From 90561ac6fd9c8d8ea5681fef545fa2681d50c6d8 Mon Sep 17 00:00:00 2001 From: kileys Date: Tue, 12 Jan 2021 19:19:53 +0000 Subject: [PATCH 01/10] Publish beam gcp bom --- sdks/java/bom/common.gradle | 122 +++++++++++++++++++++++++++++++++ sdks/java/bom/gcp/build.gradle | 31 +++++++++ settings.gradle | 1 + 3 files changed, 154 insertions(+) create mode 100644 sdks/java/bom/common.gradle create mode 100644 sdks/java/bom/gcp/build.gradle diff --git a/sdks/java/bom/common.gradle b/sdks/java/bom/common.gradle new file mode 100644 index 000000000000..d269035a05b2 --- /dev/null +++ b/sdks/java/bom/common.gradle @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'org.apache.beam.module' +apply plugin: 'maven-publish' +apply plugin: 'java-platform' + +def isRelease(Project project) { + return project.hasProperty('isRelease') +} + +publishing { + publications { + mavenJava(MavenPublication) { + groupId = project.mavenGroupId + artifactId = archivesBaseName + version = project.version + pom { + name = project.description + if (project.hasProperty("summary")) { + description = project.summary + } + url = "http://beam.apache.org" + inceptionYear = "2016" + licenses { + license { + name = "Apache License, Version 2.0" + url = "http://www.apache.org/licenses/LICENSE-2.0.txt" + distribution = "repo" + } + } + scm { + connection = "scm:git:https://gitbox.apache.org/repos/asf/beam.git" + developerConnection = "scm:git:https://gitbox.apache.org/repos/asf/beam.git" + url = "https://gitbox.apache.org/repos/asf?p=beam.git;a=summary" + } + issueManagement { + system = "jira" + url = "https://issues.apache.org/jira/browse/BEAM" + } + mailingLists { + mailingList { + name = "Beam Dev" + subscribe = "dev-subscribe@beam.apache.org" + unsubscribe = "dev-unsubscribe@beam.apache.org" + post = "dev@beam.apache.org" + archive = "http://www.mail-archive.com/dev%beam.apache.org" + } + mailingList { + name = "Beam User" + subscribe = "user-subscribe@beam.apache.org" + unsubscribe = "user-unsubscribe@beam.apache.org" + post = "user@beam.apache.org" + archive = "http://www.mail-archive.com/user%beam.apache.org" + } + mailingList { + name = "Beam Commits" + subscribe = "commits-subscribe@beam.apache.org" + unsubscribe = "commits-unsubscribe@beam.apache.org" + post = "commits@beam.apache.org" + archive = "http://www.mail-archive.com/commits%beam.apache.org" + } + } + developers { + developer { + name = "The Apache Beam Team" + email = "dev@beam.apache.org" + url = "http://beam.apache.org" + organization = "Apache Software Foundation" + organizationUrl = "http://www.apache.org" + } + } + } + + pom.withXml { + def elem = asElement() + def hdr = elem.getOwnerDocument().createComment( + ''' + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +''') + elem.insertBefore(hdr, elem.getFirstChild()) + } + + from components.javaPlatform + } + } +} + +// Only sign artifacts if we are performing a release +if (isRelease(project) && !project.hasProperty('noSigning')) { + apply plugin: "signing" + signing { + useGpgCmd() + sign publishing.publications + } +} \ No newline at end of file diff --git a/sdks/java/bom/gcp/build.gradle b/sdks/java/bom/gcp/build.gradle new file mode 100644 index 000000000000..834dfd693825 --- /dev/null +++ b/sdks/java/bom/gcp/build.gradle @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply from: '../common.gradle' + +javaPlatform { + allowDependencies() +} + +dependencies { + api platform(project.library.java.google_cloud_platform_libraries_bom) + constraints { + api project.library.java.guava + api platform(project(":sdks:java:bom")) + } +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 8ccf0bc54676..4c3d7c728c71 100644 --- a/settings.gradle +++ b/settings.gradle @@ -101,6 +101,7 @@ include ":sdks:go:examples" include ":sdks:go:test" include ":sdks:go:test:load" include ":sdks:java:bom" +include ":sdks:java:bom:gcp" include ":sdks:java:build-tools" include ":sdks:java:container" include ":sdks:java:container:java8" From 8a7269ca8437ebfac0b6783be19e392ca0c9523f Mon Sep 17 00:00:00 2001 From: kileys Date: Tue, 12 Jan 2021 19:23:54 +0000 Subject: [PATCH 02/10] move beam bom to dep --- sdks/java/bom/gcp/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/java/bom/gcp/build.gradle b/sdks/java/bom/gcp/build.gradle index 834dfd693825..4366bf051e45 100644 --- a/sdks/java/bom/gcp/build.gradle +++ b/sdks/java/bom/gcp/build.gradle @@ -23,9 +23,9 @@ javaPlatform { } dependencies { + api platform(project(":sdks:java:bom")) api platform(project.library.java.google_cloud_platform_libraries_bom) constraints { api project.library.java.guava - api platform(project(":sdks:java:bom")) } } \ No newline at end of file From b9d4d34e07df28e8d28feaac463b96c8bab0de19 Mon Sep 17 00:00:00 2001 From: kileys Date: Tue, 12 Jan 2021 21:13:47 +0000 Subject: [PATCH 03/10] refactor --- sdks/java/bom/build.gradle | 2 +- sdks/java/bom/common.gradle | 30 ++++++++++++++++++------------ sdks/java/bom/gcp/build.gradle | 6 +----- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/sdks/java/bom/build.gradle b/sdks/java/bom/build.gradle index b8a71db095ad..caa2e9dcb3df 100644 --- a/sdks/java/bom/build.gradle +++ b/sdks/java/bom/build.gradle @@ -38,7 +38,7 @@ ext { } for (p in rootProject.subprojects) { - if (!p.path.equals(project.path)) { + if (!p.path.startsWith(project.path)) { evaluationDependsOn(p.path) } } diff --git a/sdks/java/bom/common.gradle b/sdks/java/bom/common.gradle index d269035a05b2..7995351a36ff 100644 --- a/sdks/java/bom/common.gradle +++ b/sdks/java/bom/common.gradle @@ -20,6 +20,10 @@ apply plugin: 'org.apache.beam.module' apply plugin: 'maven-publish' apply plugin: 'java-platform' +javaPlatform { + allowDependencies() +} + def isRelease(Project project) { return project.hasProperty('isRelease') } @@ -91,18 +95,18 @@ publishing { def elem = asElement() def hdr = elem.getOwnerDocument().createComment( ''' - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. ''') elem.insertBefore(hdr, elem.getFirstChild()) } @@ -110,6 +114,8 @@ publishing { from components.javaPlatform } } + + repositories project.ext.repositories } // Only sign artifacts if we are performing a release @@ -119,4 +125,4 @@ if (isRelease(project) && !project.hasProperty('noSigning')) { useGpgCmd() sign publishing.publications } -} \ No newline at end of file +} diff --git a/sdks/java/bom/gcp/build.gradle b/sdks/java/bom/gcp/build.gradle index 4366bf051e45..487cd27fb7bc 100644 --- a/sdks/java/bom/gcp/build.gradle +++ b/sdks/java/bom/gcp/build.gradle @@ -18,14 +18,10 @@ apply from: '../common.gradle' -javaPlatform { - allowDependencies() -} - dependencies { api platform(project(":sdks:java:bom")) api platform(project.library.java.google_cloud_platform_libraries_bom) constraints { api project.library.java.guava } -} \ No newline at end of file +} From e49030d0f6d951358c624c864611d3388e6ad015 Mon Sep 17 00:00:00 2001 From: kileys Date: Tue, 12 Jan 2021 21:20:00 +0000 Subject: [PATCH 04/10] ignore bom in javadoc project --- sdks/java/javadoc/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/java/javadoc/build.gradle b/sdks/java/javadoc/build.gradle index fbdc03dfe73d..62d98f70ac49 100644 --- a/sdks/java/javadoc/build.gradle +++ b/sdks/java/javadoc/build.gradle @@ -28,7 +28,7 @@ applyJavaNature(publish: false) description = "Apache Beam :: SDKs :: Java :: Aggregated Javadoc" for (p in rootProject.subprojects) { - if (!p.path.equals(project.path) && !p.path.equals(':sdks:java:bom')) { + if (!p.path.equals(project.path) && !p.path.startsWith(':sdks:java:bom')) { evaluationDependsOn(p.path) } } @@ -36,7 +36,7 @@ for (p in rootProject.subprojects) { ext.getExportedJavadocProjects = { def exportedJavadocProjects = new ArrayList<>(); for (p in rootProject.subprojects) { - if (!p.path.equals(project.path) && !p.path.equals(':sdks:java:bom')) { + if (!p.path.equals(project.path) && !p.path.startsWith(':sdks:java:bom')) { def subproject = p // project(':' + p.name) if (subproject.ext.properties.containsKey('exportJavadoc') && subproject.ext.properties.exportJavadoc) { From 8f8e29648609724d9e91ac2c69c4d27caf494bbc Mon Sep 17 00:00:00 2001 From: kileys Date: Wed, 13 Jan 2021 22:38:03 +0000 Subject: [PATCH 05/10] fix urls, rename artifact --- sdks/java/bom/common.gradle | 14 +++++++------- sdks/java/bom/gcp/build.gradle | 8 ++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/sdks/java/bom/common.gradle b/sdks/java/bom/common.gradle index 7995351a36ff..cf45bdd07d5d 100644 --- a/sdks/java/bom/common.gradle +++ b/sdks/java/bom/common.gradle @@ -39,12 +39,12 @@ publishing { if (project.hasProperty("summary")) { description = project.summary } - url = "http://beam.apache.org" + url = "https://beam.apache.org" inceptionYear = "2016" licenses { license { name = "Apache License, Version 2.0" - url = "http://www.apache.org/licenses/LICENSE-2.0.txt" + url = "https://www.apache.org/licenses/LICENSE-2.0.txt" distribution = "repo" } } @@ -63,30 +63,30 @@ publishing { subscribe = "dev-subscribe@beam.apache.org" unsubscribe = "dev-unsubscribe@beam.apache.org" post = "dev@beam.apache.org" - archive = "http://www.mail-archive.com/dev%beam.apache.org" + archive = "https://www.mail-archive.com/dev@beam.apache.org" } mailingList { name = "Beam User" subscribe = "user-subscribe@beam.apache.org" unsubscribe = "user-unsubscribe@beam.apache.org" post = "user@beam.apache.org" - archive = "http://www.mail-archive.com/user%beam.apache.org" + archive = "https://www.mail-archive.com/user@beam.apache.org" } mailingList { name = "Beam Commits" subscribe = "commits-subscribe@beam.apache.org" unsubscribe = "commits-unsubscribe@beam.apache.org" post = "commits@beam.apache.org" - archive = "http://www.mail-archive.com/commits%beam.apache.org" + archive = "https://www.mail-archive.com/commits@beam.apache.org" } } developers { developer { name = "The Apache Beam Team" email = "dev@beam.apache.org" - url = "http://beam.apache.org" + url = "https://beam.apache.org" organization = "Apache Software Foundation" - organizationUrl = "http://www.apache.org" + organizationUrl = "https://www.apache.org" } } } diff --git a/sdks/java/bom/gcp/build.gradle b/sdks/java/bom/gcp/build.gradle index 487cd27fb7bc..b9c16ac72bb0 100644 --- a/sdks/java/bom/gcp/build.gradle +++ b/sdks/java/bom/gcp/build.gradle @@ -25,3 +25,11 @@ dependencies { api project.library.java.guava } } + +publishing { + publications { + mavenJava(MavenPublication) { + artifactId = 'beam-sdks-java-google-cloud-platform-bom' + } + } +} \ No newline at end of file From f829c05ac7d650faf92ec621a7a3bd2015a9d9b0 Mon Sep 17 00:00:00 2001 From: kileys Date: Fri, 15 Jan 2021 21:04:51 +0000 Subject: [PATCH 06/10] add new archetype project --- .gitignore | 1 + settings.gradle | 1 + 2 files changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index faaab28aa20b..654134e2c259 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ runners/**/vendor/**/* # Ignore generated archetypes sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/ sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/src/ +sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/archetype-resources/src/ # Ignore files generated by the Python build process. **/*.pyc diff --git a/settings.gradle b/settings.gradle index 4c3d7c728c71..11f30d70149e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -176,6 +176,7 @@ include ":sdks:java:io:synthetic" include ":sdks:java:io:influxdb" include ":sdks:java:javadoc" include ":sdks:java:maven-archetypes:examples" +include ":sdks:java:maven-archetypes:gcp-bom-examples" include ":sdks:java:maven-archetypes:starter" include ":sdks:java:testing:nexmark" include ":sdks:java:testing:expansion-service" From a6c0e15b4955c9f6c99c57d5d57f16ce1df0054a Mon Sep 17 00:00:00 2001 From: kileys Date: Wed, 20 Jan 2021 00:06:08 +0000 Subject: [PATCH 07/10] scripts to run examples with bom --- .../GoogleCloudPlatformBomArchetype.groovy | 44 +++++++++++++ .../mobilegaming-java-dataflow-bom.groovy | 61 +++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 release/src/main/groovy/GoogleCloudPlatformBomArchetype.groovy create mode 100644 release/src/main/groovy/mobilegaming-java-dataflow-bom.groovy diff --git a/release/src/main/groovy/GoogleCloudPlatformBomArchetype.groovy b/release/src/main/groovy/GoogleCloudPlatformBomArchetype.groovy new file mode 100644 index 000000000000..4920d60ebc21 --- /dev/null +++ b/release/src/main/groovy/GoogleCloudPlatformBomArchetype.groovy @@ -0,0 +1,44 @@ +#!groovy +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +class GoogleCloudPlatformBomArchetype { + def static generate(TestScripts t) { + // Generate a maven project from the snapshot repository + String output_text = t.run """mvn archetype:generate \ + --update-snapshots \ + -DarchetypeGroupId=org.apache.beam \ + -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-gcp-bom-examples \ + -DarchetypeVersion=${t.ver()} \ + -DgroupId=org.example \ + -DartifactId=word-count-beam \ + -Dversion="0.1" \ + -Dpackage=org.apache.beam.examples \ + -DinteractiveMode=false""" + + // Check if it was generated + t.see "[INFO] BUILD SUCCESS", output_text + t.run "cd word-count-beam" + output_text = t.run "ls" + t.see "pom.xml", output_text + t.see "src", output_text + String wordcounts = t.run "ls src/main/java/org/apache/beam/examples/" + t.see "WordCount.java", wordcounts + String games = t.run "ls src/main/java/org/apache/beam/examples/complete/game/" + t.see "UserScore.java", games + } +} diff --git a/release/src/main/groovy/mobilegaming-java-dataflow-bom.groovy b/release/src/main/groovy/mobilegaming-java-dataflow-bom.groovy new file mode 100644 index 000000000000..87944588e35b --- /dev/null +++ b/release/src/main/groovy/mobilegaming-java-dataflow-bom.groovy @@ -0,0 +1,61 @@ +#!groovy +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +t = new TestScripts(args) +mobileGamingCommands = new MobileGamingCommands(testScripts: t) + +/* + * Run the mobile game examples on Dataflow. + * https://beam.apache.org/get-started/mobile-gaming-example/ + */ + +t.describe ('Run Apache Beam Java SDK Mobile Gaming Examples using GCP BOM - Dataflow') + +GoogleCloudPlatformBomArchetype.generate(t) + +def runner = "DataflowRunner" +String command_output_text + +/** + * Run the UserScore example on DataflowRunner + * */ + +t.intent("Running: UserScore example with Beam GCP BOM on DataflowRunner") +t.run(mobileGamingCommands.createPipelineCommand("UserScore", runner)) +command_output_text = t.run "gsutil cat gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}* | grep user19_BananaWallaby" +t.see "total_score: 231, user: user19_BananaWallaby", command_output_text +t.success("UserScore successfully run on DataflowRunner.") +t.run "gsutil rm gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}*" + + +/** + * Run the HourlyTeamScore example on DataflowRunner + * */ + +t.intent("Running: HourlyTeamScore example with Beam GCP BOM on DataflowRunner") +t.run(mobileGamingCommands.createPipelineCommand("HourlyTeamScore", runner)) +command_output_text = t.run "gsutil cat gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}* | grep AzureBilby " +t.see "total_score: 2788, team: AzureBilby", command_output_text +t.success("HourlyTeamScore successfully run on DataflowRunner.") +t.run "gsutil rm gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}*" + +new LeaderBoardRunner().run(runner, t, mobileGamingCommands, false) +new LeaderBoardRunner().run(runner, t, mobileGamingCommands, true) + +t.done() From c02ffe7dfb9ed3c50255dc385801a37f1c49f932 Mon Sep 17 00:00:00 2001 From: kileys Date: Wed, 20 Jan 2021 00:57:59 +0000 Subject: [PATCH 08/10] add task to run bom examples --- release/build.gradle | 1 + ...w-bom.groovy => mobilegaming-java-dataflowbom.groovy} | 0 runners/google-cloud-dataflow-java/build.gradle | 9 +++++++++ 3 files changed, 10 insertions(+) rename release/src/main/groovy/{mobilegaming-java-dataflow-bom.groovy => mobilegaming-java-dataflowbom.groovy} (100%) diff --git a/release/build.gradle b/release/build.gradle index 4abe92ff5b16..cbe0bf14a175 100644 --- a/release/build.gradle +++ b/release/build.gradle @@ -37,4 +37,5 @@ task runJavaExamplesValidationTask { dependsOn ":runners:direct-java:runMobileGamingJavaDirect" dependsOn ":runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow" dependsOn ":runners:twister2:runQuickstartJavaTwister2" + dependsOn ":runners:google-cloud-dataflow-java:runMobileGamingJavaDataflowBom" } diff --git a/release/src/main/groovy/mobilegaming-java-dataflow-bom.groovy b/release/src/main/groovy/mobilegaming-java-dataflowbom.groovy similarity index 100% rename from release/src/main/groovy/mobilegaming-java-dataflow-bom.groovy rename to release/src/main/groovy/mobilegaming-java-dataflowbom.groovy diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 197eff83aa43..141fd0ea95e3 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -671,6 +671,15 @@ createJavaExamplesArchetypeValidationTask(type: 'MobileGaming', bqDataset: bqDataset, pubsubTopic: pubsubTopic) +// Generates :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflowBom +createJavaExamplesArchetypeValidationTask(type: 'MobileGaming', + runner: 'DataflowBom', + gcpProject: gcpProject, + gcpRegion: gcpRegion, + gcsBucket: gcsBucket, + bqDataset: bqDataset, + pubsubTopic: pubsubTopic) + // Standalone task for testing GCS upload, use with -PfilesToStage and -PdataflowTempRoot. task GCSUpload(type: JavaExec) { main = 'org.apache.beam.runners.dataflow.util.GCSUploadMain' From 969dfe7d69d697e16e474c06d7c060ff9e658964 Mon Sep 17 00:00:00 2001 From: kileys Date: Wed, 20 Jan 2021 02:00:47 +0000 Subject: [PATCH 09/10] add maven archetype for beam gcp bom --- .../examples/generate-sources.sh | 2 +- .../gcp-bom-examples/build.gradle | 80 +++ .../maven-archetypes/gcp-bom-examples/pom.xml | 136 ++++++ .../META-INF/maven/archetype-metadata.xml | 45 ++ .../resources/archetype-resources/pom.xml | 460 ++++++++++++++++++ 5 files changed, 722 insertions(+), 1 deletion(-) create mode 100644 sdks/java/maven-archetypes/gcp-bom-examples/build.gradle create mode 100644 sdks/java/maven-archetypes/gcp-bom-examples/pom.xml create mode 100644 sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/META-INF/maven/archetype-metadata.xml create mode 100644 sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/archetype-resources/pom.xml diff --git a/sdks/java/maven-archetypes/examples/generate-sources.sh b/sdks/java/maven-archetypes/examples/generate-sources.sh index c2b686cf9a32..2ba09fee5495 100755 --- a/sdks/java/maven-archetypes/examples/generate-sources.sh +++ b/sdks/java/maven-archetypes/examples/generate-sources.sh @@ -21,7 +21,7 @@ # Usage: Invoke with no arguments from any working directory. # The directory of this script. Assumes root of the maven-archetypes module. -HERE="$( dirname "$0" )" +HERE=${HERE-"$( dirname "$0" )"} # The directory of the examples-java module EXAMPLES_ROOT="${HERE}/../../../../examples/java" diff --git a/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle b/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle new file mode 100644 index 000000000000..101cd2e39d50 --- /dev/null +++ b/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyJavaNature(exportJavadoc: false, automaticModuleName: 'org.apache.beam.maven.archetypes.gcp.bom.examples') + +description = "Apache Beam :: SDKs :: Java :: Maven Archetypes :: Google Cloud Platform BOM Examples" +ext.summary = """A Maven Archetype to create a project +using the Beam Google Cloud Platform BOM""" + +processResources { + filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ + 'project.version': version, + 'bigquery.version': dependencies.create(project.library.java.google_api_services_bigquery).getVersion(), + 'google-api-client.version': dependencies.create(project.library.java.google_api_client).getVersion(), + 'hamcrest.version': dependencies.create(project.library.java.hamcrest_library).getVersion(), + 'jackson.version': dependencies.create(project.library.java.jackson_core).getVersion(), + 'joda.version': dependencies.create(project.library.java.joda_time).getVersion(), + 'junit.version': dependencies.create(project.library.java.junit).getVersion(), + 'pubsub.version': dependencies.create(project.library.java.google_api_services_pubsub).getVersion(), + 'slf4j.version': dependencies.create(project.library.java.slf4j_api).getVersion(), + 'spark.version': dependencies.create(project.library.java.spark_core).getVersion(), + 'nemo.version': dependencies.create(project.library.java.nemo_compiler_frontend_beam).getVersion(), + 'hadoop.version': dependencies.create(project.library.java.hadoop_client).getVersion(), + 'mockito.version': dependencies.create(project.library.java.mockito_core).getVersion(), + 'maven-compiler-plugin.version': dependencies.create(project.library.maven.maven_compiler_plugin).getVersion(), + 'maven-exec-plugin.version': dependencies.create(project.library.maven.maven_exec_plugin).getVersion(), + 'maven-jar-plugin.version': dependencies.create(project.library.maven.maven_jar_plugin).getVersion(), + 'maven-shade-plugin.version': dependencies.create(project.library.maven.maven_shade_plugin).getVersion(), + 'maven-surefire-plugin.version': dependencies.create(project.library.maven.maven_surefire_plugin).getVersion(), + 'flink.artifact.name': 'beam-runners-flink-'.concat(project(":runners:flink:${project.ext.latestFlinkVersion}").getName()), + ] +} + +/* + * We need to rely on manually specifying these evaluationDependsOn to ensure that + * the following projects are evaluated before we evaluate this project. This is because + * we are attempting to reference the "sourceSets.{main|test}.allSource" directly. + */ +evaluationDependsOn(':examples:java') + +task generateSources(type: Exec) { + inputs.file('../examples/generate-sources.sh') + .withPropertyName('generate-sources.sh') + .withPathSensitivity(PathSensitivity.RELATIVE) + inputs.files(project(':examples:java').sourceSets.main.allSource) + .withPropertyName('sourcesMain') + .withPathSensitivity(PathSensitivity.RELATIVE) + inputs.files(project(':examples:java').sourceSets.test.allSource) + .withPropertyName('sourcesTest') + .withPathSensitivity(PathSensitivity.RELATIVE) + outputs.dir('src/main/resources/archetype-resources/src') + environment "HERE", "." + commandLine '../examples/generate-sources.sh' +} + +sourceSets { + main { + output.dir('src', builtBy: 'generateSources') + } +} + +dependencies { + compile project(":examples:java") +} diff --git a/sdks/java/maven-archetypes/gcp-bom-examples/pom.xml b/sdks/java/maven-archetypes/gcp-bom-examples/pom.xml new file mode 100644 index 000000000000..b7447e3822a6 --- /dev/null +++ b/sdks/java/maven-archetypes/gcp-bom-examples/pom.xml @@ -0,0 +1,136 @@ + + + + + 4.0.0 + + org.apache.beam + beam-sdks-java-maven-archetypes-gcp-bom-examples + 2.6.0-SNAPSHOT + Apache Beam :: SDKs :: Java :: Maven Archetypes :: Google Cloud Platform BOM Examples + A Maven Archetype to create a project + using the Beam Google Cloud Platform BOM. + + + maven-archetype + + + + + org.apache.maven.archetype + archetype-packaging + 2.4 + + + + + + + maven-archetype-plugin + 2.4 + + + org.apache.maven.shared + maven-invoker + 2.2 + + + + + + default-integration-test + install + + integration-test + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.codehaus.mojo + exec-maven-plugin + [1.5.0,) + + exec + + + + + false + + + + + + + + + + + + + exec-maven-plugin + org.codehaus.mojo + 1.5.0 + + + generate-archetype-contents + generate-sources + + exec + + + ${project.basedir}/generate-sources.sh + + + + + + org.apache.maven.plugins + maven-clean-plugin + 3.1.0 + + + + src/main/resources/archetype-resources + + src/**/* + src + + false + + + + + + + diff --git a/sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/META-INF/maven/archetype-metadata.xml b/sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/META-INF/maven/archetype-metadata.xml new file mode 100644 index 000000000000..b13328d10d25 --- /dev/null +++ b/sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/META-INF/maven/archetype-metadata.xml @@ -0,0 +1,45 @@ + + + + + + + 1.8 + + + + + + src/main/java + + **/*.java + + + + + src/test/java + + **/*.java + + + + diff --git a/sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/archetype-resources/pom.xml new file mode 100644 index 000000000000..f37972663883 --- /dev/null +++ b/sdks/java/maven-archetypes/gcp-bom-examples/src/main/resources/archetype-resources/pom.xml @@ -0,0 +1,460 @@ + + + + 4.0.0 + + ${groupId} + ${artifactId} + ${version} + + jar + + + @project.version@ + + @bigquery.version@ + @google-api-client.version@ + @hamcrest.version@ + @jackson.version@ + @joda.version@ + @junit.version@ + @maven-compiler-plugin.version@ + @maven-exec-plugin.version@ + @maven-jar-plugin.version@ + @maven-shade-plugin.version@ + @mockito.version@ + @pubsub.version@ + @slf4j.version@ + @spark.version@ + @hadoop.version@ + @maven-surefire-plugin.version@ + @nemo.version@ + @flink.artifact.name@ + + + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + ${targetPlatform} + ${targetPlatform} + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + all + 4 + true + + + + org.apache.maven.surefire + surefire-junit47 + ${maven-surefire-plugin.version} + + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + package + + shade + + + ${project.artifactId}-bundled-${project.version} + + + *:* + + META-INF/LICENSE + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + + + org.codehaus.mojo + exec-maven-plugin + ${maven-exec-plugin.version} + + false + + + + + + + + + direct-runner + + true + + + + + org.apache.beam + beam-runners-direct-java + runtime + + + + + + portable-runner + + true + + + + + org.apache.beam + beam-runners-portability-java + runtime + + + + + + dataflow-runner + + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + runtime + + + + + + flink-runner + + + + org.apache.beam + + ${flink.artifact.name} + runtime + + + + + + spark-runner + + + 4.1.17.Final + + + + org.apache.beam + beam-runners-spark + runtime + + + org.apache.beam + beam-sdks-java-io-hadoop-file-system + runtime + + + org.apache.spark + spark-streaming_2.11 + runtime + + + org.slf4j + jul-to-slf4j + + + + + com.fasterxml.jackson.module + jackson-module-scala_2.11 + ${jackson.version} + runtime + + + + org.apache.beam + beam-sdks-java-io-google-cloud-platform + + + io.grpc + grpc-netty + + + io.netty + netty-handler + + + + + + + samza-runner + + + org.apache.beam + beam-runners-samza + runtime + + + + + twister2-runner + + + org.apache.beam + beam-runners-twister2 + runtime + + + + + nemo-runner + + + org.apache.nemo + nemo-compiler-frontend-beam + ${nemo.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + + + + + + jet-runner + + + org.apache.beam + beam-runners-jet + runtime + + + + + + + + + + org.apache.beam + beam-sdks-java-core + + + + + org.apache.beam + beam-sdks-java-io-google-cloud-platform + + + + + com.google.api-client + google-api-client + ${google-api-client.version} + + + + com.google.guava + guava-jdk5 + + + + + + com.google.apis + google-api-services-bigquery + ${bigquery.version} + + + + com.google.guava + guava-jdk5 + + + + + + com.google.http-client + google-http-client + + + + com.google.guava + guava-jdk5 + + + + + + com.google.apis + google-api-services-pubsub + ${pubsub.version} + + + + com.google.guava + guava-jdk5 + + + + + + joda-time + joda-time + ${joda.version} + + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + org.slf4j + slf4j-jdk14 + ${slf4j.version} + + runtime + + + + + org.hamcrest + hamcrest-core + ${hamcrest.version} + + + + org.hamcrest + hamcrest-library + ${hamcrest.version} + + + + junit + junit + ${junit.version} + + + + + org.apache.beam + beam-runners-direct-java + test + + + + org.mockito + mockito-core + ${mockito.version} + test + + + + + + + + org.apache.beam + beam-sdks-java-google-cloud-platform-bom + ${beam.version} + pom + import + + + + From c5b7295592178acbadfef6d683f934823ccf1305 Mon Sep 17 00:00:00 2001 From: kileys Date: Wed, 20 Jan 2021 02:10:30 +0000 Subject: [PATCH 10/10] add to release script --- release/src/main/scripts/run_rc_validation.sh | 11 +++++++++++ .../maven-archetypes/gcp-bom-examples/build.gradle | 1 + 2 files changed, 12 insertions(+) diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh index dfc02b83dec4..1cb7034a508d 100755 --- a/release/src/main/scripts/run_rc_validation.sh +++ b/release/src/main/scripts/run_rc_validation.sh @@ -285,6 +285,17 @@ if [[ ("$java_mobile_game_direct" = true || "$java_mobile_game_dataflow" = true) -PbqDataset=${MOBILE_GAME_DATASET} \ -PpubsubTopic=${MOBILE_GAME_PUBSUB_TOPIC} \ -PgcsBucket=${USER_GCS_BUCKET:5} # skip 'gs://' prefix + + echo "**************************************************************************" + echo "* Java mobile game on DataflowRunner using Beam GCP BOM: UserScore, HourlyTeamScore, Leaderboard" + echo "**************************************************************************" + ./gradlew :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflowBom \ + -Prepourl=${REPO_URL} \ + -Pver=${RELEASE_VER} \ + -PgcpProject=${USER_GCP_PROJECT} \ + -PbqDataset=${MOBILE_GAME_DATASET} \ + -PpubsubTopic=${MOBILE_GAME_PUBSUB_TOPIC} \ + -PgcsBucket=${USER_GCS_BUCKET:5} # skip 'gs://' prefix else echo "* Skip Java Mobile Game on DataflowRunner." fi diff --git a/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle b/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle index 101cd2e39d50..fe33e2ea88b0 100644 --- a/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle +++ b/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle @@ -19,6 +19,7 @@ plugins { id 'org.apache.beam.module' } applyJavaNature(exportJavadoc: false, automaticModuleName: 'org.apache.beam.maven.archetypes.gcp.bom.examples') +// Based off of :sdks:java:maven-archetypes:examples project description = "Apache Beam :: SDKs :: Java :: Maven Archetypes :: Google Cloud Platform BOM Examples" ext.summary = """A Maven Archetype to create a project using the Beam Google Cloud Platform BOM"""