aromanenko-dev · iemejia · Feb 1, 2018 · Feb 2, 2018 · Feb 4, 2018 · Feb 4, 2018
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,10 +1,17 @@
+DESCRIPTION HERE
+
+------------------------
+
 Follow this checklist to help us incorporate your contribution quickly and easily:
 
  - [ ] Make sure there is a [JIRA issue](https://issues.apache.org/jira/projects/BEAM/issues/) filed for the change (usually before you start working on it).  Trivial changes like typos do not require a JIRA issue.  Your pull request should address just this issue, without pulling in other changes.
- - [ ] Each commit in the pull request should have a meaningful subject line and body.
  - [ ] Format the pull request title like `[BEAM-XXX] Fixes bug in ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA issue.
- - [ ] Write a pull request description that is detailed enough to understand what the pull request does, how, and why.
+ - [ ] Write a pull request description that is detailed enough to understand:
+   - [ ] What the pull request does
+   - [ ] Why it does it
+   - [ ] How it does it
+   - [ ] Why this approach
+ - [ ] Each commit in the pull request should have a meaningful subject line and body.
  - [ ] Run `mvn clean verify` to make sure basic checks pass. A more thorough check will be performed on your pull request automatically.
  - [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf).
 
----
diff --git a/.gitignore b/.gitignore
@@ -37,12 +37,14 @@ sdks/python/LICENSE
 sdks/python/NOTICE
 sdks/python/README.md
 sdks/python/apache_beam/portability/api/*pb2*.*
+sdks/python/apache_beam/portability/common_urns.py
 
 # Ignore IntelliJ files.
 **/.idea/**/*
 **/*.iml
 **/*.ipr
 **/*.iws
+**/out/**/*
 
 # Ignore Eclipse files.
 **/.classpath

diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
@@ -19,45 +19,63 @@
 import common_job_properties
 
 // This job runs the Beam performance tests on PerfKit Benchmarker.
-job('beam_PerformanceTests_JDBC'){
+job('beam_PerformanceTests_JDBC') {
     // Set default Beam job properties.
     common_job_properties.setTopLevelMainJobProperties(delegate)
 
     // Run job in postcommit every 6 hours, don't trigger every push, and
     // don't email individual committers.
     common_job_properties.setPostCommit(
-        delegate,
-        '0 */6 * * *',
-        false,
-        'commits@beam.apache.org',
-        false)
+            delegate,
+            '0 */6 * * *',
+            false,
+            'commits@beam.apache.org',
+            false)
+
+    common_job_properties.enablePhraseTriggeringFromPullRequest(
+            delegate,
+            'Java JdbcIO Performance Test',
+            'Run Java JdbcIO Performance Test')
 
     def pipelineArgs = [
-        tempRoot: 'gs://temp-storage-for-end-to-end-tests',
-        project: 'apache-beam-testing',
-        postgresServerName: '10.36.0.11',
-        postgresUsername: 'postgres',
-        postgresDatabaseName: 'postgres',
-        postgresPassword: 'uuinkks',
-        postgresSsl: 'false'
+            tempRoot       : 'gs://temp-storage-for-perf-tests',
+            project        : 'apache-beam-testing',
+            postgresPort   : '5432',
+            numberOfRecords: '5000000'
     ]
-    def pipelineArgList = []
-    pipelineArgs.each({
-        key, value -> pipelineArgList.add("--$key=$value")
-    })
-    def pipelineArgsJoined = pipelineArgList.join(',')
-
-    def argMap = [
-      benchmarks: 'beam_integration_benchmark',
-      beam_it_module: 'sdks/java/io/jdbc',
-      beam_it_args: pipelineArgsJoined,
-      beam_it_class: 'org.apache.beam.sdk.io.jdbc.JdbcIOIT',
-      // Profile is located in $BEAM_ROOT/sdks/java/io/pom.xml.
-      beam_it_profile: 'io-it'
+
+    def testArgs = [
+            kubeconfig              : '"$HOME/.kube/config"',
+            beam_it_timeout         : '1800',
+            benchmarks              : 'beam_integration_benchmark',
+            beam_it_profile         : 'io-it',
+            beam_prebuilt           : 'true',
+            beam_sdk                : 'java',
+            beam_it_module          : 'sdks/java/io/jdbc',
+            beam_it_class           : 'org.apache.beam.sdk.io.jdbc.JdbcIOIT',
+            beam_it_options         : joinPipelineOptions(pipelineArgs),
+            beam_kubernetes_scripts : makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres.yml')
+                    + ',' + makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'),
+            beam_options_config_file: makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'),
+            bigquery_table          : 'beam_performance.jdbcioit_pkb_results'
     ]
 
-    common_job_properties.buildPerformanceTest(delegate, argMap)
+    steps {
+        // create .kube/config file for perfkit (if not exists)
+        shell('gcloud container clusters get-credentials io-datastores --zone=us-central1-a --verbosity=debug')
+    }
 
-    // [BEAM-2141] Perf tests do not pass.
-    disabled()
+    common_job_properties.buildPerformanceTest(delegate, testArgs)
 }
+
+static String joinPipelineOptions(Map pipelineArgs) {
+    List<String> pipelineArgList = []
+    pipelineArgs.each({
+        key, value -> pipelineArgList.add("\"--$key=$value\"")
+    })
+    return "[" + pipelineArgList.join(',') + "]"
+}
+
+static String makePathAbsolute(String path) {
+    return '"$WORKSPACE/' + path + '"'
+}
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy
@@ -39,5 +39,5 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Flink') {
     'Run Flink ValidatesRunner')
 
   // Maven goals for this job.
-  goals('-B -e clean verify -am -pl runners/flink -Plocal-validates-runner-tests -Pvalidates-runner-tests')
+  goals('-B -e clean verify -am -pl runners/flink -Plocal-validates-runner-tests')
 }
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy
@@ -40,5 +40,5 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Spark') {
     'Run Spark ValidatesRunner')
 
   // Maven goals for this job.
-  goals('-B -e clean verify -am -pl runners/spark -Pvalidates-runner-tests -Plocal-validates-runner-tests -Dspark.ui.enabled=false')
+  goals('-B -e clean verify -am -pl runners/spark -Plocal-validates-runner-tests -Dspark.ui.enabled=false')
 }
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_beam_PostCommit_Python_ValidatesContainer_Dataflow.groovy
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the suite of Python ValidatesContainer tests against the
+// Dataflow runner.
+job('beam_PostCommit_Python_ValidatesContainer_Dataflow') {
+  description('Runs Python ValidatesContainer suite on the Dataflow runner.')
+
+  // Set common parameters.
+  common_job_properties.setTopLevelMainJobProperties(delegate)
+
+  // Sets that this is a PostCommit job.
+  common_job_properties.setPostCommit(delegate, '30 3 * * *', false)
+
+  // Allows triggering this build against pull requests.
+  common_job_properties.enablePhraseTriggeringFromPullRequest(
+      delegate,
+      'Google Cloud Dataflow Runner Python ValidatesContainer Tests',
+      'Run Python Dataflow ValidatesContainer')
+
+  // Allow the test to only run on particular nodes
+  // TODO(BEAM-1817): Remove once the tests can run on all nodes
+  parameters {
+    nodeParam('TEST_HOST') {
+      description('select test host as either beam1, 2 or 3')
+      defaultNodes(['beam1', 'beam2', 'beam3'])
+      allowedNodes(['beam1', 'beam2', 'beam3'])
+      trigger('multiSelectionDisallowed')
+      eligibility('IgnoreOfflineNodeEligibility')
+    }
+  }
+
+  // Execute shell command to test Python SDK.
+  steps {
+    shell('cd ' + common_job_properties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh')
+  }
+}
diff --git a/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
@@ -31,22 +31,32 @@ job('beam_PostRelease_NightlySnapshot') {
 
   parameters {
     stringParam('snapshot_version',
-                '2.3.0-SNAPSHOT',
+                '',
                 'Version of the repository snapshot to install')
     stringParam('snapshot_url',
-                'https://repository.apache.org/content/repositories/snapshots',
+                '',
                 'Repository URL to install from')
   }
 
   // This is a post-commit job that runs once per day, not for every push.
   common_job_properties.setPostCommit(
       delegate,
       '0 11 * * *',
-      false,
-      'dev@beam.apache.org')
+      false)
+
+
+  // Allows triggering this build against pull requests.
+  common_job_properties.enablePhraseTriggeringFromPullRequest(
+      delegate,
+      './gradlew :release:runQuickstartsJava',
+      'Run Dataflow PostRelease')
 
   steps {
-    // Run a quickstart from https://beam.apache.org/get-started/quickstart-java/
-    shell('cd ' + common_job_properties.checkoutDir + '/release && groovy quickstart-java-direct.groovy')
+    // Run a quickstart from https://beam.apache.org/get-started/quickstart-java
+    gradle {
+      rootBuildScriptDir(common_job_properties.checkoutDir)
+      tasks(':release:runQuickstartsJava')
+      switches('-Pver=$snapshot_version -Prepourl=$snapshot_url')
+    }
   }
 }
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Go_GradleBuild.groovy b/.test-infra/jenkins/job_beam_PreCommit_Go_GradleBuild.groovy
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Go precommit which runs a gradle build, and the current set
+// of precommit tests.
+job('beam_PreCommit_Go_GradleBuild') {
+  description('Runs Go PreCommit tests for the current GitHub Pull Request.')
+
+  // Execute concurrent builds if necessary.
+  concurrentBuild()
+
+  // Set common parameters.
+  common_job_properties.setTopLevelMainJobProperties(
+    delegate,
+    'master',
+    240)
+
+  def gradle_switches = [
+    // Gradle log verbosity enough to diagnose basic build issues
+    "--info",
+    // Continue the build even if there is a failure to show as many potential failures as possible.
+    '--continue',
+    // Until we verify the build cache is working appropriately, force rerunning all tasks
+    '--rerun-tasks',
+  ]
+
+  def gradle_command_line = './gradlew ' + gradle_switches.join(' ') + ' :goPreCommit'
+  // Sets that this is a PreCommit job.
+  common_job_properties.setPreCommit(delegate, gradle_command_line, 'Run Go Gradle PreCommit')
+  steps {
+    gradle {
+      rootBuildScriptDir(common_job_properties.checkoutDir)
+      tasks(':goPreCommit')
+      for (String gradle_switch : gradle_switches) {
+        switches(gradle_switch)
+      }
+    }
+  }
+}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy
@@ -21,7 +21,7 @@ import common_job_properties
 // This is the Java precommit which runs a Gradle build, and the current set
 // of precommit tests.
 job('beam_PreCommit_Java_GradleBuild') {
-  description('Runs a build of the current GitHub Pull Request.')
+  description('Runs Java PreCommit tests for the current GitHub Pull Request.')
 
   // Execute concurrent builds if necessary.
   concurrentBuild()
@@ -38,6 +38,8 @@ job('beam_PreCommit_Java_GradleBuild') {
   }
 
   def gradle_switches = [
+    // Gradle log verbosity enough to diagnose basic build issues
+    "--info",
     // Continue the build even if there is a failure to show as many potential failures as possible.
     '--continue',
     // Until we verify the build cache is working appropriately, force rerunning all tasks

diff --git a/assembly.xml b/assembly.xml
@@ -69,6 +69,7 @@
                 <exclude>**/.gogradle/**</exclude>
                 <exclude>**/build/**</exclude>
                 <exclude>**/vendor/**</exclude>
+                <exclude>**/out/**</exclude>
                 <exclude>**/.gradletasknamecache</exclude>
 
                 <!-- files generated by the Maven build process. -->

diff --git a/build.gradle b/build.gradle
@@ -66,7 +66,7 @@ ext.library = [
     bigdataoss_util: "com.google.cloud.bigdataoss:util:$google_cloud_bigdataoss_version",
     bigtable_client_core: "com.google.cloud.bigtable:bigtable-client-core:$bigtable_version",
     bigtable_protos: "com.google.cloud.bigtable:bigtable-protos:$bigtable_proto_version",
-    byte_buddy: "net.bytebuddy:byte-buddy:1.6.8",
+    byte_buddy: "net.bytebuddy:byte-buddy:1.7.10",
     commons_compress: "org.apache.commons:commons-compress:1.14",
     commons_csv: "org.apache.commons:commons-csv:1.4",
     commons_io_1x: "commons-io:commons-io:1.3.2",
@@ -83,7 +83,7 @@ ext.library = [
     google_api_client_jackson2: "com.google.api-client:google-api-client-jackson2:$google_clients_version",
     google_api_client_java6: "com.google.api-client:google-api-client-java6:$google_clients_version",
     google_api_common: "com.google.api:api-common:1.0.0-rc2",
-    google_api_services_bigquery: "com.google.apis:google-api-services-bigquery:v2-rev355-$google_clients_version",
+    google_api_services_bigquery: "com.google.apis:google-api-services-bigquery:v2-rev374-$google_clients_version",
     google_api_services_clouddebugger: "com.google.apis:google-api-services-clouddebugger:v2-rev8-$google_clients_version",
     google_api_services_cloudresourcemanager: "com.google.apis:google-api-services-cloudresourcemanager:v1-rev6-$google_clients_version",
     google_api_services_dataflow: "com.google.apis:google-api-services-dataflow:v1b3-rev221-$google_clients_version",
@@ -180,7 +180,7 @@ buildscript {
     classpath "gradle.plugin.org.nosphere.apache:creadur-rat-gradle:0.3.1"                              // Enable Apache license enforcement
     classpath "com.commercehub.gradle.plugin:gradle-avro-plugin:0.11.0"                                 // Enable Avro code generation
     classpath "com.diffplug.spotless:spotless-plugin-gradle:3.6.0"                                      // Enable a code formatting plugin
-    classpath "gradle.plugin.com.github.blindpirate:gogradle:0.7.0"                                     // Enable Go code compilation
+    classpath "gradle.plugin.com.github.blindpirate:gogradle:0.8.1"                                     // Enable Go code compilation
     classpath "gradle.plugin.com.palantir.gradle.docker:gradle-docker:0.13.0"                           // Enable building Docker containers
     classpath "cz.malohlava:visteg:1.0.3"                                                               // Enable generating Gradle task dependencies as ".dot" files
     classpath "com.github.jengelman.gradle.plugins:shadow:2.0.1"                                        // Enable shading Java dependencies
@@ -208,6 +208,7 @@ rat {
     // Exclude files generated by the Gradle build process
     "**/.gradle/**/*",
     "**/.gogradle/**/*",
+    "**/gogradle.lock",
     "**/build/**/*",
     "**/vendor/**/*",
     "**/.gradletasknamecache",
@@ -241,6 +242,7 @@ rat {
     "**/*.iml",
     "**/*.ipr",
     "**/*.iws",
+    "**/out/**/*",
 
     // .gitignore: Ignore Eclipse files.
     "**/.classpath",
@@ -284,8 +286,11 @@ rat {
 }
 check.dependsOn rat
 
-// Define a root Java PreCommit task simplifying what is needed
+// Define root PreCommit tasks simplifying what is needed
 // to be specified on the commandline when executing locally.
+// This indirection also makes Jenkins use the branch of the PR
+// for the test definitions.
+
 def javaPreCommitRoots = [
   ":sdks:java:core",
   ":runners:direct-java",
@@ -298,3 +303,7 @@ task javaPreCommit() {
   }
   dependsOn ":examples:java:preCommit"
 }
+
+task goPreCommit() {
+  dependsOn ":sdks:go:test"
+}