diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index f882b8422092..92c1426aa55e 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -255,6 +255,31 @@ class common_job_properties { return mapToArgString(joinedArgs) } + static def setupKubernetes(def context, def namespace, def kubeconfigLocation) { + context.steps { + shell('gcloud container clusters get-credentials io-datastores --zone=us-central1-a --verbosity=debug') + shell("cp /home/jenkins/.kube/config ${kubeconfigLocation}") + + shell("kubectl --kubeconfig=${kubeconfigLocation} create namespace ${namespace}") + shell("kubectl --kubeconfig=${kubeconfigLocation} config set-context \$(kubectl config current-context) --namespace=${namespace}") + } + } + + static def cleanupKubernetes(def context, def namespace, def kubeconfigLocation) { + context.steps { + shell("kubectl --kubeconfig=${kubeconfigLocation} delete namespace ${namespace}") + shell("rm ${kubeconfigLocation}") + } + } + + static String getKubernetesNamespace(def testName) { + return "${testName}-${new Date().getTime()}" + } + + static String getKubeconfigLocationForNamespace(def namespace) { + return '"$WORKSPACE/' + "config-${namespace}" + '"' + } + // Adds the standard performance test job steps. static def buildPerformanceTest(def context, def argMap) { def pkbArgs = genPerformanceArgs(argMap) @@ -381,4 +406,27 @@ class common_job_properties { } } } + + /** + * Transforms pipeline options to a string of format like below: + * ["--pipelineOption=123", "--pipelineOption2=abc", ...] + * + * @param pipelineOptions A map of pipeline options. + */ + static String joinPipelineOptions(Map pipelineOptions) { + List pipelineArgList = [] + pipelineOptions.each({ + key, value -> pipelineArgList.add("\"--$key=$value\"") + }) + return "[" + pipelineArgList.join(',') + "]" + } + + + /** + * Returns absolute path to beam project's files. + * @param path A relative path to project resource. + */ + static String makePathAbsolute(String path) { + return '"$WORKSPACE/' + path + '"' + } } diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy index 667b11d20725..e1d8c5eb3ac3 100644 --- a/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy +++ b/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy @@ -86,22 +86,16 @@ private void create_filebasedio_performance_test_job(testConfiguration) { 'commits@beam.apache.org', false) - def pipelineArgs = [ + def pipelineOptions = [ project : 'apache-beam-testing', tempRoot : 'gs://temp-storage-for-perf-tests', numberOfRecords: '1000000', filenamePrefix : "gs://temp-storage-for-perf-tests/${testConfiguration.jobName}/\${BUILD_ID}/", ] if (testConfiguration.containsKey('extraPipelineArgs')) { - pipelineArgs << testConfiguration.extraPipelineArgs + pipelineOptions << testConfiguration.extraPipelineArgs } - def pipelineArgList = [] - pipelineArgs.each({ - key, value -> pipelineArgList.add("\"--$key=$value\"") - }) - def pipelineArgsJoined = "[" + pipelineArgList.join(',') + "]" - def argMap = [ benchmarks : 'beam_integration_benchmark', beam_it_timeout : '1200', @@ -110,7 +104,7 @@ private void create_filebasedio_performance_test_job(testConfiguration) { beam_sdk : 'java', beam_it_module : 'sdks/java/io/file-based-io-tests', beam_it_class : testConfiguration.itClass, - beam_it_options : pipelineArgsJoined, + beam_it_options : common_job_properties.joinPipelineOptions(pipelineOptions), beam_extra_mvn_properties: '["filesystem=gcs"]', bigquery_table : testConfiguration.bqTable, ] diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_HadoopInputFormat.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_HadoopInputFormat.groovy new file mode 100644 index 000000000000..093390dc9306 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PerformanceTests_HadoopInputFormat.groovy @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +job('beam_PerformanceTests_HadoopInputFormat') { + // Set default Beam job properties. + common_job_properties.setTopLevelMainJobProperties(delegate) + + // Run job in postcommit every 6 hours, don't trigger every push, and + // don't email individual committers. + common_job_properties.setPostCommit( + delegate, + '0 */6 * * *', + false, + 'commits@beam.apache.org', + false) + + common_job_properties.enablePhraseTriggeringFromPullRequest( + delegate, + 'Java HadoopInputFormatIO Performance Test', + 'Run Java HadoopInputFormatIO Performance Test') + + def pipelineOptions = [ + tempRoot : 'gs://temp-storage-for-perf-tests', + project : 'apache-beam-testing', + postgresPort : '5432', + numberOfRecords: '600000' + ] + + String namespace = common_job_properties.getKubernetesNamespace('hadoopinputformatioit') + String kubeconfig = common_job_properties.getKubeconfigLocationForNamespace(namespace) + + def testArgs = [ + kubeconfig : kubeconfig, + beam_it_timeout : '1200', + benchmarks : 'beam_integration_benchmark', + beam_it_profile : 'io-it', + beam_prebuilt : 'true', + beam_sdk : 'java', + beam_it_module : 'sdks/java/io/hadoop-input-format', + beam_it_class : 'org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIOIT', + beam_it_options : common_job_properties.joinPipelineOptions(pipelineOptions), + beam_kubernetes_scripts : common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'), + beam_options_config_file: common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'), + bigquery_table : 'beam_performance.hadoopinputformatioit_pkb_results' + ] + + common_job_properties.setupKubernetes(delegate, namespace, kubeconfig) + common_job_properties.buildPerformanceTest(delegate, testArgs) + common_job_properties.cleanupKubernetes(delegate, namespace, kubeconfig) +} + diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy index 1e5131f3ddf5..401a9104e6a9 100644 --- a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy +++ b/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy @@ -18,7 +18,6 @@ import common_job_properties -// This job runs the Beam performance tests on PerfKit Benchmarker. job('beam_PerformanceTests_JDBC') { // Set default Beam job properties. common_job_properties.setTopLevelMainJobProperties(delegate) @@ -37,15 +36,18 @@ job('beam_PerformanceTests_JDBC') { 'Java JdbcIO Performance Test', 'Run Java JdbcIO Performance Test') - def pipelineArgs = [ + def pipelineOptions = [ tempRoot : 'gs://temp-storage-for-perf-tests', project : 'apache-beam-testing', postgresPort : '5432', numberOfRecords: '5000000' ] + String namespace = common_job_properties.getKubernetesNamespace('jdbcioit') + String kubeconfig = common_job_properties.getKubeconfigLocationForNamespace(namespace) + def testArgs = [ - kubeconfig : '"$HOME/.kube/config"', + kubeconfig : kubeconfig, beam_it_timeout : '1800', benchmarks : 'beam_integration_benchmark', beam_it_profile : 'io-it', @@ -53,29 +55,14 @@ job('beam_PerformanceTests_JDBC') { beam_sdk : 'java', beam_it_module : 'sdks/java/io/jdbc', beam_it_class : 'org.apache.beam.sdk.io.jdbc.JdbcIOIT', - beam_it_options : joinPipelineOptions(pipelineArgs), - beam_kubernetes_scripts : makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres.yml') - + ',' + makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'), - beam_options_config_file: makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'), + beam_it_options : common_job_properties.joinPipelineOptions(pipelineOptions), + beam_kubernetes_scripts : common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'), + beam_options_config_file: common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'), bigquery_table : 'beam_performance.jdbcioit_pkb_results' ] - steps { - // create .kube/config file for perfkit (if not exists) - shell('gcloud container clusters get-credentials io-datastores --zone=us-central1-a --verbosity=debug') - } - + common_job_properties.setupKubernetes(delegate, namespace, kubeconfig) common_job_properties.buildPerformanceTest(delegate, testArgs) + common_job_properties.cleanupKubernetes(delegate, namespace, kubeconfig) } -static String joinPipelineOptions(Map pipelineArgs) { - List pipelineArgList = [] - pipelineArgs.each({ - key, value -> pipelineArgList.add("\"--$key=$value\"") - }) - return "[" + pipelineArgList.join(',') + "]" -} - -static String makePathAbsolute(String path) { - return '"$WORKSPACE/' + path + '"' -} \ No newline at end of file diff --git a/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml b/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml index 5d2c6648590d..7ba106a73d37 100644 --- a/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml +++ b/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml @@ -26,3 +26,30 @@ spec: selector: name: postgres type: LoadBalancer + +--- + +apiVersion: v1 +kind: ReplicationController +metadata: + name: postgres +spec: + replicas: 1 + selector: + name: postgres + template: + metadata: + name: postgres + labels: + name: postgres + spec: + containers: + - name: postgres + image: postgres + env: + - name: POSTGRES_PASSWORD + value: uuinkks + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - containerPort: 5432 diff --git a/sdks/java/io/hadoop-input-format/pom.xml b/sdks/java/io/hadoop-input-format/pom.xml index 4d86f4574b4c..d53afae6fe73 100644 --- a/sdks/java/io/hadoop-input-format/pom.xml +++ b/sdks/java/io/hadoop-input-format/pom.xml @@ -253,7 +253,7 @@ -beam_it_module=sdks/java/io/hadoop-input-format -beam_it_class=org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIOIT -beam_options_config_file=${beamRootProjectDir}/.test-infra/kubernetes/postgres/pkb-config-local.yml - -beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres.yml,${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml + -beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml -beam_it_options=${integrationTestPipelineOptions} diff --git a/sdks/java/io/jdbc/pom.xml b/sdks/java/io/jdbc/pom.xml index c4cc9c5b4b7c..5029ab8b8f6d 100644 --- a/sdks/java/io/jdbc/pom.xml +++ b/sdks/java/io/jdbc/pom.xml @@ -220,7 +220,7 @@ ${pkbBeamRunnerOption} -beam_options_config_file=${beamRootProjectDir}/.test-infra/kubernetes/postgres/pkb-config-local.yml - -beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres.yml,${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml + -beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml -beam_it_module=sdks/java/io/jdbc -beam_it_class=org.apache.beam.sdk.io.jdbc.JdbcIOIT