From bfd380ceb1dee46497882be2ca53edf30cbc7a1b Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Fri, 24 Feb 2023 14:59:23 -0500 Subject: [PATCH 1/2] Use credential for mongodb performance tests --- .../job_PerformanceTests_MongoDBIO_IT.groovy | 4 ++- ...stCommit_Python_MongoDBIO_Load_Test.groovy | 2 +- .../mongodb/load-balancer/mongo.yml | 5 +++ .../beam/sdk/io/mongodb/MongoDBIOIT.java | 34 +++++++++++++++++-- 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy b/.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy index c9bc59eb8581..73ef6c4a53ea 100644 --- a/.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy +++ b/.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy @@ -24,7 +24,7 @@ String jobName = "beam_PerformanceTests_MongoDBIO_IT" job(jobName) { common.setTopLevelMainJobProperties(delegate) - common.setAutoJob(delegate,'H H/6 * * *') + common.setAutoJob(delegate,'H H/12 * * *') common.enablePhraseTriggeringFromPullRequest( delegate, 'Java MongoDBIO Performance Test', @@ -51,6 +51,8 @@ job(jobName) { mongoDBDatabaseName : 'beam', mongoDBHostName : "\$${mongoHostName}", mongoDBPort : 27017, + mongoDBUsername : 'root', + mongoDBPassword : 'uuinkkS', runner : 'DataflowRunner', autoscalingAlgorithm: 'NONE', numWorkers : '5' diff --git a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy index bee2b8f6e251..7b2e9cef02db 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy @@ -40,7 +40,7 @@ job(jobName) { temp_location: 'gs://temp-storage-for-perf-tests/loadtests', project : 'apache-beam-testing', region : 'us-central1', - mongo_uri : "mongodb://\$${mongoHostName}:27017", + mongo_uri : "mongodb://root:uuinkkS@\$${mongoHostName}:27017", num_documents: '1000000', batch_size : '10000', runner : 'DataflowRunner', diff --git a/.test-infra/kubernetes/mongodb/load-balancer/mongo.yml b/.test-infra/kubernetes/mongodb/load-balancer/mongo.yml index 70e196511610..19d0cf134a32 100644 --- a/.test-infra/kubernetes/mongodb/load-balancer/mongo.yml +++ b/.test-infra/kubernetes/mongodb/load-balancer/mongo.yml @@ -45,5 +45,10 @@ spec: containers: - name: mongo image: mongo + env: + - name: MONGO_INITDB_ROOT_USERNAME + value: root + - name: MONGO_INITDB_ROOT_PASSWORD + value: uuinkkS ports: - containerPort: 27017 diff --git a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBIOIT.java b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBIOIT.java index b585cca76885..1294d04b44b5 100644 --- a/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBIOIT.java +++ b/sdks/java/io/mongodb/src/test/java/org/apache/beam/sdk/io/mongodb/MongoDBIOIT.java @@ -30,6 +30,7 @@ import java.util.Set; import java.util.UUID; import java.util.function.Function; +import org.apache.beam.repackaged.core.org.apache.commons.lang3.StringUtils; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.io.GenerateSequence; import org.apache.beam.sdk.io.common.HashingFn; @@ -108,6 +109,20 @@ public interface MongoDBPipelineOptions extends IOTestPipelineOptions { String getMongoDBDatabaseName(); void setMongoDBDatabaseName(String name); + + @Description("Username for mongodb server") + @Default.String("") + String getMongoDBUsername(); + + void setMongoDBUsername(String name); + + // Note that passwords are not as secure an authentication as other methods, and used here for + // a test environment only. + @Description("Password for mongodb server") + @Default.String("") + String getMongoDBPassword(); + + void setMongoDBPassword(String value); } private static final Map EXPECTED_HASHES = @@ -127,8 +142,23 @@ public static void setUp() { PipelineOptionsFactory.register(MongoDBPipelineOptions.class); options = TestPipeline.testingPipelineOptions().as(MongoDBPipelineOptions.class); collection = String.format("test_%s", new Date().getTime()); - mongoUrl = - String.format("mongodb://%s:%s", options.getMongoDBHostName(), options.getMongoDBPort()); + if (StringUtils.isEmpty(options.getMongoDBUsername())) { + mongoUrl = + String.format("mongodb://%s:%s", options.getMongoDBHostName(), options.getMongoDBPort()); + } else if (StringUtils.isEmpty(options.getMongoDBPassword())) { + mongoUrl = + String.format( + "mongodb://%s@%s:%s", + options.getMongoDBUsername(), options.getMongoDBHostName(), options.getMongoDBPort()); + } else { + mongoUrl = + String.format( + "mongodb://%s:%s@%s:%s", + options.getMongoDBUsername(), + options.getMongoDBPassword(), + options.getMongoDBHostName(), + options.getMongoDBPort()); + } mongoClient = MongoClients.create(mongoUrl); settings = InfluxDBSettings.builder() From 7b8abcbd7f5a9ae7e395090a291125aab81a2d01 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Tue, 28 Feb 2023 21:37:40 -0500 Subject: [PATCH 2/2] Set cron job freq to twice a day as for other performance tests --- .../jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy index 7b2e9cef02db..9a66b51e5423 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy @@ -22,7 +22,7 @@ String jobName = "beam_python_mongoio_load_test" job(jobName) { common.setTopLevelMainJobProperties(delegate) - common.setAutoJob(delegate, 'H H/6 * * *') + common.setAutoJob(delegate, 'H H/12 * * *') common.enablePhraseTriggeringFromPullRequest( delegate, 'Python MongoDBIO Load Test',