From 22e876574684a016c92241848240f40f89554166 Mon Sep 17 00:00:00 2001 From: Yichi Zhang Date: Fri, 6 May 2022 17:25:57 +0000 Subject: [PATCH] [BEAM-14429] Force java load test on dataflow runner v2 forceNumInitialBundles to 1 --- ..._LoadTests_CoGBK_Dataflow_V2_Java11.groovy | 10 ++++++++++ ..._LoadTests_CoGBK_Dataflow_V2_Java17.groovy | 10 ++++++++++ ...ob_LoadTests_GBK_Dataflow_V2_Java11.groovy | 19 ++++++++++++++----- ...ob_LoadTests_GBK_Dataflow_V2_Java17.groovy | 19 ++++++++++++++----- ..._LoadTests_ParDo_Dataflow_V2_Java11.groovy | 14 ++++++++++---- ..._LoadTests_ParDo_Dataflow_V2_Java17.groovy | 14 ++++++++++---- 6 files changed, 68 insertions(+), 18 deletions(-) diff --git a/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy index bd4b97c07e59..f96bd35b8253 100644 --- a/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy @@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder import CronJobBuilder import InfluxDBCredentialsHelper +// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved. + def loadTestConfigurations = { mode, isStreaming -> [ [ @@ -47,6 +49,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 20000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 1 } """.trim().replaceAll("\\s", ""), @@ -55,6 +58,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 2000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 1000 } """.trim().replaceAll("\\s", ""), @@ -86,6 +90,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 20000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 5 } """.trim().replaceAll("\\s", ""), @@ -94,6 +99,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 2000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 1000 } """.trim().replaceAll("\\s", ""), @@ -126,6 +132,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 2000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 200000 } """.trim().replaceAll("\\s", ""), @@ -134,6 +141,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 2000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 1000 } """.trim().replaceAll("\\s", ""), @@ -166,6 +174,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 2000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 1000 } """.trim().replaceAll("\\s", ""), @@ -174,6 +183,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 2000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 1000 } """.trim().replaceAll("\\s", ""), diff --git a/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy index 0978753f40e6..72646fab8bee 100644 --- a/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy @@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder import CronJobBuilder import InfluxDBCredentialsHelper +// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved. + def loadTestConfigurations = { mode, isStreaming -> [ [ @@ -47,6 +49,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 20000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 1 } """.trim().replaceAll("\\s", ""), @@ -55,6 +58,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 2000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 1000 } """.trim().replaceAll("\\s", ""), @@ -86,6 +90,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 20000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 5 } """.trim().replaceAll("\\s", ""), @@ -94,6 +99,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 2000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 1000 } """.trim().replaceAll("\\s", ""), @@ -126,6 +132,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 2000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 200000 } """.trim().replaceAll("\\s", ""), @@ -134,6 +141,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 2000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 1000 } """.trim().replaceAll("\\s", ""), @@ -166,6 +174,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 2000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 1000 } """.trim().replaceAll("\\s", ""), @@ -174,6 +183,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 2000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 1000 } """.trim().replaceAll("\\s", ""), diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy index 30c9dda37634..246ed4f0eaf7 100644 --- a/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy @@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder import CronJobBuilder import InfluxDBCredentialsHelper +// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved. + def loadTestConfigurations = { mode, isStreaming -> [ [ @@ -46,7 +48,8 @@ def loadTestConfigurations = { mode, isStreaming -> { "numRecords": 200000000, "keySizeBytes": 1, - "valueSizeBytes": 9 + "valueSizeBytes": 9, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), fanout : 1, @@ -77,7 +80,8 @@ def loadTestConfigurations = { mode, isStreaming -> { "numRecords": 20000000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), fanout : 1, @@ -109,7 +113,8 @@ def loadTestConfigurations = { mode, isStreaming -> { "numRecords": 20000, "keySizeBytes": 10000, - "valueSizeBytes": 90000 + "valueSizeBytes": 90000, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), fanout : 1, @@ -141,7 +146,8 @@ def loadTestConfigurations = { mode, isStreaming -> { "numRecords": 5000000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), fanout : 4, @@ -172,7 +178,8 @@ def loadTestConfigurations = { mode, isStreaming -> { "numRecords": 2500000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), fanout : 8, @@ -204,6 +211,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 20000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 200, "hotKeyFraction": 1 } @@ -237,6 +245,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 20000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 10, "hotKeyFraction": 1 } diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy index ca897e7eec10..f015d828870c 100644 --- a/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy @@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder import CronJobBuilder import InfluxDBCredentialsHelper +// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved. + def loadTestConfigurations = { mode, isStreaming -> [ [ @@ -46,7 +48,8 @@ def loadTestConfigurations = { mode, isStreaming -> { "numRecords": 200000000, "keySizeBytes": 1, - "valueSizeBytes": 9 + "valueSizeBytes": 9, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), fanout : 1, @@ -77,7 +80,8 @@ def loadTestConfigurations = { mode, isStreaming -> { "numRecords": 20000000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), fanout : 1, @@ -109,7 +113,8 @@ def loadTestConfigurations = { mode, isStreaming -> { "numRecords": 20000, "keySizeBytes": 10000, - "valueSizeBytes": 90000 + "valueSizeBytes": 90000, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), fanout : 1, @@ -141,7 +146,8 @@ def loadTestConfigurations = { mode, isStreaming -> { "numRecords": 5000000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), fanout : 4, @@ -172,7 +178,8 @@ def loadTestConfigurations = { mode, isStreaming -> { "numRecords": 2500000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), fanout : 8, @@ -204,6 +211,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 20000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 200, "hotKeyFraction": 1 } @@ -237,6 +245,7 @@ def loadTestConfigurations = { mode, isStreaming -> "numRecords": 20000000, "keySizeBytes": 10, "valueSizeBytes": 90, + "forceNumInitialBundles": 1, "numHotKeys": 10, "hotKeyFraction": 1 } diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy index 0a1cf8cfde53..86f5e6fe732b 100644 --- a/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy @@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder import CronJobBuilder import InfluxDBCredentialsHelper +// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved. + def commonLoadTestConfig = { jobType, isStreaming -> [ [ @@ -46,7 +48,8 @@ def commonLoadTestConfig = { jobType, isStreaming -> { "numRecords": 20000000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), iterations : 10, @@ -78,7 +81,8 @@ def commonLoadTestConfig = { jobType, isStreaming -> { "numRecords": 20000000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), iterations : 200, @@ -111,7 +115,8 @@ def commonLoadTestConfig = { jobType, isStreaming -> { "numRecords": 20000000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), iterations : 1, @@ -144,7 +149,8 @@ def commonLoadTestConfig = { jobType, isStreaming -> { "numRecords": 20000000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), iterations : 1, diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy index f09787225b96..ac40f39e62cb 100644 --- a/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy @@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder import CronJobBuilder import InfluxDBCredentialsHelper +// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved. + def commonLoadTestConfig = { jobType, isStreaming -> [ [ @@ -46,7 +48,8 @@ def commonLoadTestConfig = { jobType, isStreaming -> { "numRecords": 20000000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), iterations : 10, @@ -78,7 +81,8 @@ def commonLoadTestConfig = { jobType, isStreaming -> { "numRecords": 20000000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), iterations : 200, @@ -111,7 +115,8 @@ def commonLoadTestConfig = { jobType, isStreaming -> { "numRecords": 20000000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), iterations : 1, @@ -144,7 +149,8 @@ def commonLoadTestConfig = { jobType, isStreaming -> { "numRecords": 20000000, "keySizeBytes": 10, - "valueSizeBytes": 90 + "valueSizeBytes": 90, + "forceNumInitialBundles": 1 } """.trim().replaceAll("\\s", ""), iterations : 1,