[BEAM-13015] Add state caching benchmark and move benchmarks to their own module.

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -130,13 +130,15 @@ class BeamModulePlugin implements Plugin<Project> {
     boolean validateShadowJar = true
 
     /**
-     * Controls whether the 'jmh' source set is enabled for JMH benchmarks.
+     * Controls whether 'jmh' specific configuration is enabled to build a JMH
+     * focused module.
      *
-     * Add additional dependencies to the jmhCompile and jmhRuntime dependency
-     * sets.
+     * Add additional dependencies to the implementation configuration.
      *
      * Note that the JMH annotation processor is enabled by default and that
      * a 'jmh' task is created which executes JMH.
+     *
+     * Publishing is not allowed for JMH enabled projects.
      */
     boolean enableJmh = false
 
@@ -480,7 +482,7 @@ class BeamModulePlugin implements Plugin<Project> {
     def spotbugs_version = "4.0.6"
     def testcontainers_version = "1.15.1"
     def arrow_version = "5.0.0"
-    def jmh_version = "1.32"
+    def jmh_version = "1.34"
 
     // A map of maps containing common libraries used per language. To use:
     // dependencies {
@@ -793,6 +795,11 @@ class BeamModulePlugin implements Plugin<Project> {
       // Use the implicit it parameter of the closure to handle zero argument or one argument map calls.
       JavaNatureConfiguration configuration = it ? it as JavaNatureConfiguration : new JavaNatureConfiguration()
 
+      // Validate configuration
+      if (configuration.enableJmh && configuration.publish) {
+        throw new GradleException("Publishing of a benchmark project is not allowed. Benchmark projects are not meant to be consumed as artifacts for end users.");
+      }
+
       if (configuration.archivesBaseName) {
         project.archivesBaseName = configuration.archivesBaseName
       }
@@ -1325,25 +1332,12 @@ class BeamModulePlugin implements Plugin<Project> {
       }
 
       if (configuration.enableJmh) {
-        // We specifically use a separate source set for JMH to ensure that it does not
-        // become a required artifact
-        project.sourceSets {
-          jmh {
-            java {
-              srcDir "src/jmh/java"
-            }
-            resources {
-              srcDir "src/jmh/resources"
-            }
-          }
-        }
-
         project.dependencies {
-          jmhAnnotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$jmh_version"
-          jmhImplementation "org.openjdk.jmh:jmh-core:$jmh_version"
+          annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$jmh_version"
+          implementation "org.openjdk.jmh:jmh-core:$jmh_version"
         }
 
-        project.compileJmhJava {
+        project.compileJava {
           // Always exclude checkerframework on JMH generated code. It's slow,
           // and it often raises erroneous error because we don't have checker
           // annotations for generated code and test libraries.
@@ -1355,24 +1349,58 @@ class BeamModulePlugin implements Plugin<Project> {
           }
         }
 
-        project.task("jmh", type: JavaExec, dependsOn: project.jmhClasses, {
+        project.task("jmh", type: JavaExec, dependsOn: project.classes, {
           mainClass = "org.openjdk.jmh.Main"
-          classpath = project.sourceSets.jmh.compileClasspath + project.sourceSets.jmh.runtimeClasspath
+          classpath = project.sourceSets.main.runtimeClasspath
           // For a list of arguments, see
           // https://github.com/guozheng/jmh-tutorial/blob/master/README.md
           //
-          // Filter for a specific benchmark to run (uncomment below)
-          // Note that multiple regex are supported each as a separate argument.
-          // args 'BeamFnLoggingClientBenchmark.testLoggingWithAllOptionalParameters'
-          // args 'additional regexp...'
-          //
-          // Enumerate available benchmarks and exit (uncomment below)
+          // Enumerate available benchmarks and exit (uncomment below and disable other args)
           // args '-l'
           //
-          // Enable connecting a debugger by disabling forking (uncomment below)
+          // Enable connecting a debugger by disabling forking (uncomment below and disable other args)
           // Useful for debugging via an IDE such as Intellij
-          // args '-f0'
+          // args '-f=0'
+          // Specify -Pbenchmark=ProcessBundleBenchmark.testTinyBundle on the command
+          // line to enable running a single benchmark.
+
+          // Enable Google Cloud Profiler and upload the benchmarks to GCP.
+          if (project.hasProperty("benchmark")) {
+            args project.getProperty("benchmark")
+            // Add JVM arguments allowing one to additionally use Google's Java Profiler
+            // Agent: (see https://cloud.google.com/profiler/docs/profiling-java#installing-profiler for instructions on how to install)
+            if (project.file("/opt/cprof/profiler_java_agent.so").exists()) {
+              def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
+              def userName = System.getProperty("user.name").toLowerCase().replaceAll(" ", "_")
+              jvmArgs '-agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=' + userName + "_" + project.getProperty("benchmark").toLowerCase() + '_' + System.currentTimeMillis() + ',-cprof_project_id=' + gcpProject + ',-cprof_zone_name=us-central1-a'
+            }
+          } else {
+            // We filter for only Apache Beam benchmarks to ensure that we aren't
+            // running benchmarks that may have been packaged from another library
+            // that ends up on the runtime classpath.
+            args 'org.apache.beam'
+          }
+          args '-foe=true'
+        })
+
+        // Single shot of JMH benchmarks ensures that they can execute.
+        //
+        // Note that these tests will fail on JVMs that JMH doesn't support.
+        project.task("jmhTest", type: JavaExec, dependsOn: project.classes, {
+          mainClass = "org.openjdk.jmh.Main"
+          classpath = project.sourceSets.main.runtimeClasspath
+
+          // We filter for only Apache Beam benchmarks to ensure that we aren't
+          // running benchmarks that may have been packaged from another library
+          // that ends up on the runtime classpath.
+          args 'org.apache.beam'
+          args '-bm=ss'
+          args '-i=1'
+          args '-f=0'
+          args '-wf=0'
+          args '-foe=true'
         })
+        project.check.dependsOn("jmhTest")
       }
 
       project.ext.includeInJavaBom = configuration.publish

sdks/java/harness/build.gradle

@@ -33,7 +33,6 @@ applyJavaNature(
   ],
   automaticModuleName: 'org.apache.beam.fn.harness',
   validateShadowJar: false,
-  enableJmh: true,
   testShadowJar: true,
   shadowClosure:
   // Create an uber jar without repackaging for the SDK harness
@@ -76,22 +75,4 @@ dependencies {
   testImplementation project(":runners:core-construction-java")
   testImplementation project(path: ":sdks:java:fn-execution", configuration: "testRuntimeMigration")
   shadowTestRuntimeClasspath library.java.slf4j_jdk14
-  jmhImplementation project(path: ":sdks:java:harness", configuration: "shadow")
-  jmhImplementation project(":runners:java-fn-execution")
-  jmhRuntimeOnly library.java.slf4j_jdk14
-}
-
-jmh {
-  // Specify -Pbenchmark=ProcessBundleBenchmark.testTinyBundle on the command
-  // line to enable running a single benchmark.
-  if (project.hasProperty("benchmark")) {
-    args project.getProperty("benchmark")
-    // Add JVM arguments allowing one to additionally use Google's Java Profiler
-    // Agent: (see https://cloud.google.com/profiler/docs/profiling-java#installing-profiler for instructions on how to install)
-    if (file("/opt/cprof/profiler_java_agent.so").exists()) {
-      def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
-      def userName = System.getProperty("user.name").toLowerCase().replaceAll(" ", "_")
-      jvmArgs '-agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=' + userName + "_" + project.getProperty("benchmark").toLowerCase() + '_' + System.currentTimeMillis() + ',-cprof_project_id=' + gcpProject + ',-cprof_zone_name=us-central1-a'
-    }
-  }
 }

sdks/java/harness/jmh/build.gradle

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins { id 'org.apache.beam.module' }
+
+applyJavaNature(
+        automaticModuleName: 'org.apache.beam.fn.harness.jmh',
+        enableJmh: true,
+        publish: false)
+
+description = "Apache Beam :: SDKs :: Java :: Harness :: JMH"
+ext.summary = "This contains JMH benchmarks for the SDK Fn Harness for Beam Java"
+
+configurations {
+    jammAgent
+}
+
+dependencies {
+    implementation project(path: ":sdks:java:harness", configuration: "shadow")
+    implementation project(":runners:java-fn-execution")
+    runtimeOnly library.java.slf4j_jdk14
+    jammAgent library.java.jamm
+}
+
+jmh {
+    configurations.jammAgent.resolvedConfiguration.files.each {
+        jvmArgs '-javaagent:' + it
+    }
+}
+
+jmhTest {
+    configurations.jammAgent.resolvedConfiguration.files.each {
+        jvmArgs '-javaagent:' + it
+    }
+}
+