diff --git a/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json b/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json index 53d94cfc4f1e..1bd74515152c 100644 --- a/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json +++ b/.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 3 + "modification": 4 } \ No newline at end of file diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json index ff31524ec3ae..a9e688973762 100644 --- a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json +++ b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json @@ -6,5 +6,6 @@ "https://github.com/apache/beam/pull/33267": "noting that PR #33267 should run this test", "https://github.com/apache/beam/pull/33322": "noting that PR #33322 should run this test", "https://github.com/apache/beam/pull/34123": "noting that PR #34123 should run this test", - "https://github.com/apache/beam/pull/34080": "noting that PR #34080 should run this test" + "https://github.com/apache/beam/pull/34080": "noting that PR #34080 should run this test", + "https://github.com/apache/beam/pull/34155": "noting that PR #34155 should run this test" } diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json index 8f47cf80e792..fb9656ec5f68 100644 --- a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json +++ b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json @@ -5,5 +5,6 @@ "https://github.com/apache/beam/pull/32546": "noting that PR #32546 should run this test", "https://github.com/apache/beam/pull/33267": "noting that PR #33267 should run this test", "https://github.com/apache/beam/pull/34123": "noting that PR #34123 should run this test", - "https://github.com/apache/beam/pull/34080": "noting that PR #34080 should run this test" + "https://github.com/apache/beam/pull/34080": "noting that PR #34080 should run this test", + "https://github.com/apache/beam/pull/34155": "noting that PR #34155 should run this test" } diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json index e3fd31aabce2..c747288cc25b 100644 --- a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json +++ b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json @@ -5,5 +5,6 @@ "https://github.com/apache/beam/pull/32546": "noting that PR #32546 should run this test", "https://github.com/apache/beam/pull/33267": "noting that PR #33267 should run this test", "https://github.com/apache/beam/pull/33322": "noting that PR #33322 should run this test", - "https://github.com/apache/beam/pull/34080": "noting that PR #34080 should run this test" + "https://github.com/apache/beam/pull/34080": "noting that PR #34080 should run this test", + "https://github.com/apache/beam/pull/34155": "noting that PR #34155 should run this test" } diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index bc9d17b69635..f2695f50ae09 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -836,6 +836,8 @@ class BeamModulePlugin implements Plugin { log4j2_core : "org.apache.logging.log4j:log4j-core:$log4j2_version", log4j2_to_slf4j : "org.apache.logging.log4j:log4j-to-slf4j:$log4j2_version", log4j2_slf4j_impl : "org.apache.logging.log4j:log4j-slf4j-impl:$log4j2_version", + log4j2_slf4j2_impl : "org.apache.logging.log4j:log4j-slf4j2-impl:$log4j2_version", + log4j2_log4j12_api : "org.apache.logging.log4j:log4j-1.2-api:$log4j2_version", mockito_core : "org.mockito:mockito-core:4.11.0", mockito_inline : "org.mockito:mockito-inline:4.11.0", mongo_java_driver : "org.mongodb:mongo-java-driver:3.12.11", diff --git a/runners/spark/spark_runner.gradle b/runners/spark/spark_runner.gradle index 1d95574fd332..f33c84d0e14c 100644 --- a/runners/spark/spark_runner.gradle +++ b/runners/spark/spark_runner.gradle @@ -216,12 +216,17 @@ dependencies { validatesRunner project(":sdks:java:io:hadoop-format").sourceSets.test.output validatesRunner project(path: ":examples:java", configuration: "testRuntimeMigration") validatesRunner project(path: project.path, configuration: "testRuntimeMigration") - validatesRunner library.java.log4j2_api - validatesRunner library.java.log4j2_core hadoopVersions.each { kv -> "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-common:$kv.value" // Force paranamer 2.8 to avoid issues when using Scala 2.12 "hadoopVersion$kv.key" "com.thoughtworks.paranamer:paranamer:2.8" + if ("$spark_version" >= "3.5.0") { + // Add log4j 2.x dependencies as Spark 3.5+ uses slf4j with log4j 2.x backend + "hadoopVersion$kv.key" library.java.log4j2_api + "hadoopVersion$kv.key" library.java.log4j2_core + "hadoopVersion$kv.key" library.java.log4j2_slf4j2_impl + "hadoopVersion$kv.key" library.java.log4j2_log4j12_api + } } } @@ -240,6 +245,11 @@ configurations.all { configurations.validatesRunner { // Exclude to make sure log4j binding is used exclude group: "org.slf4j", module: "slf4j-simple" + + if ("$spark_version" >= "3.5.0") { + // Exclude log4j 1.x dependencies to prevent conflict with log4j 2.x used by spark 3.5+ + exclude group: "log4j", module: "log4j" + } } hadoopVersions.each { kv -> @@ -247,6 +257,10 @@ hadoopVersions.each { kv -> resolutionStrategy { force "org.apache.hadoop:hadoop-common:$kv.value" } + if ("$spark_version" >= "3.5.0") { + // Exclude log4j 1.x dependencies to prevent conflict with log4j 2.x used by spark 3.5+ + exclude group: "log4j", module: "log4j" + } } }