From 0a91d583e45b896da3e3a3ce9eff0dc30d2a6278 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Mon, 16 Jun 2025 13:17:59 -0400 Subject: [PATCH 1/3] create custom java container image for tests --- .../trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json | 3 +-- .../IO_Iceberg_Managed_Integration_Tests_Dataflow.json | 2 +- sdks/java/io/iceberg/build.gradle | 5 +++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json b/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json index 316d6e41388d..5abe02fc09c7 100644 --- a/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json +++ b/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json @@ -1,5 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 2, - "https://github.com/apache/beam/pull/35159": "moving WindowedValue and making an interface" + "modification": 1 } diff --git a/.github/trigger_files/IO_Iceberg_Managed_Integration_Tests_Dataflow.json b/.github/trigger_files/IO_Iceberg_Managed_Integration_Tests_Dataflow.json index 5abe02fc09c7..3a009261f4f9 100644 --- a/.github/trigger_files/IO_Iceberg_Managed_Integration_Tests_Dataflow.json +++ b/.github/trigger_files/IO_Iceberg_Managed_Integration_Tests_Dataflow.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 1 + "modification": 2 } diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index 2ac04eb067a3..06f85f406577 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -160,12 +160,17 @@ task integrationTest(type: Test) { task dataflowIntegrationTest(type: Test) { group = "Verification" + dependsOn project(":runners:google-cloud-dataflow-java:buildAndPushDockerJavaContainer") + finalizedBy project(":runners:google-cloud-dataflow-java:cleanUpDockerJavaImages") + def dockerJavaImageName = project.project(':runners:google-cloud-dataflow-java').ext.dockerJavaImageName + def args = [ "--runner=DataflowRunner", "--region=us-central1", "--project=${gcpProject}", "--tempLocation=${gcpTempLocation}", "--tempRoot=${gcpTempLocation}", + "--sdkContainerImage=${dockerJavaImageName}", "--experiments=use_runner_v2,use_staged_dataflow_worker_jar" ] if (project.hasProperty('enableManagedTransforms')) { From 7e6ab7ca7b06837147501e67616e99adfe397626 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Mon, 16 Jun 2025 13:42:30 -0400 Subject: [PATCH 2/3] syntax --- sdks/java/io/iceberg/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index 06f85f406577..c5b5b045ef85 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -160,8 +160,8 @@ task integrationTest(type: Test) { task dataflowIntegrationTest(type: Test) { group = "Verification" - dependsOn project(":runners:google-cloud-dataflow-java:buildAndPushDockerJavaContainer") - finalizedBy project(":runners:google-cloud-dataflow-java:cleanUpDockerJavaImages") + dependsOn ":runners:google-cloud-dataflow-java:buildAndPushDockerJavaContainer" + finalizedBy ":runners:google-cloud-dataflow-java:cleanUpDockerJavaImages" def dockerJavaImageName = project.project(':runners:google-cloud-dataflow-java').ext.dockerJavaImageName def args = [ From 819f4133df7f5b9b06d7fc31cd8d3565cba0ed89 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud Date: Mon, 16 Jun 2025 14:54:53 -0400 Subject: [PATCH 3/3] eval depends on df --- sdks/java/io/iceberg/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index 0d6e60e59361..e4e7e2f1095b 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -163,6 +163,7 @@ task integrationTest(type: Test) { task dataflowIntegrationTest(type: Test) { group = "Verification" + evaluationDependsOn(":runners:google-cloud-dataflow-java") dependsOn ":runners:google-cloud-dataflow-java:buildAndPushDockerJavaContainer" finalizedBy ":runners:google-cloud-dataflow-java:cleanUpDockerJavaImages" def dockerJavaImageName = project.project(':runners:google-cloud-dataflow-java').ext.dockerJavaImageName