From ba2188832a8a6e5c518848e7c5b4e94e76044026 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 1 Apr 2026 15:15:36 -0500 Subject: [PATCH 1/7] Build: Add runtime dependency guard for bundled artifacts Adds a build-time check that prevents accidental transitive dependency leaks into shipped shadow JARs and distribution archives. A checked-in runtime-deps.txt baseline lists every dependency resolved into each bundled artifact. checkRuntimeDeps compares resolved deps against the baseline and fails the build with a clear diff on mismatch, wired into the check lifecycle so it runs in CI automatically. This guards all 11 bundled modules: Spark runtime (3.4, 3.5, 4.0, 4.1), Flink runtime (1.20, 2.0, 2.1), cloud bundles (AWS, Azure, GCP), and Kafka Connect runtime. --- aws-bundle/build.gradle | 2 + aws-bundle/runtime-deps.txt | 70 +++++ azure-bundle/build.gradle | 2 + azure-bundle/runtime-deps.txt | 44 ++++ flink/v1.20/build.gradle | 2 + flink/v1.20/flink-runtime/runtime-deps.txt | 131 ++++++++++ flink/v2.0/build.gradle | 2 + flink/v2.0/flink-runtime/runtime-deps.txt | 131 ++++++++++ flink/v2.1/build.gradle | 2 + flink/v2.1/flink-runtime/runtime-deps.txt | 131 ++++++++++ gcp-bundle/build.gradle | 2 + gcp-bundle/runtime-deps.txt | 114 ++++++++ gradle/runtime-deps.gradle | 105 ++++++++ kafka-connect/build.gradle | 2 + .../kafka-connect-runtime/runtime-deps.txt | 247 ++++++++++++++++++ spark/v3.4/build.gradle | 2 + spark/v3.4/spark-runtime/runtime-deps.txt | 131 ++++++++++ spark/v3.5/build.gradle | 2 + spark/v3.5/spark-runtime/runtime-deps.txt | 131 ++++++++++ spark/v4.0/build.gradle | 2 + spark/v4.0/spark-runtime/runtime-deps.txt | 131 ++++++++++ spark/v4.1/build.gradle | 2 + spark/v4.1/spark-runtime/runtime-deps.txt | 131 ++++++++++ 23 files changed, 1519 insertions(+) create mode 100644 aws-bundle/runtime-deps.txt create mode 100644 azure-bundle/runtime-deps.txt create mode 100644 flink/v1.20/flink-runtime/runtime-deps.txt create mode 100644 flink/v2.0/flink-runtime/runtime-deps.txt create mode 100644 flink/v2.1/flink-runtime/runtime-deps.txt create mode 100644 gcp-bundle/runtime-deps.txt create mode 100644 gradle/runtime-deps.gradle create mode 100644 kafka-connect/kafka-connect-runtime/runtime-deps.txt create mode 100644 spark/v3.4/spark-runtime/runtime-deps.txt create mode 100644 spark/v3.5/spark-runtime/runtime-deps.txt create mode 100644 spark/v4.0/spark-runtime/runtime-deps.txt create mode 100644 spark/v4.1/spark-runtime/runtime-deps.txt diff --git a/aws-bundle/build.gradle b/aws-bundle/build.gradle index 5b9054812a50..b2fbcb111caf 100644 --- a/aws-bundle/build.gradle +++ b/aws-bundle/build.gradle @@ -66,4 +66,6 @@ project(":iceberg-aws-bundle") { jar { enabled = false } + + apply from: "${rootDir}/gradle/runtime-deps.gradle" } diff --git a/aws-bundle/runtime-deps.txt b/aws-bundle/runtime-deps.txt new file mode 100644 index 000000000000..7e692a6a111a --- /dev/null +++ b/aws-bundle/runtime-deps.txt @@ -0,0 +1,70 @@ +com.github.ben-manes.caffeine:caffeine:2.9.3 +com.google.errorprone:error_prone_annotations:2.10.0 +commons-codec:commons-codec:1.17.1 +commons-logging:commons-logging:1.2 +io.netty:netty-buffer:4.1.130.Final +io.netty:netty-codec-http2:4.1.130.Final +io.netty:netty-codec-http:4.1.130.Final +io.netty:netty-codec:4.1.130.Final +io.netty:netty-common:4.1.130.Final +io.netty:netty-handler:4.1.130.Final +io.netty:netty-resolver:4.1.130.Final +io.netty:netty-transport-classes-epoll:4.1.130.Final +io.netty:netty-transport-native-unix-common:4.1.130.Final +io.netty:netty-transport:4.1.130.Final +org.apache.httpcomponents:httpclient:4.5.13 +org.apache.httpcomponents:httpcore:4.4.16 +org.apache.logging.log4j:log4j-api:2.20.0 +org.apache.logging.log4j:log4j-core:2.20.0 +org.apache.logging.log4j:log4j-slf4j-impl:2.20.0 +org.checkerframework:checker-qual:3.19.0 +org.reactivestreams:reactive-streams:1.0.4 +org.slf4j:slf4j-api:2.0.17 +software.amazon.awssdk.crt:aws-crt:0.43.5 +software.amazon.awssdk:annotations:2.42.18 +software.amazon.awssdk:apache-client:2.42.18 +software.amazon.awssdk:arns:2.42.18 +software.amazon.awssdk:auth:2.42.18 +software.amazon.awssdk:aws-core:2.42.18 +software.amazon.awssdk:aws-json-protocol:2.42.18 +software.amazon.awssdk:aws-query-protocol:2.42.18 +software.amazon.awssdk:aws-xml-protocol:2.42.18 +software.amazon.awssdk:checksums-spi:2.42.18 +software.amazon.awssdk:checksums:2.42.18 +software.amazon.awssdk:cloudwatch-metric-publisher:2.42.18 +software.amazon.awssdk:cloudwatch:2.42.18 +software.amazon.awssdk:crt-core:2.42.18 +software.amazon.awssdk:dynamodb:2.42.18 +software.amazon.awssdk:endpoints-spi:2.42.18 +software.amazon.awssdk:glue:2.42.18 +software.amazon.awssdk:http-auth-aws-crt:2.42.18 +software.amazon.awssdk:http-auth-aws-eventstream:2.42.18 +software.amazon.awssdk:http-auth-aws:2.42.18 +software.amazon.awssdk:http-auth-spi:2.42.18 +software.amazon.awssdk:http-auth:2.42.18 +software.amazon.awssdk:http-client-spi:2.42.18 +software.amazon.awssdk:iam:2.42.18 +software.amazon.awssdk:identity-spi:2.42.18 +software.amazon.awssdk:json-utils:2.42.18 +software.amazon.awssdk:kms:2.42.18 +software.amazon.awssdk:lakeformation:2.42.18 +software.amazon.awssdk:metrics-spi:2.42.18 +software.amazon.awssdk:netty-nio-client:2.42.18 +software.amazon.awssdk:profiles:2.42.18 +software.amazon.awssdk:protocol-core:2.42.18 +software.amazon.awssdk:regions:2.42.18 +software.amazon.awssdk:retries-spi:2.42.18 +software.amazon.awssdk:retries:2.42.18 +software.amazon.awssdk:s3:2.42.18 +software.amazon.awssdk:s3control:2.42.18 +software.amazon.awssdk:sdk-core:2.42.18 +software.amazon.awssdk:smithy-rpcv2-protocol:2.42.18 +software.amazon.awssdk:sso:2.42.18 +software.amazon.awssdk:sts:2.42.18 +software.amazon.awssdk:third-party-jackson-core:2.42.18 +software.amazon.awssdk:third-party-jackson-dataformat-cbor:2.42.18 +software.amazon.awssdk:utils-lite:2.42.18 +software.amazon.awssdk:utils:2.42.18 +software.amazon.eventstream:eventstream:1.0.1 +software.amazon.s3.accessgrants:aws-s3-accessgrants-java-plugin:2.4.1 +software.amazon.s3.analyticsaccelerator:analyticsaccelerator-s3:1.3.1 diff --git a/azure-bundle/build.gradle b/azure-bundle/build.gradle index 0bdc30fdaa7e..6278d3fb66af 100644 --- a/azure-bundle/build.gradle +++ b/azure-bundle/build.gradle @@ -52,4 +52,6 @@ project(":iceberg-azure-bundle") { jar { enabled = false } + + apply from: "${rootDir}/gradle/runtime-deps.gradle" } diff --git a/azure-bundle/runtime-deps.txt b/azure-bundle/runtime-deps.txt new file mode 100644 index 000000000000..6ff75ae80e46 --- /dev/null +++ b/azure-bundle/runtime-deps.txt @@ -0,0 +1,44 @@ +com.azure:azure-core-http-netty:1.16.3 +com.azure:azure-core:1.57.1 +com.azure:azure-identity:1.18.2 +com.azure:azure-json:1.5.1 +com.azure:azure-security-keyvault-keys:4.10.5 +com.azure:azure-storage-blob:12.33.1 +com.azure:azure-storage-common:12.32.2 +com.azure:azure-storage-file-datalake:12.26.1 +com.azure:azure-storage-internal-avro:12.18.2 +com.azure:azure-xml:1.2.1 +com.fasterxml.jackson.core:jackson-annotations:2.18.4 +com.fasterxml.jackson.core:jackson-core:2.18.4.1 +com.fasterxml.jackson.core:jackson-databind:2.18.4 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.4 +com.microsoft.azure:msal4j-persistence-extension:1.3.0 +com.microsoft.azure:msal4j:1.23.1 +io.netty:netty-buffer:4.1.130.Final +io.netty:netty-codec-dns:4.1.128.Final +io.netty:netty-codec-http2:4.1.130.Final +io.netty:netty-codec-http:4.1.130.Final +io.netty:netty-codec-socks:4.1.130.Final +io.netty:netty-codec:4.1.130.Final +io.netty:netty-common:4.1.130.Final +io.netty:netty-handler-proxy:4.1.130.Final +io.netty:netty-handler:4.1.130.Final +io.netty:netty-resolver-dns-classes-macos:4.1.128.Final +io.netty:netty-resolver-dns-native-macos:4.1.128.Final +io.netty:netty-resolver-dns:4.1.128.Final +io.netty:netty-resolver:4.1.130.Final +io.netty:netty-tcnative-boringssl-static:2.0.74.Final +io.netty:netty-tcnative-classes:2.0.74.Final +io.netty:netty-transport-classes-epoll:4.1.130.Final +io.netty:netty-transport-classes-kqueue:4.1.130.Final +io.netty:netty-transport-native-epoll:4.1.130.Final +io.netty:netty-transport-native-kqueue:4.1.130.Final +io.netty:netty-transport-native-unix-common:4.1.130.Final +io.netty:netty-transport:4.1.130.Final +io.projectreactor.netty:reactor-netty-core:1.2.13 +io.projectreactor.netty:reactor-netty-http:1.2.13 +io.projectreactor:reactor-core:3.7.14 +net.java.dev.jna:jna-platform:5.17.0 +net.java.dev.jna:jna:5.17.0 +org.reactivestreams:reactive-streams:1.0.4 +org.slf4j:slf4j-api:2.0.17 diff --git a/flink/v1.20/build.gradle b/flink/v1.20/build.gradle index 3591bf37b1a7..c62a0305e00d 100644 --- a/flink/v1.20/build.gradle +++ b/flink/v1.20/build.gradle @@ -266,4 +266,6 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") { jar { enabled = false } + + apply from: "${rootDir}/gradle/runtime-deps.gradle" } diff --git a/flink/v1.20/flink-runtime/runtime-deps.txt b/flink/v1.20/flink-runtime/runtime-deps.txt new file mode 100644 index 000000000000..ce4f4205b179 --- /dev/null +++ b/flink/v1.20/flink-runtime/runtime-deps.txt @@ -0,0 +1,131 @@ +com.aliyun:credentials-java:0.3.12 +com.aliyun:tea:1.4.1 +com.fasterxml.jackson.core:jackson-annotations:2.21 +com.fasterxml.jackson.core:jackson-core:2.21.2 +com.fasterxml.jackson.core:jackson-databind:2.21.2 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 +com.github.ben-manes.caffeine:caffeine:2.9.3 +com.github.luben:zstd-jni:1.5.7-3 +com.google.android:annotations:4.1.1.4 +com.google.api-client:google-api-client:2.7.2 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 +com.google.api.grpc:proto-google-common-protos:2.66.0 +com.google.api.grpc:proto-google-iam-v1:1.61.0 +com.google.api:api-common:2.58.0 +com.google.api:gax-grpc:2.75.0 +com.google.api:gax-httpjson:2.75.0 +com.google.api:gax:2.75.0 +com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 +com.google.auth:google-auth-library-credentials:1.43.0 +com.google.auth:google-auth-library-oauth2-http:1.43.0 +com.google.auto.value:auto-value-annotations:1.11.0 +com.google.cloud:google-cloud-bigquery:2.60.0 +com.google.cloud:google-cloud-bigquerystorage:3.23.0 +com.google.cloud:google-cloud-core-http:2.65.0 +com.google.cloud:google-cloud-core:2.65.0 +com.google.code.findbugs:jsr305:3.0.2 +com.google.code.gson:gson:2.12.1 +com.google.errorprone:error_prone_annotations:2.42.0 +com.google.flatbuffers:flatbuffers-java:24.3.25 +com.google.guava:failureaccess:1.0.3 +com.google.guava:guava:33.5.0-jre +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +com.google.http-client:google-http-client-apache-v2:2.1.0 +com.google.http-client:google-http-client-appengine:2.1.0 +com.google.http-client:google-http-client-gson:2.1.0 +com.google.http-client:google-http-client:2.1.0 +com.google.j2objc:j2objc-annotations:3.1 +com.google.oauth-client:google-oauth-client:1.39.0 +com.google.protobuf:protobuf-java-util:4.33.2 +com.google.protobuf:protobuf-java:4.33.2 +com.squareup.okhttp3:okhttp:4.12.0 +com.squareup.okio:okio-jvm:3.6.0 +com.sun.xml.bind:jaxb-core:2.3.0 +com.sun.xml.bind:jaxb-impl:2.3.0 +dev.failsafe:failsafe:3.3.2 +io.airlift:aircompressor:2.0.3 +io.dropwizard.metrics:metrics-core:3.2.6 +io.grpc:grpc-alts:1.76.3 +io.grpc:grpc-api:1.76.3 +io.grpc:grpc-auth:1.76.3 +io.grpc:grpc-context:1.76.3 +io.grpc:grpc-core:1.76.3 +io.grpc:grpc-googleapis:1.76.3 +io.grpc:grpc-grpclb:1.76.3 +io.grpc:grpc-inprocess:1.76.3 +io.grpc:grpc-netty-shaded:1.76.3 +io.grpc:grpc-protobuf-lite:1.76.3 +io.grpc:grpc-protobuf:1.76.3 +io.grpc:grpc-stub:1.76.3 +io.grpc:grpc-util:1.76.3 +io.netty:netty-buffer:4.1.110.Final +io.netty:netty-common:4.1.110.Final +io.opencensus:opencensus-api:0.31.1 +io.opencensus:opencensus-contrib-http-util:0.31.1 +io.opentelemetry:opentelemetry-api:1.51.0 +io.opentelemetry:opentelemetry-context:1.51.0 +io.perfmark:perfmark-api:0.27.0 +org.apache.arrow:arrow-format:17.0.0 +org.apache.arrow:arrow-memory-core:17.0.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 +org.apache.arrow:arrow-memory-netty:17.0.0 +org.apache.arrow:arrow-vector:17.0.0 +org.apache.avro:avro:1.12.1 +org.apache.datasketches:datasketches-java:6.2.0 +org.apache.datasketches:datasketches-memory:3.0.2 +org.apache.flink:flink-metrics-dropwizard:1.20.1 +org.apache.httpcomponents.client5:httpclient5:5.6 +org.apache.httpcomponents.core5:httpcore5-h2:5.4 +org.apache.httpcomponents.core5:httpcore5:5.4 +org.apache.httpcomponents:httpclient:4.5.14 +org.apache.httpcomponents:httpcore:4.4.16 +org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-flink-1.20:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT +org.apache.orc:orc-core:1.9.8 +org.apache.orc:orc-shims:1.9.8 +org.apache.parquet:parquet-avro:1.17.0 +org.apache.parquet:parquet-column:1.17.0 +org.apache.parquet:parquet-common:1.17.0 +org.apache.parquet:parquet-encoding:1.17.0 +org.apache.parquet:parquet-format-structures:1.17.0 +org.apache.parquet:parquet-hadoop:1.17.0 +org.apache.parquet:parquet-jackson:1.17.0 +org.apache.parquet:parquet-variant:1.17.0 +org.checkerframework:checker-compat-qual:2.5.6 +org.checkerframework:checker-qual:3.49.0 +org.codehaus.mojo:animal-sniffer-annotations:1.24 +org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 +org.jacoco:org.jacoco.agent:0.8.8 +org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib:1.9.10 +org.json:json:20250517 +org.jspecify:jspecify:1.0.0 +org.locationtech.jts:jts-core:1.20.0 +org.projectnessie.nessie:nessie-client:0.107.4 +org.projectnessie.nessie:nessie-model:0.107.4 +org.roaringbitmap:RoaringBitmap:1.6.13 +org.threeten:threeten-extra:1.8.0 +org.threeten:threetenbp:1.7.0 diff --git a/flink/v2.0/build.gradle b/flink/v2.0/build.gradle index 5907f41b3544..478fce44b81b 100644 --- a/flink/v2.0/build.gradle +++ b/flink/v2.0/build.gradle @@ -266,4 +266,6 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") { jar { enabled = false } + + apply from: "${rootDir}/gradle/runtime-deps.gradle" } diff --git a/flink/v2.0/flink-runtime/runtime-deps.txt b/flink/v2.0/flink-runtime/runtime-deps.txt new file mode 100644 index 000000000000..4967fb6517c4 --- /dev/null +++ b/flink/v2.0/flink-runtime/runtime-deps.txt @@ -0,0 +1,131 @@ +com.aliyun:credentials-java:0.3.12 +com.aliyun:tea:1.4.1 +com.fasterxml.jackson.core:jackson-annotations:2.21 +com.fasterxml.jackson.core:jackson-core:2.21.2 +com.fasterxml.jackson.core:jackson-databind:2.21.2 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 +com.github.ben-manes.caffeine:caffeine:2.9.3 +com.github.luben:zstd-jni:1.5.7-3 +com.google.android:annotations:4.1.1.4 +com.google.api-client:google-api-client:2.7.2 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 +com.google.api.grpc:proto-google-common-protos:2.66.0 +com.google.api.grpc:proto-google-iam-v1:1.61.0 +com.google.api:api-common:2.58.0 +com.google.api:gax-grpc:2.75.0 +com.google.api:gax-httpjson:2.75.0 +com.google.api:gax:2.75.0 +com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 +com.google.auth:google-auth-library-credentials:1.43.0 +com.google.auth:google-auth-library-oauth2-http:1.43.0 +com.google.auto.value:auto-value-annotations:1.11.0 +com.google.cloud:google-cloud-bigquery:2.60.0 +com.google.cloud:google-cloud-bigquerystorage:3.23.0 +com.google.cloud:google-cloud-core-http:2.65.0 +com.google.cloud:google-cloud-core:2.65.0 +com.google.code.findbugs:jsr305:3.0.2 +com.google.code.gson:gson:2.12.1 +com.google.errorprone:error_prone_annotations:2.42.0 +com.google.flatbuffers:flatbuffers-java:24.3.25 +com.google.guava:failureaccess:1.0.3 +com.google.guava:guava:33.5.0-jre +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +com.google.http-client:google-http-client-apache-v2:2.1.0 +com.google.http-client:google-http-client-appengine:2.1.0 +com.google.http-client:google-http-client-gson:2.1.0 +com.google.http-client:google-http-client:2.1.0 +com.google.j2objc:j2objc-annotations:3.1 +com.google.oauth-client:google-oauth-client:1.39.0 +com.google.protobuf:protobuf-java-util:4.33.2 +com.google.protobuf:protobuf-java:4.33.2 +com.squareup.okhttp3:okhttp:4.12.0 +com.squareup.okio:okio-jvm:3.6.0 +com.sun.xml.bind:jaxb-core:2.3.0 +com.sun.xml.bind:jaxb-impl:2.3.0 +dev.failsafe:failsafe:3.3.2 +io.airlift:aircompressor:2.0.3 +io.dropwizard.metrics:metrics-core:3.2.6 +io.grpc:grpc-alts:1.76.3 +io.grpc:grpc-api:1.76.3 +io.grpc:grpc-auth:1.76.3 +io.grpc:grpc-context:1.76.3 +io.grpc:grpc-core:1.76.3 +io.grpc:grpc-googleapis:1.76.3 +io.grpc:grpc-grpclb:1.76.3 +io.grpc:grpc-inprocess:1.76.3 +io.grpc:grpc-netty-shaded:1.76.3 +io.grpc:grpc-protobuf-lite:1.76.3 +io.grpc:grpc-protobuf:1.76.3 +io.grpc:grpc-stub:1.76.3 +io.grpc:grpc-util:1.76.3 +io.netty:netty-buffer:4.1.110.Final +io.netty:netty-common:4.1.110.Final +io.opencensus:opencensus-api:0.31.1 +io.opencensus:opencensus-contrib-http-util:0.31.1 +io.opentelemetry:opentelemetry-api:1.51.0 +io.opentelemetry:opentelemetry-context:1.51.0 +io.perfmark:perfmark-api:0.27.0 +org.apache.arrow:arrow-format:17.0.0 +org.apache.arrow:arrow-memory-core:17.0.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 +org.apache.arrow:arrow-memory-netty:17.0.0 +org.apache.arrow:arrow-vector:17.0.0 +org.apache.avro:avro:1.12.1 +org.apache.datasketches:datasketches-java:6.2.0 +org.apache.datasketches:datasketches-memory:3.0.2 +org.apache.flink:flink-metrics-dropwizard:2.0.0 +org.apache.httpcomponents.client5:httpclient5:5.6 +org.apache.httpcomponents.core5:httpcore5-h2:5.4 +org.apache.httpcomponents.core5:httpcore5:5.4 +org.apache.httpcomponents:httpclient:4.5.14 +org.apache.httpcomponents:httpcore:4.4.16 +org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-flink-2.0:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT +org.apache.orc:orc-core:1.9.8 +org.apache.orc:orc-shims:1.9.8 +org.apache.parquet:parquet-avro:1.17.0 +org.apache.parquet:parquet-column:1.17.0 +org.apache.parquet:parquet-common:1.17.0 +org.apache.parquet:parquet-encoding:1.17.0 +org.apache.parquet:parquet-format-structures:1.17.0 +org.apache.parquet:parquet-hadoop:1.17.0 +org.apache.parquet:parquet-jackson:1.17.0 +org.apache.parquet:parquet-variant:1.17.0 +org.checkerframework:checker-compat-qual:2.5.6 +org.checkerframework:checker-qual:3.49.0 +org.codehaus.mojo:animal-sniffer-annotations:1.24 +org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 +org.jacoco:org.jacoco.agent:0.8.8 +org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib:1.9.10 +org.json:json:20250517 +org.jspecify:jspecify:1.0.0 +org.locationtech.jts:jts-core:1.20.0 +org.projectnessie.nessie:nessie-client:0.107.4 +org.projectnessie.nessie:nessie-model:0.107.4 +org.roaringbitmap:RoaringBitmap:1.6.13 +org.threeten:threeten-extra:1.8.0 +org.threeten:threetenbp:1.7.0 diff --git a/flink/v2.1/build.gradle b/flink/v2.1/build.gradle index 91081bdc2e42..24d6f557a3fb 100644 --- a/flink/v2.1/build.gradle +++ b/flink/v2.1/build.gradle @@ -266,4 +266,6 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") { jar { enabled = false } + + apply from: "${rootDir}/gradle/runtime-deps.gradle" } diff --git a/flink/v2.1/flink-runtime/runtime-deps.txt b/flink/v2.1/flink-runtime/runtime-deps.txt new file mode 100644 index 000000000000..4155b7eddc88 --- /dev/null +++ b/flink/v2.1/flink-runtime/runtime-deps.txt @@ -0,0 +1,131 @@ +com.aliyun:credentials-java:0.3.12 +com.aliyun:tea:1.4.1 +com.fasterxml.jackson.core:jackson-annotations:2.21 +com.fasterxml.jackson.core:jackson-core:2.21.2 +com.fasterxml.jackson.core:jackson-databind:2.21.2 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 +com.github.ben-manes.caffeine:caffeine:2.9.3 +com.github.luben:zstd-jni:1.5.7-3 +com.google.android:annotations:4.1.1.4 +com.google.api-client:google-api-client:2.7.2 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 +com.google.api.grpc:proto-google-common-protos:2.66.0 +com.google.api.grpc:proto-google-iam-v1:1.61.0 +com.google.api:api-common:2.58.0 +com.google.api:gax-grpc:2.75.0 +com.google.api:gax-httpjson:2.75.0 +com.google.api:gax:2.75.0 +com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 +com.google.auth:google-auth-library-credentials:1.43.0 +com.google.auth:google-auth-library-oauth2-http:1.43.0 +com.google.auto.value:auto-value-annotations:1.11.0 +com.google.cloud:google-cloud-bigquery:2.60.0 +com.google.cloud:google-cloud-bigquerystorage:3.23.0 +com.google.cloud:google-cloud-core-http:2.65.0 +com.google.cloud:google-cloud-core:2.65.0 +com.google.code.findbugs:jsr305:3.0.2 +com.google.code.gson:gson:2.12.1 +com.google.errorprone:error_prone_annotations:2.42.0 +com.google.flatbuffers:flatbuffers-java:24.3.25 +com.google.guava:failureaccess:1.0.3 +com.google.guava:guava:33.5.0-jre +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +com.google.http-client:google-http-client-apache-v2:2.1.0 +com.google.http-client:google-http-client-appengine:2.1.0 +com.google.http-client:google-http-client-gson:2.1.0 +com.google.http-client:google-http-client:2.1.0 +com.google.j2objc:j2objc-annotations:3.1 +com.google.oauth-client:google-oauth-client:1.39.0 +com.google.protobuf:protobuf-java-util:4.33.2 +com.google.protobuf:protobuf-java:4.33.2 +com.squareup.okhttp3:okhttp:4.12.0 +com.squareup.okio:okio-jvm:3.6.0 +com.sun.xml.bind:jaxb-core:2.3.0 +com.sun.xml.bind:jaxb-impl:2.3.0 +dev.failsafe:failsafe:3.3.2 +io.airlift:aircompressor:2.0.3 +io.dropwizard.metrics:metrics-core:3.2.6 +io.grpc:grpc-alts:1.76.3 +io.grpc:grpc-api:1.76.3 +io.grpc:grpc-auth:1.76.3 +io.grpc:grpc-context:1.76.3 +io.grpc:grpc-core:1.76.3 +io.grpc:grpc-googleapis:1.76.3 +io.grpc:grpc-grpclb:1.76.3 +io.grpc:grpc-inprocess:1.76.3 +io.grpc:grpc-netty-shaded:1.76.3 +io.grpc:grpc-protobuf-lite:1.76.3 +io.grpc:grpc-protobuf:1.76.3 +io.grpc:grpc-stub:1.76.3 +io.grpc:grpc-util:1.76.3 +io.netty:netty-buffer:4.1.110.Final +io.netty:netty-common:4.1.110.Final +io.opencensus:opencensus-api:0.31.1 +io.opencensus:opencensus-contrib-http-util:0.31.1 +io.opentelemetry:opentelemetry-api:1.51.0 +io.opentelemetry:opentelemetry-context:1.51.0 +io.perfmark:perfmark-api:0.27.0 +org.apache.arrow:arrow-format:17.0.0 +org.apache.arrow:arrow-memory-core:17.0.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 +org.apache.arrow:arrow-memory-netty:17.0.0 +org.apache.arrow:arrow-vector:17.0.0 +org.apache.avro:avro:1.12.1 +org.apache.datasketches:datasketches-java:6.2.0 +org.apache.datasketches:datasketches-memory:3.0.2 +org.apache.flink:flink-metrics-dropwizard:2.1.0 +org.apache.httpcomponents.client5:httpclient5:5.6 +org.apache.httpcomponents.core5:httpcore5-h2:5.4 +org.apache.httpcomponents.core5:httpcore5:5.4 +org.apache.httpcomponents:httpclient:4.5.14 +org.apache.httpcomponents:httpcore:4.4.16 +org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-flink-2.1:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT +org.apache.orc:orc-core:1.9.8 +org.apache.orc:orc-shims:1.9.8 +org.apache.parquet:parquet-avro:1.17.0 +org.apache.parquet:parquet-column:1.17.0 +org.apache.parquet:parquet-common:1.17.0 +org.apache.parquet:parquet-encoding:1.17.0 +org.apache.parquet:parquet-format-structures:1.17.0 +org.apache.parquet:parquet-hadoop:1.17.0 +org.apache.parquet:parquet-jackson:1.17.0 +org.apache.parquet:parquet-variant:1.17.0 +org.checkerframework:checker-compat-qual:2.5.6 +org.checkerframework:checker-qual:3.49.0 +org.codehaus.mojo:animal-sniffer-annotations:1.24 +org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 +org.jacoco:org.jacoco.agent:0.8.8 +org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib:1.9.10 +org.json:json:20250517 +org.jspecify:jspecify:1.0.0 +org.locationtech.jts:jts-core:1.20.0 +org.projectnessie.nessie:nessie-client:0.107.4 +org.projectnessie.nessie:nessie-model:0.107.4 +org.roaringbitmap:RoaringBitmap:1.6.13 +org.threeten:threeten-extra:1.8.0 +org.threeten:threetenbp:1.7.0 diff --git a/gcp-bundle/build.gradle b/gcp-bundle/build.gradle index 6ebe05ccdbce..84482a7b039b 100644 --- a/gcp-bundle/build.gradle +++ b/gcp-bundle/build.gradle @@ -59,4 +59,6 @@ project(":iceberg-gcp-bundle") { jar { enabled = false } + + apply from: "${rootDir}/gradle/runtime-deps.gradle" } diff --git a/gcp-bundle/runtime-deps.txt b/gcp-bundle/runtime-deps.txt new file mode 100644 index 000000000000..5fca81c617c8 --- /dev/null +++ b/gcp-bundle/runtime-deps.txt @@ -0,0 +1,114 @@ +com.fasterxml.jackson.core:jackson-annotations:2.18.2 +com.fasterxml.jackson.core:jackson-core:2.18.2 +com.fasterxml.jackson.core:jackson-databind:2.18.2 +com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.18.2 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.2 +com.fasterxml.woodstox:woodstox-core:7.0.0 +com.google.android:annotations:4.1.1.4 +com.google.api-client:google-api-client:2.7.2 +com.google.api.grpc:gapic-google-cloud-storage-v2:2.64.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:grpc-google-cloud-storage-v2:2.64.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 +com.google.api.grpc:proto-google-cloud-kms-v1:0.181.0 +com.google.api.grpc:proto-google-cloud-monitoring-v3:3.88.0 +com.google.api.grpc:proto-google-cloud-storage-v2:2.64.0 +com.google.api.grpc:proto-google-common-protos:2.66.0 +com.google.api.grpc:proto-google-iam-v1:1.61.0 +com.google.api:api-common:2.58.0 +com.google.api:gax-grpc:2.75.0 +com.google.api:gax-httpjson:2.75.0 +com.google.api:gax:2.75.0 +com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 +com.google.apis:google-api-services-storage:v1-rev20251118-2.0.0 +com.google.auth:google-auth-library-credentials:1.43.0 +com.google.auth:google-auth-library-oauth2-http:1.43.0 +com.google.auto.value:auto-value-annotations:1.11.1 +com.google.cloud.gcs.analytics:client:1.2.3 +com.google.cloud.gcs.analytics:gcs-analytics-core:1.2.3 +com.google.cloud.opentelemetry:detector-resources-support:0.33.0 +com.google.cloud.opentelemetry:exporter-metrics:0.33.0 +com.google.cloud.opentelemetry:shared-resourcemapping:0.33.0 +com.google.cloud:google-cloud-bigquery:2.60.0 +com.google.cloud:google-cloud-bigquerystorage:3.23.0 +com.google.cloud:google-cloud-core-grpc:2.65.0 +com.google.cloud:google-cloud-core-http:2.65.0 +com.google.cloud:google-cloud-core:2.65.0 +com.google.cloud:google-cloud-kms:2.90.0 +com.google.cloud:google-cloud-monitoring:3.88.0 +com.google.cloud:google-cloud-storage:2.64.0 +com.google.code.findbugs:jsr305:3.0.2 +com.google.code.gson:gson:2.12.1 +com.google.errorprone:error_prone_annotations:2.42.0 +com.google.flatbuffers:flatbuffers-java:24.3.25 +com.google.guava:failureaccess:1.0.3 +com.google.guava:guava:33.5.0-jre +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +com.google.http-client:google-http-client-apache-v2:2.1.0 +com.google.http-client:google-http-client-appengine:2.1.0 +com.google.http-client:google-http-client-gson:2.1.0 +com.google.http-client:google-http-client-jackson2:2.1.0 +com.google.http-client:google-http-client:2.1.0 +com.google.j2objc:j2objc-annotations:3.1 +com.google.oauth-client:google-oauth-client:1.39.0 +com.google.protobuf:protobuf-java-util:4.33.2 +com.google.protobuf:protobuf-java:4.33.2 +com.google.re2j:re2j:1.8 +commons-codec:commons-codec:1.18.0 +io.grpc:grpc-alts:1.76.3 +io.grpc:grpc-api:1.76.3 +io.grpc:grpc-auth:1.76.3 +io.grpc:grpc-context:1.76.3 +io.grpc:grpc-core:1.76.3 +io.grpc:grpc-googleapis:1.76.3 +io.grpc:grpc-grpclb:1.76.3 +io.grpc:grpc-inprocess:1.76.3 +io.grpc:grpc-netty-shaded:1.76.3 +io.grpc:grpc-opentelemetry:1.76.3 +io.grpc:grpc-protobuf-lite:1.76.3 +io.grpc:grpc-protobuf:1.76.3 +io.grpc:grpc-rls:1.76.3 +io.grpc:grpc-services:1.76.3 +io.grpc:grpc-stub:1.76.3 +io.grpc:grpc-util:1.76.3 +io.grpc:grpc-xds:1.76.3 +io.netty:netty-buffer:4.1.110.Final +io.netty:netty-common:4.1.110.Final +io.opencensus:opencensus-api:0.31.1 +io.opencensus:opencensus-contrib-http-util:0.31.1 +io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37.0-alpha +io.opentelemetry.semconv:opentelemetry-semconv:1.29.0-alpha +io.opentelemetry:opentelemetry-api:1.51.0 +io.opentelemetry:opentelemetry-context:1.51.0 +io.opentelemetry:opentelemetry-sdk-common:1.51.0 +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.51.0 +io.opentelemetry:opentelemetry-sdk-logs:1.51.0 +io.opentelemetry:opentelemetry-sdk-metrics:1.51.0 +io.opentelemetry:opentelemetry-sdk-trace:1.51.0 +io.opentelemetry:opentelemetry-sdk:1.51.0 +io.perfmark:perfmark-api:0.27.0 +javax.annotation:javax.annotation-api:1.3.2 +org.apache.arrow:arrow-format:17.0.0 +org.apache.arrow:arrow-memory-core:17.0.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 +org.apache.arrow:arrow-memory-netty:17.0.0 +org.apache.arrow:arrow-vector:17.0.0 +org.apache.commons:commons-lang3:3.20.0 +org.apache.httpcomponents:httpclient:4.5.14 +org.apache.httpcomponents:httpcore:4.4.16 +org.checkerframework:checker-compat-qual:2.5.6 +org.checkerframework:checker-qual:3.49.0 +org.codehaus.mojo:animal-sniffer-annotations:1.24 +org.codehaus.woodstox:stax2-api:4.2.2 +org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.json:json:20250517 +org.jspecify:jspecify:1.0.0 +org.slf4j:slf4j-api:2.0.17 +org.threeten:threeten-extra:1.8.0 +org.threeten:threetenbp:1.7.0 diff --git a/gradle/runtime-deps.gradle b/gradle/runtime-deps.gradle new file mode 100644 index 000000000000..ea8b123181f6 --- /dev/null +++ b/gradle/runtime-deps.gradle @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Guards the runtime dependency surface for shadow JAR modules. +// +// Prevents accidental transitive dependency growth in shipped shadow JARs. +// Without this guard, adding a single catalog module as 'implementation' +// instead of 'compileOnly' can silently leak dozens of transitive artifacts +// into the runtime JAR, inflating its size and introducing unlicensed code. +// +// Apply this script in any project that ships a bundled artifact: Spark and +// Flink runtime shadow JARs, cloud bundles (aws, azure, gcp), and Kafka +// Connect runtime distribution. +// +// It adds two tasks: +// +// generateRuntimeDeps - resolves runtimeClasspath and writes a sorted +// baseline of group:artifact:version coordinates +// to runtime-deps.txt in the project directory. +// +// checkRuntimeDeps - compares the resolved dependencies against the +// checked-in baseline and fails with a diff if +// they don't match. Wired into the 'check' lifecycle. +// +// Workflow: +// 1. ./gradlew check -- fails if deps changed +// 2. ./gradlew generateRuntimeDeps -- auto-updates all baselines +// 3. Update LICENSE and NOTICE if dependency licenses changed -- This is a Manual Step +// 4. Commit + +def depsFile = file("${projectDir}/runtime-deps.txt") + +def resolveRuntimeDeps = { + configurations.runtimeClasspath.resolvedConfiguration + .resolvedArtifacts + .collect { "${it.moduleVersion.id.group}:${it.moduleVersion.id.name}:${it.moduleVersion.id.version}" } + .toSorted() + .toUnique() +} + +tasks.register('generateRuntimeDeps') { + group = 'verification' + description = 'Regenerate the runtime dependency baseline after intentional dependency changes' + outputs.file(depsFile) + doLast { + def deps = resolveRuntimeDeps() + depsFile.text = deps.join('\n') + '\n' + logger.lifecycle("Wrote ${deps.size()} dependencies to ${depsFile}") + logger.lifecycle("Review the diff, then update LICENSE and NOTICE if licenses changed.") + } +} + +tasks.register('checkRuntimeDeps') { + group = 'verification' + description = 'Verify runtime dependencies match the checked-in baseline' + inputs.files(configurations.runtimeClasspath) + outputs.file(depsFile) + doLast { + if (!depsFile.exists()) { + throw new GradleException( + "Missing ${depsFile.name} in ${projectDir}.\n" + + "Run: ./gradlew ${project.path}:generateRuntimeDeps") + } + + def actual = resolveRuntimeDeps() + def expected = depsFile.readLines().findAll { it.trim() }.toSorted() + + if (actual != expected) { + def added = actual - expected + def removed = expected - actual + def msg = new StringBuilder() + msg.append("Runtime dependency baseline mismatch for ${project.name}!\n") + if (added) { + msg.append("\n Added (${added.size()}):\n") + added.each { msg.append(" + ${it}\n") } + } + if (removed) { + msg.append("\n Removed (${removed.size()}):\n") + removed.each { msg.append(" - ${it}\n") } + } + msg.append("\nTo update the baseline run:\n") + msg.append(" ./gradlew ${project.path}:generateRuntimeDeps\n") + msg.append("\nThen update LICENSE and NOTICE to reflect the dependency changes.") + throw new GradleException(msg.toString()) + } + } +} + +check.dependsOn checkRuntimeDeps diff --git a/kafka-connect/build.gradle b/kafka-connect/build.gradle index e93ad8641f79..8e4b8877d7e3 100644 --- a/kafka-connect/build.gradle +++ b/kafka-connect/build.gradle @@ -254,6 +254,8 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') { check.dependsOn integrationTest assemble.dependsOn distZip, hiveDistZip + + apply from: "${rootDir}/gradle/runtime-deps.gradle" } project(':iceberg-kafka-connect:iceberg-kafka-connect-transforms') { diff --git a/kafka-connect/kafka-connect-runtime/runtime-deps.txt b/kafka-connect/kafka-connect-runtime/runtime-deps.txt new file mode 100644 index 000000000000..2250b90179cd --- /dev/null +++ b/kafka-connect/kafka-connect-runtime/runtime-deps.txt @@ -0,0 +1,247 @@ +com.azure:azure-core-http-netty:1.16.3 +com.azure:azure-core:1.57.1 +com.azure:azure-identity:1.18.2 +com.azure:azure-json:1.5.1 +com.azure:azure-storage-blob:12.33.1 +com.azure:azure-storage-common:12.32.2 +com.azure:azure-storage-file-datalake:12.26.1 +com.azure:azure-storage-internal-avro:12.18.2 +com.azure:azure-xml:1.2.1 +com.fasterxml.jackson.core:jackson-annotations:2.21 +com.fasterxml.jackson.core:jackson-core:2.21.2 +com.fasterxml.jackson.core:jackson-databind:2.21.2 +com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.21.2 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 +com.fasterxml.woodstox:woodstox-core:6.7.0 +com.github.ben-manes.caffeine:caffeine:2.9.3 +com.github.luben:zstd-jni:1.5.7-3 +com.github.pjfanning:jersey-json:1.22.0 +com.google.android:annotations:4.1.1.4 +com.google.api-client:google-api-client:2.7.2 +com.google.api.grpc:gapic-google-cloud-storage-v2:2.64.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:grpc-google-cloud-storage-v2:2.64.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 +com.google.api.grpc:proto-google-cloud-monitoring-v3:3.88.0 +com.google.api.grpc:proto-google-cloud-storage-v2:2.64.0 +com.google.api.grpc:proto-google-common-protos:2.66.0 +com.google.api.grpc:proto-google-iam-v1:1.61.0 +com.google.api:api-common:2.58.0 +com.google.api:gax-grpc:2.75.0 +com.google.api:gax-httpjson:2.75.0 +com.google.api:gax:2.75.0 +com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 +com.google.apis:google-api-services-storage:v1-rev20251118-2.0.0 +com.google.auth:google-auth-library-credentials:1.43.0 +com.google.auth:google-auth-library-oauth2-http:1.43.0 +com.google.auto.value:auto-value-annotations:1.11.0 +com.google.cloud.opentelemetry:detector-resources-support:0.33.0 +com.google.cloud.opentelemetry:exporter-metrics:0.33.0 +com.google.cloud.opentelemetry:shared-resourcemapping:0.33.0 +com.google.cloud:google-cloud-bigquery:2.60.0 +com.google.cloud:google-cloud-bigquerystorage:3.23.0 +com.google.cloud:google-cloud-core-grpc:2.65.0 +com.google.cloud:google-cloud-core-http:2.65.0 +com.google.cloud:google-cloud-core:2.65.0 +com.google.cloud:google-cloud-monitoring:3.88.0 +com.google.cloud:google-cloud-storage:2.64.0 +com.google.code.findbugs:jsr305:3.0.2 +com.google.code.gson:gson:2.12.1 +com.google.errorprone:error_prone_annotations:2.42.0 +com.google.flatbuffers:flatbuffers-java:24.3.25 +com.google.guava:failureaccess:1.0.3 +com.google.guava:guava:33.5.0-jre +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +com.google.http-client:google-http-client-apache-v2:2.1.0 +com.google.http-client:google-http-client-appengine:2.1.0 +com.google.http-client:google-http-client-gson:2.1.0 +com.google.http-client:google-http-client-jackson2:2.1.0 +com.google.http-client:google-http-client:2.1.0 +com.google.j2objc:j2objc-annotations:3.1 +com.google.oauth-client:google-oauth-client:1.39.0 +com.google.protobuf:protobuf-java-util:4.33.2 +com.google.protobuf:protobuf-java:4.33.2 +com.google.re2j:re2j:1.8 +com.jcraft:jsch:0.1.55 +com.microsoft.azure:msal4j-persistence-extension:1.3.0 +com.microsoft.azure:msal4j:1.23.1 +com.sun.xml.bind:jaxb-impl:2.2.3-1 +commons-cli:commons-cli:1.9.0 +commons-codec:commons-codec:1.19.0 +commons-io:commons-io:2.20.0 +commons-logging:commons-logging:1.2 +commons-net:commons-net:3.9.0 +commons-pool:commons-pool:1.6 +dev.failsafe:failsafe:3.3.2 +dnsjava:dnsjava:3.6.1 +io.airlift:aircompressor:2.0.3 +io.dropwizard.metrics:metrics-core:3.2.4 +io.grpc:grpc-alts:1.76.3 +io.grpc:grpc-api:1.76.3 +io.grpc:grpc-auth:1.76.3 +io.grpc:grpc-context:1.76.3 +io.grpc:grpc-core:1.76.3 +io.grpc:grpc-googleapis:1.76.3 +io.grpc:grpc-grpclb:1.76.3 +io.grpc:grpc-inprocess:1.76.3 +io.grpc:grpc-netty-shaded:1.80.0 +io.grpc:grpc-opentelemetry:1.76.3 +io.grpc:grpc-protobuf-lite:1.76.3 +io.grpc:grpc-protobuf:1.76.3 +io.grpc:grpc-rls:1.76.3 +io.grpc:grpc-services:1.76.3 +io.grpc:grpc-stub:1.76.3 +io.grpc:grpc-util:1.76.3 +io.grpc:grpc-xds:1.76.3 +io.netty:netty-buffer:4.1.130.Final +io.netty:netty-codec-dns:4.1.128.Final +io.netty:netty-codec-http2:4.1.130.Final +io.netty:netty-codec-http:4.1.130.Final +io.netty:netty-codec-socks:4.1.130.Final +io.netty:netty-codec:4.1.130.Final +io.netty:netty-common:4.1.130.Final +io.netty:netty-handler-proxy:4.1.130.Final +io.netty:netty-handler:4.1.130.Final +io.netty:netty-resolver-dns-classes-macos:4.1.128.Final +io.netty:netty-resolver-dns-native-macos:4.1.128.Final +io.netty:netty-resolver-dns:4.1.128.Final +io.netty:netty-resolver:4.1.130.Final +io.netty:netty-tcnative-boringssl-static:2.0.74.Final +io.netty:netty-tcnative-classes:2.0.74.Final +io.netty:netty-transport-classes-epoll:4.1.130.Final +io.netty:netty-transport-classes-kqueue:4.1.130.Final +io.netty:netty-transport-native-epoll:4.1.130.Final +io.netty:netty-transport-native-kqueue:4.1.130.Final +io.netty:netty-transport-native-unix-common:4.1.130.Final +io.netty:netty-transport:4.1.130.Final +io.opencensus:opencensus-api:0.31.1 +io.opencensus:opencensus-contrib-http-util:0.31.1 +io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37.0-alpha +io.opentelemetry.semconv:opentelemetry-semconv:1.29.0-alpha +io.opentelemetry:opentelemetry-api:1.51.0 +io.opentelemetry:opentelemetry-context:1.51.0 +io.opentelemetry:opentelemetry-sdk-common:1.51.0 +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.51.0 +io.opentelemetry:opentelemetry-sdk-logs:1.51.0 +io.opentelemetry:opentelemetry-sdk-metrics:1.51.0 +io.opentelemetry:opentelemetry-sdk-trace:1.51.0 +io.opentelemetry:opentelemetry-sdk:1.51.0 +io.perfmark:perfmark-api:0.27.0 +io.projectreactor.netty:reactor-netty-core:1.2.13 +io.projectreactor.netty:reactor-netty-http:1.2.13 +io.projectreactor:reactor-core:3.7.14 +jakarta.activation:jakarta.activation-api:1.2.1 +javax.annotation:javax.annotation-api:1.3.2 +javax.servlet.jsp:jsp-api:2.1 +javax.servlet:javax.servlet-api:3.1.0 +javax.xml.bind:jaxb-api:2.2.2 +javax.xml.stream:stax-api:1.0-2 +net.java.dev.jna:jna-platform:5.17.0 +net.java.dev.jna:jna:5.17.0 +org.apache.arrow:arrow-format:17.0.0 +org.apache.arrow:arrow-memory-core:17.0.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 +org.apache.arrow:arrow-memory-netty:17.0.0 +org.apache.arrow:arrow-vector:17.0.0 +org.apache.avro:avro:1.12.1 +org.apache.commons:commons-collections4:4.4 +org.apache.commons:commons-compress:1.28.0 +org.apache.commons:commons-lang3:3.18.0 +org.apache.commons:commons-math3:3.6.1 +org.apache.commons:commons-text:1.14.0 +org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.5.0 +org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_25:1.5.0 +org.apache.hadoop:hadoop-annotations:3.4.3 +org.apache.hadoop:hadoop-common:3.4.3 +org.apache.httpcomponents.client5:httpclient5:5.6 +org.apache.httpcomponents.core5:httpcore5-h2:5.4 +org.apache.httpcomponents.core5:httpcore5:5.4 +org.apache.httpcomponents:httpclient:4.5.14 +org.apache.httpcomponents:httpcore:4.4.16 +org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-kafka-connect-events:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-kafka-connect-transforms:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-kafka-connect:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT +org.apache.orc:orc-core:1.9.8 +org.apache.orc:orc-shims:1.9.8 +org.apache.parquet:parquet-avro:1.17.0 +org.apache.parquet:parquet-column:1.17.0 +org.apache.parquet:parquet-common:1.17.0 +org.apache.parquet:parquet-encoding:1.17.0 +org.apache.parquet:parquet-format-structures:1.17.0 +org.apache.parquet:parquet-hadoop:1.17.0 +org.apache.parquet:parquet-jackson:1.17.0 +org.apache.parquet:parquet-variant:1.17.0 +org.bouncycastle:bcprov-jdk18on:1.82 +org.checkerframework:checker-compat-qual:2.5.6 +org.checkerframework:checker-qual:3.49.0 +org.codehaus.jettison:jettison:1.5.5 +org.codehaus.mojo:animal-sniffer-annotations:1.24 +org.codehaus.woodstox:stax2-api:4.2.2 +org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.json:json:20250517 +org.locationtech.jts:jts-core:1.20.0 +org.mongodb:bson:4.11.5 +org.reactivestreams:reactive-streams:1.0.4 +org.roaringbitmap:RoaringBitmap:1.6.13 +org.slf4j:slf4j-api:2.0.17 +org.threeten:threeten-extra:1.8.0 +org.threeten:threetenbp:1.7.0 +org.xerial.snappy:snappy-java:1.1.10.8 +software.amazon.awssdk.crt:aws-crt:0.43.5 +software.amazon.awssdk:annotations:2.42.18 +software.amazon.awssdk:apache-client:2.42.18 +software.amazon.awssdk:arns:2.42.18 +software.amazon.awssdk:auth:2.42.18 +software.amazon.awssdk:aws-core:2.42.18 +software.amazon.awssdk:aws-json-protocol:2.42.18 +software.amazon.awssdk:aws-query-protocol:2.42.18 +software.amazon.awssdk:aws-xml-protocol:2.42.18 +software.amazon.awssdk:checksums-spi:2.42.18 +software.amazon.awssdk:checksums:2.42.18 +software.amazon.awssdk:crt-core:2.42.18 +software.amazon.awssdk:dynamodb:2.42.18 +software.amazon.awssdk:endpoints-spi:2.42.18 +software.amazon.awssdk:glue:2.42.18 +software.amazon.awssdk:http-auth-aws-crt:2.42.18 +software.amazon.awssdk:http-auth-aws-eventstream:2.42.18 +software.amazon.awssdk:http-auth-aws:2.42.18 +software.amazon.awssdk:http-auth-spi:2.42.18 +software.amazon.awssdk:http-auth:2.42.18 +software.amazon.awssdk:http-client-spi:2.42.18 +software.amazon.awssdk:iam:2.42.18 +software.amazon.awssdk:identity-spi:2.42.18 +software.amazon.awssdk:json-utils:2.42.18 +software.amazon.awssdk:kms:2.42.18 +software.amazon.awssdk:lakeformation:2.42.18 +software.amazon.awssdk:metrics-spi:2.42.18 +software.amazon.awssdk:netty-nio-client:2.42.18 +software.amazon.awssdk:profiles:2.42.18 +software.amazon.awssdk:protocol-core:2.42.18 +software.amazon.awssdk:regions:2.42.18 +software.amazon.awssdk:retries-spi:2.42.18 +software.amazon.awssdk:retries:2.42.18 +software.amazon.awssdk:s3:2.42.18 +software.amazon.awssdk:sdk-core:2.42.18 +software.amazon.awssdk:sso:2.42.18 +software.amazon.awssdk:sts:2.42.18 +software.amazon.awssdk:third-party-jackson-core:2.42.18 +software.amazon.awssdk:utils-lite:2.42.18 +software.amazon.awssdk:utils:2.42.18 +software.amazon.eventstream:eventstream:1.0.1 diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle index bfe84b08dfa4..df7b9f36bd58 100644 --- a/spark/v3.4/build.gradle +++ b/spark/v3.4/build.gradle @@ -333,5 +333,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio jar { enabled = false } + + apply from: "${rootDir}/gradle/runtime-deps.gradle" } diff --git a/spark/v3.4/spark-runtime/runtime-deps.txt b/spark/v3.4/spark-runtime/runtime-deps.txt new file mode 100644 index 000000000000..75be819289ce --- /dev/null +++ b/spark/v3.4/spark-runtime/runtime-deps.txt @@ -0,0 +1,131 @@ +com.aliyun:credentials-java:0.3.12 +com.aliyun:tea:1.4.1 +com.fasterxml.jackson.core:jackson-annotations:2.21 +com.fasterxml.jackson.core:jackson-core:2.14.2 +com.fasterxml.jackson.core:jackson-databind:2.14.2 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 +com.github.ben-manes.caffeine:caffeine:2.9.3 +com.google.android:annotations:4.1.1.4 +com.google.api-client:google-api-client:2.7.2 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 +com.google.api.grpc:proto-google-common-protos:2.66.0 +com.google.api.grpc:proto-google-iam-v1:1.61.0 +com.google.api:api-common:2.58.0 +com.google.api:gax-grpc:2.75.0 +com.google.api:gax-httpjson:2.75.0 +com.google.api:gax:2.75.0 +com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 +com.google.auth:google-auth-library-credentials:1.43.0 +com.google.auth:google-auth-library-oauth2-http:1.43.0 +com.google.auto.value:auto-value-annotations:1.11.0 +com.google.cloud:google-cloud-bigquery:2.60.0 +com.google.cloud:google-cloud-bigquerystorage:3.23.0 +com.google.cloud:google-cloud-core-http:2.65.0 +com.google.cloud:google-cloud-core:2.65.0 +com.google.code.findbugs:jsr305:3.0.2 +com.google.code.gson:gson:2.12.1 +com.google.errorprone:error_prone_annotations:2.42.0 +com.google.flatbuffers:flatbuffers-java:24.3.25 +com.google.guava:failureaccess:1.0.3 +com.google.guava:guava:33.5.0-jre +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +com.google.http-client:google-http-client-apache-v2:2.1.0 +com.google.http-client:google-http-client-appengine:2.1.0 +com.google.http-client:google-http-client-gson:2.1.0 +com.google.http-client:google-http-client:2.1.0 +com.google.j2objc:j2objc-annotations:3.1 +com.google.oauth-client:google-oauth-client:1.39.0 +com.google.protobuf:protobuf-java-util:4.33.2 +com.google.protobuf:protobuf-java:4.33.2 +com.squareup.okhttp3:okhttp:4.12.0 +com.squareup.okio:okio-jvm:3.6.0 +com.sun.xml.bind:jaxb-core:2.3.0 +com.sun.xml.bind:jaxb-impl:2.3.0 +dev.failsafe:failsafe:3.3.2 +io.airlift:aircompressor:2.0.3 +io.grpc:grpc-alts:1.76.3 +io.grpc:grpc-api:1.76.3 +io.grpc:grpc-auth:1.76.3 +io.grpc:grpc-context:1.76.3 +io.grpc:grpc-core:1.76.3 +io.grpc:grpc-googleapis:1.76.3 +io.grpc:grpc-grpclb:1.76.3 +io.grpc:grpc-inprocess:1.76.3 +io.grpc:grpc-netty-shaded:1.76.3 +io.grpc:grpc-protobuf-lite:1.76.3 +io.grpc:grpc-protobuf:1.76.3 +io.grpc:grpc-stub:1.76.3 +io.grpc:grpc-util:1.76.3 +io.netty:netty-buffer:4.2.10.Final +io.netty:netty-common:4.2.10.Final +io.opencensus:opencensus-api:0.31.1 +io.opencensus:opencensus-contrib-http-util:0.31.1 +io.opentelemetry:opentelemetry-api:1.51.0 +io.opentelemetry:opentelemetry-context:1.51.0 +io.perfmark:perfmark-api:0.27.0 +org.apache.arrow:arrow-format:17.0.0 +org.apache.arrow:arrow-memory-core:17.0.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 +org.apache.arrow:arrow-memory-netty:17.0.0 +org.apache.arrow:arrow-vector:17.0.0 +org.apache.avro:avro:1.12.1 +org.apache.datasketches:datasketches-java:6.2.0 +org.apache.datasketches:datasketches-memory:3.0.2 +org.apache.httpcomponents.client5:httpclient5:5.6 +org.apache.httpcomponents.core5:httpcore5-h2:5.4 +org.apache.httpcomponents.core5:httpcore5:5.4 +org.apache.httpcomponents:httpclient:4.5.14 +org.apache.httpcomponents:httpcore:4.4.16 +org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-arrow:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-snowflake:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-spark-3.4_2.12:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-spark-extensions-3.4_2.12:1.11.0-SNAPSHOT +org.apache.orc:orc-core:1.9.8 +org.apache.orc:orc-shims:1.9.8 +org.apache.parquet:parquet-avro:1.17.0 +org.apache.parquet:parquet-column:1.17.0 +org.apache.parquet:parquet-common:1.17.0 +org.apache.parquet:parquet-encoding:1.17.0 +org.apache.parquet:parquet-format-structures:1.17.0 +org.apache.parquet:parquet-hadoop:1.17.0 +org.apache.parquet:parquet-jackson:1.17.0 +org.apache.parquet:parquet-variant:1.17.0 +org.checkerframework:checker-compat-qual:2.5.6 +org.checkerframework:checker-qual:3.49.0 +org.codehaus.mojo:animal-sniffer-annotations:1.24 +org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 +org.jacoco:org.jacoco.agent:0.8.8 +org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib:1.9.10 +org.json:json:20250517 +org.jspecify:jspecify:1.0.0 +org.locationtech.jts:jts-core:1.20.0 +org.projectnessie.nessie:nessie-client:0.107.4 +org.projectnessie.nessie:nessie-model:0.107.4 +org.roaringbitmap:RoaringBitmap:1.6.13 +org.threeten:threeten-extra:1.8.0 +org.threeten:threetenbp:1.7.0 diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle index 2fe3deb0ce5e..66f6d68f1c25 100644 --- a/spark/v3.5/build.gradle +++ b/spark/v3.5/build.gradle @@ -335,5 +335,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio jar { enabled = false } + + apply from: "${rootDir}/gradle/runtime-deps.gradle" } diff --git a/spark/v3.5/spark-runtime/runtime-deps.txt b/spark/v3.5/spark-runtime/runtime-deps.txt new file mode 100644 index 000000000000..c93b3c042f29 --- /dev/null +++ b/spark/v3.5/spark-runtime/runtime-deps.txt @@ -0,0 +1,131 @@ +com.aliyun:credentials-java:0.3.12 +com.aliyun:tea:1.4.1 +com.fasterxml.jackson.core:jackson-annotations:2.21 +com.fasterxml.jackson.core:jackson-core:2.15.2 +com.fasterxml.jackson.core:jackson-databind:2.15.2 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 +com.github.ben-manes.caffeine:caffeine:2.9.3 +com.google.android:annotations:4.1.1.4 +com.google.api-client:google-api-client:2.7.2 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 +com.google.api.grpc:proto-google-common-protos:2.66.0 +com.google.api.grpc:proto-google-iam-v1:1.61.0 +com.google.api:api-common:2.58.0 +com.google.api:gax-grpc:2.75.0 +com.google.api:gax-httpjson:2.75.0 +com.google.api:gax:2.75.0 +com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 +com.google.auth:google-auth-library-credentials:1.43.0 +com.google.auth:google-auth-library-oauth2-http:1.43.0 +com.google.auto.value:auto-value-annotations:1.11.0 +com.google.cloud:google-cloud-bigquery:2.60.0 +com.google.cloud:google-cloud-bigquerystorage:3.23.0 +com.google.cloud:google-cloud-core-http:2.65.0 +com.google.cloud:google-cloud-core:2.65.0 +com.google.code.findbugs:jsr305:3.0.2 +com.google.code.gson:gson:2.12.1 +com.google.errorprone:error_prone_annotations:2.42.0 +com.google.flatbuffers:flatbuffers-java:24.3.25 +com.google.guava:failureaccess:1.0.3 +com.google.guava:guava:33.5.0-jre +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +com.google.http-client:google-http-client-apache-v2:2.1.0 +com.google.http-client:google-http-client-appengine:2.1.0 +com.google.http-client:google-http-client-gson:2.1.0 +com.google.http-client:google-http-client:2.1.0 +com.google.j2objc:j2objc-annotations:3.1 +com.google.oauth-client:google-oauth-client:1.39.0 +com.google.protobuf:protobuf-java-util:4.33.2 +com.google.protobuf:protobuf-java:4.33.2 +com.squareup.okhttp3:okhttp:4.12.0 +com.squareup.okio:okio-jvm:3.6.0 +com.sun.xml.bind:jaxb-core:2.3.0 +com.sun.xml.bind:jaxb-impl:2.3.0 +dev.failsafe:failsafe:3.3.2 +io.airlift:aircompressor:2.0.3 +io.grpc:grpc-alts:1.76.3 +io.grpc:grpc-api:1.76.3 +io.grpc:grpc-auth:1.76.3 +io.grpc:grpc-context:1.76.3 +io.grpc:grpc-core:1.76.3 +io.grpc:grpc-googleapis:1.76.3 +io.grpc:grpc-grpclb:1.76.3 +io.grpc:grpc-inprocess:1.76.3 +io.grpc:grpc-netty-shaded:1.76.3 +io.grpc:grpc-protobuf-lite:1.76.3 +io.grpc:grpc-protobuf:1.76.3 +io.grpc:grpc-stub:1.76.3 +io.grpc:grpc-util:1.76.3 +io.netty:netty-buffer:4.2.10.Final +io.netty:netty-common:4.2.10.Final +io.opencensus:opencensus-api:0.31.1 +io.opencensus:opencensus-contrib-http-util:0.31.1 +io.opentelemetry:opentelemetry-api:1.51.0 +io.opentelemetry:opentelemetry-context:1.51.0 +io.perfmark:perfmark-api:0.27.0 +org.apache.arrow:arrow-format:17.0.0 +org.apache.arrow:arrow-memory-core:17.0.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 +org.apache.arrow:arrow-memory-netty:17.0.0 +org.apache.arrow:arrow-vector:17.0.0 +org.apache.avro:avro:1.12.1 +org.apache.datasketches:datasketches-java:6.2.0 +org.apache.datasketches:datasketches-memory:3.0.2 +org.apache.httpcomponents.client5:httpclient5:5.6 +org.apache.httpcomponents.core5:httpcore5-h2:5.4 +org.apache.httpcomponents.core5:httpcore5:5.4 +org.apache.httpcomponents:httpclient:4.5.14 +org.apache.httpcomponents:httpcore:4.4.16 +org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-arrow:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-snowflake:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-spark-3.5_2.12:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-spark-extensions-3.5_2.12:1.11.0-SNAPSHOT +org.apache.orc:orc-core:1.9.8 +org.apache.orc:orc-shims:1.9.8 +org.apache.parquet:parquet-avro:1.17.0 +org.apache.parquet:parquet-column:1.17.0 +org.apache.parquet:parquet-common:1.17.0 +org.apache.parquet:parquet-encoding:1.17.0 +org.apache.parquet:parquet-format-structures:1.17.0 +org.apache.parquet:parquet-hadoop:1.17.0 +org.apache.parquet:parquet-jackson:1.17.0 +org.apache.parquet:parquet-variant:1.17.0 +org.checkerframework:checker-compat-qual:2.5.6 +org.checkerframework:checker-qual:3.49.0 +org.codehaus.mojo:animal-sniffer-annotations:1.24 +org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 +org.jacoco:org.jacoco.agent:0.8.8 +org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib:1.9.10 +org.json:json:20250517 +org.jspecify:jspecify:1.0.0 +org.locationtech.jts:jts-core:1.20.0 +org.projectnessie.nessie:nessie-client:0.107.4 +org.projectnessie.nessie:nessie-model:0.107.4 +org.roaringbitmap:RoaringBitmap:1.6.13 +org.threeten:threeten-extra:1.8.0 +org.threeten:threetenbp:1.7.0 diff --git a/spark/v4.0/build.gradle b/spark/v4.0/build.gradle index acc4d7529f37..b92e5c26c8e2 100644 --- a/spark/v4.0/build.gradle +++ b/spark/v4.0/build.gradle @@ -335,5 +335,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio jar { enabled = false } + + apply from: "${rootDir}/gradle/runtime-deps.gradle" } diff --git a/spark/v4.0/spark-runtime/runtime-deps.txt b/spark/v4.0/spark-runtime/runtime-deps.txt new file mode 100644 index 000000000000..c16fa6f13e7a --- /dev/null +++ b/spark/v4.0/spark-runtime/runtime-deps.txt @@ -0,0 +1,131 @@ +com.aliyun:credentials-java:0.3.12 +com.aliyun:tea:1.4.1 +com.fasterxml.jackson.core:jackson-annotations:2.21 +com.fasterxml.jackson.core:jackson-core:2.15.2 +com.fasterxml.jackson.core:jackson-databind:2.15.2 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 +com.github.ben-manes.caffeine:caffeine:2.9.3 +com.google.android:annotations:4.1.1.4 +com.google.api-client:google-api-client:2.7.2 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 +com.google.api.grpc:proto-google-common-protos:2.66.0 +com.google.api.grpc:proto-google-iam-v1:1.61.0 +com.google.api:api-common:2.58.0 +com.google.api:gax-grpc:2.75.0 +com.google.api:gax-httpjson:2.75.0 +com.google.api:gax:2.75.0 +com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 +com.google.auth:google-auth-library-credentials:1.43.0 +com.google.auth:google-auth-library-oauth2-http:1.43.0 +com.google.auto.value:auto-value-annotations:1.11.0 +com.google.cloud:google-cloud-bigquery:2.60.0 +com.google.cloud:google-cloud-bigquerystorage:3.23.0 +com.google.cloud:google-cloud-core-http:2.65.0 +com.google.cloud:google-cloud-core:2.65.0 +com.google.code.findbugs:jsr305:3.0.2 +com.google.code.gson:gson:2.12.1 +com.google.errorprone:error_prone_annotations:2.42.0 +com.google.flatbuffers:flatbuffers-java:24.3.25 +com.google.guava:failureaccess:1.0.3 +com.google.guava:guava:33.5.0-jre +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +com.google.http-client:google-http-client-apache-v2:2.1.0 +com.google.http-client:google-http-client-appengine:2.1.0 +com.google.http-client:google-http-client-gson:2.1.0 +com.google.http-client:google-http-client:2.1.0 +com.google.j2objc:j2objc-annotations:3.1 +com.google.oauth-client:google-oauth-client:1.39.0 +com.google.protobuf:protobuf-java-util:4.33.2 +com.google.protobuf:protobuf-java:4.33.2 +com.squareup.okhttp3:okhttp:4.12.0 +com.squareup.okio:okio-jvm:3.6.0 +com.sun.xml.bind:jaxb-core:2.3.0 +com.sun.xml.bind:jaxb-impl:2.3.0 +dev.failsafe:failsafe:3.3.2 +io.airlift:aircompressor:2.0.3 +io.grpc:grpc-alts:1.76.3 +io.grpc:grpc-api:1.76.3 +io.grpc:grpc-auth:1.76.3 +io.grpc:grpc-context:1.76.3 +io.grpc:grpc-core:1.76.3 +io.grpc:grpc-googleapis:1.76.3 +io.grpc:grpc-grpclb:1.76.3 +io.grpc:grpc-inprocess:1.76.3 +io.grpc:grpc-netty-shaded:1.76.3 +io.grpc:grpc-protobuf-lite:1.76.3 +io.grpc:grpc-protobuf:1.76.3 +io.grpc:grpc-stub:1.76.3 +io.grpc:grpc-util:1.76.3 +io.netty:netty-buffer:4.2.10.Final +io.netty:netty-common:4.2.10.Final +io.opencensus:opencensus-api:0.31.1 +io.opencensus:opencensus-contrib-http-util:0.31.1 +io.opentelemetry:opentelemetry-api:1.51.0 +io.opentelemetry:opentelemetry-context:1.51.0 +io.perfmark:perfmark-api:0.27.0 +org.apache.arrow:arrow-format:17.0.0 +org.apache.arrow:arrow-memory-core:17.0.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 +org.apache.arrow:arrow-memory-netty:17.0.0 +org.apache.arrow:arrow-vector:17.0.0 +org.apache.avro:avro:1.12.1 +org.apache.datasketches:datasketches-java:6.2.0 +org.apache.datasketches:datasketches-memory:3.0.2 +org.apache.httpcomponents.client5:httpclient5:5.6 +org.apache.httpcomponents.core5:httpcore5-h2:5.4 +org.apache.httpcomponents.core5:httpcore5:5.4 +org.apache.httpcomponents:httpclient:4.5.14 +org.apache.httpcomponents:httpcore:4.4.16 +org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-arrow:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-snowflake:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-spark-4.0_2.13:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-spark-extensions-4.0_2.13:1.11.0-SNAPSHOT +org.apache.orc:orc-core:1.9.8 +org.apache.orc:orc-shims:1.9.8 +org.apache.parquet:parquet-avro:1.17.0 +org.apache.parquet:parquet-column:1.17.0 +org.apache.parquet:parquet-common:1.17.0 +org.apache.parquet:parquet-encoding:1.17.0 +org.apache.parquet:parquet-format-structures:1.17.0 +org.apache.parquet:parquet-hadoop:1.17.0 +org.apache.parquet:parquet-jackson:1.17.0 +org.apache.parquet:parquet-variant:1.17.0 +org.checkerframework:checker-compat-qual:2.5.6 +org.checkerframework:checker-qual:3.49.0 +org.codehaus.mojo:animal-sniffer-annotations:1.24 +org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 +org.jacoco:org.jacoco.agent:0.8.8 +org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib:1.9.10 +org.json:json:20250517 +org.jspecify:jspecify:1.0.0 +org.locationtech.jts:jts-core:1.20.0 +org.projectnessie.nessie:nessie-client:0.107.4 +org.projectnessie.nessie:nessie-model:0.107.4 +org.roaringbitmap:RoaringBitmap:1.6.13 +org.threeten:threeten-extra:1.8.0 +org.threeten:threetenbp:1.7.0 diff --git a/spark/v4.1/build.gradle b/spark/v4.1/build.gradle index 6a46cb4b2063..25d3fa96df0e 100644 --- a/spark/v4.1/build.gradle +++ b/spark/v4.1/build.gradle @@ -335,5 +335,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio jar { enabled = false } + + apply from: "${rootDir}/gradle/runtime-deps.gradle" } diff --git a/spark/v4.1/spark-runtime/runtime-deps.txt b/spark/v4.1/spark-runtime/runtime-deps.txt new file mode 100644 index 000000000000..bcec06071570 --- /dev/null +++ b/spark/v4.1/spark-runtime/runtime-deps.txt @@ -0,0 +1,131 @@ +com.aliyun:credentials-java:0.3.12 +com.aliyun:tea:1.4.1 +com.fasterxml.jackson.core:jackson-annotations:2.21 +com.fasterxml.jackson.core:jackson-core:2.15.2 +com.fasterxml.jackson.core:jackson-databind:2.15.2 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 +com.github.ben-manes.caffeine:caffeine:2.9.3 +com.google.android:annotations:4.1.1.4 +com.google.api-client:google-api-client:2.7.2 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 +com.google.api.grpc:proto-google-common-protos:2.66.0 +com.google.api.grpc:proto-google-iam-v1:1.61.0 +com.google.api:api-common:2.58.0 +com.google.api:gax-grpc:2.75.0 +com.google.api:gax-httpjson:2.75.0 +com.google.api:gax:2.75.0 +com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 +com.google.auth:google-auth-library-credentials:1.43.0 +com.google.auth:google-auth-library-oauth2-http:1.43.0 +com.google.auto.value:auto-value-annotations:1.11.0 +com.google.cloud:google-cloud-bigquery:2.60.0 +com.google.cloud:google-cloud-bigquerystorage:3.23.0 +com.google.cloud:google-cloud-core-http:2.65.0 +com.google.cloud:google-cloud-core:2.65.0 +com.google.code.findbugs:jsr305:3.0.2 +com.google.code.gson:gson:2.12.1 +com.google.errorprone:error_prone_annotations:2.42.0 +com.google.flatbuffers:flatbuffers-java:24.3.25 +com.google.guava:failureaccess:1.0.3 +com.google.guava:guava:33.5.0-jre +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +com.google.http-client:google-http-client-apache-v2:2.1.0 +com.google.http-client:google-http-client-appengine:2.1.0 +com.google.http-client:google-http-client-gson:2.1.0 +com.google.http-client:google-http-client:2.1.0 +com.google.j2objc:j2objc-annotations:3.1 +com.google.oauth-client:google-oauth-client:1.39.0 +com.google.protobuf:protobuf-java-util:4.33.2 +com.google.protobuf:protobuf-java:4.33.2 +com.squareup.okhttp3:okhttp:4.12.0 +com.squareup.okio:okio-jvm:3.6.0 +com.sun.xml.bind:jaxb-core:2.3.0 +com.sun.xml.bind:jaxb-impl:2.3.0 +dev.failsafe:failsafe:3.3.2 +io.airlift:aircompressor:2.0.3 +io.grpc:grpc-alts:1.76.3 +io.grpc:grpc-api:1.76.3 +io.grpc:grpc-auth:1.76.3 +io.grpc:grpc-context:1.76.3 +io.grpc:grpc-core:1.76.3 +io.grpc:grpc-googleapis:1.76.3 +io.grpc:grpc-grpclb:1.76.3 +io.grpc:grpc-inprocess:1.76.3 +io.grpc:grpc-netty-shaded:1.76.3 +io.grpc:grpc-protobuf-lite:1.76.3 +io.grpc:grpc-protobuf:1.76.3 +io.grpc:grpc-stub:1.76.3 +io.grpc:grpc-util:1.76.3 +io.netty:netty-buffer:4.2.10.Final +io.netty:netty-common:4.2.10.Final +io.opencensus:opencensus-api:0.31.1 +io.opencensus:opencensus-contrib-http-util:0.31.1 +io.opentelemetry:opentelemetry-api:1.51.0 +io.opentelemetry:opentelemetry-context:1.51.0 +io.perfmark:perfmark-api:0.27.0 +org.apache.arrow:arrow-format:17.0.0 +org.apache.arrow:arrow-memory-core:17.0.0 +org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 +org.apache.arrow:arrow-memory-netty:17.0.0 +org.apache.arrow:arrow-vector:17.0.0 +org.apache.avro:avro:1.12.1 +org.apache.datasketches:datasketches-java:6.2.0 +org.apache.datasketches:datasketches-memory:3.0.2 +org.apache.httpcomponents.client5:httpclient5:5.6 +org.apache.httpcomponents.core5:httpcore5-h2:5.4 +org.apache.httpcomponents.core5:httpcore5:5.4 +org.apache.httpcomponents:httpclient:4.5.14 +org.apache.httpcomponents:httpcore:4.4.16 +org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-arrow:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-snowflake:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-spark-4.1_2.13:1.11.0-SNAPSHOT +org.apache.iceberg:iceberg-spark-extensions-4.1_2.13:1.11.0-SNAPSHOT +org.apache.orc:orc-core:1.9.8 +org.apache.orc:orc-shims:1.9.8 +org.apache.parquet:parquet-avro:1.17.0 +org.apache.parquet:parquet-column:1.17.0 +org.apache.parquet:parquet-common:1.17.0 +org.apache.parquet:parquet-encoding:1.17.0 +org.apache.parquet:parquet-format-structures:1.17.0 +org.apache.parquet:parquet-hadoop:1.17.0 +org.apache.parquet:parquet-jackson:1.17.0 +org.apache.parquet:parquet-variant:1.17.0 +org.checkerframework:checker-compat-qual:2.5.6 +org.checkerframework:checker-qual:3.49.0 +org.codehaus.mojo:animal-sniffer-annotations:1.24 +org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 +org.jacoco:org.jacoco.agent:0.8.8 +org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 +org.jetbrains.kotlin:kotlin-stdlib:1.9.10 +org.json:json:20250517 +org.jspecify:jspecify:1.0.0 +org.locationtech.jts:jts-core:1.20.0 +org.projectnessie.nessie:nessie-client:0.107.4 +org.projectnessie.nessie:nessie-model:0.107.4 +org.roaringbitmap:RoaringBitmap:1.6.13 +org.threeten:threeten-extra:1.8.0 +org.threeten:threetenbp:1.7.0 From 156f125cb800b8861f6fd4ce2f4758d421efda37 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 1 Apr 2026 15:28:06 -0500 Subject: [PATCH 2/7] Build: Exclude org.apache.iceberg modules from runtime dependency guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Internal Iceberg module dependencies don't affect LICENSE/NOTICE compliance or shadow JAR size concerns — only third-party dependency changes matter. Filtering them out avoids false positives when modules are added or reorganized. --- flink/v1.20/flink-runtime/runtime-deps.txt | 94 +--------------- flink/v2.0/flink-runtime/runtime-deps.txt | 94 +--------------- flink/v2.1/flink-runtime/runtime-deps.txt | 94 +--------------- gradle/runtime-deps.gradle | 1 + .../kafka-connect-runtime/runtime-deps.txt | 14 --- spark/v3.4/spark-runtime/runtime-deps.txt | 101 ++---------------- spark/v3.5/spark-runtime/runtime-deps.txt | 101 ++---------------- spark/v4.0/spark-runtime/runtime-deps.txt | 101 ++---------------- spark/v4.1/spark-runtime/runtime-deps.txt | 101 ++---------------- 9 files changed, 57 insertions(+), 644 deletions(-) diff --git a/flink/v1.20/flink-runtime/runtime-deps.txt b/flink/v1.20/flink-runtime/runtime-deps.txt index ce4f4205b179..108d72bd8f12 100644 --- a/flink/v1.20/flink-runtime/runtime-deps.txt +++ b/flink/v1.20/flink-runtime/runtime-deps.txt @@ -3,48 +3,10 @@ com.aliyun:tea:1.4.1 com.fasterxml.jackson.core:jackson-annotations:2.21 com.fasterxml.jackson.core:jackson-core:2.21.2 com.fasterxml.jackson.core:jackson-databind:2.21.2 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 com.github.ben-manes.caffeine:caffeine:2.9.3 com.github.luben:zstd-jni:1.5.7-3 -com.google.android:annotations:4.1.1.4 -com.google.api-client:google-api-client:2.7.2 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 -com.google.api.grpc:proto-google-common-protos:2.66.0 -com.google.api.grpc:proto-google-iam-v1:1.61.0 -com.google.api:api-common:2.58.0 -com.google.api:gax-grpc:2.75.0 -com.google.api:gax-httpjson:2.75.0 -com.google.api:gax:2.75.0 -com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 -com.google.auth:google-auth-library-credentials:1.43.0 -com.google.auth:google-auth-library-oauth2-http:1.43.0 -com.google.auto.value:auto-value-annotations:1.11.0 -com.google.cloud:google-cloud-bigquery:2.60.0 -com.google.cloud:google-cloud-bigquerystorage:3.23.0 -com.google.cloud:google-cloud-core-http:2.65.0 -com.google.cloud:google-cloud-core:2.65.0 -com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.12.1 -com.google.errorprone:error_prone_annotations:2.42.0 -com.google.flatbuffers:flatbuffers-java:24.3.25 -com.google.guava:failureaccess:1.0.3 -com.google.guava:guava:33.5.0-jre -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava -com.google.http-client:google-http-client-apache-v2:2.1.0 -com.google.http-client:google-http-client-appengine:2.1.0 -com.google.http-client:google-http-client-gson:2.1.0 -com.google.http-client:google-http-client:2.1.0 -com.google.j2objc:j2objc-annotations:3.1 -com.google.oauth-client:google-oauth-client:1.39.0 -com.google.protobuf:protobuf-java-util:4.33.2 -com.google.protobuf:protobuf-java:4.33.2 +com.google.code.gson:gson:2.11.0 +com.google.errorprone:error_prone_annotations:2.27.0 com.squareup.okhttp3:okhttp:4.12.0 com.squareup.okio:okio-jvm:3.6.0 com.sun.xml.bind:jaxb-core:2.3.0 @@ -52,31 +14,6 @@ com.sun.xml.bind:jaxb-impl:2.3.0 dev.failsafe:failsafe:3.3.2 io.airlift:aircompressor:2.0.3 io.dropwizard.metrics:metrics-core:3.2.6 -io.grpc:grpc-alts:1.76.3 -io.grpc:grpc-api:1.76.3 -io.grpc:grpc-auth:1.76.3 -io.grpc:grpc-context:1.76.3 -io.grpc:grpc-core:1.76.3 -io.grpc:grpc-googleapis:1.76.3 -io.grpc:grpc-grpclb:1.76.3 -io.grpc:grpc-inprocess:1.76.3 -io.grpc:grpc-netty-shaded:1.76.3 -io.grpc:grpc-protobuf-lite:1.76.3 -io.grpc:grpc-protobuf:1.76.3 -io.grpc:grpc-stub:1.76.3 -io.grpc:grpc-util:1.76.3 -io.netty:netty-buffer:4.1.110.Final -io.netty:netty-common:4.1.110.Final -io.opencensus:opencensus-api:0.31.1 -io.opencensus:opencensus-contrib-http-util:0.31.1 -io.opentelemetry:opentelemetry-api:1.51.0 -io.opentelemetry:opentelemetry-context:1.51.0 -io.perfmark:perfmark-api:0.27.0 -org.apache.arrow:arrow-format:17.0.0 -org.apache.arrow:arrow-memory-core:17.0.0 -org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 -org.apache.arrow:arrow-memory-netty:17.0.0 -org.apache.arrow:arrow-vector:17.0.0 org.apache.avro:avro:1.12.1 org.apache.datasketches:datasketches-java:6.2.0 org.apache.datasketches:datasketches-memory:3.0.2 @@ -84,23 +21,6 @@ org.apache.flink:flink-metrics-dropwizard:1.20.1 org.apache.httpcomponents.client5:httpclient5:5.6 org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.httpcomponents:httpclient:4.5.14 -org.apache.httpcomponents:httpcore:4.4.16 -org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-flink-1.20:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT org.apache.orc:orc-core:1.9.8 org.apache.orc:orc-shims:1.9.8 org.apache.parquet:parquet-avro:1.17.0 @@ -111,21 +31,15 @@ org.apache.parquet:parquet-format-structures:1.17.0 org.apache.parquet:parquet-hadoop:1.17.0 org.apache.parquet:parquet-jackson:1.17.0 org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-compat-qual:2.5.6 -org.checkerframework:checker-qual:3.49.0 -org.codehaus.mojo:animal-sniffer-annotations:1.24 -org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.checkerframework:checker-qual:3.19.0 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 org.jacoco:org.jacoco.agent:0.8.8 org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.json:json:20250517 -org.jspecify:jspecify:1.0.0 org.locationtech.jts:jts-core:1.20.0 org.projectnessie.nessie:nessie-client:0.107.4 org.projectnessie.nessie:nessie-model:0.107.4 org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.8.0 -org.threeten:threetenbp:1.7.0 +org.threeten:threeten-extra:1.7.1 diff --git a/flink/v2.0/flink-runtime/runtime-deps.txt b/flink/v2.0/flink-runtime/runtime-deps.txt index 4967fb6517c4..17dad715199a 100644 --- a/flink/v2.0/flink-runtime/runtime-deps.txt +++ b/flink/v2.0/flink-runtime/runtime-deps.txt @@ -3,48 +3,10 @@ com.aliyun:tea:1.4.1 com.fasterxml.jackson.core:jackson-annotations:2.21 com.fasterxml.jackson.core:jackson-core:2.21.2 com.fasterxml.jackson.core:jackson-databind:2.21.2 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 com.github.ben-manes.caffeine:caffeine:2.9.3 com.github.luben:zstd-jni:1.5.7-3 -com.google.android:annotations:4.1.1.4 -com.google.api-client:google-api-client:2.7.2 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 -com.google.api.grpc:proto-google-common-protos:2.66.0 -com.google.api.grpc:proto-google-iam-v1:1.61.0 -com.google.api:api-common:2.58.0 -com.google.api:gax-grpc:2.75.0 -com.google.api:gax-httpjson:2.75.0 -com.google.api:gax:2.75.0 -com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 -com.google.auth:google-auth-library-credentials:1.43.0 -com.google.auth:google-auth-library-oauth2-http:1.43.0 -com.google.auto.value:auto-value-annotations:1.11.0 -com.google.cloud:google-cloud-bigquery:2.60.0 -com.google.cloud:google-cloud-bigquerystorage:3.23.0 -com.google.cloud:google-cloud-core-http:2.65.0 -com.google.cloud:google-cloud-core:2.65.0 -com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.12.1 -com.google.errorprone:error_prone_annotations:2.42.0 -com.google.flatbuffers:flatbuffers-java:24.3.25 -com.google.guava:failureaccess:1.0.3 -com.google.guava:guava:33.5.0-jre -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava -com.google.http-client:google-http-client-apache-v2:2.1.0 -com.google.http-client:google-http-client-appengine:2.1.0 -com.google.http-client:google-http-client-gson:2.1.0 -com.google.http-client:google-http-client:2.1.0 -com.google.j2objc:j2objc-annotations:3.1 -com.google.oauth-client:google-oauth-client:1.39.0 -com.google.protobuf:protobuf-java-util:4.33.2 -com.google.protobuf:protobuf-java:4.33.2 +com.google.code.gson:gson:2.11.0 +com.google.errorprone:error_prone_annotations:2.27.0 com.squareup.okhttp3:okhttp:4.12.0 com.squareup.okio:okio-jvm:3.6.0 com.sun.xml.bind:jaxb-core:2.3.0 @@ -52,31 +14,6 @@ com.sun.xml.bind:jaxb-impl:2.3.0 dev.failsafe:failsafe:3.3.2 io.airlift:aircompressor:2.0.3 io.dropwizard.metrics:metrics-core:3.2.6 -io.grpc:grpc-alts:1.76.3 -io.grpc:grpc-api:1.76.3 -io.grpc:grpc-auth:1.76.3 -io.grpc:grpc-context:1.76.3 -io.grpc:grpc-core:1.76.3 -io.grpc:grpc-googleapis:1.76.3 -io.grpc:grpc-grpclb:1.76.3 -io.grpc:grpc-inprocess:1.76.3 -io.grpc:grpc-netty-shaded:1.76.3 -io.grpc:grpc-protobuf-lite:1.76.3 -io.grpc:grpc-protobuf:1.76.3 -io.grpc:grpc-stub:1.76.3 -io.grpc:grpc-util:1.76.3 -io.netty:netty-buffer:4.1.110.Final -io.netty:netty-common:4.1.110.Final -io.opencensus:opencensus-api:0.31.1 -io.opencensus:opencensus-contrib-http-util:0.31.1 -io.opentelemetry:opentelemetry-api:1.51.0 -io.opentelemetry:opentelemetry-context:1.51.0 -io.perfmark:perfmark-api:0.27.0 -org.apache.arrow:arrow-format:17.0.0 -org.apache.arrow:arrow-memory-core:17.0.0 -org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 -org.apache.arrow:arrow-memory-netty:17.0.0 -org.apache.arrow:arrow-vector:17.0.0 org.apache.avro:avro:1.12.1 org.apache.datasketches:datasketches-java:6.2.0 org.apache.datasketches:datasketches-memory:3.0.2 @@ -84,23 +21,6 @@ org.apache.flink:flink-metrics-dropwizard:2.0.0 org.apache.httpcomponents.client5:httpclient5:5.6 org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.httpcomponents:httpclient:4.5.14 -org.apache.httpcomponents:httpcore:4.4.16 -org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-flink-2.0:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT org.apache.orc:orc-core:1.9.8 org.apache.orc:orc-shims:1.9.8 org.apache.parquet:parquet-avro:1.17.0 @@ -111,21 +31,15 @@ org.apache.parquet:parquet-format-structures:1.17.0 org.apache.parquet:parquet-hadoop:1.17.0 org.apache.parquet:parquet-jackson:1.17.0 org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-compat-qual:2.5.6 -org.checkerframework:checker-qual:3.49.0 -org.codehaus.mojo:animal-sniffer-annotations:1.24 -org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.checkerframework:checker-qual:3.19.0 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 org.jacoco:org.jacoco.agent:0.8.8 org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.json:json:20250517 -org.jspecify:jspecify:1.0.0 org.locationtech.jts:jts-core:1.20.0 org.projectnessie.nessie:nessie-client:0.107.4 org.projectnessie.nessie:nessie-model:0.107.4 org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.8.0 -org.threeten:threetenbp:1.7.0 +org.threeten:threeten-extra:1.7.1 diff --git a/flink/v2.1/flink-runtime/runtime-deps.txt b/flink/v2.1/flink-runtime/runtime-deps.txt index 4155b7eddc88..c08131f14806 100644 --- a/flink/v2.1/flink-runtime/runtime-deps.txt +++ b/flink/v2.1/flink-runtime/runtime-deps.txt @@ -3,48 +3,10 @@ com.aliyun:tea:1.4.1 com.fasterxml.jackson.core:jackson-annotations:2.21 com.fasterxml.jackson.core:jackson-core:2.21.2 com.fasterxml.jackson.core:jackson-databind:2.21.2 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 com.github.ben-manes.caffeine:caffeine:2.9.3 com.github.luben:zstd-jni:1.5.7-3 -com.google.android:annotations:4.1.1.4 -com.google.api-client:google-api-client:2.7.2 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 -com.google.api.grpc:proto-google-common-protos:2.66.0 -com.google.api.grpc:proto-google-iam-v1:1.61.0 -com.google.api:api-common:2.58.0 -com.google.api:gax-grpc:2.75.0 -com.google.api:gax-httpjson:2.75.0 -com.google.api:gax:2.75.0 -com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 -com.google.auth:google-auth-library-credentials:1.43.0 -com.google.auth:google-auth-library-oauth2-http:1.43.0 -com.google.auto.value:auto-value-annotations:1.11.0 -com.google.cloud:google-cloud-bigquery:2.60.0 -com.google.cloud:google-cloud-bigquerystorage:3.23.0 -com.google.cloud:google-cloud-core-http:2.65.0 -com.google.cloud:google-cloud-core:2.65.0 -com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.12.1 -com.google.errorprone:error_prone_annotations:2.42.0 -com.google.flatbuffers:flatbuffers-java:24.3.25 -com.google.guava:failureaccess:1.0.3 -com.google.guava:guava:33.5.0-jre -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava -com.google.http-client:google-http-client-apache-v2:2.1.0 -com.google.http-client:google-http-client-appengine:2.1.0 -com.google.http-client:google-http-client-gson:2.1.0 -com.google.http-client:google-http-client:2.1.0 -com.google.j2objc:j2objc-annotations:3.1 -com.google.oauth-client:google-oauth-client:1.39.0 -com.google.protobuf:protobuf-java-util:4.33.2 -com.google.protobuf:protobuf-java:4.33.2 +com.google.code.gson:gson:2.11.0 +com.google.errorprone:error_prone_annotations:2.27.0 com.squareup.okhttp3:okhttp:4.12.0 com.squareup.okio:okio-jvm:3.6.0 com.sun.xml.bind:jaxb-core:2.3.0 @@ -52,31 +14,6 @@ com.sun.xml.bind:jaxb-impl:2.3.0 dev.failsafe:failsafe:3.3.2 io.airlift:aircompressor:2.0.3 io.dropwizard.metrics:metrics-core:3.2.6 -io.grpc:grpc-alts:1.76.3 -io.grpc:grpc-api:1.76.3 -io.grpc:grpc-auth:1.76.3 -io.grpc:grpc-context:1.76.3 -io.grpc:grpc-core:1.76.3 -io.grpc:grpc-googleapis:1.76.3 -io.grpc:grpc-grpclb:1.76.3 -io.grpc:grpc-inprocess:1.76.3 -io.grpc:grpc-netty-shaded:1.76.3 -io.grpc:grpc-protobuf-lite:1.76.3 -io.grpc:grpc-protobuf:1.76.3 -io.grpc:grpc-stub:1.76.3 -io.grpc:grpc-util:1.76.3 -io.netty:netty-buffer:4.1.110.Final -io.netty:netty-common:4.1.110.Final -io.opencensus:opencensus-api:0.31.1 -io.opencensus:opencensus-contrib-http-util:0.31.1 -io.opentelemetry:opentelemetry-api:1.51.0 -io.opentelemetry:opentelemetry-context:1.51.0 -io.perfmark:perfmark-api:0.27.0 -org.apache.arrow:arrow-format:17.0.0 -org.apache.arrow:arrow-memory-core:17.0.0 -org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 -org.apache.arrow:arrow-memory-netty:17.0.0 -org.apache.arrow:arrow-vector:17.0.0 org.apache.avro:avro:1.12.1 org.apache.datasketches:datasketches-java:6.2.0 org.apache.datasketches:datasketches-memory:3.0.2 @@ -84,23 +21,6 @@ org.apache.flink:flink-metrics-dropwizard:2.1.0 org.apache.httpcomponents.client5:httpclient5:5.6 org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.httpcomponents:httpclient:4.5.14 -org.apache.httpcomponents:httpcore:4.4.16 -org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-flink-2.1:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT org.apache.orc:orc-core:1.9.8 org.apache.orc:orc-shims:1.9.8 org.apache.parquet:parquet-avro:1.17.0 @@ -111,21 +31,15 @@ org.apache.parquet:parquet-format-structures:1.17.0 org.apache.parquet:parquet-hadoop:1.17.0 org.apache.parquet:parquet-jackson:1.17.0 org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-compat-qual:2.5.6 -org.checkerframework:checker-qual:3.49.0 -org.codehaus.mojo:animal-sniffer-annotations:1.24 -org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.checkerframework:checker-qual:3.19.0 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 org.jacoco:org.jacoco.agent:0.8.8 org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.json:json:20250517 -org.jspecify:jspecify:1.0.0 org.locationtech.jts:jts-core:1.20.0 org.projectnessie.nessie:nessie-client:0.107.4 org.projectnessie.nessie:nessie-model:0.107.4 org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.8.0 -org.threeten:threetenbp:1.7.0 +org.threeten:threeten-extra:1.7.1 diff --git a/gradle/runtime-deps.gradle b/gradle/runtime-deps.gradle index ea8b123181f6..a0c1b67b4747 100644 --- a/gradle/runtime-deps.gradle +++ b/gradle/runtime-deps.gradle @@ -50,6 +50,7 @@ def resolveRuntimeDeps = { configurations.runtimeClasspath.resolvedConfiguration .resolvedArtifacts .collect { "${it.moduleVersion.id.group}:${it.moduleVersion.id.name}:${it.moduleVersion.id.version}" } + .findAll { !it.startsWith('org.apache.iceberg:') } .toSorted() .toUnique() } diff --git a/kafka-connect/kafka-connect-runtime/runtime-deps.txt b/kafka-connect/kafka-connect-runtime/runtime-deps.txt index 2250b90179cd..417477bfcf12 100644 --- a/kafka-connect/kafka-connect-runtime/runtime-deps.txt +++ b/kafka-connect/kafka-connect-runtime/runtime-deps.txt @@ -164,20 +164,6 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.httpcomponents:httpclient:4.5.14 org.apache.httpcomponents:httpcore:4.4.16 -org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-kafka-connect-events:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-kafka-connect-transforms:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-kafka-connect:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT org.apache.orc:orc-core:1.9.8 org.apache.orc:orc-shims:1.9.8 org.apache.parquet:parquet-avro:1.17.0 diff --git a/spark/v3.4/spark-runtime/runtime-deps.txt b/spark/v3.4/spark-runtime/runtime-deps.txt index 75be819289ce..b9429f8144fe 100644 --- a/spark/v3.4/spark-runtime/runtime-deps.txt +++ b/spark/v3.4/spark-runtime/runtime-deps.txt @@ -5,102 +5,27 @@ com.fasterxml.jackson.core:jackson-core:2.14.2 com.fasterxml.jackson.core:jackson-databind:2.14.2 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 com.github.ben-manes.caffeine:caffeine:2.9.3 -com.google.android:annotations:4.1.1.4 -com.google.api-client:google-api-client:2.7.2 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 -com.google.api.grpc:proto-google-common-protos:2.66.0 -com.google.api.grpc:proto-google-iam-v1:1.61.0 -com.google.api:api-common:2.58.0 -com.google.api:gax-grpc:2.75.0 -com.google.api:gax-httpjson:2.75.0 -com.google.api:gax:2.75.0 -com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 -com.google.auth:google-auth-library-credentials:1.43.0 -com.google.auth:google-auth-library-oauth2-http:1.43.0 -com.google.auto.value:auto-value-annotations:1.11.0 -com.google.cloud:google-cloud-bigquery:2.60.0 -com.google.cloud:google-cloud-bigquerystorage:3.23.0 -com.google.cloud:google-cloud-core-http:2.65.0 -com.google.cloud:google-cloud-core:2.65.0 -com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.12.1 -com.google.errorprone:error_prone_annotations:2.42.0 -com.google.flatbuffers:flatbuffers-java:24.3.25 -com.google.guava:failureaccess:1.0.3 -com.google.guava:guava:33.5.0-jre -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava -com.google.http-client:google-http-client-apache-v2:2.1.0 -com.google.http-client:google-http-client-appengine:2.1.0 -com.google.http-client:google-http-client-gson:2.1.0 -com.google.http-client:google-http-client:2.1.0 -com.google.j2objc:j2objc-annotations:3.1 -com.google.oauth-client:google-oauth-client:1.39.0 -com.google.protobuf:protobuf-java-util:4.33.2 -com.google.protobuf:protobuf-java:4.33.2 +com.google.code.gson:gson:2.11.0 +com.google.errorprone:error_prone_annotations:2.27.0 +com.google.flatbuffers:flatbuffers-java:23.5.26 com.squareup.okhttp3:okhttp:4.12.0 com.squareup.okio:okio-jvm:3.6.0 com.sun.xml.bind:jaxb-core:2.3.0 com.sun.xml.bind:jaxb-impl:2.3.0 dev.failsafe:failsafe:3.3.2 io.airlift:aircompressor:2.0.3 -io.grpc:grpc-alts:1.76.3 -io.grpc:grpc-api:1.76.3 -io.grpc:grpc-auth:1.76.3 -io.grpc:grpc-context:1.76.3 -io.grpc:grpc-core:1.76.3 -io.grpc:grpc-googleapis:1.76.3 -io.grpc:grpc-grpclb:1.76.3 -io.grpc:grpc-inprocess:1.76.3 -io.grpc:grpc-netty-shaded:1.76.3 -io.grpc:grpc-protobuf-lite:1.76.3 -io.grpc:grpc-protobuf:1.76.3 -io.grpc:grpc-stub:1.76.3 -io.grpc:grpc-util:1.76.3 io.netty:netty-buffer:4.2.10.Final io.netty:netty-common:4.2.10.Final -io.opencensus:opencensus-api:0.31.1 -io.opencensus:opencensus-contrib-http-util:0.31.1 -io.opentelemetry:opentelemetry-api:1.51.0 -io.opentelemetry:opentelemetry-context:1.51.0 -io.perfmark:perfmark-api:0.27.0 -org.apache.arrow:arrow-format:17.0.0 -org.apache.arrow:arrow-memory-core:17.0.0 -org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 -org.apache.arrow:arrow-memory-netty:17.0.0 -org.apache.arrow:arrow-vector:17.0.0 +org.apache.arrow:arrow-format:15.0.2 +org.apache.arrow:arrow-memory-core:15.0.2 +org.apache.arrow:arrow-memory-netty:15.0.2 +org.apache.arrow:arrow-vector:15.0.2 org.apache.avro:avro:1.12.1 org.apache.datasketches:datasketches-java:6.2.0 org.apache.datasketches:datasketches-memory:3.0.2 org.apache.httpcomponents.client5:httpclient5:5.6 org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.httpcomponents:httpclient:4.5.14 -org.apache.httpcomponents:httpcore:4.4.16 -org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-arrow:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-snowflake:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-spark-3.4_2.12:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-spark-extensions-3.4_2.12:1.11.0-SNAPSHOT org.apache.orc:orc-core:1.9.8 org.apache.orc:orc-shims:1.9.8 org.apache.parquet:parquet-avro:1.17.0 @@ -111,21 +36,17 @@ org.apache.parquet:parquet-format-structures:1.17.0 org.apache.parquet:parquet-hadoop:1.17.0 org.apache.parquet:parquet-jackson:1.17.0 org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-compat-qual:2.5.6 -org.checkerframework:checker-qual:3.49.0 -org.codehaus.mojo:animal-sniffer-annotations:1.24 -org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.checkerframework:checker-qual:3.19.0 +org.eclipse.collections:eclipse-collections-api:11.1.0 +org.eclipse.collections:eclipse-collections:11.1.0 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 org.jacoco:org.jacoco.agent:0.8.8 org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.json:json:20250517 -org.jspecify:jspecify:1.0.0 org.locationtech.jts:jts-core:1.20.0 org.projectnessie.nessie:nessie-client:0.107.4 org.projectnessie.nessie:nessie-model:0.107.4 org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.8.0 -org.threeten:threetenbp:1.7.0 +org.threeten:threeten-extra:1.7.1 diff --git a/spark/v3.5/spark-runtime/runtime-deps.txt b/spark/v3.5/spark-runtime/runtime-deps.txt index c93b3c042f29..a662942f9cd1 100644 --- a/spark/v3.5/spark-runtime/runtime-deps.txt +++ b/spark/v3.5/spark-runtime/runtime-deps.txt @@ -5,102 +5,27 @@ com.fasterxml.jackson.core:jackson-core:2.15.2 com.fasterxml.jackson.core:jackson-databind:2.15.2 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 com.github.ben-manes.caffeine:caffeine:2.9.3 -com.google.android:annotations:4.1.1.4 -com.google.api-client:google-api-client:2.7.2 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 -com.google.api.grpc:proto-google-common-protos:2.66.0 -com.google.api.grpc:proto-google-iam-v1:1.61.0 -com.google.api:api-common:2.58.0 -com.google.api:gax-grpc:2.75.0 -com.google.api:gax-httpjson:2.75.0 -com.google.api:gax:2.75.0 -com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 -com.google.auth:google-auth-library-credentials:1.43.0 -com.google.auth:google-auth-library-oauth2-http:1.43.0 -com.google.auto.value:auto-value-annotations:1.11.0 -com.google.cloud:google-cloud-bigquery:2.60.0 -com.google.cloud:google-cloud-bigquerystorage:3.23.0 -com.google.cloud:google-cloud-core-http:2.65.0 -com.google.cloud:google-cloud-core:2.65.0 -com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.12.1 -com.google.errorprone:error_prone_annotations:2.42.0 -com.google.flatbuffers:flatbuffers-java:24.3.25 -com.google.guava:failureaccess:1.0.3 -com.google.guava:guava:33.5.0-jre -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava -com.google.http-client:google-http-client-apache-v2:2.1.0 -com.google.http-client:google-http-client-appengine:2.1.0 -com.google.http-client:google-http-client-gson:2.1.0 -com.google.http-client:google-http-client:2.1.0 -com.google.j2objc:j2objc-annotations:3.1 -com.google.oauth-client:google-oauth-client:1.39.0 -com.google.protobuf:protobuf-java-util:4.33.2 -com.google.protobuf:protobuf-java:4.33.2 +com.google.code.gson:gson:2.11.0 +com.google.errorprone:error_prone_annotations:2.27.0 +com.google.flatbuffers:flatbuffers-java:23.5.26 com.squareup.okhttp3:okhttp:4.12.0 com.squareup.okio:okio-jvm:3.6.0 com.sun.xml.bind:jaxb-core:2.3.0 com.sun.xml.bind:jaxb-impl:2.3.0 dev.failsafe:failsafe:3.3.2 io.airlift:aircompressor:2.0.3 -io.grpc:grpc-alts:1.76.3 -io.grpc:grpc-api:1.76.3 -io.grpc:grpc-auth:1.76.3 -io.grpc:grpc-context:1.76.3 -io.grpc:grpc-core:1.76.3 -io.grpc:grpc-googleapis:1.76.3 -io.grpc:grpc-grpclb:1.76.3 -io.grpc:grpc-inprocess:1.76.3 -io.grpc:grpc-netty-shaded:1.76.3 -io.grpc:grpc-protobuf-lite:1.76.3 -io.grpc:grpc-protobuf:1.76.3 -io.grpc:grpc-stub:1.76.3 -io.grpc:grpc-util:1.76.3 io.netty:netty-buffer:4.2.10.Final io.netty:netty-common:4.2.10.Final -io.opencensus:opencensus-api:0.31.1 -io.opencensus:opencensus-contrib-http-util:0.31.1 -io.opentelemetry:opentelemetry-api:1.51.0 -io.opentelemetry:opentelemetry-context:1.51.0 -io.perfmark:perfmark-api:0.27.0 -org.apache.arrow:arrow-format:17.0.0 -org.apache.arrow:arrow-memory-core:17.0.0 -org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 -org.apache.arrow:arrow-memory-netty:17.0.0 -org.apache.arrow:arrow-vector:17.0.0 +org.apache.arrow:arrow-format:15.0.2 +org.apache.arrow:arrow-memory-core:15.0.2 +org.apache.arrow:arrow-memory-netty:15.0.2 +org.apache.arrow:arrow-vector:15.0.2 org.apache.avro:avro:1.12.1 org.apache.datasketches:datasketches-java:6.2.0 org.apache.datasketches:datasketches-memory:3.0.2 org.apache.httpcomponents.client5:httpclient5:5.6 org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.httpcomponents:httpclient:4.5.14 -org.apache.httpcomponents:httpcore:4.4.16 -org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-arrow:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-snowflake:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-spark-3.5_2.12:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-spark-extensions-3.5_2.12:1.11.0-SNAPSHOT org.apache.orc:orc-core:1.9.8 org.apache.orc:orc-shims:1.9.8 org.apache.parquet:parquet-avro:1.17.0 @@ -111,21 +36,17 @@ org.apache.parquet:parquet-format-structures:1.17.0 org.apache.parquet:parquet-hadoop:1.17.0 org.apache.parquet:parquet-jackson:1.17.0 org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-compat-qual:2.5.6 -org.checkerframework:checker-qual:3.49.0 -org.codehaus.mojo:animal-sniffer-annotations:1.24 -org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.checkerframework:checker-qual:3.19.0 +org.eclipse.collections:eclipse-collections-api:11.1.0 +org.eclipse.collections:eclipse-collections:11.1.0 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 org.jacoco:org.jacoco.agent:0.8.8 org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.json:json:20250517 -org.jspecify:jspecify:1.0.0 org.locationtech.jts:jts-core:1.20.0 org.projectnessie.nessie:nessie-client:0.107.4 org.projectnessie.nessie:nessie-model:0.107.4 org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.8.0 -org.threeten:threetenbp:1.7.0 +org.threeten:threeten-extra:1.7.1 diff --git a/spark/v4.0/spark-runtime/runtime-deps.txt b/spark/v4.0/spark-runtime/runtime-deps.txt index c16fa6f13e7a..a662942f9cd1 100644 --- a/spark/v4.0/spark-runtime/runtime-deps.txt +++ b/spark/v4.0/spark-runtime/runtime-deps.txt @@ -5,102 +5,27 @@ com.fasterxml.jackson.core:jackson-core:2.15.2 com.fasterxml.jackson.core:jackson-databind:2.15.2 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 com.github.ben-manes.caffeine:caffeine:2.9.3 -com.google.android:annotations:4.1.1.4 -com.google.api-client:google-api-client:2.7.2 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 -com.google.api.grpc:proto-google-common-protos:2.66.0 -com.google.api.grpc:proto-google-iam-v1:1.61.0 -com.google.api:api-common:2.58.0 -com.google.api:gax-grpc:2.75.0 -com.google.api:gax-httpjson:2.75.0 -com.google.api:gax:2.75.0 -com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 -com.google.auth:google-auth-library-credentials:1.43.0 -com.google.auth:google-auth-library-oauth2-http:1.43.0 -com.google.auto.value:auto-value-annotations:1.11.0 -com.google.cloud:google-cloud-bigquery:2.60.0 -com.google.cloud:google-cloud-bigquerystorage:3.23.0 -com.google.cloud:google-cloud-core-http:2.65.0 -com.google.cloud:google-cloud-core:2.65.0 -com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.12.1 -com.google.errorprone:error_prone_annotations:2.42.0 -com.google.flatbuffers:flatbuffers-java:24.3.25 -com.google.guava:failureaccess:1.0.3 -com.google.guava:guava:33.5.0-jre -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava -com.google.http-client:google-http-client-apache-v2:2.1.0 -com.google.http-client:google-http-client-appengine:2.1.0 -com.google.http-client:google-http-client-gson:2.1.0 -com.google.http-client:google-http-client:2.1.0 -com.google.j2objc:j2objc-annotations:3.1 -com.google.oauth-client:google-oauth-client:1.39.0 -com.google.protobuf:protobuf-java-util:4.33.2 -com.google.protobuf:protobuf-java:4.33.2 +com.google.code.gson:gson:2.11.0 +com.google.errorprone:error_prone_annotations:2.27.0 +com.google.flatbuffers:flatbuffers-java:23.5.26 com.squareup.okhttp3:okhttp:4.12.0 com.squareup.okio:okio-jvm:3.6.0 com.sun.xml.bind:jaxb-core:2.3.0 com.sun.xml.bind:jaxb-impl:2.3.0 dev.failsafe:failsafe:3.3.2 io.airlift:aircompressor:2.0.3 -io.grpc:grpc-alts:1.76.3 -io.grpc:grpc-api:1.76.3 -io.grpc:grpc-auth:1.76.3 -io.grpc:grpc-context:1.76.3 -io.grpc:grpc-core:1.76.3 -io.grpc:grpc-googleapis:1.76.3 -io.grpc:grpc-grpclb:1.76.3 -io.grpc:grpc-inprocess:1.76.3 -io.grpc:grpc-netty-shaded:1.76.3 -io.grpc:grpc-protobuf-lite:1.76.3 -io.grpc:grpc-protobuf:1.76.3 -io.grpc:grpc-stub:1.76.3 -io.grpc:grpc-util:1.76.3 io.netty:netty-buffer:4.2.10.Final io.netty:netty-common:4.2.10.Final -io.opencensus:opencensus-api:0.31.1 -io.opencensus:opencensus-contrib-http-util:0.31.1 -io.opentelemetry:opentelemetry-api:1.51.0 -io.opentelemetry:opentelemetry-context:1.51.0 -io.perfmark:perfmark-api:0.27.0 -org.apache.arrow:arrow-format:17.0.0 -org.apache.arrow:arrow-memory-core:17.0.0 -org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 -org.apache.arrow:arrow-memory-netty:17.0.0 -org.apache.arrow:arrow-vector:17.0.0 +org.apache.arrow:arrow-format:15.0.2 +org.apache.arrow:arrow-memory-core:15.0.2 +org.apache.arrow:arrow-memory-netty:15.0.2 +org.apache.arrow:arrow-vector:15.0.2 org.apache.avro:avro:1.12.1 org.apache.datasketches:datasketches-java:6.2.0 org.apache.datasketches:datasketches-memory:3.0.2 org.apache.httpcomponents.client5:httpclient5:5.6 org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.httpcomponents:httpclient:4.5.14 -org.apache.httpcomponents:httpcore:4.4.16 -org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-arrow:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-snowflake:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-spark-4.0_2.13:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-spark-extensions-4.0_2.13:1.11.0-SNAPSHOT org.apache.orc:orc-core:1.9.8 org.apache.orc:orc-shims:1.9.8 org.apache.parquet:parquet-avro:1.17.0 @@ -111,21 +36,17 @@ org.apache.parquet:parquet-format-structures:1.17.0 org.apache.parquet:parquet-hadoop:1.17.0 org.apache.parquet:parquet-jackson:1.17.0 org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-compat-qual:2.5.6 -org.checkerframework:checker-qual:3.49.0 -org.codehaus.mojo:animal-sniffer-annotations:1.24 -org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.checkerframework:checker-qual:3.19.0 +org.eclipse.collections:eclipse-collections-api:11.1.0 +org.eclipse.collections:eclipse-collections:11.1.0 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 org.jacoco:org.jacoco.agent:0.8.8 org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.json:json:20250517 -org.jspecify:jspecify:1.0.0 org.locationtech.jts:jts-core:1.20.0 org.projectnessie.nessie:nessie-client:0.107.4 org.projectnessie.nessie:nessie-model:0.107.4 org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.8.0 -org.threeten:threetenbp:1.7.0 +org.threeten:threeten-extra:1.7.1 diff --git a/spark/v4.1/spark-runtime/runtime-deps.txt b/spark/v4.1/spark-runtime/runtime-deps.txt index bcec06071570..a662942f9cd1 100644 --- a/spark/v4.1/spark-runtime/runtime-deps.txt +++ b/spark/v4.1/spark-runtime/runtime-deps.txt @@ -5,102 +5,27 @@ com.fasterxml.jackson.core:jackson-core:2.15.2 com.fasterxml.jackson.core:jackson-databind:2.15.2 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 com.github.ben-manes.caffeine:caffeine:2.9.3 -com.google.android:annotations:4.1.1.4 -com.google.api-client:google-api-client:2.7.2 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 -com.google.api.grpc:proto-google-common-protos:2.66.0 -com.google.api.grpc:proto-google-iam-v1:1.61.0 -com.google.api:api-common:2.58.0 -com.google.api:gax-grpc:2.75.0 -com.google.api:gax-httpjson:2.75.0 -com.google.api:gax:2.75.0 -com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 -com.google.auth:google-auth-library-credentials:1.43.0 -com.google.auth:google-auth-library-oauth2-http:1.43.0 -com.google.auto.value:auto-value-annotations:1.11.0 -com.google.cloud:google-cloud-bigquery:2.60.0 -com.google.cloud:google-cloud-bigquerystorage:3.23.0 -com.google.cloud:google-cloud-core-http:2.65.0 -com.google.cloud:google-cloud-core:2.65.0 -com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.12.1 -com.google.errorprone:error_prone_annotations:2.42.0 -com.google.flatbuffers:flatbuffers-java:24.3.25 -com.google.guava:failureaccess:1.0.3 -com.google.guava:guava:33.5.0-jre -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava -com.google.http-client:google-http-client-apache-v2:2.1.0 -com.google.http-client:google-http-client-appengine:2.1.0 -com.google.http-client:google-http-client-gson:2.1.0 -com.google.http-client:google-http-client:2.1.0 -com.google.j2objc:j2objc-annotations:3.1 -com.google.oauth-client:google-oauth-client:1.39.0 -com.google.protobuf:protobuf-java-util:4.33.2 -com.google.protobuf:protobuf-java:4.33.2 +com.google.code.gson:gson:2.11.0 +com.google.errorprone:error_prone_annotations:2.27.0 +com.google.flatbuffers:flatbuffers-java:23.5.26 com.squareup.okhttp3:okhttp:4.12.0 com.squareup.okio:okio-jvm:3.6.0 com.sun.xml.bind:jaxb-core:2.3.0 com.sun.xml.bind:jaxb-impl:2.3.0 dev.failsafe:failsafe:3.3.2 io.airlift:aircompressor:2.0.3 -io.grpc:grpc-alts:1.76.3 -io.grpc:grpc-api:1.76.3 -io.grpc:grpc-auth:1.76.3 -io.grpc:grpc-context:1.76.3 -io.grpc:grpc-core:1.76.3 -io.grpc:grpc-googleapis:1.76.3 -io.grpc:grpc-grpclb:1.76.3 -io.grpc:grpc-inprocess:1.76.3 -io.grpc:grpc-netty-shaded:1.76.3 -io.grpc:grpc-protobuf-lite:1.76.3 -io.grpc:grpc-protobuf:1.76.3 -io.grpc:grpc-stub:1.76.3 -io.grpc:grpc-util:1.76.3 io.netty:netty-buffer:4.2.10.Final io.netty:netty-common:4.2.10.Final -io.opencensus:opencensus-api:0.31.1 -io.opencensus:opencensus-contrib-http-util:0.31.1 -io.opentelemetry:opentelemetry-api:1.51.0 -io.opentelemetry:opentelemetry-context:1.51.0 -io.perfmark:perfmark-api:0.27.0 -org.apache.arrow:arrow-format:17.0.0 -org.apache.arrow:arrow-memory-core:17.0.0 -org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 -org.apache.arrow:arrow-memory-netty:17.0.0 -org.apache.arrow:arrow-vector:17.0.0 +org.apache.arrow:arrow-format:15.0.2 +org.apache.arrow:arrow-memory-core:15.0.2 +org.apache.arrow:arrow-memory-netty:15.0.2 +org.apache.arrow:arrow-vector:15.0.2 org.apache.avro:avro:1.12.1 org.apache.datasketches:datasketches-java:6.2.0 org.apache.datasketches:datasketches-memory:3.0.2 org.apache.httpcomponents.client5:httpclient5:5.6 org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.httpcomponents:httpclient:4.5.14 -org.apache.httpcomponents:httpcore:4.4.16 -org.apache.iceberg:iceberg-aliyun:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-api:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-arrow:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-aws:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-azure:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bigquery:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-bundled-guava:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-common:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-core:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-data:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-gcp:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-hive-metastore:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-nessie:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-orc:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-parquet:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-snowflake:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-spark-4.1_2.13:1.11.0-SNAPSHOT -org.apache.iceberg:iceberg-spark-extensions-4.1_2.13:1.11.0-SNAPSHOT org.apache.orc:orc-core:1.9.8 org.apache.orc:orc-shims:1.9.8 org.apache.parquet:parquet-avro:1.17.0 @@ -111,21 +36,17 @@ org.apache.parquet:parquet-format-structures:1.17.0 org.apache.parquet:parquet-hadoop:1.17.0 org.apache.parquet:parquet-jackson:1.17.0 org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-compat-qual:2.5.6 -org.checkerframework:checker-qual:3.49.0 -org.codehaus.mojo:animal-sniffer-annotations:1.24 -org.conscrypt:conscrypt-openjdk-uber:2.5.2 +org.checkerframework:checker-qual:3.19.0 +org.eclipse.collections:eclipse-collections-api:11.1.0 +org.eclipse.collections:eclipse-collections:11.1.0 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 org.jacoco:org.jacoco.agent:0.8.8 org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.json:json:20250517 -org.jspecify:jspecify:1.0.0 org.locationtech.jts:jts-core:1.20.0 org.projectnessie.nessie:nessie-client:0.107.4 org.projectnessie.nessie:nessie-model:0.107.4 org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.8.0 -org.threeten:threetenbp:1.7.0 +org.threeten:threeten-extra:1.7.1 From 3337f5cc0c5beb1c8279acf68c7a339519bd15ec Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 1 Apr 2026 15:34:36 -0500 Subject: [PATCH 3/7] Build: Differentiate version changes from added/removed in dependency guard Report version bumps separately from truly new or removed dependencies so reviewers can quickly distinguish routine upgrades from dependency surface changes that require LICENSE/NOTICE updates. --- gradle/runtime-deps.gradle | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/gradle/runtime-deps.gradle b/gradle/runtime-deps.gradle index a0c1b67b4747..791d4f6213c6 100644 --- a/gradle/runtime-deps.gradle +++ b/gradle/runtime-deps.gradle @@ -83,17 +83,30 @@ tasks.register('checkRuntimeDeps') { def expected = depsFile.readLines().findAll { it.trim() }.toSorted() if (actual != expected) { - def added = actual - expected - def removed = expected - actual + def groupArtifact = { coord -> coord.substring(0, coord.lastIndexOf(':')) } + def actualByModule = actual.collectEntries { [(groupArtifact(it)): it] } + def expectedByModule = expected.collectEntries { [(groupArtifact(it)): it] } + + def added = actualByModule.keySet() - expectedByModule.keySet() + def removed = expectedByModule.keySet() - actualByModule.keySet() + def shared = actualByModule.keySet().intersect(expectedByModule.keySet()) + def versionChanged = shared.findAll { actualByModule[it] != expectedByModule[it] } + def msg = new StringBuilder() msg.append("Runtime dependency baseline mismatch for ${project.name}!\n") + if (versionChanged) { + msg.append("\n Version changed (${versionChanged.size()}):\n") + versionChanged.toSorted().each { module -> + msg.append(" ~ ${expectedByModule[module]} -> ${actualByModule[module]}\n") + } + } if (added) { msg.append("\n Added (${added.size()}):\n") - added.each { msg.append(" + ${it}\n") } + added.toSorted().each { module -> msg.append(" + ${actualByModule[module]}\n") } } if (removed) { msg.append("\n Removed (${removed.size()}):\n") - removed.each { msg.append(" - ${it}\n") } + removed.toSorted().each { module -> msg.append(" - ${expectedByModule[module]}\n") } } msg.append("\nTo update the baseline run:\n") msg.append(" ./gradlew ${project.path}:generateRuntimeDeps\n") From 09f7c54d0ef3f14b7be4e6ba8155a9acef6fcc7d Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 1 Apr 2026 15:46:29 -0500 Subject: [PATCH 4/7] Build: Exclude runtime-deps.txt from RAT license check These are plain data files containing only dependency coordinates, so a license header is not practical. --- dev/.rat-excludes | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/.rat-excludes b/dev/.rat-excludes index 52a800723598..f94ef5bf8988 100644 --- a/dev/.rat-excludes +++ b/dev/.rat-excludes @@ -29,3 +29,4 @@ sitemap.xml .python-version **/*_index.md **/.venv/** +**/runtime-deps.txt From cca9eca932b62c3ac0c62307f0282dfd6a641f72 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 1 Apr 2026 16:26:01 -0500 Subject: [PATCH 5/7] Build, CI: Add top-level checkAllRuntimeDeps task and CI job Add a checkAllRuntimeDeps aggregation task in build.gradle that collects checkRuntimeDeps from all subprojects, and a dedicated check-runtime-deps CI job in java-ci.yml that runs it on every PR. Incorporates ideas from rdblue's #15857 (top-level aggregation task and dedicated CI job) into the runtime dependency guard approach. --- .github/workflows/java-ci.yml | 13 +++++++++++++ build.gradle | 9 +++++++++ 2 files changed, 22 insertions(+) diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index 4ef0a30b8225..bc6cb23b6f90 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -126,3 +126,16 @@ jobs: java-version: ${{ matrix.jvm }} - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5 # zizmor: ignore[cache-poisoning] -- cache writes are restricted to the default branch by setup-gradle - run: ./gradlew -Pquick=true javadoc + + check-runtime-deps: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: + distribution: zulu + java-version: 17 + - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5 # zizmor: ignore[cache-poisoning] -- cache writes are restricted to the default branch by setup-gradle + - run: ./gradlew checkAllRuntimeDeps diff --git a/build.gradle b/build.gradle index 35e1d6a002f3..8ff278b77ee4 100644 --- a/build.gradle +++ b/build.gradle @@ -120,6 +120,15 @@ allprojects { } } +tasks.register('checkAllRuntimeDeps') { + description = 'Validates runtime dependency baselines for all subprojects that have them' + group = 'verification' + + dependsOn subprojects.collect { subproject -> + subproject.tasks.matching { it.name == 'checkRuntimeDeps' } + } +} + subprojects { if (it.name == 'iceberg-bom') { // the BOM does not build anything, the code below expects "source code" From 4759cd6d923ce2c762b572fede53b23d834f634e Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 1 Apr 2026 16:32:03 -0500 Subject: [PATCH 6/7] Build: Move runtime-deps script to root, warn on missing baseline, remove baselines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move runtime-deps.gradle from gradle/ to the project root alongside other custom scripts (tasks.gradle, deploy.gradle, baseline.gradle). Change the missing baseline check from a hard failure to a warning so that the guard can be wired up before baselines are generated. Remove all runtime-deps.txt baseline files — these will be regenerated in a follow-up once the infrastructure is finalized. --- aws-bundle/build.gradle | 2 +- aws-bundle/runtime-deps.txt | 70 ------ azure-bundle/build.gradle | 2 +- azure-bundle/runtime-deps.txt | 44 ---- flink/v1.20/build.gradle | 2 +- flink/v1.20/flink-runtime/runtime-deps.txt | 45 ---- flink/v2.0/build.gradle | 2 +- flink/v2.0/flink-runtime/runtime-deps.txt | 45 ---- flink/v2.1/build.gradle | 2 +- flink/v2.1/flink-runtime/runtime-deps.txt | 45 ---- gcp-bundle/build.gradle | 2 +- gcp-bundle/runtime-deps.txt | 114 --------- kafka-connect/build.gradle | 2 +- .../kafka-connect-runtime/runtime-deps.txt | 233 ------------------ ...runtime-deps.gradle => runtime-deps.gradle | 33 ++- spark/v3.4/build.gradle | 2 +- spark/v3.4/spark-runtime/runtime-deps.txt | 52 ---- spark/v3.5/build.gradle | 2 +- spark/v3.5/spark-runtime/runtime-deps.txt | 52 ---- spark/v4.0/build.gradle | 2 +- spark/v4.0/spark-runtime/runtime-deps.txt | 52 ---- spark/v4.1/build.gradle | 2 +- spark/v4.1/spark-runtime/runtime-deps.txt | 52 ---- 23 files changed, 33 insertions(+), 826 deletions(-) delete mode 100644 aws-bundle/runtime-deps.txt delete mode 100644 azure-bundle/runtime-deps.txt delete mode 100644 flink/v1.20/flink-runtime/runtime-deps.txt delete mode 100644 flink/v2.0/flink-runtime/runtime-deps.txt delete mode 100644 flink/v2.1/flink-runtime/runtime-deps.txt delete mode 100644 gcp-bundle/runtime-deps.txt delete mode 100644 kafka-connect/kafka-connect-runtime/runtime-deps.txt rename gradle/runtime-deps.gradle => runtime-deps.gradle (78%) delete mode 100644 spark/v3.4/spark-runtime/runtime-deps.txt delete mode 100644 spark/v3.5/spark-runtime/runtime-deps.txt delete mode 100644 spark/v4.0/spark-runtime/runtime-deps.txt delete mode 100644 spark/v4.1/spark-runtime/runtime-deps.txt diff --git a/aws-bundle/build.gradle b/aws-bundle/build.gradle index b2fbcb111caf..c891ac5b439c 100644 --- a/aws-bundle/build.gradle +++ b/aws-bundle/build.gradle @@ -67,5 +67,5 @@ project(":iceberg-aws-bundle") { enabled = false } - apply from: "${rootDir}/gradle/runtime-deps.gradle" + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/aws-bundle/runtime-deps.txt b/aws-bundle/runtime-deps.txt deleted file mode 100644 index 7e692a6a111a..000000000000 --- a/aws-bundle/runtime-deps.txt +++ /dev/null @@ -1,70 +0,0 @@ -com.github.ben-manes.caffeine:caffeine:2.9.3 -com.google.errorprone:error_prone_annotations:2.10.0 -commons-codec:commons-codec:1.17.1 -commons-logging:commons-logging:1.2 -io.netty:netty-buffer:4.1.130.Final -io.netty:netty-codec-http2:4.1.130.Final -io.netty:netty-codec-http:4.1.130.Final -io.netty:netty-codec:4.1.130.Final -io.netty:netty-common:4.1.130.Final -io.netty:netty-handler:4.1.130.Final -io.netty:netty-resolver:4.1.130.Final -io.netty:netty-transport-classes-epoll:4.1.130.Final -io.netty:netty-transport-native-unix-common:4.1.130.Final -io.netty:netty-transport:4.1.130.Final -org.apache.httpcomponents:httpclient:4.5.13 -org.apache.httpcomponents:httpcore:4.4.16 -org.apache.logging.log4j:log4j-api:2.20.0 -org.apache.logging.log4j:log4j-core:2.20.0 -org.apache.logging.log4j:log4j-slf4j-impl:2.20.0 -org.checkerframework:checker-qual:3.19.0 -org.reactivestreams:reactive-streams:1.0.4 -org.slf4j:slf4j-api:2.0.17 -software.amazon.awssdk.crt:aws-crt:0.43.5 -software.amazon.awssdk:annotations:2.42.18 -software.amazon.awssdk:apache-client:2.42.18 -software.amazon.awssdk:arns:2.42.18 -software.amazon.awssdk:auth:2.42.18 -software.amazon.awssdk:aws-core:2.42.18 -software.amazon.awssdk:aws-json-protocol:2.42.18 -software.amazon.awssdk:aws-query-protocol:2.42.18 -software.amazon.awssdk:aws-xml-protocol:2.42.18 -software.amazon.awssdk:checksums-spi:2.42.18 -software.amazon.awssdk:checksums:2.42.18 -software.amazon.awssdk:cloudwatch-metric-publisher:2.42.18 -software.amazon.awssdk:cloudwatch:2.42.18 -software.amazon.awssdk:crt-core:2.42.18 -software.amazon.awssdk:dynamodb:2.42.18 -software.amazon.awssdk:endpoints-spi:2.42.18 -software.amazon.awssdk:glue:2.42.18 -software.amazon.awssdk:http-auth-aws-crt:2.42.18 -software.amazon.awssdk:http-auth-aws-eventstream:2.42.18 -software.amazon.awssdk:http-auth-aws:2.42.18 -software.amazon.awssdk:http-auth-spi:2.42.18 -software.amazon.awssdk:http-auth:2.42.18 -software.amazon.awssdk:http-client-spi:2.42.18 -software.amazon.awssdk:iam:2.42.18 -software.amazon.awssdk:identity-spi:2.42.18 -software.amazon.awssdk:json-utils:2.42.18 -software.amazon.awssdk:kms:2.42.18 -software.amazon.awssdk:lakeformation:2.42.18 -software.amazon.awssdk:metrics-spi:2.42.18 -software.amazon.awssdk:netty-nio-client:2.42.18 -software.amazon.awssdk:profiles:2.42.18 -software.amazon.awssdk:protocol-core:2.42.18 -software.amazon.awssdk:regions:2.42.18 -software.amazon.awssdk:retries-spi:2.42.18 -software.amazon.awssdk:retries:2.42.18 -software.amazon.awssdk:s3:2.42.18 -software.amazon.awssdk:s3control:2.42.18 -software.amazon.awssdk:sdk-core:2.42.18 -software.amazon.awssdk:smithy-rpcv2-protocol:2.42.18 -software.amazon.awssdk:sso:2.42.18 -software.amazon.awssdk:sts:2.42.18 -software.amazon.awssdk:third-party-jackson-core:2.42.18 -software.amazon.awssdk:third-party-jackson-dataformat-cbor:2.42.18 -software.amazon.awssdk:utils-lite:2.42.18 -software.amazon.awssdk:utils:2.42.18 -software.amazon.eventstream:eventstream:1.0.1 -software.amazon.s3.accessgrants:aws-s3-accessgrants-java-plugin:2.4.1 -software.amazon.s3.analyticsaccelerator:analyticsaccelerator-s3:1.3.1 diff --git a/azure-bundle/build.gradle b/azure-bundle/build.gradle index 6278d3fb66af..dad563b67ab7 100644 --- a/azure-bundle/build.gradle +++ b/azure-bundle/build.gradle @@ -53,5 +53,5 @@ project(":iceberg-azure-bundle") { enabled = false } - apply from: "${rootDir}/gradle/runtime-deps.gradle" + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/azure-bundle/runtime-deps.txt b/azure-bundle/runtime-deps.txt deleted file mode 100644 index 6ff75ae80e46..000000000000 --- a/azure-bundle/runtime-deps.txt +++ /dev/null @@ -1,44 +0,0 @@ -com.azure:azure-core-http-netty:1.16.3 -com.azure:azure-core:1.57.1 -com.azure:azure-identity:1.18.2 -com.azure:azure-json:1.5.1 -com.azure:azure-security-keyvault-keys:4.10.5 -com.azure:azure-storage-blob:12.33.1 -com.azure:azure-storage-common:12.32.2 -com.azure:azure-storage-file-datalake:12.26.1 -com.azure:azure-storage-internal-avro:12.18.2 -com.azure:azure-xml:1.2.1 -com.fasterxml.jackson.core:jackson-annotations:2.18.4 -com.fasterxml.jackson.core:jackson-core:2.18.4.1 -com.fasterxml.jackson.core:jackson-databind:2.18.4 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.4 -com.microsoft.azure:msal4j-persistence-extension:1.3.0 -com.microsoft.azure:msal4j:1.23.1 -io.netty:netty-buffer:4.1.130.Final -io.netty:netty-codec-dns:4.1.128.Final -io.netty:netty-codec-http2:4.1.130.Final -io.netty:netty-codec-http:4.1.130.Final -io.netty:netty-codec-socks:4.1.130.Final -io.netty:netty-codec:4.1.130.Final -io.netty:netty-common:4.1.130.Final -io.netty:netty-handler-proxy:4.1.130.Final -io.netty:netty-handler:4.1.130.Final -io.netty:netty-resolver-dns-classes-macos:4.1.128.Final -io.netty:netty-resolver-dns-native-macos:4.1.128.Final -io.netty:netty-resolver-dns:4.1.128.Final -io.netty:netty-resolver:4.1.130.Final -io.netty:netty-tcnative-boringssl-static:2.0.74.Final -io.netty:netty-tcnative-classes:2.0.74.Final -io.netty:netty-transport-classes-epoll:4.1.130.Final -io.netty:netty-transport-classes-kqueue:4.1.130.Final -io.netty:netty-transport-native-epoll:4.1.130.Final -io.netty:netty-transport-native-kqueue:4.1.130.Final -io.netty:netty-transport-native-unix-common:4.1.130.Final -io.netty:netty-transport:4.1.130.Final -io.projectreactor.netty:reactor-netty-core:1.2.13 -io.projectreactor.netty:reactor-netty-http:1.2.13 -io.projectreactor:reactor-core:3.7.14 -net.java.dev.jna:jna-platform:5.17.0 -net.java.dev.jna:jna:5.17.0 -org.reactivestreams:reactive-streams:1.0.4 -org.slf4j:slf4j-api:2.0.17 diff --git a/flink/v1.20/build.gradle b/flink/v1.20/build.gradle index c62a0305e00d..772133c8e1d8 100644 --- a/flink/v1.20/build.gradle +++ b/flink/v1.20/build.gradle @@ -267,5 +267,5 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") { enabled = false } - apply from: "${rootDir}/gradle/runtime-deps.gradle" + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/flink/v1.20/flink-runtime/runtime-deps.txt b/flink/v1.20/flink-runtime/runtime-deps.txt deleted file mode 100644 index 108d72bd8f12..000000000000 --- a/flink/v1.20/flink-runtime/runtime-deps.txt +++ /dev/null @@ -1,45 +0,0 @@ -com.aliyun:credentials-java:0.3.12 -com.aliyun:tea:1.4.1 -com.fasterxml.jackson.core:jackson-annotations:2.21 -com.fasterxml.jackson.core:jackson-core:2.21.2 -com.fasterxml.jackson.core:jackson-databind:2.21.2 -com.github.ben-manes.caffeine:caffeine:2.9.3 -com.github.luben:zstd-jni:1.5.7-3 -com.google.code.gson:gson:2.11.0 -com.google.errorprone:error_prone_annotations:2.27.0 -com.squareup.okhttp3:okhttp:4.12.0 -com.squareup.okio:okio-jvm:3.6.0 -com.sun.xml.bind:jaxb-core:2.3.0 -com.sun.xml.bind:jaxb-impl:2.3.0 -dev.failsafe:failsafe:3.3.2 -io.airlift:aircompressor:2.0.3 -io.dropwizard.metrics:metrics-core:3.2.6 -org.apache.avro:avro:1.12.1 -org.apache.datasketches:datasketches-java:6.2.0 -org.apache.datasketches:datasketches-memory:3.0.2 -org.apache.flink:flink-metrics-dropwizard:1.20.1 -org.apache.httpcomponents.client5:httpclient5:5.6 -org.apache.httpcomponents.core5:httpcore5-h2:5.4 -org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.orc:orc-core:1.9.8 -org.apache.orc:orc-shims:1.9.8 -org.apache.parquet:parquet-avro:1.17.0 -org.apache.parquet:parquet-column:1.17.0 -org.apache.parquet:parquet-common:1.17.0 -org.apache.parquet:parquet-encoding:1.17.0 -org.apache.parquet:parquet-format-structures:1.17.0 -org.apache.parquet:parquet-hadoop:1.17.0 -org.apache.parquet:parquet-jackson:1.17.0 -org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-qual:3.19.0 -org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 -org.jacoco:org.jacoco.agent:0.8.8 -org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.locationtech.jts:jts-core:1.20.0 -org.projectnessie.nessie:nessie-client:0.107.4 -org.projectnessie.nessie:nessie-model:0.107.4 -org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.7.1 diff --git a/flink/v2.0/build.gradle b/flink/v2.0/build.gradle index 478fce44b81b..b276cb90dd24 100644 --- a/flink/v2.0/build.gradle +++ b/flink/v2.0/build.gradle @@ -267,5 +267,5 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") { enabled = false } - apply from: "${rootDir}/gradle/runtime-deps.gradle" + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/flink/v2.0/flink-runtime/runtime-deps.txt b/flink/v2.0/flink-runtime/runtime-deps.txt deleted file mode 100644 index 17dad715199a..000000000000 --- a/flink/v2.0/flink-runtime/runtime-deps.txt +++ /dev/null @@ -1,45 +0,0 @@ -com.aliyun:credentials-java:0.3.12 -com.aliyun:tea:1.4.1 -com.fasterxml.jackson.core:jackson-annotations:2.21 -com.fasterxml.jackson.core:jackson-core:2.21.2 -com.fasterxml.jackson.core:jackson-databind:2.21.2 -com.github.ben-manes.caffeine:caffeine:2.9.3 -com.github.luben:zstd-jni:1.5.7-3 -com.google.code.gson:gson:2.11.0 -com.google.errorprone:error_prone_annotations:2.27.0 -com.squareup.okhttp3:okhttp:4.12.0 -com.squareup.okio:okio-jvm:3.6.0 -com.sun.xml.bind:jaxb-core:2.3.0 -com.sun.xml.bind:jaxb-impl:2.3.0 -dev.failsafe:failsafe:3.3.2 -io.airlift:aircompressor:2.0.3 -io.dropwizard.metrics:metrics-core:3.2.6 -org.apache.avro:avro:1.12.1 -org.apache.datasketches:datasketches-java:6.2.0 -org.apache.datasketches:datasketches-memory:3.0.2 -org.apache.flink:flink-metrics-dropwizard:2.0.0 -org.apache.httpcomponents.client5:httpclient5:5.6 -org.apache.httpcomponents.core5:httpcore5-h2:5.4 -org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.orc:orc-core:1.9.8 -org.apache.orc:orc-shims:1.9.8 -org.apache.parquet:parquet-avro:1.17.0 -org.apache.parquet:parquet-column:1.17.0 -org.apache.parquet:parquet-common:1.17.0 -org.apache.parquet:parquet-encoding:1.17.0 -org.apache.parquet:parquet-format-structures:1.17.0 -org.apache.parquet:parquet-hadoop:1.17.0 -org.apache.parquet:parquet-jackson:1.17.0 -org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-qual:3.19.0 -org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 -org.jacoco:org.jacoco.agent:0.8.8 -org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.locationtech.jts:jts-core:1.20.0 -org.projectnessie.nessie:nessie-client:0.107.4 -org.projectnessie.nessie:nessie-model:0.107.4 -org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.7.1 diff --git a/flink/v2.1/build.gradle b/flink/v2.1/build.gradle index 24d6f557a3fb..a08cb1d5ebdd 100644 --- a/flink/v2.1/build.gradle +++ b/flink/v2.1/build.gradle @@ -267,5 +267,5 @@ project(":iceberg-flink:iceberg-flink-runtime-${flinkMajorVersion}") { enabled = false } - apply from: "${rootDir}/gradle/runtime-deps.gradle" + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/flink/v2.1/flink-runtime/runtime-deps.txt b/flink/v2.1/flink-runtime/runtime-deps.txt deleted file mode 100644 index c08131f14806..000000000000 --- a/flink/v2.1/flink-runtime/runtime-deps.txt +++ /dev/null @@ -1,45 +0,0 @@ -com.aliyun:credentials-java:0.3.12 -com.aliyun:tea:1.4.1 -com.fasterxml.jackson.core:jackson-annotations:2.21 -com.fasterxml.jackson.core:jackson-core:2.21.2 -com.fasterxml.jackson.core:jackson-databind:2.21.2 -com.github.ben-manes.caffeine:caffeine:2.9.3 -com.github.luben:zstd-jni:1.5.7-3 -com.google.code.gson:gson:2.11.0 -com.google.errorprone:error_prone_annotations:2.27.0 -com.squareup.okhttp3:okhttp:4.12.0 -com.squareup.okio:okio-jvm:3.6.0 -com.sun.xml.bind:jaxb-core:2.3.0 -com.sun.xml.bind:jaxb-impl:2.3.0 -dev.failsafe:failsafe:3.3.2 -io.airlift:aircompressor:2.0.3 -io.dropwizard.metrics:metrics-core:3.2.6 -org.apache.avro:avro:1.12.1 -org.apache.datasketches:datasketches-java:6.2.0 -org.apache.datasketches:datasketches-memory:3.0.2 -org.apache.flink:flink-metrics-dropwizard:2.1.0 -org.apache.httpcomponents.client5:httpclient5:5.6 -org.apache.httpcomponents.core5:httpcore5-h2:5.4 -org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.orc:orc-core:1.9.8 -org.apache.orc:orc-shims:1.9.8 -org.apache.parquet:parquet-avro:1.17.0 -org.apache.parquet:parquet-column:1.17.0 -org.apache.parquet:parquet-common:1.17.0 -org.apache.parquet:parquet-encoding:1.17.0 -org.apache.parquet:parquet-format-structures:1.17.0 -org.apache.parquet:parquet-hadoop:1.17.0 -org.apache.parquet:parquet-jackson:1.17.0 -org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-qual:3.19.0 -org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 -org.jacoco:org.jacoco.agent:0.8.8 -org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.locationtech.jts:jts-core:1.20.0 -org.projectnessie.nessie:nessie-client:0.107.4 -org.projectnessie.nessie:nessie-model:0.107.4 -org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.7.1 diff --git a/gcp-bundle/build.gradle b/gcp-bundle/build.gradle index 84482a7b039b..1f6642c9b2ce 100644 --- a/gcp-bundle/build.gradle +++ b/gcp-bundle/build.gradle @@ -60,5 +60,5 @@ project(":iceberg-gcp-bundle") { enabled = false } - apply from: "${rootDir}/gradle/runtime-deps.gradle" + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/gcp-bundle/runtime-deps.txt b/gcp-bundle/runtime-deps.txt deleted file mode 100644 index 5fca81c617c8..000000000000 --- a/gcp-bundle/runtime-deps.txt +++ /dev/null @@ -1,114 +0,0 @@ -com.fasterxml.jackson.core:jackson-annotations:2.18.2 -com.fasterxml.jackson.core:jackson-core:2.18.2 -com.fasterxml.jackson.core:jackson-databind:2.18.2 -com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.18.2 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.2 -com.fasterxml.woodstox:woodstox-core:7.0.0 -com.google.android:annotations:4.1.1.4 -com.google.api-client:google-api-client:2.7.2 -com.google.api.grpc:gapic-google-cloud-storage-v2:2.64.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:grpc-google-cloud-storage-v2:2.64.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 -com.google.api.grpc:proto-google-cloud-kms-v1:0.181.0 -com.google.api.grpc:proto-google-cloud-monitoring-v3:3.88.0 -com.google.api.grpc:proto-google-cloud-storage-v2:2.64.0 -com.google.api.grpc:proto-google-common-protos:2.66.0 -com.google.api.grpc:proto-google-iam-v1:1.61.0 -com.google.api:api-common:2.58.0 -com.google.api:gax-grpc:2.75.0 -com.google.api:gax-httpjson:2.75.0 -com.google.api:gax:2.75.0 -com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 -com.google.apis:google-api-services-storage:v1-rev20251118-2.0.0 -com.google.auth:google-auth-library-credentials:1.43.0 -com.google.auth:google-auth-library-oauth2-http:1.43.0 -com.google.auto.value:auto-value-annotations:1.11.1 -com.google.cloud.gcs.analytics:client:1.2.3 -com.google.cloud.gcs.analytics:gcs-analytics-core:1.2.3 -com.google.cloud.opentelemetry:detector-resources-support:0.33.0 -com.google.cloud.opentelemetry:exporter-metrics:0.33.0 -com.google.cloud.opentelemetry:shared-resourcemapping:0.33.0 -com.google.cloud:google-cloud-bigquery:2.60.0 -com.google.cloud:google-cloud-bigquerystorage:3.23.0 -com.google.cloud:google-cloud-core-grpc:2.65.0 -com.google.cloud:google-cloud-core-http:2.65.0 -com.google.cloud:google-cloud-core:2.65.0 -com.google.cloud:google-cloud-kms:2.90.0 -com.google.cloud:google-cloud-monitoring:3.88.0 -com.google.cloud:google-cloud-storage:2.64.0 -com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.12.1 -com.google.errorprone:error_prone_annotations:2.42.0 -com.google.flatbuffers:flatbuffers-java:24.3.25 -com.google.guava:failureaccess:1.0.3 -com.google.guava:guava:33.5.0-jre -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava -com.google.http-client:google-http-client-apache-v2:2.1.0 -com.google.http-client:google-http-client-appengine:2.1.0 -com.google.http-client:google-http-client-gson:2.1.0 -com.google.http-client:google-http-client-jackson2:2.1.0 -com.google.http-client:google-http-client:2.1.0 -com.google.j2objc:j2objc-annotations:3.1 -com.google.oauth-client:google-oauth-client:1.39.0 -com.google.protobuf:protobuf-java-util:4.33.2 -com.google.protobuf:protobuf-java:4.33.2 -com.google.re2j:re2j:1.8 -commons-codec:commons-codec:1.18.0 -io.grpc:grpc-alts:1.76.3 -io.grpc:grpc-api:1.76.3 -io.grpc:grpc-auth:1.76.3 -io.grpc:grpc-context:1.76.3 -io.grpc:grpc-core:1.76.3 -io.grpc:grpc-googleapis:1.76.3 -io.grpc:grpc-grpclb:1.76.3 -io.grpc:grpc-inprocess:1.76.3 -io.grpc:grpc-netty-shaded:1.76.3 -io.grpc:grpc-opentelemetry:1.76.3 -io.grpc:grpc-protobuf-lite:1.76.3 -io.grpc:grpc-protobuf:1.76.3 -io.grpc:grpc-rls:1.76.3 -io.grpc:grpc-services:1.76.3 -io.grpc:grpc-stub:1.76.3 -io.grpc:grpc-util:1.76.3 -io.grpc:grpc-xds:1.76.3 -io.netty:netty-buffer:4.1.110.Final -io.netty:netty-common:4.1.110.Final -io.opencensus:opencensus-api:0.31.1 -io.opencensus:opencensus-contrib-http-util:0.31.1 -io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37.0-alpha -io.opentelemetry.semconv:opentelemetry-semconv:1.29.0-alpha -io.opentelemetry:opentelemetry-api:1.51.0 -io.opentelemetry:opentelemetry-context:1.51.0 -io.opentelemetry:opentelemetry-sdk-common:1.51.0 -io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.51.0 -io.opentelemetry:opentelemetry-sdk-logs:1.51.0 -io.opentelemetry:opentelemetry-sdk-metrics:1.51.0 -io.opentelemetry:opentelemetry-sdk-trace:1.51.0 -io.opentelemetry:opentelemetry-sdk:1.51.0 -io.perfmark:perfmark-api:0.27.0 -javax.annotation:javax.annotation-api:1.3.2 -org.apache.arrow:arrow-format:17.0.0 -org.apache.arrow:arrow-memory-core:17.0.0 -org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 -org.apache.arrow:arrow-memory-netty:17.0.0 -org.apache.arrow:arrow-vector:17.0.0 -org.apache.commons:commons-lang3:3.20.0 -org.apache.httpcomponents:httpclient:4.5.14 -org.apache.httpcomponents:httpcore:4.4.16 -org.checkerframework:checker-compat-qual:2.5.6 -org.checkerframework:checker-qual:3.49.0 -org.codehaus.mojo:animal-sniffer-annotations:1.24 -org.codehaus.woodstox:stax2-api:4.2.2 -org.conscrypt:conscrypt-openjdk-uber:2.5.2 -org.json:json:20250517 -org.jspecify:jspecify:1.0.0 -org.slf4j:slf4j-api:2.0.17 -org.threeten:threeten-extra:1.8.0 -org.threeten:threetenbp:1.7.0 diff --git a/kafka-connect/build.gradle b/kafka-connect/build.gradle index 8e4b8877d7e3..009ae719bac5 100644 --- a/kafka-connect/build.gradle +++ b/kafka-connect/build.gradle @@ -255,7 +255,7 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') { assemble.dependsOn distZip, hiveDistZip - apply from: "${rootDir}/gradle/runtime-deps.gradle" + apply from: "${rootDir}/runtime-deps.gradle" } project(':iceberg-kafka-connect:iceberg-kafka-connect-transforms') { diff --git a/kafka-connect/kafka-connect-runtime/runtime-deps.txt b/kafka-connect/kafka-connect-runtime/runtime-deps.txt deleted file mode 100644 index 417477bfcf12..000000000000 --- a/kafka-connect/kafka-connect-runtime/runtime-deps.txt +++ /dev/null @@ -1,233 +0,0 @@ -com.azure:azure-core-http-netty:1.16.3 -com.azure:azure-core:1.57.1 -com.azure:azure-identity:1.18.2 -com.azure:azure-json:1.5.1 -com.azure:azure-storage-blob:12.33.1 -com.azure:azure-storage-common:12.32.2 -com.azure:azure-storage-file-datalake:12.26.1 -com.azure:azure-storage-internal-avro:12.18.2 -com.azure:azure-xml:1.2.1 -com.fasterxml.jackson.core:jackson-annotations:2.21 -com.fasterxml.jackson.core:jackson-core:2.21.2 -com.fasterxml.jackson.core:jackson-databind:2.21.2 -com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.21.2 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 -com.fasterxml.woodstox:woodstox-core:6.7.0 -com.github.ben-manes.caffeine:caffeine:2.9.3 -com.github.luben:zstd-jni:1.5.7-3 -com.github.pjfanning:jersey-json:1.22.0 -com.google.android:annotations:4.1.1.4 -com.google.api-client:google-api-client:2.7.2 -com.google.api.grpc:gapic-google-cloud-storage-v2:2.64.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:grpc-google-cloud-storage-v2:2.64.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.23.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.195.0 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.23.0 -com.google.api.grpc:proto-google-cloud-monitoring-v3:3.88.0 -com.google.api.grpc:proto-google-cloud-storage-v2:2.64.0 -com.google.api.grpc:proto-google-common-protos:2.66.0 -com.google.api.grpc:proto-google-iam-v1:1.61.0 -com.google.api:api-common:2.58.0 -com.google.api:gax-grpc:2.75.0 -com.google.api:gax-httpjson:2.75.0 -com.google.api:gax:2.75.0 -com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0.0 -com.google.apis:google-api-services-storage:v1-rev20251118-2.0.0 -com.google.auth:google-auth-library-credentials:1.43.0 -com.google.auth:google-auth-library-oauth2-http:1.43.0 -com.google.auto.value:auto-value-annotations:1.11.0 -com.google.cloud.opentelemetry:detector-resources-support:0.33.0 -com.google.cloud.opentelemetry:exporter-metrics:0.33.0 -com.google.cloud.opentelemetry:shared-resourcemapping:0.33.0 -com.google.cloud:google-cloud-bigquery:2.60.0 -com.google.cloud:google-cloud-bigquerystorage:3.23.0 -com.google.cloud:google-cloud-core-grpc:2.65.0 -com.google.cloud:google-cloud-core-http:2.65.0 -com.google.cloud:google-cloud-core:2.65.0 -com.google.cloud:google-cloud-monitoring:3.88.0 -com.google.cloud:google-cloud-storage:2.64.0 -com.google.code.findbugs:jsr305:3.0.2 -com.google.code.gson:gson:2.12.1 -com.google.errorprone:error_prone_annotations:2.42.0 -com.google.flatbuffers:flatbuffers-java:24.3.25 -com.google.guava:failureaccess:1.0.3 -com.google.guava:guava:33.5.0-jre -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava -com.google.http-client:google-http-client-apache-v2:2.1.0 -com.google.http-client:google-http-client-appengine:2.1.0 -com.google.http-client:google-http-client-gson:2.1.0 -com.google.http-client:google-http-client-jackson2:2.1.0 -com.google.http-client:google-http-client:2.1.0 -com.google.j2objc:j2objc-annotations:3.1 -com.google.oauth-client:google-oauth-client:1.39.0 -com.google.protobuf:protobuf-java-util:4.33.2 -com.google.protobuf:protobuf-java:4.33.2 -com.google.re2j:re2j:1.8 -com.jcraft:jsch:0.1.55 -com.microsoft.azure:msal4j-persistence-extension:1.3.0 -com.microsoft.azure:msal4j:1.23.1 -com.sun.xml.bind:jaxb-impl:2.2.3-1 -commons-cli:commons-cli:1.9.0 -commons-codec:commons-codec:1.19.0 -commons-io:commons-io:2.20.0 -commons-logging:commons-logging:1.2 -commons-net:commons-net:3.9.0 -commons-pool:commons-pool:1.6 -dev.failsafe:failsafe:3.3.2 -dnsjava:dnsjava:3.6.1 -io.airlift:aircompressor:2.0.3 -io.dropwizard.metrics:metrics-core:3.2.4 -io.grpc:grpc-alts:1.76.3 -io.grpc:grpc-api:1.76.3 -io.grpc:grpc-auth:1.76.3 -io.grpc:grpc-context:1.76.3 -io.grpc:grpc-core:1.76.3 -io.grpc:grpc-googleapis:1.76.3 -io.grpc:grpc-grpclb:1.76.3 -io.grpc:grpc-inprocess:1.76.3 -io.grpc:grpc-netty-shaded:1.80.0 -io.grpc:grpc-opentelemetry:1.76.3 -io.grpc:grpc-protobuf-lite:1.76.3 -io.grpc:grpc-protobuf:1.76.3 -io.grpc:grpc-rls:1.76.3 -io.grpc:grpc-services:1.76.3 -io.grpc:grpc-stub:1.76.3 -io.grpc:grpc-util:1.76.3 -io.grpc:grpc-xds:1.76.3 -io.netty:netty-buffer:4.1.130.Final -io.netty:netty-codec-dns:4.1.128.Final -io.netty:netty-codec-http2:4.1.130.Final -io.netty:netty-codec-http:4.1.130.Final -io.netty:netty-codec-socks:4.1.130.Final -io.netty:netty-codec:4.1.130.Final -io.netty:netty-common:4.1.130.Final -io.netty:netty-handler-proxy:4.1.130.Final -io.netty:netty-handler:4.1.130.Final -io.netty:netty-resolver-dns-classes-macos:4.1.128.Final -io.netty:netty-resolver-dns-native-macos:4.1.128.Final -io.netty:netty-resolver-dns:4.1.128.Final -io.netty:netty-resolver:4.1.130.Final -io.netty:netty-tcnative-boringssl-static:2.0.74.Final -io.netty:netty-tcnative-classes:2.0.74.Final -io.netty:netty-transport-classes-epoll:4.1.130.Final -io.netty:netty-transport-classes-kqueue:4.1.130.Final -io.netty:netty-transport-native-epoll:4.1.130.Final -io.netty:netty-transport-native-kqueue:4.1.130.Final -io.netty:netty-transport-native-unix-common:4.1.130.Final -io.netty:netty-transport:4.1.130.Final -io.opencensus:opencensus-api:0.31.1 -io.opencensus:opencensus-contrib-http-util:0.31.1 -io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37.0-alpha -io.opentelemetry.semconv:opentelemetry-semconv:1.29.0-alpha -io.opentelemetry:opentelemetry-api:1.51.0 -io.opentelemetry:opentelemetry-context:1.51.0 -io.opentelemetry:opentelemetry-sdk-common:1.51.0 -io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.51.0 -io.opentelemetry:opentelemetry-sdk-logs:1.51.0 -io.opentelemetry:opentelemetry-sdk-metrics:1.51.0 -io.opentelemetry:opentelemetry-sdk-trace:1.51.0 -io.opentelemetry:opentelemetry-sdk:1.51.0 -io.perfmark:perfmark-api:0.27.0 -io.projectreactor.netty:reactor-netty-core:1.2.13 -io.projectreactor.netty:reactor-netty-http:1.2.13 -io.projectreactor:reactor-core:3.7.14 -jakarta.activation:jakarta.activation-api:1.2.1 -javax.annotation:javax.annotation-api:1.3.2 -javax.servlet.jsp:jsp-api:2.1 -javax.servlet:javax.servlet-api:3.1.0 -javax.xml.bind:jaxb-api:2.2.2 -javax.xml.stream:stax-api:1.0-2 -net.java.dev.jna:jna-platform:5.17.0 -net.java.dev.jna:jna:5.17.0 -org.apache.arrow:arrow-format:17.0.0 -org.apache.arrow:arrow-memory-core:17.0.0 -org.apache.arrow:arrow-memory-netty-buffer-patch:17.0.0 -org.apache.arrow:arrow-memory-netty:17.0.0 -org.apache.arrow:arrow-vector:17.0.0 -org.apache.avro:avro:1.12.1 -org.apache.commons:commons-collections4:4.4 -org.apache.commons:commons-compress:1.28.0 -org.apache.commons:commons-lang3:3.18.0 -org.apache.commons:commons-math3:3.6.1 -org.apache.commons:commons-text:1.14.0 -org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.5.0 -org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_25:1.5.0 -org.apache.hadoop:hadoop-annotations:3.4.3 -org.apache.hadoop:hadoop-common:3.4.3 -org.apache.httpcomponents.client5:httpclient5:5.6 -org.apache.httpcomponents.core5:httpcore5-h2:5.4 -org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.httpcomponents:httpclient:4.5.14 -org.apache.httpcomponents:httpcore:4.4.16 -org.apache.orc:orc-core:1.9.8 -org.apache.orc:orc-shims:1.9.8 -org.apache.parquet:parquet-avro:1.17.0 -org.apache.parquet:parquet-column:1.17.0 -org.apache.parquet:parquet-common:1.17.0 -org.apache.parquet:parquet-encoding:1.17.0 -org.apache.parquet:parquet-format-structures:1.17.0 -org.apache.parquet:parquet-hadoop:1.17.0 -org.apache.parquet:parquet-jackson:1.17.0 -org.apache.parquet:parquet-variant:1.17.0 -org.bouncycastle:bcprov-jdk18on:1.82 -org.checkerframework:checker-compat-qual:2.5.6 -org.checkerframework:checker-qual:3.49.0 -org.codehaus.jettison:jettison:1.5.5 -org.codehaus.mojo:animal-sniffer-annotations:1.24 -org.codehaus.woodstox:stax2-api:4.2.2 -org.conscrypt:conscrypt-openjdk-uber:2.5.2 -org.json:json:20250517 -org.locationtech.jts:jts-core:1.20.0 -org.mongodb:bson:4.11.5 -org.reactivestreams:reactive-streams:1.0.4 -org.roaringbitmap:RoaringBitmap:1.6.13 -org.slf4j:slf4j-api:2.0.17 -org.threeten:threeten-extra:1.8.0 -org.threeten:threetenbp:1.7.0 -org.xerial.snappy:snappy-java:1.1.10.8 -software.amazon.awssdk.crt:aws-crt:0.43.5 -software.amazon.awssdk:annotations:2.42.18 -software.amazon.awssdk:apache-client:2.42.18 -software.amazon.awssdk:arns:2.42.18 -software.amazon.awssdk:auth:2.42.18 -software.amazon.awssdk:aws-core:2.42.18 -software.amazon.awssdk:aws-json-protocol:2.42.18 -software.amazon.awssdk:aws-query-protocol:2.42.18 -software.amazon.awssdk:aws-xml-protocol:2.42.18 -software.amazon.awssdk:checksums-spi:2.42.18 -software.amazon.awssdk:checksums:2.42.18 -software.amazon.awssdk:crt-core:2.42.18 -software.amazon.awssdk:dynamodb:2.42.18 -software.amazon.awssdk:endpoints-spi:2.42.18 -software.amazon.awssdk:glue:2.42.18 -software.amazon.awssdk:http-auth-aws-crt:2.42.18 -software.amazon.awssdk:http-auth-aws-eventstream:2.42.18 -software.amazon.awssdk:http-auth-aws:2.42.18 -software.amazon.awssdk:http-auth-spi:2.42.18 -software.amazon.awssdk:http-auth:2.42.18 -software.amazon.awssdk:http-client-spi:2.42.18 -software.amazon.awssdk:iam:2.42.18 -software.amazon.awssdk:identity-spi:2.42.18 -software.amazon.awssdk:json-utils:2.42.18 -software.amazon.awssdk:kms:2.42.18 -software.amazon.awssdk:lakeformation:2.42.18 -software.amazon.awssdk:metrics-spi:2.42.18 -software.amazon.awssdk:netty-nio-client:2.42.18 -software.amazon.awssdk:profiles:2.42.18 -software.amazon.awssdk:protocol-core:2.42.18 -software.amazon.awssdk:regions:2.42.18 -software.amazon.awssdk:retries-spi:2.42.18 -software.amazon.awssdk:retries:2.42.18 -software.amazon.awssdk:s3:2.42.18 -software.amazon.awssdk:sdk-core:2.42.18 -software.amazon.awssdk:sso:2.42.18 -software.amazon.awssdk:sts:2.42.18 -software.amazon.awssdk:third-party-jackson-core:2.42.18 -software.amazon.awssdk:utils-lite:2.42.18 -software.amazon.awssdk:utils:2.42.18 -software.amazon.eventstream:eventstream:1.0.1 diff --git a/gradle/runtime-deps.gradle b/runtime-deps.gradle similarity index 78% rename from gradle/runtime-deps.gradle rename to runtime-deps.gradle index 791d4f6213c6..e08fce3fe47a 100644 --- a/gradle/runtime-deps.gradle +++ b/runtime-deps.gradle @@ -36,7 +36,10 @@ // // checkRuntimeDeps - compares the resolved dependencies against the // checked-in baseline and fails with a diff if -// they don't match. Wired into the 'check' lifecycle. +// they don't match. Patch-level version changes are +// ignored so that routine Dependabot bumps don't +// require a baseline update. Wired into the 'check' +// lifecycle. // // Workflow: // 1. ./gradlew check -- fails if deps changed @@ -74,24 +77,32 @@ tasks.register('checkRuntimeDeps') { outputs.file(depsFile) doLast { if (!depsFile.exists()) { - throw new GradleException( - "Missing ${depsFile.name} in ${projectDir}.\n" + + logger.warn("WARNING: Missing ${depsFile.name} in ${projectDir}. " + "Run: ./gradlew ${project.path}:generateRuntimeDeps") + return } def actual = resolveRuntimeDeps() def expected = depsFile.readLines().findAll { it.trim() }.toSorted() - if (actual != expected) { - def groupArtifact = { coord -> coord.substring(0, coord.lastIndexOf(':')) } - def actualByModule = actual.collectEntries { [(groupArtifact(it)): it] } - def expectedByModule = expected.collectEntries { [(groupArtifact(it)): it] } + def groupArtifact = { coord -> coord.substring(0, coord.lastIndexOf(':')) } + def majorMinor = { coord -> + def ver = coord.substring(coord.lastIndexOf(':') + 1) + def parts = ver.split('\\.') + parts.length >= 2 ? "${parts[0]}.${parts[1]}" : ver + } + + def actualByModule = actual.collectEntries { [(groupArtifact(it)): it] } + def expectedByModule = expected.collectEntries { [(groupArtifact(it)): it] } - def added = actualByModule.keySet() - expectedByModule.keySet() - def removed = expectedByModule.keySet() - actualByModule.keySet() - def shared = actualByModule.keySet().intersect(expectedByModule.keySet()) - def versionChanged = shared.findAll { actualByModule[it] != expectedByModule[it] } + def added = actualByModule.keySet() - expectedByModule.keySet() + def removed = expectedByModule.keySet() - actualByModule.keySet() + def shared = actualByModule.keySet().intersect(expectedByModule.keySet()) + def versionChanged = shared.findAll { + majorMinor(actualByModule[it]) != majorMinor(expectedByModule[it]) + } + if (added || removed || versionChanged) { def msg = new StringBuilder() msg.append("Runtime dependency baseline mismatch for ${project.name}!\n") if (versionChanged) { diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle index df7b9f36bd58..bb8270e3d303 100644 --- a/spark/v3.4/build.gradle +++ b/spark/v3.4/build.gradle @@ -334,6 +334,6 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } - apply from: "${rootDir}/gradle/runtime-deps.gradle" + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/spark/v3.4/spark-runtime/runtime-deps.txt b/spark/v3.4/spark-runtime/runtime-deps.txt deleted file mode 100644 index b9429f8144fe..000000000000 --- a/spark/v3.4/spark-runtime/runtime-deps.txt +++ /dev/null @@ -1,52 +0,0 @@ -com.aliyun:credentials-java:0.3.12 -com.aliyun:tea:1.4.1 -com.fasterxml.jackson.core:jackson-annotations:2.21 -com.fasterxml.jackson.core:jackson-core:2.14.2 -com.fasterxml.jackson.core:jackson-databind:2.14.2 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 -com.github.ben-manes.caffeine:caffeine:2.9.3 -com.google.code.gson:gson:2.11.0 -com.google.errorprone:error_prone_annotations:2.27.0 -com.google.flatbuffers:flatbuffers-java:23.5.26 -com.squareup.okhttp3:okhttp:4.12.0 -com.squareup.okio:okio-jvm:3.6.0 -com.sun.xml.bind:jaxb-core:2.3.0 -com.sun.xml.bind:jaxb-impl:2.3.0 -dev.failsafe:failsafe:3.3.2 -io.airlift:aircompressor:2.0.3 -io.netty:netty-buffer:4.2.10.Final -io.netty:netty-common:4.2.10.Final -org.apache.arrow:arrow-format:15.0.2 -org.apache.arrow:arrow-memory-core:15.0.2 -org.apache.arrow:arrow-memory-netty:15.0.2 -org.apache.arrow:arrow-vector:15.0.2 -org.apache.avro:avro:1.12.1 -org.apache.datasketches:datasketches-java:6.2.0 -org.apache.datasketches:datasketches-memory:3.0.2 -org.apache.httpcomponents.client5:httpclient5:5.6 -org.apache.httpcomponents.core5:httpcore5-h2:5.4 -org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.orc:orc-core:1.9.8 -org.apache.orc:orc-shims:1.9.8 -org.apache.parquet:parquet-avro:1.17.0 -org.apache.parquet:parquet-column:1.17.0 -org.apache.parquet:parquet-common:1.17.0 -org.apache.parquet:parquet-encoding:1.17.0 -org.apache.parquet:parquet-format-structures:1.17.0 -org.apache.parquet:parquet-hadoop:1.17.0 -org.apache.parquet:parquet-jackson:1.17.0 -org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-qual:3.19.0 -org.eclipse.collections:eclipse-collections-api:11.1.0 -org.eclipse.collections:eclipse-collections:11.1.0 -org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 -org.jacoco:org.jacoco.agent:0.8.8 -org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.locationtech.jts:jts-core:1.20.0 -org.projectnessie.nessie:nessie-client:0.107.4 -org.projectnessie.nessie:nessie-model:0.107.4 -org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.7.1 diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle index 66f6d68f1c25..18fca51be251 100644 --- a/spark/v3.5/build.gradle +++ b/spark/v3.5/build.gradle @@ -336,6 +336,6 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } - apply from: "${rootDir}/gradle/runtime-deps.gradle" + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/spark/v3.5/spark-runtime/runtime-deps.txt b/spark/v3.5/spark-runtime/runtime-deps.txt deleted file mode 100644 index a662942f9cd1..000000000000 --- a/spark/v3.5/spark-runtime/runtime-deps.txt +++ /dev/null @@ -1,52 +0,0 @@ -com.aliyun:credentials-java:0.3.12 -com.aliyun:tea:1.4.1 -com.fasterxml.jackson.core:jackson-annotations:2.21 -com.fasterxml.jackson.core:jackson-core:2.15.2 -com.fasterxml.jackson.core:jackson-databind:2.15.2 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 -com.github.ben-manes.caffeine:caffeine:2.9.3 -com.google.code.gson:gson:2.11.0 -com.google.errorprone:error_prone_annotations:2.27.0 -com.google.flatbuffers:flatbuffers-java:23.5.26 -com.squareup.okhttp3:okhttp:4.12.0 -com.squareup.okio:okio-jvm:3.6.0 -com.sun.xml.bind:jaxb-core:2.3.0 -com.sun.xml.bind:jaxb-impl:2.3.0 -dev.failsafe:failsafe:3.3.2 -io.airlift:aircompressor:2.0.3 -io.netty:netty-buffer:4.2.10.Final -io.netty:netty-common:4.2.10.Final -org.apache.arrow:arrow-format:15.0.2 -org.apache.arrow:arrow-memory-core:15.0.2 -org.apache.arrow:arrow-memory-netty:15.0.2 -org.apache.arrow:arrow-vector:15.0.2 -org.apache.avro:avro:1.12.1 -org.apache.datasketches:datasketches-java:6.2.0 -org.apache.datasketches:datasketches-memory:3.0.2 -org.apache.httpcomponents.client5:httpclient5:5.6 -org.apache.httpcomponents.core5:httpcore5-h2:5.4 -org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.orc:orc-core:1.9.8 -org.apache.orc:orc-shims:1.9.8 -org.apache.parquet:parquet-avro:1.17.0 -org.apache.parquet:parquet-column:1.17.0 -org.apache.parquet:parquet-common:1.17.0 -org.apache.parquet:parquet-encoding:1.17.0 -org.apache.parquet:parquet-format-structures:1.17.0 -org.apache.parquet:parquet-hadoop:1.17.0 -org.apache.parquet:parquet-jackson:1.17.0 -org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-qual:3.19.0 -org.eclipse.collections:eclipse-collections-api:11.1.0 -org.eclipse.collections:eclipse-collections:11.1.0 -org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 -org.jacoco:org.jacoco.agent:0.8.8 -org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.locationtech.jts:jts-core:1.20.0 -org.projectnessie.nessie:nessie-client:0.107.4 -org.projectnessie.nessie:nessie-model:0.107.4 -org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.7.1 diff --git a/spark/v4.0/build.gradle b/spark/v4.0/build.gradle index b92e5c26c8e2..62111e104e26 100644 --- a/spark/v4.0/build.gradle +++ b/spark/v4.0/build.gradle @@ -336,6 +336,6 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } - apply from: "${rootDir}/gradle/runtime-deps.gradle" + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/spark/v4.0/spark-runtime/runtime-deps.txt b/spark/v4.0/spark-runtime/runtime-deps.txt deleted file mode 100644 index a662942f9cd1..000000000000 --- a/spark/v4.0/spark-runtime/runtime-deps.txt +++ /dev/null @@ -1,52 +0,0 @@ -com.aliyun:credentials-java:0.3.12 -com.aliyun:tea:1.4.1 -com.fasterxml.jackson.core:jackson-annotations:2.21 -com.fasterxml.jackson.core:jackson-core:2.15.2 -com.fasterxml.jackson.core:jackson-databind:2.15.2 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 -com.github.ben-manes.caffeine:caffeine:2.9.3 -com.google.code.gson:gson:2.11.0 -com.google.errorprone:error_prone_annotations:2.27.0 -com.google.flatbuffers:flatbuffers-java:23.5.26 -com.squareup.okhttp3:okhttp:4.12.0 -com.squareup.okio:okio-jvm:3.6.0 -com.sun.xml.bind:jaxb-core:2.3.0 -com.sun.xml.bind:jaxb-impl:2.3.0 -dev.failsafe:failsafe:3.3.2 -io.airlift:aircompressor:2.0.3 -io.netty:netty-buffer:4.2.10.Final -io.netty:netty-common:4.2.10.Final -org.apache.arrow:arrow-format:15.0.2 -org.apache.arrow:arrow-memory-core:15.0.2 -org.apache.arrow:arrow-memory-netty:15.0.2 -org.apache.arrow:arrow-vector:15.0.2 -org.apache.avro:avro:1.12.1 -org.apache.datasketches:datasketches-java:6.2.0 -org.apache.datasketches:datasketches-memory:3.0.2 -org.apache.httpcomponents.client5:httpclient5:5.6 -org.apache.httpcomponents.core5:httpcore5-h2:5.4 -org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.orc:orc-core:1.9.8 -org.apache.orc:orc-shims:1.9.8 -org.apache.parquet:parquet-avro:1.17.0 -org.apache.parquet:parquet-column:1.17.0 -org.apache.parquet:parquet-common:1.17.0 -org.apache.parquet:parquet-encoding:1.17.0 -org.apache.parquet:parquet-format-structures:1.17.0 -org.apache.parquet:parquet-hadoop:1.17.0 -org.apache.parquet:parquet-jackson:1.17.0 -org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-qual:3.19.0 -org.eclipse.collections:eclipse-collections-api:11.1.0 -org.eclipse.collections:eclipse-collections:11.1.0 -org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 -org.jacoco:org.jacoco.agent:0.8.8 -org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.locationtech.jts:jts-core:1.20.0 -org.projectnessie.nessie:nessie-client:0.107.4 -org.projectnessie.nessie:nessie-model:0.107.4 -org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.7.1 diff --git a/spark/v4.1/build.gradle b/spark/v4.1/build.gradle index 25d3fa96df0e..355a85ab81a9 100644 --- a/spark/v4.1/build.gradle +++ b/spark/v4.1/build.gradle @@ -336,6 +336,6 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } - apply from: "${rootDir}/gradle/runtime-deps.gradle" + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/spark/v4.1/spark-runtime/runtime-deps.txt b/spark/v4.1/spark-runtime/runtime-deps.txt deleted file mode 100644 index a662942f9cd1..000000000000 --- a/spark/v4.1/spark-runtime/runtime-deps.txt +++ /dev/null @@ -1,52 +0,0 @@ -com.aliyun:credentials-java:0.3.12 -com.aliyun:tea:1.4.1 -com.fasterxml.jackson.core:jackson-annotations:2.21 -com.fasterxml.jackson.core:jackson-core:2.15.2 -com.fasterxml.jackson.core:jackson-databind:2.15.2 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.21.2 -com.github.ben-manes.caffeine:caffeine:2.9.3 -com.google.code.gson:gson:2.11.0 -com.google.errorprone:error_prone_annotations:2.27.0 -com.google.flatbuffers:flatbuffers-java:23.5.26 -com.squareup.okhttp3:okhttp:4.12.0 -com.squareup.okio:okio-jvm:3.6.0 -com.sun.xml.bind:jaxb-core:2.3.0 -com.sun.xml.bind:jaxb-impl:2.3.0 -dev.failsafe:failsafe:3.3.2 -io.airlift:aircompressor:2.0.3 -io.netty:netty-buffer:4.2.10.Final -io.netty:netty-common:4.2.10.Final -org.apache.arrow:arrow-format:15.0.2 -org.apache.arrow:arrow-memory-core:15.0.2 -org.apache.arrow:arrow-memory-netty:15.0.2 -org.apache.arrow:arrow-vector:15.0.2 -org.apache.avro:avro:1.12.1 -org.apache.datasketches:datasketches-java:6.2.0 -org.apache.datasketches:datasketches-memory:3.0.2 -org.apache.httpcomponents.client5:httpclient5:5.6 -org.apache.httpcomponents.core5:httpcore5-h2:5.4 -org.apache.httpcomponents.core5:httpcore5:5.4 -org.apache.orc:orc-core:1.9.8 -org.apache.orc:orc-shims:1.9.8 -org.apache.parquet:parquet-avro:1.17.0 -org.apache.parquet:parquet-column:1.17.0 -org.apache.parquet:parquet-common:1.17.0 -org.apache.parquet:parquet-encoding:1.17.0 -org.apache.parquet:parquet-format-structures:1.17.0 -org.apache.parquet:parquet-hadoop:1.17.0 -org.apache.parquet:parquet-jackson:1.17.0 -org.apache.parquet:parquet-variant:1.17.0 -org.checkerframework:checker-qual:3.19.0 -org.eclipse.collections:eclipse-collections-api:11.1.0 -org.eclipse.collections:eclipse-collections:11.1.0 -org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1.1 -org.jacoco:org.jacoco.agent:0.8.8 -org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 -org.jetbrains.kotlin:kotlin-stdlib:1.9.10 -org.locationtech.jts:jts-core:1.20.0 -org.projectnessie.nessie:nessie-client:0.107.4 -org.projectnessie.nessie:nessie-model:0.107.4 -org.roaringbitmap:RoaringBitmap:1.6.13 -org.threeten:threeten-extra:1.7.1 From 956c635783a0a43124816ba2f0dcc6044ef08469 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 1 Apr 2026 16:45:36 -0500 Subject: [PATCH 7/7] CI: Suppress compilation noise in check-runtime-deps job Add -q flag to the Gradle invocation so that only build failures (the dependency mismatch diff) are printed. Compilation warnings from upstream project dependencies are not relevant to this check. --- .github/workflows/java-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index bc6cb23b6f90..e8ac497ab04a 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -138,4 +138,4 @@ jobs: distribution: zulu java-version: 17 - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5 # zizmor: ignore[cache-poisoning] -- cache writes are restricted to the default branch by setup-gradle - - run: ./gradlew checkAllRuntimeDeps + - run: ./gradlew checkAllRuntimeDeps -q