From b2f114daa2bfb2e05d37a0745cb50b0ddde5a705 Mon Sep 17 00:00:00 2001 From: William Hyun Date: Sat, 26 Jul 2025 15:56:33 -0700 Subject: [PATCH 1/2] [SPARK-52967][BUILD] Upgrade ORC to 2.2.0 --- LICENSE-binary | 2 +- dev/deps/spark-deps-hadoop-3-hive-2.3 | 8 ++++---- pom.xml | 6 +++++- project/SparkBuild.scala | 1 + 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 9483bd3723c7c..61a6a830dad3e 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -386,7 +386,6 @@ org.glassfish.jersey.core:jersey-common org.glassfish.jersey.core:jersey-server org.glassfish.jersey.inject:jersey-hk2 org.javassist:javassist -org.jetbrains:annotations org.json4s:json4s-ast_2.13 org.json4s:json4s-core_2.13 org.json4s:json4s-jackson-core_2.13 @@ -462,6 +461,7 @@ org.codehaus.janino:janino org.fusesource.leveldbjni:leveldbjni-all org.jline:jline org.jpmml:pmml-model +org.locationtech.jts:jts-core org.threeten:threeten-extra python/lib/py4j-*-src.zip diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 85ba4f6253398..e229bf450c5e0 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -9,7 +9,6 @@ aliyun-java-sdk-core/4.5.10//aliyun-java-sdk-core-4.5.10.jar aliyun-java-sdk-kms/2.11.0//aliyun-java-sdk-kms-2.11.0.jar aliyun-java-sdk-ram/3.1.0//aliyun-java-sdk-ram-3.1.0.jar aliyun-sdk-oss/3.13.2//aliyun-sdk-oss-3.13.2.jar -annotations/17.0.0//annotations-17.0.0.jar antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar antlr4-runtime/4.13.1//antlr4-runtime-4.13.1.jar aopalliance-repackaged/3.0.6//aopalliance-repackaged-3.0.6.jar @@ -154,6 +153,7 @@ json4s-jackson_2.13/4.0.7//json4s-jackson_2.13-4.0.7.jar json4s-scalap_2.13/4.0.7//json4s-scalap_2.13-4.0.7.jar jsr305/3.0.0//jsr305-3.0.0.jar jta/1.1//jta-1.1.jar +jts-core/1.20.0//jts-core-1.20.0.jar jul-to-slf4j/2.0.17//jul-to-slf4j-2.0.17.jar kryo-shaded/4.0.3//kryo-shaded-4.0.3.jar kubernetes-client-api/7.3.1//kubernetes-client-api-7.3.1.jar @@ -232,10 +232,10 @@ opencsv/2.3//opencsv-2.3.jar opentracing-api/0.33.0//opentracing-api-0.33.0.jar opentracing-noop/0.33.0//opentracing-noop-0.33.0.jar opentracing-util/0.33.0//opentracing-util-0.33.0.jar -orc-core/2.1.3/shaded-protobuf/orc-core-2.1.3-shaded-protobuf.jar +orc-core/2.2.0/shaded-protobuf/orc-core-2.2.0-shaded-protobuf.jar orc-format/1.1.0/shaded-protobuf/orc-format-1.1.0-shaded-protobuf.jar -orc-mapreduce/2.1.3/shaded-protobuf/orc-mapreduce-2.1.3-shaded-protobuf.jar -orc-shims/2.1.3//orc-shims-2.1.3.jar +orc-mapreduce/2.2.0/shaded-protobuf/orc-mapreduce-2.2.0-shaded-protobuf.jar +orc-shims/2.2.0//orc-shims-2.2.0.jar oro/2.0.8//oro-2.0.8.jar osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar paranamer/2.8.3//paranamer-2.8.3.jar diff --git a/pom.xml b/pom.xml index c12739202c846..bdb865b0480a4 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ 10.16.1.1 1.15.2 - 2.1.3 + 2.2.0 shaded-protobuf 11.0.25 5.0.0 @@ -379,6 +379,10 @@ false + + orc + https://repository.apache.org/content/repositories/orgapacheorc-1102/ + diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 3ef0e174fe787..1d82f0b4edeea 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -305,6 +305,7 @@ object SparkBuild extends PomBuild { "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/", DefaultMavenRepository, Resolver.mavenLocal, + "orc" at "https://repository.apache.org/content/repositories/orgapacheorc-1102/", Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ), externalResolvers := resolvers.value, From d5889e3d42b8f6d27e6f16ff92fbe89e64b435d1 Mon Sep 17 00:00:00 2001 From: William Hyun Date: Tue, 29 Jul 2025 23:39:49 -0700 Subject: [PATCH 2/2] Use Official ORC 2.2.0 Release --- pom.xml | 4 ---- project/SparkBuild.scala | 1 - 2 files changed, 5 deletions(-) diff --git a/pom.xml b/pom.xml index 6d5ca307643f8..2ecf89af8d07b 100644 --- a/pom.xml +++ b/pom.xml @@ -379,10 +379,6 @@ false - - orc - https://repository.apache.org/content/repositories/orgapacheorc-1102/ - diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 1d82f0b4edeea..3ef0e174fe787 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -305,7 +305,6 @@ object SparkBuild extends PomBuild { "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/", DefaultMavenRepository, Resolver.mavenLocal, - "orc" at "https://repository.apache.org/content/repositories/orgapacheorc-1102/", Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ), externalResolvers := resolvers.value,