diff --git a/ci/docker/conda-python-spark.dockerfile b/ci/docker/conda-python-spark.dockerfile index d3f0a224582..a20f1ff3521 100644 --- a/ci/docker/conda-python-spark.dockerfile +++ b/ci/docker/conda-python-spark.dockerfile @@ -36,6 +36,10 @@ ARG spark=master COPY ci/scripts/install_spark.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_spark.sh ${spark} /spark +# patch spark to build with current Arrow Java +COPY ci/etc/integration_spark_ARROW-9438.patch /arrow/ci/etc/ +RUN patch -d /spark -p1 -i /arrow/ci/etc/integration_spark_ARROW-9438.patch + # build cpp with tests ENV CC=gcc \ CXX=g++ \ diff --git a/ci/etc/integration_spark_ARROW-9438.patch b/ci/etc/integration_spark_ARROW-9438.patch new file mode 100644 index 00000000000..2baed303717 --- /dev/null +++ b/ci/etc/integration_spark_ARROW-9438.patch @@ -0,0 +1,72 @@ +From 0b5388a945a7e5c5706cf00d0754540a6c68254d Mon Sep 17 00:00:00 2001 +From: Bryan Cutler +Date: Mon, 13 Jul 2020 23:12:25 -0700 +Subject: [PATCH] Update Arrow Java for 1.0.0 + +--- + pom.xml | 17 ++++++++++++++--- + sql/catalyst/pom.xml | 4 ++++ + 2 files changed, 18 insertions(+), 3 deletions(-) + +diff --git a/pom.xml b/pom.xml +index 08ca13bfe9..6619fca200 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -199,7 +199,7 @@ + If you are changing Arrow version specification, please check ./python/pyspark/sql/utils.py, + and ./python/setup.py too. + --> +- 0.15.1 ++ 1.0.0-SNAPSHOT + + org.fusesource.leveldbjni + +@@ -2288,7 +2288,7 @@ + + + com.fasterxml.jackson.core +- jackson-databind ++ jackson-core + + + io.netty +@@ -2298,9 +2298,20 @@ + io.netty + netty-common + ++ ++ ++ ++ org.apache.arrow ++ arrow-memory-netty ++ ${arrow.version} ++ + + io.netty +- netty-handler ++ netty-buffer ++ ++ ++ io.netty ++ netty-common + + + +diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml +index 9edbb7fec9..6b79eb722f 100644 +--- a/sql/catalyst/pom.xml ++++ b/sql/catalyst/pom.xml +@@ -117,6 +117,10 @@ + org.apache.arrow + arrow-vector + ++ ++ org.apache.arrow ++ arrow-memory-netty ++ + + + target/scala-${scala.binary.version}/classes +-- +2.17.1 + diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index d25e2e3eff9..158790d3320 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -9,6 +9,7 @@ *.snap .github/ISSUE_TEMPLATE/question.md ci/etc/rprofile +ci/etc/*.patch cpp/CHANGELOG_PARQUET.md cpp/src/arrow/io/mman.h cpp/src/arrow/util/random.h