From 910520efbd656ec960e1c4ec228b928fcee80be9 Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 27 Jul 2016 13:53:46 +0100 Subject: [PATCH 01/12] [SPARK-16751] Upgrade Derby, remove from package ## What changes were proposed in this pull request? Version of derby upgraded based on important security info at VersionEye. Test scope added so we don't include it in our final package anyway. NB: I think this should be backported to all previous releases as it is a security problem https://www.versioneye.com/java/org.apache.derby:derby/10.11.1.1 The CVE number is 2015-1832. I also suggest we add a SECURITY tag for JIRAs ## How was this patch tested? Existing tests with the change making sure that we see no new failures. I checked derby 10.12.x and not derby 10.11.x is downloaded to our ~/.m2 folder. I then used dev/make-distribution.sh and checked the dist/jars folder for Spark 2.0: no derby jar is present. I don't know if this would also remove it from the assembly jar in our 1.x branches. --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b69292d188090..f9fed59dcd8b1 100644 --- a/pom.xml +++ b/pom.xml @@ -134,7 +134,7 @@ 1.2.1.spark2 1.2.1 - 10.11.1.1 + 10.12.1.1 1.8.1 1.6.0 9.2.16.v20160414 @@ -565,6 +565,7 @@ org.apache.derby derby ${derby.version} + test io.dropwizard.metrics From 3c8ad8f657122f37553c63ca8425a23eaa49839d Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 27 Jul 2016 14:29:08 +0100 Subject: [PATCH 02/12] Remove derby dependency for Hadoop 2.2 profile --- dev/deps/spark-deps-hadoop-2.2 | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2 index ff1587314030e..d7a690bc50fb4 100644 --- a/dev/deps/spark-deps-hadoop-2.2 +++ b/dev/deps/spark-deps-hadoop-2.2 @@ -46,7 +46,6 @@ curator-recipes-2.4.0.jar datanucleus-api-jdo-3.2.6.jar datanucleus-core-3.2.10.jar datanucleus-rdbms-3.2.9.jar -derby-10.11.1.1.jar eigenbase-properties-1.1.5.jar guava-14.0.1.jar guice-3.0.jar From 8a14014288559314eaba7a36e0d7ea3fc6d0eb40 Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 27 Jul 2016 14:29:31 +0100 Subject: [PATCH 03/12] Remove derby dependency for Hadoop 2.3 profile --- dev/deps/spark-deps-hadoop-2.3 | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3 index 2b5764f868690..cdf8d2ac8e6ce 100644 --- a/dev/deps/spark-deps-hadoop-2.3 +++ b/dev/deps/spark-deps-hadoop-2.3 @@ -48,7 +48,6 @@ curator-recipes-2.4.0.jar datanucleus-api-jdo-3.2.6.jar datanucleus-core-3.2.10.jar datanucleus-rdbms-3.2.9.jar -derby-10.11.1.1.jar eigenbase-properties-1.1.5.jar guava-14.0.1.jar guice-3.0.jar From dd72a1b1bdbb9163ac226174da61b3bb5ee3f62a Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 27 Jul 2016 14:29:50 +0100 Subject: [PATCH 04/12] Remove derby dependency for Hadoop 2.4 profile --- dev/deps/spark-deps-hadoop-2.4 | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4 index 3f53fdb09c64a..f22eaafe11dfc 100644 --- a/dev/deps/spark-deps-hadoop-2.4 +++ b/dev/deps/spark-deps-hadoop-2.4 @@ -48,7 +48,6 @@ curator-recipes-2.4.0.jar datanucleus-api-jdo-3.2.6.jar datanucleus-core-3.2.10.jar datanucleus-rdbms-3.2.9.jar -derby-10.11.1.1.jar eigenbase-properties-1.1.5.jar guava-14.0.1.jar guice-3.0.jar From 404209df71f8e2880097c4cc2f025187694c9d44 Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 27 Jul 2016 14:30:09 +0100 Subject: [PATCH 05/12] Remove derby dependency for Hadoop 2.6 profile --- dev/deps/spark-deps-hadoop-2.6 | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index d3a7ab8bb457d..1222bcaa6878c 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -52,7 +52,6 @@ curator-recipes-2.6.0.jar datanucleus-api-jdo-3.2.6.jar datanucleus-core-3.2.10.jar datanucleus-rdbms-3.2.9.jar -derby-10.11.1.1.jar eigenbase-properties-1.1.5.jar gson-2.2.4.jar guava-14.0.1.jar From 80168406f7d974a9caf0b108ef304f888b308cde Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 27 Jul 2016 14:30:25 +0100 Subject: [PATCH 06/12] Remove derby dependency for Hadoop 2.7 profile --- dev/deps/spark-deps-hadoop-2.7 | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 05317a044d65e..bd8392ffda4ba 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -52,7 +52,6 @@ curator-recipes-2.6.0.jar datanucleus-api-jdo-3.2.6.jar datanucleus-core-3.2.10.jar datanucleus-rdbms-3.2.9.jar -derby-10.11.1.1.jar eigenbase-properties-1.1.5.jar gson-2.2.4.jar guava-14.0.1.jar From 3233b15a7fee7f0d6031315c7aafa8e06c849557 Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 27 Jul 2016 22:26:01 +0100 Subject: [PATCH 07/12] Revert test scope change for derby We actually do want to include it in the Spark distribution --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index f9fed59dcd8b1..9b7be371bb134 100644 --- a/pom.xml +++ b/pom.xml @@ -565,7 +565,6 @@ org.apache.derby derby ${derby.version} - test io.dropwizard.metrics From 0d7deade799ce0cf00905961bd6360e12966ceae Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 27 Jul 2016 22:43:12 +0100 Subject: [PATCH 08/12] Add updated derby (10.12.1.1) in 2.2 deps file --- dev/deps/spark-deps-hadoop-2.2 | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2 index d7a690bc50fb4..9350b9df50c0d 100644 --- a/dev/deps/spark-deps-hadoop-2.2 +++ b/dev/deps/spark-deps-hadoop-2.2 @@ -46,6 +46,7 @@ curator-recipes-2.4.0.jar datanucleus-api-jdo-3.2.6.jar datanucleus-core-3.2.10.jar datanucleus-rdbms-3.2.9.jar +derby-10.12.1.1.jar eigenbase-properties-1.1.5.jar guava-14.0.1.jar guice-3.0.jar From 67d5c2806977cf0c0c0054b28f062bea38954d9b Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 27 Jul 2016 22:43:59 +0100 Subject: [PATCH 09/12] Add updated derby (10.12.1.1) in 2.3 deps file --- dev/deps/spark-deps-hadoop-2.3 | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3 index cdf8d2ac8e6ce..2e1a6a3dc60cc 100644 --- a/dev/deps/spark-deps-hadoop-2.3 +++ b/dev/deps/spark-deps-hadoop-2.3 @@ -48,6 +48,7 @@ curator-recipes-2.4.0.jar datanucleus-api-jdo-3.2.6.jar datanucleus-core-3.2.10.jar datanucleus-rdbms-3.2.9.jar +derby-10.12.1.1.jar eigenbase-properties-1.1.5.jar guava-14.0.1.jar guice-3.0.jar From 97655595256faaeeee58a3e482155984248fbbae Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 27 Jul 2016 22:44:38 +0100 Subject: [PATCH 10/12] Add updated derby (10.12.1.1) in 2.4 deps file --- dev/deps/spark-deps-hadoop-2.4 | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4 index f22eaafe11dfc..9baf87e5329ff 100644 --- a/dev/deps/spark-deps-hadoop-2.4 +++ b/dev/deps/spark-deps-hadoop-2.4 @@ -48,6 +48,7 @@ curator-recipes-2.4.0.jar datanucleus-api-jdo-3.2.6.jar datanucleus-core-3.2.10.jar datanucleus-rdbms-3.2.9.jar +derby-10.12.1.1.jar eigenbase-properties-1.1.5.jar guava-14.0.1.jar guice-3.0.jar From f8573fde32e70451604143e6430c6c56450da043 Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 27 Jul 2016 22:45:11 +0100 Subject: [PATCH 11/12] Add updated derby (10.12.1.1) in 2.6 deps file --- dev/deps/spark-deps-hadoop-2.6 | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 1222bcaa6878c..9112452b5cb50 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -52,6 +52,7 @@ curator-recipes-2.6.0.jar datanucleus-api-jdo-3.2.6.jar datanucleus-core-3.2.10.jar datanucleus-rdbms-3.2.9.jar +derby-10.12.1.1.jar eigenbase-properties-1.1.5.jar gson-2.2.4.jar guava-14.0.1.jar From f3815cfd1b6b6d7ba29dc8f2e12b1334ad415847 Mon Sep 17 00:00:00 2001 From: Adam Roberts Date: Wed, 27 Jul 2016 22:45:48 +0100 Subject: [PATCH 12/12] Add updated derby (10.12.1.1) in 2.7 deps file --- dev/deps/spark-deps-hadoop-2.7 | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index bd8392ffda4ba..b0e3e9304b198 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -52,6 +52,7 @@ curator-recipes-2.6.0.jar datanucleus-api-jdo-3.2.6.jar datanucleus-core-3.2.10.jar datanucleus-rdbms-3.2.9.jar +derby-10.12.1.1.jar eigenbase-properties-1.1.5.jar gson-2.2.4.jar guava-14.0.1.jar