From cd013929055fba47e68b2058d0370a49ee30a172 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 27 May 2019 14:32:00 +0200 Subject: [PATCH 1/2] Bump Apache Avro to 1.9.0 Apache Avro 1.9.0 brings a lot of new features: * Deprecate Joda-Time in favor of Java8 JSR310 and setting it as default * Remove support for Hadoop 1.x * Move from Jackson 1.x to 2.9 * Add ZStandard Codec * Lots of updates on the dependencies to fix CVE's * Remove Jackson classes from public API * Apache Avro is built by default with Java 8 * Apache Avro is compiled and tested with Java 11 to guarantee compatibility * Apache Avro MapReduce is compiled and tested with Hadoop 3 * Apache Avro is now leaner, multiple dependencies were removed: guava, paranamer, commons-codec, and commons-logging * Introduce JMH Performance Testing Framework * Add Snappy support for C++ DataFile * and many, many more! --- extensions-core/avro-extensions/pom.xml | 19 +------------------ .../input/parquet/BaseParquetInputTest.java | 2 +- .../input/parquet/CompatParquetInputTest.java | 2 +- .../parquet/DecimalParquetInputTest.java | 2 +- .../parquet/FlattenSpecParquetInputTest.java | 2 +- .../parquet/TimestampsParquetInputTest.java | 2 +- .../input/parquet/WikiParquetInputTest.java | 2 +- 7 files changed, 7 insertions(+), 24 deletions(-) diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index e8393995c37e..4b69c7ce6c4e 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -37,7 +37,7 @@ 0.1.3 3.0.1 - 1.8.2 + 1.9.0 @@ -57,30 +57,13 @@ org.slf4j slf4j-api - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-mapper-asl - org.apache.avro avro-mapred - hadoop2 ${avro.version} - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-mapper-asl - org.slf4j slf4j-api diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/BaseParquetInputTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/BaseParquetInputTest.java index 253eea53e7cf..3a0da9f89e93 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/BaseParquetInputTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/BaseParquetInputTest.java @@ -19,7 +19,7 @@ package org.apache.druid.data.input.parquet; -import avro.shaded.com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap; import org.apache.directory.api.util.Strings; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.InputRowParser; diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetInputTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetInputTest.java index 5addaecf37e6..9287599ce06f 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetInputTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/CompatParquetInputTest.java @@ -19,7 +19,7 @@ package org.apache.druid.data.input.parquet; -import avro.shaded.com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.InputRow; import org.apache.druid.indexer.HadoopDruidIndexerConfig; import org.apache.druid.indexer.path.StaticPathSpec; diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/DecimalParquetInputTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/DecimalParquetInputTest.java index 2660d33bc07c..a587789fb481 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/DecimalParquetInputTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/DecimalParquetInputTest.java @@ -19,7 +19,7 @@ package org.apache.druid.data.input.parquet; -import avro.shaded.com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.InputRow; import org.apache.druid.indexer.HadoopDruidIndexerConfig; import org.apache.hadoop.conf.Configuration; diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetInputTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetInputTest.java index ee3aa6dc1121..28fa8bb265a2 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetInputTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/FlattenSpecParquetInputTest.java @@ -19,7 +19,7 @@ package org.apache.druid.data.input.parquet; -import avro.shaded.com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.InputRow; import org.apache.druid.indexer.HadoopDruidIndexerConfig; import org.apache.druid.indexer.path.StaticPathSpec; diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetInputTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetInputTest.java index e06b3817ee9b..a451da700d76 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetInputTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/TimestampsParquetInputTest.java @@ -19,7 +19,7 @@ package org.apache.druid.data.input.parquet; -import avro.shaded.com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.InputRow; import org.apache.druid.indexer.HadoopDruidIndexerConfig; import org.apache.druid.indexer.path.StaticPathSpec; diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/WikiParquetInputTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/WikiParquetInputTest.java index cc1245434f68..07ae255406fb 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/WikiParquetInputTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/WikiParquetInputTest.java @@ -19,7 +19,7 @@ package org.apache.druid.data.input.parquet; -import avro.shaded.com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.InputRow; import org.apache.druid.indexer.HadoopDruidIndexerConfig; import org.apache.druid.indexer.path.StaticPathSpec; From 32c5b8d96f73d2079822b3f8707e436f61f2cd6d Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 09:58:44 +0200 Subject: [PATCH 2/2] Add exclusions for Jackson --- extensions-core/avro-extensions/pom.xml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index 4b69c7ce6c4e..c56a5bb2c1f3 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -57,6 +57,14 @@ org.slf4j slf4j-api + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + @@ -68,6 +76,14 @@ org.slf4j slf4j-api + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind +