diff --git a/LICENSE b/LICENSE index 44feea070250..ddc008611134 100644 --- a/LICENSE +++ b/LICENSE @@ -300,3 +300,13 @@ This product includes code from Delta Lake. Copyright: 2020 The Delta Lake Project Authors. Home page: https://delta.io/ License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Apache Commons. + +* Core ArrayUtil. + +Copyright: 2020 The Apache Software Foundation +Home page: https://commons.apache.org/ +License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/build.gradle b/build.gradle index 82ed2fcda4cd..c6bf29735ce8 100644 --- a/build.gradle +++ b/build.gradle @@ -1069,8 +1069,6 @@ project(':iceberg-pig') { compile project(':iceberg-core') compile project(':iceberg-parquet') - compile "org.apache.commons:commons-lang3" - compileOnly("org.apache.pig:pig") { exclude group: "junit", module: "junit" } diff --git a/core/src/main/java/org/apache/iceberg/util/ArrayUtil.java b/core/src/main/java/org/apache/iceberg/util/ArrayUtil.java index 685f193a74c7..b968571e2055 100644 --- a/core/src/main/java/org/apache/iceberg/util/ArrayUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/ArrayUtil.java @@ -19,6 +19,7 @@ package org.apache.iceberg.util; +import java.lang.reflect.Array; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -28,6 +29,14 @@ public class ArrayUtil { private ArrayUtil() { } + public static final boolean[] EMPTY_BOOLEAN_ARRAY = new boolean[0]; + public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + public static final short[] EMPTY_SHORT_ARRAY = new short[0]; + public static final int[] EMPTY_INT_ARRAY = new int[0]; + public static final long[] EMPTY_LONG_ARRAY = new long[0]; + public static final float[] EMPTY_FLOAT_ARRAY = new float[0]; + public static final double[] EMPTY_DOUBLE_ARRAY = new double[0]; + public static List toIntList(int[] ints) { if (ints != null) { return IntStream.of(ints).boxed().collect(Collectors.toList()); @@ -59,4 +68,239 @@ public static long[] toLongArray(List longs) { return null; } } + + /** + * Converts an array of object Booleans to primitives. + *

+ * This method returns {@code null} for a {@code null} input array. + *

+ * This code is borrowed from `org.apache.commons:commons-lang3`. + * + * @param array a {@code Boolean} array, may be {@code null} + * @return a {@code boolean} array, {@code null} if null array input + * @throws NullPointerException if array content is {@code null} + */ + public static boolean[] toPrimitive(final Boolean[] array) { + if (array == null) { + return null; + } else if (array.length == 0) { + return EMPTY_BOOLEAN_ARRAY; + } + final boolean[] result = new boolean[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i].booleanValue(); + } + return result; + } + + /** + * Converts an array of object Bytes to primitives. + *

+ * This method returns {@code null} for a {@code null} input array. + *

+ * This code is borrowed from `org.apache.commons:commons-lang3`. + * + * @param array a {@code Byte} array, may be {@code null} + * @return a {@code byte} array, {@code null} if null array input + * @throws NullPointerException if array content is {@code null} + */ + public static byte[] toPrimitive(final Byte[] array) { + if (array == null) { + return null; + } else if (array.length == 0) { + return EMPTY_BYTE_ARRAY; + } + final byte[] result = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i].byteValue(); + } + return result; + } + + /** + * Converts an array of object Shorts to primitives. + *

+ * This method returns {@code null} for a {@code null} input array. + *

+ * This code is borrowed from `org.apache.commons:commons-lang3`. + * + * @param array a {@code Short} array, may be {@code null} + * @return a {@code byte} array, {@code null} if null array input + * @throws NullPointerException if array content is {@code null} + */ + public static short[] toPrimitive(final Short[] array) { + if (array == null) { + return null; + } else if (array.length == 0) { + return EMPTY_SHORT_ARRAY; + } + final short[] result = new short[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i].shortValue(); + } + return result; + } + + /** + * Converts an array of object Integers to primitives. + *

+ * This method returns {@code null} for a {@code null} input array. + *

+ * This code is borrowed from `org.apache.commons:commons-lang3`. + * + * @param array a {@code Integer} array, may be {@code null} + * @return an {@code int} array, {@code null} if null array input + * @throws NullPointerException if array content is {@code null} + */ + public static int[] toPrimitive(final Integer[] array) { + if (array == null) { + return null; + } else if (array.length == 0) { + return EMPTY_INT_ARRAY; + } + final int[] result = new int[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i].intValue(); + } + return result; + } + + /** + * Converts an array of object Longs to primitives. + *

+ * This method returns {@code null} for a {@code null} input array. + *

+ * This code is borrowed from `org.apache.commons:commons-lang3`. + * + * @param array a {@code Long} array, may be {@code null} + * @return a {@code long} array, {@code null} if null array input + * @throws NullPointerException if array content is {@code null} + */ + public static long[] toPrimitive(final Long[] array) { + if (array == null) { + return null; + } else if (array.length == 0) { + return EMPTY_LONG_ARRAY; + } + final long[] result = new long[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i].longValue(); + } + return result; + } + + /** + * Converts an array of object Floats to primitives. + *

+ * This method returns {@code null} for a {@code null} input array. + *

+ * This code is borrowed from `org.apache.commons:commons-lang3`. + * + * @param array a {@code Float} array, may be {@code null} + * @return a {@code float} array, {@code null} if null array input + * @throws NullPointerException if array content is {@code null} + */ + public static float[] toPrimitive(final Float[] array) { + if (array == null) { + return null; + } else if (array.length == 0) { + return EMPTY_FLOAT_ARRAY; + } + final float[] result = new float[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i].floatValue(); + } + return result; + } + + /** + * Converts an array of object Doubles to primitives. + *

+ * This method returns {@code null} for a {@code null} input array. + *

+ * This code is borrowed from `org.apache.commons:commons-lang3`. + * + * @param array a {@code Double} array, may be {@code null} + * @return a {@code double} array, {@code null} if null array input + * @throws NullPointerException if array content is {@code null} + */ + public static double[] toPrimitive(final Double[] array) { + if (array == null) { + return null; + } else if (array.length == 0) { + return EMPTY_DOUBLE_ARRAY; + } + final double[] result = new double[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i].doubleValue(); + } + return result; + } + + /** + * Copies the given array and adds the given element at the end of the new array. + *

+ * The new array contains the same elements of the input + * array plus the given element in the last position. The component type of + * the new array is the same as that of the input array. + *

+ * If the input array is {@code null}, a new one element array is returned + * whose component type is the same as the element, unless the element itself is null, + * in which case the return type is Object[] + * + *

+   * ArrayUtils.add(null, null)      = IllegalArgumentException
+   * ArrayUtils.add(null, "a")       = ["a"]
+   * ArrayUtils.add(["a"], null)     = ["a", null]
+   * ArrayUtils.add(["a"], "b")      = ["a", "b"]
+   * ArrayUtils.add(["a", "b"], "c") = ["a", "b", "c"]
+   * 
+ * + * This code is borrowed from `org.apache.commons:commons-lang3`. + * + * @param the component type of the array + * @param array the array to "add" the element to, may be {@code null} + * @param element the object to add, may be {@code null} + * @return A new array containing the existing elements plus the new element + * The returned array type will be that of the input array (unless null), + * in which case it will have the same type as the element. + * If both are null, an IllegalArgumentException is thrown + * @since 2.1 + * @throws IllegalArgumentException if both arguments are null + */ + public static T[] add(final T[] array, final T element) { + Class type; + if (array != null) { + type = array.getClass().getComponentType(); + } else if (element != null) { + type = element.getClass(); + } else { + throw new IllegalArgumentException("Arguments cannot both be null"); + } + @SuppressWarnings("unchecked") // type must be T + final T[] newArray = (T[]) copyArrayGrow1(array, type); + newArray[newArray.length - 1] = element; + return newArray; + } + + /** + * Returns a copy of the given array of size 1 greater than the argument. + * The last value of the array is left to the default value. + *

+ * This code is borrowed from `org.apache.commons:commons-lang3`. + * + * @param array The array to copy, must not be {@code null}. + * @param newArrayComponentType If {@code array} is {@code null}, create a + * size 1 array of this type. + * @return A new copy of the array of size 1 greater than the input. + */ + private static Object copyArrayGrow1(final Object array, final Class newArrayComponentType) { + if (array != null) { + final int arrayLength = Array.getLength(array); + final Object newArray = Array.newInstance(array.getClass().getComponentType(), arrayLength + 1); + System.arraycopy(array, 0, newArray, 0, arrayLength); + return newArray; + } + return Array.newInstance(newArrayComponentType, 1); + } } diff --git a/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java b/core/src/main/java/org/apache/iceberg/util/SerializationUtil.java similarity index 93% rename from mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java rename to core/src/main/java/org/apache/iceberg/util/SerializationUtil.java index bd35717719e5..52b14d96b0ca 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/SerializationUtil.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iceberg.mr; +package org.apache.iceberg.util; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -35,7 +35,7 @@ private SerializationUtil() { public static byte[] serializeToBytes(Object obj) { try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(baos)) { + ObjectOutputStream oos = new ObjectOutputStream(baos)) { oos.writeObject(obj); return baos.toByteArray(); } catch (IOException e) { @@ -50,7 +50,7 @@ public static T deserializeFromBytes(byte[] bytes) { } try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes); - ObjectInputStream ois = new ObjectInputStream(bais)) { + ObjectInputStream ois = new ObjectInputStream(bais)) { return (T) ois.readObject(); } catch (IOException e) { throw new UncheckedIOException("Failed to deserialize object", e); diff --git a/flink-runtime/LICENSE b/flink-runtime/LICENSE index 496a9e8a3882..ef2599b94d8c 100644 --- a/flink-runtime/LICENSE +++ b/flink-runtime/LICENSE @@ -461,3 +461,13 @@ License text: | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This binary includes code from Apache Commons. + +* Core ArrayUtil. + +Copyright: 2020 The Apache Software Foundation +Home page: https://commons.apache.org/ +License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java b/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java index 6f95d652ceab..319568bb6e99 100644 --- a/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java +++ b/flink/src/main/java/org/apache/iceberg/flink/data/FlinkParquetReaders.java @@ -26,7 +26,6 @@ import java.time.ZoneOffset; import java.util.List; import java.util.Map; -import org.apache.commons.lang3.ArrayUtils; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; import org.apache.flink.table.data.GenericRowData; @@ -45,6 +44,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ArrayUtil; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.GroupType; @@ -760,37 +760,37 @@ public RowData getRow(int pos, int numFields) { @Override public boolean[] toBooleanArray() { - return ArrayUtils.toPrimitive((Boolean[]) values); + return ArrayUtil.toPrimitive((Boolean[]) values); } @Override public byte[] toByteArray() { - return ArrayUtils.toPrimitive((Byte[]) values); + return ArrayUtil.toPrimitive((Byte[]) values); } @Override public short[] toShortArray() { - return ArrayUtils.toPrimitive((Short[]) values); + return ArrayUtil.toPrimitive((Short[]) values); } @Override public int[] toIntArray() { - return ArrayUtils.toPrimitive((Integer[]) values); + return ArrayUtil.toPrimitive((Integer[]) values); } @Override public long[] toLongArray() { - return ArrayUtils.toPrimitive((Long[]) values); + return ArrayUtil.toPrimitive((Long[]) values); } @Override public float[] toFloatArray() { - return ArrayUtils.toPrimitive((Float[]) values); + return ArrayUtil.toPrimitive((Float[]) values); } @Override public double[] toDoubleArray() { - return ArrayUtils.toPrimitive((Double[]) values); + return ArrayUtil.toPrimitive((Double[]) values); } } } diff --git a/hive-runtime/LICENSE b/hive-runtime/LICENSE index 8a42341a6db7..21881bb7664b 100644 --- a/hive-runtime/LICENSE +++ b/hive-runtime/LICENSE @@ -473,3 +473,12 @@ file: | This product includes software developed at | The Apache Software Foundation (http://www.apache.org/). +-------------------------------------------------------------------------------- + +This binary includes code from Apache Commons. + +* Core ArrayUtil. + +Copyright: 2020 The Apache Software Foundation +Home page: https://commons.apache.org/ +License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java index 566194243ee4..b5beee06244f 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java +++ b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java @@ -26,6 +26,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.util.SerializationUtil; public class InputFormatConfig { diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java index 62334f57c6fd..5974e7f2b94d 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java @@ -37,10 +37,10 @@ import org.apache.iceberg.data.Record; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.mr.InputFormatConfig; -import org.apache.iceberg.mr.SerializationUtil; import org.apache.iceberg.mr.mapred.Container; import org.apache.iceberg.mr.mapred.MapredIcebergInputFormat; import org.apache.iceberg.mr.mapreduce.IcebergSplit; +import org.apache.iceberg.util.SerializationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSplit.java b/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSplit.java index 81735b300665..a1e7b1aa4cde 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSplit.java +++ b/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSplit.java @@ -24,9 +24,9 @@ import java.io.IOException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.FileSplit; -import org.apache.iceberg.mr.SerializationUtil; import org.apache.iceberg.mr.mapreduce.IcebergSplit; import org.apache.iceberg.mr.mapreduce.IcebergSplitContainer; +import org.apache.iceberg.util.SerializationUtil; // Hive requires file formats to return splits that are instances of `FileSplit`. public class HiveIcebergSplit extends FileSplit implements IcebergSplitContainer { diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 3ea1cb71466a..aa5b9db6cf18 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/mr/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -46,10 +46,10 @@ import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.mr.Catalogs; import org.apache.iceberg.mr.InputFormatConfig; -import org.apache.iceberg.mr.SerializationUtil; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.util.SerializationUtil; import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT; import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT_DEFAULT; diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java index e319269d7c9f..033e5075be1c 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java @@ -63,7 +63,6 @@ import org.apache.iceberg.io.InputFile; import org.apache.iceberg.mr.Catalogs; import org.apache.iceberg.mr.InputFormatConfig; -import org.apache.iceberg.mr.SerializationUtil; import org.apache.iceberg.orc.ORC; import org.apache.iceberg.parquet.Parquet; import org.apache.iceberg.relocated.com.google.common.collect.Lists; @@ -71,6 +70,7 @@ import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.util.PartitionUtil; +import org.apache.iceberg.util.SerializationUtil; /** * Generic Mrv2 InputFormat API for Iceberg. diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java index 0d6c44074539..43b78d661a0d 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java @@ -32,7 +32,7 @@ import org.apache.iceberg.hadoop.Util; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.mr.InputFormatConfig; -import org.apache.iceberg.mr.SerializationUtil; +import org.apache.iceberg.util.SerializationUtil; // Since this class extends `mapreduce.InputSplit and implements `mapred.InputSplit`, it can be returned by both MR v1 // and v2 file formats. diff --git a/pig/src/main/java/org/apache/iceberg/pig/IcebergPigInputFormat.java b/pig/src/main/java/org/apache/iceberg/pig/IcebergPigInputFormat.java index 2c157392acee..e4bf5983d15b 100644 --- a/pig/src/main/java/org/apache/iceberg/pig/IcebergPigInputFormat.java +++ b/pig/src/main/java/org/apache/iceberg/pig/IcebergPigInputFormat.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.commons.lang3.SerializationUtils; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; @@ -52,6 +51,7 @@ import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ByteBuffers; +import org.apache.iceberg.util.SerializationUtil; import org.apache.pig.data.DataByteArray; import org.apache.pig.impl.util.ObjectSerializer; import org.slf4j.Logger; @@ -64,8 +64,8 @@ public class IcebergPigInputFormat extends InputFormat { static final String ICEBERG_PROJECTED_FIELDS = "iceberg.projected.fields"; static final String ICEBERG_FILTER_EXPRESSION = "iceberg.filter.expression"; - private Table table; - private String signature; + private final Table table; + private final String signature; private List splits; IcebergPigInputFormat(Table table, String signature) { @@ -129,7 +129,7 @@ public String[] getLocations() { @Override public void write(DataOutput out) throws IOException { - byte[] data = SerializationUtils.serialize(this.task); + byte[] data = SerializationUtil.serializeToBytes(this.task); out.writeInt(data.length); out.write(data); } @@ -139,7 +139,7 @@ public void readFields(DataInput in) throws IOException { byte[] data = new byte[in.readInt()]; in.readFully(data); - this.task = (CombinedScanTask) SerializationUtils.deserialize(data); + this.task = SerializationUtil.deserializeFromBytes(data); } } diff --git a/spark-runtime/LICENSE b/spark-runtime/LICENSE index aaf5c4f47933..ab3ce976a8d2 100644 --- a/spark-runtime/LICENSE +++ b/spark-runtime/LICENSE @@ -587,3 +587,12 @@ Copyright: 2020 Dremio Corporation. Home page: https://projectnessie.org/ License: http://www.apache.org/licenses/LICENSE-2.0 +-------------------------------------------------------------------------------- + +This binary includes code from Apache Commons. + +* Core ArrayUtil. + +Copyright: 2020 The Apache Software Foundation +Home page: https://commons.apache.org/ +License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/spark3-runtime/LICENSE b/spark3-runtime/LICENSE index 32f66f2a5952..dab3b906188a 100644 --- a/spark3-runtime/LICENSE +++ b/spark3-runtime/LICENSE @@ -609,3 +609,13 @@ This product includes code from Delta Lake. Copyright: 2020 The Delta Lake Project Authors. Home page: https://delta.io/ License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This binary includes code from Apache Commons. + +* Core ArrayUtil. + +Copyright: 2020 The Apache Software Foundation +Home page: https://commons.apache.org/ +License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java index 9d75efd8a496..d7a8a58b1ddb 100644 --- a/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java +++ b/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.lang3.ArrayUtils; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; @@ -49,6 +48,7 @@ import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ArrayUtil; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.PropertyUtil; import org.apache.spark.sql.Dataset; @@ -622,7 +622,7 @@ private static Dataset loadCatalogMetadataTable(SparkSession spark, String if (catalogAndIdentifier.catalog instanceof BaseCatalog) { BaseCatalog catalog = (BaseCatalog) catalogAndIdentifier.catalog; Identifier baseId = catalogAndIdentifier.identifier; - Identifier metaId = Identifier.of(ArrayUtils.add(baseId.namespace(), baseId.name()), type.name()); + Identifier metaId = Identifier.of(ArrayUtil.add(baseId.namespace(), baseId.name()), type.name()); Table metaTable = catalog.loadTable(metaId); return Dataset.ofRows(spark, DataSourceV2Relation.create(metaTable, Some.apply(catalog), Some.apply(metaId))); } diff --git a/versions.props b/versions.props index 43ce4017a93c..08626174c064 100644 --- a/versions.props +++ b/versions.props @@ -10,7 +10,6 @@ org.apache.spark:spark-hive_2.11 = 2.4.7 org.apache.spark:spark-avro_2.11 = 2.4.7 org.apache.spark:spark-hive_2.12 = 3.0.1 org.apache.pig:pig = 0.14.0 -org.apache.commons:commons-lang3 = 3.9 com.fasterxml.jackson.*:* = 2.10.0 com.google.guava:guava = 28.0-jre com.github.ben-manes.caffeine:caffeine = 2.7.0