From 1875242bba50898585b5217ee4d45a8e4ef2ed3d Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Fri, 7 Apr 2023 16:48:23 -0700 Subject: [PATCH 1/8] Parquet: Update parquet to 1.13.0 --- versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.props b/versions.props index 7af5f0eaa9d4..4beeec63426b 100644 --- a/versions.props +++ b/versions.props @@ -5,7 +5,7 @@ org.apache.hadoop:* = 2.7.3 org.apache.hive:* = 2.3.9 org.apache.httpcomponents.client5:* = 5.2.1 org.apache.orc:* = 1.8.3 -org.apache.parquet:* = 1.12.3 +org.apache.parquet:* = 1.13.0 org.apache.pig:pig = 0.14.0 com.fasterxml.jackson.*:* = 2.14.1 com.google.code.findbugs:jsr305 = 3.0.2 From 904698068c83748113bfacffcd272dcea32b7c90 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sat, 8 Apr 2023 11:43:13 -0700 Subject: [PATCH 2/8] fix aliyun failures --- .../apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java | 2 +- build.gradle | 1 + versions.props | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java index 75766a671490..0bc45007ca27 100644 --- a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java +++ b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java @@ -36,8 +36,8 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import org.apache.commons.codec.binary.Hex; import org.apache.commons.io.FileUtils; -import org.apache.directory.api.util.Hex; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.io.ByteStreams; diff --git a/build.gradle b/build.gradle index ea8c0a6f699e..d445d3c9a8f8 100644 --- a/build.gradle +++ b/build.gradle @@ -449,6 +449,7 @@ project(':iceberg-aliyun') { exclude module: 'logback-classic' exclude module: 'spring-boot-starter-logging' } + testImplementation 'commons-io:commons-io' } } diff --git a/versions.props b/versions.props index 4beeec63426b..a17a82c8ccab 100644 --- a/versions.props +++ b/versions.props @@ -49,3 +49,4 @@ org.eclipse.jetty:* = 9.4.43.v20210629 org.testcontainers:* = 1.17.6 io.delta:delta-core_* = 2.2.0 org.awaitility:awaitility = 4.2.0 +commons-io:commons-io = 2.8.0 From 6b1ffddbb9793c3b8aca19d9cfedac54fd07be69 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 1 May 2023 15:36:22 -0700 Subject: [PATCH 3/8] Disable dictionary encoding to make sure BF always gets created --- .../org/apache/iceberg/TableProperties.java | 4 ++++ .../org/apache/iceberg/parquet/Parquet.java | 19 +++++++++++++++++++ .../parquet/TestBloomRowGroupFilter.java | 2 ++ 3 files changed, 25 insertions(+) diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index b14354def6ac..a2f63f79a061 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -135,6 +135,10 @@ private TableProperties() {} public static final String DELETE_PARQUET_PAGE_ROW_LIMIT = "write.delete.parquet.page-row-limit"; public static final int PARQUET_PAGE_ROW_LIMIT_DEFAULT = 20_000; + public static final String PARQUET_DICT_ENABLED = "write.parquet.enable.dictionary"; + + public static final boolean PARQUET_DICT_ENABLED_DEFAULT = true; + public static final String PARQUET_DICT_SIZE_BYTES = "write.parquet.dict-size-bytes"; public static final String DELETE_PARQUET_DICT_SIZE_BYTES = "write.delete.parquet.dict-size-bytes"; diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java b/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java index 8b1e6c056403..817f7c44e2a0 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java @@ -33,6 +33,8 @@ import static org.apache.iceberg.TableProperties.PARQUET_COMPRESSION_DEFAULT; import static org.apache.iceberg.TableProperties.PARQUET_COMPRESSION_LEVEL; import static org.apache.iceberg.TableProperties.PARQUET_COMPRESSION_LEVEL_DEFAULT; +import static org.apache.iceberg.TableProperties.PARQUET_DICT_ENABLED; +import static org.apache.iceberg.TableProperties.PARQUET_DICT_ENABLED_DEFAULT; import static org.apache.iceberg.TableProperties.PARQUET_DICT_SIZE_BYTES; import static org.apache.iceberg.TableProperties.PARQUET_DICT_SIZE_BYTES_DEFAULT; import static org.apache.iceberg.TableProperties.PARQUET_PAGE_ROW_LIMIT; @@ -244,6 +246,7 @@ public FileAppender build() throws IOException { int rowGroupSize = context.rowGroupSize(); int pageSize = context.pageSize(); int pageRowLimit = context.pageRowLimit(); + boolean dictionaryEnabled = context.dictionaryEnabled(); int dictionaryPageSize = context.dictionaryPageSize(); String compressionLevel = context.compressionLevel(); CompressionCodecName codec = context.codec(); @@ -286,6 +289,7 @@ public FileAppender build() throws IOException { .withWriterVersion(writerVersion) .withPageSize(pageSize) .withPageRowCountLimit(pageRowLimit) + .withDictionaryEncoding(dictionaryEnabled) .withDictionaryPageSize(dictionaryPageSize) .withMinRowCountForPageSizeCheck(rowGroupCheckMinRecordCount) .withMaxRowCountForPageSizeCheck(rowGroupCheckMaxRecordCount) @@ -323,6 +327,7 @@ public FileAppender build() throws IOException { .withRowGroupSize(rowGroupSize) .withPageSize(pageSize) .withPageRowCountLimit(pageRowLimit) + .withDictionaryEncoding(dictionaryEnabled) .withDictionaryPageSize(dictionaryPageSize); for (Map.Entry entry : columnBloomFilterEnabled.entrySet()) { @@ -339,6 +344,8 @@ private static class Context { private final int rowGroupSize; private final int pageSize; private final int pageRowLimit; + + private final boolean dictionaryEnabled; private final int dictionaryPageSize; private final CompressionCodecName codec; private final String compressionLevel; @@ -351,6 +358,7 @@ private Context( int rowGroupSize, int pageSize, int pageRowLimit, + boolean dictionaryEnabled, int dictionaryPageSize, CompressionCodecName codec, String compressionLevel, @@ -361,6 +369,7 @@ private Context( this.rowGroupSize = rowGroupSize; this.pageSize = pageSize; this.pageRowLimit = pageRowLimit; + this.dictionaryEnabled = dictionaryEnabled; this.dictionaryPageSize = dictionaryPageSize; this.codec = codec; this.compressionLevel = compressionLevel; @@ -386,6 +395,10 @@ static Context dataContext(Map config) { config, PARQUET_PAGE_ROW_LIMIT, PARQUET_PAGE_ROW_LIMIT_DEFAULT); Preconditions.checkArgument(pageRowLimit > 0, "Page row count limit must be > 0"); + boolean dictionaryEnabled = + PropertyUtil.propertyAsBoolean( + config, PARQUET_DICT_ENABLED, PARQUET_DICT_ENABLED_DEFAULT); + int dictionaryPageSize = PropertyUtil.propertyAsInt( config, PARQUET_DICT_SIZE_BYTES, PARQUET_DICT_SIZE_BYTES_DEFAULT); @@ -429,6 +442,7 @@ static Context dataContext(Map config) { rowGroupSize, pageSize, pageRowLimit, + dictionaryEnabled, dictionaryPageSize, codec, compressionLevel, @@ -500,6 +514,7 @@ static Context deleteContext(Map config) { rowGroupSize, pageSize, pageRowLimit, + dataContext.dictionaryEnabled(), dictionaryPageSize, codec, compressionLevel, @@ -529,6 +544,10 @@ int pageRowLimit() { return pageRowLimit; } + boolean dictionaryEnabled() { + return dictionaryEnabled; + } + int dictionaryPageSize() { return dictionaryPageSize; } diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestBloomRowGroupFilter.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestBloomRowGroupFilter.java index 34a92a9b4483..acb318a23286 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestBloomRowGroupFilter.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestBloomRowGroupFilter.java @@ -19,6 +19,7 @@ package org.apache.iceberg.parquet; import static org.apache.iceberg.TableProperties.PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX; +import static org.apache.iceberg.TableProperties.PARQUET_DICT_ENABLED; import static org.apache.iceberg.avro.AvroSchemaUtil.convert; import static org.apache.iceberg.expressions.Expressions.and; import static org.apache.iceberg.expressions.Expressions.equal; @@ -197,6 +198,7 @@ public void createInputFile() throws IOException { try (FileAppender appender = Parquet.write(outFile) .schema(FILE_SCHEMA) + .set(PARQUET_DICT_ENABLED, "false") .set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "_id", "true") .set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "_long", "true") .set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "_double", "true") From 9d32c5bb6c31f4a9ab67b7655d56da56de4f09d9 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 1 May 2023 15:48:32 -0700 Subject: [PATCH 4/8] Update the doc --- .../org/apache/iceberg/TableProperties.java | 1 - docs/configuration.md | 91 ++++++++++--------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index a2f63f79a061..770865a5693c 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -136,7 +136,6 @@ private TableProperties() {} public static final int PARQUET_PAGE_ROW_LIMIT_DEFAULT = 20_000; public static final String PARQUET_DICT_ENABLED = "write.parquet.enable.dictionary"; - public static final boolean PARQUET_DICT_ENABLED_DEFAULT = true; public static final String PARQUET_DICT_SIZE_BYTES = "write.parquet.dict-size-bytes"; diff --git a/docs/configuration.md b/docs/configuration.md index 15e36b34e4b0..99a25fec128c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -47,51 +47,52 @@ Iceberg tables support table properties to configure table behavior, like the de ### Write properties -| Property | Default | Description | -|-----------------------------------------------------|----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| write.format.default | parquet | Default file format for the table; parquet, avro, or orc | -| write.delete.format.default | data file format | Default delete file format for the table; parquet, avro, or orc | -| write.parquet.row-group-size-bytes | 134217728 (128 MB) | Parquet row group size | -| write.parquet.page-size-bytes | 1048576 (1 MB) | Parquet page size | -| write.parquet.page-row-limit | 20000 | Parquet page row limit | -| write.parquet.dict-size-bytes | 2097152 (2 MB) | Parquet dictionary page size | -| write.parquet.compression-codec | gzip | Parquet compression codec: zstd, brotli, lz4, gzip, snappy, uncompressed | -| write.parquet.compression-level | null | Parquet compression level | -| write.parquet.bloom-filter-enabled.column.col1 | (not set) | Enables writing a bloom filter for the column: col1 | -| write.parquet.bloom-filter-max-bytes | 1048576 (1 MB) | The maximum number of bytes for a bloom filter bitset | -| write.avro.compression-codec | gzip | Avro compression codec: gzip(deflate with 9 level), zstd, snappy, uncompressed | -| write.avro.compression-level | null | Avro compression level | -| write.orc.stripe-size-bytes | 67108864 (64 MB) | Define the default ORC stripe size, in bytes | -| write.orc.block-size-bytes | 268435456 (256 MB) | Define the default file system block size for ORC files | -| write.orc.compression-codec | zlib | ORC compression codec: zstd, lz4, lzo, zlib, snappy, none | -| write.orc.compression-strategy | speed | ORC compression strategy: speed, compression | -| write.orc.bloom.filter.columns | (not set) | Comma separated list of column names for which a Bloom filter must be created | -| write.orc.bloom.filter.fpp | 0.05 | False positive probability for Bloom filter (must > 0.0 and < 1.0) | -| write.location-provider.impl | null | Optional custom implementation for LocationProvider | -| write.metadata.compression-codec | none | Metadata compression codec; none or gzip | -| write.metadata.metrics.max-inferred-column-defaults | 100 | Defines the maximum number of columns for which metrics are collected | -| write.metadata.metrics.default | truncate(16) | Default metrics mode for all columns in the table; none, counts, truncate(length), or full | -| write.metadata.metrics.column.col1 | (not set) | Metrics mode for column 'col1' to allow per-column tuning; none, counts, truncate(length), or full | -| write.target-file-size-bytes | 536870912 (512 MB) | Controls the size of files generated to target about this many bytes | -| write.delete.target-file-size-bytes | 67108864 (64 MB) | Controls the size of delete files generated to target about this many bytes | -| write.distribution-mode | none | Defines distribution of write data: __none__: don't shuffle rows; __hash__: hash distribute by partition key ; __range__: range distribute by partition key or sort key if table has an SortOrder | -| write.delete.distribution-mode | hash | Defines distribution of write delete data | -| write.update.distribution-mode | hash | Defines distribution of write update data | -| write.merge.distribution-mode | none | Defines distribution of write merge data | -| write.wap.enabled | false | Enables write-audit-publish writes | -| write.summary.partition-limit | 0 | Includes partition-level summary stats in snapshot summaries if the changed partition count is less than this limit | -| write.metadata.delete-after-commit.enabled | false | Controls whether to delete the oldest **tracked** version metadata files after commit | -| write.metadata.previous-versions-max | 100 | The max number of previous version metadata files to keep before deleting after commit | -| write.spark.fanout.enabled | false | Enables the fanout writer in Spark that does not require data to be clustered; uses more memory | -| write.object-storage.enabled | false | Enables the object storage location provider that adds a hash component to file paths | -| write.data.path | table location + /data | Base location for data files | -| write.metadata.path | table location + /metadata | Base location for metadata files | -| write.delete.mode | copy-on-write | Mode used for delete commands: copy-on-write or merge-on-read (v2 only) | -| write.delete.isolation-level | serializable | Isolation level for delete commands: serializable or snapshot | -| write.update.mode | copy-on-write | Mode used for update commands: copy-on-write or merge-on-read (v2 only) | -| write.update.isolation-level | serializable | Isolation level for update commands: serializable or snapshot | -| write.merge.mode | copy-on-write | Mode used for merge commands: copy-on-write or merge-on-read (v2 only) | -| write.merge.isolation-level | serializable | Isolation level for merge commands: serializable or snapshot | +| Property | Default | Description | +|------------------------------------------------------|-----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| write.format.default | parquet | Default file format for the table; parquet, avro, or orc | +| write.delete.format.default | data file format | Default delete file format for the table; parquet, avro, or orc | +| write.parquet.row-group-size-bytes | 134217728 (128 MB) | Parquet row group size | +| write.parquet.page-size-bytes | 1048576 (1 MB) | Parquet page size | +| write.parquet.page-row-limit | 20000 | Parquet page row limit | + | write.parquet.dictionary.enabled | true | Enable dictionary encoding | + | write.parquet.dict-size-bytes | 2097152 (2 MB) | Parquet dictionary page size | +| write.parquet.compression-codec | gzip | Parquet compression codec: zstd, brotli, lz4, gzip, snappy, uncompressed | +| write.parquet.compression-level | null | Parquet compression level | +| write.parquet.bloom-filter-enabled.column.col1 | (not set) | Hint to parquet to write a bloom filter for the column: col1 | +| write.parquet.bloom-filter-max-bytes | 1048576 (1 MB) | The maximum number of bytes for a bloom filter bitset | +| write.avro.compression-codec | gzip | Avro compression codec: gzip(deflate with 9 level), zstd, snappy, uncompressed | +| write.avro.compression-level | null | Avro compression level | +| write.orc.stripe-size-bytes | 67108864 (64 MB) | Define the default ORC stripe size, in bytes | +| write.orc.block-size-bytes | 268435456 (256 MB) | Define the default file system block size for ORC files | +| write.orc.compression-codec | zlib | ORC compression codec: zstd, lz4, lzo, zlib, snappy, none | +| write.orc.compression-strategy | speed | ORC compression strategy: speed, compression | +| write.orc.bloom.filter.columns | (not set) | Comma separated list of column names for which a Bloom filter must be created | +| write.orc.bloom.filter.fpp | 0.05 | False positive probability for Bloom filter (must > 0.0 and < 1.0) | +| write.location-provider.impl | null | Optional custom implementation for LocationProvider | +| write.metadata.compression-codec | none | Metadata compression codec; none or gzip | +| write.metadata.metrics.max-inferred-column-defaults | 100 | Defines the maximum number of columns for which metrics are collected | +| write.metadata.metrics.default | truncate(16) | Default metrics mode for all columns in the table; none, counts, truncate(length), or full | +| write.metadata.metrics.column.col1 | (not set) | Metrics mode for column 'col1' to allow per-column tuning; none, counts, truncate(length), or full | +| write.target-file-size-bytes | 536870912 (512 MB) | Controls the size of files generated to target about this many bytes | +| write.delete.target-file-size-bytes | 67108864 (64 MB) | Controls the size of delete files generated to target about this many bytes | +| write.distribution-mode | none | Defines distribution of write data: __none__: don't shuffle rows; __hash__: hash distribute by partition key ; __range__: range distribute by partition key or sort key if table has an SortOrder | +| write.delete.distribution-mode | hash | Defines distribution of write delete data | +| write.update.distribution-mode | hash | Defines distribution of write update data | +| write.merge.distribution-mode | none | Defines distribution of write merge data | +| write.wap.enabled | false | Enables write-audit-publish writes | +| write.summary.partition-limit | 0 | Includes partition-level summary stats in snapshot summaries if the changed partition count is less than this limit | +| write.metadata.delete-after-commit.enabled | false | Controls whether to delete the oldest **tracked** version metadata files after commit | +| write.metadata.previous-versions-max | 100 | The max number of previous version metadata files to keep before deleting after commit | +| write.spark.fanout.enabled | false | Enables the fanout writer in Spark that does not require data to be clustered; uses more memory | +| write.object-storage.enabled | false | Enables the object storage location provider that adds a hash component to file paths | +| write.data.path | table location + /data | Base location for data files | +| write.metadata.path | table location + /metadata | Base location for metadata files | +| write.delete.mode | copy-on-write | Mode used for delete commands: copy-on-write or merge-on-read (v2 only) | +| write.delete.isolation-level | serializable | Isolation level for delete commands: serializable or snapshot | +| write.update.mode | copy-on-write | Mode used for update commands: copy-on-write or merge-on-read (v2 only) | +| write.update.isolation-level | serializable | Isolation level for update commands: serializable or snapshot | +| write.merge.mode | copy-on-write | Mode used for merge commands: copy-on-write or merge-on-read (v2 only) | +| write.merge.isolation-level | serializable | Isolation level for merge commands: serializable or snapshot | ### Table behavior properties From e3eb27284adbd946a2df265e6bd05134d8eb1c35 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 16 May 2023 16:31:09 -0700 Subject: [PATCH 5/8] point to 1.13.1 using staging repo until officially released --- build.gradle | 5 +++++ versions.props | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d445d3c9a8f8..6dbaeddfdb34 100644 --- a/build.gradle +++ b/build.gradle @@ -137,6 +137,11 @@ allprojects { repositories { mavenCentral() mavenLocal() + maven { + url "https://repository.apache.org/content/groups/staging/" + mavenContent { + } + } } } diff --git a/versions.props b/versions.props index a17a82c8ccab..073930944fc5 100644 --- a/versions.props +++ b/versions.props @@ -5,7 +5,7 @@ org.apache.hadoop:* = 2.7.3 org.apache.hive:* = 2.3.9 org.apache.httpcomponents.client5:* = 5.2.1 org.apache.orc:* = 1.8.3 -org.apache.parquet:* = 1.13.0 +org.apache.parquet:* = 1.13.1 org.apache.pig:pig = 0.14.0 com.fasterxml.jackson.*:* = 2.14.1 com.google.code.findbugs:jsr305 = 3.0.2 From 6e99c43d01c95c1e4cd8e653b7d09a758b7805ce Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 18 May 2023 18:31:59 -0700 Subject: [PATCH 6/8] revert staging remote repo --- build.gradle | 5 ----- 1 file changed, 5 deletions(-) diff --git a/build.gradle b/build.gradle index 6dbaeddfdb34..d445d3c9a8f8 100644 --- a/build.gradle +++ b/build.gradle @@ -137,11 +137,6 @@ allprojects { repositories { mavenCentral() mavenLocal() - maven { - url "https://repository.apache.org/content/groups/staging/" - mavenContent { - } - } } } From 7f2649d71e42fc09ced2c0fb611691f8894ef8a7 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Fri, 19 May 2023 08:20:00 -0700 Subject: [PATCH 7/8] remove apache commons dep --- .../iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java | 2 +- build.gradle | 1 - versions.props | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java index 0bc45007ca27..75766a671490 100644 --- a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java +++ b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java @@ -36,8 +36,8 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import org.apache.commons.codec.binary.Hex; import org.apache.commons.io.FileUtils; +import org.apache.directory.api.util.Hex; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.io.ByteStreams; diff --git a/build.gradle b/build.gradle index d445d3c9a8f8..ea8c0a6f699e 100644 --- a/build.gradle +++ b/build.gradle @@ -449,7 +449,6 @@ project(':iceberg-aliyun') { exclude module: 'logback-classic' exclude module: 'spring-boot-starter-logging' } - testImplementation 'commons-io:commons-io' } } diff --git a/versions.props b/versions.props index 073930944fc5..c9028be3cfcf 100644 --- a/versions.props +++ b/versions.props @@ -48,5 +48,4 @@ com.esotericsoftware:kryo = 4.0.2 org.eclipse.jetty:* = 9.4.43.v20210629 org.testcontainers:* = 1.17.6 io.delta:delta-core_* = 2.2.0 -org.awaitility:awaitility = 4.2.0 -commons-io:commons-io = 2.8.0 +org.awaitility:awaitility = 4.2.0 \ No newline at end of file From 9f978041dbe8871b585822d6551dd5a4e3bd8b86 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Fri, 19 May 2023 11:22:29 -0700 Subject: [PATCH 8/8] address review feedback --- parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java | 1 - 1 file changed, 1 deletion(-) diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java b/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java index 817f7c44e2a0..7caeb7c7cd71 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java @@ -344,7 +344,6 @@ private static class Context { private final int rowGroupSize; private final int pageSize; private final int pageRowLimit; - private final boolean dictionaryEnabled; private final int dictionaryPageSize; private final CompressionCodecName codec;