diff --git a/docs/content/migration/iceberg-compatibility.md b/docs/content/migration/iceberg-compatibility.md index 73f64fb60007..66b6f17b2e31 100644 --- a/docs/content/migration/iceberg-compatibility.md +++ b/docs/content/migration/iceberg-compatibility.md @@ -389,6 +389,12 @@ you also need to set some (or all) of the following table options when creating String Hive client class name for Iceberg Hive Catalog. + +
metadata.iceberg.glue.skip-archive
+ false + Boolean + Skip archive for AWS Glue catalog. + diff --git a/docs/layouts/shortcodes/generated/iceberg_configuration.html b/docs/layouts/shortcodes/generated/iceberg_configuration.html index b51be14a9345..a71cabb67991 100644 --- a/docs/layouts/shortcodes/generated/iceberg_configuration.html +++ b/docs/layouts/shortcodes/generated/iceberg_configuration.html @@ -44,6 +44,12 @@ String Metastore database name for Iceberg Catalog. Set this as an iceberg database alias if using a centralized Catalog. + +
metadata.iceberg.glue.skip-archive
+ false + Boolean + Skip archive for AWS Glue catalog. +
metadata.iceberg.hadoop-conf-dir
(none) diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java index b7237dcd5b43..b538a7a60625 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java @@ -106,6 +106,12 @@ public class IcebergOptions { "Metastore table name for Iceberg Catalog." + "Set this as an iceberg table alias if using a centralized Catalog."); + public static final ConfigOption GLUE_SKIP_ARCHIVE = + key("metadata.iceberg.glue.skip-archive") + .booleanType() + .defaultValue(false) + .withDescription("Skip archive for AWS Glue catalog."); + /** Where to store Iceberg metadata. */ public enum StorageType implements DescribedEnum { DISABLED("disabled", "Disable Iceberg compatibility support."), diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java index 44b2a8a7e90a..5eb2bfd27ab6 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java @@ -30,9 +30,11 @@ import org.apache.paimon.utils.Preconditions; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.SerDeInfo; @@ -145,9 +147,20 @@ private void commitMetadataImpl(Path newMetadataPath, @Nullable Path baseMetadat .put("previous_metadata_location", baseMetadataPath.toString()); } + Options options = new Options(table.options()); + boolean skipAWSGlueArchive = options.get(IcebergOptions.GLUE_SKIP_ARCHIVE); + EnvironmentContext environmentContext = new EnvironmentContext(); + environmentContext.putToProperties(StatsSetupConst.CASCADE, StatsSetupConst.TRUE); + environmentContext.putToProperties( + "skipAWSGlueArchive", Boolean.toString(skipAWSGlueArchive)); + clients.execute( client -> - client.alter_table(icebergHiveDatabase, icebergHiveTable, hiveTable, true)); + client.alter_table_with_environmentContext( + icebergHiveDatabase, + icebergHiveTable, + hiveTable, + environmentContext)); } private boolean databaseExists(String databaseName) throws Exception { diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java index c97f6f209048..c0c68888b8a3 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java @@ -164,7 +164,8 @@ public void testAppendOnlyTable() throws Exception { tEnv.executeSql("CREATE DATABASE my_paimon.test_db"); tEnv.executeSql( "CREATE TABLE my_paimon.test_db.t ( pt INT, id INT, data STRING ) PARTITIONED BY (pt) WITH " - + "( 'metadata.iceberg.storage' = 'hive-catalog', 'metadata.iceberg.uri' = '', 'file.format' = 'avro' )"); + + "( 'metadata.iceberg.storage' = 'hive-catalog', 'metadata.iceberg.uri' = '', 'file.format' = 'avro'," + + " 'metadata.iceberg.glue.skip-archive' = 'true' )"); tEnv.executeSql( "INSERT INTO my_paimon.test_db.t VALUES " + "(1, 1, 'apple'), (1, 2, 'pear'), (2, 1, 'cat'), (2, 2, 'dog')")