From a160915e1dbccfbdb3bf39d125b831d57ea6600e Mon Sep 17 00:00:00 2001 From: wgcn <1026688210@qq.com> Date: Tue, 5 Mar 2024 19:56:58 +0800 Subject: [PATCH 1/4] improve HiveSerDe --- .../org/apache/paimon/hive/HiveSchema.java | 19 ++++++++++++++++-- .../paimon/hive/PaimonStorageHandler.java | 17 +++++++++++++--- .../paimon/hive/HiveTableSchemaTest.java | 20 +++++++++++++++++++ 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java index dc6fc99219f8..6d11c179e45a 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java @@ -19,6 +19,7 @@ package org.apache.paimon.hive; import org.apache.paimon.CoreOptions; +import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; @@ -29,6 +30,7 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.JsonSerdeUtil; import org.apache.paimon.utils.StringUtils; import org.apache.paimon.shade.guava30.com.google.common.base.Splitter; @@ -96,7 +98,11 @@ public List fieldComments() { /** Extract {@link HiveSchema} from Hive serde properties. */ public static HiveSchema extract(@Nullable Configuration configuration, Properties properties) { String location = LocationKeyExtractor.getPaimonLocation(configuration, properties); - Optional tableSchema = getExistingSchema(configuration, location); + Optional tableSchema = getTableSchemaFromCache(properties); + tableSchema = + tableSchema.isPresent() + ? tableSchema + : HiveSchema.getExistingSchema(configuration, location); String columnProperty = properties.getProperty(hive_metastoreConstants.META_TABLE_COLUMNS); // Create hive external table with empty ddl @@ -192,7 +198,16 @@ public static HiveSchema extract(@Nullable Configuration configuration, Properti return new HiveSchema(builder.build()); } - private static Optional getExistingSchema( + @VisibleForTesting + static Optional getTableSchemaFromCache(Properties properties) { + String paimonSchemaStr = properties.getProperty(PaimonStorageHandler.PAIMON_SCHEMA); + if (paimonSchemaStr != null) { + return Optional.of(JsonSerdeUtil.fromJson(paimonSchemaStr, TableSchema.class)); + } + return Optional.empty(); + } + + public static Optional getExistingSchema( @Nullable Configuration configuration, @Nullable String location) { if (location == null) { return Optional.empty(); diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java index b410833a9b11..05940daf611b 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java @@ -21,6 +21,7 @@ import org.apache.paimon.hive.mapred.PaimonInputFormat; import org.apache.paimon.hive.mapred.PaimonOutputCommitter; import org.apache.paimon.hive.mapred.PaimonOutputFormat; +import org.apache.paimon.schema.TableSchema; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.HiveMetaHook; @@ -38,6 +39,7 @@ import org.apache.hadoop.mapred.OutputFormat; import java.util.Map; +import java.util.Optional; import java.util.Properties; /** {@link HiveStorageHandler} for paimon. This is the entrance class of Hive API. */ @@ -46,6 +48,8 @@ public class PaimonStorageHandler implements HiveStoragePredicateHandler, HiveSt private static final String MAPRED_OUTPUT_COMMITTER = "mapred.output.committer.class"; private static final String PAIMON_WRITE = "paimon.write"; + public static final String PAIMON_SCHEMA = "paimon.schema"; + private Configuration conf; @Override @@ -76,9 +80,16 @@ public HiveAuthorizationProvider getAuthorizationProvider() throws HiveException @Override public void configureInputJobProperties(TableDesc tableDesc, Map map) { Properties properties = tableDesc.getProperties(); - map.put( - LocationKeyExtractor.INTERNAL_LOCATION, - LocationKeyExtractor.getPaimonLocation(conf, properties)); + String paimonLocation = LocationKeyExtractor.getPaimonLocation(conf, properties); + map.put(LocationKeyExtractor.INTERNAL_LOCATION, paimonLocation); + + // cache the schema into table properties + Optional existingSchema = HiveSchema.getExistingSchema(null, paimonLocation); + if (existingSchema.isPresent()) { + String tableSchema = existingSchema.get().toString(); + tableDesc.getProperties().put(PAIMON_SCHEMA, tableSchema); + map.put(PAIMON_SCHEMA, tableSchema); + } } public void configureInputJobCredentials(TableDesc tableDesc, Map map) {} diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java index 326667a47619..1f4d97106b02 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java @@ -22,6 +22,7 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.TableSchema; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; @@ -33,8 +34,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.Optional; import java.util.Properties; +import static org.apache.paimon.hive.PaimonStorageHandler.PAIMON_SCHEMA; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -342,4 +345,21 @@ private Properties createTableWithExistsDDL() { properties.setProperty("location", tempDir.toString()); return properties; } + + @Test + public void testReadSchemaFromProperties() throws Exception { + createSchema(); + // cache the TableSchema to properties + Optional existingSchema = + HiveSchema.getExistingSchema(null, tempDir.toString()); + assertThat(existingSchema).isPresent(); + Properties properties = new Properties(); + TableSchema tableSchema = existingSchema.get(); + properties.put(PAIMON_SCHEMA, tableSchema.toString()); + + // get the TableSchema from properties + Optional tableSchemaFromCache = HiveSchema.getTableSchemaFromCache(properties); + assertThat(tableSchemaFromCache).isPresent(); + assertThat(tableSchemaFromCache.get()).isEqualTo(tableSchema); + } } From 03043799f49b9431e74282e622ce208ac92651fb Mon Sep 17 00:00:00 2001 From: wgcn <1026688210@qq.com> Date: Tue, 12 Mar 2024 17:12:30 +0800 Subject: [PATCH 2/4] refactor --- .../java/org/apache/paimon/hive/HiveSchema.java | 2 +- .../java/org/apache/paimon/hive/PaimonSerDe.java | 14 +++++++++++--- .../apache/paimon/hive/PaimonStorageHandler.java | 15 ++++----------- .../apache/paimon/hive/HiveTableSchemaTest.java | 4 ++-- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java index 6d11c179e45a..9256c4b99562 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java @@ -200,7 +200,7 @@ public static HiveSchema extract(@Nullable Configuration configuration, Properti @VisibleForTesting static Optional getTableSchemaFromCache(Properties properties) { - String paimonSchemaStr = properties.getProperty(PaimonStorageHandler.PAIMON_SCHEMA); + String paimonSchemaStr = properties.getProperty(PaimonStorageHandler.PAIMON_HIVE_SCHEMA); if (paimonSchemaStr != null) { return Optional.of(JsonSerdeUtil.fromJson(paimonSchemaStr, TableSchema.class)); } diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonSerDe.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonSerDe.java index 2934b7e77d9f..79fa42596e6f 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonSerDe.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonSerDe.java @@ -19,6 +19,7 @@ package org.apache.paimon.hive; import org.apache.paimon.hive.objectinspector.PaimonInternalRowObjectInspector; +import org.apache.paimon.utils.JsonSerdeUtil; import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; @@ -53,11 +54,18 @@ public class PaimonSerDe extends AbstractSerDe { @Override public void initialize(@Nullable Configuration configuration, Properties properties) throws SerDeException { - HiveSchema schema = HiveSchema.extract(configuration, properties); - this.tableSchema = schema; + String hiveSchemaStr = properties.getProperty(PaimonStorageHandler.PAIMON_HIVE_SCHEMA); + if (hiveSchemaStr != null) { + this.tableSchema = JsonSerdeUtil.fromJson(hiveSchemaStr, HiveSchema.class); + } else { + this.tableSchema = HiveSchema.extract(configuration, properties); + } + inspector = new PaimonInternalRowObjectInspector( - schema.fieldNames(), schema.fieldTypes(), schema.fieldComments()); + tableSchema.fieldNames(), + tableSchema.fieldTypes(), + tableSchema.fieldComments()); } @Override diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java index 05940daf611b..8a94757627c3 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java @@ -21,7 +21,7 @@ import org.apache.paimon.hive.mapred.PaimonInputFormat; import org.apache.paimon.hive.mapred.PaimonOutputCommitter; import org.apache.paimon.hive.mapred.PaimonOutputFormat; -import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.utils.JsonSerdeUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.HiveMetaHook; @@ -39,7 +39,6 @@ import org.apache.hadoop.mapred.OutputFormat; import java.util.Map; -import java.util.Optional; import java.util.Properties; /** {@link HiveStorageHandler} for paimon. This is the entrance class of Hive API. */ @@ -48,7 +47,7 @@ public class PaimonStorageHandler implements HiveStoragePredicateHandler, HiveSt private static final String MAPRED_OUTPUT_COMMITTER = "mapred.output.committer.class"; private static final String PAIMON_WRITE = "paimon.write"; - public static final String PAIMON_SCHEMA = "paimon.schema"; + public static final String PAIMON_HIVE_SCHEMA = "paimon.hive.schema"; private Configuration conf; @@ -82,14 +81,8 @@ public void configureInputJobProperties(TableDesc tableDesc, Map Properties properties = tableDesc.getProperties(); String paimonLocation = LocationKeyExtractor.getPaimonLocation(conf, properties); map.put(LocationKeyExtractor.INTERNAL_LOCATION, paimonLocation); - - // cache the schema into table properties - Optional existingSchema = HiveSchema.getExistingSchema(null, paimonLocation); - if (existingSchema.isPresent()) { - String tableSchema = existingSchema.get().toString(); - tableDesc.getProperties().put(PAIMON_SCHEMA, tableSchema); - map.put(PAIMON_SCHEMA, tableSchema); - } + HiveSchema hiveSchema = HiveSchema.extract(null, properties); + tableDesc.getProperties().put(PAIMON_HIVE_SCHEMA, JsonSerdeUtil.toJson(hiveSchema)); } public void configureInputJobCredentials(TableDesc tableDesc, Map map) {} diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java index 1f4d97106b02..cd21c663e92a 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java @@ -37,7 +37,7 @@ import java.util.Optional; import java.util.Properties; -import static org.apache.paimon.hive.PaimonStorageHandler.PAIMON_SCHEMA; +import static org.apache.paimon.hive.PaimonStorageHandler.PAIMON_HIVE_SCHEMA; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -355,7 +355,7 @@ public void testReadSchemaFromProperties() throws Exception { assertThat(existingSchema).isPresent(); Properties properties = new Properties(); TableSchema tableSchema = existingSchema.get(); - properties.put(PAIMON_SCHEMA, tableSchema.toString()); + properties.put(PAIMON_HIVE_SCHEMA, tableSchema.toString()); // get the TableSchema from properties Optional tableSchemaFromCache = HiveSchema.getTableSchemaFromCache(properties); From 93266c4657719cba5028fcaeee75424bb7e64ceb Mon Sep 17 00:00:00 2001 From: wgcn <1026688210@qq.com> Date: Tue, 12 Mar 2024 21:19:48 +0800 Subject: [PATCH 3/4] cache hiveschema --- .../apache/paimon/utils/JsonSerdeUtil.java | 8 ++++++ .../org/apache/paimon/hive/HiveSchema.java | 21 +++------------ .../org/apache/paimon/hive/PaimonSerDe.java | 12 ++++++--- .../paimon/hive/PaimonStorageHandler.java | 9 +++++-- .../paimon/hive/HiveTableSchemaTest.java | 26 ++++++++++--------- 5 files changed, 41 insertions(+), 35 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java b/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java index d45a9336847d..676276a30e58 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java @@ -120,6 +120,14 @@ public static T getNodeAs( fieldName, clazz.getName(), node.getClass().getName())); } + public static T fromJson(String json, TypeReference typeReference) { + try { + return OBJECT_MAPPER_INSTANCE.readValue(json, typeReference); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + public static T fromJson(String json, Class clazz) { try { return OBJECT_MAPPER_INSTANCE.reader().readValue(json, clazz); diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java index 9256c4b99562..f637651413ed 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java @@ -19,7 +19,6 @@ package org.apache.paimon.hive; import org.apache.paimon.CoreOptions; -import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; @@ -30,7 +29,6 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowType; -import org.apache.paimon.utils.JsonSerdeUtil; import org.apache.paimon.utils.StringUtils; import org.apache.paimon.shade.guava30.com.google.common.base.Splitter; @@ -69,7 +67,7 @@ public class HiveSchema { private static final Logger LOG = LoggerFactory.getLogger(HiveSchema.class); private final RowType rowType; - private HiveSchema(RowType rowType) { + HiveSchema(RowType rowType) { this.rowType = rowType; } @@ -98,11 +96,7 @@ public List fieldComments() { /** Extract {@link HiveSchema} from Hive serde properties. */ public static HiveSchema extract(@Nullable Configuration configuration, Properties properties) { String location = LocationKeyExtractor.getPaimonLocation(configuration, properties); - Optional tableSchema = getTableSchemaFromCache(properties); - tableSchema = - tableSchema.isPresent() - ? tableSchema - : HiveSchema.getExistingSchema(configuration, location); + Optional tableSchema = getExistingSchema(configuration, location); String columnProperty = properties.getProperty(hive_metastoreConstants.META_TABLE_COLUMNS); // Create hive external table with empty ddl @@ -198,16 +192,7 @@ public static HiveSchema extract(@Nullable Configuration configuration, Properti return new HiveSchema(builder.build()); } - @VisibleForTesting - static Optional getTableSchemaFromCache(Properties properties) { - String paimonSchemaStr = properties.getProperty(PaimonStorageHandler.PAIMON_HIVE_SCHEMA); - if (paimonSchemaStr != null) { - return Optional.of(JsonSerdeUtil.fromJson(paimonSchemaStr, TableSchema.class)); - } - return Optional.empty(); - } - - public static Optional getExistingSchema( + private static Optional getExistingSchema( @Nullable Configuration configuration, @Nullable String location) { if (location == null) { return Optional.empty(); diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonSerDe.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonSerDe.java index 79fa42596e6f..fe6a31b53c29 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonSerDe.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonSerDe.java @@ -19,9 +19,12 @@ package org.apache.paimon.hive; import org.apache.paimon.hive.objectinspector.PaimonInternalRowObjectInspector; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.RowType; import org.apache.paimon.utils.JsonSerdeUtil; import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde2.AbstractSerDe; @@ -33,6 +36,7 @@ import javax.annotation.Nullable; +import java.util.List; import java.util.Map; import java.util.Properties; @@ -54,9 +58,11 @@ public class PaimonSerDe extends AbstractSerDe { @Override public void initialize(@Nullable Configuration configuration, Properties properties) throws SerDeException { - String hiveSchemaStr = properties.getProperty(PaimonStorageHandler.PAIMON_HIVE_SCHEMA); - if (hiveSchemaStr != null) { - this.tableSchema = JsonSerdeUtil.fromJson(hiveSchemaStr, HiveSchema.class); + String dataFieldStr = properties.getProperty(PaimonStorageHandler.PAIMON_TABLE_FIELDS); + if (dataFieldStr != null) { + List dataFields = + JsonSerdeUtil.fromJson(dataFieldStr, new TypeReference>() {}); + this.tableSchema = new HiveSchema(new RowType(dataFields)); } else { this.tableSchema = HiveSchema.extract(configuration, properties); } diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java index 8a94757627c3..5987fd0c9dfb 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonStorageHandler.java @@ -47,7 +47,7 @@ public class PaimonStorageHandler implements HiveStoragePredicateHandler, HiveSt private static final String MAPRED_OUTPUT_COMMITTER = "mapred.output.committer.class"; private static final String PAIMON_WRITE = "paimon.write"; - public static final String PAIMON_HIVE_SCHEMA = "paimon.hive.schema"; + public static final String PAIMON_TABLE_FIELDS = "paimon.table.fields"; private Configuration conf; @@ -81,8 +81,13 @@ public void configureInputJobProperties(TableDesc tableDesc, Map Properties properties = tableDesc.getProperties(); String paimonLocation = LocationKeyExtractor.getPaimonLocation(conf, properties); map.put(LocationKeyExtractor.INTERNAL_LOCATION, paimonLocation); + String dataFieldJsonStr = getDataFieldsJsonStr(properties); + tableDesc.getProperties().put(PAIMON_TABLE_FIELDS, dataFieldJsonStr); + } + + static String getDataFieldsJsonStr(Properties properties) { HiveSchema hiveSchema = HiveSchema.extract(null, properties); - tableDesc.getProperties().put(PAIMON_HIVE_SCHEMA, JsonSerdeUtil.toJson(hiveSchema)); + return JsonSerdeUtil.toJson(hiveSchema.fields()); } public void configureInputJobCredentials(TableDesc tableDesc, Map map) {} diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java index cd21c663e92a..971003eefad0 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java @@ -22,11 +22,14 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; -import org.apache.paimon.schema.TableSchema; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.JsonSerdeUtil; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference; + +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -34,10 +37,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.Optional; +import java.util.List; import java.util.Properties; -import static org.apache.paimon.hive.PaimonStorageHandler.PAIMON_HIVE_SCHEMA; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -350,16 +352,16 @@ private Properties createTableWithExistsDDL() { public void testReadSchemaFromProperties() throws Exception { createSchema(); // cache the TableSchema to properties - Optional existingSchema = - HiveSchema.getExistingSchema(null, tempDir.toString()); - assertThat(existingSchema).isPresent(); Properties properties = new Properties(); - TableSchema tableSchema = existingSchema.get(); - properties.put(PAIMON_HIVE_SCHEMA, tableSchema.toString()); + properties.put(hive_metastoreConstants.META_TABLE_LOCATION, tempDir.toString()); + + HiveSchema hiveSchema = HiveSchema.extract(null, properties); + + List dataFields = hiveSchema.fields(); + String dataFieldStr = JsonSerdeUtil.toJson(dataFields); - // get the TableSchema from properties - Optional tableSchemaFromCache = HiveSchema.getTableSchemaFromCache(properties); - assertThat(tableSchemaFromCache).isPresent(); - assertThat(tableSchemaFromCache.get()).isEqualTo(tableSchema); + List dataFieldsDeserialized = + JsonSerdeUtil.fromJson(dataFieldStr, new TypeReference>() {}); + assertThat(dataFields).isEqualTo(dataFieldsDeserialized); } } From dc813b76897f6a944421e8da5b91f5ae4619bafd Mon Sep 17 00:00:00 2001 From: wgcn <1026688210@qq.com> Date: Wed, 13 Mar 2024 09:22:11 +0800 Subject: [PATCH 4/4] cache hiveschema --- .../java/org/apache/paimon/hive/HiveTableSchemaTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java index 971003eefad0..07cd00c8e67e 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java @@ -349,7 +349,7 @@ private Properties createTableWithExistsDDL() { } @Test - public void testReadSchemaFromProperties() throws Exception { + public void testReadHiveSchemaFromProperties() throws Exception { createSchema(); // cache the TableSchema to properties Properties properties = new Properties(); @@ -362,6 +362,7 @@ public void testReadSchemaFromProperties() throws Exception { List dataFieldsDeserialized = JsonSerdeUtil.fromJson(dataFieldStr, new TypeReference>() {}); - assertThat(dataFields).isEqualTo(dataFieldsDeserialized); + HiveSchema newHiveSchema = new HiveSchema(new RowType(dataFieldsDeserialized)); + assertThat(newHiveSchema).usingRecursiveComparison().isEqualTo(hiveSchema); } }