From 06bd40895fd7dbb7163d2832d2c43ed71b2c71f9 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 9 Jan 2025 16:32:25 +0800 Subject: [PATCH] [enhance](mtmv)When obtaining the partition list fails, treat the paimon table as an unpartitioned table (#46641) When retrieving data of type Paimon Date in version 0.9 from the system table, the value is an integer and cannot be converted to type Date. This issue has been fixed in Paimon's latest code. This PR downgrades this situation without affecting user data queries --- .../paimon/run01.sql | 47 ++++++++++++++++++- .../paimon/PaimonExternalTable.java | 8 ++++ .../paimon/PaimonPartitionInfo.java | 5 ++ .../doris/datasource/paimon/PaimonUtil.java | 11 ++++- 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql index 7aa4170eab0985..7722d09463686e 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql @@ -22,4 +22,49 @@ insert into test_tb_mix_format values (1,1,'b'),(2,1,'b'),(3,1,'b'),(4,1,'b'),(5 -- update some data, these splits will be readed by jni insert into test_tb_mix_format values (1,2,'b'),(2,2,'b'),(3,2,'b'),(4,2,'b'),(5,2,'b'); -- delete foramt in table properties, doris should get format by file name -alter table test_tb_mix_format unset TBLPROPERTIES ('file.format'); \ No newline at end of file +alter table test_tb_mix_format unset TBLPROPERTIES ('file.format'); + +drop table if exists two_partition; +CREATE TABLE two_partition ( + id BIGINT, + create_date STRING, + region STRING +) PARTITIONED BY (create_date,region) TBLPROPERTIES ( + 'primary-key' = 'create_date,region,id', + 'bucket'=10, + 'file.format'='orc' +); + +insert into two_partition values(1,'2020-01-01','bj'); +insert into two_partition values(2,'2020-01-01','sh'); +insert into two_partition values(3,'2038-01-01','bj'); +insert into two_partition values(4,'2038-01-01','sh'); +insert into two_partition values(5,'2038-01-02','bj'); + +drop table if exists null_partition; +CREATE TABLE null_partition ( + id BIGINT, + region STRING +) PARTITIONED BY (region) TBLPROPERTIES ( + 'primary-key' = 'region,id', + 'bucket'=10, + 'file.format'='orc' +); +-- null NULL "null" all will be in partition [null] +insert into null_partition values(1,'bj'); +insert into null_partition values(2,null); +insert into null_partition values(3,NULL); +insert into null_partition values(4,'null'); +insert into null_partition values(5,'NULL'); + +drop table if exists date_partition; +CREATE TABLE date_partition ( + id BIGINT, + create_date DATE +) PARTITIONED BY (create_date) TBLPROPERTIES ( + 'primary-key' = 'create_date,id', + 'bucket'=10, + 'file.format'='orc' +); + +insert into date_partition values(1,date '2020-01-01'); \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 3f22ce4c46b32f..efd8e6bb7a7a42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -144,9 +144,17 @@ public long fetchRowCount() { @Override public List getPartitionColumns(Optional snapshot) { + if (isPartitionInvalid(snapshot)) { + return Collections.emptyList(); + } return getPaimonSchemaCacheValue(snapshot).getPartitionColumns(); } + private boolean isPartitionInvalid(Optional snapshot) { + PaimonSnapshotCacheValue paimonSnapshotCacheValue = getOrFetchSnapshotCacheValue(snapshot); + return paimonSnapshotCacheValue.getPartitionInfo().isPartitionInvalid(); + } + @Override public MvccSnapshot loadSnapshot() { return new PaimonMvccSnapshot(getPaimonSnapshotCacheValue()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index 4d3326f8e48376..88515a2510d2c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -45,4 +45,9 @@ public Map getNameToPartitionItem() { public Map getNameToPartition() { return nameToPartition; } + + public boolean isPartitionInvalid() { + // when transfer to partitionItem failed, will not equal + return nameToPartitionItem.size() != nameToPartition.size(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 1f7576dca51d93..b3df41bc5cef17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -117,7 +117,7 @@ public static PaimonPartition rowToPartition(InternalRow row) { } public static PaimonPartitionInfo generatePartitionInfo(List partitionColumns, - List paimonPartitions) throws AnalysisException { + List paimonPartitions) { Map nameToPartitionItem = Maps.newHashMap(); Map nameToPartition = Maps.newHashMap(); PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(nameToPartitionItem, nameToPartition); @@ -127,7 +127,14 @@ public static PaimonPartitionInfo generatePartitionInfo(List partitionCo for (PaimonPartition paimonPartition : paimonPartitions) { String partitionName = getPartitionName(partitionColumns, paimonPartition.getPartitionValues()); nameToPartition.put(partitionName, paimonPartition); - nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns)); + try { + // partition values return by paimon api, may have problem, + // to avoid affecting the query, we catch exceptions here + nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns)); + } catch (Exception e) { + LOG.warn("toListPartitionItem failed, partitionColumns: {}, partitionValues: {}", partitionColumns, + paimonPartition.getPartitionValues(), e); + } } return partitionInfo; }