From d48d809d0a361876406dcde17cf41943c9c06d38 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 8 Jan 2025 19:04:56 +0800 Subject: [PATCH 1/5] 1 --- .../doris/datasource/paimon/PaimonExternalTable.java | 8 +++++++- .../apache/doris/datasource/paimon/PaimonUtil.java | 11 +++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 7264d83621d8be..588e0ee9b73d43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -176,7 +176,13 @@ public Set getPartitionColumnNames(Optional snapshot) { @Override public List getPartitionColumns(Optional snapshot) { - return getPaimonSchemaCacheValue(snapshot).getPartitionColumns(); + PaimonSchemaCacheValue paimonSchemaCacheValue = getPaimonSchemaCacheValue(snapshot); + if (paimonSchemaCacheValue) + return paimonSchemaCacheValue.getPartitionColumns(); + } + + private boolean isPartitionInvalid(PaimonSchemaCacheValue paimonSchemaCacheValue) { + } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 1f7576dca51d93..b3df41bc5cef17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -117,7 +117,7 @@ public static PaimonPartition rowToPartition(InternalRow row) { } public static PaimonPartitionInfo generatePartitionInfo(List partitionColumns, - List paimonPartitions) throws AnalysisException { + List paimonPartitions) { Map nameToPartitionItem = Maps.newHashMap(); Map nameToPartition = Maps.newHashMap(); PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(nameToPartitionItem, nameToPartition); @@ -127,7 +127,14 @@ public static PaimonPartitionInfo generatePartitionInfo(List partitionCo for (PaimonPartition paimonPartition : paimonPartitions) { String partitionName = getPartitionName(partitionColumns, paimonPartition.getPartitionValues()); nameToPartition.put(partitionName, paimonPartition); - nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns)); + try { + // partition values return by paimon api, may have problem, + // to avoid affecting the query, we catch exceptions here + nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns)); + } catch (Exception e) { + LOG.warn("toListPartitionItem failed, partitionColumns: {}, partitionValues: {}", partitionColumns, + paimonPartition.getPartitionValues(), e); + } } return partitionInfo; } From e5b70f88bef708677f59d38b38055533b5552d7e Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 8 Jan 2025 19:44:31 +0800 Subject: [PATCH 2/5] 1 --- .../datasource/paimon/PaimonExternalTable.java | 15 ++++++++++----- .../datasource/paimon/PaimonPartitionInfo.java | 5 +++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 588e0ee9b73d43..347e8df5c8f75e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -165,6 +165,9 @@ public Map getAndCopyPartitionItems(Optional snapshot) { + if (isPartitionInvalid(snapshot)) { + return PartitionType.UNPARTITIONED; + } return getPartitionColumns(snapshot).size() > 0 ? PartitionType.LIST : PartitionType.UNPARTITIONED; } @@ -176,13 +179,15 @@ public Set getPartitionColumnNames(Optional snapshot) { @Override public List getPartitionColumns(Optional snapshot) { - PaimonSchemaCacheValue paimonSchemaCacheValue = getPaimonSchemaCacheValue(snapshot); - if (paimonSchemaCacheValue) - return paimonSchemaCacheValue.getPartitionColumns(); + if (isPartitionInvalid(snapshot)) { + return Collections.emptyList(); + } + return getPaimonSchemaCacheValue(snapshot).getPartitionColumns(); } - private boolean isPartitionInvalid(PaimonSchemaCacheValue paimonSchemaCacheValue) { - + private boolean isPartitionInvalid(Optional snapshot) { + PaimonSnapshotCacheValue paimonSnapshotCacheValue = getOrFetchSnapshotCacheValue(snapshot); + return paimonSnapshotCacheValue.getPartitionInfo().isPartitionInvalid(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index 4d3326f8e48376..88515a2510d2c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -45,4 +45,9 @@ public Map getNameToPartitionItem() { public Map getNameToPartition() { return nameToPartition; } + + public boolean isPartitionInvalid() { + // when transfer to partitionItem failed, will not equal + return nameToPartitionItem.size() != nameToPartition.size(); + } } From cb996e532aaccdb22f32dbd00f1ec660de34cc19 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 8 Jan 2025 19:49:08 +0800 Subject: [PATCH 3/5] 1 --- .../create_preinstalled_scripts/paimon/run01.sql | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql index 5cc0a0ea685e37..7722d09463686e 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql @@ -55,4 +55,16 @@ insert into null_partition values(1,'bj'); insert into null_partition values(2,null); insert into null_partition values(3,NULL); insert into null_partition values(4,'null'); -insert into null_partition values(5,'NULL'); \ No newline at end of file +insert into null_partition values(5,'NULL'); + +drop table if exists date_partition; +CREATE TABLE date_partition ( + id BIGINT, + create_date DATE +) PARTITIONED BY (create_date) TBLPROPERTIES ( + 'primary-key' = 'create_date,id', + 'bucket'=10, + 'file.format'='orc' +); + +insert into date_partition values(1,date '2020-01-01'); \ No newline at end of file From 691eeeee6c9124f779e42a60f48f8a318d8e82d7 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 8 Jan 2025 20:02:25 +0800 Subject: [PATCH 4/5] 1 --- .../suites/mtmv_p0/test_paimon_mtmv.groovy | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy index 48d63e03ec3db5..98477ab92d542e 100644 --- a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy @@ -264,6 +264,36 @@ suite("test_paimon_mtmv", "p0,external,mtmv,external_docker,external_docker_dori order_qt_null_partition "SELECT * FROM ${mvName} " sql """drop materialized view if exists ${mvName};""" + // date type will has problem + order_qt_date_partition_base_table "SELECT * FROM ${catalogName}.`test_paimon_spark`.date_partition" + test { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (`create_date`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * FROM ${catalogName}.`test_paimon_spark`.date_partition; + """ + exception "Unable to find a suitable base table" + } + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * FROM ${catalogName}.`test_paimon_spark`.date_partition; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} auto; + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_date_partition "SELECT * FROM ${mvName} " + + sql """drop materialized view if exists ${mvName};""" sql """drop catalog if exists ${catalogName}""" } From ed7d320c0624c9ae04bb81d6fd6ccf9d78fb2271 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 8 Jan 2025 20:04:46 +0800 Subject: [PATCH 5/5] 1 --- regression-test/data/mtmv_p0/test_paimon_mtmv.out | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/regression-test/data/mtmv_p0/test_paimon_mtmv.out b/regression-test/data/mtmv_p0/test_paimon_mtmv.out index ba6fc06c1d2491..0cfe4bd293cf51 100644 --- a/regression-test/data/mtmv_p0/test_paimon_mtmv.out +++ b/regression-test/data/mtmv_p0/test_paimon_mtmv.out @@ -140,3 +140,9 @@ true 4 null 5 NULL +-- !date_partition_base_table -- +1 2020-01-01 + +-- !date_partition -- +1 2020-01-01 +