From b078da121b1e036dfed1acced52549578bde7f6f Mon Sep 17 00:00:00 2001 From: vinlee19 Date: Mon, 21 Jul 2025 21:03:43 +0800 Subject: [PATCH 1/5] show partitions command for paimon --- .../plans/commands/ShowPartitionsCommand.java | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java index 558117ed209594..44d4bf46502c73 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; @@ -46,6 +47,8 @@ import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable; +import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.properties.OrderKey; @@ -211,7 +214,8 @@ protected void validate(ConnectContext ctx) throws AnalysisException { // disallow unsupported catalog if (!(catalog.isInternalCatalog() || catalog instanceof HMSExternalCatalog - || catalog instanceof MaxComputeExternalCatalog || catalog instanceof IcebergExternalCatalog)) { + || catalog instanceof MaxComputeExternalCatalog || catalog instanceof IcebergExternalCatalog) + || catalog instanceof PaimonExternalCatalog) { throw new AnalysisException(String.format("Catalog of type '%s' is not allowed in ShowPartitionsCommand", catalog.getType())); } @@ -289,6 +293,13 @@ protected void analyze() throws UserException { return; } + if (table instanceof PaimonExternalTable) { + if (((PaimonExternalTable) table).getPartitionType(Optional.empty()).equals(PartitionType.UNPARTITIONED)) { + throw new AnalysisException("Table " + tblName + " is not a partitioned table"); + } + return; + } + table.readLock(); try { // build proc path @@ -360,6 +371,34 @@ private ShowResultSet handleShowIcebergTablePartitions() { return new ShowResultSet(getMetaData(), rows); } + private ShowResultSet handleShowPaimonTablePartitions() { + PaimonExternalCatalog paimonCatalog = (PaimonExternalCatalog) catalog; + String db = ClusterNamespace.getNameFromFullName(tableName.getDb()); + String tbl = tableName.getTbl(); + PaimonExternalTable paimonTable = (PaimonExternalTable) paimonCatalog.getDb(db).get().getTable(tbl).get(); + + Map partitions = paimonTable.getAndCopyPartitionItems(Optional.empty()); + List> rows = new ArrayList<>(); + for (Map.Entry entry : partitions.entrySet()) { + List row = new ArrayList<>(); + Range items = entry.getValue().getItems(); + row.add(entry.getKey()); + row.add(items.lowerEndpoint().toString()); + row.add(items.upperEndpoint().toString()); + rows.add(row); + } + // sort by partition name + if (orderByPairs != null && orderByPairs.get(0).isDesc()) { + rows.sort(Comparator.comparing(x -> x.get(0), Comparator.reverseOrder())); + } else { + rows.sort(Comparator.comparing(x -> x.get(0))); + } + + rows = applyLimit(limit, offset, rows); + + return new ShowResultSet(getMetaData(), rows); + } + private ShowResultSet handleShowHMSTablePartitions() throws AnalysisException { HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; List> rows = new ArrayList<>(); @@ -446,7 +485,7 @@ public ShowResultSetMetaData getMetaData() { for (String col : result.getColumnNames()) { builder.addColumn(new Column(col, ScalarType.createVarchar(30))); } - } else if (catalog instanceof IcebergExternalCatalog) { + } else if (catalog instanceof IcebergExternalCatalog || catalog instanceof PaimonExternalCatalog) { builder.addColumn(new Column("Partition", ScalarType.createVarchar(60))); builder.addColumn(new Column("Lower Bound", ScalarType.createVarchar(100))); builder.addColumn(new Column("Upper Bound", ScalarType.createVarchar(100))); From 4d45f4e3e5c76703179c1550128c1cbd034175f9 Mon Sep 17 00:00:00 2001 From: vinlee19 Date: Mon, 21 Jul 2025 22:23:11 +0800 Subject: [PATCH 2/5] Revert "show partitions command for paimon" This reverts commit b078da121b1e036dfed1acced52549578bde7f6f. --- .../plans/commands/ShowPartitionsCommand.java | 43 +------------------ 1 file changed, 2 insertions(+), 41 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java index 44d4bf46502c73..558117ed209594 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionsCommand.java @@ -24,7 +24,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.PartitionKey; -import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; @@ -47,8 +46,6 @@ import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable; -import org.apache.doris.datasource.paimon.PaimonExternalCatalog; -import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.properties.OrderKey; @@ -214,8 +211,7 @@ protected void validate(ConnectContext ctx) throws AnalysisException { // disallow unsupported catalog if (!(catalog.isInternalCatalog() || catalog instanceof HMSExternalCatalog - || catalog instanceof MaxComputeExternalCatalog || catalog instanceof IcebergExternalCatalog) - || catalog instanceof PaimonExternalCatalog) { + || catalog instanceof MaxComputeExternalCatalog || catalog instanceof IcebergExternalCatalog)) { throw new AnalysisException(String.format("Catalog of type '%s' is not allowed in ShowPartitionsCommand", catalog.getType())); } @@ -293,13 +289,6 @@ protected void analyze() throws UserException { return; } - if (table instanceof PaimonExternalTable) { - if (((PaimonExternalTable) table).getPartitionType(Optional.empty()).equals(PartitionType.UNPARTITIONED)) { - throw new AnalysisException("Table " + tblName + " is not a partitioned table"); - } - return; - } - table.readLock(); try { // build proc path @@ -371,34 +360,6 @@ private ShowResultSet handleShowIcebergTablePartitions() { return new ShowResultSet(getMetaData(), rows); } - private ShowResultSet handleShowPaimonTablePartitions() { - PaimonExternalCatalog paimonCatalog = (PaimonExternalCatalog) catalog; - String db = ClusterNamespace.getNameFromFullName(tableName.getDb()); - String tbl = tableName.getTbl(); - PaimonExternalTable paimonTable = (PaimonExternalTable) paimonCatalog.getDb(db).get().getTable(tbl).get(); - - Map partitions = paimonTable.getAndCopyPartitionItems(Optional.empty()); - List> rows = new ArrayList<>(); - for (Map.Entry entry : partitions.entrySet()) { - List row = new ArrayList<>(); - Range items = entry.getValue().getItems(); - row.add(entry.getKey()); - row.add(items.lowerEndpoint().toString()); - row.add(items.upperEndpoint().toString()); - rows.add(row); - } - // sort by partition name - if (orderByPairs != null && orderByPairs.get(0).isDesc()) { - rows.sort(Comparator.comparing(x -> x.get(0), Comparator.reverseOrder())); - } else { - rows.sort(Comparator.comparing(x -> x.get(0))); - } - - rows = applyLimit(limit, offset, rows); - - return new ShowResultSet(getMetaData(), rows); - } - private ShowResultSet handleShowHMSTablePartitions() throws AnalysisException { HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; List> rows = new ArrayList<>(); @@ -485,7 +446,7 @@ public ShowResultSetMetaData getMetaData() { for (String col : result.getColumnNames()) { builder.addColumn(new Column(col, ScalarType.createVarchar(30))); } - } else if (catalog instanceof IcebergExternalCatalog || catalog instanceof PaimonExternalCatalog) { + } else if (catalog instanceof IcebergExternalCatalog) { builder.addColumn(new Column("Partition", ScalarType.createVarchar(60))); builder.addColumn(new Column("Lower Bound", ScalarType.createVarchar(100))); builder.addColumn(new Column("Upper Bound", ScalarType.createVarchar(100))); From bc47b1ed299d0a10340738b5422dd25956bb8bf8 Mon Sep 17 00:00:00 2001 From: vinlee19 Date: Wed, 23 Jul 2025 20:13:55 +0800 Subject: [PATCH 3/5] add mvmt regression --- regression-test/data/mtmv_p0/test_paimon_mtmv.out | 3 +++ .../suites/mtmv_p0/test_paimon_mtmv.groovy | 14 ++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/regression-test/data/mtmv_p0/test_paimon_mtmv.out b/regression-test/data/mtmv_p0/test_paimon_mtmv.out index 0cfe4bd293cf51..5c7547c0687c86 100644 --- a/regression-test/data/mtmv_p0/test_paimon_mtmv.out +++ b/regression-test/data/mtmv_p0/test_paimon_mtmv.out @@ -143,6 +143,9 @@ true -- !date_partition_base_table -- 1 2020-01-01 +-- !date_type_partition -- +1 2020-01-01 + -- !date_partition -- 1 2020-01-01 diff --git a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy index abd0f64450ef4e..c4ea889f92307a 100644 --- a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy @@ -268,8 +268,7 @@ suite("test_paimon_mtmv", "p0,external,mtmv,external_docker,external_docker_dori // date type will has problem order_qt_date_partition_base_table "SELECT * FROM ${catalogName}.`test_paimon_spark`.date_partition" - test { - sql """ + sql """ CREATE MATERIALIZED VIEW ${mvName} BUILD DEFERRED REFRESH AUTO ON MANUAL partition by (`create_date`) @@ -278,8 +277,15 @@ suite("test_paimon_mtmv", "p0,external,mtmv,external_docker,external_docker_dori AS SELECT * FROM ${catalogName}.`test_paimon_spark`.date_partition; """ - exception "Unable to find a suitable base table" - } + sql """ + REFRESH MATERIALIZED VIEW ${mvName} auto; + """ + waitingMTMVTaskFinishedByMvName(mvName) + def showPaimonDateTypePartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPaimonDateTypePartitionsResult: " + showPaimonDateTypePartitionsResult.toString()) + assertTrue(showPaimonDateTypePartitionsResult.toString().contains("p_20200101")) + order_qt_date_type_partition "select * FROM ${mvName}" + sql """drop materialized view if exists ${mvName};""" sql """ CREATE MATERIALIZED VIEW ${mvName} From 8cd1ae637cc78b2acd86f76766bf1486f02990ef Mon Sep 17 00:00:00 2001 From: vinlee19 Date: Tue, 29 Jul 2025 22:05:27 +0800 Subject: [PATCH 4/5] fix can not convert paimon date partition --- .../doris/datasource/paimon/PaimonUtil.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index a0ede02a32a0ff..549e45299bda8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -59,6 +59,7 @@ import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; import org.apache.paimon.types.VarCharType; +import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.InstantiationUtil; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Projection; @@ -116,12 +117,23 @@ public static PaimonPartitionInfo generatePartitionInfo(List partitionCo Map nameToPartitionItem = Maps.newHashMap(); Map nameToPartition = Maps.newHashMap(); PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(nameToPartitionItem, nameToPartition); + List types = partitionColumns.stream() + .map(Column::getType) + .collect(Collectors.toList()); for (Partition partition : paimonPartitions) { + int index = 0; Map spec = partition.spec(); StringBuilder sb = new StringBuilder(); for (Map.Entry entry : spec.entrySet()) { - sb.append(entry.getKey()).append("=").append(entry.getValue()).append("/"); + sb.append(entry.getKey()).append("="); + // Paimon stores DATE type as days since 1970-01-01 (epoch), so we convert the integer to a date string. + if (types.get(index).isDateV2()) { + sb.append(DateTimeUtils.formatDate(Integer.parseInt(entry.getValue()))).append("/"); + } else { + sb.append(entry.getValue()).append("/"); + } + index++; } if (sb.length() > 0) { sb.deleteCharAt(sb.length() - 1); @@ -131,7 +143,7 @@ public static PaimonPartitionInfo generatePartitionInfo(List partitionCo try { // partition values return by paimon api, may have problem, // to avoid affecting the query, we catch exceptions here - nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns)); + nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, types)); } catch (Exception e) { LOG.warn("toListPartitionItem failed, partitionColumns: {}, partitionValues: {}", partitionColumns, partition.spec(), e); @@ -140,11 +152,8 @@ public static PaimonPartitionInfo generatePartitionInfo(List partitionCo return partitionInfo; } - public static ListPartitionItem toListPartitionItem(String partitionName, List partitionColumns) + public static ListPartitionItem toListPartitionItem(String partitionName, List types) throws AnalysisException { - List types = partitionColumns.stream() - .map(Column::getType) - .collect(Collectors.toList()); // Partition name will be in format: nation=cn/city=beijing // parse it to get values "cn" and "beijing" List partitionValues = HiveUtil.toPartitionValues(partitionName); From a50be83779afc2900603ab1998a712d49d8ef2ee Mon Sep 17 00:00:00 2001 From: vinlee19 Date: Wed, 30 Jul 2025 21:03:32 +0800 Subject: [PATCH 5/5] optimize get column name logic --- .../java/org/apache/doris/datasource/paimon/PaimonUtil.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 549e45299bda8e..8f82f3c3aa0695 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -120,20 +120,20 @@ public static PaimonPartitionInfo generatePartitionInfo(List partitionCo List types = partitionColumns.stream() .map(Column::getType) .collect(Collectors.toList()); + Map columnNameToType = partitionColumns.stream() + .collect(Collectors.toMap(Column::getName, Column::getType)); for (Partition partition : paimonPartitions) { - int index = 0; Map spec = partition.spec(); StringBuilder sb = new StringBuilder(); for (Map.Entry entry : spec.entrySet()) { sb.append(entry.getKey()).append("="); // Paimon stores DATE type as days since 1970-01-01 (epoch), so we convert the integer to a date string. - if (types.get(index).isDateV2()) { + if (columnNameToType.getOrDefault(entry.getKey(), Type.NULL).isDateV2()) { sb.append(DateTimeUtils.formatDate(Integer.parseInt(entry.getValue()))).append("/"); } else { sb.append(entry.getValue()).append("/"); } - index++; } if (sb.length() > 0) { sb.deleteCharAt(sb.length() - 1);