From 6d534270186dc7a4f8a3c0679c9f02f2085e39ac Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Tue, 26 Dec 2023 10:50:19 +0800 Subject: [PATCH 1/2] fix paimon ctas --- .../planner/external/FileQueryScanNode.java | 22 +++++-------------- .../planner/external/MaxComputeScanNode.java | 9 +++++++- .../planner/external/hudi/HudiScanNode.java | 9 +++++++- .../external/iceberg/IcebergScanNode.java | 9 +++++++- .../external/paimon/PaimonScanNode.java | 13 ++++++++--- .../paimon/test_paimon_catalog.groovy | 18 +++++++++++++++ 6 files changed, 57 insertions(+), 23 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java index 658ac49659ba3e..419f2a752bee1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java @@ -38,12 +38,6 @@ import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.planner.PlanNodeId; -import org.apache.doris.planner.external.hudi.HudiScanNode; -import org.apache.doris.planner.external.hudi.HudiSplit; -import org.apache.doris.planner.external.iceberg.IcebergScanNode; -import org.apache.doris.planner.external.iceberg.IcebergSplit; -import org.apache.doris.planner.external.paimon.PaimonScanNode; -import org.apache.doris.planner.external.paimon.PaimonSplit; import org.apache.doris.qe.ConnectContext; import org.apache.doris.spi.Split; import org.apache.doris.statistics.StatisticalType; @@ -260,6 +254,9 @@ public TFileScanRangeParams getFileScanRangeParams() { return params; } + protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { + } + @Override public void createScanRangeLocations() throws UserException { long start = System.currentTimeMillis(); @@ -353,17 +350,7 @@ public void createScanRangeLocations() throws UserException { rangeDesc.setTableFormatParams(tableFormatFileDesc); } - // external data lake table - if (fileSplit instanceof IcebergSplit) { - // TODO: extract all data lake split to factory - IcebergScanNode.setIcebergParams(rangeDesc, (IcebergSplit) fileSplit); - } else if (fileSplit instanceof PaimonSplit) { - PaimonScanNode.setPaimonParams(rangeDesc, (PaimonSplit) fileSplit); - } else if (fileSplit instanceof HudiSplit) { - HudiScanNode.setHudiParams(rangeDesc, (HudiSplit) fileSplit); - } else if (fileSplit instanceof MaxComputeSplit) { - MaxComputeScanNode.setScanParams(rangeDesc, (MaxComputeSplit) fileSplit); - } + setScanParams(rangeDesc, fileSplit); curLocations.getScanRange().getExtScanRange().getFileScanRange().addToRanges(rangeDesc); TScanRangeLocation location = new TScanRangeLocation(); @@ -473,6 +460,7 @@ private TFileRangeDesc createFileRangeDesc(FileSplit fileSplit, List col protected abstract TFileFormatType getFileFormatType() throws UserException; + protected TFileCompressType getFileCompressType(FileSplit fileSplit) throws UserException { return Util.inferFileCompressTypeByPath(fileSplit.getPath().toString()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java index ae0b424ad815af..8fe7c8dff2283d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java @@ -63,7 +63,14 @@ public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeNa catalog = (MaxComputeExternalCatalog) table.getCatalog(); } - public static void setScanParams(TFileRangeDesc rangeDesc, MaxComputeSplit maxComputeSplit) { + @Override + protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { + if (split instanceof MaxComputeSplit) { + setScanParams(rangeDesc, (MaxComputeSplit) split); + } + } + + public void setScanParams(TFileRangeDesc rangeDesc, MaxComputeSplit maxComputeSplit) { TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc(); tableFormatFileDesc.setTableFormatType(TableFormatType.MAX_COMPUTE.value()); TMaxComputeFileDesc fileDesc = new TMaxComputeFileDesc(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java index f73947262e48c0..9d601e71daa951 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java @@ -132,7 +132,14 @@ protected Map getLocationProperties() throws UserException { } } - public static void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) { + @Override + protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { + if (split instanceof HudiSplit) { + setHudiParams(rangeDesc, (HudiSplit) split); + } + } + + public void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) { TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc(); tableFormatFileDesc.setTableFormatType(hudiSplit.getTableFormatType().value()); THudiFileDesc fileDesc = new THudiFileDesc(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java index e8e90d5096a8c9..ae3efccadc5ff1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java @@ -129,7 +129,14 @@ protected void doInitialize() throws UserException { super.doInitialize(); } - public static void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSplit) { + @Override + protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { + if (split instanceof IcebergSplit) { + setIcebergParams(rangeDesc, (IcebergSplit) split); + } + } + + public void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSplit) { TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc(); tableFormatFileDesc.setTableFormatType(icebergSplit.getTableFormatType().value()); TIcebergFileDesc fileDesc = new TIcebergFileDesc(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java index 7e8a4b78773c80..1e23d2f781df20 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java @@ -54,8 +54,8 @@ import java.util.stream.Collectors; public class PaimonScanNode extends FileQueryScanNode { - private static PaimonSource source = null; - private static List predicates; + private PaimonSource source = null; + private List predicates; public PaimonScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) { super(id, desc, "PAIMON_SCAN_NODE", StatisticalType.PAIMON_SCAN_NODE, needCheckColumnPriv); @@ -91,7 +91,14 @@ public static String encodeObjectToString(T t) { } } - public static void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) { + @Override + protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { + if (split instanceof PaimonSplit) { + setPaimonParams(rangeDesc, (PaimonSplit) split); + } + } + + public void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) { TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc(); tableFormatFileDesc.setTableFormatType(paimonSplit.getTableFormatType().value()); TPaimonFileDesc fileDesc = new TPaimonFileDesc(); diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy index ee74abd7ed887c..dce8b0e8888d7a 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy @@ -274,5 +274,23 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_ qt_c98 c98 qt_c99 c99 qt_c100 c100 + + // test view from jion paimon + sql """ switch internal """ + String view_db = "test_view_for_paimon" + sql """ drop database if exists ${view_db}""" + sql """ create database if not exists ${view_db}""" + sql """use ${view_db}""" + sql """ create view test_tst_1 as select * from ${catalog_name}.`db1`.all_table; """ + sql """ create view test_tst_2 as select * from ${catalog_name}.`db1`.all_table_with_parquet; """ + sql """ create view test_tst_5 as select * from ${catalog_name}.`db1`.array_nested; """ + sql """ create table test_tst_6 properties ("replication_num" = "1") as + select f.c2,f.c3,c.c1 from + (select a.c2,b.c3 from test_tst_1 a inner join test_tst_2 b on a.c2=b.c2) f + inner join test_tst_5 c on f.c2=c.c1; + """ + def view1 = """select * from test_tst_6 order by c1""" + + // qt_view1 view1 } } From 0aaf15e100ca63b71f75cbfe84a05d75c41bef9e Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Tue, 26 Dec 2023 15:30:40 +0800 Subject: [PATCH 2/2] add comment --- .../org/apache/doris/planner/external/FileQueryScanNode.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java index 419f2a752bee1e..9bae707c2f3ed8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java @@ -254,6 +254,7 @@ public TFileScanRangeParams getFileScanRangeParams() { return params; } + // Set some parameters of scan to support different types of file data sources protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { }