From 2e03bb842e74acf7e28df2a2ea9e952f72d22c8c Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 18 Aug 2023 14:20:30 +0800 Subject: [PATCH 1/2] [Fix](RoutineLoad)Fix when Unique (MoW) routineload imports unspecified Sequence column --- .../doris/planner/external/LoadScanProvider.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java index 9089f0c2c76758..1280023e140a5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java @@ -190,7 +190,7 @@ private void initColumns(FileLoadScanNode.ParamCreateContext context, Analyzer a .filter(c -> c.getColumnName().equalsIgnoreCase(finalSequenceCol)).findAny(); // if `columnDescs.descs` is empty, that means it's not a partial update load, and user not specify // column name. - if (foundCol.isPresent() || columnDescs.descs.isEmpty()) { + if (foundCol.isPresent() || columnDescs.descs.isEmpty() || shouldAddSequenceColumn(columnDescs)) { columnDescs.descs.add(new ImportColumnDesc(Column.SEQUENCE_COL, new SlotRef(null, sequenceCol))); } else if (!fileGroupInfo.isPartialUpdate()) { @@ -226,6 +226,17 @@ private void initColumns(FileLoadScanNode.ParamCreateContext context, Analyzer a } } + /** + * if not set sequence column and column size is null or only have deleted sign ,return true + */ + private boolean shouldAddSequenceColumn(LoadTaskInfo.ImportColumnDescs columnDescs) { + if (columnDescs.descs.isEmpty()) { + return true; + } + return columnDescs.descs.size() == 1 && columnDescs.descs.get(0).getColumnName() + .equalsIgnoreCase(Column.DELETE_SIGN); + } + private TFileFormatType formatType(String fileFormat, String path) throws UserException { if (fileFormat != null) { String lowerFileFormat = fileFormat.toLowerCase(); From b3d911e6661fb6271c990d1cd7f2358053f8cd52 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 18 Aug 2023 15:30:05 +0800 Subject: [PATCH 2/2] reduce unless code --- .../org/apache/doris/planner/external/LoadScanProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java index 1280023e140a5e..25be5de433ac68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java @@ -190,7 +190,7 @@ private void initColumns(FileLoadScanNode.ParamCreateContext context, Analyzer a .filter(c -> c.getColumnName().equalsIgnoreCase(finalSequenceCol)).findAny(); // if `columnDescs.descs` is empty, that means it's not a partial update load, and user not specify // column name. - if (foundCol.isPresent() || columnDescs.descs.isEmpty() || shouldAddSequenceColumn(columnDescs)) { + if (foundCol.isPresent() || shouldAddSequenceColumn(columnDescs)) { columnDescs.descs.add(new ImportColumnDesc(Column.SEQUENCE_COL, new SlotRef(null, sequenceCol))); } else if (!fileGroupInfo.isPartialUpdate()) {