From 7ccff6addd0efebfe7dc142ca863bc2df1abbd5c Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Thu, 22 Apr 2021 10:12:10 +0800 Subject: [PATCH 1/9] Data export function, add certain columns that can be exported to the table Data export function, add certain columns that can be exported to the table --- .../org/apache/doris/analysis/ExportStmt.java | 14 +++++++++ .../org/apache/doris/analysis/LoadStmt.java | 3 ++ .../java/org/apache/doris/load/ExportJob.java | 29 +++++++++++++------ 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java index 160aaf6bb503dd..efa903c1daeef8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java @@ -17,6 +17,7 @@ package org.apache.doris.analysis; +import com.google.common.base.Splitter; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.FsBroker; @@ -43,6 +44,7 @@ import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -69,6 +71,7 @@ public class ExportStmt extends StatementBase { private Map properties = Maps.newHashMap(); private String columnSeparator; private String lineDelimiter; + private List columns = new ArrayList<>(); private TableRef tableRef; @@ -85,6 +88,10 @@ public ExportStmt(TableRef tableRef, Expr whereExpr, String path, this.lineDelimiter = DEFAULT_LINE_DELIMITER; } + public List getColumns() { + return columns; + } + public TableName getTblName() { return tblName; } @@ -264,6 +271,13 @@ private void checkProperties(Map properties) throws UserExceptio properties, ExportStmt.DEFAULT_COLUMN_SEPARATOR)); this.lineDelimiter = Separator.convertSeparator(PropertyAnalyzer.analyzeLineDelimiter( properties, ExportStmt.DEFAULT_LINE_DELIMITER)); + if(properties.containsKey(LoadStmt.EXPORT_KEY_IN_PARAM_COLUMNS)){ + String cols = properties.get(LoadStmt.EXPORT_KEY_IN_PARAM_COLUMNS); + if(!Strings.isNullOrEmpty(cols)){ + Splitter split = Splitter.on(',').trimResults().omitEmptyStrings(); + this.columns = split.splitToList(cols.toLowerCase()); + } + } // exec_mem_limit if (properties.containsKey(LoadStmt.EXEC_MEM_LIMIT)) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java index 29156183220e01..d097c0b6dd89c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java @@ -114,6 +114,9 @@ public class LoadStmt extends DdlStmt { public static final String KEY_IN_PARAM_SEQUENCE_COL = "sequence_col"; public static final String KEY_IN_PARAM_BACKEND_ID = "backend_id"; + //export + public static final String EXPORT_KEY_IN_PARAM_COLUMNS = "columns"; + private final LabelName label; private final List dataDescriptions; private final BrokerDesc brokerDesc; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index 89513cbcb30619..923fefe4c51409 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -167,6 +167,9 @@ public enum JobState { private OriginStatement origStmt; protected Map sessionVariables = Maps.newHashMap(); + private List exportColumns ; + + public ExportJob() { this.id = -1; this.dbId = -1; @@ -220,7 +223,7 @@ public void setJob(ExportStmt stmt) throws UserException { } this.tableId = exportTable.getId(); this.tableName = stmt.getTblName(); - genExecFragment(); + genExecFragment(stmt); } finally { exportTable.readUnlock(); } @@ -235,8 +238,8 @@ public void setJob(ExportStmt stmt) throws UserException { } } - private void genExecFragment() throws UserException { - registerToDesc(); + private void genExecFragment(ExportStmt stmt) throws UserException { + registerToDesc(stmt); String tmpExportPathStr = getExportPath(); // broker will upload file to tp path and than rename to the final file if (brokerDesc.getStorageType() == StorageBackend.StorageType.BROKER) { @@ -252,17 +255,25 @@ private void genExecFragment() throws UserException { plan(); } - private void registerToDesc() { + private void registerToDesc(ExportStmt stmt) { TableRef ref = new TableRef(tableName, null, partitions == null ? null : new PartitionNames(false, partitions)); BaseTableRef tableRef = new BaseTableRef(ref, exportTable, tableName); exportTupleDesc = desc.createTupleDescriptor(); exportTupleDesc.setTable(exportTable); exportTupleDesc.setRef(tableRef); + this.exportColumns = stmt.getColumns(); for (Column col : exportTable.getBaseSchema()) { - SlotDescriptor slot = desc.addSlotDescriptor(exportTupleDesc); - slot.setIsMaterialized(true); - slot.setColumn(col); - slot.setIsNullable(col.isAllowNull()); + if(!this.exportColumns.isEmpty() && this.exportColumns.contains(col.getName().toLowerCase())) { + SlotDescriptor slot = desc.addSlotDescriptor(exportTupleDesc); + slot.setIsMaterialized(true); + slot.setColumn(col); + slot.setIsNullable(col.isAllowNull()); + } else { + SlotDescriptor slot = desc.addSlotDescriptor(exportTupleDesc); + slot.setIsMaterialized(true); + slot.setColumn(col); + slot.setIsNullable(col.isAllowNull()); + } } desc.computeMemLayout(); } @@ -782,7 +793,7 @@ public void readFields(DataInput in) throws IOException { String value = Text.readString(in); sessionVariables.put(key, value); } - + if (origStmt.originStmt.isEmpty()) { return; } From 4a8ee26362970b22d92228c073a99e49f42ef293 Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Thu, 22 Apr 2021 13:22:18 +0800 Subject: [PATCH 2/9] Modify the data export usage document Modify the data export usage document --- docs/zh-CN/administrator-guide/export-manual.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/zh-CN/administrator-guide/export-manual.md b/docs/zh-CN/administrator-guide/export-manual.md index a3f43be24a0be5..4e739b5b0b0150 100644 --- a/docs/zh-CN/administrator-guide/export-manual.md +++ b/docs/zh-CN/administrator-guide/export-manual.md @@ -111,6 +111,7 @@ TO "hdfs://host/path/to/export/" PROPERTIES ( "column_separator"=",", + "columns":"col1,col2" "exec_mem_limit"="2147483648", "timeout" = "3600" ) @@ -122,6 +123,7 @@ WITH BROKER "hdfs" ``` * `column_separator`：列分隔符。默认为 `\t`。支持不可见字符，比如 '\x07'。 +* columns：要导出的列，使用英文状态逗号隔开，如果不填这个参数默认是导出表的所有列 * `line_delimiter`：行分隔符。默认为 `\n`。支持不可见字符，比如 '\x07'。 * `exec_mem_limit`： 表示 Export 作业中，一个查询计划在单个 BE 上的内存使用限制。默认 2GB。单位字节。 * `timeout`：作业超时时间。默认 2小时。单位秒。 From 87e38f9b6436c6d09cf6f5c1d34f401daaedbb0f Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Thu, 22 Apr 2021 19:33:56 +0800 Subject: [PATCH 3/9] Code format modify Code format modify --- .../org/apache/doris/analysis/ExportStmt.java | 14 +++----------- .../org/apache/doris/analysis/LoadStmt.java | 3 --- .../java/org/apache/doris/load/ExportJob.java | 18 +++++++++++------- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java index efa903c1daeef8..48fc0141f17342 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java @@ -17,7 +17,6 @@ package org.apache.doris.analysis; -import com.google.common.base.Splitter; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.FsBroker; @@ -44,7 +43,6 @@ import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -71,7 +69,7 @@ public class ExportStmt extends StatementBase { private Map properties = Maps.newHashMap(); private String columnSeparator; private String lineDelimiter; - private List columns = new ArrayList<>(); + private String columns ; private TableRef tableRef; @@ -88,7 +86,7 @@ public ExportStmt(TableRef tableRef, Expr whereExpr, String path, this.lineDelimiter = DEFAULT_LINE_DELIMITER; } - public List getColumns() { + public String getColumns() { return columns; } @@ -271,13 +269,7 @@ private void checkProperties(Map properties) throws UserExceptio properties, ExportStmt.DEFAULT_COLUMN_SEPARATOR)); this.lineDelimiter = Separator.convertSeparator(PropertyAnalyzer.analyzeLineDelimiter( properties, ExportStmt.DEFAULT_LINE_DELIMITER)); - if(properties.containsKey(LoadStmt.EXPORT_KEY_IN_PARAM_COLUMNS)){ - String cols = properties.get(LoadStmt.EXPORT_KEY_IN_PARAM_COLUMNS); - if(!Strings.isNullOrEmpty(cols)){ - Splitter split = Splitter.on(',').trimResults().omitEmptyStrings(); - this.columns = split.splitToList(cols.toLowerCase()); - } - } + this.columns = properties.get(LoadStmt.KEY_IN_PARAM_COLUMNS); // exec_mem_limit if (properties.containsKey(LoadStmt.EXEC_MEM_LIMIT)) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java index d097c0b6dd89c8..29156183220e01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java @@ -114,9 +114,6 @@ public class LoadStmt extends DdlStmt { public static final String KEY_IN_PARAM_SEQUENCE_COL = "sequence_col"; public static final String KEY_IN_PARAM_BACKEND_ID = "backend_id"; - //export - public static final String EXPORT_KEY_IN_PARAM_COLUMNS = "columns"; - private final LabelName label; private final List dataDescriptions; private final BrokerDesc brokerDesc; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index 923fefe4c51409..04b43d1e79dfe7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -17,6 +17,7 @@ package org.apache.doris.load; +import com.google.common.base.Splitter; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.BaseTableRef; import org.apache.doris.analysis.BrokerDesc; @@ -46,7 +47,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeMetaVersion; -import org.apache.doris.common.Pair; import org.apache.doris.common.Status; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; @@ -215,6 +215,10 @@ public void setJob(ExportStmt stmt) throws UserException { this.exportTable = db.getTable(stmt.getTblName().getTbl()); + if (!Strings.isNullOrEmpty(stmt.getColumns())) { + Splitter split = Splitter.on(',').trimResults().omitEmptyStrings(); + this.exportColumns = split.splitToList(stmt.getColumns().toLowerCase()); + } exportTable.readLock(); try { this.dbId = db.getId(); @@ -223,7 +227,7 @@ public void setJob(ExportStmt stmt) throws UserException { } this.tableId = exportTable.getId(); this.tableName = stmt.getTblName(); - genExecFragment(stmt); + genExecFragment(); } finally { exportTable.readUnlock(); } @@ -238,8 +242,8 @@ public void setJob(ExportStmt stmt) throws UserException { } } - private void genExecFragment(ExportStmt stmt) throws UserException { - registerToDesc(stmt); + private void genExecFragment() throws UserException { + registerToDesc(); String tmpExportPathStr = getExportPath(); // broker will upload file to tp path and than rename to the final file if (brokerDesc.getStorageType() == StorageBackend.StorageType.BROKER) { @@ -255,15 +259,15 @@ private void genExecFragment(ExportStmt stmt) throws UserException { plan(); } - private void registerToDesc(ExportStmt stmt) { + private void registerToDesc() { TableRef ref = new TableRef(tableName, null, partitions == null ? null : new PartitionNames(false, partitions)); BaseTableRef tableRef = new BaseTableRef(ref, exportTable, tableName); exportTupleDesc = desc.createTupleDescriptor(); exportTupleDesc.setTable(exportTable); exportTupleDesc.setRef(tableRef); - this.exportColumns = stmt.getColumns(); for (Column col : exportTable.getBaseSchema()) { - if(!this.exportColumns.isEmpty() && this.exportColumns.contains(col.getName().toLowerCase())) { + String colName = col.getName().toLowerCase(); + if (!this.exportColumns.isEmpty() && this.exportColumns.contains(colName)) { SlotDescriptor slot = desc.addSlotDescriptor(exportTupleDesc); slot.setIsMaterialized(true); slot.setColumn(col); From 6c738081707cd0d6d811810d6aacf7c6a78fad86 Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Thu, 22 Apr 2021 19:40:14 +0800 Subject: [PATCH 4/9] There may be a null problem in the modification There may be a null problem in the modification --- .../src/main/java/org/apache/doris/load/ExportJob.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index 04b43d1e79dfe7..193878572e61ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -93,11 +93,7 @@ import java.io.StringReader; import java.net.URI; import java.net.URISyntaxException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; // NOTE: we must be carefully if we send next request @@ -167,7 +163,7 @@ public enum JobState { private OriginStatement origStmt; protected Map sessionVariables = Maps.newHashMap(); - private List exportColumns ; + private List exportColumns = Lists.newArrayList(); public ExportJob() { From f0158a9f2d2f61aae3e6b063c59960b9ed68d0ef Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Fri, 23 Apr 2021 09:05:44 +0800 Subject: [PATCH 5/9] columns persistence columns persistence --- .../main/java/org/apache/doris/load/ExportJob.java | 14 +++++++++++--- .../main/java/org/apache/doris/load/ExportMgr.java | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index 193878572e61ed..622b9602c89888 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -17,7 +17,6 @@ package org.apache.doris.load; -import com.google.common.base.Splitter; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.BaseTableRef; import org.apache.doris.analysis.BrokerDesc; @@ -82,6 +81,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.common.base.Splitter; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; @@ -164,6 +164,7 @@ public enum JobState { protected Map sessionVariables = Maps.newHashMap(); private List exportColumns = Lists.newArrayList(); + private String columns ; public ExportJob() { @@ -181,6 +182,7 @@ public ExportJob() { this.exportPath = ""; this.columnSeparator = "\t"; this.lineDelimiter = "\n"; + this.columns = ""; } public ExportJob(long jobId) { @@ -210,8 +212,8 @@ public void setJob(ExportStmt stmt) throws UserException { this.partitions = stmt.getPartitions(); this.exportTable = db.getTable(stmt.getTblName().getTbl()); - - if (!Strings.isNullOrEmpty(stmt.getColumns())) { + this.columns = stmt.getColumns(); + if (!Strings.isNullOrEmpty(this.columns)) { Splitter split = Splitter.on(',').trimResults().omitEmptyStrings(); this.exportColumns = split.splitToList(stmt.getColumns().toLowerCase()); } @@ -458,6 +460,10 @@ private void genCoordinators(List fragments, List nodes) LOG.info("create {} coordinators for export job: {}", coordList.size(), id); } + public String getColumns() { + return columns; + } + public long getId() { return id; } @@ -691,6 +697,7 @@ public void write(DataOutput out) throws IOException { Text.writeString(out, exportPath); Text.writeString(out, columnSeparator); Text.writeString(out, lineDelimiter); + Text.writeString(out, columns); out.writeInt(properties.size()); for (Map.Entry property : properties.entrySet()) { Text.writeString(out, property.getKey()); @@ -742,6 +749,7 @@ public void readFields(DataInput in) throws IOException { exportPath = Text.readString(in); columnSeparator = Text.readString(in); lineDelimiter = Text.readString(in); + columns = Text.readString(in); if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_53) { int count = in.readInt(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java index efb90f35fbd078..87d18071d7ab50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java @@ -193,6 +193,7 @@ public List> getExportJobInfosByIdOrState( infoMap.put("column separator", job.getColumnSeparator()); infoMap.put("line delimiter", job.getLineDelimiter()); infoMap.put("exec mem limit", job.getExecMemLimit()); + infoMap.put("columns",job.getColumns()); infoMap.put("coord num", job.getCoordList().size()); infoMap.put("tablet num", job.getTabletLocations() == null ? -1 : job.getTabletLocations().size()); jobInfo.add(new Gson().toJson(infoMap)); From 62f70a7fddba737bf34a29543e1a34b458a38db5 Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Fri, 23 Apr 2021 17:07:03 +0800 Subject: [PATCH 6/9] modify columns Persistence modify columns Persistence --- .../main/java/org/apache/doris/analysis/ExportStmt.java | 3 +++ .../src/main/java/org/apache/doris/load/ExportJob.java | 9 ++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java index 48fc0141f17342..66b17b703df1dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java @@ -60,6 +60,8 @@ public class ExportStmt extends StatementBase { private static final String DEFAULT_COLUMN_SEPARATOR = "\t"; private static final String DEFAULT_LINE_DELIMITER = "\n"; + private static final String DEFAULT_COLUMNS = ""; + private TableName tblName; private List partitions; @@ -84,6 +86,7 @@ public ExportStmt(TableRef tableRef, Expr whereExpr, String path, this.brokerDesc = brokerDesc; this.columnSeparator = DEFAULT_COLUMN_SEPARATOR; this.lineDelimiter = DEFAULT_LINE_DELIMITER; + this.columns = DEFAULT_COLUMNS; } public String getColumns() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index 622b9602c89888..35faf365111529 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -43,11 +43,7 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; -import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.FeMetaVersion; -import org.apache.doris.common.Status; -import org.apache.doris.common.UserException; +import org.apache.doris.common.*; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.SqlParserUtils; @@ -697,7 +693,6 @@ public void write(DataOutput out) throws IOException { Text.writeString(out, exportPath); Text.writeString(out, columnSeparator); Text.writeString(out, lineDelimiter); - Text.writeString(out, columns); out.writeInt(properties.size()); for (Map.Entry property : properties.entrySet()) { Text.writeString(out, property.getKey()); @@ -749,7 +744,6 @@ public void readFields(DataInput in) throws IOException { exportPath = Text.readString(in); columnSeparator = Text.readString(in); lineDelimiter = Text.readString(in); - columns = Text.readString(in); if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_53) { int count = in.readInt(); @@ -759,6 +753,7 @@ public void readFields(DataInput in) throws IOException { this.properties.put(propertyKey, propertyValue); } } + this.columns = this.properties.get(LoadStmt.KEY_IN_PARAM_COLUMNS); boolean hasPartition = in.readBoolean(); if (hasPartition) { From 7e12a831c2f092ffc8b7f92acd9652e998795cd0 Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Fri, 23 Apr 2021 17:14:55 +0800 Subject: [PATCH 7/9] code style code style --- .../src/main/java/org/apache/doris/load/ExportJob.java | 6 +++++- .../src/main/java/org/apache/doris/load/ExportMgr.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index 35faf365111529..25870d0808a17e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -89,7 +89,11 @@ import java.io.StringReader; import java.net.URI; import java.net.URISyntaxException; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; // NOTE: we must be carefully if we send next request diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java index 87d18071d7ab50..577aeae5646090 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java @@ -193,7 +193,7 @@ public List> getExportJobInfosByIdOrState( infoMap.put("column separator", job.getColumnSeparator()); infoMap.put("line delimiter", job.getLineDelimiter()); infoMap.put("exec mem limit", job.getExecMemLimit()); - infoMap.put("columns",job.getColumns()); + infoMap.put("columns", job.getColumns()); infoMap.put("coord num", job.getCoordList().size()); infoMap.put("tablet num", job.getTabletLocations() == null ? -1 : job.getTabletLocations().size()); jobInfo.add(new Gson().toJson(infoMap)); From fa978654e01ce3c96dd2a3f04cdb958e2a2f99c4 Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Fri, 23 Apr 2021 17:39:08 +0800 Subject: [PATCH 8/9] remove import * remove import * --- .../src/main/java/org/apache/doris/load/ExportJob.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index 25870d0808a17e..c730cb3ffca912 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -43,7 +43,12 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; -import org.apache.doris.common.*; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.Pair; +import org.apache.doris.common.Status; +import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.SqlParserUtils; From f5afa19b19a5676794eafe5e282144af9a736331 Mon Sep 17 00:00:00 2001 From: "jiafeng.zhang" Date: Mon, 26 Apr 2021 11:15:09 +0800 Subject: [PATCH 9/9] modify readFields exportColumns modify readFields exportColumns --- .../src/main/java/org/apache/doris/load/ExportJob.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index c730cb3ffca912..2150023df84122 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -763,7 +763,10 @@ public void readFields(DataInput in) throws IOException { } } this.columns = this.properties.get(LoadStmt.KEY_IN_PARAM_COLUMNS); - + if (!Strings.isNullOrEmpty(this.columns)) { + Splitter split = Splitter.on(',').trimResults().omitEmptyStrings(); + this.exportColumns = split.splitToList(this.columns); + } boolean hasPartition = in.readBoolean(); if (hasPartition) { partitions = Lists.newArrayList();