From b01e62169f3801126358b0148b6ff280aa2217b4 Mon Sep 17 00:00:00 2001 From: EmmyMiao87 <522274284@qq.com> Date: Fri, 7 May 2021 12:45:38 +0800 Subject: [PATCH] [Bug-fix] Export specified column The code logic error causes the user to specify the export column, which may not be effective. The PR fix this problem. --- docs/en/administrator-guide/export-manual.md | 1 + .../Data Manipulation/EXPORT.md | 4 +++ .../Data Manipulation/EXPORT.md | 4 +++ .../org/apache/doris/analysis/ExportStmt.java | 2 +- .../java/org/apache/doris/load/ExportJob.java | 28 +++++++++++-------- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/docs/en/administrator-guide/export-manual.md b/docs/en/administrator-guide/export-manual.md index 0a044325149cdd..b25b8643ed1174 100644 --- a/docs/en/administrator-guide/export-manual.md +++ b/docs/en/administrator-guide/export-manual.md @@ -118,6 +118,7 @@ WITH BROKER "hdfs" ``` * `column_separator`: Column separator. The default is `\t`. Supports invisible characters, such as'\x07'. +* `column`: columns to be exported, separated by commas, if this parameter is not filled in, all columns of the table will be exported by default. * `line_delimiter`: Line separator. The default is `\n`. Supports invisible characters, such as'\x07'. * `exec_mem_limit`: Represents the memory usage limitation of a query plan on a single BE in an Export job. Default 2GB. Unit bytes. * `timeout`: homework timeout. Default 2 hours. Unit seconds. diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/EXPORT.md b/docs/en/sql-reference/sql-statements/Data Manipulation/EXPORT.md index 0038827968698a..bec28aa214d180 100644 --- a/docs/en/sql-reference/sql-statements/Data Manipulation/EXPORT.md +++ b/docs/en/sql-reference/sql-statements/Data Manipulation/EXPORT.md @@ -58,6 +58,7 @@ under the License. The following parameters can be specified: column_separator: Specifies the exported column separator, defaulting to t. Supports invisible characters, such as'\x07'. + column: Specify the columns to be exported, separated by commas. If you do not fill in this parameter, the default is to export all the columns of the table. line_delimiter: Specifies the exported line separator, defaulting to\n. Supports invisible characters, such as'\x07'. exec_mem_limit: Exports the upper limit of memory usage for a single BE node, defaulting to 2GB in bytes. timeout: The time-out for importing jobs is 1 day by default, in seconds. @@ -92,5 +93,8 @@ under the License. 6. Export all data in the testTbl table to hdfs, using the invisible character "\x07" as the column and row separator. EXPORT TABLE testTbl TO "hdfs://hdfs_host:port/a/b/c" PROPERTIES ("column_separator"="\\x07", "line_delimiter" = "\\x07") WITH BROKER "broker_name" ("username"="xxx", "password"="yyy") + 7. Export column k1, v1 from the testTbl to the local. + EXPORT TABLE testTbl TO "file:///home/data/a" PROPERTIES ("columns" = "k1,v1"); + ## keyword EXPORT diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/EXPORT.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/EXPORT.md index e1a08c91daee18..adf68455943c05 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/EXPORT.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/EXPORT.md @@ -58,6 +58,7 @@ under the License. 可以指定如下参数： column_separator: 指定导出的列分隔符，默认为\t。支持不可见字符，比如 '\x07'。 + column: 指定待导出的列，使用英文逗号隔开，如果不填这个参数默认是导出表的所有列。 line_delimiter: 指定导出的行分隔符，默认为\n。支持不可见字符，比如 '\x07'。 exec_mem_limit: 导出在单个 BE 节点的内存使用上限，默认为 2GB，单位为字节。 timeout：导入作业的超时时间，默认为1天，单位是秒。 @@ -91,6 +92,9 @@ under the License. 6. 将 testTbl 表中的所有数据导出到 hdfs 上，以不可见字符 "\x07" 作为列或者行分隔符。 EXPORT TABLE testTbl TO "hdfs://hdfs_host:port/a/b/c" PROPERTIES ("column_separator"="\\x07", "line_delimiter" = "\\x07") WITH BROKER "broker_name" ("username"="xxx", "password"="yyy") + + 7. 将 testTbl 表的 k1, v1 列导出到本地。 + EXPORT TABLE testTbl TO "file:///home/data/a" PROPERTIES ("columns" = "k1,v1"); ## keyword EXPORT diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java index 66b17b703df1dc..0810b44be6730a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java @@ -134,7 +134,7 @@ public boolean needAuditEncryption() { } @Override - public void analyze(Analyzer analyzer) throws AnalysisException, UserException { + public void analyze(Analyzer analyzer) throws UserException { super.analyze(analyzer); tableRef = analyzer.resolveTableRef(tableRef); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index 2150023df84122..1946c03bac5950 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -78,11 +78,11 @@ import org.apache.doris.thrift.TUniqueId; import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.google.common.base.Splitter; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; @@ -268,18 +268,22 @@ private void registerToDesc() { exportTupleDesc = desc.createTupleDescriptor(); exportTupleDesc.setTable(exportTable); exportTupleDesc.setRef(tableRef); - for (Column col : exportTable.getBaseSchema()) { - String colName = col.getName().toLowerCase(); - if (!this.exportColumns.isEmpty() && this.exportColumns.contains(colName)) { - SlotDescriptor slot = desc.addSlotDescriptor(exportTupleDesc); - slot.setIsMaterialized(true); - slot.setColumn(col); - slot.setIsNullable(col.isAllowNull()); - } else { + if (exportColumns.isEmpty()) { + for (Column column : exportTable.getBaseSchema()) { SlotDescriptor slot = desc.addSlotDescriptor(exportTupleDesc); slot.setIsMaterialized(true); - slot.setColumn(col); - slot.setIsNullable(col.isAllowNull()); + slot.setColumn(column); + slot.setIsNullable(column.isAllowNull()); + } + } else { + for (Column column : exportTable.getBaseSchema()) { + String colName = column.getName().toLowerCase(); + if (exportColumns.contains(colName)) { + SlotDescriptor slot = desc.addSlotDescriptor(exportTupleDesc); + slot.setIsMaterialized(true); + slot.setColumn(column); + slot.setIsNullable(column.isAllowNull()); + } } } desc.computeMemLayout(); @@ -765,7 +769,7 @@ public void readFields(DataInput in) throws IOException { this.columns = this.properties.get(LoadStmt.KEY_IN_PARAM_COLUMNS); if (!Strings.isNullOrEmpty(this.columns)) { Splitter split = Splitter.on(',').trimResults().omitEmptyStrings(); - this.exportColumns = split.splitToList(this.columns); + this.exportColumns = split.splitToList(this.columns.toLowerCase()); } boolean hasPartition = in.readBoolean(); if (hasPartition) {