From 29c61dcdbb512bad836148a913609b738d10d610 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 24 Sep 2020 20:38:22 +0800 Subject: [PATCH 1/4] first --- fe/fe-core/src/main/cup/sql_parser.cup | 4 + .../java/org/apache/doris/alter/Alter.java | 61 ++++++++++++++ .../org/apache/doris/alter/AlterOpType.java | 6 +- .../apache/doris/alter/AlterOperations.java | 6 ++ .../doris/analysis/ReplaceTableClause.java | 84 +++++++++++++++++++ .../org/apache/doris/catalog/Catalog.java | 12 +-- .../org/apache/doris/catalog/OlapTable.java | 12 +++ .../doris/common/util/PropertyAnalyzer.java | 2 + 8 files changed, 175 insertions(+), 12 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ReplaceTableClause.java diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 73b0b47d5bc47d..02f7fa87c63885 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -953,6 +953,10 @@ alter_table_clause ::= {: RESULT = new ReplacePartitionClause(partitions, tempPartitions, properties); :} + | KW_REPLACE KW_WITH KW_TABLE ident:tblName opt_properties:properties + {: + RESULT = new ReplaceTableClause(tblName, properties); + :} | KW_RENAME ident:newTableName {: RESULT = new TableRenameClause(newTableName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 803ffbea7f3ed3..7d06c31c388c41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -30,6 +30,7 @@ import org.apache.doris.analysis.ModifyTablePropertiesClause; import org.apache.doris.analysis.PartitionRenameClause; import org.apache.doris.analysis.ReplacePartitionClause; +import org.apache.doris.analysis.ReplaceTableClause; import org.apache.doris.analysis.RollupRenameClause; import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRenameClause; @@ -57,6 +58,7 @@ import org.apache.doris.persist.ModifyPartitionInfo; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TTabletType; + import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -233,6 +235,8 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException { } } else if (currentAlterOps.hasRenameOp()) { processRename(db, olapTable, alterClauses); + } else if (currentAlterOps.hasReplaceTableOp()) { + processReplaceTable(db, olapTable, alterClauses); } else if (currentAlterOps.contains(AlterOpType.MODIFY_TABLE_PROPERTY_SYNC)) { needProcessOutsideDatabaseLock = true; } else { @@ -278,6 +282,63 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException { } } + /** + * The replace table operation works as follow: + * For example, REPLACE TABLE A WITH TABLE B. + * + * 1. If "swapTable" is true, A will be renamed to B, and B will be renamed to A + * 1.1 check if A can be renamed to B (checking name conflict, etc...) + * 1.2 check if B can be renamed to A (checking name conflict, etc...) + * 1.3 rename B to A, drop old A, and add new A to database. + * 1.4 rename A to B, drop old B, and add new B to database. + * + * 2. If "swapTable" is false, A will be dropped, and B will be renamed to A + * 1.1 check if B can be renamed to A (checking name conflict, etc...) + * 1.2 rename B to A, drop old A, and add new A to database. + * + * @param db + * @param origTable + * @param alterClauses + * @throws UserException + */ + private void processReplaceTable(Database db, OlapTable origTable, List alterClauses) throws UserException { + ReplaceTableClause clause = (ReplaceTableClause) alterClauses.get(0); + Preconditions.checkState(db.isWriteLockHeldByCurrentThread()); + + String oldTblName = origTable.getName(); + String newTblName = clause.getTblName(); + Table newTbl = db.getTable(newTblName); + if (newTbl == null || newTbl.getType() != TableType.OLAP) { + throw new DdlException("Table " + newTblName + " does not exist or is not OLAP table"); + } + OlapTable olapNewTbl = (OlapTable) newTbl; + + boolean swapTable = clause.isSwapTable(); + + // First, we need to check whether the table to be operated on can be renamed + olapNewTbl.checkAndSetName(oldTblName, true); + if (swapTable) { + origTable.checkAndSetName(newTblName, true); + } + + // "rename B to A" + olapNewTbl.checkAndSetName(oldTblName, false); + // "drop old A, and add new A to database." + db.dropTable(oldTblName); + db.dropTable(newTblName); + db.createTable(olapNewTbl); + + if (swapTable) { + // "rename A to B, drop old B, and add new B to database." + origTable.checkAndSetName(newTblName, false); + db.createTable(origTable); + } + + // TODO(cmy): add edit log + + LOG.info("finish replacing table {} with table {}, is swap: {}", oldTblName, newTblName, swapTable); + } + public void processAlterView(AlterViewStmt stmt, ConnectContext ctx) throws UserException { TableName dbTableName = stmt.getTbl(); String dbName = dbTableName.getDb(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java index fbeedd10e4525b..bbf5f43fe3d218 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java @@ -18,6 +18,7 @@ package org.apache.doris.alter; public enum AlterOpType { + INVALID_OP, // rollup ADD_ROLLUP, DROP_ROLLUP, @@ -35,9 +36,8 @@ public enum AlterOpType { MODIFY_TABLE_PROPERTY_SYNC, // Some operations are performed synchronously, so we distinguish them by suffix _SYNC // others operation, such as add/drop backend. currently we do not care about them ALTER_OTHER, - - INVALID_OP, - ENABLE_FEATURE; + ENABLE_FEATURE, + REPLACE_TABLE; // true means 2 operations have no conflict. diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOperations.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOperations.java index ad3f4d054bead7..b2ef49807f50c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOperations.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOperations.java @@ -76,6 +76,10 @@ public boolean hasRenameOp() { return currentOps.contains(AlterOpType.RENAME); } + public boolean hasReplaceTableOp() { + return currentOps.contains(AlterOpType.REPLACE_TABLE); + } + public boolean contains(AlterOpType op) { return currentOps.contains(op); } @@ -103,4 +107,6 @@ public boolean hasEnableFeatureOP() { public String toString() { return Joiner.on(", ").join(currentOps); } + + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ReplaceTableClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ReplaceTableClause.java new file mode 100644 index 00000000000000..051b2f623e4f32 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ReplaceTableClause.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.alter.AlterOpType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.PropertyAnalyzer; + +import org.apache.parquet.Strings; + +import java.util.Map; + +// clause which is used to replace table +// eg: +// ALTER TABLE tbl REPLACE WITH TABLE tbl2; +public class ReplaceTableClause extends AlterTableClause { + private String tblName; + private Map properties; + + // parsed from properties. + // if false, after replace, there will be only one table exist with. + // if true, the new table and the old table will be exchanged. + // default is true. + private boolean swapTable; + + public ReplaceTableClause(String tblName, Map properties) { + super(AlterOpType.REPLACE_PARTITION); + this.tblName = tblName; + this.properties = properties; + } + + public String getTblName() { + return tblName; + } + + public boolean isSwapTable() { + return swapTable; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException { + if (Strings.isNullOrEmpty(tblName)) { + throw new AnalysisException("No table specified"); + } + + this.swapTable = PropertyAnalyzer.analyzeBooleanProp(properties, PropertyAnalyzer.PROPERTIES_SWAP_TABLE, true); + + if (properties != null && !properties.isEmpty()) { + throw new AnalysisException("Unknown properties: " + properties.keySet()); + } + } + + @Override + public Map getProperties() { + return this.properties; + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder(); + sb.append("REPLACE WITH TABLE ").append(tblName); + return sb.toString(); + } + + @Override + public String toString() { + return toSql(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 8b47b1161171d3..cfb9f48a6d2673 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -5012,6 +5012,7 @@ public void cancelBackup(CancelBackupStmt stmt) throws DdlException { getBackupHandler().cancel(stmt); } + // entry of rename table operation public void renameTable(Database db, OlapTable table, TableRenameClause tableRenameClause) throws DdlException { if (table.getState() != OlapTableState.NORMAL) { throw new DdlException("Table[" + table.getName() + "] is under " + table.getState()); @@ -5028,16 +5029,9 @@ public void renameTable(Database db, OlapTable table, TableRenameClause tableRen throw new DdlException("Table name[" + newTableName + "] is already used"); } - // check if rollup has same name - for (String idxName : table.getIndexNameToId().keySet()) { - if (idxName.equals(newTableName)) { - throw new DdlException("New name conflicts with rollup index name: " + idxName); - } - } + table.checkAndSetName(newTableName, false); - table.setName(newTableName); - - db.dropTable(tableName); + db.dropTable(table.getName()); db.createTable(table); TableInfo tableInfo = TableInfo.createForTableRename(db.getId(), table.getId(), newTableName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index e30e896946b55d..2a3a201ef66305 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -231,6 +231,18 @@ public Map getIndexesMap() { return indexMap; } + public void checkAndSetName(String newName, boolean onlyCheck) throws DdlException { + // check if rollup has same name + for (String idxName : getIndexNameToId().keySet()) { + if (idxName.equals(newName)) { + throw new DdlException("New name conflicts with rollup index name: " + idxName); + } + } + if (!onlyCheck) { + setName(newName); + } + } + public void setName(String newName) { // change name in indexNameToId long baseIndexId = indexNameToId.remove(this.name); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 184c336616bc64..cdffd5fb34b0d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -92,6 +92,8 @@ public class PropertyAnalyzer { public static final String PROPERTIES_FUNCTION_COLUMN = "function_column"; public static final String PROPERTIES_SEQUENCE_TYPE = "sequence_type"; + public static final String PROPERTIES_SWAP_TABLE = "swap"; + public static DataProperty analyzeDataProperty(Map properties, DataProperty oldDataProperty) throws AnalysisException { if (properties == null) { From 2d5d78b3008101a64543d866dd61c4f921875088 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 24 Sep 2020 22:20:29 +0800 Subject: [PATCH 2/4] second --- .../java/org/apache/doris/alter/Alter.java | 86 ++++++++++++------- .../org/apache/doris/persist/EditLog.java | 9 ++ .../apache/doris/persist/OperationType.java | 1 + 3 files changed, 65 insertions(+), 31 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 7d06c31c388c41..8a0abc851264de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -56,16 +56,17 @@ import org.apache.doris.persist.AlterViewInfo; import org.apache.doris.persist.BatchModifyPartitionsInfo; import org.apache.doris.persist.ModifyPartitionInfo; +import org.apache.doris.persist.ReplaceTableOperationLog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TTabletType; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.util.Arrays; import java.util.List; import java.util.Map; @@ -282,25 +283,7 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException { } } - /** - * The replace table operation works as follow: - * For example, REPLACE TABLE A WITH TABLE B. - * - * 1. If "swapTable" is true, A will be renamed to B, and B will be renamed to A - * 1.1 check if A can be renamed to B (checking name conflict, etc...) - * 1.2 check if B can be renamed to A (checking name conflict, etc...) - * 1.3 rename B to A, drop old A, and add new A to database. - * 1.4 rename A to B, drop old B, and add new B to database. - * - * 2. If "swapTable" is false, A will be dropped, and B will be renamed to A - * 1.1 check if B can be renamed to A (checking name conflict, etc...) - * 1.2 rename B to A, drop old A, and add new A to database. - * - * @param db - * @param origTable - * @param alterClauses - * @throws UserException - */ + // entry of processing replace table private void processReplaceTable(Database db, OlapTable origTable, List alterClauses) throws UserException { ReplaceTableClause clause = (ReplaceTableClause) alterClauses.get(0); Preconditions.checkState(db.isWriteLockHeldByCurrentThread()); @@ -321,22 +304,63 @@ private void processReplaceTable(Database db, OlapTable origTable, List + * 1. If "swapTable" is true, A will be renamed to B, and B will be renamed to A + * 1.1 check if A can be renamed to B (checking name conflict, etc...) + * 1.2 check if B can be renamed to A (checking name conflict, etc...) + * 1.3 rename B to A, drop old A, and add new A to database. + * 1.4 rename A to B, drop old B, and add new B to database. + *

+ * 2. If "swapTable" is false, A will be dropped, and B will be renamed to A + * 1.1 check if B can be renamed to A (checking name conflict, etc...) + * 1.2 rename B to A, drop old A, and add new A to database. + */ + private void replaceTableInternal(Database db, OlapTable origTable, OlapTable newTbl, boolean swapTable) + throws DdlException { + String oldTblName = origTable.getName(); + String newTblName = newTbl.getName(); + + // drop origin table and new table db.dropTable(oldTblName); db.dropTable(newTblName); - db.createTable(olapNewTbl); + + // rename new table name to origin table name and add it to database + newTbl.checkAndSetName(oldTblName, false); + db.createTable(newTbl); if (swapTable) { - // "rename A to B, drop old B, and add new B to database." + // rename origin table name to new table name and add it to database origTable.checkAndSetName(newTblName, false); db.createTable(origTable); } - - // TODO(cmy): add edit log - - LOG.info("finish replacing table {} with table {}, is swap: {}", oldTblName, newTblName, swapTable); } public void processAlterView(AlterViewStmt stmt, ConnectContext ctx) throws UserException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 46042f9c305867..df32a4601b1f57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -794,6 +794,11 @@ public static void loadJournal(Catalog catalog, JournalEntity journal) { catalog.replayGlobalVariableV2(info); break; } + case OperationType.OP_REPLACE_TABLE: { + ReplaceTableOperationLog log = (ReplaceTableOperationLog) journal.getData(); + catalog.getAlterInstance().replayReplaceTable(log); + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); @@ -1364,4 +1369,8 @@ public void logAlterRoutineLoadJob(AlterRoutineLoadJobOperationLog log) { public void logGlobalVariableV2(GlobalVarPersistInfo info) { logEdit(OperationType.OP_GLOBAL_VARIABLE_V2, info); } + + public void logReplaceTable(ReplaceTableOperationLog log) { + logEdit(OperationType.OP_REPLACE_TABLE, log); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index c7ac919954fb14..0d30edcd9f2c26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -45,6 +45,7 @@ public class OperationType { public static final short OP_MODIFY_VIEW_DEF = 119; public static final short OP_REPLACE_TEMP_PARTITION = 210; public static final short OP_BATCH_MODIFY_PARTITION = 211; + public static final short OP_REPLACE_TABLE = 212; // 20~29 120~129 220~229 ... public static final short OP_START_ROLLUP = 20; From f8e2a744257ac728898bada4d435416a31996675 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 25 Sep 2020 00:00:40 +0800 Subject: [PATCH 3/4] add doc --- docs/.vuepress/sidebar/en.js | 1 + docs/.vuepress/sidebar/zh-CN.js | 1 + .../alter-table/alter-table-replace-table.md | 72 ++++++++++++++ .../alter-table/alter-table-temp-partition.md | 2 +- .../alter-table/alter-table-replace-table.md | 73 ++++++++++++++ .../alter-table/alter-table-temp-partition.md | 2 +- .../org/apache/doris/alter/AlterOpType.java | 5 +- .../doris/analysis/ReplaceTableClause.java | 2 +- .../persist/ReplaceTableOperationLog.java | 56 +++++++++++ .../org/apache/doris/alter/AlterTest.java | 99 ++++++++++++++++++- .../persist/ReplaceTableOperationLogTest.java | 57 +++++++++++ 11 files changed, 362 insertions(+), 8 deletions(-) create mode 100644 docs/en/administrator-guide/alter-table/alter-table-replace-table.md create mode 100644 docs/zh-CN/administrator-guide/alter-table/alter-table-replace-table.md create mode 100644 fe/fe-core/src/main/java/org/apache/doris/persist/ReplaceTableOperationLog.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/persist/ReplaceTableOperationLogTest.java diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js index 0d07657e08b09c..ae0e9ed309c107 100644 --- a/docs/.vuepress/sidebar/en.js +++ b/docs/.vuepress/sidebar/en.js @@ -67,6 +67,7 @@ module.exports = [ "alter-table-rollup", "alter-table-schema-change", "alter-table-temp-partition", + "alter-table-replace-table", ], sidebarDepth: 2, }, diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js index 2d694e79873f36..cc502963cb1ace 100644 --- a/docs/.vuepress/sidebar/zh-CN.js +++ b/docs/.vuepress/sidebar/zh-CN.js @@ -68,6 +68,7 @@ module.exports = [ "alter-table-rollup", "alter-table-schema-change", "alter-table-temp-partition", + "alter-table-replace-table", ], sidebarDepth: 2, }, diff --git a/docs/en/administrator-guide/alter-table/alter-table-replace-table.md b/docs/en/administrator-guide/alter-table/alter-table-replace-table.md new file mode 100644 index 00000000000000..c7d644585d0d4d --- /dev/null +++ b/docs/en/administrator-guide/alter-table/alter-table-replace-table.md @@ -0,0 +1,72 @@ +--- +{ + "title": "Replace Table", + "language": "en" +} +--- + + + +# Replace Table + +In version 0.14, Doris supports atomic replacement of two tables. +This operation only applies to OLAP tables. + +For partition level replacement operations, please refer to [Temporary Partition Document](./alter-table-temp-partition.md) + +## Syntax + +``` +ALTER TABLE [db.]tbl1 REPLACE WITH tbl2 +[PROPERTIES('swap' = 'true')]; +``` + +Replace table `tbl1` with table `tbl2`. + +If the `swap` parameter is `true`, after replacement, the data in the table named `tbl1` is the data in the original `tbl2` table. The data in the table named `tbl2` is the data in the original table `tbl1`. That is, the data of the two tables are interchanged. + +If the `swap` parameter is `true`, after replacement, the data in the table named `tbl1` is the data in the original `tbl2` table. The table named `tbl2` is dropped. + +## Principle + +The replacement table function actually turns the following set of operations into an atomic operation. + +Suppose you want to replace table A with table B, and `swap` is `true`, the operation is as follows: + +1. Rename table B to table A. +2. Rename table A to table B. + +If `swap` is `true`, the operation is as follows: + +1. Drop table A. +2. Rename table B to table A. + +## Notice + +1. The `swap` parameter defaults to `true`. That is, the replacement table operation is equivalent to the exchange of two table data. +2. If the `swap` parameter is set to `false`, the replaced table (table A) will be dropped and cannot be recovered. +3. The replacement operation can only occur between two OLAP tables, and the table structure of the two tables is not checked for consistency. +4. The replacement operation will not change the original permission settings. Because the permission check is based on the table name. + +## Best Practices + +1. Atomic Overwrite Operation + + In some cases, the user wants to be able to rewrite the data of a certain table, but if it is dropped and then imported, there will be a period of time in which the data cannot be viewed. At this time, the user can first use the `CREATE TABLE LIKE` statement to create a new table with the same structure, import the new data into the new table, and replace the old table atomically through the replacement operation to achieve the goal. For partition level atomic overwrite operation, please refer to [Temporary partition document](./alter-table-temp-partition.md) diff --git a/docs/en/administrator-guide/alter-table/alter-table-temp-partition.md b/docs/en/administrator-guide/alter-table/alter-table-temp-partition.md index 0c363c9700273e..412df27b9401c8 100644 --- a/docs/en/administrator-guide/alter-table/alter-table-temp-partition.md +++ b/docs/en/administrator-guide/alter-table/alter-table-temp-partition.md @@ -230,7 +230,7 @@ Users can load data into temporary partitions or specify temporary partitions fo 1. Atomic overwrite - In some cases, the user wants to be able to rewrite the data of a certain partition, but if it is dropped first and then loaded, there will be a period of time when the data cannot be seen. This is, the user can first create a corresponding temporary partition, load new data into the temporary partition, and then replace the original partition atomically through the `REPLACE` operation to achieve the purpose. + In some cases, the user wants to be able to rewrite the data of a certain partition, but if it is dropped first and then loaded, there will be a period of time when the data cannot be seen. At this moment, the user can first create a corresponding temporary partition, load new data into the temporary partition, and then replace the original partition atomically through the `REPLACE` operation to achieve the purpose. For atomic overwrite operations of non-partitioned tables, please refer to [Replace Table Document](./alter-table-replace-table.md)      2. Modify the number of buckets diff --git a/docs/zh-CN/administrator-guide/alter-table/alter-table-replace-table.md b/docs/zh-CN/administrator-guide/alter-table/alter-table-replace-table.md new file mode 100644 index 00000000000000..4188dc152d4bea --- /dev/null +++ b/docs/zh-CN/administrator-guide/alter-table/alter-table-replace-table.md @@ -0,0 +1,73 @@ +--- +{ + "title": "替换表", + "language": "zh-CN" +} +--- + + + +# 替换表 + +在 0.14 版本中，Doris 支持对两个表进行原子的替换操作。 +该操作仅适用于 OLAP 表。 + +分区级别的替换操作，请参阅 [临时分区文档](./alter-table-temp-partition.md) + +## 语法说明 + +``` +ALTER TABLE [db.]tbl1 REPLACE WITH tbl2 +[PROPERTIES('swap' = 'true')]; +``` + +将表 tbl1 替换为表 tbl2。 + +如果 `swap` 参数为 `true`，则替换后，名称为 `tbl1` 表中的数据为原 `tbl2` 表中的数据。而名称为 `tbl2` 表中的数据为原 `tbl1` 表中的数据。即两张表数据发生了互换。 + +如果 `swap` 参数为 `true`，则替换后，名称为 `tbl1` 表中的数据为原 `tbl2` 表中的数据。而名称为 `tbl2` 表被删除。 + + +## 原理 + +替换表功能，实际上是将以下操作集合变成一个原子操作。 + +假设要将表 A 替换为表 B，且 `swap` 为 `true`，则操作如下： + +1. 将表 B 重名为表 A。 +2. 将表 A 重名为表 B。 + +如果 `swap` 为 `true`，则操作如下： + +1. 删除表 A。 +2. 将表 B 重名为表 A。 + +## 注意事项 + +1. `swap` 参数默认为 `true`。即替换表操作相当于将两张表数据进行交换。 +2. 如果设置 `swap` 参数为 `false`，则被替换的表（表A）将被删除，且无法恢复。 +3. 替换操作仅能发生在两张 OLAP 表之间，且不会检查两张表的表结构是否一致。 +4. 替换操作不会改变原有的权限设置。因为权限检查以表名称为准。 + +## 最佳实践 + +1. 原子的覆盖写操作 + + 某些情况下，用户希望能够重写某张表的数据，但如果采用先删除再导入的方式进行，在中间会有一段时间无法查看数据。这时，用户可以先使用 `CREATE TABLE LIKE` 语句创建一个相同结构的新表，将新的数据导入到新表后，通过替换操作，原子的替换旧表，以达到目的。分区级别的原子覆盖写操作，请参阅 [临时分区文档](./alter-table-temp-partition.md) diff --git a/docs/zh-CN/administrator-guide/alter-table/alter-table-temp-partition.md b/docs/zh-CN/administrator-guide/alter-table/alter-table-temp-partition.md index 32b328f6a4e407..a9493bda211626 100644 --- a/docs/zh-CN/administrator-guide/alter-table/alter-table-temp-partition.md +++ b/docs/zh-CN/administrator-guide/alter-table/alter-table-temp-partition.md @@ -230,7 +230,7 @@ PROPERTIES ( 1. 原子的覆盖写操作 - 某些情况下，用户希望能够重写某一分区的数据，但如果采用先删除再导入的方式进行，在中间会有一段时间无法查看数据。这是，用户可以先创建一个对应的临时分区，将新的数据导入到临时分区后，通过替换操作，原子的替换原有分区，以达到目的。 + 某些情况下，用户希望能够重写某一分区的数据，但如果采用先删除再导入的方式进行，在中间会有一段时间无法查看数据。这时，用户可以先创建一个对应的临时分区，将新的数据导入到临时分区后，通过替换操作，原子的替换原有分区，以达到目的。对于非分区表的原子覆盖写操作，请参阅[替换表文档](./alter-table-replace-table.md) 2. 修改分桶数 diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java index bbf5f43fe3d218..7bc69aebcd86a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java @@ -18,7 +18,6 @@ package org.apache.doris.alter; public enum AlterOpType { - INVALID_OP, // rollup ADD_ROLLUP, DROP_ROLLUP, @@ -37,8 +36,8 @@ public enum AlterOpType { // others operation, such as add/drop backend. currently we do not care about them ALTER_OTHER, ENABLE_FEATURE, - REPLACE_TABLE; - + REPLACE_TABLE, + INVALID_OP; // INVALID_OP must be the last one // true means 2 operations have no conflict. public static Boolean[][] COMPATIBITLITY_MATRIX; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ReplaceTableClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ReplaceTableClause.java index 051b2f623e4f32..06f6fa1e4dfd19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ReplaceTableClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ReplaceTableClause.java @@ -39,7 +39,7 @@ public class ReplaceTableClause extends AlterTableClause { private boolean swapTable; public ReplaceTableClause(String tblName, Map properties) { - super(AlterOpType.REPLACE_PARTITION); + super(AlterOpType.REPLACE_TABLE); this.tblName = tblName; this.properties = properties; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/ReplaceTableOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/ReplaceTableOperationLog.java new file mode 100644 index 00000000000000..3251e58539fce3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/ReplaceTableOperationLog.java @@ -0,0 +1,56 @@ +package org.apache.doris.persist; + +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public class ReplaceTableOperationLog implements Writable { + @SerializedName(value = "dbId") + private long dbId; + @SerializedName(value = "origTblId") + private long origTblId; + @SerializedName(value = "newTblName") + private long newTblId; + @SerializedName(value = "swapTable") + private boolean swapTable; + + public ReplaceTableOperationLog(long dbId, long origTblId, long newTblId, boolean swapTable) { + this.dbId = dbId; + this.origTblId = origTblId; + this.newTblId = newTblId; + this.swapTable = swapTable; + } + + public long getDbId() { + return dbId; + } + + public long getOrigTblId() { + return origTblId; + } + + public long getNewTblId() { + return newTblId; + } + + public boolean isSwapTable() { + return swapTable; + } + + @Override + public void write(DataOutput out) throws IOException { + String json = GsonUtils.GSON.toJson(this); + Text.writeString(out, json); + } + + public static ReplaceTableOperationLog read(DataInput in) throws IOException { + String json = Text.readString(in); + return GsonUtils.GSON.fromJson(json, ReplaceTableOperationLog.class); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java index 0f9946c113c583..c30ecbc74a3d66 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.DataProperty; import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Type; @@ -34,13 +35,13 @@ import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.utframe.UtFrameUtils; -import com.google.common.collect.Lists; - import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import com.google.common.collect.Lists; + import java.io.File; import java.util.List; import java.util.Map; @@ -453,4 +454,98 @@ public void testSetDynamicPropertiesInDynamicPartitionTable() throws Exception { String alterStmt6 = "alter table test." + tableName + " set (\"dynamic_partition.buckets\" = \"5\");"; alterTable(alterStmt6, false); } + + @Test + public void testReplaceTable() throws Exception { + String stmt1 = "CREATE TABLE test.replace1\n" + + "(\n" + + " k1 int, k2 int, k3 int sum\n" + + ")\n" + + "AGGREGATE KEY(k1, k2)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 10\n" + + "rollup (\n" + + "r1(k1),\n" + + "r2(k2, k3)\n" + + ")\n" + + "PROPERTIES(\"replication_num\" = \"1\");"; + + + String stmt2 = "CREATE TABLE test.r1\n" + + "(\n" + + " k1 int, k2 int\n" + + ")\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 11\n" + + "PROPERTIES(\"replication_num\" = \"1\");"; + + String stmt3 = "CREATE TABLE test.replace2\n" + + "(\n" + + " k1 int, k2 int\n" + + ")\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 11\n" + + "PROPERTIES(\"replication_num\" = \"1\");"; + + String stmt4 = "CREATE TABLE test.replace3\n" + + "(\n" + + " k1 int, k2 int, k3 int sum\n" + + ")\n" + + "PARTITION BY RANGE(k1)\n" + + "(\n" + + "\tPARTITION p1 values less than(\"100\"),\n" + + "\tPARTITION p2 values less than(\"200\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 1\n" + + "rollup (\n" + + "r3(k1),\n" + + "r4(k2, k3)\n" + + ")\n" + + "PROPERTIES(\"replication_num\" = \"1\");"; + + createTable(stmt1); + createTable(stmt2); + createTable(stmt3); + createTable(stmt4); + Database db = Catalog.getCurrentCatalog().getDb("default_cluster:test"); + + // name conflict + String replaceStmt = "ALTER TABLE test.replace1 REPLACE WITH TABLE r1"; + alterTable(replaceStmt, true); + + // replace1 with replace2 + replaceStmt = "ALTER TABLE test.replace1 REPLACE WITH TABLE replace2"; + OlapTable replace1 = (OlapTable) db.getTable("replace1"); + OlapTable replace2 = (OlapTable) db.getTable("replace2"); + Assert.assertEquals(3, replace1.getPartition("replace1").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); + Assert.assertEquals(1, replace2.getPartition("replace2").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); + + alterTable(replaceStmt, false); + + replace1 = (OlapTable) db.getTable("replace1"); + replace2 = (OlapTable) db.getTable("replace2"); + Assert.assertEquals(1, replace1.getPartition("replace1").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); + Assert.assertEquals(3, replace2.getPartition("replace2").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); + Assert.assertEquals("replace1", replace1.getIndexNameById(replace1.getBaseIndexId())); + Assert.assertEquals("replace2", replace2.getIndexNameById(replace2.getBaseIndexId())); + + // replace with no swap + replaceStmt = "ALTER TABLE test.replace1 REPLACE WITH TABLE replace2 properties('swap' = 'false')"; + alterTable(replaceStmt, false); + replace1 = (OlapTable) db.getTable("replace1"); + replace2 = (OlapTable) db.getTable("replace2"); + Assert.assertNull(replace2); + Assert.assertEquals(3, replace1.getPartition("replace1").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); + Assert.assertEquals("replace1", replace1.getIndexNameById(replace1.getBaseIndexId())); + + replaceStmt = "ALTER TABLE test.replace1 REPLACE WITH TABLE replace3 properties('swap' = 'true')"; + alterTable(replaceStmt, false); + replace1 = (OlapTable) db.getTable("replace1"); + OlapTable replace3 = (OlapTable) db.getTable("replace3"); + Assert.assertEquals(3, replace1.getPartition("p1").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); + Assert.assertEquals(3, replace1.getPartition("p2").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); + Assert.assertNotNull(replace1.getIndexIdByName("r3")); + Assert.assertNotNull(replace1.getIndexIdByName("r4")); + + Assert.assertEquals(3, replace3.getPartition("replace3").getMaterializedIndices(MaterializedIndex.IndexExtState.VISIBLE).size()); + Assert.assertNotNull(replace3.getIndexIdByName("r1")); + Assert.assertNotNull(replace3.getIndexIdByName("r2")); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/ReplaceTableOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/ReplaceTableOperationLogTest.java new file mode 100644 index 00000000000000..48bb81f8925f56 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/ReplaceTableOperationLogTest.java @@ -0,0 +1,57 @@ +package org.apache.doris.persist; + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; + +public class ReplaceTableOperationLogTest { + @Test + public void testSerialization() throws Exception { + // 1. Write objects to file + File file = new File("./ReplaceTableOperationLogTest"); + file.createNewFile(); + DataOutputStream dos = new DataOutputStream(new FileOutputStream(file)); + + ReplaceTableOperationLog log = new ReplaceTableOperationLog(1,2,3,true); + log.write(dos); + + dos.flush(); + dos.close(); + + // 2. Read objects from file + DataInputStream dis = new DataInputStream(new FileInputStream(file)); + + ReplaceTableOperationLog readLog = ReplaceTableOperationLog.read(dis); + Assert.assertTrue(readLog.getDbId() == log.getDbId()); + Assert.assertTrue(readLog.getNewTblId() == log.getNewTblId()); + Assert.assertTrue(readLog.getOrigTblId() == log.getOrigTblId()); + Assert.assertTrue(readLog.isSwapTable() == log.isSwapTable()); + + // 3. delete files + dis.close(); + file.delete(); + } +} + From 3a6aaccecb290c96c605b5cd2129c4913b6512f6 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 25 Sep 2020 00:14:49 +0800 Subject: [PATCH 4/4] fix bug --- .../org/apache/doris/journal/JournalEntity.java | 11 +++++++++-- .../doris/persist/ReplaceTableOperationLog.java | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 9de554e3d70302..74d1f21bec4249 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -72,6 +72,7 @@ import org.apache.doris.persist.RecoverInfo; import org.apache.doris.persist.RemoveAlterJobV2OperationLog; import org.apache.doris.persist.ReplacePartitionOperationLog; +import org.apache.doris.persist.ReplaceTableOperationLog; import org.apache.doris.persist.ReplicaPersistInfo; import org.apache.doris.persist.RoutineLoadOperation; import org.apache.doris.persist.SetReplicaStatusOperationLog; @@ -84,11 +85,11 @@ import org.apache.doris.system.Frontend; import org.apache.doris.transaction.TransactionState; -import com.google.common.base.Preconditions; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.google.common.base.Preconditions; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -587,6 +588,11 @@ public void readFields(DataInput in) throws IOException { isRead = true; break; } + case OperationType.OP_REPLACE_TABLE: { + data = ReplaceTableOperationLog.read(in); + isRead = true; + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); @@ -596,3 +602,4 @@ public void readFields(DataInput in) throws IOException { Preconditions.checkState(isRead); } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/ReplaceTableOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/ReplaceTableOperationLog.java index 3251e58539fce3..c5b0a05f0e6456 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/ReplaceTableOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/ReplaceTableOperationLog.java @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.doris.persist; import org.apache.doris.common.io.Text;