From 1450fdd10bdae8e17add9eab409f97dabcb4c69b Mon Sep 17 00:00:00 2001 From: codeDing18 Date: Wed, 7 May 2025 21:24:10 +0800 Subject: [PATCH] [Enhancement] (nereids)implement TruncateTableCommand in nereids --- .../org/apache/doris/nereids/DorisParser.g4 | 6 +- .../java/org/apache/doris/catalog/Env.java | 18 ++ .../apache/doris/datasource/CatalogIf.java | 3 + .../doris/datasource/ExternalCatalog.java | 27 ++ .../doris/datasource/InternalCatalog.java | 237 ++++++++++++++ .../nereids/parser/LogicalPlanBuilder.java | 14 + .../doris/nereids/trees/plans/PlanType.java | 3 +- .../plans/commands/TruncateTableCommand.java | 119 ++++++++ .../trees/plans/visitor/CommandVisitor.java | 5 + .../commands/TruncateTableCommandTest.java | 289 ++++++++++++++++++ .../ddl/test_truncate_table_nereids.out | 23 ++ .../hive/ddl/test_hive_truncate_table.groovy | 87 +++--- .../ddl/test_truncate_table_nereids.groovy | 192 ++++++++++++ 13 files changed, 976 insertions(+), 47 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommand.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommandTest.java create mode 100644 regression-test/data/nereids_p0/ddl/test_truncate_table_nereids.out create mode 100644 regression-test/suites/nereids_p0/ddl/test_truncate_table_nereids.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 9c47dda6cbe25c..e996a24ce261a9 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -73,7 +73,6 @@ statementBase unsupportedStatement : unsupportedUseStatement - | unsupportedDmlStatement | unsupportedCreateStatement | unsupportedDropStatement | unsupportedStatsStatement @@ -165,6 +164,7 @@ supportedDmlStatement (stageAndPattern | (LEFT_PAREN SELECT selectColumnClause FROM stageAndPattern whereClause? RIGHT_PAREN)) properties=propertyClause? #copyInto + | TRUNCATE TABLE multipartIdentifier specifiedPartition? FORCE? #truncateTable ; supportedCreateStatement @@ -907,10 +907,6 @@ unsupportedUseStatement : USE ((catalog=identifier DOT)? database=identifier)? ATSIGN cluster=identifier #useCloudCluster ; -unsupportedDmlStatement - : TRUNCATE TABLE multipartIdentifier specifiedPartition? FORCE? #truncateTable - ; - stageAndPattern : ATSIGN (stage=identifier | TILDE) (LEFT_PAREN pattern=STRING_LITERAL RIGHT_PAREN)? diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index b9617f62d2187d..ad35b25572c6c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -208,6 +208,7 @@ import org.apache.doris.nereids.trees.plans.commands.AnalyzeCommand; import org.apache.doris.nereids.trees.plans.commands.CreateMaterializedViewCommand; import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand.IdType; +import org.apache.doris.nereids.trees.plans.commands.TruncateTableCommand; import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVPropertyInfo; import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVRefreshInfo; import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; @@ -6089,6 +6090,23 @@ public void truncateTable(TruncateTableStmt stmt) throws DdlException { catalogIf.truncateTable(stmt); } + /* + * Truncate specified table or partitions. + * The main idea is: + * + * 1. using the same schema to create new table(partitions) + * 2. use the new created table(partitions) to replace the old ones. + * + * if no partition specified, it will truncate all partitions of this table, including all temp partitions, + * otherwise, it will only truncate those specified partitions. + * + */ + public void truncateTable(TruncateTableCommand command) throws DdlException { + CatalogIf catalogIf = catalogMgr.getCatalogOrException(command.getTableNameInfo().getCtl(), + catalog -> new DdlException(("Unknown catalog " + catalog))); + catalogIf.truncateTable(command); + } + public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundException { if (Strings.isNullOrEmpty(info.getCtl()) || info.getCtl().equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { // In previous versions(before 2.1.8), there is no catalog info in TruncateTableInfo, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index f876f67c092268..849a0cd9a90cdb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -31,6 +31,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.UserException; +import org.apache.doris.nereids.trees.plans.commands.TruncateTableCommand; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -205,6 +206,8 @@ void dropTable(String dbName, String tableName, boolean isView, boolean isMtmv, void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlException; + void truncateTable(TruncateTableCommand truncateTableCommand) throws DdlException; + // Convert from remote database name to local database name, overridden by subclass if necessary default String fromRemoteDatabaseName(String remoteDatabaseName) { return remoteDatabaseName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index d9c620ee0bba66..fabb809362b604 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -59,6 +59,7 @@ import org.apache.doris.datasource.test.TestExternalDatabase; import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalDatabase; import org.apache.doris.fs.remote.dfs.DFSFileSystem; +import org.apache.doris.nereids.trees.plans.commands.TruncateTableCommand; import org.apache.doris.persist.CreateDbInfo; import org.apache.doris.persist.CreateTableInfo; import org.apache.doris.persist.DropDbInfo; @@ -1185,6 +1186,32 @@ public void truncateTable(TruncateTableStmt stmt) throws DdlException { } } + @Override + public void truncateTable(TruncateTableCommand command) throws DdlException { + makeSureInitialized(); + if (metadataOps == null) { + throw new UnsupportedOperationException("Truncate table not supported in " + getName()); + } + try { + String db = command.getTableNameInfo().getDb(); + String tbl = command.getTableNameInfo().getTbl(); + + // delete all table data if null + List partitions = null; + if (command.getPartitionNamesInfo().isPresent()) { + partitions = command.getPartitionNamesInfo().get().getPartitionNames(); + } + + metadataOps.truncateTable(db, tbl, partitions); + TruncateTableInfo info = new TruncateTableInfo(getName(), db, tbl, partitions); + Env.getCurrentEnv().getEditLog().logTruncateTable(info); + } catch (Exception e) { + LOG.warn("Failed to truncate table {}.{} in catalog {}", command.getTableNameInfo().getDb(), + command.getTableNameInfo().getTbl(), getName(), e); + throw e; + } + } + public void replayTruncateTable(TruncateTableInfo info) { if (metadataOps != null) { metadataOps.afterTruncateTable(info.getDb(), info.getTable()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index a98f68ac15c875..766b006533dddf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -146,6 +146,7 @@ import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand.IdType; +import org.apache.doris.nereids.trees.plans.commands.TruncateTableCommand; import org.apache.doris.nereids.trees.plans.commands.info.DropMTMVInfo; import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; import org.apache.doris.persist.AlterDatabasePropertyInfo; @@ -3779,6 +3780,242 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti LOG.info("finished to truncate table {}, partitions: {}", tblRef.getName().toSql(), tblRef.getPartitionNames()); } + /* + * Truncate specified table or partitions. + * The main idea is: + * + * 1. using the same schema to create new table(partitions) + * 2. use the new created table(partitions) to replace the old ones. + * + * if no partition specified, it will truncate all partitions of this table, including all temp partitions, + * otherwise, it will only truncate those specified partitions. + * + */ + public void truncateTable(TruncateTableCommand truncateTableCommand) throws DdlException { + boolean isForceDrop = truncateTableCommand.isForceDrop(); + String database = truncateTableCommand.getTableNameInfo().getDb(); + String tbl = truncateTableCommand.getTableNameInfo().getTbl(); + + // check, and save some info which need to be checked again later + Map origPartitions = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + Map partitionsDistributionInfo = Maps.newHashMap(); + OlapTable copiedTbl; + + boolean truncateEntireTable = !truncateTableCommand.getPartitionNamesInfo().isPresent(); + + Database db = (Database) getDbOrDdlException(database); + OlapTable olapTable = db.getOlapTableOrDdlException(tbl); + + if (olapTable instanceof MTMV && !MTMVUtil.allowModifyMTMVData(ConnectContext.get())) { + throw new DdlException("Not allowed to perform current operation on async materialized view"); + } + + HashMap updateRecords = new HashMap<>(); + + BinlogConfig binlogConfig; + olapTable.readLock(); + try { + olapTable.checkNormalStateForAlter(); + if (!truncateEntireTable) { + for (String partName : truncateTableCommand.getPartitionNamesInfo().get().getPartitionNames()) { + Partition partition = olapTable.getPartition(partName); + if (partition == null) { + throw new DdlException("Partition " + partName + " does not exist"); + } + // If need absolutely correct, should check running txn here. + // But if the txn is in prepare state, cann't known which partitions had load data. + if ((isForceDrop) && (!partition.hasData())) { + // if not force drop, then need to add partition to + // recycle bin, so behavior for recover would be clear + continue; + } + origPartitions.put(partName, partition.getId()); + partitionsDistributionInfo.put(partition.getId(), partition.getDistributionInfo()); + updateRecords.put(partition.getId(), partition.getBaseIndex().getRowCount()); + } + } else { + for (Partition partition : olapTable.getPartitions()) { + // If need absolutely correct, should check running txn here. + // But if the txn is in prepare state, cann't known which partitions had load data. + if ((isForceDrop) && (!partition.hasData())) { + // if not force drop, then need to add partition to + // recycle bin, so behavior for recover would be clear + continue; + } + origPartitions.put(partition.getName(), partition.getId()); + partitionsDistributionInfo.put(partition.getId(), partition.getDistributionInfo()); + updateRecords.put(partition.getId(), partition.getBaseIndex().getRowCount()); + } + } + // if table currently has no partitions, this sql like empty command and do nothing, should return directly. + // but if truncate whole table, the temporary partitions also need drop + if (origPartitions.isEmpty() && (!truncateEntireTable || olapTable.getAllTempPartitions().isEmpty())) { + LOG.info("finished to truncate table {}, no partition contains data, do nothing", + truncateTableCommand.getTableNameInfo().toSql()); + return; + } + copiedTbl = olapTable.selectiveCopy(origPartitions.keySet(), IndexExtState.VISIBLE, false); + + binlogConfig = new BinlogConfig(olapTable.getBinlogConfig()); + } finally { + olapTable.readUnlock(); + } + + // 2. use the copied table to create partitions + List newPartitions = Lists.newArrayList(); + // tabletIdSet to save all newly created tablet ids. + Set tabletIdSet = Sets.newHashSet(); + Runnable failedCleanCallback = () -> { + for (Long tabletId : tabletIdSet) { + Env.getCurrentInvertedIndex().deleteTablet(tabletId); + } + }; + try { + long bufferSize = IdGeneratorUtil.getBufferSizeForTruncateTable(copiedTbl, origPartitions.values()); + IdGeneratorBuffer idGeneratorBuffer = + origPartitions.isEmpty() ? null : Env.getCurrentEnv().getIdGeneratorBuffer(bufferSize); + + Map oldToNewPartitionId = new HashMap(); + List newPartitionIds = new ArrayList(); + for (Map.Entry entry : origPartitions.entrySet()) { + long oldPartitionId = entry.getValue(); + long newPartitionId = idGeneratorBuffer.getNextId(); + oldToNewPartitionId.put(oldPartitionId, newPartitionId); + newPartitionIds.add(newPartitionId); + } + + List indexIds = copiedTbl.getIndexIdToMeta().keySet().stream().collect(Collectors.toList()); + beforeCreatePartitions(db.getId(), copiedTbl.getId(), newPartitionIds, indexIds, true); + + for (Map.Entry entry : origPartitions.entrySet()) { + // the new partition must use new id + // If we still use the old partition id, the behavior of current load jobs on this partition + // will be undefined. + // By using a new id, load job will be aborted(just like partition is dropped), + // which is the right behavior. + long oldPartitionId = entry.getValue(); + long newPartitionId = oldToNewPartitionId.get(oldPartitionId); + Partition newPartition = createPartitionWithIndices(db.getId(), copiedTbl, + newPartitionId, entry.getKey(), + copiedTbl.getIndexIdToMeta(), partitionsDistributionInfo.get(oldPartitionId), + copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId), + copiedTbl.getPartitionInfo().getReplicaAllocation(oldPartitionId), null /* version info */, + copiedTbl.getCopiedBfColumns(), tabletIdSet, + copiedTbl.isInMemory(), + copiedTbl.getPartitionInfo().getTabletType(oldPartitionId), + olapTable.getPartitionInfo().getDataProperty(oldPartitionId).getStoragePolicy(), + idGeneratorBuffer, binlogConfig, + copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified()); + newPartitions.add(newPartition); + } + + afterCreatePartitions(db.getId(), copiedTbl.getId(), newPartitionIds, indexIds, true); + + } catch (DdlException e) { + // create partition failed, remove all newly created tablets + failedCleanCallback.run(); + throw e; + } + Preconditions.checkState(origPartitions.size() == newPartitions.size()); + + // all partitions are created successfully, try to replace the old partitions. + // before replacing, we need to check again. + // Things may be changed outside the table lock. + List oldPartitions = Lists.newArrayList(); + boolean hasWriteLock = false; + try { + olapTable = (OlapTable) db.getTableOrDdlException(copiedTbl.getId()); + olapTable.writeLockOrDdlException(); + hasWriteLock = true; + olapTable.checkNormalStateForAlter(); + // check partitions + for (Map.Entry entry : origPartitions.entrySet()) { + Partition partition = olapTable.getPartition(entry.getValue()); + if (partition == null || !partition.getName().equalsIgnoreCase(entry.getKey())) { + throw new DdlException("Partition [" + entry.getKey() + "] is changed" + + " during truncating table, please retry"); + } + } + + // check if meta changed + // rollup index may be added or dropped, and schema may be changed during creating partition operation. + boolean metaChanged = false; + if (olapTable.getIndexNameToId().size() != copiedTbl.getIndexNameToId().size()) { + metaChanged = true; + } else { + // compare schemaHash + Map copiedIndexIdToSchemaHash = copiedTbl.getIndexIdToSchemaHash(); + for (Map.Entry entry : olapTable.getIndexIdToSchemaHash().entrySet()) { + long indexId = entry.getKey(); + if (!copiedIndexIdToSchemaHash.containsKey(indexId)) { + metaChanged = true; + break; + } + if (!copiedIndexIdToSchemaHash.get(indexId).equals(entry.getValue())) { + metaChanged = true; + break; + } + } + + List oldSchema = copiedTbl.getFullSchema(); + List newSchema = olapTable.getFullSchema(); + if (oldSchema.size() != newSchema.size()) { + LOG.warn("schema column size diff, old schema {}, new schema {}", oldSchema, newSchema); + metaChanged = true; + } else { + List oldSchemaCopy = Lists.newArrayList(oldSchema); + List newSchemaCopy = Lists.newArrayList(newSchema); + oldSchemaCopy.sort((Column a, Column b) -> a.getUniqueId() - b.getUniqueId()); + newSchemaCopy.sort((Column a, Column b) -> a.getUniqueId() - b.getUniqueId()); + for (int i = 0; i < oldSchemaCopy.size(); ++i) { + if (!oldSchemaCopy.get(i).equals(newSchemaCopy.get(i))) { + LOG.warn("schema diff, old schema {}, new schema {}", oldSchemaCopy.get(i), + newSchemaCopy.get(i)); + metaChanged = true; + break; + } + } + } + } + if (DebugPointUtil.isEnable("InternalCatalog.truncateTable.metaChanged")) { + metaChanged = true; + LOG.warn("debug set truncate table meta changed"); + } + + if (metaChanged) { + throw new DdlException("Table[" + copiedTbl.getName() + "]'s meta has been changed. try again."); + } + + //replace + Map recyclePartitionParamMap = new HashMap<>(); + oldPartitions = truncateTableInternal(olapTable, newPartitions, + truncateEntireTable, recyclePartitionParamMap, isForceDrop); + + // write edit log + TruncateTableInfo info = + new TruncateTableInfo(db.getId(), db.getFullName(), olapTable.getId(), olapTable.getName(), + newPartitions, truncateEntireTable, + truncateTableCommand.toSqlWithoutTable(), oldPartitions, isForceDrop); + Env.getCurrentEnv().getEditLog().logTruncateTable(info); + } catch (DdlException e) { + failedCleanCallback.run(); + throw e; + } finally { + if (hasWriteLock) { + olapTable.writeUnlock(); + } + } + + PartitionNames partitionNames = truncateEntireTable ? null + : new PartitionNames(false, truncateTableCommand.getPartitionNamesInfo().get().getPartitionNames()); + Env.getCurrentEnv().getAnalysisManager().dropStats(olapTable, partitionNames); + Env.getCurrentEnv().getAnalysisManager().updateUpdatedRows(updateRecords, db.getId(), olapTable.getId(), 0); + LOG.info("finished to truncate table {}, partitions: {}", + truncateTableCommand.getTableNameInfo().toSql(), + !truncateTableCommand.getPartitionNamesInfo().isPresent() + ? "null" : truncateTableCommand.getPartitionNamesInfo().get().getPartitionNames()); + } + private List truncateTableInternal(OlapTable olapTable, List newPartitions, boolean isEntireTable, Map recyclePartitionParamMap, boolean isforceDrop) { // use new partitions to replace the old ones. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 0b0e9659f69469..9a1bc5327ac5c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -725,6 +725,7 @@ import org.apache.doris.nereids.trees.plans.commands.TransactionBeginCommand; import org.apache.doris.nereids.trees.plans.commands.TransactionCommitCommand; import org.apache.doris.nereids.trees.plans.commands.TransactionRollbackCommand; +import org.apache.doris.nereids.trees.plans.commands.TruncateTableCommand; import org.apache.doris.nereids.trees.plans.commands.UnlockTablesCommand; import org.apache.doris.nereids.trees.plans.commands.UnsetDefaultStorageVaultCommand; import org.apache.doris.nereids.trees.plans.commands.UnsetVariableCommand; @@ -6098,6 +6099,19 @@ public LogicalPlan visitUseDatabase(UseDatabaseContext ctx) { : new UseCommand(ctx.database.getText()); } + @Override + public LogicalPlan visitTruncateTable(DorisParser.TruncateTableContext ctx) { + TableNameInfo tableName = new TableNameInfo(visitMultipartIdentifier(ctx.multipartIdentifier())); + Optional partitionNamesInfo = ctx.specifiedPartition() == null + ? Optional.empty() : Optional.of(visitSpecifiedPartitionContext(ctx.specifiedPartition())); + + return new TruncateTableCommand( + tableName, + partitionNamesInfo, + ctx.FORCE() != null + ); + } + @Override public LogicalPlan visitShowConvertLsc(ShowConvertLscContext ctx) { if (ctx.database == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index fded17aead116f..1f785e0201ada1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -369,5 +369,6 @@ public enum PlanType { CREATE_USER_COMMAND, CREATE_RESOURCE_COMMAND, CREATE_STAGE_COMMAND, - DROP_STAGE_COMMAND + DROP_STAGE_COMMAND, + TRUNCATE_TABLE_COMMAND } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommand.java new file mode 100644 index 00000000000000..24b2dcbec6e934 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommand.java @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.StmtType; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.util.InternalDatabaseUtil; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.info.PartitionNamesInfo; +import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +import java.util.Objects; +import java.util.Optional; + +/** + * TRUNCATE TABLE tbl [PARTITION(p1, p2, ...)] + */ +public class TruncateTableCommand extends Command implements ForwardWithSync { + private final TableNameInfo tableNameInfo; + private final Optional partitionNamesInfo; + private final boolean forceDrop; + + public TruncateTableCommand(TableNameInfo tableNameInfo, + Optional partitionNamesInfo, boolean forceDrop) { + super(PlanType.TRUNCATE_TABLE_COMMAND); + this.tableNameInfo = Objects.requireNonNull(tableNameInfo, "tableNameInfo is null"); + this.partitionNamesInfo = Objects.requireNonNull(partitionNamesInfo, "partitionNamesInfo is null"); + this.forceDrop = Objects.requireNonNull(forceDrop, "forceDrop is null"); + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + validate(ctx); + Env.getCurrentEnv().truncateTable(this); + } + + /** + * validate + */ + public void validate(ConnectContext ctx) throws AnalysisException { + tableNameInfo.analyze(ctx); + + InternalDatabaseUtil.checkDatabase(tableNameInfo.getDb(), ctx); + // check access + // it requires LOAD privilege, because we consider this operation as 'delete data', which is also a + // 'load' operation. + if (!Env.getCurrentEnv().getAccessManager() + .checkTblPriv(ConnectContext.get(), tableNameInfo.getCtl(), tableNameInfo.getDb(), + tableNameInfo.getTbl(), PrivPredicate.LOAD)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "LOAD"); + } + + // check partition if specified. do not support truncate temp partitions + if (partitionNamesInfo.isPresent()) { + partitionNamesInfo.get().validate(); + if (partitionNamesInfo.get().isTemp()) { + throw new AnalysisException("Not support truncate temp partitions"); + } + } + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitTruncateTableCommand(this, context); + } + + @Override + public StmtType stmtType() { + return StmtType.TRUNCATE; + } + + /** + * toSqlWithoutTable + */ + public String toSqlWithoutTable() { + StringBuilder sb = new StringBuilder(); + if (partitionNamesInfo.isPresent()) { + sb.append(partitionNamesInfo.get().toSql()); + } + if (isForceDrop()) { + sb.append(" FORCE"); + } + return sb.toString(); + } + + public TableNameInfo getTableNameInfo() { + return tableNameInfo; + } + + public Optional getPartitionNamesInfo() { + return partitionNamesInfo; + } + + public boolean isForceDrop() { + return forceDrop; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index bd2eec6f4e4559..228d11ee9b0460 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -200,6 +200,7 @@ import org.apache.doris.nereids.trees.plans.commands.TransactionBeginCommand; import org.apache.doris.nereids.trees.plans.commands.TransactionCommitCommand; import org.apache.doris.nereids.trees.plans.commands.TransactionRollbackCommand; +import org.apache.doris.nereids.trees.plans.commands.TruncateTableCommand; import org.apache.doris.nereids.trees.plans.commands.UnlockTablesCommand; import org.apache.doris.nereids.trees.plans.commands.UnsetDefaultStorageVaultCommand; import org.apache.doris.nereids.trees.plans.commands.UnsetVariableCommand; @@ -1079,4 +1080,8 @@ default R visitCreateStageCommand(CreateStageCommand createStageCommand, C conte default R visitDropStageCommand(DropStageCommand dropStageCommand, C context) { return visitCommand(dropStageCommand, context); } + + default R visitTruncateTableCommand(TruncateTableCommand truncateTableCommand, C context) { + return visitCommand(truncateTableCommand, context); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommandTest.java new file mode 100644 index 00000000000000..90fac35fc2cea8 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/TruncateTableCommandTest.java @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.common.util.DebugPointUtil; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.info.PartitionNamesInfo; +import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class TruncateTableCommandTest extends TestWithFeService { + @Override + protected void runBeforeAll() throws Exception { + Config.enable_debug_points = true; + createDatabase("test"); + connectContext.setDatabase("test"); + + String createTableStr = "create table test.tbl(d1 date, k1 int, k2 bigint)" + + "duplicate key(d1, k1) " + + "PARTITION BY RANGE(d1)" + + "(PARTITION p20210901 VALUES [('2021-09-01'), ('2021-09-02')))" + + "distributed by hash(k1) buckets 2 " + + "properties('replication_num' = '1');"; + createTable(createTableStr); + + String createTable2 = "CREATE TABLE test.case_sensitive_table (\n" + + " `date_id` date NULL COMMENT \"\",\n" + + " `column2` tinyint(4) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`date_id`, `column2`)\n" + + "COMMENT \"OLAP\"\n" + + "PARTITION BY RANGE(`date_id`)\n" + + "(\n" + + "PARTITION p20211006 VALUES [('2021-10-06'), ('2021-10-07')),\n" + + "PARTITION P20211007 VALUES [('2021-10-07'), ('2021-10-08')),\n" + + "PARTITION P20211008 VALUES [('2021-10-08'), ('2021-10-09')))\n" + + "DISTRIBUTED BY HASH(`column2`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\"\n" + + ");"; + + createTable(createTable2); + } + + @Test + public void testValidate(@Mocked Env env, @Mocked AccessControllerManager accessManager) { + new Expectations() { + { + env.getAccessManager(); + result = accessManager; + accessManager.checkTblPriv((ConnectContext) any, anyString, anyString, anyString, PrivPredicate.LOAD); + result = true; + } + }; + + String truncateStr = "TRUNCATE TABLE internal.test.case_sensitive_table PARTITION P20211008; \n"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan plan = nereidsParser.parseSingle(truncateStr); + Assertions.assertTrue(plan instanceof TruncateTableCommand); + Assertions.assertDoesNotThrow(() -> ((TruncateTableCommand) plan).validate(connectContext)); + + // test no database + TableNameInfo tableNameInfo = new TableNameInfo("internal", "", "test"); + TruncateTableCommand truncateTableCommand = new TruncateTableCommand(tableNameInfo, Optional.empty(), false); + connectContext.setDatabase(""); + Assertions.assertThrows(AnalysisException.class, () -> truncateTableCommand.validate(connectContext)); + connectContext.setDatabase("test"); //reset database + + // test no table + tableNameInfo = new TableNameInfo("internal", "test", ""); + TruncateTableCommand truncateTableCommand1 = new TruncateTableCommand(tableNameInfo, Optional.empty(), false); + Assertions.assertThrows(AnalysisException.class, () -> truncateTableCommand1.validate(connectContext)); + + // test no partition + tableNameInfo = new TableNameInfo("internal", "test", "test"); + PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false); + TruncateTableCommand truncateTableCommand2 = new TruncateTableCommand(tableNameInfo, Optional.of(partitionNamesInfo), false); + Assertions.assertThrows(AnalysisException.class, () -> truncateTableCommand2.validate(connectContext)); + } + + @Test + public void testTruncateWithCaseInsensitivePartitionName() throws Exception { + //now in order to support auto create partition, need set partition name is case sensitive + Database db = Env.getCurrentInternalCatalog().getDbNullable("test"); + OlapTable tbl = db.getOlapTableOrDdlException("case_sensitive_table"); + + long p20211006Id = tbl.getPartition("p20211006").getId(); + long p20211007Id = tbl.getPartition("P20211007").getId(); + long p20211008Id = tbl.getPartition("P20211008").getId(); + + // truncate P20211008(real name is P20211008) + Partition p20211008 = tbl.getPartition("P20211008"); + p20211008.updateVisibleVersion(2L); + p20211008.setNextVersion(p20211008.getVisibleVersion() + 1); + String truncateStr = "TRUNCATE TABLE internal.test.case_sensitive_table PARTITION P20211008; \n"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan plan = nereidsParser.parseSingle(truncateStr); + Assertions.assertTrue(plan instanceof TruncateTableCommand); + Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan); + Assertions.assertNotEquals(p20211008Id, tbl.getPartition("P20211008").getId()); + + // 2. truncate P20211007 + truncateStr = "TRUNCATE TABLE internal.test.case_sensitive_table PARTITION P20211007 force; \n"; + plan = nereidsParser.parseSingle(truncateStr); + Assertions.assertTrue(plan instanceof TruncateTableCommand); + Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan); + + Assertions.assertEquals(p20211006Id, tbl.getPartition("p20211006").getId()); + Assertions.assertEquals(p20211007Id, tbl.getPartition("P20211007").getId()); + Assertions.assertNotNull(tbl.getPartition("p20211006")); + Assertions.assertNotNull(tbl.getPartition("P20211007")); + Assertions.assertNotNull(tbl.getPartition("P20211008")); + } + + @Test + public void testTruncateTable() throws Exception { + String stmtStr = "ALTER TABLE internal.test.tbl ADD PARTITION p20210902 VALUES [('2021-09-02'), ('2021-09-03'))" + + " DISTRIBUTED BY HASH(`k1`) BUCKETS 3;"; + alterTable(stmtStr); + stmtStr = "ALTER TABLE internal.test.tbl ADD PARTITION p20210903 VALUES [('2021-09-03'), ('2021-09-04'))" + + " DISTRIBUTED BY HASH(`k1`) BUCKETS 4;"; + alterTable(stmtStr); + stmtStr = "ALTER TABLE internal.test.tbl ADD PARTITION p20210904 VALUES [('2021-09-04'), ('2021-09-05'))" + + " DISTRIBUTED BY HASH(`k1`) BUCKETS 5;"; + alterTable(stmtStr); + checkShowTabletResultNum("internal.test.tbl", "p20210901", 2); + checkShowTabletResultNum("internal.test.tbl", "p20210902", 3); + checkShowTabletResultNum("internal.test.tbl", "p20210903", 4); + checkShowTabletResultNum("internal.test.tbl", "p20210904", 5); + + String truncateStr = "truncate table internal.test.tbl;"; + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan plan = nereidsParser.parseSingle(truncateStr); + Assertions.assertTrue(plan instanceof TruncateTableCommand); + Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan); + + checkShowTabletResultNum("internal.test.tbl", "p20210901", 2); + checkShowTabletResultNum("internal.test.tbl", "p20210902", 3); + checkShowTabletResultNum("internal.test.tbl", "p20210903", 4); + checkShowTabletResultNum("internal.test.tbl", "p20210904", 5); + + truncateStr = "truncate table internal.test.tbl partition(p20210901, p20210902, p20210903, p20210904);"; + plan = nereidsParser.parseSingle(truncateStr); + Assertions.assertTrue(plan instanceof TruncateTableCommand); + Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan); + + checkShowTabletResultNum("internal.test.tbl", "p20210901", 2); + checkShowTabletResultNum("internal.test.tbl", "p20210902", 3); + checkShowTabletResultNum("internal.test.tbl", "p20210903", 4); + checkShowTabletResultNum("internal.test.tbl", "p20210904", 5); + + truncateStr = "truncate table internal.test.tbl partition (p20210901);"; + plan = nereidsParser.parseSingle(truncateStr); + Assertions.assertTrue(plan instanceof TruncateTableCommand); + Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan); + checkShowTabletResultNum("internal.test.tbl", "p20210901", 2); + + truncateStr = "truncate table internal.test.tbl partition (p20210902);"; + plan = nereidsParser.parseSingle(truncateStr); + Assertions.assertTrue(plan instanceof TruncateTableCommand); + Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan); + checkShowTabletResultNum("internal.test.tbl", "p20210902", 3); + + truncateStr = "truncate table internal.test.tbl partition (p20210903);"; + plan = nereidsParser.parseSingle(truncateStr); + Assertions.assertTrue(plan instanceof TruncateTableCommand); + Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan); + checkShowTabletResultNum("internal.test.tbl", "p20210903", 4); + + truncateStr = "truncate table internal.test.tbl partition (p20210904);"; + plan = nereidsParser.parseSingle(truncateStr); + Assertions.assertTrue(plan instanceof TruncateTableCommand); + Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan); + checkShowTabletResultNum("internal.test.tbl", "p20210904", 5); + } + + @Test + public void testTruncateTableFailed() throws Exception { + String createTableStr = "create table internal.test.tbl2(d1 date, k1 int, k2 bigint)" + + "duplicate key(d1, k1) " + + "PARTITION BY RANGE(d1)" + + "(PARTITION p20210901 VALUES [('2021-09-01'), ('2021-09-02')))" + + "distributed by hash(k1) buckets 2 " + + "properties('replication_num' = '1');"; + createTable(createTableStr); + String partitionName = "p20210901"; + Database db = Env.getCurrentInternalCatalog().getDbNullable("test"); + OlapTable tbl2 = db.getOlapTableOrDdlException("tbl2"); + Assertions.assertNotNull(tbl2); + + Partition p20210901 = tbl2.getPartition(partitionName); + Assertions.assertNotNull(p20210901); + + long partitionId = p20210901.getId(); + p20210901.setVisibleVersionAndTime(2L, System.currentTimeMillis()); + + try { + List backendIds = Env.getCurrentSystemInfo().getAllBackendIds(); + Map> oldBackendTablets = Maps.newHashMap(); + for (long backendId : backendIds) { + Set tablets = Sets.newHashSet(Env.getCurrentInvertedIndex().getTabletIdsByBackendId(backendId)); + oldBackendTablets.put(backendId, tablets); + } + + DebugPointUtil.addDebugPoint("InternalCatalog.truncateTable.metaChanged", new DebugPointUtil.DebugPoint()); + String truncateStr = "truncate table internal.test.tbl2 partition (" + partitionName + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan plan = nereidsParser.parseSingle(truncateStr); + Assertions.assertTrue(plan instanceof TruncateTableCommand); + + ExceptionChecker.expectThrowsWithMsg(DdlException.class, + "Table[tbl2]'s meta has been changed. try again", + () -> Env.getCurrentEnv().truncateTable((TruncateTableCommand) plan)); + + Assertions.assertEquals(partitionId, tbl2.getPartition(partitionName).getId()); + for (long backendId : backendIds) { + Set tablets = Sets.newHashSet(Env.getCurrentInvertedIndex().getTabletIdsByBackendId(backendId)); + Assertions.assertEquals(oldBackendTablets.get(backendId), tablets); + } + } finally { + DebugPointUtil.removeDebugPoint("InternalCatalog.truncateTable.metaChanged"); + } + } + + private List> checkShowTabletResultNum(String tbl, String partition, int expected) throws Exception { + String showStr = "show tablets from " + tbl + " partition(" + partition + ")"; + + LogicalPlan plan = new NereidsParser().parseSingle(showStr); + Assertions.assertTrue(plan instanceof ShowTabletsFromTableCommand); + ShowResultSet showResultSet = ((ShowTabletsFromTableCommand) plan).doRun(connectContext, null); + + List> rows = showResultSet.getResultRows(); + Assertions.assertEquals(expected, rows.size()); + return rows; + } + + private void alterTable(String sql) throws Exception { + try { + LogicalPlan plan = new NereidsParser().parseSingle(sql); + Assertions.assertTrue(plan instanceof AlterTableCommand); + ((AlterTableCommand) plan).run(connectContext, null); + } catch (Exception e) { + throw e; + } + } +} diff --git a/regression-test/data/nereids_p0/ddl/test_truncate_table_nereids.out b/regression-test/data/nereids_p0/ddl/test_truncate_table_nereids.out new file mode 100644 index 00000000000000..61f081e758737a --- /dev/null +++ b/regression-test/data/nereids_p0/ddl/test_truncate_table_nereids.out @@ -0,0 +1,23 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_1 -- +2020-01-01 1.00 a 1 +2020-03-10 1.00 a 1 + +-- !select_2 -- + +-- !select_3 -- +2020-02-10 1.00 a 1 + +-- !select_4 -- + +-- !select_5 -- +2020-01-01 1.00 a 1 +2020-03-10 1.00 a 1 + +-- !select_6 -- + +-- !select_7 -- +2020-02-10 1.00 a 1 + +-- !select_8 -- + diff --git a/regression-test/suites/external_table_p0/hive/ddl/test_hive_truncate_table.groovy b/regression-test/suites/external_table_p0/hive/ddl/test_hive_truncate_table.groovy index 94082d0287f469..4341f310b37d05 100644 --- a/regression-test/suites/external_table_p0/hive/ddl/test_hive_truncate_table.groovy +++ b/regression-test/suites/external_table_p0/hive/ddl/test_hive_truncate_table.groovy @@ -22,6 +22,7 @@ suite("test_hive_truncate_table", "p0,external,hive,external_docker,external_doc String hms_port = context.config.otherConfigs.get("hive3HmsPort") String hdfs_port = context.config.otherConfigs.get("hive3HdfsPort") String catalog_name = "test_hive3_truncate_table" + String database_name = "hive_truncate" sql """drop catalog if exists ${catalog_name};""" sql """ @@ -37,64 +38,68 @@ suite("test_hive_truncate_table", "p0,external,hive,external_docker,external_doc logger.info("catalog " + catalog_name + " created") sql """switch ${catalog_name};""" logger.info("switched to catalog " + catalog_name) - sql """create database if not exists `hive_truncate`;""" - sql """use `hive_truncate`;""" + + sql """create database if not exists ${database_name};""" + sql """use ${database_name};""" + logger.info("use database " + database_name) // 1. test no partition table - sql """create table if not exists `table_no_pars`(col1 bigint, col2 string) engine=hive; """ - sql """truncate table table_no_pars;""" + String table_name = "table_no_pars" + sql """create table if not exists ${table_name}(col1 bigint, col2 string); """ + checkNereidsExecute("truncate table ${table_name};") - sql """insert into `table_no_pars` values(3234424, '44'); """ - sql """insert into `table_no_pars` values(222, 'aoe'); """ + sql """insert into ${table_name} values(3234424, '44'); """ + sql """insert into ${table_name} values(222, 'aoe'); """ order_qt_truncate_01_pre """ select * from table_no_pars; """ - sql """truncate table table_no_pars;""" - order_qt_truncate_01 """ select * from table_no_pars; """ + checkNereidsExecute("truncate table ${table_name};") + order_qt_truncate_01 """ select * from ${table_name}; """ - sql """insert into `table_no_pars` values(3234424, '44'); """ - sql """truncate table hive_truncate.table_no_pars;""" - order_qt_truncate_02 """ select * from table_no_pars; """ + sql """insert into ${table_name} values(3234424, '44'); """ + checkNereidsExecute("truncate table ${table_name};") + order_qt_truncate_02 """ select * from ${table_name}; """ - sql """insert into `table_no_pars` values(222, 'aoe'); """ - sql """truncate table ${catalog_name}.hive_truncate.table_no_pars;""" - order_qt_truncate_03 """ select * from table_no_pars; """ - sql """insert into `table_no_pars` values(222, 'aoe'); """ - sql """drop table table_no_pars """ + sql """insert into ${table_name} values(222, 'aoe'); """ + checkNereidsExecute("truncate table ${table_name};") + order_qt_truncate_03 """ select * from ${table_name}; """ + sql """insert into ${table_name} values(222, 'aoe'); """ + sql """drop table ${table_name}""" // 2. test partition table - sql """create table if not exists `table_with_pars`(col1 bigint, col2 string, pt1 varchar, pt2 date) engine=hive + table_name = "table_with_pars"; + sql """create table if not exists ${table_name}(col1 bigint, col2 string, pt1 varchar, pt2 date) partition by list(pt1, pt2)() """ - sql """truncate table table_with_pars;""" + checkNereidsExecute("truncate table ${table_name};") - sql """insert into `table_with_pars` values(33, 'awe', 'wuu', '2023-02-04') """ - sql """insert into `table_with_pars` values(5535, '3', 'dre', '2023-04-04') """ - order_qt_truncate_04_pre """ select * from table_with_pars; """ - sql """truncate table table_with_pars;""" - order_qt_truncate_04 """ select * from table_with_pars; """ + sql """insert into ${table_name} values(33, 'awe', 'wuu', '2023-02-04') """ + sql """insert into ${table_name} values(5535, '3', 'dre', '2023-04-04') """ + order_qt_truncate_04_pre """ select * from ${table_name}; """ + checkNereidsExecute("truncate table ${table_name};") + order_qt_truncate_04 """ select * from ${table_name}; """ // 3. test partition table and truncate partitions - sql """insert into `table_with_pars` values(44, 'etg', 'wuweu', '2022-02-04') """ - sql """insert into `table_with_pars` values(88, 'etg', 'wuweu', '2022-01-04') """ - sql """insert into `table_with_pars` values(095, 'etgf', 'hiyr', '2021-05-06') """ - sql """insert into `table_with_pars` values(555, 'etgf', 'wet', '2021-05-06') """ - // sql """truncate table hive_truncate.table_with_pars partition pt1;""" + sql """insert into ${table_name} values(44, 'etg', 'wuweu', '2022-02-04') """ + sql """insert into ${table_name} values(88, 'etg', 'wuweu', '2022-01-04') """ + sql """insert into ${table_name} values(095, 'etgf', 'hiyr', '2021-05-06') """ + sql """insert into ${table_name} values(555, 'etgf', 'wet', '2021-05-06') """ + // checkNereidsExecute("truncate table hive_truncate.table_with_pars partition pt1;") // order_qt_truncate_05 """ select * from table_with_pars; """ - // sql """truncate table hive_truncate.table_with_pars partition pt2;""" - order_qt_truncate_06 """ select * from table_with_pars; """ + // checkNereidsExecute("truncate table hive_truncate.table_with_pars partition pt2;") + order_qt_truncate_06 """ select * from ${table_name}; """ - sql """insert into `table_with_pars` values(22, 'ttt', 'gggw', '2022-02-04')""" - sql """insert into `table_with_pars` values(44, 'etg', 'wuweu', '2022-02-04') """ - sql """insert into `table_with_pars` values(88, 'etg', 'wuweu', '2022-01-04') """ - sql """insert into `table_with_pars` values(095, 'etgf', 'hiyr', '2021-05-06') """ - sql """insert into `table_with_pars` values(555, 'etgf', 'wet', '2021-05-06') """ - // sql """truncate table ${catalog_name}.hive_truncate.table_with_pars partition pt1;""" + sql """insert into ${table_name} values(22, 'ttt', 'gggw', '2022-02-04')""" + sql """insert into ${table_name} values(44, 'etg', 'wuweu', '2022-02-04') """ + sql """insert into ${table_name} values(88, 'etg', 'wuweu', '2022-01-04') """ + sql """insert into ${table_name} values(095, 'etgf', 'hiyr', '2021-05-06') """ + sql """insert into ${table_name} values(555, 'etgf', 'wet', '2021-05-06') """ + // checkNereidsExecute("truncate table ${catalog_name}.hive_truncate.table_with_pars partition pt1;") // order_qt_truncate_07 """ select * from table_with_pars; """ - // sql """truncate table ${catalog_name}.hive_truncate.table_with_pars partition pt2;""" + // checkNereidsExecute("truncate table ${catalog_name}.hive_truncate.table_with_pars partition pt2;") // order_qt_truncate_08 """ select * from table_with_pars; """ - sql """truncate table table_with_pars""" - order_qt_truncate_09 """ select * from table_with_pars; """ + checkNereidsExecute("truncate table ${table_name}") + order_qt_truncate_09 """ select * from ${table_name}; """ - sql """drop table table_with_pars """ - sql """drop database hive_truncate;""" + sql """drop table ${table_name};""" + sql """drop database ${database_name};""" sql """drop catalog ${catalog_name};""" } } diff --git a/regression-test/suites/nereids_p0/ddl/test_truncate_table_nereids.groovy b/regression-test/suites/nereids_p0/ddl/test_truncate_table_nereids.groovy new file mode 100644 index 00000000000000..9c8b00e49bf8d0 --- /dev/null +++ b/regression-test/suites/nereids_p0/ddl/test_truncate_table_nereids.groovy @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_truncate_table_nereids") { + def testTable = "test_truncate_table_with_partition_nereids" + + sql "DROP TABLE IF EXISTS ${testTable}" + + sql """ + CREATE TABLE ${testTable} + ( + k1 DATE, + k2 DECIMAL(10, 2) DEFAULT "10.5", + k3 CHAR(10) COMMENT "string column", + k4 INT NOT NULL DEFAULT "1" COMMENT "int column" + ) + PARTITION BY RANGE(k1) + ( + PARTITION p1 VALUES LESS THAN ("2020-02-01"), + PARTITION p2 VALUES LESS THAN ("2020-03-01"), + PARTITION p3 VALUES LESS THAN ("2020-04-01") + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + def getPartitionIds = { -> + def result = sql_return_maparray("show partitions from ${testTable}") + return result.collectEntries { [it.PartitionName, it.PartitionId as long] } + } + + def partitionIds1 = getPartitionIds() + assertEquals(["p1", "p2", "p3"].toSet(), partitionIds1.keySet()) + + sql "insert into ${testTable} values ('2020-01-01', 1.0, 'a', 1)" + sql "insert into ${testTable} values ('2020-03-10', 1.0, 'a', 1)" + order_qt_select_1 "SELECT * FROM ${testTable}" + + // if truncate without force, empty partions also kept in recycle bin. + checkNereidsExecute("truncate table ${testTable};") + + def partitionIds2 = getPartitionIds() + assertEquals(["p1", "p2", "p3"].toSet(), partitionIds2.keySet()) + assertNotEquals(partitionIds1.get("p1"), partitionIds2.get("p1")) + assertNotEquals(partitionIds1.get("p2"), partitionIds2.get("p2")) + assertNotEquals(partitionIds1.get("p3"), partitionIds2.get("p3")) + order_qt_select_2 "SELECT * FROM ${testTable}" + + sql "insert into ${testTable} values ('2020-02-10', 1.0, 'a', 1)" + order_qt_select_3 "SELECT * FROM ${testTable}" + checkNereidsExecute("truncate table ${testTable} partitions (p1, p2);") + order_qt_select_4 "SELECT * FROM ${testTable}" + + def partitionIds3 = getPartitionIds() + assertEquals(["p1", "p2", "p3"].toSet(), partitionIds3.keySet()) + assertNotEquals(partitionIds2.get("p1"), partitionIds3.get("p1")) + assertNotEquals(partitionIds2.get("p2"), partitionIds3.get("p2")) + assertEquals(partitionIds2.get("p3"), partitionIds3.get("p3")) + + sql "DROP TABLE IF EXISTS ${testTable}" + + // test truncate partition table which has no partition + testTable = "test_truncate_no_partition_table_nereids" + sql "DROP TABLE IF EXISTS ${testTable}" + sql """ + CREATE TABLE ${testTable} + ( + k1 DATE, + k2 DECIMAL(10, 2) DEFAULT "10.5", + k3 CHAR(10) COMMENT "string column", + k4 INT NOT NULL DEFAULT "1" COMMENT "int column" + ) + PARTITION BY RANGE(k1) + () + DISTRIBUTED BY HASH(k2) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + List> result = sql "show partitions from ${testTable}" + logger.info("${result}") + assertEquals(result.size(), 0) + + checkNereidsExecute("truncate table ${testTable};") + + result = sql "show partitions from ${testTable}" + logger.info("${result}") + assertEquals(result.size(), 0) + + sql "DROP TABLE IF EXISTS ${testTable}" + + // test truncate non partition table + def testNonPartitionTable = "test_truncate_non_partition_table" + sql "DROP TABLE IF EXISTS ${testNonPartitionTable}" + sql """ + CREATE TABLE ${testNonPartitionTable} + ( + k1 DATE, + k2 DECIMAL(10, 2) DEFAULT "10.5", + k3 CHAR(10) COMMENT "string column", + k4 INT NOT NULL DEFAULT "1" COMMENT "int column" + ) + DISTRIBUTED BY HASH(k2) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + List> result1 = sql "show partitions from ${testNonPartitionTable}" + logger.info("${result1}") + assertEquals(result1.size(), 1) + + checkNereidsExecute("truncate table ${testNonPartitionTable};") + result1 = sql "show partitions from ${testNonPartitionTable}" + logger.info("${result1}") + assertEquals(result1.size(), 1) + + sql "DROP TABLE IF EXISTS ${testNonPartitionTable}" + + // test truncate table force + testTable = "test_truncate_table_force_nereids" + + sql "DROP TABLE IF EXISTS ${testTable}" + + sql """ + CREATE TABLE ${testTable} + ( + k1 DATE, + k2 DECIMAL(10, 2) DEFAULT "10.5", + k3 CHAR(10) COMMENT "string column", + k4 INT NOT NULL DEFAULT "1" COMMENT "int column" + ) + PARTITION BY RANGE(k1) + ( + PARTITION p1 VALUES LESS THAN ("2020-02-01"), + PARTITION p2 VALUES LESS THAN ("2020-03-01"), + PARTITION p3 VALUES LESS THAN ("2020-04-01") + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + partitionIds1 = getPartitionIds() + assertEquals(["p1", "p2", "p3"].toSet(), partitionIds1.keySet()) + + sql "insert into ${testTable} values ('2020-01-01', 1.0, 'a', 1)" + sql "insert into ${testTable} values ('2020-03-10', 1.0, 'a', 1)" + order_qt_select_5 "SELECT * FROM ${testTable}" + + checkNereidsExecute("truncate table ${testTable} force;") + // if we use force, the empty partitions skiped, + // so that partition ID doesnt change. + partitionIds2 = getPartitionIds() + assertEquals(["p1", "p2", "p3"].toSet(), partitionIds2.keySet()) + assertNotEquals(partitionIds1.get("p1"), partitionIds2.get("p1")) + assertEquals(partitionIds1.get("p2"), partitionIds2.get("p2")) + assertNotEquals(partitionIds1.get("p3"), partitionIds2.get("p3")) + order_qt_select_6 "SELECT * FROM ${testTable}" + + sql "insert into ${testTable} values ('2020-02-10', 1.0, 'a', 1)" + order_qt_select_7 "SELECT * FROM ${testTable}" + + checkNereidsExecute("truncate table ${testTable} partitions (p1, p2) force;") + order_qt_select_8 "SELECT * FROM ${testTable}" + + partitionIds3 = getPartitionIds() + assertEquals(["p1", "p2", "p3"].toSet(), partitionIds3.keySet()) + assertEquals(partitionIds2.get("p1"), partitionIds3.get("p1")) + assertNotEquals(partitionIds2.get("p2"), partitionIds3.get("p2")) + assertEquals(partitionIds2.get("p3"), partitionIds3.get("p3")) + + sql "DROP TABLE IF EXISTS ${testTable}" +} +