From 5645743dcd94c6b81f3663e2777872af800e8788 Mon Sep 17 00:00:00 2001 From: deardeng <565620795@qq.com> Date: Sun, 14 Apr 2024 09:44:25 +0800 Subject: [PATCH 1/3] drop table when create table fail --- .../doris/datasource/InternalCatalog.java | 24 +++- .../test_create_table_exception.groovy | 127 ++++++++++++++++++ 2 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 regression-test/suites/partition_p0/test_create_table_exception.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 8c027412fcf3ca..c649538f1e0726 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -958,21 +958,26 @@ public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop, return true; } - public void replayDropTable(Database db, long tableId, boolean isForceDrop, - Long recycleTime) throws MetaNotFoundException { + public void dropTable(Database db, long tableId, boolean isForceDrop, + Long recycleTime) throws MetaNotFoundException { Table table = db.getTableOrMetaException(tableId); db.writeLock(); table.writeLock(); try { unprotectDropTable(db, table, isForceDrop, true, recycleTime); - Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(), db.getId(), - tableId); + Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(), db.getId(), tableId); + //Env.getCurrentEnv().getAnalysisManager().removeTableStats(table.getId()); } finally { table.writeUnlock(); db.writeUnlock(); } } + public void replayDropTable(Database db, long tableId, boolean isForceDrop, + Long recycleTime) throws MetaNotFoundException { + dropTable(db, tableId, isForceDrop, recycleTime); + } + public void replayEraseTable(long tableId) { Env.getCurrentRecycleBin().replayEraseTable(tableId); } @@ -2600,6 +2605,11 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx if (!result.first) { ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName); } + if (DebugPointUtil.isEnable("FE.createOlapTable.exception")) { + LOG.info("debug point FE.createOlapTable.exception, throw e"); + // not commit, not log edit + throw new DdlException("debug point FE.createOlapTable.exception"); + } if (result.second) { if (Env.getCurrentColocateIndex().isColocateTable(tableId)) { @@ -2632,6 +2642,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx TimeUtils.getCurrentFormatTime()); } } catch (DdlException e) { + LOG.warn("create table failed {} - {}", tabletIdSet, e.getMessage()); for (Long tabletId : tabletIdSet) { Env.getCurrentInvertedIndex().deleteTablet(tabletId); } @@ -2639,6 +2650,11 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx if (Env.getCurrentColocateIndex().isColocateTable(tableId)) { Env.getCurrentColocateIndex().removeTable(tableId); } + try { + dropTable(db, tableId, true, 0L); + } catch (Exception ex) { + LOG.warn("drop table", ex); + } throw e; } diff --git a/regression-test/suites/partition_p0/test_create_table_exception.groovy b/regression-test/suites/partition_p0/test_create_table_exception.groovy new file mode 100644 index 00000000000000..49cadcd3af4bf9 --- /dev/null +++ b/regression-test/suites/partition_p0/test_create_table_exception.groovy @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.NodeType +import org.apache.doris.regression.suite.SuiteCluster + +suite("test_create_table_exception") { + def options = new ClusterOptions() + options.enableDebugPoints() + options.setFeNum(3) + options.feConfigs.add('max_dynamic_partition_num=2000') + + docker(options) { + sleep 2000 + def table1 = "normal_table" + def table2 = "range_table" + def table3 = "dynamic_partition_table" + try { + GetDebugPoint().enableDebugPointForAllFEs('FE.createOlapTable.exception', null) + def createTable = { -> + try_sql """ + CREATE TABLE $table1 ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 10 + PROPERTIES ( + "replication_num"="3" + ); + """ + + try_sql """ + CREATE TABLE IF NOT EXISTS $table2 ( + lo_orderdate int(11) NOT NULL COMMENT "", + lo_orderkey bigint(20) NOT NULL COMMENT "", + lo_linenumber bigint(20) NOT NULL COMMENT "", + lo_custkey int(11) NOT NULL COMMENT "", + lo_partkey int(11) NOT NULL COMMENT "", + lo_suppkey int(11) NOT NULL COMMENT "", + lo_orderpriority varchar(64) NOT NULL COMMENT "", + lo_shippriority int(11) NOT NULL COMMENT "", + lo_quantity bigint(20) NOT NULL COMMENT "", + lo_extendedprice bigint(20) NOT NULL COMMENT "", + lo_ordtotalprice bigint(20) NOT NULL COMMENT "", + lo_discount bigint(20) NOT NULL COMMENT "", + lo_revenue bigint(20) NOT NULL COMMENT "", + lo_supplycost bigint(20) NOT NULL COMMENT "", + lo_tax bigint(20) NOT NULL COMMENT "", + lo_commitdate bigint(20) NOT NULL COMMENT "", + lo_shipmode varchar(64) NOT NULL COMMENT "" ) + ENGINE=OLAP + UNIQUE KEY(lo_orderdate, lo_orderkey, lo_linenumber) + COMMENT "OLAP" + PARTITION BY RANGE(lo_orderdate) ( + PARTITION p1992 VALUES [("-2147483648"), ("19930101")), + PARTITION p1993 VALUES [("19930101"), ("19940101")), + PARTITION p1994 VALUES [("19940101"), ("19950101")), + PARTITION p1995 VALUES [("19950101"), ("19960101")), + PARTITION p1996 VALUES [("19960101"), ("19970101")), + PARTITION p1997 VALUES [("19970101"), ("19980101")), + PARTITION p1998 VALUES [("19980101"), ("19990101"))) + DISTRIBUTED BY HASH(lo_orderkey) BUCKETS 48; + """ + + try_sql """ + CREATE TABLE $table3 ( + time date, + key1 int, + key2 int, + value1 int, + value2 int + ) ENGINE = OLAP UNIQUE KEY( + `time`, + `key1`, + `key2` + ) COMMENT 'OLAP' PARTITION BY RANGE(`time`)() + DISTRIBUTED BY HASH(`key1`) BUCKETS 6 PROPERTIES ( + "file_cache_ttl_seconds" = "0", + "bloom_filter_columns" = "time", + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.time_zone" = "Asia/Shanghai", + "dynamic_partition.start" = "-730", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "2", + "dynamic_partition.create_history_partition" = "true", + "dynamic_partition.history_partition_num" = "-1", + "dynamic_partition.hot_partition_num" = "0", + "dynamic_partition.reserved_history_periods" = "NULL", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true" + ); + """ + } + createTable() + def result = sql """show tables;""" + assertEquals(result.size(), 0) + GetDebugPoint().disableDebugPointForAllFEs('FE.createOlapTable.exception') + createTable() + result = sql """show tables;""" + log.info(result.toString()) + assertEquals(result.size(), 3) + } finally { + GetDebugPoint().disableDebugPointForAllFEs('FE.createOlapTable.exception') + sql """drop table if exists ${table1}""" + sql """drop table if exists ${table2}""" + sql """drop table if exists ${table3}""" + } + } +} From 382debf4839d5169cda4fbab0bc2480c0586e1e2 Mon Sep 17 00:00:00 2001 From: yujun Date: Tue, 9 Jul 2024 09:38:52 +0800 Subject: [PATCH 2/3] [fix](create table) create table fail not write drop table editlog (#37488) Fix: for creating table, if create dynamic partition throw exception (pr #35778), then create table will fail, then drop table, but it forget to write an editlog. --- .../doris/datasource/InternalCatalog.java | 29 ++++++++----- .../test_create_table_exception.groovy | 43 +++++++++++++------ 2 files changed, 49 insertions(+), 23 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index c649538f1e0726..29a8d20321b5e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -958,13 +958,13 @@ public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop, return true; } - public void dropTable(Database db, long tableId, boolean isForceDrop, + private void dropTable(Database db, long tableId, boolean isForceDrop, boolean isReplay, Long recycleTime) throws MetaNotFoundException { Table table = db.getTableOrMetaException(tableId); db.writeLock(); table.writeLock(); try { - unprotectDropTable(db, table, isForceDrop, true, recycleTime); + unprotectDropTable(db, table, isForceDrop, isReplay, recycleTime); Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(), db.getId(), tableId); //Env.getCurrentEnv().getAnalysisManager().removeTableStats(table.getId()); } finally { @@ -975,7 +975,7 @@ public void dropTable(Database db, long tableId, boolean isForceDrop, public void replayDropTable(Database db, long tableId, boolean isForceDrop, Long recycleTime) throws MetaNotFoundException { - dropTable(db, tableId, isForceDrop, recycleTime); + dropTable(db, tableId, isForceDrop, true, recycleTime); } public void replayEraseTable(long tableId) { @@ -2473,6 +2473,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx // if failed in any step, use this set to do clear things Set tabletIdSet = new HashSet<>(); // create partition + boolean hadLogEditCreateTable = false; try { if (partitionInfo.getType() == PartitionType.UNPARTITIONED) { if (storagePolicy.equals("") && properties != null && !properties.isEmpty()) { @@ -2605,11 +2606,6 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx if (!result.first) { ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName); } - if (DebugPointUtil.isEnable("FE.createOlapTable.exception")) { - LOG.info("debug point FE.createOlapTable.exception, throw e"); - // not commit, not log edit - throw new DdlException("debug point FE.createOlapTable.exception"); - } if (result.second) { if (Env.getCurrentColocateIndex().isColocateTable(tableId)) { @@ -2623,6 +2619,9 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx LOG.info("duplicate create table[{};{}], skip next steps", tableName, tableId); tableHasExists = true; } else { + // if table not exists, then db.createTableWithLock will write an editlog. + hadLogEditCreateTable = true; + // we have added these index to memory, only need to persist here if (Env.getCurrentColocateIndex().isColocateTable(tableId)) { GroupId groupId = Env.getCurrentColocateIndex().getGroup(tableId); @@ -2641,17 +2640,27 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx .createOrUpdateRuntimeInfo(tableId, DynamicPartitionScheduler.LAST_UPDATE_TIME, TimeUtils.getCurrentFormatTime()); } + + if (DebugPointUtil.isEnable("FE.createOlapTable.exception")) { + LOG.info("debug point FE.createOlapTable.exception, throw e"); + throw new DdlException("debug point FE.createOlapTable.exception"); + } } catch (DdlException e) { LOG.warn("create table failed {} - {}", tabletIdSet, e.getMessage()); for (Long tabletId : tabletIdSet) { Env.getCurrentInvertedIndex().deleteTablet(tabletId); } - // only remove from memory, because we have not persist it + // edit log write DropTableInfo will result in deleting colocate group, + // but follow fe may need wait 30s (recycle bin mgr run every 30s). if (Env.getCurrentColocateIndex().isColocateTable(tableId)) { Env.getCurrentColocateIndex().removeTable(tableId); } try { - dropTable(db, tableId, true, 0L); + dropTable(db, tableId, true, false, 0L); + if (hadLogEditCreateTable) { + DropInfo info = new DropInfo(db.getId(), tableId, olapTable.getName(), -1L, true, 0L); + Env.getCurrentEnv().getEditLog().logDropTable(info); + } } catch (Exception ex) { LOG.warn("drop table", ex); } diff --git a/regression-test/suites/partition_p0/test_create_table_exception.groovy b/regression-test/suites/partition_p0/test_create_table_exception.groovy index 49cadcd3af4bf9..96f097c76705f2 100644 --- a/regression-test/suites/partition_p0/test_create_table_exception.groovy +++ b/regression-test/suites/partition_p0/test_create_table_exception.groovy @@ -31,9 +31,9 @@ suite("test_create_table_exception") { def table3 = "dynamic_partition_table" try { GetDebugPoint().enableDebugPointForAllFEs('FE.createOlapTable.exception', null) - def createTable = { -> + def createTable = { tableIdx -> try_sql """ - CREATE TABLE $table1 ( + CREATE TABLE ${table1}_${tableIdx} ( `k1` int(11) NULL, `k2` int(11) NULL ) @@ -41,12 +41,13 @@ suite("test_create_table_exception") { COMMENT 'OLAP' DISTRIBUTED BY HASH(`k1`) BUCKETS 10 PROPERTIES ( + "colocate_with" = "col_grp_${tableIdx}", "replication_num"="3" ); """ try_sql """ - CREATE TABLE IF NOT EXISTS $table2 ( + CREATE TABLE IF NOT EXISTS ${table2}_${tableIdx} ( lo_orderdate int(11) NOT NULL COMMENT "", lo_orderkey bigint(20) NOT NULL COMMENT "", lo_linenumber bigint(20) NOT NULL COMMENT "", @@ -79,7 +80,7 @@ suite("test_create_table_exception") { """ try_sql """ - CREATE TABLE $table3 ( + CREATE TABLE ${table3}_${tableIdx} ( time date, key1 int, key2 int, @@ -109,19 +110,35 @@ suite("test_create_table_exception") { ); """ } - createTable() + createTable(1) def result = sql """show tables;""" assertEquals(result.size(), 0) + + def checkResult = { -> + def tables = sql """show tables;""" + log.info("tables=" + tables) + assertEquals(3, tables.size()) + + def groups = sql """ show proc "/colocation_group" """ + log.info("groups=" + groups) + assertEquals(1, groups.size()) + } + GetDebugPoint().disableDebugPointForAllFEs('FE.createOlapTable.exception') - createTable() - result = sql """show tables;""" - log.info(result.toString()) - assertEquals(result.size(), 3) + createTable(2) + checkResult() + + sleep 1000 + cluster.restartFrontends(cluster.getMasterFe().index) + sleep 32_000 + def newMasterFe = cluster.getMasterFe() + def newMasterFeUrl = "jdbc:mysql://${newMasterFe.host}:${newMasterFe.queryPort}/?useLocalSessionState=false&allowLoadLocalInfile=false" + newMasterFeUrl = context.config.buildUrlWithDb(newMasterFeUrl, context.dbName) + connect('root', '', newMasterFeUrl) { + checkResult() + } + } finally { - GetDebugPoint().disableDebugPointForAllFEs('FE.createOlapTable.exception') - sql """drop table if exists ${table1}""" - sql """drop table if exists ${table2}""" - sql """drop table if exists ${table3}""" } } } From 64290ffbf1b359ca77a868496a7c61e4cebd267a Mon Sep 17 00:00:00 2001 From: yujun777 Date: Tue, 9 Jul 2024 11:39:38 +0800 Subject: [PATCH 3/3] update --- .../suites/partition_p0/test_create_table_exception.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/regression-test/suites/partition_p0/test_create_table_exception.groovy b/regression-test/suites/partition_p0/test_create_table_exception.groovy index 96f097c76705f2..c3069d8e592fe4 100644 --- a/regression-test/suites/partition_p0/test_create_table_exception.groovy +++ b/regression-test/suites/partition_p0/test_create_table_exception.groovy @@ -92,7 +92,6 @@ suite("test_create_table_exception") { `key2` ) COMMENT 'OLAP' PARTITION BY RANGE(`time`)() DISTRIBUTED BY HASH(`key1`) BUCKETS 6 PROPERTIES ( - "file_cache_ttl_seconds" = "0", "bloom_filter_columns" = "time", "dynamic_partition.enable" = "true", "dynamic_partition.time_unit" = "DAY",