Skip to content

[fix](create table) create table fail not write drop table editlog #37488

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 9 commits into from
Jul 9, 2024
Merged

[fix](create table) create table fail not write drop table editlog #37488

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
8bada3c
fix no drop table
yujun777 Jul 8, 2024
af0ff33
no need log drop colocate
yujun777 Jul 8, 2024
71471f3
update
yujun777 Jul 8, 2024
a0d5c3e
update
yujun777 Jul 8, 2024
a49e8de
update
yujun777 Jul 8, 2024
94103d0
update
yujun777 Jul 8, 2024
87b7136
update
yujun777 Jul 8, 2024
472787e
udpate
yujun777 Jul 8, 2024
9261095
update
yujun777 Jul 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 19 additions & 10 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1001,13 +1001,13 @@ public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop,
return true;
}

public void dropTable(Database db, long tableId, boolean isForceDrop,
private void dropTable(Database db, long tableId, boolean isForceDrop, boolean isReplay,
Long recycleTime) throws MetaNotFoundException {
Table table = db.getTableOrMetaException(tableId);
db.writeLock();
table.writeLock();
try {
unprotectDropTable(db, table, isForceDrop, true, recycleTime);
unprotectDropTable(db, table, isForceDrop, isReplay, recycleTime);
Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(), db.getId(), tableId);
Env.getCurrentEnv().getAnalysisManager().removeTableStats(table.getId());
} finally {
Expand All @@ -1018,7 +1018,7 @@ public void dropTable(Database db, long tableId, boolean isForceDrop,

public void replayDropTable(Database db, long tableId, boolean isForceDrop,
Long recycleTime) throws MetaNotFoundException {
dropTable(db, tableId, isForceDrop, recycleTime);
dropTable(db, tableId, isForceDrop, true, recycleTime);
}

public void replayEraseTable(long tableId) {
Expand Down Expand Up @@ -2804,6 +2804,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
// if failed in any step, use this set to do clear things
Set<Long> tabletIdSet = new HashSet<>();
// create partition
boolean hadLogEditCreateTable = false;
try {
if (partitionInfo.getType() == PartitionType.UNPARTITIONED) {
if (properties != null && !properties.isEmpty()) {
Expand Down Expand Up @@ -2935,11 +2936,6 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
if (!result.first) {
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName);
}
if (DebugPointUtil.isEnable("FE.createOlapTable.exception")) {
LOG.info("debug point FE.createOlapTable.exception, throw e");
// not commit, not log edit
throw new DdlException("debug point FE.createOlapTable.exception");
}

if (result.second) {
if (Env.getCurrentColocateIndex().isColocateTable(tableId)) {
Expand All @@ -2952,6 +2948,9 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
}
LOG.info("duplicate create table[{};{}], skip next steps", tableName, tableId);
} else {
// if table not exists, then db.createTableWithLock will write an editlog.
hadLogEditCreateTable = true;

// we have added these index to memory, only need to persist here
if (Env.getCurrentColocateIndex().isColocateTable(tableId)) {
GroupId groupId = Env.getCurrentColocateIndex().getGroup(tableId);
Expand All @@ -2970,17 +2969,27 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
.createOrUpdateRuntimeInfo(tableId, DynamicPartitionScheduler.LAST_UPDATE_TIME,
TimeUtils.getCurrentFormatTime());
}

if (DebugPointUtil.isEnable("FE.createOlapTable.exception")) {
LOG.info("debug point FE.createOlapTable.exception, throw e");
throw new DdlException("debug point FE.createOlapTable.exception");
}
} catch (DdlException e) {
LOG.warn("create table failed {} - {}", tabletIdSet, e.getMessage());
for (Long tabletId : tabletIdSet) {
Env.getCurrentInvertedIndex().deleteTablet(tabletId);
}
// only remove from memory, because we have not persist it
// edit log write DropTableInfo will result in deleting colocate group,
// but follow fe may need wait 30s (recycle bin mgr run every 30s).
if (Env.getCurrentColocateIndex().isColocateTable(tableId)) {
Env.getCurrentColocateIndex().removeTable(tableId);
}
try {
dropTable(db, tableId, true, 0L);
dropTable(db, tableId, true, false, 0L);
if (hadLogEditCreateTable) {
DropInfo info = new DropInfo(db.getId(), tableId, olapTable.getName(), -1L, true, 0L);
Env.getCurrentEnv().getEditLog().logDropTable(info);
}
} catch (Exception ex) {
LOG.warn("drop table", ex);
}
Expand Down
43 changes: 30 additions & 13 deletions regression-test/suites/partition_p0/test_create_table_exception.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,22 +31,23 @@ suite("test_create_table_exception") {
def table3 = "dynamic_partition_table"
try {
GetDebugPoint().enableDebugPointForAllFEs('FE.createOlapTable.exception', null)
def createTable = { ->
def createTable = { tableIdx ->
try_sql """
CREATE TABLE $table1 (
CREATE TABLE ${table1}_${tableIdx} (
`k1` int(11) NULL,
`k2` int(11) NULL
)
DUPLICATE KEY(`k1`, `k2`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 10
PROPERTIES (
"colocate_with" = "col_grp_${tableIdx}",
"replication_num"="3"
);
"""

try_sql """
CREATE TABLE IF NOT EXISTS $table2 (
CREATE TABLE IF NOT EXISTS ${table2}_${tableIdx} (
lo_orderdate int(11) NOT NULL COMMENT "",
lo_orderkey bigint(20) NOT NULL COMMENT "",
lo_linenumber bigint(20) NOT NULL COMMENT "",
Expand Down Expand Up @@ -79,7 +80,7 @@ suite("test_create_table_exception") {
"""

try_sql """
CREATE TABLE $table3 (
CREATE TABLE ${table3}_${tableIdx} (
time date,
key1 int,
key2 int,
Expand Down Expand Up @@ -109,19 +110,35 @@ suite("test_create_table_exception") {
);
"""
}
createTable()
createTable(1)
def result = sql """show tables;"""
assertEquals(result.size(), 0)

def checkResult = { ->
def tables = sql """show tables;"""
log.info("tables=" + tables)
assertEquals(3, tables.size())

def groups = sql """ show proc "/colocation_group" """
log.info("groups=" + groups)
assertEquals(1, groups.size())
}

GetDebugPoint().disableDebugPointForAllFEs('FE.createOlapTable.exception')
createTable()
result = sql """show tables;"""
log.info(result.toString())
assertEquals(result.size(), 3)
createTable(2)
checkResult()

sleep 1000
cluster.restartFrontends(cluster.getMasterFe().index)
sleep 32_000
def newMasterFe = cluster.getMasterFe()
def newMasterFeUrl = "jdbc:mysql://${newMasterFe.host}:${newMasterFe.queryPort}/?useLocalSessionState=false&allowLoadLocalInfile=false"
newMasterFeUrl = context.config.buildUrlWithDb(newMasterFeUrl, context.dbName)
connect('root', '', newMasterFeUrl) {
checkResult()
}

} finally {
GetDebugPoint().disableDebugPointForAllFEs('FE.createOlapTable.exception')
sql """drop table if exists ${table1}"""
sql """drop table if exists ${table2}"""
sql """drop table if exists ${table3}"""
}
}
}