From f74ff0e6be670e9fb45555a999987a0f5733aa10 Mon Sep 17 00:00:00 2001 From: lamb Date: Wed, 10 Apr 2024 13:33:59 +0800 Subject: [PATCH 1/4] save --- .../apache/doris/datasource/InternalCatalog.java | 11 +++++++++-- .../suites/table_p0/test_table_version.groovy | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index d0e52b9ff05216..82da01610b823f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1817,9 +1817,16 @@ public void dropPartition(Database db, OlapTable olapTable, DropPartitionClause recycleTime = Env.getCurrentRecycleBin().getRecycleTimeById(partition.getId()); } } - long version = olapTable.getNextVersion(); + + long version = -1L; long versionTime = System.currentTimeMillis(); - olapTable.updateVisibleVersionAndTime(version, versionTime); + // table version in cloud mode should already updated + if (Config.isCloudMode()) { + version = olapTable.getVisibleVersion(); + } else { + version = olapTable.getNextVersion(); + olapTable.updateVisibleVersionAndTime(version, versionTime); + } // log DropPartitionInfo info = new DropPartitionInfo(db.getId(), olapTable.getId(), partitionName, isTempPartition, clause.isForceDrop(), recycleTime, version, versionTime); diff --git a/regression-test/suites/table_p0/test_table_version.groovy b/regression-test/suites/table_p0/test_table_version.groovy index 3b60a49b1c4807..a500a221731a10 100644 --- a/regression-test/suites/table_p0/test_table_version.groovy +++ b/regression-test/suites/table_p0/test_table_version.groovy @@ -48,12 +48,26 @@ suite("test_table_version") { visibleVersion = getTableVersion(dbId,tableNameNum); assertEquals(2, visibleVersion); + sql """ + insert into ${tableNameNum} values(1,"2017-02-15",1); + """ + visibleVersion = getTableVersion(dbId,tableNameNum); + assertEquals(3, visibleVersion); + + // drop an empty partition will not add table version sql """ alter table ${tableNameNum} drop partition p201703_all force; """ visibleVersion = getTableVersion(dbId,tableNameNum); assertEquals(3, visibleVersion); + // drop an non-empty partition will add table version + sql """ + alter table ${tableNameNum} drop partition p201702_2000 force; + """ + visibleVersion = getTableVersion(dbId,tableNameNum); + assertEquals(4, visibleVersion); + sql """ ALTER TABLE ${tableNameNum} ADD TEMPORARY PARTITION p201702_2000_1 VALUES [('2017-02-01'), ('2017-03-01')); """ From a43f28ec74cfb01f34406524d3e806dcee6377c2 Mon Sep 17 00:00:00 2001 From: lamb Date: Wed, 10 Apr 2024 14:39:00 +0800 Subject: [PATCH 2/4] save --- .../org/apache/doris/catalog/OlapTable.java | 3 ++- .../doris/datasource/InternalCatalog.java | 23 +++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index e50ba4a0ca46f6..eb563945735183 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -2300,12 +2300,13 @@ public boolean checkPartitionNameExist(String partitionName, boolean isTempParti // drop temp partition. if needDropTablet is true, tablets of this temp partition // will be dropped from tablet inverted index. - public void dropTempPartition(String partitionName, boolean needDropTablet) { + public Partition dropTempPartition(String partitionName, boolean needDropTablet) { Partition partition = getPartition(partitionName, true); if (partition != null) { partitionInfo.dropPartition(partition.getId()); tempPartitions.dropPartition(partitionName, needDropTablet); } + return partition; } /* diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 82da01610b823f..d2de21d451645d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1795,10 +1795,11 @@ public void dropPartition(Database db, OlapTable olapTable, DropPartitionClause // drop long recycleTime = 0; + + Partition partition = null; if (isTempPartition) { - olapTable.dropTempPartition(partitionName, true); + partition = olapTable.dropTempPartition(partitionName, true); } else { - Partition partition = null; if (!clause.isForceDrop()) { partition = olapTable.getPartition(partitionName); if (partition != null) { @@ -1812,21 +1813,23 @@ public void dropPartition(Database db, OlapTable olapTable, DropPartitionClause } } } - olapTable.dropPartition(db.getId(), partitionName, clause.isForceDrop()); + partition = olapTable.dropPartition(db.getId(), partitionName, clause.isForceDrop()); if (!clause.isForceDrop() && partition != null) { recycleTime = Env.getCurrentRecycleBin().getRecycleTimeById(partition.getId()); } } - long version = -1L; - long versionTime = System.currentTimeMillis(); - // table version in cloud mode should already updated - if (Config.isCloudMode()) { - version = olapTable.getVisibleVersion(); - } else { - version = olapTable.getNextVersion(); + long version = olapTable.getVisibleVersion(); + long versionTime = olapTable.getVisibleVersionTime(); + // Only update table version if drop a non-empty partition + if (partition != null && partition.hasData()) { + if (Config.isNotCloudMode()) { + version = olapTable.getNextVersion(); + } + versionTime = System.currentTimeMillis(); olapTable.updateVisibleVersionAndTime(version, versionTime); } + // log DropPartitionInfo info = new DropPartitionInfo(db.getId(), olapTable.getId(), partitionName, isTempPartition, clause.isForceDrop(), recycleTime, version, versionTime); From b2585e6dc474fc1615421250818c3c7010c3f21d Mon Sep 17 00:00:00 2001 From: lamb Date: Wed, 10 Apr 2024 14:46:18 +0800 Subject: [PATCH 3/4] save --- .../java/org/apache/doris/datasource/InternalCatalog.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index d2de21d451645d..5c9f3fd4e4e278 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1823,11 +1823,11 @@ public void dropPartition(Database db, OlapTable olapTable, DropPartitionClause long versionTime = olapTable.getVisibleVersionTime(); // Only update table version if drop a non-empty partition if (partition != null && partition.hasData()) { + versionTime = System.currentTimeMillis(); if (Config.isNotCloudMode()) { version = olapTable.getNextVersion(); + olapTable.updateVisibleVersionAndTime(version, versionTime); } - versionTime = System.currentTimeMillis(); - olapTable.updateVisibleVersionAndTime(version, versionTime); } // log From b39c18306535860b58f440e887eb96f555de4caf Mon Sep 17 00:00:00 2001 From: lamb Date: Wed, 10 Apr 2024 16:04:55 +0800 Subject: [PATCH 4/4] save --- .../suites/table_p0/test_table_version.groovy | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/regression-test/suites/table_p0/test_table_version.groovy b/regression-test/suites/table_p0/test_table_version.groovy index a500a221731a10..f8c4c605a52841 100644 --- a/regression-test/suites/table_p0/test_table_version.groovy +++ b/regression-test/suites/table_p0/test_table_version.groovy @@ -33,7 +33,8 @@ suite("test_table_version") { PARTITION BY RANGE(`date`) (PARTITION p201701_1000 VALUES [('0000-01-01'), ('2017-02-01')), PARTITION p201702_2000 VALUES [('2017-02-01'), ('2017-03-01')), - PARTITION p201703_all VALUES [('2017-03-01'), ('2017-04-01'))) + PARTITION p201703_3000 VALUES [('2017-03-01'), ('2017-04-01')), + PARTITION p201704_all VALUES [('2017-04-01'), ('2017-05-01'))) DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 PROPERTIES ('replication_num' = '1') ; """ @@ -49,21 +50,21 @@ suite("test_table_version") { assertEquals(2, visibleVersion); sql """ - insert into ${tableNameNum} values(1,"2017-02-15",1); + insert into ${tableNameNum} values(1,"2017-03-15",1); """ visibleVersion = getTableVersion(dbId,tableNameNum); assertEquals(3, visibleVersion); // drop an empty partition will not add table version sql """ - alter table ${tableNameNum} drop partition p201703_all force; + alter table ${tableNameNum} drop partition p201704_all force; """ visibleVersion = getTableVersion(dbId,tableNameNum); assertEquals(3, visibleVersion); // drop an non-empty partition will add table version sql """ - alter table ${tableNameNum} drop partition p201702_2000 force; + alter table ${tableNameNum} drop partition p201703_3000 force; """ visibleVersion = getTableVersion(dbId,tableNameNum); assertEquals(4, visibleVersion); @@ -75,7 +76,7 @@ suite("test_table_version") { ALTER TABLE ${tableNameNum} REPLACE PARTITION (p201702_2000) WITH TEMPORARY PARTITION (p201702_2000_1); """ visibleVersion = getTableVersion(dbId,tableNameNum); - assertEquals(4, visibleVersion); + assertEquals(5, visibleVersion); sql """drop table if exists `${tableNameNum}`""" }