From 5acd952804da3627974aa0d32897cfcbe76dbe61 Mon Sep 17 00:00:00 2001 From: tsy Date: Tue, 14 Nov 2023 19:59:27 +0800 Subject: [PATCH 1/4] fix updated rows --- .../apache/doris/transaction/PublishVersionDaemon.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java index 922e8645d9f17a..e3bb5b79bbf047 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java @@ -129,7 +129,7 @@ private void publishVersion() { AgentTaskExecutor.submit(batchTask); } - Map tableIdToNumDeltaRows = Maps.newHashMap(); + Map tableIdToTotalDeltaNumRows = Maps.newHashMap(); // try to finish the transaction, if failed just retry in next loop for (TransactionState transactionState : readyTransactionStates) { Stream publishVersionTaskStream = transactionState @@ -141,15 +141,15 @@ private void publishVersion() { Map tableIdToDeltaNumRows = task.getTableIdToDeltaNumRows(); tableIdToDeltaNumRows.forEach((tableId, numRows) -> { - tableIdToDeltaNumRows + tableIdToTotalDeltaNumRows .computeIfPresent(tableId, (id, orgNumRows) -> orgNumRows + numRows); - tableIdToNumDeltaRows.putIfAbsent(tableId, numRows); + tableIdToTotalDeltaNumRows.putIfAbsent(tableId, numRows); }); } }); boolean hasBackendAliveAndUnfinishedTask = publishVersionTaskStream .anyMatch(task -> !task.isFinished() && infoService.checkBackendAlive(task.getBackendId())); - transactionState.setTableIdToTotalNumDeltaRows(tableIdToNumDeltaRows); + transactionState.setTableIdToTotalNumDeltaRows(tableIdToTotalDeltaNumRows); boolean shouldFinishTxn = !hasBackendAliveAndUnfinishedTask || transactionState.isPublishTimeout() || DebugPointUtil.isEnable("PublishVersionDaemon.not_wait_unfinished_tasks"); From 50baf9748af7570c1901298da7eaf79fd42ca9ca Mon Sep 17 00:00:00 2001 From: tsy Date: Tue, 14 Nov 2023 20:41:06 +0800 Subject: [PATCH 2/4] add regression test --- .../suites/statistics/analyze_stats.groovy | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy index 92692559193341..bc8e3ec7bf06ef 100644 --- a/regression-test/suites/statistics/analyze_stats.groovy +++ b/regression-test/suites/statistics/analyze_stats.groovy @@ -1214,5 +1214,28 @@ PARTITION `p599` VALUES IN (599) assert all_finished(show_result) + sql """ + CREATE TABLE test_updated_rows ( + `col1` varchar(16) NOT NULL, + `col2` int(11) NOT NULL, + `col3` int(11) NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`col1`) + DISTRIBUTED BY HASH(`col1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false" + ); + """ + sql """ INSERT INTO test_updated_rows VALUES('1',1,1); """ + def cnt = sql """ SHOW TABLE STATS test_updated_rows """ + assertEquals(Integer.valueOf(cnt[1][0]), 1) + sql """ INSERT INTO test_updated_rows SELECT * FROM test_updated_rows """ + sql """ INSERT INTO test_updated_rows SELECT * FROM test_updated_rows """ + sql """ INSERT INTO test_updated_rows SELECT * FROM test_updated_rows """ + assertEquals(Integer.valueOf(cnt[1][0]), 8) } From d9f8528fca0186b0cc217a857071fa4955db8bca Mon Sep 17 00:00:00 2001 From: tsy Date: Wed, 15 Nov 2023 10:11:17 +0800 Subject: [PATCH 3/4] mod case --- .../suites/statistics/analyze_stats.groovy | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy index bc8e3ec7bf06ef..029243082c15ec 100644 --- a/regression-test/suites/statistics/analyze_stats.groovy +++ b/regression-test/suites/statistics/analyze_stats.groovy @@ -1140,6 +1140,35 @@ PARTITION `p599` VALUES IN (599) afterDropped = sql """SHOW TABLE STATS test_meta_management""" assert check_column(afterDropped, "[col1, col2, col3]") + sql """ DROP TABLE IF EXISTS test_updated_rows """ + sql """ + CREATE TABLE test_updated_rows ( + `col1` varchar(16) NOT NULL, + `col2` int(11) NOT NULL, + `col3` int(11) NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`col1`) + DISTRIBUTED BY HASH(`col1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false" + ); + """ + + sql """ INSERT INTO test_updated_rows VALUES('1',1,1); """ + sql """ANALYZE TABLE test_updated_rows WITH SYNC""" + def cnt1 = sql """ SHOW TABLE STATS test_updated_rows """ + assertEquals(Integer.valueOf(cnt1[0][0]), 1) + sql """ INSERT INTO test_updated_rows SELECT * FROM test_updated_rows """ + sql """ INSERT INTO test_updated_rows SELECT * FROM test_updated_rows """ + sql """ INSERT INTO test_updated_rows SELECT * FROM test_updated_rows """ + sql """ANALYZE TABLE test_updated_rows WITH SYNC""" + def cnt2 = sql """ SHOW TABLE STATS test_updated_rows """ + assertEquals(Integer.valueOf(cnt2[0][0]), 8) + // test analyze specific column sql """CREATE TABLE test_analyze_specific_column (col1 varchar(11451) not null, col2 int not null, col3 int not null) DUPLICATE KEY(col1) @@ -1213,29 +1242,4 @@ PARTITION `p599` VALUES IN (599) } assert all_finished(show_result) - - sql """ - CREATE TABLE test_updated_rows ( - `col1` varchar(16) NOT NULL, - `col2` int(11) NOT NULL, - `col3` int(11) NOT NULL - ) ENGINE=OLAP - DUPLICATE KEY(`col1`) - DISTRIBUTED BY HASH(`col1`) BUCKETS 3 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "storage_format" = "V2", - "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" - ); - """ - - sql """ INSERT INTO test_updated_rows VALUES('1',1,1); """ - def cnt = sql """ SHOW TABLE STATS test_updated_rows """ - assertEquals(Integer.valueOf(cnt[1][0]), 1) - sql """ INSERT INTO test_updated_rows SELECT * FROM test_updated_rows """ - sql """ INSERT INTO test_updated_rows SELECT * FROM test_updated_rows """ - sql """ INSERT INTO test_updated_rows SELECT * FROM test_updated_rows """ - assertEquals(Integer.valueOf(cnt[1][0]), 8) } From 6b28f62c64b5343cebfa70fe645d2620a6108cac Mon Sep 17 00:00:00 2001 From: tsy Date: Wed, 15 Nov 2023 15:53:25 +0800 Subject: [PATCH 4/4] mod case --- regression-test/suites/statistics/analyze_stats.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy index 029243082c15ec..08f3f931dbc0ca 100644 --- a/regression-test/suites/statistics/analyze_stats.groovy +++ b/regression-test/suites/statistics/analyze_stats.groovy @@ -1158,6 +1158,7 @@ PARTITION `p599` VALUES IN (599) ); """ + sql """ANALYZE TABLE test_updated_rows WITH SYNC""" sql """ INSERT INTO test_updated_rows VALUES('1',1,1); """ sql """ANALYZE TABLE test_updated_rows WITH SYNC""" def cnt1 = sql """ SHOW TABLE STATS test_updated_rows """