From 38c80cd54b404bfc15573fc8bf4e6a65a521c3d1 Mon Sep 17 00:00:00 2001 From: huanghaibin <284824253@qq.com> Date: Wed, 20 Dec 2023 10:31:05 +0800 Subject: [PATCH 1/5] [fix](group commit)Fix the issue of duplicate addition of wal path when encouter exception --- be/src/runtime/group_commit_mgr.cpp | 2 -- .../java/org/apache/doris/common/util/PropertyAnalyzer.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/be/src/runtime/group_commit_mgr.cpp b/be/src/runtime/group_commit_mgr.cpp index 16c841cc0f3808..d7b168c7eee6df 100644 --- a/be/src/runtime/group_commit_mgr.cpp +++ b/be/src/runtime/group_commit_mgr.cpp @@ -333,7 +333,6 @@ Status GroupCommitTable::_finish_group_commit_load(int64_t db_id, int64_t table_ << ", executor status=" << status.to_string() << ", request commit status=" << st.to_string(); if (!prepare_failed) { - RETURN_IF_ERROR(_exec_env->wal_mgr()->add_wal_path(_db_id, table_id, txn_id, label)); std::string wal_path; RETURN_IF_ERROR(_exec_env->wal_mgr()->get_wal_path(txn_id, wal_path)); RETURN_IF_ERROR(_exec_env->wal_mgr()->add_recover_wal( @@ -350,7 +349,6 @@ Status GroupCommitTable::_finish_group_commit_load(int64_t db_id, int64_t table_ // TODO handle execute and commit error if (!prepare_failed && !result_status.ok() && !(result_status.is())) { - RETURN_IF_ERROR(_exec_env->wal_mgr()->add_wal_path(_db_id, table_id, txn_id, label)); std::string wal_path; RETURN_IF_ERROR(_exec_env->wal_mgr()->get_wal_path(txn_id, wal_path)); RETURN_IF_ERROR(_exec_env->wal_mgr()->add_recover_wal(std::to_string(db_id), diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 36c9d2b5dcd195..ca0310769ee277 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -1174,7 +1174,7 @@ public static int analyzeGroupCommitIntervalMs(Map properties) t try { groupCommitIntervalMs = Integer.parseInt(groupIntervalCommitMsStr); } catch (Exception e) { - throw new AnalysisException("schema version format error"); + throw new AnalysisException("parse group_commit_interval_ms format error"); } properties.remove(PROPERTIES_GROUP_COMMIT_INTERVAL_MS); From 3abf215bf1c6cb9609691cc5b5f509a1a1eede34 Mon Sep 17 00:00:00 2001 From: huanghaibin <284824253@qq.com> Date: Wed, 20 Dec 2023 10:57:12 +0800 Subject: [PATCH 2/5] edit --- be/src/olap/wal_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/wal_manager.cpp b/be/src/olap/wal_manager.cpp index f03075609ea1e2..4e2d3efa544d40 100644 --- a/be/src/olap/wal_manager.cpp +++ b/be/src/olap/wal_manager.cpp @@ -39,7 +39,7 @@ namespace doris { WalManager::WalManager(ExecEnv* exec_env, const std::string& wal_dir_list) : _exec_env(exec_env), _stop_background_threads_latch(1), _stop(false) { - doris::vectorized::WalReader::string_split(wal_dir_list, ",", _wal_dirs); + doris::vectorized::WalReader::string_split(wal_dir_list, ";", _wal_dirs); _all_wal_disk_bytes = std::make_shared(0); _cv = std::make_shared(); static_cast(ThreadPoolBuilder("GroupCommitReplayWalThreadPool") From 3a343b8684bfbc001cd715428575dd5061c10902 Mon Sep 17 00:00:00 2001 From: huanghaibin <284824253@qq.com> Date: Wed, 20 Dec 2023 12:13:59 +0800 Subject: [PATCH 3/5] edit --- be/src/common/config.cpp | 2 +- be/src/common/config.h | 2 +- be/src/runtime/exec_env_init.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index e75ac2c1c755d4..e66c51cc531217 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1112,7 +1112,7 @@ DEFINE_Int32(grace_shutdown_wait_seconds, "120"); DEFINE_Int16(bitmap_serialize_version, "1"); // group commit insert config -DEFINE_String(group_commit_replay_wal_dir, "./wal"); +DEFINE_String(group_commit_wal_path, "./wal"); DEFINE_Int32(group_commit_replay_wal_retry_num, "10"); DEFINE_Int32(group_commit_replay_wal_retry_interval_seconds, "5"); DEFINE_Int32(group_commit_relay_wal_threads, "10"); diff --git a/be/src/common/config.h b/be/src/common/config.h index c911963a87fe17..e63ec7724e9a5e 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1185,7 +1185,7 @@ DECLARE_Int32(grace_shutdown_wait_seconds); DECLARE_Int16(bitmap_serialize_version); // group commit insert config -DECLARE_String(group_commit_replay_wal_dir); +DECLARE_String(group_commit_wal_path); DECLARE_Int32(group_commit_replay_wal_retry_num); DECLARE_Int32(group_commit_replay_wal_retry_interval_seconds); DECLARE_mInt32(group_commit_relay_wal_threads); diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 422826790df8d6..13cb861aac2eb5 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -222,7 +222,7 @@ Status ExecEnv::_init(const std::vector& store_paths, _memtable_memory_limiter = std::make_unique(); _load_stream_stub_pool = std::make_unique(); _delta_writer_v2_pool = std::make_unique(); - _wal_manager = WalManager::create_shared(this, config::group_commit_replay_wal_dir); + _wal_manager = WalManager::create_shared(this, config::group_commit_wal_path); _backend_client_cache->init_metrics("backend"); _frontend_client_cache->init_metrics("frontend"); From f4bdbd43279b5df15cd123b1178d08234113e07c Mon Sep 17 00:00:00 2001 From: huanghaibin <284824253@qq.com> Date: Wed, 20 Dec 2023 15:29:19 +0800 Subject: [PATCH 4/5] fix bug --- be/src/olap/wal_table.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/olap/wal_table.cpp b/be/src/olap/wal_table.cpp index 54d158e7a95965..bde0e8dd69d65f 100644 --- a/be/src/olap/wal_table.cpp +++ b/be/src/olap/wal_table.cpp @@ -70,7 +70,8 @@ Status WalTable::replay_wals() { for (auto& [wal, info] : _replay_wal_map) { auto& [retry_num, start_ts, replaying] = info; if (replaying) { - continue; + LOG(INFO) << wal << " is replaying, skip this round"; + return Status::OK(); } if (retry_num >= config::group_commit_replay_wal_retry_num) { LOG(WARNING) << "All replay wal failed, db=" << _db_id << ", table=" << _table_id From 78a43e531b5fd741faf2647ebb02266763a5ed45 Mon Sep 17 00:00:00 2001 From: huanghaibin <284824253@qq.com> Date: Wed, 20 Dec 2023 15:47:29 +0800 Subject: [PATCH 5/5] edit --- be/src/vec/exec/format/wal/wal_reader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/vec/exec/format/wal/wal_reader.cpp b/be/src/vec/exec/format/wal/wal_reader.cpp index f0e5e29ca8e17c..01846acc04aa18 100644 --- a/be/src/vec/exec/format/wal/wal_reader.cpp +++ b/be/src/vec/exec/format/wal/wal_reader.cpp @@ -67,7 +67,7 @@ Status WalReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { } block->swap(dst_block); *read_rows = block->rows(); - LOG(INFO) << "read block rows:" << *read_rows; + VLOG_DEBUG << "read block rows:" << *read_rows; return Status::OK(); }