From d5d87588de41f7729ef06bec480b3bbb5cf45187 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Fri, 23 Aug 2019 17:40:53 +0800 Subject: [PATCH 1/7] Add local unique rowset id generator Fix bug Compile success Build unit test successfully Rowsetid version error Add rowsetid gc logic Fix bug Add unique rowset id unit test --- be/src/agent/agent_server.cpp | 6 +- be/src/olap/compaction.cpp | 6 +- be/src/olap/data_dir.cpp | 22 ++--- be/src/olap/data_dir.h | 10 +- be/src/olap/delta_writer.cpp | 6 +- be/src/olap/olap_common.h | 76 +++++++++++++- be/src/olap/olap_define.h | 2 +- be/src/olap/olap_snapshot_converter.cpp | 22 +++-- be/src/olap/olap_snapshot_converter.h | 4 +- be/src/olap/options.h | 2 + be/src/olap/push_handler.cpp | 8 +- be/src/olap/rowset/CMakeLists.txt | 5 +- be/src/olap/rowset/beta_rowset.cpp | 2 +- be/src/olap/rowset/rowset.h | 2 +- be/src/olap/rowset/rowset_id_generator.cpp | 76 -------------- be/src/olap/rowset/rowset_id_generator.h | 20 +--- be/src/olap/rowset/rowset_meta.h | 23 ++++- be/src/olap/rowset/rowset_meta_manager.cpp | 39 +++++--- be/src/olap/rowset/rowset_meta_manager.h | 14 +-- be/src/olap/rowset/rowset_writer_context.h | 3 +- be/src/olap/rowset/segment_group.cpp | 14 +-- be/src/olap/rowset/segment_group.h | 12 +-- be/src/olap/rowset/segment_writer.cpp | 6 ++ .../rowset/unique_rowset_id_generator.cpp | 50 ++++++++++ .../olap/rowset/unique_rowset_id_generator.h | 48 +++++++++ be/src/olap/schema_change.cpp | 24 ++--- be/src/olap/snapshot_manager.cpp | 48 +-------- be/src/olap/storage_engine.cpp | 4 +- be/src/olap/storage_engine.h | 9 ++ be/src/olap/tablet.cpp | 19 ---- be/src/olap/tablet.h | 9 -- be/src/olap/tablet_manager.cpp | 7 +- be/src/olap/tablet_meta.cpp | 59 ----------- be/src/olap/tablet_meta.h | 17 ---- be/src/olap/task/engine_clone_task.cpp | 2 +- be/src/olap/txn_manager.cpp | 4 +- be/src/service/doris_main.cpp | 2 + be/src/util/uid_util.h | 2 + be/test/olap/CMakeLists.txt | 1 + be/test/olap/olap_snapshot_converter_test.cpp | 12 +++ be/test/olap/rowset/alpha_rowset_test.cpp | 8 +- .../olap/rowset/rowset_meta_manager_test.cpp | 19 +++- be/test/olap/rowset/rowset_meta_test.cpp | 8 +- .../unique_rowset_id_generator_test.cpp | 98 +++++++++++++++++++ be/test/olap/tablet_mgr_test.cpp | 29 +++--- be/test/olap/test_data/header.txt | 3 +- be/test/olap/txn_manager_test.cpp | 20 +++- .../org/apache/doris/task/SnapshotTask.java | 2 +- gensrc/proto/olap_file.proto | 2 + 49 files changed, 505 insertions(+), 381 deletions(-) delete mode 100644 be/src/olap/rowset/rowset_id_generator.cpp create mode 100644 be/src/olap/rowset/unique_rowset_id_generator.cpp create mode 100644 be/src/olap/rowset/unique_rowset_id_generator.h create mode 100644 be/test/olap/rowset/unique_rowset_id_generator_test.cpp diff --git a/be/src/agent/agent_server.cpp b/be/src/agent/agent_server.cpp index 81ffcf57202808..6fca3f76b7921c 100644 --- a/be/src/agent/agent_server.cpp +++ b/be/src/agent/agent_server.cpp @@ -426,7 +426,11 @@ void AgentServer::make_snapshot(TAgentResult& return_value, TStatus status; vector error_msgs; TStatusCode::type status_code = TStatusCode::OK; - return_value.__set_snapshot_version(PREFERRED_SNAPSHOT_VERSION); + int32_t return_snapshot_version = PREFERRED_SNAPSHOT_VERSION; + if (snapshot_request.preferred_snapshot_version < PREFERRED_SNAPSHOT_VERSION) { + return_snapshot_version = 1; + } + return_value.__set_snapshot_version(return_snapshot_version); string snapshot_path; OLAPStatus make_snapshot_status = SnapshotManager::instance()->make_snapshot(snapshot_request, &snapshot_path); diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index b4049a25efd888..628311afb9ea7f 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -28,7 +28,7 @@ Compaction::Compaction(TabletSharedPtr tablet) _state(CompactionState::INITED) { } -Compaction::~Compaction() { } +Compaction::~Compaction() {} OLAPStatus Compaction::do_compaction() { LOG(INFO) << "start " << compaction_name() << ". tablet=" << _tablet->full_name(); @@ -83,8 +83,8 @@ OLAPStatus Compaction::do_compaction() { } OLAPStatus Compaction::construct_output_rowset_writer() { - RowsetId rowset_id = 0; - RETURN_NOT_OK(_tablet->next_rowset_id(&rowset_id)); + RowsetId rowset_id; + RETURN_NOT_OK(StorageEngine::instance()->next_rowset_id(&rowset_id)); RowsetWriterContext context; context.rowset_id = rowset_id; context.tablet_uid = _tablet->tablet_uid(); diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index c186c5927c5730..9a5779026c4a14 100755 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -106,12 +106,6 @@ Status DataDir::init() { RETURN_IF_ERROR(_init_file_system()); RETURN_IF_ERROR(_init_meta()); - _id_generator = new RowsetIdGenerator(_meta); - auto res = _id_generator->init(); - if (res != OLAP_SUCCESS) { - return Status::InternalError("Id generator initialized failed."); - } - _is_used = true; return Status::OK(); } @@ -582,7 +576,9 @@ OLAPStatus DataDir::_convert_old_tablet() { for (auto& rowset_pb : pending_rowsets) { string meta_binary; rowset_pb.SerializeToString(&meta_binary); - status = RowsetMetaManager::save(_meta, rowset_pb.tablet_uid(), rowset_pb.rowset_id() , meta_binary); + RowsetId rowset_id; + rowset_id.init(rowset_pb.rowset_id_v2()); + status = RowsetMetaManager::save(_meta, rowset_pb.tablet_uid(), rowset_id, meta_binary); if (status != OLAP_SUCCESS) { LOG(FATAL) << "convert olap header to tablet meta failed when save rowset meta tablet=" << tablet_id << "." << schema_hash; @@ -903,11 +899,12 @@ void DataDir::perform_path_gc() { } } else { bool valid = tablet->check_path(path); + // TODO(ygl): should change a method to do gc if (!valid) { - RowsetId rowset_id = -1; + RowsetId rowset_id; bool is_rowset_file = _tablet_manager->get_rowset_id_from_path(path, &rowset_id); if (is_rowset_file) { - std::string rowset_path_id = ROWSET_ID_PREFIX + std::to_string(rowset_id); + std::string rowset_path_id = ROWSET_ID_PREFIX + rowset_id.to_string(); bool exist_in_pending = _check_pending_ids(rowset_path_id); if (!exist_in_pending) { _process_garbage_path(path); @@ -959,18 +956,19 @@ void DataDir::perform_path_gc_by_rowsetid() { // tablet schema hash path or rowset file path // gc thread should get tablet include deleted tablet // or it will delete rowset file before tablet is garbage collected - RowsetId rowset_id = -1; + RowsetId rowset_id; bool is_rowset_file = _tablet_manager->get_rowset_id_from_path(path, &rowset_id); if (is_rowset_file) { TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id, schema_hash); if (tablet != nullptr) { bool valid = tablet->check_rowset_id(rowset_id); if (!valid) { - // if the rowset id is less than tablet's initial end rowset id + // if the rowset id is in using rowset set // and the rowsetid is not in unused_rowsets // and the rowsetid is not in committed rowsets // then delete the path. - if (rowset_id < tablet->initial_end_rowset_id() + // TODO(ygl): check rowset id + if (!StorageEngine::instance()->rowset_id_in_use(rowset_id) && !StorageEngine::instance()->check_rowset_id_in_unused_rowsets(rowset_id) && !RowsetMetaManager::check_rowset_meta(_meta, tablet->tablet_uid(), rowset_id)) { _process_garbage_path(path); diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h index e5b9a914f34f69..8316811c4e7140 100644 --- a/be/src/olap/data_dir.h +++ b/be/src/olap/data_dir.h @@ -92,15 +92,7 @@ class DataDir { void find_tablet_in_trash(int64_t tablet_id, std::vector* paths); static std::string get_root_path_from_schema_hash_path_in_trash(const std::string& schema_hash_dir_in_trash); -/* - OLAPStatus next_id(RowsetId* id) { - return _id_generator->get_next_id(id); - } - - OLAPStatus set_next_id(RowsetId new_rowset_id) { - return _id_generator->set_next_id(new_rowset_id); - } -*/ + // load data from meta and data files OLAPStatus load(); diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index 35c4d18fc49691..1d59f6fd066b8e 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -46,7 +46,7 @@ DeltaWriter::~DeltaWriter() { SAFE_DELETE(_mem_table); SAFE_DELETE(_schema); if (_rowset_writer != nullptr) { - _rowset_writer->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + std::to_string(_rowset_writer->rowset_id())); + _rowset_writer->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + _rowset_writer->rowset_id().to_string()); } } @@ -115,8 +115,8 @@ OLAPStatus DeltaWriter::init() { } } - RowsetId rowset_id = 0; // get rowset_id from id generator - OLAPStatus status = _tablet->next_rowset_id(&rowset_id); + RowsetId rowset_id; // get rowset_id from id generator + OLAPStatus status = StorageEngine::instance()->next_rowset_id(&rowset_id); if (status != OLAP_SUCCESS) { LOG(WARNING) << "generate rowset id failed, status:" << status; return OLAP_ERR_ROWSET_GENERATE_ID_FAILED; diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index da815a3cc3e699..dc6dcbc430a0a9 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -241,7 +242,80 @@ typedef std::set UniqueIdSet; // Column unique Id -> column id map typedef std::map UniqueIdToColumnIdMap; -typedef int64_t RowsetId; +// 8 bit, id version +// 120 bit, backend random number generated by uuid +// 64 bit, inc number from 0 +struct RowsetId { + int8_t version = 0; + int64_t hi = 0; + int64_t mi = 0; + int64_t lo = 0; + + void init(const std::string& rowset_id_str) { + // for new rowsetid its a 48 hex string + // if the len < 48, then it is an old format rowset id + if (rowset_id_str.length() < 48) { + hi = 1; + hi = hi << 56; + mi = 0; + lo = std::stol(rowset_id_str, nullptr, 10); + version = 1; + } else { + from_hex(&hi, rowset_id_str.substr(0, 16)); + from_hex(&mi, rowset_id_str.substr(16, 16)); + from_hex(&lo, rowset_id_str.substr(32, 16)); + version = hi >> 56; + } + } + + // to compatiable with old version + void init(int64_t rowset_id) { + init(1, 0, 0, rowset_id); + } + + void init(int64_t id_version, int64_t high, int64_t middle, int64_t low) { + version = id_version; + hi = (id_version << 56) + (high & 0x00ffffffffffffff); + mi = middle; + lo = low; + } + + std::string to_string() const { + if (version < 2) { + return std::to_string(lo); + } else { + char buf[48]; + to_hex(hi, buf); + to_hex(mi, buf + 16); + to_hex(lo, buf + 32); + return {buf, 48}; + } + } + + // std::unordered_map need this api + bool operator==(const RowsetId& rhs) const { + return hi == rhs.hi && mi == rhs.mi && lo == rhs.lo; + } + + bool operator!=(const RowsetId& rhs) const { + return hi != rhs.hi || mi != rhs.mi || lo != rhs.lo; + } + + bool operator<(const RowsetId& rhs) const { + if (hi != rhs.hi) { + return hi < rhs.hi; + } else if (mi != rhs.mi) { + return mi < rhs.mi; + } else { + return lo < rhs.lo; + } + } + + friend std::ostream &operator<<(std::ostream &out, const RowsetId& rowset_id) { + out << rowset_id.to_string(); + return out; + } +}; } // namespace doris diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h index f3575ed547d50c..cd331fe75ef34b 100644 --- a/be/src/olap/olap_define.h +++ b/be/src/olap/olap_define.h @@ -53,7 +53,7 @@ static constexpr uint32_t OLAP_COMPACTION_DEFAULT_CANDIDATE_SIZE = 10; // the max length supported for string type static const uint16_t OLAP_STRING_MAX_LENGTH = 65535; -static const int32_t PREFERRED_SNAPSHOT_VERSION = 2; +static const int32_t PREFERRED_SNAPSHOT_VERSION = 3; // the max bytes for stored string length using StringOffsetType = uint32_t; diff --git a/be/src/olap/olap_snapshot_converter.cpp b/be/src/olap/olap_snapshot_converter.cpp index 72bc90eeaf6a35..0a556f8b45d139 100755 --- a/be/src/olap/olap_snapshot_converter.cpp +++ b/be/src/olap/olap_snapshot_converter.cpp @@ -119,11 +119,12 @@ OLAPStatus OlapSnapshotConverter::to_tablet_meta_pb(const OLAPHeaderMessage& ola schema->set_next_column_unique_id(olap_header.next_column_unique_id()); } - RowsetId next_id = 10000; std::unordered_map _rs_version_map; for (auto& delta : olap_header.delta()) { + RowsetId next_id; + RETURN_NOT_OK(StorageEngine::instance()->next_rowset_id(&next_id)); RowsetMetaPB* rowset_meta = tablet_meta_pb->add_rs_metas(); - convert_to_rowset_meta(delta, ++next_id, olap_header.tablet_id(), olap_header.schema_hash(), rowset_meta); + convert_to_rowset_meta(delta, next_id, olap_header.tablet_id(), olap_header.schema_hash(), rowset_meta); Version rowset_version = { delta.start_version(), delta.end_version() }; _rs_version_map[rowset_version] = rowset_meta; } @@ -137,13 +138,17 @@ OLAPStatus OlapSnapshotConverter::to_tablet_meta_pb(const OLAPHeaderMessage& ola *rowset_meta = *(exist_rs->second); continue; } + RowsetId next_id; + RETURN_NOT_OK(StorageEngine::instance()->next_rowset_id(&next_id)); RowsetMetaPB* rowset_meta = tablet_meta_pb->add_inc_rs_metas(); - convert_to_rowset_meta(inc_delta, ++next_id, olap_header.tablet_id(), olap_header.schema_hash(), rowset_meta); + convert_to_rowset_meta(inc_delta, next_id, olap_header.tablet_id(), olap_header.schema_hash(), rowset_meta); } for (auto& pending_delta : olap_header.pending_delta()) { + RowsetId next_id; + RETURN_NOT_OK(StorageEngine::instance()->next_rowset_id(&next_id)); RowsetMetaPB rowset_meta; - convert_to_rowset_meta(pending_delta, ++next_id, olap_header.tablet_id(), olap_header.schema_hash(), &rowset_meta); + convert_to_rowset_meta(pending_delta, next_id, olap_header.tablet_id(), olap_header.schema_hash(), &rowset_meta); pending_rowsets->emplace_back(std::move(rowset_meta)); } if (olap_header.has_schema_change_status()) { @@ -155,7 +160,6 @@ OLAPStatus OlapSnapshotConverter::to_tablet_meta_pb(const OLAPHeaderMessage& ola } tablet_meta_pb->set_tablet_state(TabletStatePB::PB_RUNNING); *(tablet_meta_pb->mutable_tablet_uid()) = TabletUid().to_proto(); - tablet_meta_pb->set_end_rowset_id(++next_id); VLOG(3) << "convert tablet meta tablet id = " << olap_header.tablet_id() << " schema hash = " << olap_header.schema_hash() << " successfully."; return OLAP_SUCCESS; @@ -196,8 +200,8 @@ OLAPStatus OlapSnapshotConverter::convert_to_pdelta(const RowsetMetaPB& rowset_m } OLAPStatus OlapSnapshotConverter::convert_to_rowset_meta(const PDelta& delta, - int64_t rowset_id, int64_t tablet_id, int32_t schema_hash, RowsetMetaPB* rowset_meta_pb) { - rowset_meta_pb->set_rowset_id(rowset_id); + const RowsetId& rowset_id, int64_t tablet_id, int32_t schema_hash, RowsetMetaPB* rowset_meta_pb) { + rowset_meta_pb->set_rowset_id_v2(rowset_id.to_string()); rowset_meta_pb->set_tablet_id(tablet_id); rowset_meta_pb->set_tablet_schema_hash(schema_hash); rowset_meta_pb->set_rowset_type(RowsetTypePB::ALPHA_ROWSET); @@ -243,8 +247,8 @@ OLAPStatus OlapSnapshotConverter::convert_to_rowset_meta(const PDelta& delta, } OLAPStatus OlapSnapshotConverter::convert_to_rowset_meta(const PPendingDelta& pending_delta, - int64_t rowset_id, int64_t tablet_id, int32_t schema_hash, RowsetMetaPB* rowset_meta_pb) { - rowset_meta_pb->set_rowset_id(rowset_id); + const RowsetId& rowset_id, int64_t tablet_id, int32_t schema_hash, RowsetMetaPB* rowset_meta_pb) { + rowset_meta_pb->set_rowset_id_v2(rowset_id.to_string()); rowset_meta_pb->set_tablet_id(tablet_id); rowset_meta_pb->set_tablet_schema_hash(schema_hash); rowset_meta_pb->set_rowset_type(RowsetTypePB::ALPHA_ROWSET); diff --git a/be/src/olap/olap_snapshot_converter.h b/be/src/olap/olap_snapshot_converter.h index 772956af4d7396..ea568ad1699e3b 100644 --- a/be/src/olap/olap_snapshot_converter.h +++ b/be/src/olap/olap_snapshot_converter.h @@ -52,10 +52,10 @@ class OlapSnapshotConverter { OLAPStatus convert_to_pdelta(const RowsetMetaPB& rowset_meta_pb, PDelta* delta); - OLAPStatus convert_to_rowset_meta(const PDelta& delta, int64_t rowset_id, int64_t tablet_id, + OLAPStatus convert_to_rowset_meta(const PDelta& delta, const RowsetId& rowset_id, int64_t tablet_id, int32_t schema_hash, RowsetMetaPB* rowset_meta_pb); - OLAPStatus convert_to_rowset_meta(const PPendingDelta& pending_delta, int64_t rowset_id, + OLAPStatus convert_to_rowset_meta(const PPendingDelta& pending_delta, const RowsetId& rowset_id, int64_t tablet_id, int32_t schema_hash, RowsetMetaPB* rowset_meta_pb); OLAPStatus to_column_pb(const ColumnMessage& column_msg, ColumnPB* column_pb); diff --git a/be/src/olap/options.h b/be/src/olap/options.h index b7af8d33ed0d14..0dd0d9a7b952fe 100644 --- a/be/src/olap/options.h +++ b/be/src/olap/options.h @@ -21,6 +21,7 @@ #include #include "olap/olap_define.h" +#include "util/uid_util.h" namespace doris { @@ -37,6 +38,7 @@ OLAPStatus parse_conf_store_paths(const std::string& config_path, std::vector store_paths; + UniqueId backend_uid; }; } diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index f7775816cfb419..9329eda2c62b26 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -270,12 +270,8 @@ OLAPStatus PushHandler::_convert(TabletSharedPtr cur_tablet, } RowsetWriterContext context; uint32_t num_rows = 0; - RowsetId rowset_id = 0; - res = cur_tablet->next_rowset_id(&rowset_id); - if (res != OLAP_SUCCESS) { - LOG(WARNING) << "generate rowset id failed, res:" << res; - return OLAP_ERR_ROWSET_GENERATE_ID_FAILED; - } + RowsetId rowset_id; + RETURN_NOT_OK(StorageEngine::instance()->next_rowset_id(&rowset_id)); PUniqueId load_id; load_id.set_hi(0); load_id.set_lo(0); diff --git a/be/src/olap/rowset/CMakeLists.txt b/be/src/olap/rowset/CMakeLists.txt index 6357dcba3fede1..28cdde7159f401 100644 --- a/be/src/olap/rowset/CMakeLists.txt +++ b/be/src/olap/rowset/CMakeLists.txt @@ -33,15 +33,14 @@ add_library(Rowset STATIC run_length_byte_writer.cpp run_length_integer_reader.cpp run_length_integer_writer.cpp + unique_rowset_id_generator.cpp segment_reader.cpp segment_writer.cpp rowset.cpp - rowset_id_generator.cpp rowset_meta_manager.cpp alpha_rowset.cpp alpha_rowset_reader.cpp alpha_rowset_writer.cpp alpha_rowset_meta.cpp beta_rowset.cpp - beta_rowset_writer.cpp - rowset_id_generator.cpp) + beta_rowset_writer.cpp) diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 33930583f20880..b2cbfd00d6286c 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -26,7 +26,7 @@ namespace doris { std::string BetaRowset::segment_file_path(const std::string& dir, RowsetId rowset_id, int segment_id) { - return strings::Substitute("$0/$1_$2.dat", dir, rowset_id, segment_id); + return strings::Substitute("$0/$1_$2.dat", dir, rowset_id.to_string(), segment_id); } BetaRowset::BetaRowset(const TabletSchema* schema, diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index 8775b3aae65052..0f9e145346be2d 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -114,7 +114,7 @@ class Rowset : public std::enable_shared_from_this { // return an unique identifier string for this rowset std::string unique_id() const { - return _rowset_path + "/" + std::to_string(rowset_id()); + return _rowset_path + "/" + rowset_id().to_string(); } bool need_delete_file() const { diff --git a/be/src/olap/rowset/rowset_id_generator.cpp b/be/src/olap/rowset/rowset_id_generator.cpp deleted file mode 100644 index 55aca7f05253b7..00000000000000 --- a/be/src/olap/rowset/rowset_id_generator.cpp +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -#include -#include - -#include "olap/rowset/rowset_id_generator.h" -#include "olap/olap_meta.h" - -namespace doris { - -static RowsetId k_batch_interval = 10000; - -OLAPStatus RowsetIdGenerator::init() { - _next_id = k_batch_interval; - _id_batch_end = (k_batch_interval << 1); - // get last stored value from meta - std::string value; - OLAPStatus s = _meta->get(DEFAULT_COLUMN_FAMILY_INDEX, END_ROWSET_ID, &value); - if (s == OLAP_SUCCESS) { - _next_id = std::stol(value); - _id_batch_end = _next_id + k_batch_interval; - } else if (s != OLAP_ERR_META_KEY_NOT_FOUND) { - return s; - } - // else: meta-key not found, we will initialize a initial state - s = _meta->put(DEFAULT_COLUMN_FAMILY_INDEX, END_ROWSET_ID, std::to_string(_id_batch_end)); - if (s != OLAP_SUCCESS) { - return s; - } - return OLAP_SUCCESS; -} - -OLAPStatus RowsetIdGenerator::get_next_id(RowsetId* gen_rowset_id) { - std::lock_guard l(_lock); - if (_next_id >= _id_batch_end) { - _id_batch_end += k_batch_interval; - auto s = _meta->put(DEFAULT_COLUMN_FAMILY_INDEX, END_ROWSET_ID, std::to_string(_id_batch_end)); - if (s != OLAP_SUCCESS) { - return s; - } - } - *gen_rowset_id = _next_id; - ++_next_id; - return OLAP_SUCCESS; -} - -OLAPStatus RowsetIdGenerator::set_next_id(RowsetId new_rowset_id) { - std::lock_guard l(_lock); - // must be < not <= - if (new_rowset_id < _next_id) { - return OLAP_SUCCESS; - } - if (new_rowset_id >= _id_batch_end) { - _id_batch_end = new_rowset_id + k_batch_interval; - auto s = _meta->put(DEFAULT_COLUMN_FAMILY_INDEX, END_ROWSET_ID, std::to_string(_id_batch_end)); - RETURN_NOT_OK(s); - } - _next_id = new_rowset_id + 1; - return OLAP_SUCCESS; -} - -} // doris diff --git a/be/src/olap/rowset/rowset_id_generator.h b/be/src/olap/rowset/rowset_id_generator.h index c6a1436713a3c7..d205d3cb2f9841 100644 --- a/be/src/olap/rowset/rowset_id_generator.h +++ b/be/src/olap/rowset/rowset_id_generator.h @@ -28,27 +28,17 @@ class OlapMeta; class RowsetIdGenerator { public: - RowsetIdGenerator(OlapMeta* meta) : _meta(meta) { } - ~RowsetIdGenerator() {} - - // This function would try to restore sate from meta first, - // If there is no such state, will initialize a state, and store - // it into meta. - OLAPStatus init(); + RowsetIdGenerator() {} + virtual ~RowsetIdGenerator() {} // generator a id according to data dir // rowsetid is not globally unique, it is dir level // it saves the batch end id into meta env - OLAPStatus get_next_id(RowsetId* rowset_id); - - OLAPStatus set_next_id(RowsetId new_rowset_id); + virtual OLAPStatus next_id(RowsetId* rowset_id) = 0; -private: - OlapMeta* _meta = nullptr; + virtual bool id_in_use(RowsetId& rowset_id) = 0; - std::mutex _lock; - RowsetId _next_id = -1; - RowsetId _id_batch_end = -1; + virtual void release_id(RowsetId& rowset_id) = 0; }; // RowsetIdGenerator } // namespace doris diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h index d8a4b84227d3f7..c0f494defaf37b 100644 --- a/be/src/olap/rowset/rowset_meta.h +++ b/be/src/olap/rowset/rowset_meta.h @@ -44,11 +44,13 @@ class RowsetMeta { if (!ret) { return false; } + _init(); return true; } virtual bool init_from_pb(const RowsetMetaPB& rowset_meta_pb) { _rowset_meta_pb = rowset_meta_pb; + _init(); return true; } @@ -57,6 +59,7 @@ class RowsetMeta { if (!ret) { return false; } + _init(); return true; } @@ -71,12 +74,15 @@ class RowsetMeta { return ret; } - int64_t rowset_id() const { - return _rowset_meta_pb.rowset_id(); + RowsetId rowset_id() const { + return _rowset_id; } - void set_rowset_id(int64_t rowset_id) { - _rowset_meta_pb.set_rowset_id(rowset_id); + void set_rowset_id(RowsetId rowset_id) { + // rowset id is a required field, just set it to 0 + _rowset_meta_pb.set_rowset_id(0); + _rowset_id = rowset_id; + _rowset_meta_pb.set_rowset_id_v2(rowset_id.to_string()); } int64_t tablet_id() const { @@ -312,8 +318,17 @@ class RowsetMeta { return _rowset_meta_pb.mutable_alpha_rowset_extra_meta_pb(); } + void _init() { + if (_rowset_meta_pb.rowset_id() > 0) { + _rowset_id.init(_rowset_meta_pb.rowset_id()); + } else { + _rowset_id.init(_rowset_meta_pb.rowset_id_v2()); + } + } + private: RowsetMetaPB _rowset_meta_pb; + RowsetId _rowset_id; }; } // namespace doris diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp b/be/src/olap/rowset/rowset_meta_manager.cpp index abc6792e293a54..f865ca0fa9817b 100644 --- a/be/src/olap/rowset/rowset_meta_manager.cpp +++ b/be/src/olap/rowset/rowset_meta_manager.cpp @@ -24,6 +24,7 @@ #include #include "olap/olap_define.h" +#include "olap/storage_engine.h" #include "olap/utils.h" #include "common/logging.h" #include "json2pb/json_to_pb.h" @@ -33,8 +34,8 @@ namespace doris { const std::string ROWSET_PREFIX = "rst_"; -bool RowsetMetaManager::check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, int64_t rowset_id) { - std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + std::to_string(rowset_id); +bool RowsetMetaManager::check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id) { + std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string(); std::string value; OLAPStatus s = meta->get(META_COLUMN_FAMILY_INDEX, key, &value); if (s != OLAP_SUCCESS) { @@ -43,8 +44,8 @@ bool RowsetMetaManager::check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, return true; } -OLAPStatus RowsetMetaManager::get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, int64_t rowset_id, RowsetMetaSharedPtr rowset_meta) { - std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + std::to_string(rowset_id); +OLAPStatus RowsetMetaManager::get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, RowsetMetaSharedPtr rowset_meta) { + std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string(); std::string value; OLAPStatus s = meta->get(META_COLUMN_FAMILY_INDEX, key, &value); if (s == OLAP_ERR_META_KEY_NOT_FOUND) { @@ -64,7 +65,7 @@ OLAPStatus RowsetMetaManager::get_rowset_meta(OlapMeta* meta, TabletUid tablet_u return OLAP_SUCCESS; } -OLAPStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, int64_t rowset_id, std::string* json_rowset_meta) { +OLAPStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, std::string* json_rowset_meta) { RowsetMetaSharedPtr rowset_meta_ptr(new(std::nothrow) RowsetMeta()); OLAPStatus status = get_rowset_meta(meta, tablet_uid, rowset_id, rowset_meta_ptr); if (status != OLAP_SUCCESS) { @@ -72,14 +73,14 @@ OLAPStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, TabletUid tab } bool ret = rowset_meta_ptr->json_rowset_meta(json_rowset_meta); if (!ret) { - std::string error_msg = "get json rowset meta failed. rowset id:" + std::to_string(rowset_id); + std::string error_msg = "get json rowset meta failed. rowset id:" + rowset_id.to_string(); return OLAP_ERR_SERIALIZE_PROTOBUF_ERROR; } return OLAP_SUCCESS; } -OLAPStatus RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, int64_t rowset_id, RowsetMeta* rowset_meta) { - std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + std::to_string(rowset_id); +OLAPStatus RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, RowsetMeta* rowset_meta) { + std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string(); std::string value; bool ret = rowset_meta->serialize(&value); if (!ret) { @@ -88,17 +89,23 @@ OLAPStatus RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, int64_t return OLAP_ERR_SERIALIZE_PROTOBUF_ERROR; } OLAPStatus status = meta->put(META_COLUMN_FAMILY_INDEX, key, value); + if (status == OLAP_SUCCESS) { + StorageEngine::instance()->release_rowset_id(rowset_id); + } return status; } -OLAPStatus RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, int64_t rowset_id, const string& meta_binary) { - std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + std::to_string(rowset_id); +OLAPStatus RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, const string& meta_binary) { + std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string(); OLAPStatus status = meta->put(META_COLUMN_FAMILY_INDEX, key, meta_binary); + if (status == OLAP_SUCCESS) { + StorageEngine::instance()->release_rowset_id(rowset_id); + } return status; } -OLAPStatus RowsetMetaManager::remove(OlapMeta* meta, TabletUid tablet_uid, int64_t rowset_id) { - std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + std::to_string(rowset_id); +OLAPStatus RowsetMetaManager::remove(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id) { + std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string(); LOG(INFO) << "start to remove rowset, key:" << key; OLAPStatus status = meta->remove(META_COLUMN_FAMILY_INDEX, key); LOG(INFO) << "remove rowset key:" << key << " finished"; @@ -106,7 +113,7 @@ OLAPStatus RowsetMetaManager::remove(OlapMeta* meta, TabletUid tablet_uid, int64 } OLAPStatus RowsetMetaManager::traverse_rowset_metas(OlapMeta* meta, - std::function const& func) { + std::function const& func) { auto traverse_rowset_meta_func = [&func](const std::string& key, const std::string& value) -> bool { std::vector parts; // key format: rst_uuid_rowset_id @@ -115,8 +122,8 @@ OLAPStatus RowsetMetaManager::traverse_rowset_metas(OlapMeta* meta, LOG(WARNING) << "invalid rowset key:" << key << ", splitted size:" << parts.size(); return true; } - uint64_t rowset_id = std::stol(parts[2].c_str(), NULL, 10); - // TODO(ygl): parset tablet id from parts[1] + RowsetId rowset_id; + rowset_id.init(parts[2]); std::vector uid_parts; split_string(parts[1], '-', &uid_parts); TabletUid tablet_uid(uid_parts[0], uid_parts[1]); @@ -142,7 +149,7 @@ OLAPStatus RowsetMetaManager::load_json_rowset_meta(OlapMeta* meta, const std::s LOG(WARNING) << error_msg; return OLAP_ERR_SERIALIZE_PROTOBUF_ERROR; } - uint64_t rowset_id = rowset_meta.rowset_id(); + RowsetId rowset_id = rowset_meta.rowset_id(); TabletUid tablet_uid = rowset_meta.tablet_uid(); OLAPStatus status = save(meta, tablet_uid, rowset_id, &rowset_meta); return status; diff --git a/be/src/olap/rowset/rowset_meta_manager.h b/be/src/olap/rowset/rowset_meta_manager.h index 0411243d7bbadb..d04a911c1126c9 100644 --- a/be/src/olap/rowset/rowset_meta_manager.h +++ b/be/src/olap/rowset/rowset_meta_manager.h @@ -31,20 +31,20 @@ namespace doris { // Helper class for managing rowset meta of one root path. class RowsetMetaManager { public: - static bool check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, int64_t rowset_id); + static bool check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id); - static OLAPStatus get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, int64_t rowset_id, RowsetMetaSharedPtr rowset_meta); + static OLAPStatus get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, RowsetMetaSharedPtr rowset_meta); - static OLAPStatus get_json_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, int64_t rowset_id, std::string* json_rowset_meta); + static OLAPStatus get_json_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, std::string* json_rowset_meta); - static OLAPStatus save(OlapMeta* meta, TabletUid tablet_uid, int64_t rowset_id, RowsetMeta* rowset_meta); + static OLAPStatus save(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, RowsetMeta* rowset_meta); - static OLAPStatus save(OlapMeta* meta, TabletUid tablet_uid, int64_t rowset_id, const string& meta_binary); + static OLAPStatus save(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, const string& meta_binary); - static OLAPStatus remove(OlapMeta* meta, TabletUid tablet_uid, int64_t rowset_id); + static OLAPStatus remove(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id); static OLAPStatus traverse_rowset_metas(OlapMeta* meta, - std::function const& func); + std::function const& func); static OLAPStatus load_json_rowset_meta(OlapMeta* meta, const std::string& rowset_meta_path); }; diff --git a/be/src/olap/rowset/rowset_writer_context.h b/be/src/olap/rowset/rowset_writer_context.h index 66b74e5cd49496..2123045420db40 100644 --- a/be/src/olap/rowset/rowset_writer_context.h +++ b/be/src/olap/rowset/rowset_writer_context.h @@ -29,7 +29,6 @@ using RowsetWriterContextBuilderSharedPtr = std::shared_ptr 0) { tmp_sg_id = _segment_group_id; } - std::string file_name = std::to_string(_rowset_id) + "_" + std::string file_name = _rowset_id.to_string() + "_" + std::to_string(tmp_sg_id) + "_" + std::to_string(segment_id) + suffix; return file_name; } -std::string SegmentGroup::_construct_file_name(int64_t rowset_id, int32_t segment_id, const string& suffix) const { - std::string file_name = std::to_string(rowset_id) + "_" +std::string SegmentGroup::_construct_file_name(RowsetId rowset_id, int32_t segment_id, const string& suffix) const { + std::string file_name = rowset_id.to_string() + "_" + std::to_string(_segment_group_id) + "_" + std::to_string(segment_id) + suffix; return file_name; } @@ -584,7 +584,7 @@ OLAPStatus SegmentGroup::add_short_key(const RowCursor& short_key, const uint32_ boost::filesystem::path data_dir_path = tablet_path.parent_path().parent_path().parent_path().parent_path(); std::string data_dir_string = data_dir_path.string(); DataDir* data_dir = StorageEngine::instance()->get_store(data_dir_string); - data_dir->add_pending_ids(ROWSET_ID_PREFIX + std::to_string(_rowset_id)); + data_dir->add_pending_ids(ROWSET_ID_PREFIX + _rowset_id.to_string()); res = _current_file_handler.open_with_mode( file_path.c_str(), O_CREAT | O_EXCL | O_WRONLY, S_IRUSR | S_IWUSR); if (res != OLAP_SUCCESS) { @@ -855,7 +855,7 @@ OLAPStatus SegmentGroup::remove_old_files(std::vector* links_to_rem return OLAP_SUCCESS; } -OLAPStatus SegmentGroup::link_segments_to_path(const std::string& dest_path, int64_t rowset_id) { +OLAPStatus SegmentGroup::link_segments_to_path(const std::string& dest_path, RowsetId rowset_id) { if (dest_path.empty()) { LOG(WARNING) << "dest path is empty, return error"; return OLAP_ERR_INPUT_PARAMETER_ERROR; diff --git a/be/src/olap/rowset/segment_group.h b/be/src/olap/rowset/segment_group.h index 8297e3abaaf786..9d8d99cea98efb 100644 --- a/be/src/olap/rowset/segment_group.h +++ b/be/src/olap/rowset/segment_group.h @@ -48,11 +48,11 @@ namespace doris { class SegmentGroup { friend class MemIndex; public: - SegmentGroup(int64_t tablet_id, int64_t rowset_id, const TabletSchema* tablet_schema, + SegmentGroup(int64_t tablet_id, RowsetId rowset_id, const TabletSchema* tablet_schema, const std::string& rowset_path_prefix, Version version, VersionHash version_hash, bool delete_flag, int segment_group_id, int32_t num_segments); - SegmentGroup(int64_t tablet_id, int64_t rowset_id, const TabletSchema* tablet_schema, + SegmentGroup(int64_t tablet_id, RowsetId rowset_id, const TabletSchema* tablet_schema, const std::string& rowset_path_prefix, bool delete_flag, int32_t segment_group_id, int32_t num_segments, bool is_pending, TPartitionId partition_id, TTransactionId transaction_id); @@ -248,7 +248,7 @@ class SegmentGroup { int64_t get_tablet_id(); - int64_t rowset_id() { + RowsetId rowset_id() { return _rowset_id; } @@ -262,12 +262,12 @@ class SegmentGroup { OLAPStatus copy_files_to(const std::string& dir); - OLAPStatus link_segments_to_path(const std::string& dest_path, int64_t rowset_id); + OLAPStatus link_segments_to_path(const std::string& dest_path, RowsetId rowset_id); private: std::string _construct_file_name(int32_t segment_id, const std::string& suffix) const; - std::string _construct_file_name(int64_t rowset_id, int32_t segment_id, const std::string& suffix) const; + std::string _construct_file_name(RowsetId rowset_id, int32_t segment_id, const std::string& suffix) const; std::string _construct_old_pending_file_path(const std::string& path_prefix, int32_t segment_id, const std::string& suffix) const; @@ -281,7 +281,7 @@ class SegmentGroup { private: int64_t _tablet_id; - int64_t _rowset_id; + RowsetId _rowset_id; const TabletSchema* _schema; std::string _rowset_path_prefix; // path of rowset Version _version; // version of associated data file diff --git a/be/src/olap/rowset/segment_writer.cpp b/be/src/olap/rowset/segment_writer.cpp index 70ffccb53f9e94..86677225dc72d4 100644 --- a/be/src/olap/rowset/segment_writer.cpp +++ b/be/src/olap/rowset/segment_writer.cpp @@ -213,6 +213,12 @@ OLAPStatus SegmentWriter::finalize(uint32_t* segment_file_size) { boost::filesystem::path data_dir_path = tablet_path.parent_path().parent_path().parent_path().parent_path(); std::string data_dir_string = data_dir_path.string(); DataDir* data_dir = StorageEngine::instance()->get_store(data_dir_string); + data_dir->add_pending_ids(ROWSET_ID_PREFIX + _segment_group->rowset_id().to_string()); + if (OLAP_SUCCESS != (res = file_handle.open_with_mode( + _file_name, O_CREAT | O_EXCL | O_WRONLY , S_IRUSR | S_IWUSR))) { + LOG(WARNING) << "fail to open file. [file_name=" << _file_name << "]"; + return res; + } res = _make_file_header(file_header.mutable_message()); if (OLAP_SUCCESS != res) { diff --git a/be/src/olap/rowset/unique_rowset_id_generator.cpp b/be/src/olap/rowset/unique_rowset_id_generator.cpp new file mode 100644 index 00000000000000..32b4ec97d73e75 --- /dev/null +++ b/be/src/olap/rowset/unique_rowset_id_generator.cpp @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/rowset/unique_rowset_id_generator.h" +#include "util/spinlock.h" +#include "util/uid_util.h" + +namespace doris { + +UniqueRowsetIdGenerator::UniqueRowsetIdGenerator(UniqueId backend_uid) : + _backend_uid(backend_uid), _inc_id(1) { +} +// generator a id according to data dir +// rowsetid is not globally unique, it is dir level +// it saves the batch end id into meta env +OLAPStatus UniqueRowsetIdGenerator::next_id(RowsetId* rowset_id) { + std::lock_guard l(_lock); + rowset_id->init(_version, _backend_uid.hi, _backend_uid.lo, ++_inc_id); + _valid_rowset_ids.insert(*rowset_id); + return OLAP_SUCCESS; +} + +bool UniqueRowsetIdGenerator::id_in_use(RowsetId& rowset_id) { + // if rowset_id == 1, then it is an old version rowsetid, not gc it + // because old version rowset id is not generated by this code, so that not delete them + if (rowset_id.version < _version) { + return true; + } + return _valid_rowset_ids.find(rowset_id) != _valid_rowset_ids.end(); +} + +void UniqueRowsetIdGenerator::release_id(RowsetId& rowset_id) { + _valid_rowset_ids.erase(rowset_id); +} + +} // namespace doris diff --git a/be/src/olap/rowset/unique_rowset_id_generator.h b/be/src/olap/rowset/unique_rowset_id_generator.h new file mode 100644 index 00000000000000..76595393e06c96 --- /dev/null +++ b/be/src/olap/rowset/unique_rowset_id_generator.h @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "olap/rowset/rowset_id_generator.h" +#include "util/spinlock.h" +#include "util/uid_util.h" + +namespace doris { + +class UniqueRowsetIdGenerator : public RowsetIdGenerator { +public: + UniqueRowsetIdGenerator(UniqueId backend_uid); + ~UniqueRowsetIdGenerator() {} + + // generator a id according to data dir + // rowsetid is not globally unique, it is dir level + // it saves the batch end id into meta env + OLAPStatus next_id(RowsetId* rowset_id); + + bool id_in_use(RowsetId& rowset_id); + + void release_id(RowsetId& rowset_id); + +private: + SpinLock _lock; + UniqueId _backend_uid; + int64_t _version = 2; // modify it when create new version id generator + int64_t _inc_id = 0; + std::set _valid_rowset_ids; +}; // FeBasedRowsetIdGenerator + +} // namespace doris diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 4551ef402b14b8..9b24dedd900a0b 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1074,8 +1074,8 @@ bool SchemaChangeWithSorting::_internal_sorting(const vector& row_blo LOG(WARNING) << "new rowset builder failed"; return false; } - RowsetId rowset_id = 0; - OLAPStatus status = new_tablet->next_rowset_id(&rowset_id); + RowsetId rowset_id; + OLAPStatus status = StorageEngine::instance()->next_rowset_id(&rowset_id); if (status != OLAP_SUCCESS) { LOG(WARNING) << "get next rowset id failed"; return false; @@ -1098,10 +1098,10 @@ bool SchemaChangeWithSorting::_internal_sorting(const vector& row_blo rowset_writer->init(context); if (!merger.merge(row_block_arr, rowset_writer, &merged_rows)) { LOG(WARNING) << "failed to merge row blocks."; - new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + std::to_string(rowset_writer->rowset_id())); + new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + rowset_writer->rowset_id().to_string()); return false; } - new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + std::to_string(rowset_writer->rowset_id())); + new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + rowset_writer->rowset_id().to_string()); add_merged_rows(merged_rows); *rowset = rowset_writer->build(); return true; @@ -1667,8 +1667,8 @@ OLAPStatus SchemaChangeHandler::schema_version_convert( RowsetReaderSharedPtr rowset_reader = (*base_rowset)->create_reader(); rowset_reader->init(&_reader_context); - RowsetId rowset_id = 0; - RETURN_NOT_OK(new_tablet->next_rowset_id(&rowset_id)); + RowsetId rowset_id; + RETURN_NOT_OK(StorageEngine::instance()->next_rowset_id(&rowset_id)); RowsetWriterContext writer_context; writer_context.rowset_id = rowset_id; writer_context.tablet_uid = new_tablet->tablet_uid(); @@ -1696,11 +1696,11 @@ OLAPStatus SchemaChangeHandler::schema_version_convert( << "-" << (*base_rowset)->version().second; } res = OLAP_ERR_INPUT_PARAMETER_ERROR; - new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + std::to_string(rowset_writer->rowset_id())); + new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + rowset_writer->rowset_id().to_string()); goto SCHEMA_VERSION_CONVERT_ERR; } *new_rowset = rowset_writer->build(); - new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + std::to_string(rowset_writer->rowset_id())); + new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + rowset_writer->rowset_id().to_string()); if (*new_rowset == nullptr) { LOG(WARNING) << "build rowset failed."; res = OLAP_ERR_MALLOC_ERROR; @@ -1891,9 +1891,9 @@ OLAPStatus SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangePa // set status for monitor // 只要有一个new_table为running，ref table就设置为running // NOTE 如果第一个sub_table先fail，这里会继续按正常走 - RowsetId rowset_id = 0; + RowsetId rowset_id; TabletSharedPtr new_tablet = sc_params.new_tablet; - res = sc_params.new_tablet->next_rowset_id(&rowset_id); + res = StorageEngine::instance()->next_rowset_id(&rowset_id); if (res != OLAP_SUCCESS) { LOG(WARNING) << "generate next id failed"; goto PROCESS_ALTER_EXIT; @@ -1923,10 +1923,10 @@ OLAPStatus SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangePa << " version=" << rs_reader->version().first << "-" << rs_reader->version().second; res = OLAP_ERR_INPUT_PARAMETER_ERROR; - new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + std::to_string(rowset_writer->rowset_id())); + new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + rowset_writer->rowset_id().to_string()); goto PROCESS_ALTER_EXIT; } - new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + std::to_string(rowset_writer->rowset_id())); + new_tablet->data_dir()->remove_pending_ids(ROWSET_ID_PREFIX + rowset_writer->rowset_id().to_string()); // 将新版本的数据加入header // 为了防止死锁的出现，一定要先锁住旧表，再锁住新表 sc_params.new_tablet->obtain_push_lock(); diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index 407ac465f0dfef..e201132d52acf4 100755 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -157,40 +157,11 @@ OLAPStatus SnapshotManager::convert_rowset_ids(DataDir& data_dir, const string& TabletSchema tablet_schema; RETURN_NOT_OK(tablet_schema.init_from_pb(new_tablet_meta_pb.schema())); - RowsetId max_rowset_id = 0; - for (auto& visible_rowset : cloned_tablet_meta_pb.rs_metas()) { - if (visible_rowset.rowset_id() > max_rowset_id) { - max_rowset_id = visible_rowset.rowset_id(); - } - } - - for (auto& inc_rowset : cloned_tablet_meta_pb.inc_rs_metas()) { - if (inc_rowset.rowset_id() > max_rowset_id) { - max_rowset_id = inc_rowset.rowset_id(); - } - } - RowsetId next_rowset_id = 0; - if (tablet == nullptr) { - next_rowset_id = 10000; - } else { - RETURN_NOT_OK(tablet->next_rowset_id(&next_rowset_id)); - } - if (next_rowset_id <= max_rowset_id) { - next_rowset_id = max_rowset_id + 1; - if (tablet != nullptr) { - RETURN_NOT_OK(tablet->set_next_rowset_id(next_rowset_id)); - } - } - std::unordered_map _rs_version_map; for (auto& visible_rowset : cloned_tablet_meta_pb.rs_metas()) { RowsetMetaPB* rowset_meta = new_tablet_meta_pb.add_rs_metas(); - RowsetId rowset_id = 0; - if (tablet != nullptr) { - RETURN_NOT_OK(tablet->next_rowset_id(&rowset_id)); - } else { - rowset_id = ++next_rowset_id; - } + RowsetId rowset_id; + RETURN_NOT_OK(StorageEngine::instance()->next_rowset_id(&rowset_id)); RETURN_NOT_OK(_rename_rowset_id(visible_rowset, clone_dir, data_dir, tablet_schema, rowset_id, rowset_meta)); rowset_meta->set_tablet_id(tablet_id); rowset_meta->set_tablet_schema_hash(schema_hash); @@ -207,23 +178,12 @@ OLAPStatus SnapshotManager::convert_rowset_ids(DataDir& data_dir, const string& continue; } RowsetMetaPB* rowset_meta = new_tablet_meta_pb.add_inc_rs_metas(); - RowsetId rowset_id = 0; - if (tablet != nullptr) { - RETURN_NOT_OK(tablet->next_rowset_id(&rowset_id)); - } else { - rowset_id = ++next_rowset_id; - } + RowsetId rowset_id; + RETURN_NOT_OK(StorageEngine::instance()->next_rowset_id(&rowset_id)); RETURN_NOT_OK(_rename_rowset_id(inc_rowset, clone_dir, data_dir, tablet_schema, rowset_id, rowset_meta)); rowset_meta->set_tablet_id(tablet_id); rowset_meta->set_tablet_schema_hash(schema_hash); } - RowsetId new_next_rowset_id = 0; - if (tablet != nullptr) { - RETURN_NOT_OK(tablet->next_rowset_id(&new_next_rowset_id)); - } else { - new_next_rowset_id = next_rowset_id + 1; - } - new_tablet_meta_pb.set_end_rowset_id(new_next_rowset_id); res = TabletMeta::save(cloned_meta_file, new_tablet_meta_pb); if (res != OLAP_SUCCESS) { diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index caba4108028c17..16bde7642611c2 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -49,6 +49,7 @@ #include "olap/rowset/alpha_rowset_meta.h" #include "olap/rowset/column_data_writer.h" #include "olap/olap_snapshot_converter.h" +#include "olap/rowset/unique_rowset_id_generator.h" #include "util/time.h" #include "util/doris_metrics.h" #include "util/pretty_printer.h" @@ -112,7 +113,8 @@ StorageEngine::StorageEngine(const EngineOptions& options) _is_report_disk_state_already(false), _is_report_tablet_already(false), _tablet_manager(new TabletManager()), - _txn_manager(new TxnManager()) { + _txn_manager(new TxnManager()), + _rowset_id_generator(new UniqueRowsetIdGenerator(options.backend_uid)) { if (_s_instance == nullptr) { _s_instance = this; } diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index dad4ce34529655..71e75d460344b7 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -47,6 +47,7 @@ #include "olap/tablet_sync_service.h" #include "olap/txn_manager.h" #include "olap/task/engine_task.h" +#include "olap/rowset/rowset_id_generator.h" namespace doris { @@ -196,6 +197,12 @@ class StorageEngine { // TODO(ygl) TabletSyncService* tablet_sync_service() { return nullptr; } + OLAPStatus next_rowset_id(RowsetId* rowset_id) { return _rowset_id_generator->next_id(rowset_id); }; + + bool rowset_id_in_use(RowsetId& rowset_id) { return _rowset_id_generator->id_in_use(rowset_id); }; + + void release_rowset_id(RowsetId& rowset_id) { return _rowset_id_generator->release_id(rowset_id); }; + private: OLAPStatus check_all_root_path_cluster_id(); @@ -333,6 +340,8 @@ class StorageEngine { std::unique_ptr _tablet_manager; std::unique_ptr _txn_manager; + RowsetIdGenerator* _rowset_id_generator; + DISALLOW_COPY_AND_ASSIGN(StorageEngine); }; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 225799601ca9f4..f7e06841200750 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -887,18 +887,6 @@ bool Tablet::check_rowset_id(RowsetId rowset_id) { return false; } -// lock here, function that call next_rowset_id should not have meta lock -OLAPStatus Tablet::next_rowset_id(RowsetId* id) { - WriteLock wrlock(&_meta_lock); - return _tablet_meta->get_next_rowset_id(id, _data_dir); -} - -// lock here, function that call set_next_rowset_id should not have meta lock -OLAPStatus Tablet::set_next_rowset_id(RowsetId new_rowset_id) { - WriteLock wrlock(&_meta_lock); - return _tablet_meta->set_next_rowset_id(new_rowset_id, _data_dir); -} - void Tablet::_print_missed_versions(const std::vector& missed_versions) const { std::stringstream ss; ss << full_name() << " has "<< missed_versions.size() << " missed version:"; @@ -913,13 +901,6 @@ void Tablet::_print_missed_versions(const std::vector& missed_versions) if (rowset == nullptr) { return OLAP_ERR_ROWSET_INVALID; } - // check if the rowset id is valid - if (rowset->rowset_id() >= _tablet_meta->get_cur_rowset_id()) { - LOG(FATAL) << "rowset id is larger than next rowsetid, it is fatal error" - << " rowset_id=" << rowset->rowset_id() - << " next_id=" << _tablet_meta->get_cur_rowset_id(); - return OLAP_ERR_ROWSET_INVALID; - } Version version = {rowset->start_version(), rowset->end_version()}; RowsetSharedPtr exist_rs = get_rowset_by_version(version); // if there exist a rowset with version_hash == 0, should delete it diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 472a968ec5bf99..aeb7f01174a7b9 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -216,19 +216,10 @@ class Tablet : public std::enable_shared_from_this { void delete_all_files(); bool check_path(const std::string& check_path); - - // check rowset_id is valid bool check_rowset_id(RowsetId rowset_id); - OLAPStatus next_rowset_id(RowsetId* id); - OLAPStatus set_next_rowset_id(RowsetId new_rowset_id); - OLAPStatus set_partition_id(int64_t partition_id); - RowsetId initial_end_rowset_id() { - return _tablet_meta->initial_end_rowset_id(); - } - TabletInfo get_tablet_info(); void pick_candicate_rowsets_to_cumulative_compaction(std::vector* candidate_rowsets); diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index eb876f55394a89..2f98c6030cbf52 100755 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -705,12 +705,12 @@ bool TabletManager::get_tablet_id_and_schema_hash_from_path(const std::string& p } bool TabletManager::get_rowset_id_from_path(const std::string& path, RowsetId* rowset_id) { - static std::regex rgx ("/data/\\d+/\\d+/\\d+/(\\d+)_.*"); + static std::regex rgx ("/data/\\d+/\\d+/\\d+/([A-Fa-f0-9]+)_.*"); std::smatch sm; bool ret = std::regex_search(path, sm, rgx); if (ret) { if (sm.size() == 2) { - *rowset_id = std::strtoll(sm.str(1).c_str(), nullptr, 10); + rowset_id->init(sm.str(1)); return true; } else { return false; @@ -1225,7 +1225,8 @@ OLAPStatus TabletManager::_create_inital_rowset( res = OLAP_ERR_INPUT_PARAMETER_ERROR; break; } - RowsetId rowset_id = 1; + RowsetId rowset_id; + RETURN_NOT_OK(StorageEngine::instance()->next_rowset_id(&rowset_id)); // if we know this is the first rowset in this tablet, then not call // tablet to generate rowset id, just set it to 1 // RETURN_NOT_OK(tablet->next_rowset_id(&rowset_id)); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 74b42b85038fb3..969c4a34ad8544 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -86,7 +86,6 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, tablet_meta_pb.set_cumulative_layer_point(-1); tablet_meta_pb.set_tablet_state(PB_RUNNING); *(tablet_meta_pb.mutable_tablet_uid()) = tablet_uid.to_proto(); - tablet_meta_pb.set_end_rowset_id(10000); TabletSchemaPB* schema = tablet_meta_pb.mutable_schema(); schema->set_num_short_key_columns(tablet_schema.short_key_column_count); schema->set_num_rows_per_row_block(config::default_num_rows_per_column_file_block); @@ -263,29 +262,6 @@ OLAPStatus TabletMeta::save_meta(DataDir* data_dir) { } OLAPStatus TabletMeta::_save_meta(DataDir* data_dir) { - // check if rowset id all valid, should remove it later - for (auto& rs_meta : _rs_metas) { - if (rs_meta->rowset_id() >= _next_rowset_id) { - LOG(FATAL) << "meta contains invalid rowsetid " - << " tablet=" << full_name() - << " rowset_id=" << rs_meta->rowset_id() - << " next_rowset_id=" << _next_rowset_id; - } - } - for (auto& rs_meta : _inc_rs_metas) { - if (rs_meta->rowset_id() >= _next_rowset_id) { - LOG(FATAL) << "meta contains invalid rowsetid " - << " tablet=" << full_name() - << " rowset_id=" << rs_meta->rowset_id() - << " next_rowset_id=" << _next_rowset_id; - } - } - // check if _end_rowset_id > 10000 - if (_end_rowset_id < 10000) { - LOG(FATAL) << "end_rowset_id is invalid" - << " tablet=" << full_name() - << " end_rowset_id=" << _end_rowset_id; - } // check if tablet uid is valid if (_tablet_uid.hi == 0 && _tablet_uid.lo == 0) { LOG(FATAL) << "tablet_uid is invalid" @@ -338,9 +314,6 @@ OLAPStatus TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { _creation_time = tablet_meta_pb.creation_time(); _cumulative_layer_point = tablet_meta_pb.cumulative_layer_point(); _tablet_uid = TabletUid(tablet_meta_pb.tablet_uid()); - _end_rowset_id = tablet_meta_pb.end_rowset_id(); - _initial_end_rowset_id = tablet_meta_pb.end_rowset_id(); - _next_rowset_id = _end_rowset_id + 1; // init _tablet_state switch (tablet_meta_pb.tablet_state()) { @@ -404,7 +377,6 @@ OLAPStatus TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) { tablet_meta_pb->set_creation_time(creation_time()); tablet_meta_pb->set_cumulative_layer_point(cumulative_layer_point()); *(tablet_meta_pb->mutable_tablet_uid()) = tablet_uid().to_proto(); - tablet_meta_pb->set_end_rowset_id(_end_rowset_id); switch (tablet_state()) { case TABLET_NOTREADY: tablet_meta_pb->set_tablet_state(PB_NOTREADY); @@ -679,37 +651,6 @@ bool TabletMeta::version_for_delete_predicate(const Version& version) { return false; } -OLAPStatus TabletMeta::get_next_rowset_id(RowsetId* gen_rowset_id, DataDir* data_dir) { - WriteLock wrlock(&_meta_lock); - if (_next_rowset_id >= _end_rowset_id) { - ++_next_rowset_id; - _end_rowset_id = _next_rowset_id + _batch_interval; - RETURN_NOT_OK(_save_meta(data_dir)); - } - *gen_rowset_id = _next_rowset_id; - ++_next_rowset_id; - return OLAP_SUCCESS; -} - -OLAPStatus TabletMeta::set_next_rowset_id(RowsetId new_rowset_id, DataDir* data_dir) { - WriteLock wrlock(&_meta_lock); - // must be < not <= - if (new_rowset_id < _next_rowset_id) { - return OLAP_SUCCESS; - } - if (new_rowset_id >= _end_rowset_id) { - _end_rowset_id = new_rowset_id + _batch_interval; - RETURN_NOT_OK(_save_meta(data_dir)); - } - _next_rowset_id = new_rowset_id + 1; - return OLAP_SUCCESS; -} - -RowsetId TabletMeta::get_cur_rowset_id() { - return _next_rowset_id; -} - - // return value not reference // MVCC modification for alter task, upper application get a alter task mirror AlterTabletTaskSharedPtr TabletMeta::TabletMeta::alter_task() { diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 9fbd8cbeb4bbf2..60831a7fb2e3b9 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -179,22 +179,10 @@ class TabletMeta { OLAPStatus delete_alter_task(); OLAPStatus set_alter_state(AlterTabletState alter_state); - // rowsetid is not globally unique, it is tablet level - // it saves the batch end id into meta env - OLAPStatus get_next_rowset_id(RowsetId* rowset_id, DataDir* data_dir); - - OLAPStatus set_next_rowset_id(RowsetId new_rowset_id, DataDir* data_dir); - - RowsetId get_cur_rowset_id(); - std::string full_name() const; OLAPStatus set_partition_id(int64_t partition_id); - RowsetId initial_end_rowset_id() { - return _initial_end_rowset_id; - } - private: OLAPStatus _save_meta(DataDir* data_dir); @@ -207,11 +195,6 @@ class TabletMeta { int64_t _creation_time; int64_t _cumulative_layer_point; TabletUid _tablet_uid; - RowsetId _next_rowset_id = 10000; - RowsetId _end_rowset_id; - RowsetId _initial_end_rowset_id; - RowsetId _batch_interval = 10000; - TabletState _tablet_state; TabletSchema _schema; diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index 13300f5b318207..cc7348456bb321 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -526,7 +526,7 @@ AgentStatus EngineCloneTask::_clone_copy( << ", total file size: " << total_file_size << " B" << ", cost: " << total_time_ms << " ms" << ", rate: " << copy_rate << " B/s"; - if (make_snapshot_result.snapshot_version < PREFERRED_SNAPSHOT_VERSION) { + if (make_snapshot_result.snapshot_version == 1) { OLAPStatus convert_status = _convert_to_new_snapshot(data_dir, local_data_path, clone_req.tablet_id); if (convert_status != OLAP_SUCCESS) { status = DORIS_ERROR; diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp index a856510c797a43..d7eb37adf46020 100755 --- a/be/src/olap/txn_manager.cpp +++ b/be/src/olap/txn_manager.cpp @@ -337,7 +337,7 @@ OLAPStatus TxnManager::delete_txn(OlapMeta* meta, TPartitionId partition_id, TTr << " partition_id: " << key.first << ", transaction_id: " << key.second << ", tablet: " << tablet_info.to_string() - << ", rowset: " << (load_info.rowset != nullptr ? load_info.rowset->rowset_id(): 0); + << ", rowset: " << (load_info.rowset != nullptr ? load_info.rowset->rowset_id().to_string(): "0"); } } } @@ -388,7 +388,7 @@ void TxnManager::force_rollback_tablet_related_txns(OlapMeta* meta, TTabletId ta << " partition_id: " << it.first.first << ", transaction_id: " << it.first.second << ", tablet: " << tablet_info.to_string() - << ", rowset: " << (load_info.rowset != nullptr ? load_info.rowset->rowset_id(): 0); + << ", rowset: " << (load_info.rowset != nullptr ? load_info.rowset->rowset_id().to_string() : "0"); it.second.erase(tablet_info); } if (it.second.empty()) { diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index c8a22e8fad104d..a916039e2fa576 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -56,6 +56,7 @@ #include #include "common/resource_tls.h" #include "util/frontend_helper.h" +#include "util/uid_util.h" static void help(const char*); @@ -153,6 +154,7 @@ int main(int argc, char** argv) { // options doris::EngineOptions options; options.store_paths = paths; + options.backend_uid = doris::UniqueId(); doris::StorageEngine* engine = nullptr; auto st = doris::StorageEngine::open(options, &engine); if (!st.ok()) { diff --git a/be/src/util/uid_util.h b/be/src/util/uid_util.h index 0806dab0afb4a3..a626aa3ad24335 100644 --- a/be/src/util/uid_util.h +++ b/be/src/util/uid_util.h @@ -63,6 +63,8 @@ struct UniqueId { int64_t hi; int64_t lo; + // !!!! Not modify this method, it is very important. it will generate a random uid + // it need modify it contact yiguolei UniqueId() { auto uuid = boost::uuids::basic_random_generator()(); memcpy(&hi, uuid.data, sizeof(int64_t)); diff --git a/be/test/olap/CMakeLists.txt b/be/test/olap/CMakeLists.txt index 2b3056ed39b172..8af914139fd002 100644 --- a/be/test/olap/CMakeLists.txt +++ b/be/test/olap/CMakeLists.txt @@ -62,6 +62,7 @@ ADD_BE_TEST(tablet_mgr_test) ADD_BE_TEST(rowset/rowset_meta_manager_test) ADD_BE_TEST(rowset/rowset_meta_test) ADD_BE_TEST(rowset/alpha_rowset_test) +ADD_BE_TEST(rowset/unique_rowset_id_generator_test) ADD_BE_TEST(olap_snapshot_converter_test) ADD_BE_TEST(txn_manager_test) ADD_BE_TEST(generic_iterators_test) diff --git a/be/test/olap/olap_snapshot_converter_test.cpp b/be/test/olap/olap_snapshot_converter_test.cpp index e6db1f6a3d1319..fef45dba9ade68 100755 --- a/be/test/olap/olap_snapshot_converter_test.cpp +++ b/be/test/olap/olap_snapshot_converter_test.cpp @@ -45,15 +45,27 @@ using std::string; namespace doris { +static StorageEngine* k_engine = nullptr; + class OlapSnapshotConverterTest : public testing::Test { public: virtual void SetUp() { + std::vector paths; + paths.emplace_back("_engine_data_path", -1); + EngineOptions options; + options.store_paths = paths; + options.backend_uid = doris::UniqueId(); + if (k_engine == nullptr) { + k_engine = new StorageEngine(options); + } + auto cache = new_lru_cache(config::file_descriptor_cache_capacity); FileHandler::set_fd_cache(cache); string test_engine_data_path = "./be/test/olap/test_data/converter_test_data/data"; _engine_data_path = "./be/test/olap/test_data/converter_test_data/tmp"; boost::filesystem::remove_all(_engine_data_path); create_dirs(_engine_data_path); + _data_dir = new DataDir(_engine_data_path, 1000000000); _data_dir->init(); _meta_path = "./meta"; diff --git a/be/test/olap/rowset/alpha_rowset_test.cpp b/be/test/olap/rowset/alpha_rowset_test.cpp index a5c28081a578b6..6aad201d8c627c 100644 --- a/be/test/olap/rowset/alpha_rowset_test.cpp +++ b/be/test/olap/rowset/alpha_rowset_test.cpp @@ -85,7 +85,9 @@ void tear_down() { void create_rowset_writer_context(TabletSchema* tablet_schema, DataDir* data_dir, RowsetWriterContext* rowset_writer_context) { - rowset_writer_context->rowset_id = 10000; + RowsetId rowset_id; + rowset_id.init(10000); + rowset_writer_context->rowset_id = rowset_id; rowset_writer_context->tablet_id = 12345; rowset_writer_context->tablet_schema_hash = 1111; rowset_writer_context->partition_id = 10; @@ -226,7 +228,9 @@ TEST_F(AlphaRowsetTest, TestAlphaRowsetReader) { _alpha_rowset_writer->flush(); RowsetSharedPtr alpha_rowset = _alpha_rowset_writer->build(); ASSERT_TRUE(alpha_rowset != nullptr); - ASSERT_EQ(10000, alpha_rowset->rowset_id()); + RowsetId rowset_id; + rowset_id.init(10000); + ASSERT_EQ(rowset_id, alpha_rowset->rowset_id()); ASSERT_EQ(1, alpha_rowset->num_rows()); RowsetReaderSharedPtr rowset_reader = alpha_rowset->create_reader(); ASSERT_TRUE(rowset_reader != nullptr); diff --git a/be/test/olap/rowset/rowset_meta_manager_test.cpp b/be/test/olap/rowset/rowset_meta_manager_test.cpp index 63c5125d49e370..22246ae728819c 100644 --- a/be/test/olap/rowset/rowset_meta_manager_test.cpp +++ b/be/test/olap/rowset/rowset_meta_manager_test.cpp @@ -23,6 +23,7 @@ #include "gmock/gmock.h" #include "olap/olap_meta.h" #include "olap/rowset/rowset_meta_manager.h" +#include "olap/storage_engine.h" #include "olap/new_status.h" #include "boost/filesystem.hpp" #include "json2pb/json_to_pb.h" @@ -38,11 +39,23 @@ using std::string; namespace doris { +static StorageEngine* k_engine = nullptr; + const std::string rowset_meta_path = "./be/test/olap/test_data/rowset_meta.json"; class RowsetMetaManagerTest : public testing::Test { public: virtual void SetUp() { + + std::vector paths; + paths.emplace_back("_engine_data_path", -1); + EngineOptions options; + options.store_paths = paths; + options.backend_uid = doris::UniqueId(); + if (k_engine == nullptr) { + k_engine = new StorageEngine(options); + } + std::string meta_path = "./meta"; ASSERT_TRUE(boost::filesystem::create_directory(meta_path)); _meta = new(std::nothrow) OlapMeta(meta_path); @@ -74,7 +87,8 @@ class RowsetMetaManagerTest : public testing::Test { }; TEST_F(RowsetMetaManagerTest, TestSaveAndGetAndRemove) { - uint64_t rowset_id = 10000; + RowsetId rowset_id; + rowset_id.init(10000); RowsetMeta rowset_meta; rowset_meta.init_from_json(_json_rowset_meta); ASSERT_EQ(rowset_meta.rowset_id(), rowset_id); @@ -94,7 +108,8 @@ TEST_F(RowsetMetaManagerTest, TestSaveAndGetAndRemove) { } TEST_F(RowsetMetaManagerTest, TestLoad) { - uint64_t rowset_id = 10000; + RowsetId rowset_id; + rowset_id.init(10000); OLAPStatus status = RowsetMetaManager::load_json_rowset_meta(_meta, rowset_meta_path); ASSERT_TRUE(status == OLAP_SUCCESS); ASSERT_TRUE(RowsetMetaManager::check_rowset_meta(_meta, _tablet_uid, rowset_id)); diff --git a/be/test/olap/rowset/rowset_meta_test.cpp b/be/test/olap/rowset/rowset_meta_test.cpp index 56f2059b386ed4..7320ec73325716 100644 --- a/be/test/olap/rowset/rowset_meta_test.cpp +++ b/be/test/olap/rowset/rowset_meta_test.cpp @@ -72,7 +72,9 @@ class RowsetMetaTest : public testing::Test { }; void do_check(RowsetMeta rowset_meta) { - ASSERT_EQ(540081, rowset_meta.rowset_id()); + RowsetId rowset_id; + rowset_id.init(540081); + ASSERT_EQ(rowset_id, rowset_meta.rowset_id()); ASSERT_EQ(15673, rowset_meta.tablet_id()); ASSERT_EQ(4042, rowset_meta.txn_id()); ASSERT_EQ(567997577, rowset_meta.tablet_schema_hash()); @@ -112,7 +114,9 @@ TEST_F(RowsetMetaTest, TestInitWithInvalidData) { } void do_check_for_alpha(AlphaRowsetMeta alpha_rowset_meta) { - ASSERT_EQ(540081, alpha_rowset_meta.rowset_id()); + RowsetId rowset_id; + rowset_id.init(540081); + ASSERT_EQ(rowset_id, alpha_rowset_meta.rowset_id()); ASSERT_EQ(15673, alpha_rowset_meta.tablet_id()); ASSERT_EQ(4042, alpha_rowset_meta.txn_id()); ASSERT_EQ(567997577, alpha_rowset_meta.tablet_schema_hash()); diff --git a/be/test/olap/rowset/unique_rowset_id_generator_test.cpp b/be/test/olap/rowset/unique_rowset_id_generator_test.cpp new file mode 100644 index 00000000000000..6e6efe2f6531a7 --- /dev/null +++ b/be/test/olap/rowset/unique_rowset_id_generator_test.cpp @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/rowset/unique_rowset_id_generator.h" + +#include +#include + +namespace doris { +class UniqueRowsetIdGeneratorTest : public testing::Test { +public: + UniqueRowsetIdGeneratorTest() { } + virtual ~UniqueRowsetIdGeneratorTest() { + } +}; + +TEST_F(UniqueRowsetIdGeneratorTest, RowsetIdFormatTest) { + { + RowsetId rowset_id; + rowset_id.init(123); + ASSERT_TRUE(rowset_id.version == 1); + ASSERT_TRUE(rowset_id.lo == 123); + ASSERT_TRUE(rowset_id.mi == 0); + ASSERT_STREQ("123", rowset_id.to_string().c_str()); + } + { + RowsetId rowset_id; + rowset_id.init("123"); + ASSERT_TRUE(rowset_id.version == 1); + ASSERT_TRUE(rowset_id.lo == 123); + ASSERT_TRUE(rowset_id.mi == 0); + ASSERT_STREQ("123", rowset_id.to_string().c_str()); + } + { + RowsetId rowset_id; + rowset_id.init("024a07c4e21cec017ea9b7e88bf6768a0000000000000019"); + ASSERT_TRUE(rowset_id.version == 2); + ASSERT_TRUE(rowset_id.lo == 19); + ASSERT_STREQ("024a07c4e21cec017ea9b7e88bf6768a0000000000000019", rowset_id.to_string().c_str()); + } +} + + +TEST_F(UniqueRowsetIdGeneratorTest, GenerateIdTest) { + UniqueId backend_uid; + UniqueRowsetIdGenerator id_generator(backend_uid); + { + RowsetId rowset_id; + id_generator.next_id(&rowset_id); + ASSERT_TRUE(rowset_id.lo == 1); + ASSERT_TRUE(rowset_id.version == 2); + ASSERT_TRUE(backend_uid.lo == rowset_id.mi); + ASSERT_TRUE(backend_uid.hi != rowset_id.hi); + ASSERT_TRUE(rowset_id.hi != 0); + bool in_use = id_generator.id_in_use(rowset_id); + ASSERT_TRUE(in_use == true); + id_generator.release_id(rowset_id); + in_use = id_generator.id_in_use(rowset_id); + ASSERT_TRUE(in_use == false); + + int64_t hi = 2; + hi = (hi << 56) + (backend_uid.hi & 0x00ffffffffffffff); + ASSERT_TRUE(hi == rowset_id.hi); + + int64_t low = rowset_id.lo + 1; + id_generator.next_id(&rowset_id); + ASSERT_TRUE(rowset_id.lo == low); + in_use = id_generator.id_in_use(rowset_id); + ASSERT_TRUE(in_use == true); + + std::string rowset_mid_str = rowset_id.to_string().substr(16,16); + std::string backend_mid_str = backend_uid.to_string().substr(17, 16); + ASSERT_STREQ(rowset_mid_str.c_str(), backend_mid_str.c_str()); + } +} + +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + doris::CpuInfo::init(); + return RUN_ALL_TESTS(); +} + diff --git a/be/test/olap/tablet_mgr_test.cpp b/be/test/olap/tablet_mgr_test.cpp index 498fa25cd9bfce..debcefb6a81408 100644 --- a/be/test/olap/tablet_mgr_test.cpp +++ b/be/test/olap/tablet_mgr_test.cpp @@ -42,6 +42,8 @@ using std::string; namespace doris { +static StorageEngine* k_engine = nullptr; + class TabletMgrTest : public testing::Test { public: virtual void SetUp() { @@ -51,9 +53,19 @@ class TabletMgrTest : public testing::Test { _engine_data_path = "./be/test/olap/test_data/converter_test_data/tmp"; boost::filesystem::remove_all(_engine_data_path); create_dirs(_engine_data_path); + create_dirs(_engine_data_path + "/meta"); + + std::vector paths; + paths.emplace_back("_engine_data_path", -1); + EngineOptions options; + options.store_paths = paths; + options.backend_uid = doris::UniqueId(); + if (k_engine == nullptr) { + k_engine = new StorageEngine(options); + } + _data_dir = new DataDir(_engine_data_path, 1000000000); _data_dir->init(); - _meta_path = "./meta"; string tmp_data_path = _engine_data_path + "/data"; if (boost::filesystem::exists(tmp_data_path)) { boost::filesystem::remove_all(tmp_data_path); @@ -65,23 +77,10 @@ class TabletMgrTest : public testing::Test { + "/" + std::to_string(0) + "/" + std::to_string(_tablet_id) + "/" + std::to_string(_schema_hash); - if (boost::filesystem::exists(_meta_path)) { - boost::filesystem::remove_all(_meta_path); - } - ASSERT_TRUE(boost::filesystem::create_directory(_meta_path)); - ASSERT_TRUE(boost::filesystem::exists(_meta_path)); - _meta = new(std::nothrow) OlapMeta(_meta_path); - ASSERT_NE(nullptr, _meta); - OLAPStatus st = _meta->init(); - ASSERT_TRUE(st == OLAP_SUCCESS); } virtual void TearDown() { - delete _meta; delete _data_dir; - if (boost::filesystem::exists(_meta_path)) { - ASSERT_TRUE(boost::filesystem::remove_all(_meta_path)); - } if (boost::filesystem::exists(_engine_data_path)) { ASSERT_TRUE(boost::filesystem::remove_all(_engine_data_path)); } @@ -90,11 +89,9 @@ class TabletMgrTest : public testing::Test { private: DataDir* _data_dir; - OlapMeta* _meta; std::string _json_rowset_meta; TxnManager _txn_mgr; std::string _engine_data_path; - std::string _meta_path; int64_t _tablet_id; int32_t _schema_hash; string _tablet_data_path; diff --git a/be/test/olap/test_data/header.txt b/be/test/olap/test_data/header.txt index f596d7f2793259..c7efbc9ff0df0c 100644 --- a/be/test/olap/test_data/header.txt +++ b/be/test/olap/test_data/header.txt @@ -183,6 +183,5 @@ "tablet_uid": { "hi": 10, "lo": 10 - }, - "end_rowset_id": 10000 + } } diff --git a/be/test/olap/txn_manager_test.cpp b/be/test/olap/txn_manager_test.cpp index 9996f63cb12b38..af897454ed81d1 100644 --- a/be/test/olap/txn_manager_test.cpp +++ b/be/test/olap/txn_manager_test.cpp @@ -41,12 +41,24 @@ using std::string; namespace doris { +static StorageEngine* k_engine = nullptr; + const std::string rowset_meta_path = "./be/test/olap/test_data/rowset_meta.json"; const std::string rowset_meta_path_2 = "./be/test/olap/test_data/rowset_meta2.json"; class TxnManagerTest : public testing::Test { public: virtual void SetUp() { + + std::vector paths; + paths.emplace_back("_engine_data_path", -1); + EngineOptions options; + options.store_paths = paths; + options.backend_uid = doris::UniqueId(); + if (k_engine == nullptr) { + k_engine = new StorageEngine(options); + } + std::string meta_path = "./meta"; boost::filesystem::remove_all("./meta"); ASSERT_TRUE(boost::filesystem::create_directory(meta_path)); @@ -65,8 +77,8 @@ class TxnManagerTest : public testing::Test { _json_rowset_meta = _json_rowset_meta + buffer + "\n"; } _json_rowset_meta = _json_rowset_meta.substr(0, _json_rowset_meta.size() - 1); - - uint64_t rowset_id = 10000; + RowsetId rowset_id; + rowset_id.init(10000); RowsetMetaSharedPtr rowset_meta(new AlphaRowsetMeta()); rowset_meta->init_from_json(_json_rowset_meta); ASSERT_EQ(rowset_meta->rowset_id(), rowset_id); @@ -82,8 +94,8 @@ class TxnManagerTest : public testing::Test { _json_rowset_meta = _json_rowset_meta + buffer2 + "\n"; std::cout << _json_rowset_meta << std::endl; } - _json_rowset_meta = _json_rowset_meta.substr(0, _json_rowset_meta.size() - 1); - rowset_id = 10001; + _json_rowset_meta = _json_rowset_meta.substr(0, _json_rowset_meta.size() - 1); + rowset_id.init(10001); RowsetMetaSharedPtr rowset_meta2(new AlphaRowsetMeta()); rowset_meta2->init_from_json(_json_rowset_meta); ASSERT_EQ(rowset_meta2->rowset_id(), rowset_id); diff --git a/fe/src/main/java/org/apache/doris/task/SnapshotTask.java b/fe/src/main/java/org/apache/doris/task/SnapshotTask.java index 2ea7184ccbb0b6..332d66709d91b9 100644 --- a/fe/src/main/java/org/apache/doris/task/SnapshotTask.java +++ b/fe/src/main/java/org/apache/doris/task/SnapshotTask.java @@ -79,7 +79,7 @@ public TSnapshotRequest toThrift() { request.setVersion(version); request.setVersion_hash(versionHash); request.setList_files(true); - request.setPreferred_snapshot_version(2); + request.setPreferred_snapshot_version(3); request.setTimeout(timeoutMs / 1000); return request; } diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 7dfc27ffd5b21c..953dc59f00ec49 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -110,6 +110,8 @@ message RowsetMetaPB { optional PUniqueId tablet_uid = 21; // total number of segments optional int64 num_segments = 22; + // rowset id definition, it will replace required rowset id + optional string rowset_id_v2 = 23; // spare field id for future use optional AlphaRowsetExtraMetaPB alpha_rowset_extra_meta_pb = 50; } From ee2f33e4c5d1137684070242e1af777726d1b030 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Tue, 27 Aug 2019 14:35:05 +0800 Subject: [PATCH 2/7] Using low as rowset version --- be/src/olap/olap_common.h | 31 ++++++++++++------- be/src/olap/rowset/alpha_rowset_writer.cpp | 5 +++ .../unique_rowset_id_generator_test.cpp | 21 +++++++------ 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index dc6dcbc430a0a9..85fdda602022ac 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -36,8 +36,12 @@ #include "util/hash_util.hpp" #include "util/uid_util.h" +#define LOW_56_BITS 0x00ffffffffffffff + namespace doris { +static const int64_t MAX_ROWSET_ID = 1L << 56; + typedef int32_t SchemaHash; typedef int64_t VersionHash; typedef __int128 int128_t; @@ -255,16 +259,16 @@ struct RowsetId { // for new rowsetid its a 48 hex string // if the len < 48, then it is an old format rowset id if (rowset_id_str.length() < 48) { - hi = 1; - hi = hi << 56; - mi = 0; - lo = std::stol(rowset_id_str, nullptr, 10); - version = 1; + int64_t low = std::stol(rowset_id_str, nullptr, 10); + init(1, 0, 0, low); } else { - from_hex(&hi, rowset_id_str.substr(0, 16)); - from_hex(&mi, rowset_id_str.substr(16, 16)); - from_hex(&lo, rowset_id_str.substr(32, 16)); - version = hi >> 56; + int64_t high = 0; + int64_t middle = 0; + int64_t low = 0; + from_hex(&high, rowset_id_str.substr(0, 16)); + from_hex(&middle, rowset_id_str.substr(16, 16)); + from_hex(&low, rowset_id_str.substr(32, 16)); + init(low >> 56, high, middle, low & LOW_56_BITS); } } @@ -275,14 +279,17 @@ struct RowsetId { void init(int64_t id_version, int64_t high, int64_t middle, int64_t low) { version = id_version; - hi = (id_version << 56) + (high & 0x00ffffffffffffff); + if (low >= MAX_ROWSET_ID) { + LOG(FATAL) << "low is too large" << low; + } + hi = high; mi = middle; - lo = low; + lo = (id_version << 56) + (low & LOW_56_BITS); } std::string to_string() const { if (version < 2) { - return std::to_string(lo); + return std::to_string(lo & LOW_56_BITS); } else { char buf[48]; to_hex(hi, buf); diff --git a/be/src/olap/rowset/alpha_rowset_writer.cpp b/be/src/olap/rowset/alpha_rowset_writer.cpp index 77b7f735874733..1383b36318ece6 100644 --- a/be/src/olap/rowset/alpha_rowset_writer.cpp +++ b/be/src/olap/rowset/alpha_rowset_writer.cpp @@ -148,6 +148,11 @@ OLAPStatus AlphaRowsetWriter::flush() { } RowsetSharedPtr AlphaRowsetWriter::build() { + if (_current_rowset_meta->rowset_id().version == 0) { + LOG(WARNING) << "invalid rowset id, version == 0, rowset id=" + << _current_rowset_meta->rowset_id().to_string(); + return nullptr; + } if (_writer_state != WRITER_FLUSHED) { LOG(WARNING) << "invalid writer state before build, state:" << _writer_state; return nullptr; diff --git a/be/test/olap/rowset/unique_rowset_id_generator_test.cpp b/be/test/olap/rowset/unique_rowset_id_generator_test.cpp index 6e6efe2f6531a7..6b85ed4c95820b 100644 --- a/be/test/olap/rowset/unique_rowset_id_generator_test.cpp +++ b/be/test/olap/rowset/unique_rowset_id_generator_test.cpp @@ -29,29 +29,34 @@ class UniqueRowsetIdGeneratorTest : public testing::Test { }; TEST_F(UniqueRowsetIdGeneratorTest, RowsetIdFormatTest) { + int64_t max_id = 1; + max_id = max_id << 56; { RowsetId rowset_id; rowset_id.init(123); ASSERT_TRUE(rowset_id.version == 1); - ASSERT_TRUE(rowset_id.lo == 123); + ASSERT_TRUE(rowset_id.lo == (123 + max_id)); ASSERT_TRUE(rowset_id.mi == 0); + ASSERT_TRUE(rowset_id.hi == 0); ASSERT_STREQ("123", rowset_id.to_string().c_str()); } { RowsetId rowset_id; rowset_id.init("123"); ASSERT_TRUE(rowset_id.version == 1); - ASSERT_TRUE(rowset_id.lo == 123); + ASSERT_TRUE(rowset_id.lo == (123 + max_id)); ASSERT_TRUE(rowset_id.mi == 0); + ASSERT_TRUE(rowset_id.hi == 0); ASSERT_STREQ("123", rowset_id.to_string().c_str()); } + /** { RowsetId rowset_id; rowset_id.init("024a07c4e21cec017ea9b7e88bf6768a0000000000000019"); ASSERT_TRUE(rowset_id.version == 2); ASSERT_TRUE(rowset_id.lo == 19); ASSERT_STREQ("024a07c4e21cec017ea9b7e88bf6768a0000000000000019", rowset_id.to_string().c_str()); - } + }*/ } @@ -59,12 +64,14 @@ TEST_F(UniqueRowsetIdGeneratorTest, GenerateIdTest) { UniqueId backend_uid; UniqueRowsetIdGenerator id_generator(backend_uid); { + int64_t max_id = 2; + max_id = max_id << 56; RowsetId rowset_id; id_generator.next_id(&rowset_id); - ASSERT_TRUE(rowset_id.lo == 1); + ASSERT_TRUE(rowset_id.lo == (1 + max_id)); ASSERT_TRUE(rowset_id.version == 2); ASSERT_TRUE(backend_uid.lo == rowset_id.mi); - ASSERT_TRUE(backend_uid.hi != rowset_id.hi); + ASSERT_TRUE(backend_uid.hi == rowset_id.hi); ASSERT_TRUE(rowset_id.hi != 0); bool in_use = id_generator.id_in_use(rowset_id); ASSERT_TRUE(in_use == true); @@ -72,10 +79,6 @@ TEST_F(UniqueRowsetIdGeneratorTest, GenerateIdTest) { in_use = id_generator.id_in_use(rowset_id); ASSERT_TRUE(in_use == false); - int64_t hi = 2; - hi = (hi << 56) + (backend_uid.hi & 0x00ffffffffffffff); - ASSERT_TRUE(hi == rowset_id.hi); - int64_t low = rowset_id.lo + 1; id_generator.next_id(&rowset_id); ASSERT_TRUE(rowset_id.lo == low); From 1af363d2345adf67572b8539b23c23f37a897ddb Mon Sep 17 00:00:00 2001 From: yiguolei Date: Tue, 27 Aug 2019 16:05:15 +0800 Subject: [PATCH 3/7] Set rowset id to 0 when convert from old tablet --- be/src/olap/olap_snapshot_converter.cpp | 2 ++ .../olap/rowset/unique_rowset_id_generator_test.cpp | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/be/src/olap/olap_snapshot_converter.cpp b/be/src/olap/olap_snapshot_converter.cpp index 0a556f8b45d139..99e11565132c3b 100755 --- a/be/src/olap/olap_snapshot_converter.cpp +++ b/be/src/olap/olap_snapshot_converter.cpp @@ -201,6 +201,7 @@ OLAPStatus OlapSnapshotConverter::convert_to_pdelta(const RowsetMetaPB& rowset_m OLAPStatus OlapSnapshotConverter::convert_to_rowset_meta(const PDelta& delta, const RowsetId& rowset_id, int64_t tablet_id, int32_t schema_hash, RowsetMetaPB* rowset_meta_pb) { + rowset_meta_pb->set_rowset_id(0); rowset_meta_pb->set_rowset_id_v2(rowset_id.to_string()); rowset_meta_pb->set_tablet_id(tablet_id); rowset_meta_pb->set_tablet_schema_hash(schema_hash); @@ -248,6 +249,7 @@ OLAPStatus OlapSnapshotConverter::convert_to_rowset_meta(const PDelta& delta, OLAPStatus OlapSnapshotConverter::convert_to_rowset_meta(const PPendingDelta& pending_delta, const RowsetId& rowset_id, int64_t tablet_id, int32_t schema_hash, RowsetMetaPB* rowset_meta_pb) { + rowset_meta_pb->set_rowset_id(0); rowset_meta_pb->set_rowset_id_v2(rowset_id.to_string()); rowset_meta_pb->set_tablet_id(tablet_id); rowset_meta_pb->set_tablet_schema_hash(schema_hash); diff --git a/be/test/olap/rowset/unique_rowset_id_generator_test.cpp b/be/test/olap/rowset/unique_rowset_id_generator_test.cpp index 6b85ed4c95820b..5f63958d7ef704 100644 --- a/be/test/olap/rowset/unique_rowset_id_generator_test.cpp +++ b/be/test/olap/rowset/unique_rowset_id_generator_test.cpp @@ -49,14 +49,14 @@ TEST_F(UniqueRowsetIdGeneratorTest, RowsetIdFormatTest) { ASSERT_TRUE(rowset_id.hi == 0); ASSERT_STREQ("123", rowset_id.to_string().c_str()); } - /** + { RowsetId rowset_id; - rowset_id.init("024a07c4e21cec017ea9b7e88bf6768a0000000000000019"); + rowset_id.init("c04f58d989cab2f2efd45faa204491890200000000000003"); ASSERT_TRUE(rowset_id.version == 2); - ASSERT_TRUE(rowset_id.lo == 19); - ASSERT_STREQ("024a07c4e21cec017ea9b7e88bf6768a0000000000000019", rowset_id.to_string().c_str()); - }*/ + ASSERT_TRUE(rowset_id.lo == (3 + max_id)); + ASSERT_STREQ("c04f58d989cab2f2efd45faa204491890200000000000003", rowset_id.to_string().c_str()); + } } From 77da4328d405971145c5c828a86562e0d6027ee7 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Wed, 28 Aug 2019 19:04:56 +0800 Subject: [PATCH 4/7] Fix according to comment --- be/src/agent/agent_server.cpp | 4 ++++ be/src/olap/olap_common.h | 4 ++-- be/src/olap/rowset/beta_rowset.cpp | 2 +- be/src/olap/rowset/beta_rowset.h | 2 +- be/src/olap/rowset/rowset_id_generator.h | 11 ++++++++--- be/src/olap/rowset/unique_rowset_id_generator.cpp | 4 ++-- be/src/olap/rowset/unique_rowset_id_generator.h | 8 ++++---- .../olap/rowset/unique_rowset_id_generator_test.cpp | 10 ++++++++++ 8 files changed, 32 insertions(+), 13 deletions(-) diff --git a/be/src/agent/agent_server.cpp b/be/src/agent/agent_server.cpp index 6fca3f76b7921c..b7cfc91cbcec98 100644 --- a/be/src/agent/agent_server.cpp +++ b/be/src/agent/agent_server.cpp @@ -427,6 +427,10 @@ void AgentServer::make_snapshot(TAgentResult& return_value, vector error_msgs; TStatusCode::type status_code = TStatusCode::OK; int32_t return_snapshot_version = PREFERRED_SNAPSHOT_VERSION; + // if the request's snapshot version is less than current be's snapshot version + // it means the request be is under old version. just set the request version to 1 + // current be will generate snapshot files like tabletid_schemahash_startversion_endversion + // format. Every be is able to parse this format snapshot files. if (snapshot_request.preferred_snapshot_version < PREFERRED_SNAPSHOT_VERSION) { return_snapshot_version = 1; } diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 85fdda602022ac..2733146b3a0101 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -280,7 +280,7 @@ struct RowsetId { void init(int64_t id_version, int64_t high, int64_t middle, int64_t low) { version = id_version; if (low >= MAX_ROWSET_ID) { - LOG(FATAL) << "low is too large" << low; + LOG(FATAL) << "low is too large:" << low; } hi = high; mi = middle; @@ -318,7 +318,7 @@ struct RowsetId { } } - friend std::ostream &operator<<(std::ostream &out, const RowsetId& rowset_id) { + friend std::ostream& operator<<(std::ostream &out, const RowsetId& rowset_id) { out << rowset_id.to_string(); return out; } diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index b2cbfd00d6286c..7d7e8509914efe 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -25,7 +25,7 @@ namespace doris { -std::string BetaRowset::segment_file_path(const std::string& dir, RowsetId rowset_id, int segment_id) { +std::string BetaRowset::segment_file_path(const std::string& dir, const RowsetId& rowset_id, int segment_id) { return strings::Substitute("$0/$1_$2.dat", dir, rowset_id.to_string(), segment_id); } diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h index 98b4c685e5e7d8..08822d18cbcc01 100644 --- a/be/src/olap/rowset/beta_rowset.h +++ b/be/src/olap/rowset/beta_rowset.h @@ -38,7 +38,7 @@ class BetaRowset : public Rowset { virtual ~BetaRowset() {} - static std::string segment_file_path(const std::string& segment_dir, RowsetId rowset_id, int segment_id); + static std::string segment_file_path(const std::string& segment_dir, const RowsetId& rowset_id, int segment_id); OLAPStatus init() override; diff --git a/be/src/olap/rowset/rowset_id_generator.h b/be/src/olap/rowset/rowset_id_generator.h index d205d3cb2f9841..fd3def414f5d32 100644 --- a/be/src/olap/rowset/rowset_id_generator.h +++ b/be/src/olap/rowset/rowset_id_generator.h @@ -36,9 +36,14 @@ class RowsetIdGenerator { // it saves the batch end id into meta env virtual OLAPStatus next_id(RowsetId* rowset_id) = 0; - virtual bool id_in_use(RowsetId& rowset_id) = 0; - - virtual void release_id(RowsetId& rowset_id) = 0; + // check whether the rowset id is userful or validate + // for example, during gc logic, gc thread finds a file + // and it could not find it under rowset list. but it maybe in use + // during load procedure. Gc thread will check it using this method. + virtual bool id_in_use(const RowsetId& rowset_id) = 0; + + // remove the rowsetid from useful rowsetid list. + virtual void release_id(const RowsetId& rowset_id) = 0; }; // RowsetIdGenerator } // namespace doris diff --git a/be/src/olap/rowset/unique_rowset_id_generator.cpp b/be/src/olap/rowset/unique_rowset_id_generator.cpp index 32b4ec97d73e75..d3b1b8ba53d9e2 100644 --- a/be/src/olap/rowset/unique_rowset_id_generator.cpp +++ b/be/src/olap/rowset/unique_rowset_id_generator.cpp @@ -34,7 +34,7 @@ OLAPStatus UniqueRowsetIdGenerator::next_id(RowsetId* rowset_id) { return OLAP_SUCCESS; } -bool UniqueRowsetIdGenerator::id_in_use(RowsetId& rowset_id) { +bool UniqueRowsetIdGenerator::id_in_use(const RowsetId& rowset_id) { // if rowset_id == 1, then it is an old version rowsetid, not gc it // because old version rowset id is not generated by this code, so that not delete them if (rowset_id.version < _version) { @@ -43,7 +43,7 @@ bool UniqueRowsetIdGenerator::id_in_use(RowsetId& rowset_id) { return _valid_rowset_ids.find(rowset_id) != _valid_rowset_ids.end(); } -void UniqueRowsetIdGenerator::release_id(RowsetId& rowset_id) { +void UniqueRowsetIdGenerator::release_id(const RowsetId& rowset_id) { _valid_rowset_ids.erase(rowset_id); } diff --git a/be/src/olap/rowset/unique_rowset_id_generator.h b/be/src/olap/rowset/unique_rowset_id_generator.h index 76595393e06c96..19b362708307d6 100644 --- a/be/src/olap/rowset/unique_rowset_id_generator.h +++ b/be/src/olap/rowset/unique_rowset_id_generator.h @@ -31,16 +31,16 @@ class UniqueRowsetIdGenerator : public RowsetIdGenerator { // generator a id according to data dir // rowsetid is not globally unique, it is dir level // it saves the batch end id into meta env - OLAPStatus next_id(RowsetId* rowset_id); + OLAPStatus next_id(RowsetId* rowset_id) override; - bool id_in_use(RowsetId& rowset_id); + bool id_in_use(const RowsetId& rowset_id) override; - void release_id(RowsetId& rowset_id); + void release_id(const RowsetId& rowset_id) override; private: SpinLock _lock; UniqueId _backend_uid; - int64_t _version = 2; // modify it when create new version id generator + const int64_t _version = 2; // modify it when create new version id generator int64_t _inc_id = 0; std::set _valid_rowset_ids; }; // FeBasedRowsetIdGenerator diff --git a/be/test/olap/rowset/unique_rowset_id_generator_test.cpp b/be/test/olap/rowset/unique_rowset_id_generator_test.cpp index 5f63958d7ef704..1f4e20411faab1 100644 --- a/be/test/olap/rowset/unique_rowset_id_generator_test.cpp +++ b/be/test/olap/rowset/unique_rowset_id_generator_test.cpp @@ -62,7 +62,17 @@ TEST_F(UniqueRowsetIdGeneratorTest, RowsetIdFormatTest) { TEST_F(UniqueRowsetIdGeneratorTest, GenerateIdTest) { UniqueId backend_uid; + UniqueId backend_uid2; + ASSERT_TRUE(backend_uid != backend_uid2); UniqueRowsetIdGenerator id_generator(backend_uid); + UniqueRowsetIdGenerator id_generator2(backend_uid2); + { + RowsetId rowset_id1; + id_generator.next_id(&rowset_id1); + RowsetId rowset_id2; + id_generator2.next_id(&rowset_id2); + ASSERT_TRUE(rowset_id1.hi != rowset_id2.hi); + } { int64_t max_id = 2; max_id = max_id << 56; From 2f00d2afbf588a9f49ecc3d70bd496efc556b402 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 29 Aug 2019 15:08:38 +0800 Subject: [PATCH 5/7] Modify typo error according to review --- be/src/olap/olap_common.h | 13 +++++++------ be/src/olap/rowset/rowset_meta_manager.cpp | 14 +++++++------- be/src/olap/rowset/rowset_meta_manager.h | 14 +++++++------- be/src/olap/rowset/segment_group.cpp | 4 ++-- be/src/olap/rowset/segment_group.h | 6 +++--- be/src/olap/rowset/segment_writer.cpp | 2 +- be/src/olap/rowset/unique_rowset_id_generator.cpp | 7 +++---- be/src/olap/rowset/unique_rowset_id_generator.h | 2 +- be/src/olap/storage_engine.cpp | 1 + be/src/olap/storage_engine.h | 4 ++-- 10 files changed, 34 insertions(+), 33 deletions(-) diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 2733146b3a0101..44ac329918c0cc 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -246,9 +246,9 @@ typedef std::set UniqueIdSet; // Column unique Id -> column id map typedef std::map UniqueIdToColumnIdMap; -// 8 bit, id version -// 120 bit, backend random number generated by uuid -// 64 bit, inc number from 0 +// 128 bit backend uid, it is a uuid bit, id version +// 8 bit rowset id version +// 56 bit, inc number from 0 struct RowsetId { int8_t version = 0; int64_t hi = 0; @@ -301,11 +301,11 @@ struct RowsetId { // std::unordered_map need this api bool operator==(const RowsetId& rhs) const { - return hi == rhs.hi && mi == rhs.mi && lo == rhs.lo; + return lo == rhs.lo && hi == rhs.hi && mi == rhs.mi ; } bool operator!=(const RowsetId& rhs) const { - return hi != rhs.hi || mi != rhs.mi || lo != rhs.lo; + return lo != rhs.lo || hi != rhs.hi || mi != rhs.mi ; } bool operator<(const RowsetId& rhs) const { @@ -318,10 +318,11 @@ struct RowsetId { } } - friend std::ostream& operator<<(std::ostream &out, const RowsetId& rowset_id) { + friend std::ostream& operator<<(std::ostream& out, const RowsetId& rowset_id) { out << rowset_id.to_string(); return out; } + }; } // namespace doris diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp b/be/src/olap/rowset/rowset_meta_manager.cpp index f865ca0fa9817b..1212681ea46153 100644 --- a/be/src/olap/rowset/rowset_meta_manager.cpp +++ b/be/src/olap/rowset/rowset_meta_manager.cpp @@ -34,7 +34,7 @@ namespace doris { const std::string ROWSET_PREFIX = "rst_"; -bool RowsetMetaManager::check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id) { +bool RowsetMetaManager::check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id) { std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string(); std::string value; OLAPStatus s = meta->get(META_COLUMN_FAMILY_INDEX, key, &value); @@ -44,7 +44,7 @@ bool RowsetMetaManager::check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, return true; } -OLAPStatus RowsetMetaManager::get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, RowsetMetaSharedPtr rowset_meta) { +OLAPStatus RowsetMetaManager::get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, RowsetMetaSharedPtr rowset_meta) { std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string(); std::string value; OLAPStatus s = meta->get(META_COLUMN_FAMILY_INDEX, key, &value); @@ -65,7 +65,7 @@ OLAPStatus RowsetMetaManager::get_rowset_meta(OlapMeta* meta, TabletUid tablet_u return OLAP_SUCCESS; } -OLAPStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, std::string* json_rowset_meta) { +OLAPStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, std::string* json_rowset_meta) { RowsetMetaSharedPtr rowset_meta_ptr(new(std::nothrow) RowsetMeta()); OLAPStatus status = get_rowset_meta(meta, tablet_uid, rowset_id, rowset_meta_ptr); if (status != OLAP_SUCCESS) { @@ -79,7 +79,7 @@ OLAPStatus RowsetMetaManager::get_json_rowset_meta(OlapMeta* meta, TabletUid tab return OLAP_SUCCESS; } -OLAPStatus RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, RowsetMeta* rowset_meta) { +OLAPStatus RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, RowsetMeta* rowset_meta) { std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string(); std::string value; bool ret = rowset_meta->serialize(&value); @@ -95,7 +95,7 @@ OLAPStatus RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, RowsetI return status; } -OLAPStatus RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, const string& meta_binary) { +OLAPStatus RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, const string& meta_binary) { std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string(); OLAPStatus status = meta->put(META_COLUMN_FAMILY_INDEX, key, meta_binary); if (status == OLAP_SUCCESS) { @@ -104,7 +104,7 @@ OLAPStatus RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, RowsetI return status; } -OLAPStatus RowsetMetaManager::remove(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id) { +OLAPStatus RowsetMetaManager::remove(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id) { std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string(); LOG(INFO) << "start to remove rowset, key:" << key; OLAPStatus status = meta->remove(META_COLUMN_FAMILY_INDEX, key); @@ -113,7 +113,7 @@ OLAPStatus RowsetMetaManager::remove(OlapMeta* meta, TabletUid tablet_uid, Rowse } OLAPStatus RowsetMetaManager::traverse_rowset_metas(OlapMeta* meta, - std::function const& func) { + std::function const& func) { auto traverse_rowset_meta_func = [&func](const std::string& key, const std::string& value) -> bool { std::vector parts; // key format: rst_uuid_rowset_id diff --git a/be/src/olap/rowset/rowset_meta_manager.h b/be/src/olap/rowset/rowset_meta_manager.h index d04a911c1126c9..9a66e0a9187f64 100644 --- a/be/src/olap/rowset/rowset_meta_manager.h +++ b/be/src/olap/rowset/rowset_meta_manager.h @@ -31,20 +31,20 @@ namespace doris { // Helper class for managing rowset meta of one root path. class RowsetMetaManager { public: - static bool check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id); + static bool check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id); - static OLAPStatus get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, RowsetMetaSharedPtr rowset_meta); + static OLAPStatus get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, RowsetMetaSharedPtr rowset_meta); - static OLAPStatus get_json_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, std::string* json_rowset_meta); + static OLAPStatus get_json_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, std::string* json_rowset_meta); - static OLAPStatus save(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, RowsetMeta* rowset_meta); + static OLAPStatus save(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, RowsetMeta* rowset_meta); - static OLAPStatus save(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id, const string& meta_binary); + static OLAPStatus save(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, const string& meta_binary); - static OLAPStatus remove(OlapMeta* meta, TabletUid tablet_uid, RowsetId rowset_id); + static OLAPStatus remove(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id); static OLAPStatus traverse_rowset_metas(OlapMeta* meta, - std::function const& func); + std::function const& func); static OLAPStatus load_json_rowset_meta(OlapMeta* meta, const std::string& rowset_meta_path); }; diff --git a/be/src/olap/rowset/segment_group.cpp b/be/src/olap/rowset/segment_group.cpp index 201a8122bc45bb..de3a5c940350da 100644 --- a/be/src/olap/rowset/segment_group.cpp +++ b/be/src/olap/rowset/segment_group.cpp @@ -65,7 +65,7 @@ namespace doris { } \ } while (0); -SegmentGroup::SegmentGroup(int64_t tablet_id, RowsetId rowset_id, const TabletSchema* schema, +SegmentGroup::SegmentGroup(int64_t tablet_id, const RowsetId& rowset_id, const TabletSchema* schema, const std::string& rowset_path_prefix, Version version, VersionHash version_hash, bool delete_flag, int32_t segment_group_id, int32_t num_segments) : _tablet_id(tablet_id), @@ -101,7 +101,7 @@ SegmentGroup::SegmentGroup(int64_t tablet_id, RowsetId rowset_id, const TabletSc } } -SegmentGroup::SegmentGroup(int64_t tablet_id, RowsetId rowset_id, const TabletSchema* schema, +SegmentGroup::SegmentGroup(int64_t tablet_id, const RowsetId& rowset_id, const TabletSchema* schema, const std::string& rowset_path_prefix, bool delete_flag, int32_t segment_group_id, int32_t num_segments, bool is_pending, TPartitionId partition_id, TTransactionId transaction_id) : _tablet_id(tablet_id), diff --git a/be/src/olap/rowset/segment_group.h b/be/src/olap/rowset/segment_group.h index 9d8d99cea98efb..ecc48d7de829e3 100644 --- a/be/src/olap/rowset/segment_group.h +++ b/be/src/olap/rowset/segment_group.h @@ -48,11 +48,11 @@ namespace doris { class SegmentGroup { friend class MemIndex; public: - SegmentGroup(int64_t tablet_id, RowsetId rowset_id, const TabletSchema* tablet_schema, + SegmentGroup(int64_t tablet_id, const RowsetId& rowset_id, const TabletSchema* tablet_schema, const std::string& rowset_path_prefix, Version version, VersionHash version_hash, bool delete_flag, int segment_group_id, int32_t num_segments); - SegmentGroup(int64_t tablet_id, RowsetId rowset_id, const TabletSchema* tablet_schema, + SegmentGroup(int64_t tablet_id, const RowsetId& rowset_id, const TabletSchema* tablet_schema, const std::string& rowset_path_prefix, bool delete_flag, int32_t segment_group_id, int32_t num_segments, bool is_pending, TPartitionId partition_id, TTransactionId transaction_id); @@ -248,7 +248,7 @@ class SegmentGroup { int64_t get_tablet_id(); - RowsetId rowset_id() { + const RowsetId& rowset_id() { return _rowset_id; } diff --git a/be/src/olap/rowset/segment_writer.cpp b/be/src/olap/rowset/segment_writer.cpp index 86677225dc72d4..c7ddef4825ee16 100644 --- a/be/src/olap/rowset/segment_writer.cpp +++ b/be/src/olap/rowset/segment_writer.cpp @@ -231,7 +231,7 @@ OLAPStatus SegmentWriter::finalize(uint32_t* segment_file_size) { return OLAP_ERR_DISK_REACH_CAPACITY_LIMIT; } - data_dir->add_pending_ids(ROWSET_ID_PREFIX + std::to_string(_segment_group->rowset_id())); + data_dir->add_pending_ids(ROWSET_ID_PREFIX + _segment_group->rowset_id().to_string()); if (OLAP_SUCCESS != (res = file_handle.open_with_mode( _file_name, O_CREAT | O_EXCL | O_WRONLY , S_IRUSR | S_IWUSR))) { LOG(WARNING) << "fail to open file. [file_name=" << _file_name << "]"; diff --git a/be/src/olap/rowset/unique_rowset_id_generator.cpp b/be/src/olap/rowset/unique_rowset_id_generator.cpp index d3b1b8ba53d9e2..04c3d30f17dff1 100644 --- a/be/src/olap/rowset/unique_rowset_id_generator.cpp +++ b/be/src/olap/rowset/unique_rowset_id_generator.cpp @@ -21,12 +21,11 @@ namespace doris { -UniqueRowsetIdGenerator::UniqueRowsetIdGenerator(UniqueId backend_uid) : +UniqueRowsetIdGenerator::UniqueRowsetIdGenerator(const UniqueId& backend_uid) : _backend_uid(backend_uid), _inc_id(1) { } -// generator a id according to data dir -// rowsetid is not globally unique, it is dir level -// it saves the batch end id into meta env + +// generate a unique rowset id and save it in a set to check whether it is valid in the future OLAPStatus UniqueRowsetIdGenerator::next_id(RowsetId* rowset_id) { std::lock_guard l(_lock); rowset_id->init(_version, _backend_uid.hi, _backend_uid.lo, ++_inc_id); diff --git a/be/src/olap/rowset/unique_rowset_id_generator.h b/be/src/olap/rowset/unique_rowset_id_generator.h index 19b362708307d6..bdc0c9b8b929ed 100644 --- a/be/src/olap/rowset/unique_rowset_id_generator.h +++ b/be/src/olap/rowset/unique_rowset_id_generator.h @@ -25,7 +25,7 @@ namespace doris { class UniqueRowsetIdGenerator : public RowsetIdGenerator { public: - UniqueRowsetIdGenerator(UniqueId backend_uid); + UniqueRowsetIdGenerator(const UniqueId& backend_uid); ~UniqueRowsetIdGenerator() {} // generator a id according to data dir diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index 16bde7642611c2..afec65e1a45c10 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -88,6 +88,7 @@ static Status _validate_options(const EngineOptions& options) { Status StorageEngine::open(const EngineOptions& options, StorageEngine** engine_ptr) { RETURN_IF_ERROR(_validate_options(options)); + LOG(INFO) << "starting backend using uid:" << options.backend_uid.to_string(); std::unique_ptr engine(new StorageEngine(options)); auto st = engine->open(); if (st != OLAP_SUCCESS) { diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 71e75d460344b7..fd325767c6a9a7 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -201,7 +201,7 @@ class StorageEngine { bool rowset_id_in_use(RowsetId& rowset_id) { return _rowset_id_generator->id_in_use(rowset_id); }; - void release_rowset_id(RowsetId& rowset_id) { return _rowset_id_generator->release_id(rowset_id); }; + void release_rowset_id(const RowsetId& rowset_id) { return _rowset_id_generator->release_id(rowset_id); }; private: OLAPStatus check_all_root_path_cluster_id(); @@ -340,7 +340,7 @@ class StorageEngine { std::unique_ptr _tablet_manager; std::unique_ptr _txn_manager; - RowsetIdGenerator* _rowset_id_generator; + std::unique_ptr _rowset_id_generator; DISALLOW_COPY_AND_ASSIGN(StorageEngine); }; From 00a9c4fe81121e1f5ba159bead0c094d11861cb8 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 29 Aug 2019 19:24:20 +0800 Subject: [PATCH 6/7] Add lock when check or release rowset id --- be/src/olap/rowset/unique_rowset_id_generator.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/be/src/olap/rowset/unique_rowset_id_generator.cpp b/be/src/olap/rowset/unique_rowset_id_generator.cpp index 04c3d30f17dff1..ab3a24a49cb7c8 100644 --- a/be/src/olap/rowset/unique_rowset_id_generator.cpp +++ b/be/src/olap/rowset/unique_rowset_id_generator.cpp @@ -34,6 +34,7 @@ OLAPStatus UniqueRowsetIdGenerator::next_id(RowsetId* rowset_id) { } bool UniqueRowsetIdGenerator::id_in_use(const RowsetId& rowset_id) { + std::lock_guard l(_lock); // if rowset_id == 1, then it is an old version rowsetid, not gc it // because old version rowset id is not generated by this code, so that not delete them if (rowset_id.version < _version) { @@ -43,6 +44,7 @@ bool UniqueRowsetIdGenerator::id_in_use(const RowsetId& rowset_id) { } void UniqueRowsetIdGenerator::release_id(const RowsetId& rowset_id) { + std::lock_guard l(_lock); _valid_rowset_ids.erase(rowset_id); } From f12dab0087177a940506619eb5934808f5d6d8a7 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Mon, 2 Sep 2019 15:27:33 +0800 Subject: [PATCH 7/7] Using const & as parameter for rowsetid --- be/src/olap/rowset/segment_group.cpp | 4 ++-- be/src/olap/rowset/segment_group.h | 4 ++-- be/src/olap/snapshot_manager.cpp | 2 +- be/src/olap/snapshot_manager.h | 2 +- be/src/olap/storage_engine.cpp | 2 +- be/src/olap/storage_engine.h | 4 ++-- be/src/olap/tablet.cpp | 2 +- be/src/olap/tablet.h | 2 +- fe/src/main/java/org/apache/doris/task/SnapshotTask.java | 4 +++- 9 files changed, 14 insertions(+), 12 deletions(-) diff --git a/be/src/olap/rowset/segment_group.cpp b/be/src/olap/rowset/segment_group.cpp index de3a5c940350da..1a7f44a1eb0904 100644 --- a/be/src/olap/rowset/segment_group.cpp +++ b/be/src/olap/rowset/segment_group.cpp @@ -160,7 +160,7 @@ std::string SegmentGroup::_construct_file_name(int32_t segment_id, const string& return file_name; } -std::string SegmentGroup::_construct_file_name(RowsetId rowset_id, int32_t segment_id, const string& suffix) const { +std::string SegmentGroup::_construct_file_name(const RowsetId& rowset_id, int32_t segment_id, const string& suffix) const { std::string file_name = rowset_id.to_string() + "_" + std::to_string(_segment_group_id) + "_" + std::to_string(segment_id) + suffix; return file_name; @@ -855,7 +855,7 @@ OLAPStatus SegmentGroup::remove_old_files(std::vector* links_to_rem return OLAP_SUCCESS; } -OLAPStatus SegmentGroup::link_segments_to_path(const std::string& dest_path, RowsetId rowset_id) { +OLAPStatus SegmentGroup::link_segments_to_path(const std::string& dest_path, const RowsetId& rowset_id) { if (dest_path.empty()) { LOG(WARNING) << "dest path is empty, return error"; return OLAP_ERR_INPUT_PARAMETER_ERROR; diff --git a/be/src/olap/rowset/segment_group.h b/be/src/olap/rowset/segment_group.h index ecc48d7de829e3..d9b4bb70b7d737 100644 --- a/be/src/olap/rowset/segment_group.h +++ b/be/src/olap/rowset/segment_group.h @@ -262,12 +262,12 @@ class SegmentGroup { OLAPStatus copy_files_to(const std::string& dir); - OLAPStatus link_segments_to_path(const std::string& dest_path, RowsetId rowset_id); + OLAPStatus link_segments_to_path(const std::string& dest_path, const RowsetId& rowset_id); private: std::string _construct_file_name(int32_t segment_id, const std::string& suffix) const; - std::string _construct_file_name(RowsetId rowset_id, int32_t segment_id, const std::string& suffix) const; + std::string _construct_file_name(const RowsetId& rowset_id, int32_t segment_id, const std::string& suffix) const; std::string _construct_old_pending_file_path(const std::string& path_prefix, int32_t segment_id, const std::string& suffix) const; diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index e201132d52acf4..614e3b26d27aa2 100755 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -195,7 +195,7 @@ OLAPStatus SnapshotManager::convert_rowset_ids(DataDir& data_dir, const string& } OLAPStatus SnapshotManager::_rename_rowset_id(const RowsetMetaPB& rs_meta_pb, const string& new_path, - DataDir& data_dir, TabletSchema& tablet_schema, RowsetId& rowset_id, RowsetMetaPB* new_rs_meta_pb) { + DataDir& data_dir, TabletSchema& tablet_schema, const RowsetId& rowset_id, RowsetMetaPB* new_rs_meta_pb) { OLAPStatus res = OLAP_SUCCESS; RowsetMetaSharedPtr alpha_rowset_meta(new AlphaRowsetMeta()); alpha_rowset_meta->init_from_pb(rs_meta_pb); diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h index c2b3214a1c63d8..ec3c3f480257e8 100644 --- a/be/src/olap/snapshot_manager.h +++ b/be/src/olap/snapshot_manager.h @@ -99,7 +99,7 @@ class SnapshotManager { DataDir* store); OLAPStatus _rename_rowset_id(const RowsetMetaPB& rs_meta_pb, const string& new_path, - DataDir& data_dir, TabletSchema& tablet_schema, RowsetId& next_id, RowsetMetaPB* new_rs_meta_pb); + DataDir& data_dir, TabletSchema& tablet_schema, const RowsetId& next_id, RowsetMetaPB* new_rs_meta_pb); private: static SnapshotManager* _s_instance; diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index afec65e1a45c10..86a08990c7ed7a 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -891,7 +891,7 @@ OLAPStatus StorageEngine::execute_task(EngineTask* task) { } // check whether any unused rowsets's id equal to rowset_id -bool StorageEngine::check_rowset_id_in_unused_rowsets(RowsetId rowset_id) { +bool StorageEngine::check_rowset_id_in_unused_rowsets(const RowsetId& rowset_id) { _gc_mutex.lock(); for (auto& _unused_rowset_pair : _unused_rowsets) { if (_unused_rowset_pair.second->rowset_id() == rowset_id) { diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index fd325767c6a9a7..4e9436b80d6d78 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -192,14 +192,14 @@ class StorageEngine { TabletManager* tablet_manager() { return _tablet_manager.get(); } TxnManager* txn_manager() { return _txn_manager.get(); } - bool check_rowset_id_in_unused_rowsets(RowsetId rowset_id); + bool check_rowset_id_in_unused_rowsets(const RowsetId& rowset_id); // TODO(ygl) TabletSyncService* tablet_sync_service() { return nullptr; } OLAPStatus next_rowset_id(RowsetId* rowset_id) { return _rowset_id_generator->next_id(rowset_id); }; - bool rowset_id_in_use(RowsetId& rowset_id) { return _rowset_id_generator->id_in_use(rowset_id); }; + bool rowset_id_in_use(const RowsetId& rowset_id) { return _rowset_id_generator->id_in_use(rowset_id); }; void release_rowset_id(const RowsetId& rowset_id) { return _rowset_id_generator->release_id(rowset_id); }; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index f7e06841200750..671449d67ab9d4 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -871,7 +871,7 @@ bool Tablet::check_path(const std::string& path_to_check) { return false; } -bool Tablet::check_rowset_id(RowsetId rowset_id) { +bool Tablet::check_rowset_id(const RowsetId& rowset_id) { ReadLock rdlock(&_meta_lock); for (auto& version_rowset : _rs_version_map) { if (version_rowset.second->rowset_id() == rowset_id) { diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index aeb7f01174a7b9..4a85d2d49819ac 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -216,7 +216,7 @@ class Tablet : public std::enable_shared_from_this { void delete_all_files(); bool check_path(const std::string& check_path); - bool check_rowset_id(RowsetId rowset_id); + bool check_rowset_id(const RowsetId& rowset_id); OLAPStatus set_partition_id(int64_t partition_id); diff --git a/fe/src/main/java/org/apache/doris/task/SnapshotTask.java b/fe/src/main/java/org/apache/doris/task/SnapshotTask.java index 332d66709d91b9..a1c2e180298a05 100644 --- a/fe/src/main/java/org/apache/doris/task/SnapshotTask.java +++ b/fe/src/main/java/org/apache/doris/task/SnapshotTask.java @@ -22,6 +22,8 @@ import org.apache.doris.thrift.TTaskType; public class SnapshotTask extends AgentTask { + private static final int PREFERRED_SNAPSHOT_VERSION = 3; + private long jobId; private long version; @@ -79,7 +81,7 @@ public TSnapshotRequest toThrift() { request.setVersion(version); request.setVersion_hash(versionHash); request.setList_files(true); - request.setPreferred_snapshot_version(3); + request.setPreferred_snapshot_version(PREFERRED_SNAPSHOT_VERSION); request.setTimeout(timeoutMs / 1000); return request; }