[Refactor] Remove version hash from BE and related test in BE

be/src/agent/task_worker_pool.cpp

@@ -383,7 +383,8 @@ void TaskWorkerPool::_create_tablet_worker_thread_callback() {
             tablet_info.tablet_id = tablet->table_id();
             tablet_info.schema_hash = tablet->schema_hash();
             tablet_info.version = create_tablet_req.version;
-            tablet_info.version_hash = create_tablet_req.version_hash;
+            // Useless but it is a required field in TTabletInfo
+            tablet_info.version_hash = 0;
             tablet_info.row_count = 0;
             tablet_info.data_size = 0;
             tablet_info.__set_path_hash(tablet->data_dir()->path_hash());
@@ -662,7 +663,6 @@ void TaskWorkerPool::_push_worker_thread_callback() {
         finish_task_request.__set_signature(agent_task_req.signature);
         if (push_req.push_type == TPushType::DELETE) {
             finish_task_request.__set_request_version(push_req.version);
-            finish_task_request.__set_request_version_hash(push_req.version_hash);
         }
 
         if (status == DORIS_SUCCESS) {
@@ -1074,7 +1074,7 @@ void TaskWorkerPool::_check_consistency_worker_thread_callback() {
         uint32_t checksum = 0;
         EngineChecksumTask engine_task(
                 check_consistency_req.tablet_id, check_consistency_req.schema_hash,
-                check_consistency_req.version, check_consistency_req.version_hash, &checksum);
+                check_consistency_req.version, &checksum);
         OLAPStatus res = _env->storage_engine()->execute_task(&engine_task);
         if (res != OLAP_SUCCESS) {
             LOG(WARNING) << "check consistency failed. status: " << res
@@ -1095,7 +1095,6 @@ void TaskWorkerPool::_check_consistency_worker_thread_callback() {
         finish_task_request.__set_task_status(task_status);
         finish_task_request.__set_tablet_checksum(static_cast<int64_t>(checksum));
         finish_task_request.__set_request_version(check_consistency_req.version);
-        finish_task_request.__set_request_version_hash(check_consistency_req.version_hash);
 
         _finish_task(finish_task_request);
         _remove_task_info(agent_task_req.task_type, agent_task_req.signature);
@@ -1404,15 +1403,13 @@ void TaskWorkerPool::_make_snapshot_thread_callback() {
             LOG(WARNING) << "make_snapshot failed. tablet_id:" << snapshot_request.tablet_id
                          << ", schema_hash:" << snapshot_request.schema_hash
                          << ", version:" << snapshot_request.version
-                         << ", version_hash:" << snapshot_request.version_hash
                          << ", status: " << make_snapshot_status;
             error_msgs.push_back("make_snapshot failed. status: " +
                                  boost::lexical_cast<string>(make_snapshot_status));
         } else {
             LOG(INFO) << "make_snapshot success. tablet_id:" << snapshot_request.tablet_id
                       << ", schema_hash:" << snapshot_request.schema_hash
                       << ", version:" << snapshot_request.version
-                      << ", version_hash:" << snapshot_request.version_hash
                       << ", snapshot_path:" << snapshot_path;
             if (snapshot_request.__isset.list_files) {
                 // list and save all snapshot files
@@ -1427,7 +1424,6 @@ void TaskWorkerPool::_make_snapshot_thread_callback() {
                     LOG(WARNING) << "make_snapshot failed. tablet_id:" << snapshot_request.tablet_id
                                  << ", schema_hash:" << snapshot_request.schema_hash
                                  << ", version:" << snapshot_request.version
-                                 << ", version_hash:" << snapshot_request.version_hash
                                  << ",list file failed: " << st.get_error_msg();
                     error_msgs.push_back("make_snapshot failed. list file failed: " +
                                          st.get_error_msg());

be/src/exec/olap_scan_node.cpp

@@ -425,7 +425,6 @@ Status OlapScanNode::close(RuntimeState* state) {
 //  1: required list<Types.TNetworkAddress> hosts
 //  2: required string schema_hash
 //  3: required string version
-//  4: required string version_hash
 //  5: required Types.TTabletId tablet_id
 //  6: required string db_name
 //  7: optional list<TKeyRange> partition_column_ranges

be/src/http/action/checksum_action.cpp

@@ -39,7 +39,6 @@ const std::string TABLET_ID = "tablet_id";
 // do not use name "VERSION",
 // or will be conflict with "VERSION" in thrift/config.h
 const std::string TABLET_VERSION = "version";
-const std::string VERSION_HASH = "version_hash";
 const std::string SCHEMA_HASH = "schema_hash";
 
 ChecksumAction::ChecksumAction(ExecEnv* exec_env) : _exec_env(exec_env) {}
@@ -67,14 +66,6 @@ void ChecksumAction::handle(HttpRequest* req) {
         return;
     }
 
-    // Get version hash
-    const std::string& version_hash_str = req->param(VERSION_HASH);
-    if (version_hash_str.empty()) {
-        std::string error_msg = std::string("parameter " + VERSION_HASH + " not specified in url.");
-        HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, error_msg);
-        return;
-    }
-
     // Get schema hash
     const std::string& schema_hash_str = req->param(SCHEMA_HASH);
     if (schema_hash_str.empty()) {
@@ -86,23 +77,20 @@ void ChecksumAction::handle(HttpRequest* req) {
     // valid str format
     int64_t tablet_id;
     int64_t version;
-    int64_t version_hash;
     int32_t schema_hash;
     try {
         tablet_id = boost::lexical_cast<int64_t>(tablet_id_str);
         version = boost::lexical_cast<int64_t>(version_str);
-        version_hash = boost::lexical_cast<int64_t>(version_hash_str);
         schema_hash = boost::lexical_cast<int64_t>(schema_hash_str);
     } catch (boost::bad_lexical_cast& e) {
         std::string error_msg = std::string("param format is invalid: ") + std::string(e.what());
         HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, error_msg);
         return;
     }
 
-    VLOG_ROW << "get checksum tablet info: " << tablet_id << "-" << version << "-" << version_hash
-             << "-" << schema_hash;
+    VLOG_ROW << "get checksum tablet info: " << tablet_id << "-" << version << "-" << schema_hash;
 
-    int64_t checksum = do_checksum(tablet_id, version, version_hash, schema_hash, req);
+    int64_t checksum = do_checksum(tablet_id, version, schema_hash, req);
     if (checksum == -1L) {
         std::string error_msg = std::string("checksum failed");
         HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, error_msg);
@@ -117,11 +105,11 @@ void ChecksumAction::handle(HttpRequest* req) {
     LOG(INFO) << "deal with checksum request finished! tablet id: " << tablet_id;
 }
 
-int64_t ChecksumAction::do_checksum(int64_t tablet_id, int64_t version, int64_t version_hash,
+int64_t ChecksumAction::do_checksum(int64_t tablet_id, int64_t version,
                                     int32_t schema_hash, HttpRequest* req) {
     OLAPStatus res = OLAP_SUCCESS;
     uint32_t checksum;
-    EngineChecksumTask engine_task(tablet_id, schema_hash, version, version_hash, &checksum);
+    EngineChecksumTask engine_task(tablet_id, schema_hash, version, &checksum);
     res = engine_task.execute();
     if (res != OLAP_SUCCESS) {
         LOG(WARNING) << "checksum failed. status: " << res << ", signature: " << tablet_id;

be/src/http/action/checksum_action.h

@@ -35,7 +35,7 @@ class ChecksumAction : public HttpHandler {
     void handle(HttpRequest* req) override;
 
 private:
-    int64_t do_checksum(int64_t tablet_id, int64_t version, int64_t version_hash,
+    int64_t do_checksum(int64_t tablet_id, int64_t version,
                         int32_t schema_hash, HttpRequest* req);
 
     ExecEnv* _exec_env;

be/src/olap/compaction.cpp

@@ -80,7 +80,6 @@ OLAPStatus Compaction::do_compaction_impl(int64_t permits) {
 
     _output_version =
             Version(_input_rowsets.front()->start_version(), _input_rowsets.back()->end_version());
-    _tablet->compute_version_hash_from_rowsets(_input_rowsets, &_output_version_hash);
 
     LOG(INFO) << "start " << compaction_name() << ". tablet=" << _tablet->full_name()
               << ", output_version=" << _output_version << ", permits: " << permits;
@@ -163,7 +162,6 @@ OLAPStatus Compaction::construct_output_rowset_writer() {
     context.tablet_schema = &(_tablet->tablet_schema());
     context.rowset_state = VISIBLE;
     context.version = _output_version;
-    context.version_hash = _output_version_hash;
     context.segments_overlap = NONOVERLAPPING;
     context.parent_mem_tracker = _writer_tracker;
     // The test results show that one rs writer is low-memory-footprint, there is no need to tracker its mem pool

be/src/olap/compaction.h

@@ -103,7 +103,6 @@ class Compaction {
     CompactionState _state;
 
     Version _output_version;
-    VersionHash _output_version_hash;
 
     DISALLOW_COPY_AND_ASSIGN(Compaction);
 };

be/src/olap/olap_common.h

@@ -44,7 +44,6 @@ namespace doris {
 static const int64_t MAX_ROWSET_ID = 1L << 56;
 
 typedef int32_t SchemaHash;
-typedef int64_t VersionHash;
 typedef __int128 int128_t;
 typedef unsigned __int128 uint128_t;
 

be/src/olap/olap_define.h

@@ -240,7 +240,6 @@ enum OLAPStatus {
     OLAP_ERR_BE_VERSION_NOT_MATCH = -800,
     OLAP_ERR_BE_REPLACE_VERSIONS_ERROR = -801,
     OLAP_ERR_BE_MERGE_ERROR = -802,
-    OLAP_ERR_BE_COMPUTE_VERSION_HASH_ERROR = -803,
     OLAP_ERR_CAPTURE_ROWSET_ERROR = -804,
     OLAP_ERR_BE_SAVE_HEADER_ERROR = -805,
     OLAP_ERR_BE_INIT_OLAP_DATA = -806,

be/src/olap/rowset/alpha_rowset.cpp

@@ -35,15 +35,14 @@ OLAPStatus AlphaRowset::do_load(bool use_cache) {
         // validate segment group
         if (segment_group->validate() != OLAP_SUCCESS) {
             LOG(WARNING) << "fail to validate segment_group. [version=" << start_version() << "-"
-                         << end_version() << " version_hash=" << version_hash();
+                         << end_version();
             // if load segment group failed, rowset init failed
             return OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR;
         }
         OLAPStatus res = segment_group->load(use_cache);
         if (res != OLAP_SUCCESS) {
             LOG(WARNING) << "fail to load segment_group. res=" << res << ", "
-                         << "version=" << start_version() << "-" << end_version() << ", "
-                         << "version_hash=" << version_hash();
+                         << "version=" << start_version() << "-" << end_version();
             return res;
         }
     }
@@ -79,15 +78,14 @@ OLAPStatus AlphaRowset::remove() {
     return OLAP_SUCCESS;
 }
 
-void AlphaRowset::make_visible_extra(Version version, VersionHash version_hash) {
+void AlphaRowset::make_visible_extra(Version version) {
     AlphaRowsetMetaSharedPtr alpha_rowset_meta =
             std::dynamic_pointer_cast<AlphaRowsetMeta>(_rowset_meta);
     std::vector<SegmentGroupPB> published_segment_groups;
     alpha_rowset_meta->get_segment_groups(&published_segment_groups);
     int32_t segment_group_idx = 0;
     for (auto& segment_group : _segment_groups) {
         segment_group->set_version(version);
-        segment_group->set_version_hash(version_hash);
         segment_group->set_pending_finished();
         published_segment_groups.at(segment_group_idx).clear_load_id();
         ++segment_group_idx;
@@ -322,7 +320,7 @@ OLAPStatus AlphaRowset::init() {
         } else {
             segment_group.reset(new SegmentGroup(
                     _rowset_meta->tablet_id(), _rowset_meta->rowset_id(), _schema, _rowset_path_desc.filepath,
-                    _rowset_meta->version(), _rowset_meta->version_hash(), false,
+                    _rowset_meta->version(), false,
                     segment_group_meta.segment_group_id(), segment_group_meta.num_segments()));
         }
         if (segment_group == nullptr) {

be/src/olap/rowset/alpha_rowset.h

@@ -80,7 +80,7 @@ class AlphaRowset : public Rowset {
     void do_close() override {}
 
     // add custom logic when rowset is published
-    void make_visible_extra(Version version, VersionHash version_hash) override;
+    void make_visible_extra(Version version) override;
 
 private:
     std::shared_ptr<SegmentGroup> _segment_group_with_largest_size();

be/src/olap/rowset/alpha_rowset_reader.cpp

@@ -120,10 +120,6 @@ Version AlphaRowsetReader::version() {
     return _alpha_rowset_meta->version();
 }
 
-VersionHash AlphaRowsetReader::version_hash() {
-    return _alpha_rowset_meta->version_hash();
-}
-
 int64_t AlphaRowsetReader::filtered_rows() {
     return _stats->rows_del_filtered;
 }

be/src/olap/rowset/alpha_rowset_reader.h

@@ -73,8 +73,6 @@ class AlphaRowsetReader : public RowsetReader {
 
     Version version() override;
 
-    VersionHash version_hash() override;
-
     RowsetSharedPtr rowset() override;
 
     int64_t filtered_rows() override;

be/src/olap/rowset/alpha_rowset_writer.cpp

@@ -68,7 +68,6 @@ OLAPStatus AlphaRowsetWriter::init(const RowsetWriterContext& rowset_writer_cont
         _current_rowset_meta->set_load_id(_rowset_writer_context.load_id);
     } else {
         _current_rowset_meta->set_version(_rowset_writer_context.version);
-        _current_rowset_meta->set_version_hash(_rowset_writer_context.version_hash);
     }
     RETURN_NOT_OK(_init());
     return OLAP_SUCCESS;
@@ -259,7 +258,7 @@ OLAPStatus AlphaRowsetWriter::_init() {
         _cur_segment_group = new (std::nothrow) SegmentGroup(
                 _rowset_writer_context.tablet_id, _rowset_writer_context.rowset_id,
                 _rowset_writer_context.tablet_schema, _rowset_writer_context.path_desc.filepath,
-                _rowset_writer_context.version, _rowset_writer_context.version_hash, false,
+                _rowset_writer_context.version, false,
                 _segment_group_id, 0);
     }
     DCHECK(_cur_segment_group != nullptr) << "failed to malloc SegmentGroup";

be/src/olap/rowset/beta_rowset_reader.h

@@ -46,8 +46,6 @@ class BetaRowsetReader : public RowsetReader {
 
     Version version() override { return _rowset->version(); }
 
-    VersionHash version_hash() override { return _rowset->version_hash(); }
-
     RowsetSharedPtr rowset() override { return std::dynamic_pointer_cast<Rowset>(_rowset); }
 
     // Return the total number of filtered rows, will be used for validation of schema change

be/src/olap/rowset/beta_rowset_writer.cpp

@@ -82,7 +82,6 @@ OLAPStatus BetaRowsetWriter::init(const RowsetWriterContext& rowset_writer_conte
         _rowset_meta->set_load_id(_context.load_id);
     } else {
         _rowset_meta->set_version(_context.version);
-        _rowset_meta->set_version_hash(_context.version_hash);
     }
     _rowset_meta->set_tablet_uid(_context.tablet_uid);
 

be/src/olap/rowset/column_data.h

@@ -47,7 +47,6 @@ class ColumnData {
 
     // 为了与之前兼容, 暴露部分index的接口
     Version version() const { return _segment_group->version(); }
-    VersionHash version_hash() const { return _segment_group->version_hash(); }
     bool delete_flag() const { return _segment_group->delete_flag(); }
     uint32_t num_segments() const { return _segment_group->num_segments(); }
 

be/src/olap/rowset/rowset.cpp

@@ -59,10 +59,9 @@ OLAPStatus Rowset::load(bool use_cache) {
     return OLAP_SUCCESS;
 }
 
-void Rowset::make_visible(Version version, VersionHash version_hash) {
+void Rowset::make_visible(Version version) {
     _is_pending = false;
     _rowset_meta->set_version(version);
-    _rowset_meta->set_version_hash(version_hash);
     _rowset_meta->set_rowset_state(VISIBLE);
     // update create time to the visible time,
     // it's used to skip recently published version during compaction
@@ -72,7 +71,7 @@ void Rowset::make_visible(Version version, VersionHash version_hash) {
         _rowset_meta->mutable_delete_predicate()->set_version(version.first);
         return;
     }
-    make_visible_extra(version, version_hash);
+    make_visible_extra(version);
 }
 
 } // namespace doris

be/src/olap/rowset/rowset.h

@@ -140,12 +140,11 @@ class Rowset : public std::enable_shared_from_this<Rowset> {
     bool is_pending() const { return _is_pending; }
 
     // publish rowset to make it visible to read
-    void make_visible(Version version, VersionHash version_hash);
+    void make_visible(Version version);
 
     // helper class to access RowsetMeta
     int64_t start_version() const { return rowset_meta()->version().first; }
     int64_t end_version() const { return rowset_meta()->version().second; }
-    VersionHash version_hash() const { return rowset_meta()->version_hash(); }
     size_t index_disk_size() const { return rowset_meta()->index_disk_size(); }
     size_t data_disk_size() const { return rowset_meta()->total_disk_size(); }
     bool empty() const { return rowset_meta()->empty(); }
@@ -269,7 +268,7 @@ class Rowset : public std::enable_shared_from_this<Rowset> {
     virtual void do_close() = 0;
 
     // allow subclass to add custom logic when rowset is being published
-    virtual void make_visible_extra(Version version, VersionHash version_hash) {}
+    virtual void make_visible_extra(Version version) {}
 
     const TabletSchema* _schema;
     FilePathDesc _rowset_path_desc;

be/src/olap/rowset/rowset_converter.cpp

@@ -52,7 +52,6 @@ OLAPStatus RowsetConverter::_convert_rowset(const RowsetMetaSharedPtr& src_rowse
     context.segments_overlap = src_rowset_meta->segments_overlap();
     if (context.rowset_state == VISIBLE) {
         context.version = src_rowset_meta->version();
-        context.version_hash = src_rowset_meta->version_hash();
     } else {
         context.txn_id = src_rowset_meta->txn_id();
         context.load_id = src_rowset_meta->load_id();

be/src/olap/rowset/rowset_meta.h

@@ -133,12 +133,6 @@ class RowsetMeta {
 
     void set_end_version(int64_t end_version) { _rowset_meta_pb.set_end_version(end_version); }
 
-    VersionHash version_hash() const { return _rowset_meta_pb.version_hash(); }
-
-    void set_version_hash(VersionHash version_hash) {
-        _rowset_meta_pb.set_version_hash(version_hash);
-    }
-
     int64_t num_rows() const { return _rowset_meta_pb.num_rows(); }
 
     void set_num_rows(int64_t num_rows) { _rowset_meta_pb.set_num_rows(num_rows); }

be/src/olap/rowset/rowset_reader.h

@@ -56,8 +56,6 @@ class RowsetReader {
 
     virtual Version version() = 0;
 
-    virtual VersionHash version_hash() = 0;
-
     virtual RowsetSharedPtr rowset() = 0;
 
     virtual int64_t filtered_rows() = 0;

be/src/olap/rowset/rowset_writer_context.h

@@ -36,7 +36,6 @@ struct RowsetWriterContext {
               tablet_schema(nullptr),
               rowset_state(PREPARED),
               version(Version(0, 0)),
-              version_hash(0),
               txn_id(0),
               tablet_uid(0, 0),
               segments_overlap(OVERLAP_UNKNOWN) {
@@ -55,7 +54,6 @@ struct RowsetWriterContext {
     RowsetStatePB rowset_state;
     // properties for non-pending rowset
     Version version;
-    VersionHash version_hash;
 
     // properties for pending rowset
     int64_t txn_id;