diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h index 360a3a02025a34..0e545bb30bc1ab 100644 --- a/be/src/olap/rowset/rowset_meta.h +++ b/be/src/olap/rowset/rowset_meta.h @@ -56,7 +56,7 @@ class RowsetMeta { virtual bool init_from_pb(const RowsetMetaPB& rowset_meta_pb) { if (rowset_meta_pb.has_tablet_schema()) { _schema = TabletSchemaCache::instance()->insert( - rowset_meta_pb.tablet_schema().SerializeAsString()); + TabletSchema::deterministic_string_serialize(rowset_meta_pb.tablet_schema())); } // Release ownership of TabletSchemaPB from `rowset_meta_pb` and then set it back to `rowset_meta_pb`, // this won't break const semantics of `rowset_meta_pb`, because `rowset_meta_pb` is not changed @@ -373,7 +373,7 @@ class RowsetMeta { } if (rowset_meta_pb.has_tablet_schema()) { _schema = TabletSchemaCache::instance()->insert( - rowset_meta_pb.tablet_schema().SerializeAsString()); + TabletSchema::deterministic_string_serialize(rowset_meta_pb.tablet_schema())); rowset_meta_pb.clear_tablet_schema(); } _rowset_meta_pb = rowset_meta_pb; diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index e8135a420259cd..05e2c771aac48e 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -166,8 +166,8 @@ Status SnapshotManager::convert_rowset_ids(const std::string& clone_dir, int64_t } new_tablet_meta_pb.set_schema_hash(schema_hash); TabletSchemaSPtr tablet_schema; - tablet_schema = - TabletSchemaCache::instance()->insert(new_tablet_meta_pb.schema().SerializeAsString()); + tablet_schema = TabletSchemaCache::instance()->insert( + TabletSchema::deterministic_string_serialize(new_tablet_meta_pb.schema())); std::unordered_map rs_version_map; std::unordered_map rowset_id_mapping; diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index fefd291e3f7e09..8c810b528b9b57 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -20,12 +20,16 @@ #include #include #include +#include +#include +#include #include #include // IWYU pragma: no_include #include // IWYU pragma: keep #include +#include // IWYU pragma: no_include #include "common/compiler_util.h" // IWYU pragma: keep @@ -609,7 +613,7 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index) const { index->add_col_unique_id(col_unique_id); } index->set_index_type(_index_type); - for (auto& kv : _properties) { + for (const auto& kv : _properties) { (*index->mutable_properties())[kv.first] = kv.second; } } @@ -726,7 +730,7 @@ void TabletSchema::copy_from(const TabletSchema& tablet_schema) { std::string TabletSchema::to_key() const { TabletSchemaPB pb; to_schema_pb(&pb); - return pb.SerializeAsString(); + return TabletSchema::deterministic_string_serialize(pb); } void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version, @@ -1097,4 +1101,13 @@ bool operator!=(const TabletSchema& a, const TabletSchema& b) { return !(a == b); } +std::string TabletSchema::deterministic_string_serialize(const TabletSchemaPB& schema_pb) { + std::string output; + google::protobuf::io::StringOutputStream string_output_stream(&output); + google::protobuf::io::CodedOutputStream output_stream(&string_output_stream); + output_stream.SetSerializationDeterministic(true); + schema_pb.SerializeToCodedStream(&output_stream); + return output; +} + } // namespace doris diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index abc75fe7a5bebe..11f87f2938749a 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -208,6 +208,9 @@ class TabletSchema { // void create_from_pb(const TabletSchemaPB& schema, TabletSchema* tablet_schema). TabletSchema() = default; void init_from_pb(const TabletSchemaPB& schema); + // Notice: Use deterministic way to serialize protobuf, + // since serialize Map in protobuf may could lead to un-deterministic by default + static std::string deterministic_string_serialize(const TabletSchemaPB& schema_pb); void to_schema_pb(TabletSchemaPB* tablet_meta_pb) const; void append_column(TabletColumn column, bool is_dropped_column = false); void append_index(TabletIndex index); @@ -310,6 +313,27 @@ class TabletSchema { str += "]"; return str; } + + // Dump [(name, type, is_nullable), ...] + string dump_structure() const { + string str = "["; + for (auto p : _cols) { + if (str.size() > 1) { + str += ", "; + } + str += "("; + str += p.name(); + str += ", "; + str += TabletColumn::get_string_by_field_type(p.type()); + str += ", "; + str += "is_nullable:"; + str += (p.is_nullable() ? "true" : "false"); + str += ")"; + } + str += "]"; + return str; + } + vectorized::Block create_block_by_cids(const std::vector& cids); private: