[Improve](Tablet Schema) Use deterministic way to serialize protobuf (#30906)

be/src/olap/rowset/rowset_meta.h

@@ -56,7 +56,7 @@ class RowsetMeta {
     virtual bool init_from_pb(const RowsetMetaPB& rowset_meta_pb) {
         if (rowset_meta_pb.has_tablet_schema()) {
             _schema = TabletSchemaCache::instance()->insert(
-                    rowset_meta_pb.tablet_schema().SerializeAsString());
+                    TabletSchema::deterministic_string_serialize(rowset_meta_pb.tablet_schema()));
         }
         // Release ownership of TabletSchemaPB from `rowset_meta_pb` and then set it back to `rowset_meta_pb`,
         // this won't break const semantics of `rowset_meta_pb`, because `rowset_meta_pb` is not changed
@@ -373,7 +373,7 @@ class RowsetMeta {
         }
         if (rowset_meta_pb.has_tablet_schema()) {
             _schema = TabletSchemaCache::instance()->insert(
-                    rowset_meta_pb.tablet_schema().SerializeAsString());
+                    TabletSchema::deterministic_string_serialize(rowset_meta_pb.tablet_schema()));
             rowset_meta_pb.clear_tablet_schema();
         }
         _rowset_meta_pb = rowset_meta_pb;

be/src/olap/snapshot_manager.cpp

@@ -166,8 +166,8 @@ Status SnapshotManager::convert_rowset_ids(const std::string& clone_dir, int64_t
     }
     new_tablet_meta_pb.set_schema_hash(schema_hash);
     TabletSchemaSPtr tablet_schema;
-    tablet_schema =
-            TabletSchemaCache::instance()->insert(new_tablet_meta_pb.schema().SerializeAsString());
+    tablet_schema = TabletSchemaCache::instance()->insert(
+            TabletSchema::deterministic_string_serialize(new_tablet_meta_pb.schema()));
 
     std::unordered_map<Version, RowsetMetaPB*, HashOfVersion> rs_version_map;
     std::unordered_map<RowsetId, RowsetId, HashOfRowsetId> rowset_id_mapping;

be/src/olap/tablet_schema.cpp

@@ -20,12 +20,16 @@
 #include <gen_cpp/Descriptors_types.h>
 #include <gen_cpp/olap_file.pb.h>
 #include <glog/logging.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
 
 #include <algorithm>
 #include <cctype>
 // IWYU pragma: no_include <bits/std_abs.h>
 #include <cmath> // IWYU pragma: keep
 #include <ostream>
+#include <vector>
 
 // IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
 #include "common/compiler_util.h" // IWYU pragma: keep
@@ -609,7 +613,7 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index) const {
         index->add_col_unique_id(col_unique_id);
     }
     index->set_index_type(_index_type);
-    for (auto& kv : _properties) {
+    for (const auto& kv : _properties) {
         (*index->mutable_properties())[kv.first] = kv.second;
     }
 }
@@ -726,7 +730,7 @@ void TabletSchema::copy_from(const TabletSchema& tablet_schema) {
 std::string TabletSchema::to_key() const {
     TabletSchemaPB pb;
     to_schema_pb(&pb);
-    return pb.SerializeAsString();
+    return TabletSchema::deterministic_string_serialize(pb);
 }
 
 void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version,
@@ -1097,4 +1101,13 @@ bool operator!=(const TabletSchema& a, const TabletSchema& b) {
     return !(a == b);
 }
 
+std::string TabletSchema::deterministic_string_serialize(const TabletSchemaPB& schema_pb) {
+    std::string output;
+    google::protobuf::io::StringOutputStream string_output_stream(&output);
+    google::protobuf::io::CodedOutputStream output_stream(&string_output_stream);
+    output_stream.SetSerializationDeterministic(true);
+    schema_pb.SerializeToCodedStream(&output_stream);
+    return output;
+}
+
 } // namespace doris

be/src/olap/tablet_schema.h

@@ -208,6 +208,9 @@ class TabletSchema {
     // void create_from_pb(const TabletSchemaPB& schema, TabletSchema* tablet_schema).
     TabletSchema() = default;
     void init_from_pb(const TabletSchemaPB& schema);
+    // Notice: Use deterministic way to serialize protobuf,
+    // since serialize Map in protobuf may could lead to un-deterministic by default
+    static std::string deterministic_string_serialize(const TabletSchemaPB& schema_pb);
     void to_schema_pb(TabletSchemaPB* tablet_meta_pb) const;
     void append_column(TabletColumn column, bool is_dropped_column = false);
     void append_index(TabletIndex index);
@@ -310,6 +313,27 @@ class TabletSchema {
         str += "]";
         return str;
     }
+
+    // Dump [(name, type, is_nullable), ...]
+    string dump_structure() const {
+        string str = "[";
+        for (auto p : _cols) {
+            if (str.size() > 1) {
+                str += ", ";
+            }
+            str += "(";
+            str += p.name();
+            str += ", ";
+            str += TabletColumn::get_string_by_field_type(p.type());
+            str += ", ";
+            str += "is_nullable:";
+            str += (p.is_nullable() ? "true" : "false");
+            str += ")";
+        }
+        str += "]";
+        return str;
+    }
+
     vectorized::Block create_block_by_cids(const std::vector<uint32_t>& cids);
 
 private: