From e5dd7f9ba159ce9c992e81e1ed5d6caae7a5f963 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Tue, 19 Aug 2025 17:10:05 +0800 Subject: [PATCH 01/18] feat(BREAKING): update load options --- .../embed/load.pc.cc | 86 +++++++------------ .../embed/load.pc.h | 20 ++--- .../embed/util.pc.cc | 14 +-- .../embed/util.pc.h | 3 +- cmd/protoc-gen-cpp-tableau-loader/hub.go | 5 +- cmd/protoc-gen-go-tableau-loader/hub.go | 4 +- go.mod | 2 +- go.sum | 4 +- .../src/hub/custom/item/custom_item_conf.h | 2 +- .../src/protoconf/hub.pc.cc | 5 +- .../src/protoconf/load.pc.cc | 86 +++++++------------ .../src/protoconf/load.pc.h | 20 ++--- .../src/protoconf/util.pc.cc | 14 +-- .../src/protoconf/util.pc.h | 3 +- .../protoconf/loader/hub.pc.go | 4 +- 15 files changed, 100 insertions(+), 172 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc index 2f5b68f6..3c49cb1b 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc @@ -10,49 +10,22 @@ namespace tableau { // Forward declaration of the PatchMessage function bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); -std::shared_ptr ParseLoadOptions(std::shared_ptr opts) { - std::shared_ptr new_opts = std::make_shared(); - // set default values - new_opts->mode = LoadMode::kAll; - new_opts->read_func = util::ReadFile; - new_opts->load_func = LoadMessager; - - if (opts == nullptr) { - return new_opts; - } - if (opts->ignore_unknown_fields.has_value()) { - new_opts->ignore_unknown_fields = opts->ignore_unknown_fields; - } - if (!opts->patch_dirs.empty()) { - new_opts->patch_dirs = opts->patch_dirs; - } - if (opts->mode != LoadMode::kNone) { - new_opts->mode = opts->mode; - } - if (opts->read_func != nullptr) { - new_opts->read_func = opts->read_func; - } - if (opts->load_func != nullptr) { - new_opts->load_func = opts->load_func; - } - return new_opts; -} - std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, const std::string& name) { - std::shared_ptr mopts; - if (auto iter = opts->messager_options.find(name); iter != opts->messager_options.end()) { - mopts = iter->second; - } else { - mopts = std::make_shared(); + std::shared_ptr mopts = std::make_shared(); + if (!opts) { + return mopts; } - if (mopts->ignore_unknown_fields.has_value()) { + if (auto iter = opts->messager_options.find(name); iter != opts->messager_options.end() && iter->second) { + mopts = std::make_shared(*iter->second); + } + if (!mopts->ignore_unknown_fields.has_value()) { mopts->ignore_unknown_fields = opts->ignore_unknown_fields; } if (mopts->patch_dirs.empty()) { mopts->patch_dirs = opts->patch_dirs; } - if (mopts->mode == LoadMode::kNone) { + if (!mopts->mode.has_value()) { mopts->mode = opts->mode; } if (mopts->read_func == nullptr) { @@ -66,12 +39,11 @@ std::shared_ptr ParseMessagerOptions(std::shared_ptr options /* = nullptr*/) { - if (options == nullptr) { - return LoadMessager(msg, path, fmt, nullptr); - } - if (options->mode == LoadMode::kOnlyMain) { + auto mode = options->GetMode(); + auto load_func = options->GetLoadFunc(); + if (mode == LoadMode::kOnlyMain) { // ignore patch files when LoadMode::kModeOnlyMain specified - return options->load_func(msg, path, fmt, nullptr); + return load_func(msg, path, fmt, nullptr); } const std::string& name = msg.GetDescriptor()->name(); std::vector patch_paths; @@ -92,27 +64,27 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem } } if (existed_patch_paths.empty()) { - if (options->mode == LoadMode::kOnlyPatch) { + if (mode == LoadMode::kOnlyPatch) { // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. return true; } // no valid patch path provided, then just load from the "main" file. - return options->load_func(msg, path, fmt, options); + return load_func(msg, path, fmt, options); } switch (patch) { case tableau::PATCH_REPLACE: { // just use the last "patch" file std::filesystem::path& patch_path = existed_patch_paths.back(); - if (!options->load_func(msg, patch_path, util::GetFormat(patch_path), options)) { + if (!load_func(msg, patch_path, util::GetFormat(patch_path), options)) { return false; } break; } case tableau::PATCH_MERGE: { - if (options->mode != LoadMode::kOnlyPatch) { + if (mode != LoadMode::kOnlyPatch) { // load msg from the "main" file - if (!options->load_func(msg, path, fmt, options)) { + if (!load_func(msg, path, fmt, options)) { return false; } } @@ -121,7 +93,7 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem std::unique_ptr _auto_release(msg.New()); // load patch_msg from each "patch" file for (auto&& patch_path : existed_patch_paths) { - if (!options->load_func(*patch_msg_ptr, patch_path, util::GetFormat(patch_path), options)) { + if (!load_func(*patch_msg_ptr, patch_path, util::GetFormat(patch_path), options)) { return false; } if (!PatchMessage(msg, *patch_msg_ptr)) { @@ -142,18 +114,15 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); std::string content; - ReadFunc read_func = util::ReadFile; - if (options != nullptr && options->read_func) { - read_func = options->read_func; - } - bool ok = read_func(path, content); + bool ok = options->GetReadFunc()(path, content); if (!ok) { return false; } switch (fmt) { case Format::kJSON: { - return util::JSON2Message(content, msg, options); + return util::JSON2Message(content, msg, options->GetIgnoreUnknownFields()); } case Format::kText: { return util::Text2Message(content, msg); @@ -170,9 +139,10 @@ bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& p bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); const std::string& name = msg.GetDescriptor()->name(); std::filesystem::path path; - if (options && !options->path.empty()) { + if (!options->path.empty()) { // path specified in Paths, then use it instead of dir. path = options->path; fmt = util::GetFormat(path); @@ -188,10 +158,7 @@ bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::pa if (worksheet_options.patch() != tableau::PATCH_NONE) { return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); } - if (options) { - return options->load_func(msg, path, fmt, options); - } - return LoadMessager(msg, path, fmt); + return options->GetLoadFunc()(msg, path, fmt, options); } #ifdef _WIN32 @@ -347,4 +314,9 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); return true; } + +bool BaseOptions::GetIgnoreUnknownFields() const { return ignore_unknown_fields.value_or(false); } +LoadMode BaseOptions::GetMode() const { return mode.value_or(LoadMode::kAll); } +ReadFunc BaseOptions::GetReadFunc() const { return read_func ? read_func : util::ReadFile; } +LoadFunc BaseOptions::GetLoadFunc() const { return load_func ? load_func : LoadMessager; } } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h index e8ba9118..b1835560 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h @@ -11,7 +11,6 @@ namespace tableau { enum class LoadMode { - kNone, kAll, // Load all related files kOnlyMain, // Only load the main file kOnlyPatch, // Only load the patch files @@ -39,15 +38,21 @@ struct BaseOptions { // Specify the loading mode for config patching. // - For LoadOptions, default is LoadMode::kModeAll. // - For MessagerOptions, inherit from LoadOptions if not set. - LoadMode mode; + std::optional mode; // You can specify custom read function to read a config file's content. // - For LoadOptions, default is util::ReadFile. // - For MessagerOptions, inherit from LoadOptions if not set. ReadFunc read_func; // You can specify custom load function to load a messager's content. - // - For LoadOptions, default is LoadMessage. + // - For LoadOptions, default is LoadMessager. // - For MessagerOptions, inherit from LoadOptions if not set. LoadFunc load_func; + + public: + bool GetIgnoreUnknownFields() const; + LoadMode GetMode() const; + ReadFunc GetReadFunc() const; + LoadFunc GetLoadFunc() const; }; // LoadOptionsOptions is the options struct, which contains both global-level and @@ -56,7 +61,7 @@ struct LoadOptions : public BaseOptions { // messager_options maps each messager name to a MessageOptions. // If specified, then the messager will be parsed with the given options // directly. - std::unordered_map> messager_options; + std::unordered_map> messager_options; }; // MessagerOptions defines the options for loading a messager. @@ -78,10 +83,7 @@ class Messager { public: virtual ~Messager() = default; - static const std::string& Name() { - static const std::string kEmpty = ""; - return kEmpty; - } + static const std::string& Name() = delete; const Stats& GetStats() { return stats_; } // Load fills message from file in the specified directory and format. virtual bool Load(const std::filesystem::path& dir, Format fmt, @@ -97,8 +99,6 @@ class Messager { Stats stats_; }; -// ParseLoadOptions parses load options with default global-level options. -std::shared_ptr ParseLoadOptions(std::shared_ptr opts); // ParseMessagerOptions parses messager options with both global-level and // messager-level options taken into consideration. std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc index 11236ea3..ced3cad7 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc @@ -58,16 +58,10 @@ const std::string& Format2Ext(Format fmt) { } } -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, - std::shared_ptr options /* = nullptr */) { - google::protobuf::util::Status status; - if (options != nullptr) { - google::protobuf::util::JsonParseOptions parse_options; - parse_options.ignore_unknown_fields = options->ignore_unknown_fields.value_or(false); - status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); - } else { - status = google::protobuf::util::JsonStringToMessage(json, &msg); - } +bool JSON2Message(const std::string& json, google::protobuf::Message& msg, bool ignore_unknown_fields) { + google::protobuf::util::JsonParseOptions parse_options; + parse_options.ignore_unknown_fields = ignore_unknown_fields; + auto status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); if (!status.ok()) { SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kJSONExt + ": " + status.ToString()); return false; diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h index 72350710..78c5b269 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h @@ -57,8 +57,7 @@ Format GetFormat(const std::filesystem::path& path); // and the error message can be obtained by GetErrMsg(). const std::string& Format2Ext(Format fmt); -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, - std::shared_ptr options = nullptr); +bool JSON2Message(const std::string& json, google::protobuf::Message& msg, bool ignore_unknown_fields); bool Text2Message(const std::string& text, google::protobuf::Message& msg); bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); diff --git a/cmd/protoc-gen-cpp-tableau-loader/hub.go b/cmd/protoc-gen-cpp-tableau-loader/hub.go index dee3a2cf..5128f855 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/hub.go +++ b/cmd/protoc-gen-cpp-tableau-loader/hub.go @@ -348,12 +348,11 @@ std::shared_ptr Hub::InternalLoad(const std::filesystem::path& dir, std::shared_ptr options /* = nullptr */) const { // intercept protobuf error logs auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); - auto opts = ParseLoadOptions(options); auto msger_map = NewMessagerMap(); for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); - auto mopts = ParseMessagerOptions(opts, name); + auto mopts = ParseMessagerOptions(options, name); bool ok = iter.second->Load(dir, fmt, mopts); if (!ok) { ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str()); @@ -408,7 +407,7 @@ bool Hub::Postprocess(std::shared_ptr msger_map) { auto msger = iter.second; bool ok = msger->ProcessAfterLoadAll(tmp_hub); if (!ok) { - SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + msger->Name()); + SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + iter.first); return false; } } diff --git a/cmd/protoc-gen-go-tableau-loader/hub.go b/cmd/protoc-gen-go-tableau-loader/hub.go index 1db4316d..d2e6147d 100644 --- a/cmd/protoc-gen-go-tableau-loader/hub.go +++ b/cmd/protoc-gen-go-tableau-loader/hub.go @@ -288,11 +288,11 @@ func (h *Hub) GetMessager(name string) Messager { } // Load fills messages from files in the specified directory and format. -func (h *Hub) Load(dir string, format format.Format, options ...load.Option) error { +func (h *Hub) Load(dir string, format format.Format, options ...load.LoadOption) error { messagerMap := h.NewMessagerMap() opts := load.ParseOptions(options...) for name, msger := range messagerMap { - mopts := load.ParseMessagerOptionsFromOptions(opts, name) + mopts := load.ParseMessagerOptions(opts, name) if err := msger.Load(dir, format, mopts); err != nil { return errors.WithMessagef(err, "failed to load: %v", name) } diff --git a/go.mod b/go.mod index 51acdf91..7e08fade 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 github.com/stretchr/testify v1.10.0 - github.com/tableauio/tableau v0.14.1 + github.com/tableauio/tableau v0.14.2-0.20250819090353-59b7724f572d golang.org/x/exp v0.0.0-20230418202329-0354be287a23 google.golang.org/protobuf v1.34.2 ) diff --git a/go.sum b/go.sum index b3c597dd..c22a07ed 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subchen/go-xmldom v1.1.2 h1:7evI2YqfYYOnuj+PBwyaOZZYjl3iWq35P6KfBUw9jeU= github.com/subchen/go-xmldom v1.1.2/go.mod h1:6Pg/HuX5/T4Jlj0IPJF1sRxKVoI/rrKP6LIMge9d5/8= -github.com/tableauio/tableau v0.14.1 h1:o6X9Tmjytb5GdB5XJdH9DYs8v7wURb5hxtX75VIoXP8= -github.com/tableauio/tableau v0.14.1/go.mod h1:EitY0lyJa8gP6tBkzWhS2WvYhC8hyWPGXoR4QPQ+Qmw= +github.com/tableauio/tableau v0.14.2-0.20250819090353-59b7724f572d h1:L6jKadyW0Almj3CvAsmqWn+mQ3urG0q/vXr4EVOWu/4= +github.com/tableauio/tableau v0.14.2-0.20250819090353-59b7724f572d/go.mod h1:EitY0lyJa8gP6tBkzWhS2WvYhC8hyWPGXoR4QPQ+Qmw= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 h1:6932x8ltq1w4utjmfMPVj09jdMlkY0aiA6+Skbtl3/c= diff --git a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h index 637d9e41..63bc6ffd 100644 --- a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h +++ b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h @@ -3,7 +3,7 @@ #include "protoconf/item_conf.pc.h" class CustomItemConf : public tableau::Messager { public: - static const std::string& Name() { return kCustomName; }; + static const std::string& Name() { return kCustomName; } virtual bool Load(const std::filesystem::path&, tableau::Format, std::shared_ptr options = nullptr) override { return true; diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index 51e591dc..62fe81f8 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -58,12 +58,11 @@ std::shared_ptr Hub::InternalLoad(const std::filesystem::path& dir, std::shared_ptr options /* = nullptr */) const { // intercept protobuf error logs auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); - auto opts = ParseLoadOptions(options); auto msger_map = NewMessagerMap(); for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); - auto mopts = ParseMessagerOptions(opts, name); + auto mopts = ParseMessagerOptions(options, name); bool ok = iter.second->Load(dir, fmt, mopts); if (!ok) { ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str()); @@ -118,7 +117,7 @@ bool Hub::Postprocess(std::shared_ptr msger_map) { auto msger = iter.second; bool ok = msger->ProcessAfterLoadAll(tmp_hub); if (!ok) { - SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + msger->Name()); + SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + iter.first); return false; } } diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc index 6c3bc34e..30aa36e1 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -15,49 +15,22 @@ namespace tableau { // Forward declaration of the PatchMessage function bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); -std::shared_ptr ParseLoadOptions(std::shared_ptr opts) { - std::shared_ptr new_opts = std::make_shared(); - // set default values - new_opts->mode = LoadMode::kAll; - new_opts->read_func = util::ReadFile; - new_opts->load_func = LoadMessager; - - if (opts == nullptr) { - return new_opts; - } - if (opts->ignore_unknown_fields.has_value()) { - new_opts->ignore_unknown_fields = opts->ignore_unknown_fields; - } - if (!opts->patch_dirs.empty()) { - new_opts->patch_dirs = opts->patch_dirs; - } - if (opts->mode != LoadMode::kNone) { - new_opts->mode = opts->mode; - } - if (opts->read_func != nullptr) { - new_opts->read_func = opts->read_func; - } - if (opts->load_func != nullptr) { - new_opts->load_func = opts->load_func; - } - return new_opts; -} - std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, const std::string& name) { - std::shared_ptr mopts; - if (auto iter = opts->messager_options.find(name); iter != opts->messager_options.end()) { - mopts = iter->second; - } else { - mopts = std::make_shared(); + std::shared_ptr mopts = std::make_shared(); + if (!opts) { + return mopts; } - if (mopts->ignore_unknown_fields.has_value()) { + if (auto iter = opts->messager_options.find(name); iter != opts->messager_options.end() && iter->second) { + mopts = std::make_shared(*iter->second); + } + if (!mopts->ignore_unknown_fields.has_value()) { mopts->ignore_unknown_fields = opts->ignore_unknown_fields; } if (mopts->patch_dirs.empty()) { mopts->patch_dirs = opts->patch_dirs; } - if (mopts->mode == LoadMode::kNone) { + if (!mopts->mode.has_value()) { mopts->mode = opts->mode; } if (mopts->read_func == nullptr) { @@ -71,12 +44,11 @@ std::shared_ptr ParseMessagerOptions(std::shared_ptr options /* = nullptr*/) { - if (options == nullptr) { - return LoadMessager(msg, path, fmt, nullptr); - } - if (options->mode == LoadMode::kOnlyMain) { + auto mode = options->GetMode(); + auto load_func = options->GetLoadFunc(); + if (mode == LoadMode::kOnlyMain) { // ignore patch files when LoadMode::kModeOnlyMain specified - return options->load_func(msg, path, fmt, nullptr); + return load_func(msg, path, fmt, nullptr); } const std::string& name = msg.GetDescriptor()->name(); std::vector patch_paths; @@ -97,27 +69,27 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem } } if (existed_patch_paths.empty()) { - if (options->mode == LoadMode::kOnlyPatch) { + if (mode == LoadMode::kOnlyPatch) { // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. return true; } // no valid patch path provided, then just load from the "main" file. - return options->load_func(msg, path, fmt, options); + return load_func(msg, path, fmt, options); } switch (patch) { case tableau::PATCH_REPLACE: { // just use the last "patch" file std::filesystem::path& patch_path = existed_patch_paths.back(); - if (!options->load_func(msg, patch_path, util::GetFormat(patch_path), options)) { + if (!load_func(msg, patch_path, util::GetFormat(patch_path), options)) { return false; } break; } case tableau::PATCH_MERGE: { - if (options->mode != LoadMode::kOnlyPatch) { + if (mode != LoadMode::kOnlyPatch) { // load msg from the "main" file - if (!options->load_func(msg, path, fmt, options)) { + if (!load_func(msg, path, fmt, options)) { return false; } } @@ -126,7 +98,7 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem std::unique_ptr _auto_release(msg.New()); // load patch_msg from each "patch" file for (auto&& patch_path : existed_patch_paths) { - if (!options->load_func(*patch_msg_ptr, patch_path, util::GetFormat(patch_path), options)) { + if (!load_func(*patch_msg_ptr, patch_path, util::GetFormat(patch_path), options)) { return false; } if (!PatchMessage(msg, *patch_msg_ptr)) { @@ -147,18 +119,15 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); std::string content; - ReadFunc read_func = util::ReadFile; - if (options != nullptr && options->read_func) { - read_func = options->read_func; - } - bool ok = read_func(path, content); + bool ok = options->GetReadFunc()(path, content); if (!ok) { return false; } switch (fmt) { case Format::kJSON: { - return util::JSON2Message(content, msg, options); + return util::JSON2Message(content, msg, options->GetIgnoreUnknownFields()); } case Format::kText: { return util::Text2Message(content, msg); @@ -175,9 +144,10 @@ bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& p bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); const std::string& name = msg.GetDescriptor()->name(); std::filesystem::path path; - if (options && !options->path.empty()) { + if (!options->path.empty()) { // path specified in Paths, then use it instead of dir. path = options->path; fmt = util::GetFormat(path); @@ -193,10 +163,7 @@ bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::pa if (worksheet_options.patch() != tableau::PATCH_NONE) { return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); } - if (options) { - return options->load_func(msg, path, fmt, options); - } - return LoadMessager(msg, path, fmt); + return options->GetLoadFunc()(msg, path, fmt, options); } #ifdef _WIN32 @@ -352,4 +319,9 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); return true; } + +bool BaseOptions::GetIgnoreUnknownFields() const { return ignore_unknown_fields.value_or(false); } +LoadMode BaseOptions::GetMode() const { return mode.value_or(LoadMode::kAll); } +ReadFunc BaseOptions::GetReadFunc() const { return read_func ? read_func : util::ReadFile; } +LoadFunc BaseOptions::GetLoadFunc() const { return load_func ? load_func : LoadMessager; } } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.h b/test/cpp-tableau-loader/src/protoconf/load.pc.h index 8c1bbb83..f196de02 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.h @@ -16,7 +16,6 @@ namespace tableau { enum class LoadMode { - kNone, kAll, // Load all related files kOnlyMain, // Only load the main file kOnlyPatch, // Only load the patch files @@ -44,15 +43,21 @@ struct BaseOptions { // Specify the loading mode for config patching. // - For LoadOptions, default is LoadMode::kModeAll. // - For MessagerOptions, inherit from LoadOptions if not set. - LoadMode mode; + std::optional mode; // You can specify custom read function to read a config file's content. // - For LoadOptions, default is util::ReadFile. // - For MessagerOptions, inherit from LoadOptions if not set. ReadFunc read_func; // You can specify custom load function to load a messager's content. - // - For LoadOptions, default is LoadMessage. + // - For LoadOptions, default is LoadMessager. // - For MessagerOptions, inherit from LoadOptions if not set. LoadFunc load_func; + + public: + bool GetIgnoreUnknownFields() const; + LoadMode GetMode() const; + ReadFunc GetReadFunc() const; + LoadFunc GetLoadFunc() const; }; // LoadOptionsOptions is the options struct, which contains both global-level and @@ -61,7 +66,7 @@ struct LoadOptions : public BaseOptions { // messager_options maps each messager name to a MessageOptions. // If specified, then the messager will be parsed with the given options // directly. - std::unordered_map> messager_options; + std::unordered_map> messager_options; }; // MessagerOptions defines the options for loading a messager. @@ -83,10 +88,7 @@ class Messager { public: virtual ~Messager() = default; - static const std::string& Name() { - static const std::string kEmpty = ""; - return kEmpty; - } + static const std::string& Name() = delete; const Stats& GetStats() { return stats_; } // Load fills message from file in the specified directory and format. virtual bool Load(const std::filesystem::path& dir, Format fmt, @@ -102,8 +104,6 @@ class Messager { Stats stats_; }; -// ParseLoadOptions parses load options with default global-level options. -std::shared_ptr ParseLoadOptions(std::shared_ptr opts); // ParseMessagerOptions parses messager options with both global-level and // messager-level options taken into consideration. std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.cc b/test/cpp-tableau-loader/src/protoconf/util.pc.cc index 95dc95de..df0f07b2 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.cc @@ -63,16 +63,10 @@ const std::string& Format2Ext(Format fmt) { } } -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, - std::shared_ptr options /* = nullptr */) { - google::protobuf::util::Status status; - if (options != nullptr) { - google::protobuf::util::JsonParseOptions parse_options; - parse_options.ignore_unknown_fields = options->ignore_unknown_fields.value_or(false); - status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); - } else { - status = google::protobuf::util::JsonStringToMessage(json, &msg); - } +bool JSON2Message(const std::string& json, google::protobuf::Message& msg, bool ignore_unknown_fields) { + google::protobuf::util::JsonParseOptions parse_options; + parse_options.ignore_unknown_fields = ignore_unknown_fields; + auto status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); if (!status.ok()) { SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kJSONExt + ": " + status.ToString()); return false; diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.h b/test/cpp-tableau-loader/src/protoconf/util.pc.h index 2a5f7732..9fcc8cfb 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.h @@ -62,8 +62,7 @@ Format GetFormat(const std::filesystem::path& path); // and the error message can be obtained by GetErrMsg(). const std::string& Format2Ext(Format fmt); -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, - std::shared_ptr options = nullptr); +bool JSON2Message(const std::string& json, google::protobuf::Message& msg, bool ignore_unknown_fields); bool Text2Message(const std::string& text, google::protobuf::Message& msg); bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); diff --git a/test/go-tableau-loader/protoconf/loader/hub.pc.go b/test/go-tableau-loader/protoconf/loader/hub.pc.go index 9c3c4f2c..65f7f4d1 100644 --- a/test/go-tableau-loader/protoconf/loader/hub.pc.go +++ b/test/go-tableau-loader/protoconf/loader/hub.pc.go @@ -240,11 +240,11 @@ func (h *Hub) GetMessager(name string) Messager { } // Load fills messages from files in the specified directory and format. -func (h *Hub) Load(dir string, format format.Format, options ...load.Option) error { +func (h *Hub) Load(dir string, format format.Format, options ...load.LoadOption) error { messagerMap := h.NewMessagerMap() opts := load.ParseOptions(options...) for name, msger := range messagerMap { - mopts := load.ParseMessagerOptionsFromOptions(opts, name) + mopts := load.ParseMessagerOptions(opts, name) if err := msger.Load(dir, format, mopts); err != nil { return errors.WithMessagef(err, "failed to load: %v", name) } From ab5e21215524d98792e2bd4d30a81594b47df8d1 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Tue, 19 Aug 2025 17:34:14 +0800 Subject: [PATCH 02/18] feat: update and add bat script for golang --- test/cpp-tableau-loader/gen.bat | 29 +++++++++++----------- test/go-tableau-loader/gen.bat | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 15 deletions(-) create mode 100644 test/go-tableau-loader/gen.bat diff --git a/test/cpp-tableau-loader/gen.bat b/test/cpp-tableau-loader/gen.bat index 39b8aa34..178a00bc 100644 --- a/test/cpp-tableau-loader/gen.bat +++ b/test/cpp-tableau-loader/gen.bat @@ -14,7 +14,7 @@ set "PROTOCONF_IN=%repoRoot%\test\proto" set "PROTOCONF_OUT=%ROOTDIR%\src\protoconf" REM remove old generated files -if exist "%PROTOCONF_OUT%" rmdir /s /q "%PROTOCONF_OUT%" +rmdir /s /q "%PROTOCONF_OUT%" 2>nul mkdir "%PROTOCONF_OUT%" REM build @@ -27,17 +27,17 @@ set "PATH=%PATH%;%PLGUIN_DIR%" set protoFiles= pushd "%PROTOCONF_IN%" for /R %%f in (*.proto) do ( - set protoFiles=!protoFiles! "%%f" + set protoFiles=!protoFiles! "%%f" ) popd "%PROTOC%" ^ - --cpp-tableau-loader_out="%PROTOCONF_OUT%" ^ - --cpp-tableau-loader_opt=paths=source_relative,shards=2 ^ - --cpp_out="%PROTOCONF_OUT%" ^ - --proto_path="%PROTOBUF_PROTO%" ^ - --proto_path="%TABLEAU_PROTO%" ^ - --proto_path="%PROTOCONF_IN%" ^ - !protoFiles! +--cpp-tableau-loader_out="%PROTOCONF_OUT%" ^ +--cpp-tableau-loader_opt=paths=source_relative,shards=2 ^ +--cpp_out="%PROTOCONF_OUT%" ^ +--proto_path="%PROTOBUF_PROTO%" ^ +--proto_path="%TABLEAU_PROTO%" ^ +--proto_path="%PROTOCONF_IN%" ^ +!protoFiles! set "TABLEAU_IN=%TABLEAU_PROTO%\tableau\protobuf" set "TABLEAU_OUT=%ROOTDIR%\src" @@ -46,11 +46,10 @@ if exist "%TABLEAU_OUT%\tableau" rmdir /s /q "%TABLEAU_OUT%\tableau" mkdir "%TABLEAU_OUT%\tableau" "%PROTOC%" ^ - --cpp_out="%TABLEAU_OUT%" ^ - --proto_path="%PROTOBUF_PROTO%" ^ - --proto_path="%TABLEAU_PROTO%" ^ - "%TABLEAU_IN%\tableau.proto" ^ - "%TABLEAU_IN%\wellknown.proto" +--cpp_out="%TABLEAU_OUT%" ^ +--proto_path="%PROTOBUF_PROTO%" ^ +--proto_path="%TABLEAU_PROTO%" ^ +"%TABLEAU_IN%\tableau.proto" "%TABLEAU_IN%\wellknown.proto" endlocal -endlocal \ No newline at end of file +endlocal diff --git a/test/go-tableau-loader/gen.bat b/test/go-tableau-loader/gen.bat new file mode 100644 index 00000000..98b8ea36 --- /dev/null +++ b/test/go-tableau-loader/gen.bat @@ -0,0 +1,44 @@ +@echo off +setlocal +setlocal enabledelayedexpansion + +for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i +cd /d "%repoRoot%" + +set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\Debug\protoc.exe" +set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" +set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" +set "PLGUIN_DIR=%repoRoot%\cmd\protoc-gen-go-tableau-loader" +set "PROTOCONF_IN=%repoRoot%\test\proto" +set "PROTOCONF_OUT=%repoRoot%\test\go-tableau-loader\protoconf" +set "LOADER_OUT=%PROTOCONF_OUT%\loader" + +REM remove old generated files +rmdir /s /q "%PROTOCONF_OUT%" "%LOADER_OUT%" 2>nul +mkdir "%PROTOCONF_OUT%" "%LOADER_OUT%" + +REM build +pushd "%PLGUIN_DIR%" +go build +popd + +set "PATH=%PATH%;%PLGUIN_DIR%" + +set protoFiles= +pushd "%PROTOCONF_IN%" +for /R %%f in (*.proto) do ( + set protoFiles=!protoFiles! "%%f" +) +popd +"%PROTOC%" ^ +--go-tableau-loader_out="%LOADER_OUT%" ^ +--go-tableau-loader_opt=paths=source_relative,pkg=loader ^ +--go_out="%PROTOCONF_OUT%" ^ +--go_opt=paths=source_relative ^ +--proto_path="%PROTOBUF_PROTO%" ^ +--proto_path="%TABLEAU_PROTO%" ^ +--proto_path="%PROTOCONF_IN%" ^ +!protoFiles! + +endlocal +endlocal From e25af722d6cba2e3a38421645c812398246920a6 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Tue, 19 Aug 2025 21:28:31 +0800 Subject: [PATCH 03/18] feat: optimize cpp code --- .../embed/load.pc.cc | 280 +++++------------- .../embed/load.pc.h | 35 ++- .../embed/logger.pc.cc | 64 +--- .../embed/logger.pc.h | 4 +- .../embed/util.pc.cc | 171 +++++++++-- .../embed/util.pc.h | 8 +- cmd/protoc-gen-cpp-tableau-loader/hub.go | 11 +- cmd/protoc-gen-go-tableau-loader/hub.go | 2 +- go.mod | 2 +- go.sum | 4 +- .../src/hub/custom/item/custom_item_conf.cpp | 6 +- test/cpp-tableau-loader/src/hub/hub.cpp | 13 +- test/cpp-tableau-loader/src/main.cpp | 92 +++--- .../src/protoconf/hub.pc.cc | 10 +- .../cpp-tableau-loader/src/protoconf/hub.pc.h | 1 + .../src/protoconf/load.pc.cc | 280 +++++------------- .../src/protoconf/load.pc.h | 35 ++- .../src/protoconf/logger.pc.cc | 64 +--- .../src/protoconf/logger.pc.h | 4 +- .../src/protoconf/util.pc.cc | 171 +++++++++-- .../src/protoconf/util.pc.h | 8 +- .../protoconf/loader/hub.pc.go | 2 +- 22 files changed, 576 insertions(+), 691 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc index 3c49cb1b..a29958bc 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc @@ -1,44 +1,73 @@ #include "load.pc.h" -#include +#include +#include #include "logger.pc.h" #include "util.pc.h" namespace tableau { - -// Forward declaration of the PatchMessage function -bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); - -std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, - const std::string& name) { +std::shared_ptr LoadOptions::ParseMessagerOptionsByName(const std::string& name) const { std::shared_ptr mopts = std::make_shared(); - if (!opts) { - return mopts; - } - if (auto iter = opts->messager_options.find(name); iter != opts->messager_options.end() && iter->second) { + if (auto iter = messager_options.find(name); iter != messager_options.end() && iter->second) { mopts = std::make_shared(*iter->second); } if (!mopts->ignore_unknown_fields.has_value()) { - mopts->ignore_unknown_fields = opts->ignore_unknown_fields; + mopts->ignore_unknown_fields = ignore_unknown_fields; } if (mopts->patch_dirs.empty()) { - mopts->patch_dirs = opts->patch_dirs; + mopts->patch_dirs = patch_dirs; } if (!mopts->mode.has_value()) { - mopts->mode = opts->mode; + mopts->mode = mode; } - if (mopts->read_func == nullptr) { - mopts->read_func = opts->read_func; + if (!mopts->read_func) { + mopts->read_func = read_func; } - if (mopts->load_func == nullptr) { - mopts->load_func = opts->load_func; + if (!mopts->load_func) { + mopts->load_func = load_func; } return mopts; } +bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, + std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); + std::string content; + bool ok = options->GetReadFunc()(path, content); + if (!ok) { + return false; + } + return Unmarshal(content, msg, fmt, options); +} + +bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt, + std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); + const std::string& name = msg.GetDescriptor()->name(); + std::filesystem::path path; + if (!options->path.empty()) { + // path specified in Paths, then use it instead of dir. + path = options->path; + fmt = util::GetFormat(path); + } + if (path.empty()) { + std::filesystem::path filename = name + util::Format2Ext(fmt); + path = dir / filename; + } + + const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); + // access the extension directly using the generated identifier + const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet); + if (worksheet_options.patch() != tableau::PATCH_NONE) { + return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); + } + return options->GetLoadFunc()(msg, path, fmt, options); +} + bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, tableau::Patch patch, std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); auto mode = options->GetMode(); auto load_func = options->GetLoadFunc(); if (mode == LoadMode::kOnlyMain) { @@ -96,7 +125,7 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem if (!load_func(*patch_msg_ptr, patch_path, util::GetFormat(patch_path), options)) { return false; } - if (!PatchMessage(msg, *patch_msg_ptr)) { + if (!util::PatchMessage(msg, *patch_msg_ptr)) { return false; } } @@ -112,23 +141,33 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem return true; } -bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, - std::shared_ptr options /* = nullptr*/) { +bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Format fmt, + std::shared_ptr options /* = nullptr*/) { options = options ? options : std::make_shared(); - std::string content; - bool ok = options->GetReadFunc()(path, content); - if (!ok) { - return false; - } switch (fmt) { case Format::kJSON: { - return util::JSON2Message(content, msg, options->GetIgnoreUnknownFields()); + google::protobuf::util::JsonParseOptions parse_options; + parse_options.ignore_unknown_fields = options->GetIgnoreUnknownFields(); + auto status = google::protobuf::util::JsonStringToMessage(content, &msg, parse_options); + if (!status.ok()) { + SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kJSONExt + ": " + status.ToString()); + return false; + } + return true; } case Format::kText: { - return util::Text2Message(content, msg); + if (!google::protobuf::TextFormat::ParseFromString(content, &msg)) { + SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kTextExt); + return false; + } + return true; } case Format::kBin: { - return util::Bin2Message(content, msg); + if (!msg.ParseFromString(content)) { + SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kBinExt); + return false; + } + return true; } default: { SetErrMsg("unknown format: " + std::to_string(static_cast(fmt))); @@ -136,187 +175,4 @@ bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& p } } } - -bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt, - std::shared_ptr options /* = nullptr*/) { - options = options ? options : std::make_shared(); - const std::string& name = msg.GetDescriptor()->name(); - std::filesystem::path path; - if (!options->path.empty()) { - // path specified in Paths, then use it instead of dir. - path = options->path; - fmt = util::GetFormat(path); - } - if (path.empty()) { - std::filesystem::path filename = name + util::Format2Ext(fmt); - path = dir / filename; - } - - const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); - // access the extension directly using the generated identifier - const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet); - if (worksheet_options.patch() != tableau::PATCH_NONE) { - return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); - } - return options->GetLoadFunc()(msg, path, fmt, options); -} - -#ifdef _WIN32 -#undef GetMessage -#endif - -// PatchMessage patches src into dst, which must be a message with the same descriptor. -// -// # Default PatchMessage mechanism -// - scalar: Populated scalar fields in src are copied to dst. -// - message: Populated singular messages in src are merged into dst by -// recursively calling [xproto.PatchMessage], or replace dst message if -// "PATCH_REPLACE" is specified for this field. -// - list: The elements of every list field in src are appended to the -// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" -// is specified for this field. -// - map: The entries of every map field in src are MERGED (different from -// the behavior of proto.Merge) into the corresponding map field in dst, -// or replace dst map if "PATCH_REPLACE" is specified for this field. -// - unknown: The unknown fields of src are appended to the unknown -// fields of dst (TODO: untested). -// -// # References: -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details -bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { - const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); - const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); - // Ensure both messages are of the same type - if (dst_descriptor != src_descriptor) { - SetErrMsg("dst and src are not messages with the same descriptor"); - ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), - src_descriptor->name().c_str()); - return false; - } - - // Get the reflection and descriptor for the messages - const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); - const google::protobuf::Reflection* src_reflection = src.GetReflection(); - - // List all populated fields - std::vector fields; - src_reflection->ListFields(src, &fields); - - // Iterates over every populated field. - for (auto fd : fields) { - const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); - tableau::Patch patch = opts.prop().patch(); - if (patch == tableau::PATCH_REPLACE) { - dst_reflection->ClearField(&dst, fd); - } - if (fd->is_map()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 - auto key_fd = fd->message_type()->map_key(); - int src_count = src_reflection->FieldSize(src, fd); - int dst_count = dst_reflection->FieldSize(dst, fd); - switch (key_fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - std::unordered_map dst_key_index_map; \ - for (int i = 0; i < dst_count; i++) { \ - auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ - TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ - dst_key_index_map[key] = i; \ - } \ - for (int j = 0; j < src_count; j++) { \ - auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ - TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ - auto it = dst_key_index_map.find(key); \ - if (it != dst_key_index_map.end()) { \ - int index = it->second; \ - auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ - PatchMessage(dst_entry, src_entry); \ - } else { \ - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ - } \ - } \ - break; \ - } - - HANDLE_TYPE(INT32, Int32, int32_t); - HANDLE_TYPE(INT64, Int64, int64_t); - HANDLE_TYPE(UINT32, UInt32, uint32_t); - HANDLE_TYPE(UINT64, UInt64, uint64_t); - HANDLE_TYPE(BOOL, Bool, bool); - HANDLE_TYPE(STRING, String, std::string); - default: { - // other types are impossible to be protobuf map key - ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); - break; - } -#undef HANDLE_TYPE - } - } else if (fd->is_repeated()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 - int count = src_reflection->FieldSize(src, fd); - for (int j = 0; j < count; j++) { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ - break; \ - } - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { - const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); - break; - } - } - } - } else { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ - dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ - break; - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: - const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); - PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); - break; - } - } - } - - dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); - return true; -} - -bool BaseOptions::GetIgnoreUnknownFields() const { return ignore_unknown_fields.value_or(false); } -LoadMode BaseOptions::GetMode() const { return mode.value_or(LoadMode::kAll); } -ReadFunc BaseOptions::GetReadFunc() const { return read_func ? read_func : util::ReadFile; } -LoadFunc BaseOptions::GetLoadFunc() const { return load_func ? load_func : LoadMessager; } } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h index b1835560..79508edf 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h @@ -4,9 +4,13 @@ #include #include #include +#include #include #include +#include +#include +#include "tableau/protobuf/tableau.pb.h" #include "util.pc.h" namespace tableau { @@ -19,6 +23,15 @@ enum class LoadMode { struct MessagerOptions; class Hub; +bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt = Format::kJSON, + std::shared_ptr options = nullptr); +bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt = Format::kJSON, + std::shared_ptr options = nullptr); +bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, + tableau::Patch patch, std::shared_ptr options = nullptr); +bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Format fmt, + std::shared_ptr options = nullptr); + // ReadFunc reads the config file and returns its content. using ReadFunc = std::function; // LoadFunc defines a func which can load message's content based on the given @@ -49,10 +62,10 @@ struct BaseOptions { LoadFunc load_func; public: - bool GetIgnoreUnknownFields() const; - LoadMode GetMode() const; - ReadFunc GetReadFunc() const; - LoadFunc GetLoadFunc() const; + inline bool GetIgnoreUnknownFields() const { return ignore_unknown_fields.value_or(false); } + inline LoadMode GetMode() const { return mode.value_or(LoadMode::kAll); } + inline ReadFunc GetReadFunc() const { return read_func ? read_func : util::ReadFile; } + inline LoadFunc GetLoadFunc() const { return load_func ? load_func : LoadMessager; } }; // LoadOptionsOptions is the options struct, which contains both global-level and @@ -62,6 +75,11 @@ struct LoadOptions : public BaseOptions { // If specified, then the messager will be parsed with the given options // directly. std::unordered_map> messager_options; + + public: + // ParseMessagerOptions parses messager options with both global-level and + // messager-level options taken into consideration. + std::shared_ptr ParseMessagerOptionsByName(const std::string& name) const; }; // MessagerOptions defines the options for loading a messager. @@ -98,13 +116,4 @@ class Messager { virtual bool ProcessAfterLoad() { return true; }; Stats stats_; }; - -// ParseMessagerOptions parses messager options with both global-level and -// messager-level options taken into consideration. -std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, - const std::string& name); -bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); -bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc index 8f2e199c..7feb715e 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc @@ -1,29 +1,14 @@ #include "logger.pc.h" -#include -#include -#include -#ifdef _WIN32 -#include -#include -#else -#include -#include -#include -#endif - +#include #include +#include +#include #include #include #include "util.pc.h" -#ifdef _WIN32 -#define gettid() GetCurrentThreadId() -#else -#define gettid() syscall(SYS_gettid) -#endif - namespace tableau { namespace log { @@ -84,42 +69,17 @@ void Logger::Log(const SourceLocation& loc, Level level, const char* format, ... } void DefaultWrite(std::ostream* os, const SourceLocation& loc, const LevelInfo& lvl, const std::string& content) { - // clang-format off - *os << NowStr() << "|" - // << std::this_thread::get_id() << "|" - << gettid() << "|" - << lvl.name << "|" - << loc.filename << ":" << loc.line << "|" - << loc.funcname << "|" - << content - << std::endl << std::flush; - // clang-format on + *os << NowStr() << "|" << lvl.name << "|" << loc.filename << ":" << loc.line << "|" << loc.funcname << "|" << content + << std::endl + << std::flush; } -const char* NowStr() { - static char fmt[64], buf[64]; - struct tm tm; - -#ifdef _WIN32 - SYSTEMTIME wtm; - GetLocalTime(&wtm); - tm.tm_year = wtm.wYear - 1900; - tm.tm_mon = wtm.wMonth - 1; - tm.tm_mday = wtm.wDay; - tm.tm_hour = wtm.wHour; - tm.tm_min = wtm.wMinute; - tm.tm_sec = wtm.wSecond; - unsigned int usec = wtm.wMilliseconds * 1000; -#else - struct timeval tv; - gettimeofday(&tv, NULL); - localtime_r(&tv.tv_sec, &tm); - unsigned int usec = tv.tv_usec; -#endif - - strftime(fmt, sizeof fmt, "%Y-%m-%d %H:%M:%S.%%06u", &tm); - snprintf(buf, sizeof buf, fmt, usec); - return buf; +std::ostream& operator<<(std::ostream& os, const NowStr&) { + auto now = std::chrono::system_clock::now(); + auto now_time_t = std::chrono::system_clock::to_time_t(now); + auto now_us = std::chrono::duration_cast(now.time_since_epoch()) % 1000000; + return os << std::put_time(std::localtime(&now_time_t), "%F %T") << "." << std::setw(6) << std::setfill('0') + << now_us.count(); } } // namespace log diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h index ca212403..c3618469 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h @@ -64,7 +64,9 @@ class Logger { Writer writer_; }; -const char* NowStr(); +class NowStr {}; +std::ostream& operator<<(std::ostream&, const NowStr&); + Logger* DefaultLogger(); void SetDefaultLogger(Logger* logger); diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc index ced3cad7..17c649e0 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc @@ -1,15 +1,9 @@ #include "util.pc.h" -#include -#include - -#include #include -#include -#include -#include "load.pc.h" #include "logger.pc.h" +#include "tableau/protobuf/tableau.pb.h" namespace tableau { static thread_local std::string g_err_msg; @@ -58,30 +52,157 @@ const std::string& Format2Ext(Format fmt) { } } -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, bool ignore_unknown_fields) { - google::protobuf::util::JsonParseOptions parse_options; - parse_options.ignore_unknown_fields = ignore_unknown_fields; - auto status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); - if (!status.ok()) { - SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kJSONExt + ": " + status.ToString()); +#ifdef _WIN32 +#undef GetMessage +#endif + +// PatchMessage patches src into dst, which must be a message with the same descriptor. +// +// # Default PatchMessage mechanism +// - scalar: Populated scalar fields in src are copied to dst. +// - message: Populated singular messages in src are merged into dst by +// recursively calling [xproto.PatchMessage], or replace dst message if +// "PATCH_REPLACE" is specified for this field. +// - list: The elements of every list field in src are appended to the +// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" +// is specified for this field. +// - map: The entries of every map field in src are MERGED (different from +// the behavior of proto.Merge) into the corresponding map field in dst, +// or replace dst map if "PATCH_REPLACE" is specified for this field. +// - unknown: The unknown fields of src are appended to the unknown +// fields of dst (TODO: untested). +// +// # References: +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { + const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); + const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); + // Ensure both messages are of the same type + if (dst_descriptor != src_descriptor) { + SetErrMsg("dst and src are not messages with the same descriptor"); + ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), + src_descriptor->name().c_str()); return false; } - return true; -} -bool Text2Message(const std::string& text, google::protobuf::Message& msg) { - if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) { - SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kTextExt); - return false; + // Get the reflection and descriptor for the messages + const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); + const google::protobuf::Reflection* src_reflection = src.GetReflection(); + + // List all populated fields + std::vector fields; + src_reflection->ListFields(src, &fields); + + // Iterates over every populated field. + for (auto fd : fields) { + const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); + tableau::Patch patch = opts.prop().patch(); + if (patch == tableau::PATCH_REPLACE) { + dst_reflection->ClearField(&dst, fd); + } + if (fd->is_map()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 + auto key_fd = fd->message_type()->map_key(); + int src_count = src_reflection->FieldSize(src, fd); + int dst_count = dst_reflection->FieldSize(dst, fd); + switch (key_fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + std::unordered_map dst_key_index_map; \ + for (int i = 0; i < dst_count; i++) { \ + auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ + TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ + dst_key_index_map[key] = i; \ + } \ + for (int j = 0; j < src_count; j++) { \ + auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ + TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ + auto it = dst_key_index_map.find(key); \ + if (it != dst_key_index_map.end()) { \ + int index = it->second; \ + auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ + PatchMessage(dst_entry, src_entry); \ + } else { \ + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ + } \ + } \ + break; \ } - return true; -} -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) { - if (!msg.ParseFromString(bin)) { - SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kBinExt); - return false; + HANDLE_TYPE(INT32, Int32, int32_t); + HANDLE_TYPE(INT64, Int64, int64_t); + HANDLE_TYPE(UINT32, UInt32, uint32_t); + HANDLE_TYPE(UINT64, UInt64, uint64_t); + HANDLE_TYPE(BOOL, Bool, bool); + HANDLE_TYPE(STRING, String, std::string); + default: { + // other types are impossible to be protobuf map key + ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); + break; + } +#undef HANDLE_TYPE + } + } else if (fd->is_repeated()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 + int count = src_reflection->FieldSize(src, fd); + for (int j = 0; j < count; j++) { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ + break; \ + } + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); + break; + } + } + } + } else { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ + dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ + break; + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); + PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); + break; + } + } } + + dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); return true; } diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h index 78c5b269..a69b86a6 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h @@ -3,11 +3,8 @@ #include #include -#include #include -#include "tableau/protobuf/tableau.pb.h" - namespace tableau { const std::string& GetErrMsg(); void SetErrMsg(const std::string& msg); @@ -57,9 +54,8 @@ Format GetFormat(const std::filesystem::path& path); // and the error message can be obtained by GetErrMsg(). const std::string& Format2Ext(Format fmt); -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, bool ignore_unknown_fields); -bool Text2Message(const std::string& text, google::protobuf::Message& msg); -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); +// PatchMessage patches src into dst, which must be a message with the same descriptor. +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg); diff --git a/cmd/protoc-gen-cpp-tableau-loader/hub.go b/cmd/protoc-gen-cpp-tableau-loader/hub.go index 5128f855..2fbaecc6 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/hub.go +++ b/cmd/protoc-gen-cpp-tableau-loader/hub.go @@ -141,6 +141,7 @@ func generateHubCppRegistry(g *protogen.GeneratedFile, protofiles []string, file const hubHpp = `#pragma once #include #include +#include #include #include #include @@ -349,10 +350,11 @@ std::shared_ptr Hub::InternalLoad(const std::filesystem::path& dir, // intercept protobuf error logs auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); auto msger_map = NewMessagerMap(); + options = options ? options : std::make_shared(); for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); - auto mopts = ParseMessagerOptions(options, name); + auto mopts = options->ParseMessagerOptionsByName(name); bool ok = iter.second->Load(dir, fmt, mopts); if (!ok) { ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str()); @@ -371,7 +373,7 @@ std::shared_ptr Hub::InternalLoad(const std::filesystem::path& dir, std::shared_ptr Hub::NewMessagerMap() const { std::shared_ptr msger_map = std::make_shared(); for (auto&& it : Registry::registrar) { - if (options_ == nullptr || options_->filter == nullptr || options_->filter(it.first)) { + if (!options_ || !options_->filter || options_->filter(it.first)) { (*msger_map)[it.first] = it.second(); } } @@ -391,7 +393,7 @@ const std::shared_ptr Hub::GetMessager(const std::string& name) const } std::shared_ptr Hub::GetMessagerContainerWithProvider() const { - if (options_ != nullptr && options_->provider != nullptr) { + if (options_ && options_->provider ) { return options_->provider(*this); } return msger_container_; @@ -418,8 +420,7 @@ std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainerWithProv const msgContainerCpp = ` MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) - : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), - last_loaded_time_(std::time(nullptr)) {` + : msger_map_(msger_map ? msger_map : std::make_shared()), last_loaded_time_(std::time(nullptr)) {` const registryCpp = `} diff --git a/cmd/protoc-gen-go-tableau-loader/hub.go b/cmd/protoc-gen-go-tableau-loader/hub.go index d2e6147d..46f0f08a 100644 --- a/cmd/protoc-gen-go-tableau-loader/hub.go +++ b/cmd/protoc-gen-go-tableau-loader/hub.go @@ -292,7 +292,7 @@ func (h *Hub) Load(dir string, format format.Format, options ...load.LoadOption) messagerMap := h.NewMessagerMap() opts := load.ParseOptions(options...) for name, msger := range messagerMap { - mopts := load.ParseMessagerOptions(opts, name) + mopts := opts.ParseMessagerOptionsByName(name) if err := msger.Load(dir, format, mopts); err != nil { return errors.WithMessagef(err, "failed to load: %v", name) } diff --git a/go.mod b/go.mod index 7e08fade..36b866e4 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 github.com/stretchr/testify v1.10.0 - github.com/tableauio/tableau v0.14.2-0.20250819090353-59b7724f572d + github.com/tableauio/tableau v0.14.2-0.20250819113156-e34406b9af4d golang.org/x/exp v0.0.0-20230418202329-0354be287a23 google.golang.org/protobuf v1.34.2 ) diff --git a/go.sum b/go.sum index c22a07ed..9b2d12df 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subchen/go-xmldom v1.1.2 h1:7evI2YqfYYOnuj+PBwyaOZZYjl3iWq35P6KfBUw9jeU= github.com/subchen/go-xmldom v1.1.2/go.mod h1:6Pg/HuX5/T4Jlj0IPJF1sRxKVoI/rrKP6LIMge9d5/8= -github.com/tableauio/tableau v0.14.2-0.20250819090353-59b7724f572d h1:L6jKadyW0Almj3CvAsmqWn+mQ3urG0q/vXr4EVOWu/4= -github.com/tableauio/tableau v0.14.2-0.20250819090353-59b7724f572d/go.mod h1:EitY0lyJa8gP6tBkzWhS2WvYhC8hyWPGXoR4QPQ+Qmw= +github.com/tableauio/tableau v0.14.2-0.20250819113156-e34406b9af4d h1:cf2SWF+D0Zag80CbNh5e/2+QMRltnrDwiNwQzCss+no= +github.com/tableauio/tableau v0.14.2-0.20250819113156-e34406b9af4d/go.mod h1:EitY0lyJa8gP6tBkzWhS2WvYhC8hyWPGXoR4QPQ+Qmw= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 h1:6932x8ltq1w4utjmfMPVj09jdMlkY0aiA6+Skbtl3/c= diff --git a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.cpp b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.cpp index 86600409..c9bdd2d1 100644 --- a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.cpp +++ b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.cpp @@ -1,15 +1,17 @@ #include "hub/custom/item/custom_item_conf.h" +#include "logger.pc.h" + const std::string CustomItemConf::kCustomName = "CustomItemConf"; bool CustomItemConf::ProcessAfterLoadAll(const tableau::Hub& hub) { auto conf = hub.Get(1); if (!conf) { - std::cout << "hub get item 1 failed!" << std::endl; + ATOM_ERROR("hub get item 1 failed!"); return false; } special_item_conf_ = *conf; // value copy - std::cout << "custom item conf processed" << std::endl; + ATOM_DEBUG("custom item conf processed"); return true; } diff --git a/test/cpp-tableau-loader/src/hub/hub.cpp b/test/cpp-tableau-loader/src/hub/hub.cpp index 1df497f7..e19fb525 100644 --- a/test/cpp-tableau-loader/src/hub/hub.cpp +++ b/test/cpp-tableau-loader/src/hub/hub.cpp @@ -5,16 +5,9 @@ void LogWrite(std::ostream* os, const tableau::log::SourceLocation& loc, const tableau::log::LevelInfo& lvl, const std::string& content) { - // clang-format off - *os << tableau::log::NowStr() << " " - // << std::this_thread::get_id() << "|" - // << gettid() << " " - << lvl.name << " [" - << loc.filename << ":" << loc.line << "][" - << loc.funcname << "]" - << content - << std::endl << std::flush; - // clang-format on + *os << tableau::log::NowStr() << " " << lvl.name << " [" << loc.filename << ":" << loc.line << "][" << loc.funcname + << "]" << content << std::endl + << std::flush; } bool DefaultFilter(const std::string& name) { diff --git a/test/cpp-tableau-loader/src/main.cpp b/test/cpp-tableau-loader/src/main.cpp index f3a310ab..2fbc0588 100644 --- a/test/cpp-tableau-loader/src/main.cpp +++ b/test/cpp-tableau-loader/src/main.cpp @@ -32,68 +32,68 @@ bool TestPatch() { options->read_func = CustomReadFile; // patchconf - std::cout << "-----TestPatch patchconf" << std::endl; + ATOM_DEBUG("-----TestPatch patchconf"); options->patch_dirs = {"../../testdata/patchconf/"}; bool ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "failed to load with patchconf" << std::endl; + ATOM_ERROR("failed to load with patchconf"); return false; } // print recursive patch conf auto mgr = Hub::Instance().Get(); if (!mgr) { - std::cout << "protobuf hub get RecursivePatchConf failed!" << std::endl; + ATOM_ERROR("protobuf hub get RecursivePatchConf failed!"); return false; } - std::cout << "RecursivePatchConf: " << std::endl << mgr->Data().ShortDebugString() << std::endl; + ATOM_DEBUG("RecursivePatchConf: %s", mgr->Data().ShortDebugString().c_str()); tableau::RecursivePatchConf result; ok = result.Load("../../testdata/patchresult/", tableau::Format::kJSON); if (!ok) { - std::cout << "failed to load with patch result" << std::endl; + ATOM_ERROR("failed to load with patch result"); return false; } - std::cout << "Expected patch result: " << std::endl << result.Data().ShortDebugString() << std::endl; + ATOM_DEBUG("Expected patch result: %s", result.Data().ShortDebugString().c_str()); if (!google::protobuf::util::MessageDifferencer::Equals(mgr->Data(), result.Data())) { - std::cout << "patch result not correct" << std::endl; + ATOM_ERROR("patch result not correct"); return false; } // patchconf2 - std::cout << "-----TestPatch patchconf2" << std::endl; + ATOM_DEBUG("-----TestPatch patchconf2"); options->patch_dirs = {"../../testdata/patchconf2/"}; ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "failed to load with patchconf2" << std::endl; + ATOM_ERROR("failed to load with patchconf2"); return false; } // patchconf2 different format - std::cout << "-----TestPatch patchconf2 different format" << std::endl; + ATOM_DEBUG("-----TestPatch patchconf2 different format"); options->patch_dirs = {"../../testdata/patchconf2/"}; auto mopts = std::make_shared(); mopts->patch_paths = {"../../testdata/patchconf2/PatchMergeConf.txt"}; options->messager_options["PatchMergeConf"] = mopts; ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "failed to load with patchconf2" << std::endl; + ATOM_ERROR("failed to load with patchconf2"); return false; } // multiple patch files - std::cout << "-----TestPatch multiple patch files" << std::endl; + ATOM_DEBUG("-----TestPatch multiple patch files"); mopts = std::make_shared(); mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", "../../testdata/patchconf2/PatchMergeConf.json"}; options->messager_options["PatchMergeConf"] = mopts; ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "failed to load with multiple patch files" << std::endl; + ATOM_ERROR("failed to load with multiple patch files"); return false; } // mode only main - std::cout << "-----TestPatch ModeOnlyMain" << std::endl; + ATOM_DEBUG("-----TestPatch ModeOnlyMain"); mopts = std::make_shared(); mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", "../../testdata/patchconf2/PatchMergeConf.json"}; @@ -101,18 +101,18 @@ bool TestPatch() { options->mode = tableau::LoadMode::kOnlyMain; ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "failed to load with mode only main" << std::endl; + ATOM_ERROR("failed to load with mode only main"); return false; } auto patch_mgr = Hub::Instance().Get(); if (!patch_mgr) { - std::cout << "protobuf hub get PatchMergeConf failed!" << std::endl; - return 1; + ATOM_ERROR("protobuf hub get PatchMergeConf failed!"); + return false; } - std::cout << "PatchMergeConf: " << patch_mgr->Data().ShortDebugString() << std::endl; + ATOM_DEBUG("PatchMergeConf: %s", patch_mgr->Data().ShortDebugString().c_str()); // mode only patch - std::cout << "-----TestPatch ModeOnlyPatch" << std::endl; + ATOM_DEBUG("-----TestPatch ModeOnlyPatch"); mopts = std::make_shared(); mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", "../../testdata/patchconf2/PatchMergeConf.json"}; @@ -120,7 +120,7 @@ bool TestPatch() { options->mode = tableau::LoadMode::kOnlyPatch; ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "failed to load with mode only patch" << std::endl; + ATOM_ERROR("failed to load with mode only patch"); return false; } return true; @@ -137,7 +137,7 @@ int main() { bool ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "protobuf hub load failed: " << tableau::GetErrMsg() << std::endl; + ATOM_ERROR("protobuf hub load failed: %s", tableau::GetErrMsg().c_str()); return 1; } auto msger_map = Hub::Instance().GetMessagerMap(); @@ -148,19 +148,19 @@ int main() { auto item_mgr = Hub::Instance().Get(); if (!item_mgr) { - std::cout << "protobuf hub get Item failed!" << std::endl; + ATOM_ERROR("protobuf hub get Item failed!"); return 1; } // std::cout << "item1: " << item_mgr->Data().DebugString() << std::endl; - std::cout << "-----Index: multi-column index test" << std::endl; + ATOM_DEBUG("-----Index: multi-column index test"); tableau::ItemConf::Index_AwardItemKey key{1, "apple"}; auto item = item_mgr->FindFirstAwardItem(key); if (!item) { - std::cout << "ItemConf FindFirstAwardItem failed!" << std::endl; + ATOM_ERROR("ItemConf FindFirstAwardItem failed!"); return 1; } - std::cout << "item: " << item->ShortDebugString() << std::endl; + ATOM_DEBUG("item: %s", item->ShortDebugString().c_str()); // auto activity_conf = Hub::Instance().Get(); // if (!activity_conf) { @@ -187,67 +187,67 @@ int main() { Hub::Instance().GetOrderedMap( 100001); if (!chapter_ordered_map) { - std::cout << "ActivityConf GetOrderedMap chapter failed!" << std::endl; + ATOM_ERROR("ActivityConf GetOrderedMap chapter failed!"); return 1; } for (auto&& it : *chapter_ordered_map) { - std::cout << "---" << it.first << "-----section_ordered_map" << std::endl; + ATOM_DEBUG("---%d-----section_ordered_map", it.first); for (auto&& kv : it.second.first) { - std::cout << kv.first << std::endl; + ATOM_DEBUG("%d", kv.first); } - std::cout << "---" << it.first << " -----section_map" << std::endl; + ATOM_DEBUG("---%d-----section_map", it.first); for (auto&& kv : it.second.second->section_map()) { - std::cout << kv.first << std::endl; + ATOM_DEBUG("%d", kv.first); } - std::cout << "chapter_id: " << it.second.second->chapter_id() << std::endl; - std::cout << "chapter_name: " << it.second.second->chapter_name() << std::endl; - std::cout << "award_id:" << it.second.second->award_id() << std::endl; + ATOM_DEBUG("chapter_id: %d", it.second.second->chapter_id()); + ATOM_DEBUG("chapter_name: %s", it.second.second->chapter_name().c_str()); + ATOM_DEBUG("award_id: %d", it.second.second->award_id()); } const auto* rank_ordered_map = Hub::Instance().GetOrderedMap(100001, 1, 2); if (!rank_ordered_map) { - std::cout << "ActivityConf GetOrderedMap rank failed!" << std::endl; + ATOM_ERROR("ActivityConf GetOrderedMap rank failed!"); return 1; } - std::cout << "-----rank_ordered_map" << std::endl; + ATOM_DEBUG("-----rank_ordered_map"); for (auto&& it : *rank_ordered_map) { - std::cout << it.first << std::endl; + ATOM_DEBUG("%d", it.first); } auto activity_conf = Hub::Instance().Get(); if (!activity_conf) { - std::cout << "protobuf hub get ActivityConf failed!" << std::endl; + ATOM_ERROR("protobuf hub get ActivityConf failed!"); return 1; } - std::cout << "-----Index accessers test" << std::endl; + ATOM_DEBUG("-----Index accessers test"); auto index_chapters = activity_conf->FindChapter(1); if (!index_chapters) { - std::cout << "ActivityConf FindChapter failed!" << std::endl; + ATOM_ERROR("ActivityConf FindChapter failed!"); return 1; } - std::cout << "-----FindChapter" << std::endl; + ATOM_DEBUG("-----FindChapter"); for (auto&& chapter : *index_chapters) { - std::cout << chapter->ShortDebugString() << std::endl; + ATOM_DEBUG("%s", chapter->ShortDebugString().c_str()); } auto index_first_chapter = activity_conf->FindFirstChapter(1); if (!index_first_chapter) { - std::cout << "ActivityConf FindFirstChapter failed!" << std::endl; + ATOM_ERROR("ActivityConf FindFirstChapter failed!"); return 1; } - std::cout << "-----FindFirstChapter" << std::endl; - std::cout << index_first_chapter->ShortDebugString() << std::endl; + ATOM_DEBUG("-----FindFirstChapter"); + ATOM_DEBUG("%s", index_first_chapter->ShortDebugString().c_str()); - std::cout << "specialItemName: " << Hub::Instance().Get()->GetSpecialItemName() << std::endl; + ATOM_DEBUG("specialItemName: %s", Hub::Instance().Get()->GetSpecialItemName().c_str()); if (!TestPatch()) { - std::cerr << "TestPatch failed!" << std::endl; + ATOM_ERROR("TestPatch failed!"); return 1; } return 0; diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index 62fe81f8..afb799e5 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -59,10 +59,11 @@ std::shared_ptr Hub::InternalLoad(const std::filesystem::path& dir, // intercept protobuf error logs auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); auto msger_map = NewMessagerMap(); + options = options ? options : std::make_shared(); for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); - auto mopts = ParseMessagerOptions(options, name); + auto mopts = options->ParseMessagerOptionsByName(name); bool ok = iter.second->Load(dir, fmt, mopts); if (!ok) { ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str()); @@ -81,7 +82,7 @@ std::shared_ptr Hub::InternalLoad(const std::filesystem::path& dir, std::shared_ptr Hub::NewMessagerMap() const { std::shared_ptr msger_map = std::make_shared(); for (auto&& it : Registry::registrar) { - if (options_ == nullptr || options_->filter == nullptr || options_->filter(it.first)) { + if (!options_ || !options_->filter || options_->filter(it.first)) { (*msger_map)[it.first] = it.second(); } } @@ -101,7 +102,7 @@ const std::shared_ptr Hub::GetMessager(const std::string& name) const } std::shared_ptr Hub::GetMessagerContainerWithProvider() const { - if (options_ != nullptr && options_->provider != nullptr) { + if (options_ && options_->provider ) { return options_->provider(*this); } return msger_container_; @@ -127,8 +128,7 @@ bool Hub::Postprocess(std::shared_ptr msger_map) { std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainerWithProvider()->last_loaded_time_; } MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) - : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), - last_loaded_time_(std::time(nullptr)) { + : msger_map_(msger_map ? msger_map : std::make_shared()), last_loaded_time_(std::time(nullptr)) { InitShard0(); InitShard1(); } diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index da0d6771..714498d0 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -6,6 +6,7 @@ #pragma once #include #include +#include #include #include #include diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc index 30aa36e1..82036607 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -5,45 +5,74 @@ #include "load.pc.h" -#include +#include +#include #include "logger.pc.h" #include "util.pc.h" namespace tableau { - -// Forward declaration of the PatchMessage function -bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); - -std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, - const std::string& name) { +std::shared_ptr LoadOptions::ParseMessagerOptionsByName(const std::string& name) const { std::shared_ptr mopts = std::make_shared(); - if (!opts) { - return mopts; - } - if (auto iter = opts->messager_options.find(name); iter != opts->messager_options.end() && iter->second) { + if (auto iter = messager_options.find(name); iter != messager_options.end() && iter->second) { mopts = std::make_shared(*iter->second); } if (!mopts->ignore_unknown_fields.has_value()) { - mopts->ignore_unknown_fields = opts->ignore_unknown_fields; + mopts->ignore_unknown_fields = ignore_unknown_fields; } if (mopts->patch_dirs.empty()) { - mopts->patch_dirs = opts->patch_dirs; + mopts->patch_dirs = patch_dirs; } if (!mopts->mode.has_value()) { - mopts->mode = opts->mode; + mopts->mode = mode; } - if (mopts->read_func == nullptr) { - mopts->read_func = opts->read_func; + if (!mopts->read_func) { + mopts->read_func = read_func; } - if (mopts->load_func == nullptr) { - mopts->load_func = opts->load_func; + if (!mopts->load_func) { + mopts->load_func = load_func; } return mopts; } +bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, + std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); + std::string content; + bool ok = options->GetReadFunc()(path, content); + if (!ok) { + return false; + } + return Unmarshal(content, msg, fmt, options); +} + +bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt, + std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); + const std::string& name = msg.GetDescriptor()->name(); + std::filesystem::path path; + if (!options->path.empty()) { + // path specified in Paths, then use it instead of dir. + path = options->path; + fmt = util::GetFormat(path); + } + if (path.empty()) { + std::filesystem::path filename = name + util::Format2Ext(fmt); + path = dir / filename; + } + + const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); + // access the extension directly using the generated identifier + const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet); + if (worksheet_options.patch() != tableau::PATCH_NONE) { + return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); + } + return options->GetLoadFunc()(msg, path, fmt, options); +} + bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, tableau::Patch patch, std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); auto mode = options->GetMode(); auto load_func = options->GetLoadFunc(); if (mode == LoadMode::kOnlyMain) { @@ -101,7 +130,7 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem if (!load_func(*patch_msg_ptr, patch_path, util::GetFormat(patch_path), options)) { return false; } - if (!PatchMessage(msg, *patch_msg_ptr)) { + if (!util::PatchMessage(msg, *patch_msg_ptr)) { return false; } } @@ -117,23 +146,33 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem return true; } -bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, - std::shared_ptr options /* = nullptr*/) { +bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Format fmt, + std::shared_ptr options /* = nullptr*/) { options = options ? options : std::make_shared(); - std::string content; - bool ok = options->GetReadFunc()(path, content); - if (!ok) { - return false; - } switch (fmt) { case Format::kJSON: { - return util::JSON2Message(content, msg, options->GetIgnoreUnknownFields()); + google::protobuf::util::JsonParseOptions parse_options; + parse_options.ignore_unknown_fields = options->GetIgnoreUnknownFields(); + auto status = google::protobuf::util::JsonStringToMessage(content, &msg, parse_options); + if (!status.ok()) { + SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kJSONExt + ": " + status.ToString()); + return false; + } + return true; } case Format::kText: { - return util::Text2Message(content, msg); + if (!google::protobuf::TextFormat::ParseFromString(content, &msg)) { + SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kTextExt); + return false; + } + return true; } case Format::kBin: { - return util::Bin2Message(content, msg); + if (!msg.ParseFromString(content)) { + SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kBinExt); + return false; + } + return true; } default: { SetErrMsg("unknown format: " + std::to_string(static_cast(fmt))); @@ -141,187 +180,4 @@ bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& p } } } - -bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt, - std::shared_ptr options /* = nullptr*/) { - options = options ? options : std::make_shared(); - const std::string& name = msg.GetDescriptor()->name(); - std::filesystem::path path; - if (!options->path.empty()) { - // path specified in Paths, then use it instead of dir. - path = options->path; - fmt = util::GetFormat(path); - } - if (path.empty()) { - std::filesystem::path filename = name + util::Format2Ext(fmt); - path = dir / filename; - } - - const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); - // access the extension directly using the generated identifier - const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet); - if (worksheet_options.patch() != tableau::PATCH_NONE) { - return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); - } - return options->GetLoadFunc()(msg, path, fmt, options); -} - -#ifdef _WIN32 -#undef GetMessage -#endif - -// PatchMessage patches src into dst, which must be a message with the same descriptor. -// -// # Default PatchMessage mechanism -// - scalar: Populated scalar fields in src are copied to dst. -// - message: Populated singular messages in src are merged into dst by -// recursively calling [xproto.PatchMessage], or replace dst message if -// "PATCH_REPLACE" is specified for this field. -// - list: The elements of every list field in src are appended to the -// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" -// is specified for this field. -// - map: The entries of every map field in src are MERGED (different from -// the behavior of proto.Merge) into the corresponding map field in dst, -// or replace dst map if "PATCH_REPLACE" is specified for this field. -// - unknown: The unknown fields of src are appended to the unknown -// fields of dst (TODO: untested). -// -// # References: -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details -bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { - const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); - const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); - // Ensure both messages are of the same type - if (dst_descriptor != src_descriptor) { - SetErrMsg("dst and src are not messages with the same descriptor"); - ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), - src_descriptor->name().c_str()); - return false; - } - - // Get the reflection and descriptor for the messages - const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); - const google::protobuf::Reflection* src_reflection = src.GetReflection(); - - // List all populated fields - std::vector fields; - src_reflection->ListFields(src, &fields); - - // Iterates over every populated field. - for (auto fd : fields) { - const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); - tableau::Patch patch = opts.prop().patch(); - if (patch == tableau::PATCH_REPLACE) { - dst_reflection->ClearField(&dst, fd); - } - if (fd->is_map()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 - auto key_fd = fd->message_type()->map_key(); - int src_count = src_reflection->FieldSize(src, fd); - int dst_count = dst_reflection->FieldSize(dst, fd); - switch (key_fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - std::unordered_map dst_key_index_map; \ - for (int i = 0; i < dst_count; i++) { \ - auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ - TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ - dst_key_index_map[key] = i; \ - } \ - for (int j = 0; j < src_count; j++) { \ - auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ - TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ - auto it = dst_key_index_map.find(key); \ - if (it != dst_key_index_map.end()) { \ - int index = it->second; \ - auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ - PatchMessage(dst_entry, src_entry); \ - } else { \ - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ - } \ - } \ - break; \ - } - - HANDLE_TYPE(INT32, Int32, int32_t); - HANDLE_TYPE(INT64, Int64, int64_t); - HANDLE_TYPE(UINT32, UInt32, uint32_t); - HANDLE_TYPE(UINT64, UInt64, uint64_t); - HANDLE_TYPE(BOOL, Bool, bool); - HANDLE_TYPE(STRING, String, std::string); - default: { - // other types are impossible to be protobuf map key - ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); - break; - } -#undef HANDLE_TYPE - } - } else if (fd->is_repeated()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 - int count = src_reflection->FieldSize(src, fd); - for (int j = 0; j < count; j++) { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ - break; \ - } - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { - const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); - break; - } - } - } - } else { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ - dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ - break; - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: - const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); - PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); - break; - } - } - } - - dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); - return true; -} - -bool BaseOptions::GetIgnoreUnknownFields() const { return ignore_unknown_fields.value_or(false); } -LoadMode BaseOptions::GetMode() const { return mode.value_or(LoadMode::kAll); } -ReadFunc BaseOptions::GetReadFunc() const { return read_func ? read_func : util::ReadFile; } -LoadFunc BaseOptions::GetLoadFunc() const { return load_func ? load_func : LoadMessager; } } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.h b/test/cpp-tableau-loader/src/protoconf/load.pc.h index f196de02..90d06059 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.h @@ -9,9 +9,13 @@ #include #include #include +#include #include #include +#include +#include +#include "tableau/protobuf/tableau.pb.h" #include "util.pc.h" namespace tableau { @@ -24,6 +28,15 @@ enum class LoadMode { struct MessagerOptions; class Hub; +bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt = Format::kJSON, + std::shared_ptr options = nullptr); +bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt = Format::kJSON, + std::shared_ptr options = nullptr); +bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, + tableau::Patch patch, std::shared_ptr options = nullptr); +bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Format fmt, + std::shared_ptr options = nullptr); + // ReadFunc reads the config file and returns its content. using ReadFunc = std::function; // LoadFunc defines a func which can load message's content based on the given @@ -54,10 +67,10 @@ struct BaseOptions { LoadFunc load_func; public: - bool GetIgnoreUnknownFields() const; - LoadMode GetMode() const; - ReadFunc GetReadFunc() const; - LoadFunc GetLoadFunc() const; + inline bool GetIgnoreUnknownFields() const { return ignore_unknown_fields.value_or(false); } + inline LoadMode GetMode() const { return mode.value_or(LoadMode::kAll); } + inline ReadFunc GetReadFunc() const { return read_func ? read_func : util::ReadFile; } + inline LoadFunc GetLoadFunc() const { return load_func ? load_func : LoadMessager; } }; // LoadOptionsOptions is the options struct, which contains both global-level and @@ -67,6 +80,11 @@ struct LoadOptions : public BaseOptions { // If specified, then the messager will be parsed with the given options // directly. std::unordered_map> messager_options; + + public: + // ParseMessagerOptions parses messager options with both global-level and + // messager-level options taken into consideration. + std::shared_ptr ParseMessagerOptionsByName(const std::string& name) const; }; // MessagerOptions defines the options for loading a messager. @@ -103,13 +121,4 @@ class Messager { virtual bool ProcessAfterLoad() { return true; }; Stats stats_; }; - -// ParseMessagerOptions parses messager options with both global-level and -// messager-level options taken into consideration. -std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, - const std::string& name); -bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); -bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/logger.pc.cc b/test/cpp-tableau-loader/src/protoconf/logger.pc.cc index ce98982c..ebd42df7 100644 --- a/test/cpp-tableau-loader/src/protoconf/logger.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/logger.pc.cc @@ -5,30 +5,15 @@ #include "logger.pc.h" -#include -#include -#include -#ifdef _WIN32 -#include -#include -#else -#include -#include -#include -#endif - +#include #include +#include +#include #include #include #include "util.pc.h" -#ifdef _WIN32 -#define gettid() GetCurrentThreadId() -#else -#define gettid() syscall(SYS_gettid) -#endif - namespace tableau { namespace log { @@ -89,42 +74,17 @@ void Logger::Log(const SourceLocation& loc, Level level, const char* format, ... } void DefaultWrite(std::ostream* os, const SourceLocation& loc, const LevelInfo& lvl, const std::string& content) { - // clang-format off - *os << NowStr() << "|" - // << std::this_thread::get_id() << "|" - << gettid() << "|" - << lvl.name << "|" - << loc.filename << ":" << loc.line << "|" - << loc.funcname << "|" - << content - << std::endl << std::flush; - // clang-format on + *os << NowStr() << "|" << lvl.name << "|" << loc.filename << ":" << loc.line << "|" << loc.funcname << "|" << content + << std::endl + << std::flush; } -const char* NowStr() { - static char fmt[64], buf[64]; - struct tm tm; - -#ifdef _WIN32 - SYSTEMTIME wtm; - GetLocalTime(&wtm); - tm.tm_year = wtm.wYear - 1900; - tm.tm_mon = wtm.wMonth - 1; - tm.tm_mday = wtm.wDay; - tm.tm_hour = wtm.wHour; - tm.tm_min = wtm.wMinute; - tm.tm_sec = wtm.wSecond; - unsigned int usec = wtm.wMilliseconds * 1000; -#else - struct timeval tv; - gettimeofday(&tv, NULL); - localtime_r(&tv.tv_sec, &tm); - unsigned int usec = tv.tv_usec; -#endif - - strftime(fmt, sizeof fmt, "%Y-%m-%d %H:%M:%S.%%06u", &tm); - snprintf(buf, sizeof buf, fmt, usec); - return buf; +std::ostream& operator<<(std::ostream& os, const NowStr&) { + auto now = std::chrono::system_clock::now(); + auto now_time_t = std::chrono::system_clock::to_time_t(now); + auto now_us = std::chrono::duration_cast(now.time_since_epoch()) % 1000000; + return os << std::put_time(std::localtime(&now_time_t), "%F %T") << "." << std::setw(6) << std::setfill('0') + << now_us.count(); } } // namespace log diff --git a/test/cpp-tableau-loader/src/protoconf/logger.pc.h b/test/cpp-tableau-loader/src/protoconf/logger.pc.h index 434d4364..28803812 100644 --- a/test/cpp-tableau-loader/src/protoconf/logger.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/logger.pc.h @@ -69,7 +69,9 @@ class Logger { Writer writer_; }; -const char* NowStr(); +class NowStr {}; +std::ostream& operator<<(std::ostream&, const NowStr&); + Logger* DefaultLogger(); void SetDefaultLogger(Logger* logger); diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.cc b/test/cpp-tableau-loader/src/protoconf/util.pc.cc index df0f07b2..05da86b8 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.cc @@ -5,16 +5,10 @@ #include "util.pc.h" -#include -#include - -#include #include -#include -#include -#include "load.pc.h" #include "logger.pc.h" +#include "tableau/protobuf/tableau.pb.h" namespace tableau { static thread_local std::string g_err_msg; @@ -63,30 +57,157 @@ const std::string& Format2Ext(Format fmt) { } } -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, bool ignore_unknown_fields) { - google::protobuf::util::JsonParseOptions parse_options; - parse_options.ignore_unknown_fields = ignore_unknown_fields; - auto status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); - if (!status.ok()) { - SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kJSONExt + ": " + status.ToString()); +#ifdef _WIN32 +#undef GetMessage +#endif + +// PatchMessage patches src into dst, which must be a message with the same descriptor. +// +// # Default PatchMessage mechanism +// - scalar: Populated scalar fields in src are copied to dst. +// - message: Populated singular messages in src are merged into dst by +// recursively calling [xproto.PatchMessage], or replace dst message if +// "PATCH_REPLACE" is specified for this field. +// - list: The elements of every list field in src are appended to the +// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" +// is specified for this field. +// - map: The entries of every map field in src are MERGED (different from +// the behavior of proto.Merge) into the corresponding map field in dst, +// or replace dst map if "PATCH_REPLACE" is specified for this field. +// - unknown: The unknown fields of src are appended to the unknown +// fields of dst (TODO: untested). +// +// # References: +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { + const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); + const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); + // Ensure both messages are of the same type + if (dst_descriptor != src_descriptor) { + SetErrMsg("dst and src are not messages with the same descriptor"); + ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), + src_descriptor->name().c_str()); return false; } - return true; -} -bool Text2Message(const std::string& text, google::protobuf::Message& msg) { - if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) { - SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kTextExt); - return false; + // Get the reflection and descriptor for the messages + const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); + const google::protobuf::Reflection* src_reflection = src.GetReflection(); + + // List all populated fields + std::vector fields; + src_reflection->ListFields(src, &fields); + + // Iterates over every populated field. + for (auto fd : fields) { + const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); + tableau::Patch patch = opts.prop().patch(); + if (patch == tableau::PATCH_REPLACE) { + dst_reflection->ClearField(&dst, fd); + } + if (fd->is_map()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 + auto key_fd = fd->message_type()->map_key(); + int src_count = src_reflection->FieldSize(src, fd); + int dst_count = dst_reflection->FieldSize(dst, fd); + switch (key_fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + std::unordered_map dst_key_index_map; \ + for (int i = 0; i < dst_count; i++) { \ + auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ + TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ + dst_key_index_map[key] = i; \ + } \ + for (int j = 0; j < src_count; j++) { \ + auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ + TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ + auto it = dst_key_index_map.find(key); \ + if (it != dst_key_index_map.end()) { \ + int index = it->second; \ + auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ + PatchMessage(dst_entry, src_entry); \ + } else { \ + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ + } \ + } \ + break; \ } - return true; -} -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) { - if (!msg.ParseFromString(bin)) { - SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kBinExt); - return false; + HANDLE_TYPE(INT32, Int32, int32_t); + HANDLE_TYPE(INT64, Int64, int64_t); + HANDLE_TYPE(UINT32, UInt32, uint32_t); + HANDLE_TYPE(UINT64, UInt64, uint64_t); + HANDLE_TYPE(BOOL, Bool, bool); + HANDLE_TYPE(STRING, String, std::string); + default: { + // other types are impossible to be protobuf map key + ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); + break; + } +#undef HANDLE_TYPE + } + } else if (fd->is_repeated()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 + int count = src_reflection->FieldSize(src, fd); + for (int j = 0; j < count; j++) { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ + break; \ + } + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); + break; + } + } + } + } else { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ + dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ + break; + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); + PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); + break; + } + } } + + dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); return true; } diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.h b/test/cpp-tableau-loader/src/protoconf/util.pc.h index 9fcc8cfb..ab6544b3 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.h @@ -8,11 +8,8 @@ #include #include -#include #include -#include "tableau/protobuf/tableau.pb.h" - namespace tableau { const std::string& GetErrMsg(); void SetErrMsg(const std::string& msg); @@ -62,9 +59,8 @@ Format GetFormat(const std::filesystem::path& path); // and the error message can be obtained by GetErrMsg(). const std::string& Format2Ext(Format fmt); -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, bool ignore_unknown_fields); -bool Text2Message(const std::string& text, google::protobuf::Message& msg); -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); +// PatchMessage patches src into dst, which must be a message with the same descriptor. +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg); diff --git a/test/go-tableau-loader/protoconf/loader/hub.pc.go b/test/go-tableau-loader/protoconf/loader/hub.pc.go index 65f7f4d1..06a2d63d 100644 --- a/test/go-tableau-loader/protoconf/loader/hub.pc.go +++ b/test/go-tableau-loader/protoconf/loader/hub.pc.go @@ -244,7 +244,7 @@ func (h *Hub) Load(dir string, format format.Format, options ...load.LoadOption) messagerMap := h.NewMessagerMap() opts := load.ParseOptions(options...) for name, msger := range messagerMap { - mopts := load.ParseMessagerOptions(opts, name) + mopts := opts.ParseMessagerOptionsByName(name) if err := msger.Load(dir, format, mopts); err != nil { return errors.WithMessagef(err, "failed to load: %v", name) } From 5e1d503917d0fd4c9a91f5a437e16db5943baa19 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Wed, 20 Aug 2025 14:38:00 +0800 Subject: [PATCH 04/18] feat: use namespace load --- .../embed/load.pc.cc | 4 ++- .../embed/load.pc.h | 20 +++++++------- .../embed/logger.pc.cc | 26 ++++++++++++++++--- cmd/protoc-gen-cpp-tableau-loader/hub.go | 16 ++++++------ cmd/protoc-gen-cpp-tableau-loader/messager.go | 4 +-- .../src/hub/custom/item/custom_item_conf.h | 2 +- test/cpp-tableau-loader/src/main.cpp | 20 +++++++------- .../src/protoconf/hero_conf.pc.cc | 4 +-- .../src/protoconf/hero_conf.pc.h | 4 +-- .../src/protoconf/hub.pc.cc | 10 +++---- .../cpp-tableau-loader/src/protoconf/hub.pc.h | 6 ++--- .../src/protoconf/item_conf.pc.cc | 2 +- .../src/protoconf/item_conf.pc.h | 2 +- .../src/protoconf/load.pc.cc | 4 ++- .../src/protoconf/load.pc.h | 20 +++++++------- .../src/protoconf/logger.pc.cc | 26 ++++++++++++++++--- .../src/protoconf/patch_conf.pc.cc | 6 ++--- .../src/protoconf/patch_conf.pc.h | 6 ++--- .../src/protoconf/test_conf.pc.cc | 8 +++--- .../src/protoconf/test_conf.pc.h | 8 +++--- 20 files changed, 121 insertions(+), 77 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc index a29958bc..12841e46 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc @@ -7,7 +7,8 @@ #include "util.pc.h" namespace tableau { -std::shared_ptr LoadOptions::ParseMessagerOptionsByName(const std::string& name) const { +namespace load { +std::shared_ptr Options::ParseMessagerOptionsByName(const std::string& name) const { std::shared_ptr mopts = std::make_shared(); if (auto iter = messager_options.find(name); iter != messager_options.end() && iter->second) { mopts = std::make_shared(*iter->second); @@ -175,4 +176,5 @@ bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Forma } } } +} // namespace load } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h index 79508edf..e622f7fa 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h @@ -14,6 +14,7 @@ #include "util.pc.h" namespace tableau { +namespace load { enum class LoadMode { kAll, // Load all related files kOnlyMain, // Only load the main file @@ -21,7 +22,6 @@ enum class LoadMode { }; struct MessagerOptions; -class Hub; bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt = Format::kJSON, std::shared_ptr options = nullptr); @@ -49,16 +49,13 @@ struct BaseOptions { // Specify the directory paths for config patching. std::vector patch_dirs; // Specify the loading mode for config patching. - // - For LoadOptions, default is LoadMode::kModeAll. - // - For MessagerOptions, inherit from LoadOptions if not set. + // Default is LoadMode::kModeAll. std::optional mode; // You can specify custom read function to read a config file's content. - // - For LoadOptions, default is util::ReadFile. - // - For MessagerOptions, inherit from LoadOptions if not set. + // Default is util::ReadFile. ReadFunc read_func; // You can specify custom load function to load a messager's content. - // - For LoadOptions, default is LoadMessager. - // - For MessagerOptions, inherit from LoadOptions if not set. + // Default is LoadMessager. LoadFunc load_func; public: @@ -68,9 +65,9 @@ struct BaseOptions { inline LoadFunc GetLoadFunc() const { return load_func ? load_func : LoadMessager; } }; -// LoadOptionsOptions is the options struct, which contains both global-level and +// Options is the options struct, which contains both global-level and // messager-level options. -struct LoadOptions : public BaseOptions { +struct Options : public BaseOptions { // messager_options maps each messager name to a MessageOptions. // If specified, then the messager will be parsed with the given options // directly. @@ -92,6 +89,9 @@ struct MessagerOptions : public BaseOptions { // If specified, then main messager will be patched. std::vector patch_paths; }; +} // namespace load + +class Hub; class Messager { public: @@ -105,7 +105,7 @@ class Messager { const Stats& GetStats() { return stats_; } // Load fills message from file in the specified directory and format. virtual bool Load(const std::filesystem::path& dir, Format fmt, - std::shared_ptr options = nullptr) = 0; + std::shared_ptr options = nullptr) = 0; // Message returns the inner message data. virtual const google::protobuf::Message* Message() const { return nullptr; } // callback after all messagers loaded. diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc index 7feb715e..9bd03c6f 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc @@ -1,5 +1,12 @@ #include "logger.pc.h" +#ifdef _WIN32 +#include +#else +#include +#include +#endif + #include #include #include @@ -9,6 +16,12 @@ #include "util.pc.h" +#ifdef _WIN32 +#define gettid() GetCurrentThreadId() +#else +#define gettid() syscall(SYS_gettid) +#endif + namespace tableau { namespace log { @@ -69,9 +82,16 @@ void Logger::Log(const SourceLocation& loc, Level level, const char* format, ... } void DefaultWrite(std::ostream* os, const SourceLocation& loc, const LevelInfo& lvl, const std::string& content) { - *os << NowStr() << "|" << lvl.name << "|" << loc.filename << ":" << loc.line << "|" << loc.funcname << "|" << content - << std::endl - << std::flush; + // clang-format off + *os << NowStr() << "|" + // << std::this_thread::get_id() << "|" + << gettid() << "|" + << lvl.name << "|" + << loc.filename << ":" << loc.line << "|" + << loc.funcname << "|" + << content + << std::endl << std::flush; + // clang-format on } std::ostream& operator<<(std::ostream& os, const NowStr&) { diff --git a/cmd/protoc-gen-cpp-tableau-loader/hub.go b/cmd/protoc-gen-cpp-tableau-loader/hub.go index 2fbaecc6..042c95e7 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/hub.go +++ b/cmd/protoc-gen-cpp-tableau-loader/hub.go @@ -181,13 +181,13 @@ class Hub { /***** Synchronous Loading *****/ // Load fills messages (in MessagerContainer) from files in the specified directory and format. bool Load(const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); + std::shared_ptr options = nullptr); /***** Asynchronous Loading *****/ // Load configs into temp MessagerContainer, and you should call LoopOnce() in you app's main loop, // in order to take the temp MessagerContainer into effect. bool AsyncLoad(const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); + std::shared_ptr options = nullptr); int LoopOnce(); // You'd better initialize the scheduler in the main thread. void InitScheduler(); @@ -215,7 +215,7 @@ class Hub { private: std::shared_ptr InternalLoad(const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr) const; + std::shared_ptr options = nullptr) const; std::shared_ptr NewMessagerMap() const; std::shared_ptr GetMessagerContainerWithProvider() const; const std::shared_ptr GetMessager(const std::string& name) const; @@ -311,7 +311,7 @@ void Hub::InitOnce(std::shared_ptr options) { } bool Hub::Load(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */, - std::shared_ptr options /* = nullptr */) { + std::shared_ptr options /* = nullptr */) { auto msger_map = InternalLoad(dir, fmt, options); if (!msger_map) { return false; @@ -325,7 +325,7 @@ bool Hub::Load(const std::filesystem::path& dir, Format fmt /* = Format::kJSON * } bool Hub::AsyncLoad(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */, - std::shared_ptr options /* = nullptr */) { + std::shared_ptr options /* = nullptr */) { auto msger_map = InternalLoad(dir, fmt, options); if (!msger_map) { return false; @@ -346,11 +346,11 @@ void Hub::InitScheduler() { } std::shared_ptr Hub::InternalLoad(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */, - std::shared_ptr options /* = nullptr */) const { + std::shared_ptr options /* = nullptr */) const { // intercept protobuf error logs auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); auto msger_map = NewMessagerMap(); - options = options ? options : std::make_shared(); + options = options ? options : std::make_shared(); for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); @@ -393,7 +393,7 @@ const std::shared_ptr Hub::GetMessager(const std::string& name) const } std::shared_ptr Hub::GetMessagerContainerWithProvider() const { - if (options_ && options_->provider ) { + if (options_ && options_->provider) { return options_->provider(*this); } return msger_container_; diff --git a/cmd/protoc-gen-cpp-tableau-loader/messager.go b/cmd/protoc-gen-cpp-tableau-loader/messager.go index 8734d486..a653c03e 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/messager.go +++ b/cmd/protoc-gen-cpp-tableau-loader/messager.go @@ -85,7 +85,7 @@ func genHppMessage(g *protogen.GeneratedFile, message *protogen.Message) { g.P("class ", message.Desc.Name(), " : public Messager {") g.P(" public:") g.P(helper.Indent(1), "static const std::string& Name() { return kProtoName; }") - g.P(helper.Indent(1), "virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override;") + g.P(helper.Indent(1), "virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override;") g.P(helper.Indent(1), "const ", cppFullName, "& Data() const { return data_; }") g.P(helper.Indent(1), "const google::protobuf::Message* Message() const override { return &data_; }") g.P() @@ -159,7 +159,7 @@ func genCppMessage(g *protogen.GeneratedFile, message *protogen.Message) { g.P("const std::string ", messagerName, "::kProtoName = ", cppFullName, `::GetDescriptor()->name();`) g.P() - g.P("bool ", messagerName, "::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) {") + g.P("bool ", messagerName, "::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) {") g.P(helper.Indent(1), "tableau::util::TimeProfiler profiler;") g.P(helper.Indent(1), "bool loaded = LoadMessagerInDir(data_, dir, fmt, options);") g.P(helper.Indent(1), "bool ok = loaded ? ProcessAfterLoad() : false;") diff --git a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h index 63bc6ffd..e1f7527e 100644 --- a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h +++ b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h @@ -5,7 +5,7 @@ class CustomItemConf : public tableau::Messager { public: static const std::string& Name() { return kCustomName; } virtual bool Load(const std::filesystem::path&, tableau::Format, - std::shared_ptr options = nullptr) override { + std::shared_ptr options = nullptr) override { return true; } virtual bool ProcessAfterLoadAll(const tableau::Hub& hub) override; diff --git a/test/cpp-tableau-loader/src/main.cpp b/test/cpp-tableau-loader/src/main.cpp index 2fbc0588..649aa714 100644 --- a/test/cpp-tableau-loader/src/main.cpp +++ b/test/cpp-tableau-loader/src/main.cpp @@ -13,7 +13,7 @@ #include "protoconf/patch_conf.pc.h" #include "protoconf/test_conf.pc.h" -bool LoadWithPatch(std::shared_ptr options) { +bool LoadWithPatch(std::shared_ptr options) { return Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); } @@ -28,7 +28,7 @@ bool CustomReadFile(const std::filesystem::path& filename, std::string& content) } bool TestPatch() { - auto options = std::make_shared(); + auto options = std::make_shared(); options->read_func = CustomReadFile; // patchconf @@ -71,7 +71,7 @@ bool TestPatch() { // patchconf2 different format ATOM_DEBUG("-----TestPatch patchconf2 different format"); options->patch_dirs = {"../../testdata/patchconf2/"}; - auto mopts = std::make_shared(); + auto mopts = std::make_shared(); mopts->patch_paths = {"../../testdata/patchconf2/PatchMergeConf.txt"}; options->messager_options["PatchMergeConf"] = mopts; ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); @@ -82,7 +82,7 @@ bool TestPatch() { // multiple patch files ATOM_DEBUG("-----TestPatch multiple patch files"); - mopts = std::make_shared(); + mopts = std::make_shared(); mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", "../../testdata/patchconf2/PatchMergeConf.json"}; options->messager_options["PatchMergeConf"] = mopts; @@ -94,11 +94,11 @@ bool TestPatch() { // mode only main ATOM_DEBUG("-----TestPatch ModeOnlyMain"); - mopts = std::make_shared(); + mopts = std::make_shared(); mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", "../../testdata/patchconf2/PatchMergeConf.json"}; options->messager_options["PatchMergeConf"] = mopts; - options->mode = tableau::LoadMode::kOnlyMain; + options->mode = tableau::load::LoadMode::kOnlyMain; ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { ATOM_ERROR("failed to load with mode only main"); @@ -113,11 +113,11 @@ bool TestPatch() { // mode only patch ATOM_DEBUG("-----TestPatch ModeOnlyPatch"); - mopts = std::make_shared(); + mopts = std::make_shared(); mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", "../../testdata/patchconf2/PatchMergeConf.json"}; options->messager_options["PatchMergeConf"] = mopts; - options->mode = tableau::LoadMode::kOnlyPatch; + options->mode = tableau::load::LoadMode::kOnlyPatch; ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { ATOM_ERROR("failed to load with mode only patch"); @@ -128,10 +128,10 @@ bool TestPatch() { int main() { Hub::Instance().InitOnce(); - auto options = std::make_shared(); + auto options = std::make_shared(); options->ignore_unknown_fields = true; options->patch_dirs = {"../../testdata/patchconf/"}; - auto mopts = std::make_shared(); + auto mopts = std::make_shared(); mopts->path = "../../testdata/conf/ItemConf.json"; options->messager_options["ItemConf"] = mopts; diff --git a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc index 1c5a900e..fee254e7 100644 --- a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc @@ -12,7 +12,7 @@ namespace tableau { const std::string HeroConf::kProtoName = protoconf::HeroConf::GetDescriptor()->name(); -bool HeroConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool HeroConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; @@ -71,7 +71,7 @@ const HeroConf::Hero_Attr_OrderedMap* HeroConf::GetOrderedMap(const std::string& const std::string HeroBaseConf::kProtoName = protoconf::HeroBaseConf::GetDescriptor()->name(); -bool HeroBaseConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool HeroBaseConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; diff --git a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h index 17b5c180..325d1d7b 100644 --- a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h @@ -16,7 +16,7 @@ namespace tableau { class HeroConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::HeroConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -47,7 +47,7 @@ class HeroConf : public Messager { class HeroBaseConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::HeroBaseConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index afb799e5..6364fd43 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -20,7 +20,7 @@ void Hub::InitOnce(std::shared_ptr options) { } bool Hub::Load(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */, - std::shared_ptr options /* = nullptr */) { + std::shared_ptr options /* = nullptr */) { auto msger_map = InternalLoad(dir, fmt, options); if (!msger_map) { return false; @@ -34,7 +34,7 @@ bool Hub::Load(const std::filesystem::path& dir, Format fmt /* = Format::kJSON * } bool Hub::AsyncLoad(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */, - std::shared_ptr options /* = nullptr */) { + std::shared_ptr options /* = nullptr */) { auto msger_map = InternalLoad(dir, fmt, options); if (!msger_map) { return false; @@ -55,11 +55,11 @@ void Hub::InitScheduler() { } std::shared_ptr Hub::InternalLoad(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */, - std::shared_ptr options /* = nullptr */) const { + std::shared_ptr options /* = nullptr */) const { // intercept protobuf error logs auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); auto msger_map = NewMessagerMap(); - options = options ? options : std::make_shared(); + options = options ? options : std::make_shared(); for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); @@ -102,7 +102,7 @@ const std::shared_ptr Hub::GetMessager(const std::string& name) const } std::shared_ptr Hub::GetMessagerContainerWithProvider() const { - if (options_ && options_->provider ) { + if (options_ && options_->provider) { return options_->provider(*this); } return msger_container_; diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index 714498d0..e37e2ba1 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -46,13 +46,13 @@ class Hub { /***** Synchronous Loading *****/ // Load fills messages (in MessagerContainer) from files in the specified directory and format. bool Load(const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); + std::shared_ptr options = nullptr); /***** Asynchronous Loading *****/ // Load configs into temp MessagerContainer, and you should call LoopOnce() in you app's main loop, // in order to take the temp MessagerContainer into effect. bool AsyncLoad(const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); + std::shared_ptr options = nullptr); int LoopOnce(); // You'd better initialize the scheduler in the main thread. void InitScheduler(); @@ -80,7 +80,7 @@ class Hub { private: std::shared_ptr InternalLoad(const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr) const; + std::shared_ptr options = nullptr) const; std::shared_ptr NewMessagerMap() const; std::shared_ptr GetMessagerContainerWithProvider() const; const std::shared_ptr GetMessager(const std::string& name) const; diff --git a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc index 596d6760..140568e3 100644 --- a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc @@ -12,7 +12,7 @@ namespace tableau { const std::string ItemConf::kProtoName = protoconf::ItemConf::GetDescriptor()->name(); -bool ItemConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool ItemConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; diff --git a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h index cd1f85b2..12a6ea69 100644 --- a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h @@ -16,7 +16,7 @@ namespace tableau { class ItemConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::ItemConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc index 82036607..94931aed 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -12,7 +12,8 @@ #include "util.pc.h" namespace tableau { -std::shared_ptr LoadOptions::ParseMessagerOptionsByName(const std::string& name) const { +namespace load { +std::shared_ptr Options::ParseMessagerOptionsByName(const std::string& name) const { std::shared_ptr mopts = std::make_shared(); if (auto iter = messager_options.find(name); iter != messager_options.end() && iter->second) { mopts = std::make_shared(*iter->second); @@ -180,4 +181,5 @@ bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Forma } } } +} // namespace load } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.h b/test/cpp-tableau-loader/src/protoconf/load.pc.h index 90d06059..b8313cb8 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.h @@ -19,6 +19,7 @@ #include "util.pc.h" namespace tableau { +namespace load { enum class LoadMode { kAll, // Load all related files kOnlyMain, // Only load the main file @@ -26,7 +27,6 @@ enum class LoadMode { }; struct MessagerOptions; -class Hub; bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt = Format::kJSON, std::shared_ptr options = nullptr); @@ -54,16 +54,13 @@ struct BaseOptions { // Specify the directory paths for config patching. std::vector patch_dirs; // Specify the loading mode for config patching. - // - For LoadOptions, default is LoadMode::kModeAll. - // - For MessagerOptions, inherit from LoadOptions if not set. + // Default is LoadMode::kModeAll. std::optional mode; // You can specify custom read function to read a config file's content. - // - For LoadOptions, default is util::ReadFile. - // - For MessagerOptions, inherit from LoadOptions if not set. + // Default is util::ReadFile. ReadFunc read_func; // You can specify custom load function to load a messager's content. - // - For LoadOptions, default is LoadMessager. - // - For MessagerOptions, inherit from LoadOptions if not set. + // Default is LoadMessager. LoadFunc load_func; public: @@ -73,9 +70,9 @@ struct BaseOptions { inline LoadFunc GetLoadFunc() const { return load_func ? load_func : LoadMessager; } }; -// LoadOptionsOptions is the options struct, which contains both global-level and +// Options is the options struct, which contains both global-level and // messager-level options. -struct LoadOptions : public BaseOptions { +struct Options : public BaseOptions { // messager_options maps each messager name to a MessageOptions. // If specified, then the messager will be parsed with the given options // directly. @@ -97,6 +94,9 @@ struct MessagerOptions : public BaseOptions { // If specified, then main messager will be patched. std::vector patch_paths; }; +} // namespace load + +class Hub; class Messager { public: @@ -110,7 +110,7 @@ class Messager { const Stats& GetStats() { return stats_; } // Load fills message from file in the specified directory and format. virtual bool Load(const std::filesystem::path& dir, Format fmt, - std::shared_ptr options = nullptr) = 0; + std::shared_ptr options = nullptr) = 0; // Message returns the inner message data. virtual const google::protobuf::Message* Message() const { return nullptr; } // callback after all messagers loaded. diff --git a/test/cpp-tableau-loader/src/protoconf/logger.pc.cc b/test/cpp-tableau-loader/src/protoconf/logger.pc.cc index ebd42df7..27b09460 100644 --- a/test/cpp-tableau-loader/src/protoconf/logger.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/logger.pc.cc @@ -5,6 +5,13 @@ #include "logger.pc.h" +#ifdef _WIN32 +#include +#else +#include +#include +#endif + #include #include #include @@ -14,6 +21,12 @@ #include "util.pc.h" +#ifdef _WIN32 +#define gettid() GetCurrentThreadId() +#else +#define gettid() syscall(SYS_gettid) +#endif + namespace tableau { namespace log { @@ -74,9 +87,16 @@ void Logger::Log(const SourceLocation& loc, Level level, const char* format, ... } void DefaultWrite(std::ostream* os, const SourceLocation& loc, const LevelInfo& lvl, const std::string& content) { - *os << NowStr() << "|" << lvl.name << "|" << loc.filename << ":" << loc.line << "|" << loc.funcname << "|" << content - << std::endl - << std::flush; + // clang-format off + *os << NowStr() << "|" + // << std::this_thread::get_id() << "|" + << gettid() << "|" + << lvl.name << "|" + << loc.filename << ":" << loc.line << "|" + << loc.funcname << "|" + << content + << std::endl << std::flush; + // clang-format on } std::ostream& operator<<(std::ostream& os, const NowStr&) { diff --git a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc index b08fa224..db80e10c 100644 --- a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc @@ -12,7 +12,7 @@ namespace tableau { const std::string PatchReplaceConf::kProtoName = protoconf::PatchReplaceConf::GetDescriptor()->name(); -bool PatchReplaceConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool PatchReplaceConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; @@ -22,7 +22,7 @@ bool PatchReplaceConf::Load(const std::filesystem::path& dir, Format fmt, std::s const std::string PatchMergeConf::kProtoName = protoconf::PatchMergeConf::GetDescriptor()->name(); -bool PatchMergeConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool PatchMergeConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; @@ -40,7 +40,7 @@ const protoconf::Item* PatchMergeConf::Get(uint32_t id) const { const std::string RecursivePatchConf::kProtoName = protoconf::RecursivePatchConf::GetDescriptor()->name(); -bool RecursivePatchConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool RecursivePatchConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; diff --git a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h index 2770cae2..3aaae6ef 100644 --- a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h @@ -16,7 +16,7 @@ namespace tableau { class PatchReplaceConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::PatchReplaceConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -29,7 +29,7 @@ class PatchReplaceConf : public Messager { class PatchMergeConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::PatchMergeConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -44,7 +44,7 @@ class PatchMergeConf : public Messager { class RecursivePatchConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::RecursivePatchConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } diff --git a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc index fac05ce7..59b3e691 100644 --- a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc @@ -12,7 +12,7 @@ namespace tableau { const std::string ActivityConf::kProtoName = protoconf::ActivityConf::GetDescriptor()->name(); -bool ActivityConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool ActivityConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; @@ -240,7 +240,7 @@ const protoconf::Section::SectionItem* ActivityConf::FindFirstAward(uint32_t id) const std::string ChapterConf::kProtoName = protoconf::ChapterConf::GetDescriptor()->name(); -bool ChapterConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool ChapterConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; @@ -258,7 +258,7 @@ const protoconf::ChapterConf::Chapter* ChapterConf::Get(uint64_t id) const { const std::string ThemeConf::kProtoName = protoconf::ThemeConf::GetDescriptor()->name(); -bool ThemeConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool ThemeConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; @@ -288,7 +288,7 @@ const std::string* ThemeConf::Get(const std::string& name, const std::string& pa const std::string TaskConf::kProtoName = protoconf::TaskConf::GetDescriptor()->name(); -bool TaskConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool TaskConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; diff --git a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h index f3226a91..9036b8d5 100644 --- a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h @@ -16,7 +16,7 @@ namespace tableau { class ActivityConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::ActivityConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -103,7 +103,7 @@ class ActivityConf : public Messager { class ChapterConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::ChapterConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -118,7 +118,7 @@ class ChapterConf : public Messager { class ThemeConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::ThemeConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -134,7 +134,7 @@ class ThemeConf : public Messager { class TaskConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::TaskConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } From 4dccd8c1d4e422e90518f2b80603b90837f2a9ed Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Wed, 20 Aug 2025 17:20:44 +0800 Subject: [PATCH 05/18] feat: keep submodules --- .gitattributes | 1 + .gitignore | 1 - README.md | 12 ++++++++++++ cmd/protoc-gen-go-tableau-loader/hub.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- init.bat | 6 ++++-- test/cpp-tableau-loader/gen.bat | 2 +- test/cpp-tableau-loader/gen.sh | 2 +- test/cpp-tableau-loader/src/CMakeLists.txt | 2 +- test/go-tableau-loader/gen.bat | 2 +- test/go-tableau-loader/main.go | 9 ++++++--- test/go-tableau-loader/protoconf/loader/hub.pc.go | 2 +- third_party/_submodules/tableau | 2 +- 14 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..94f480de --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf \ No newline at end of file diff --git a/.gitignore b/.gitignore index 87c614f9..f686f554 100644 --- a/.gitignore +++ b/.gitignore @@ -28,7 +28,6 @@ *.app # custom files and directories -third_party/_submodules _out/ build/ bin/ diff --git a/README.md b/README.md index 23faf9df..c1f06156 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,18 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). - clang: `cmake ../src/ -DCMAKE_CXX_COMPILER=clang++` - Build: `make -j8`, then the **bin** dir will be generated at `test/cpp-tableau-loader/bin`. +### Dev at Windows + +- Install: **CMake 3.22** or above +- Change dir: `cd test\cpp-tableau-loader` +- Generate protoconf: `.\gen.bat` +- Create build dir: `mkdir build 2>nul` and `cd build` +- Run cmake: + - C++17: `cmake ..\src\` + - C++20: `cmake ..\src\ -DCMAKE_CXX_STANDARD=20` + - clang: `cmake ..\src\ -DCMAKE_CXX_COMPILER=clang++` +- Build: `cmake --build .`, then the **bin** dir will be generated at `test\cpp-tableau-loader\bin\Debug`. + ### References - [Protocol Buffers C++ Installation](https://github.com/protocolbuffers/protobuf/tree/master/src) diff --git a/cmd/protoc-gen-go-tableau-loader/hub.go b/cmd/protoc-gen-go-tableau-loader/hub.go index 46f0f08a..7c178dc9 100644 --- a/cmd/protoc-gen-go-tableau-loader/hub.go +++ b/cmd/protoc-gen-go-tableau-loader/hub.go @@ -288,7 +288,7 @@ func (h *Hub) GetMessager(name string) Messager { } // Load fills messages from files in the specified directory and format. -func (h *Hub) Load(dir string, format format.Format, options ...load.LoadOption) error { +func (h *Hub) Load(dir string, format format.Format, options ...load.Option) error { messagerMap := h.NewMessagerMap() opts := load.ParseOptions(options...) for name, msger := range messagerMap { diff --git a/go.mod b/go.mod index 36b866e4..24cffe86 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 github.com/stretchr/testify v1.10.0 - github.com/tableauio/tableau v0.14.2-0.20250819113156-e34406b9af4d + github.com/tableauio/tableau v0.14.2-0.20250820060046-e718142607df golang.org/x/exp v0.0.0-20230418202329-0354be287a23 google.golang.org/protobuf v1.34.2 ) diff --git a/go.sum b/go.sum index 9b2d12df..f898e11e 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subchen/go-xmldom v1.1.2 h1:7evI2YqfYYOnuj+PBwyaOZZYjl3iWq35P6KfBUw9jeU= github.com/subchen/go-xmldom v1.1.2/go.mod h1:6Pg/HuX5/T4Jlj0IPJF1sRxKVoI/rrKP6LIMge9d5/8= -github.com/tableauio/tableau v0.14.2-0.20250819113156-e34406b9af4d h1:cf2SWF+D0Zag80CbNh5e/2+QMRltnrDwiNwQzCss+no= -github.com/tableauio/tableau v0.14.2-0.20250819113156-e34406b9af4d/go.mod h1:EitY0lyJa8gP6tBkzWhS2WvYhC8hyWPGXoR4QPQ+Qmw= +github.com/tableauio/tableau v0.14.2-0.20250820060046-e718142607df h1:rPX0e6cjrmMIjdO8qriF621sSytxXJWnTii7ESfPW58= +github.com/tableauio/tableau v0.14.2-0.20250820060046-e718142607df/go.mod h1:EitY0lyJa8gP6tBkzWhS2WvYhC8hyWPGXoR4QPQ+Qmw= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 h1:6932x8ltq1w4utjmfMPVj09jdMlkY0aiA6+Skbtl3/c= diff --git a/init.bat b/init.bat index fa2c85e5..84dac8c7 100644 --- a/init.bat +++ b/init.bat @@ -13,7 +13,9 @@ git submodule update --init --recursive REM Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) cd cmake -cmake . -cmake --build . +mkdir build 2>nul +cd build +cmake .. -DCMAKE_POLICY_VERSION_MINIMUM="3.5" +cmake --build . --config Debug endlocal \ No newline at end of file diff --git a/test/cpp-tableau-loader/gen.bat b/test/cpp-tableau-loader/gen.bat index 178a00bc..207f7d00 100644 --- a/test/cpp-tableau-loader/gen.bat +++ b/test/cpp-tableau-loader/gen.bat @@ -5,7 +5,7 @@ setlocal enabledelayedexpansion for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i cd /d "%repoRoot%" -set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\Debug\protoc.exe" +set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\Debug\protoc.exe" set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" set "ROOTDIR=%repoRoot%\test\cpp-tableau-loader" diff --git a/test/cpp-tableau-loader/gen.sh b/test/cpp-tableau-loader/gen.sh index 43d83360..83cc14ac 100755 --- a/test/cpp-tableau-loader/gen.sh +++ b/test/cpp-tableau-loader/gen.sh @@ -33,7 +33,7 @@ ${PROTOC} \ --proto_path="$PROTOCONF_IN" \ "$PROTOCONF_IN"/**/*.proto -TABLEAU_IN="./third_party/_submodules/tableau/proto/tableau/protobuf" +TABLEAU_IN="$TABLEAU_PROTO/tableau/protobuf" TABLEAU_OUT="${ROOTDIR}/src" # remove old generated files rm -rfv "$TABLEAU_OUT/tableau" diff --git a/test/cpp-tableau-loader/src/CMakeLists.txt b/test/cpp-tableau-loader/src/CMakeLists.txt index 81402304..e5408936 100644 --- a/test/cpp-tableau-loader/src/CMakeLists.txt +++ b/test/cpp-tableau-loader/src/CMakeLists.txt @@ -45,7 +45,7 @@ SET(COMMON_INCLUDE_DIR # common lib if(MSVC) SET(COMMON_LIB - ${PROTOBUF_ROOT_DIR}/cmake/Debug/libprotobufd.lib + ${PROTOBUF_ROOT_DIR}/cmake/build/Debug/libprotobufd.lib ) SET(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") else() diff --git a/test/go-tableau-loader/gen.bat b/test/go-tableau-loader/gen.bat index 98b8ea36..970aea11 100644 --- a/test/go-tableau-loader/gen.bat +++ b/test/go-tableau-loader/gen.bat @@ -5,7 +5,7 @@ setlocal enabledelayedexpansion for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i cd /d "%repoRoot%" -set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\Debug\protoc.exe" +set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\Debug\protoc.exe" set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" set "PLGUIN_DIR=%repoRoot%\cmd\protoc-gen-go-tableau-loader" diff --git a/test/go-tableau-loader/main.go b/test/go-tableau-loader/main.go index 329c6838..83bb9b7c 100644 --- a/test/go-tableau-loader/main.go +++ b/test/go-tableau-loader/main.go @@ -15,9 +15,12 @@ import ( func main() { err := hub.GetHub().Load("../testdata/conf/", format.JSON, load.IgnoreUnknownFields(), - load.Paths(map[string]string{ - "ItemConf": "../testdata/conf/ItemConf.json", - })) + load.WithMessagerOptions(map[string]*load.MessagerOptions{ + "ItemConf": { + Path: "../testdata/conf/ItemConf.json", + }, + }), + ) if err != nil { panic(err) } diff --git a/test/go-tableau-loader/protoconf/loader/hub.pc.go b/test/go-tableau-loader/protoconf/loader/hub.pc.go index 06a2d63d..899bdd5e 100644 --- a/test/go-tableau-loader/protoconf/loader/hub.pc.go +++ b/test/go-tableau-loader/protoconf/loader/hub.pc.go @@ -240,7 +240,7 @@ func (h *Hub) GetMessager(name string) Messager { } // Load fills messages from files in the specified directory and format. -func (h *Hub) Load(dir string, format format.Format, options ...load.LoadOption) error { +func (h *Hub) Load(dir string, format format.Format, options ...load.Option) error { messagerMap := h.NewMessagerMap() opts := load.ParseOptions(options...) for name, msger := range messagerMap { diff --git a/third_party/_submodules/tableau b/third_party/_submodules/tableau index a00850ec..e7181426 160000 --- a/third_party/_submodules/tableau +++ b/third_party/_submodules/tableau @@ -1 +1 @@ -Subproject commit a00850ecc9b066a9691353ed7fe23d01f67be75e +Subproject commit e718142607dfb52ca2a8c8f240312f62f8379416 From c2c3dcad1ce2377385fca30b0b1ab0cd5a948771 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Wed, 20 Aug 2025 19:41:13 +0800 Subject: [PATCH 06/18] feat: update go.mod --- cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc | 4 ++++ cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h | 2 -- go.mod | 2 +- go.sum | 4 ++-- test/cpp-tableau-loader/src/protoconf/load.pc.cc | 4 ++++ test/cpp-tableau-loader/src/protoconf/load.pc.h | 2 -- third_party/_submodules/tableau | 2 +- 7 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc index 12841e46..88209fee 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc @@ -31,6 +31,10 @@ std::shared_ptr Options::ParseMessagerOptionsByName(const return mopts; } +// Forward declaration +bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, + tableau::Patch patch, std::shared_ptr options = nullptr); + bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, std::shared_ptr options /* = nullptr*/) { options = options ? options : std::make_shared(); diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h index e622f7fa..8554a32c 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h @@ -27,8 +27,6 @@ bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& p std::shared_ptr options = nullptr); bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt = Format::kJSON, std::shared_ptr options = nullptr); -bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, - tableau::Patch patch, std::shared_ptr options = nullptr); bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Format fmt, std::shared_ptr options = nullptr); diff --git a/go.mod b/go.mod index 24cffe86..35e2b4d1 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 github.com/stretchr/testify v1.10.0 - github.com/tableauio/tableau v0.14.2-0.20250820060046-e718142607df + github.com/tableauio/tableau v0.14.2-0.20250820113117-0b0be0ac122e golang.org/x/exp v0.0.0-20230418202329-0354be287a23 google.golang.org/protobuf v1.34.2 ) diff --git a/go.sum b/go.sum index f898e11e..90ba32f1 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subchen/go-xmldom v1.1.2 h1:7evI2YqfYYOnuj+PBwyaOZZYjl3iWq35P6KfBUw9jeU= github.com/subchen/go-xmldom v1.1.2/go.mod h1:6Pg/HuX5/T4Jlj0IPJF1sRxKVoI/rrKP6LIMge9d5/8= -github.com/tableauio/tableau v0.14.2-0.20250820060046-e718142607df h1:rPX0e6cjrmMIjdO8qriF621sSytxXJWnTii7ESfPW58= -github.com/tableauio/tableau v0.14.2-0.20250820060046-e718142607df/go.mod h1:EitY0lyJa8gP6tBkzWhS2WvYhC8hyWPGXoR4QPQ+Qmw= +github.com/tableauio/tableau v0.14.2-0.20250820113117-0b0be0ac122e h1:AJa5W5gNVcCPR+euRvN5NytR9CGE63NkEjG7OsOXzlY= +github.com/tableauio/tableau v0.14.2-0.20250820113117-0b0be0ac122e/go.mod h1:EitY0lyJa8gP6tBkzWhS2WvYhC8hyWPGXoR4QPQ+Qmw= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 h1:6932x8ltq1w4utjmfMPVj09jdMlkY0aiA6+Skbtl3/c= diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc index 94931aed..852f73fc 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -36,6 +36,10 @@ std::shared_ptr Options::ParseMessagerOptionsByName(const return mopts; } +// Forward declaration +bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, + tableau::Patch patch, std::shared_ptr options = nullptr); + bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, std::shared_ptr options /* = nullptr*/) { options = options ? options : std::make_shared(); diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.h b/test/cpp-tableau-loader/src/protoconf/load.pc.h index b8313cb8..2bd4b9ce 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.h @@ -32,8 +32,6 @@ bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& p std::shared_ptr options = nullptr); bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt = Format::kJSON, std::shared_ptr options = nullptr); -bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, - tableau::Patch patch, std::shared_ptr options = nullptr); bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Format fmt, std::shared_ptr options = nullptr); diff --git a/third_party/_submodules/tableau b/third_party/_submodules/tableau index e7181426..0b0be0ac 160000 --- a/third_party/_submodules/tableau +++ b/third_party/_submodules/tableau @@ -1 +1 @@ -Subproject commit e718142607dfb52ca2a8c8f240312f62f8379416 +Subproject commit 0b0be0ac122eb466e4f342451545f8689a1a5285 From 7dfc444fd9b62693636002c7693ab9c2f6db0490 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 21 Aug 2025 10:41:39 +0800 Subject: [PATCH 07/18] feat: revert submodule change --- third_party/_submodules/tableau | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/_submodules/tableau b/third_party/_submodules/tableau index 0b0be0ac..a00850ec 160000 --- a/third_party/_submodules/tableau +++ b/third_party/_submodules/tableau @@ -1 +1 @@ -Subproject commit 0b0be0ac122eb466e4f342451545f8689a1a5285 +Subproject commit a00850ecc9b066a9691353ed7fe23d01f67be75e From 3043e12f1bea2e3e1a8e7d5723623fc9339ce528 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 22 Aug 2025 11:06:36 +0800 Subject: [PATCH 08/18] feat: use Now instead of NowStr --- cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc | 4 ++-- cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h | 4 ++-- test/cpp-tableau-loader/src/hub/hub.cpp | 2 +- test/cpp-tableau-loader/src/protoconf/logger.pc.cc | 4 ++-- test/cpp-tableau-loader/src/protoconf/logger.pc.h | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc index 9bd03c6f..4ecc2908 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc @@ -83,7 +83,7 @@ void Logger::Log(const SourceLocation& loc, Level level, const char* format, ... void DefaultWrite(std::ostream* os, const SourceLocation& loc, const LevelInfo& lvl, const std::string& content) { // clang-format off - *os << NowStr() << "|" + *os << Now() << "|" // << std::this_thread::get_id() << "|" << gettid() << "|" << lvl.name << "|" @@ -94,7 +94,7 @@ void DefaultWrite(std::ostream* os, const SourceLocation& loc, const LevelInfo& // clang-format on } -std::ostream& operator<<(std::ostream& os, const NowStr&) { +std::ostream& operator<<(std::ostream& os, const Now&) { auto now = std::chrono::system_clock::now(); auto now_time_t = std::chrono::system_clock::to_time_t(now); auto now_us = std::chrono::duration_cast(now.time_since_epoch()) % 1000000; diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h index c3618469..33a161b8 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h @@ -64,8 +64,8 @@ class Logger { Writer writer_; }; -class NowStr {}; -std::ostream& operator<<(std::ostream&, const NowStr&); +class Now {}; +std::ostream& operator<<(std::ostream&, const Now&); Logger* DefaultLogger(); void SetDefaultLogger(Logger* logger); diff --git a/test/cpp-tableau-loader/src/hub/hub.cpp b/test/cpp-tableau-loader/src/hub/hub.cpp index e19fb525..03cbf33e 100644 --- a/test/cpp-tableau-loader/src/hub/hub.cpp +++ b/test/cpp-tableau-loader/src/hub/hub.cpp @@ -5,7 +5,7 @@ void LogWrite(std::ostream* os, const tableau::log::SourceLocation& loc, const tableau::log::LevelInfo& lvl, const std::string& content) { - *os << tableau::log::NowStr() << " " << lvl.name << " [" << loc.filename << ":" << loc.line << "][" << loc.funcname + *os << tableau::log::Now() << " " << lvl.name << " [" << loc.filename << ":" << loc.line << "][" << loc.funcname << "]" << content << std::endl << std::flush; } diff --git a/test/cpp-tableau-loader/src/protoconf/logger.pc.cc b/test/cpp-tableau-loader/src/protoconf/logger.pc.cc index 27b09460..747ce28d 100644 --- a/test/cpp-tableau-loader/src/protoconf/logger.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/logger.pc.cc @@ -88,7 +88,7 @@ void Logger::Log(const SourceLocation& loc, Level level, const char* format, ... void DefaultWrite(std::ostream* os, const SourceLocation& loc, const LevelInfo& lvl, const std::string& content) { // clang-format off - *os << NowStr() << "|" + *os << Now() << "|" // << std::this_thread::get_id() << "|" << gettid() << "|" << lvl.name << "|" @@ -99,7 +99,7 @@ void DefaultWrite(std::ostream* os, const SourceLocation& loc, const LevelInfo& // clang-format on } -std::ostream& operator<<(std::ostream& os, const NowStr&) { +std::ostream& operator<<(std::ostream& os, const Now&) { auto now = std::chrono::system_clock::now(); auto now_time_t = std::chrono::system_clock::to_time_t(now); auto now_us = std::chrono::duration_cast(now.time_since_epoch()) % 1000000; diff --git a/test/cpp-tableau-loader/src/protoconf/logger.pc.h b/test/cpp-tableau-loader/src/protoconf/logger.pc.h index 28803812..592c82b7 100644 --- a/test/cpp-tableau-loader/src/protoconf/logger.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/logger.pc.h @@ -69,8 +69,8 @@ class Logger { Writer writer_; }; -class NowStr {}; -std::ostream& operator<<(std::ostream&, const NowStr&); +class Now {}; +std::ostream& operator<<(std::ostream&, const Now&); Logger* DefaultLogger(); void SetDefaultLogger(Logger* logger); From 10ebc2c5be044d6d6855b1153ff689e4ef3d9b2f Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 22 Aug 2025 12:01:24 +0800 Subject: [PATCH 09/18] feat: add notes --- init.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/init.bat b/init.bat index 84dac8c7..5ef018dc 100644 --- a/init.bat +++ b/init.bat @@ -12,10 +12,11 @@ git checkout v3.19.3 git submodule update --init --recursive REM Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) +REM Refer: https://github.com/protocolbuffers/protobuf/blob/3.19.x/cmake/README.md#cmake-configuration cd cmake mkdir build 2>nul cd build cmake .. -DCMAKE_POLICY_VERSION_MINIMUM="3.5" cmake --build . --config Debug -endlocal \ No newline at end of file +endlocal From ecd0db55f3ee6c7f654d041d508da67348c312fe Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 22 Aug 2025 16:13:02 +0800 Subject: [PATCH 10/18] feat: update windows scripts --- README.md | 11 +++++------ init.bat | 12 ++++++++---- test/cpp-tableau-loader/gen.bat | 2 +- test/cpp-tableau-loader/src/CMakeLists.txt | 2 +- test/go-tableau-loader/gen.bat | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c1f06156..a4f75224 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). - Change dir: `cd test/cpp-tableau-loader` - Generate protoconf: `bash ./gen.sh` - Create build dir: `mkdir build && cd build` -- Run cmake: +- Run cmake: - C++17: `cmake ../src/` - C++20: `cmake ../src/ -DCMAKE_CXX_STANDARD=20` - clang: `cmake ../src/ -DCMAKE_CXX_COMPILER=clang++` @@ -30,11 +30,10 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). - Change dir: `cd test\cpp-tableau-loader` - Generate protoconf: `.\gen.bat` - Create build dir: `mkdir build 2>nul` and `cd build` -- Run cmake: - - C++17: `cmake ..\src\` - - C++20: `cmake ..\src\ -DCMAKE_CXX_STANDARD=20` - - clang: `cmake ..\src\ -DCMAKE_CXX_COMPILER=clang++` -- Build: `cmake --build .`, then the **bin** dir will be generated at `test\cpp-tableau-loader\bin\Debug`. +- Run cmake: + - C++17: `cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Debug ..\src\` + - C++20: `cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Debug ..\src\ -DCMAKE_CXX_STANDARD=20` +- Build: `nmake`, then the **bin** dir will be generated at `test\cpp-tableau-loader\bin`. ### References diff --git a/init.bat b/init.bat index 5ef018dc..2ff2e836 100644 --- a/init.bat +++ b/init.bat @@ -14,9 +14,13 @@ git submodule update --init --recursive REM Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) REM Refer: https://github.com/protocolbuffers/protobuf/blob/3.19.x/cmake/README.md#cmake-configuration cd cmake -mkdir build 2>nul -cd build -cmake .. -DCMAKE_POLICY_VERSION_MINIMUM="3.5" -cmake --build . --config Debug +mkdir build\debug 2>nul +REM use Debug configuration +cd build\debug +cmake -G "NMake Makefiles" ^ + -DCMAKE_BUILD_TYPE=Debug ^ + -DCMAKE_POLICY_VERSION_MINIMUM="3.5" ^ + ../.. +nmake endlocal diff --git a/test/cpp-tableau-loader/gen.bat b/test/cpp-tableau-loader/gen.bat index 207f7d00..25df1aca 100644 --- a/test/cpp-tableau-loader/gen.bat +++ b/test/cpp-tableau-loader/gen.bat @@ -5,7 +5,7 @@ setlocal enabledelayedexpansion for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i cd /d "%repoRoot%" -set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\Debug\protoc.exe" +set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\debug\protoc.exe" set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" set "ROOTDIR=%repoRoot%\test\cpp-tableau-loader" diff --git a/test/cpp-tableau-loader/src/CMakeLists.txt b/test/cpp-tableau-loader/src/CMakeLists.txt index e5408936..df4fb5b0 100644 --- a/test/cpp-tableau-loader/src/CMakeLists.txt +++ b/test/cpp-tableau-loader/src/CMakeLists.txt @@ -45,7 +45,7 @@ SET(COMMON_INCLUDE_DIR # common lib if(MSVC) SET(COMMON_LIB - ${PROTOBUF_ROOT_DIR}/cmake/build/Debug/libprotobufd.lib + ${PROTOBUF_ROOT_DIR}/cmake/build/debug/libprotobufd.lib ) SET(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") else() diff --git a/test/go-tableau-loader/gen.bat b/test/go-tableau-loader/gen.bat index 970aea11..78520f73 100644 --- a/test/go-tableau-loader/gen.bat +++ b/test/go-tableau-loader/gen.bat @@ -5,7 +5,7 @@ setlocal enabledelayedexpansion for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i cd /d "%repoRoot%" -set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\Debug\protoc.exe" +set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\debug\protoc.exe" set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" set "PLGUIN_DIR=%repoRoot%\cmd\protoc-gen-go-tableau-loader" From 16436202a3a05b3c5c2cf87c8ffc775647ea6f87 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 22 Aug 2025 17:14:57 +0800 Subject: [PATCH 11/18] feat: improve Windows build process --- .gitattributes | 17 ++- .gitmodules | 2 + README.md | 30 +++-- init.bat | 9 +- test/cpp-tableau-loader/gen.bat | 2 +- test/cpp-tableau-loader/src/CMakeLists.txt | 2 +- .../src/protoconf/scheduler.pc.cc | 106 +++++++++--------- .../src/protoconf/scheduler.pc.h | 60 +++++----- 8 files changed, 126 insertions(+), 102 deletions(-) diff --git a/.gitattributes b/.gitattributes index 94f480de..e6c41c5a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,16 @@ -* text=auto eol=lf \ No newline at end of file +# Treat all files in this repo as binary, with no git magic updating +# line endings. This produces predictable results in different environments. +# +# Windows users contributing to this repo will need to use a modern version +# of git and editors capable of LF line endings. +# +# Windows .bat files are known to have multiple bugs when run with LF +# endings, and so they are checked in with CRLF endings, with a test +# to catch problems. (See https://github.com/golang/go/issues/37791.) +# +# We'll prevent accidental CRLF line endings from entering the repo +# via MR checks and tests. +# +# See https://github.com/golang/go/issues/9281. +# See https://adaptivepatchwork.com/2012/03/01/mind-the-end-of-your-line. +* -text \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 5877cd25..d0be858b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,8 @@ [submodule "third_party/_submodules/protobuf"] path = third_party/_submodules/protobuf url = https://github.com/protocolbuffers/protobuf + ignore = dirty [submodule "third_party/_submodules/tableau"] path = third_party/_submodules/tableau url = https://github.com/tableauio/tableau + ignore = dirty diff --git a/README.md b/README.md index a4f75224..d28acf27 100644 --- a/README.md +++ b/README.md @@ -6,34 +6,40 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). > TODO: [devcontainer](https://code.visualstudio.com/docs/devcontainers/containers) -- Supported OS: Windows, macOS, Linux -- Init protobuf: `bash init.sh` - C++ standard: at least C++17 +- Install: [CMake 3.22](https://github.com/Kitware/CMake/releases/tag/v3.31.8) or above +- Init protobuf: + - macOS or Linux: `bash init.sh` + - Windows: + - Install [Visual Studio 2022](https://visualstudio.microsoft.com/downloads/) + - Environment Setup: Open the appropriate `Developer Command Prompt for VS 2022` from the *Start* menu to ensure `cl.exe` and other build tools are in your `PATH`. + - Change dir to **loader** repo + - Run: `.\init.bat` ## C++ ### Dev at Linux -- Install: **CMake 3.22** or above - Change dir: `cd test/cpp-tableau-loader` - Generate protoconf: `bash ./gen.sh` -- Create build dir: `mkdir build && cd build` -- Run cmake: +- Create build dir: `mkdir build` and `cd build` +- CMake: - C++17: `cmake ../src/` - C++20: `cmake ../src/ -DCMAKE_CXX_STANDARD=20` - clang: `cmake ../src/ -DCMAKE_CXX_COMPILER=clang++` -- Build: `make -j8`, then the **bin** dir will be generated at `test/cpp-tableau-loader/bin`. +- Build: `cmake --build .` +- Run: `../bin/loader` ### Dev at Windows -- Install: **CMake 3.22** or above - Change dir: `cd test\cpp-tableau-loader` - Generate protoconf: `.\gen.bat` -- Create build dir: `mkdir build 2>nul` and `cd build` -- Run cmake: - - C++17: `cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Debug ..\src\` - - C++20: `cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Debug ..\src\ -DCMAKE_CXX_STANDARD=20` -- Build: `nmake`, then the **bin** dir will be generated at `test\cpp-tableau-loader\bin`. +- Create build dir: `mkdir build` and `cd build` +- CMake: + - C++17: `cmake -G "NMake Makefiles" ..\src\` + - C++20: `cmake -G "NMake Makefiles" ..\src\ -DCMAKE_CXX_STANDARD=20` +- Build: `cmake --build .` +- Run: `..\bin\loader.exe` ### References diff --git a/init.bat b/init.bat index 2ff2e836..79dc55a6 100644 --- a/init.bat +++ b/init.bat @@ -14,13 +14,14 @@ git submodule update --init --recursive REM Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) REM Refer: https://github.com/protocolbuffers/protobuf/blob/3.19.x/cmake/README.md#cmake-configuration cd cmake -mkdir build\debug 2>nul +mkdir build 2>nul +cd build REM use Debug configuration -cd build\debug cmake -G "NMake Makefiles" ^ -DCMAKE_BUILD_TYPE=Debug ^ -DCMAKE_POLICY_VERSION_MINIMUM="3.5" ^ - ../.. -nmake + .. +REM Compile the code +cmake --build . endlocal diff --git a/test/cpp-tableau-loader/gen.bat b/test/cpp-tableau-loader/gen.bat index 25df1aca..dc4b7c62 100644 --- a/test/cpp-tableau-loader/gen.bat +++ b/test/cpp-tableau-loader/gen.bat @@ -5,7 +5,7 @@ setlocal enabledelayedexpansion for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i cd /d "%repoRoot%" -set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\debug\protoc.exe" +set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\protoc.exe" set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" set "ROOTDIR=%repoRoot%\test\cpp-tableau-loader" diff --git a/test/cpp-tableau-loader/src/CMakeLists.txt b/test/cpp-tableau-loader/src/CMakeLists.txt index df4fb5b0..e946a6b6 100644 --- a/test/cpp-tableau-loader/src/CMakeLists.txt +++ b/test/cpp-tableau-loader/src/CMakeLists.txt @@ -45,7 +45,7 @@ SET(COMMON_INCLUDE_DIR # common lib if(MSVC) SET(COMMON_LIB - ${PROTOBUF_ROOT_DIR}/cmake/build/debug/libprotobufd.lib + ${PROTOBUF_ROOT_DIR}/cmake/build/libprotobufd.lib ) SET(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") else() diff --git a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc index 4ab4a4a9..92f0aea6 100644 --- a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc @@ -3,57 +3,57 @@ // - protoc-gen-cpp-tableau-loader v0.9.0 // - protoc v3.19.3 -#include "scheduler.pc.h" - -namespace tableau { -namespace internal { -// Thread-local storage (TLS) -thread_local Scheduler* tls_sched = nullptr; - -Scheduler& Scheduler::Current() { - if (tls_sched == nullptr) { - tls_sched = new Scheduler; - } - return *tls_sched; -} - -int Scheduler::LoopOnce() { - AssertInLoopThread(); - - int count = 0; - std::vector jobs; - { - // scoped for auto-release lock. - // wake up immediately when there are pending tasks. - std::unique_lock lock(mutex_); - jobs.swap(jobs_); - } - for (auto&& job : jobs) { - job(); - } - count += static_cast(jobs.size()); - return count; -} - -void Scheduler::Post(const Job& job) { - std::unique_lock lock(mutex_); - jobs_.push_back(job); -} - -void Scheduler::Dispatch(const Job& job) { - if (IsLoopThread()) { - job(); // run it immediately - } else { - Post(job); // post and run it at next loop - } -} - -bool Scheduler::IsLoopThread() const { return thread_id_ == std::this_thread::get_id(); } - -void Scheduler::AssertInLoopThread() const { - if (!IsLoopThread()) { - abort(); - } -} -} // namespace internal +#include "scheduler.pc.h" + +namespace tableau { +namespace internal { +// Thread-local storage (TLS) +thread_local Scheduler* tls_sched = nullptr; + +Scheduler& Scheduler::Current() { + if (tls_sched == nullptr) { + tls_sched = new Scheduler; + } + return *tls_sched; +} + +int Scheduler::LoopOnce() { + AssertInLoopThread(); + + int count = 0; + std::vector jobs; + { + // scoped for auto-release lock. + // wake up immediately when there are pending tasks. + std::unique_lock lock(mutex_); + jobs.swap(jobs_); + } + for (auto&& job : jobs) { + job(); + } + count += static_cast(jobs.size()); + return count; +} + +void Scheduler::Post(const Job& job) { + std::unique_lock lock(mutex_); + jobs_.push_back(job); +} + +void Scheduler::Dispatch(const Job& job) { + if (IsLoopThread()) { + job(); // run it immediately + } else { + Post(job); // post and run it at next loop + } +} + +bool Scheduler::IsLoopThread() const { return thread_id_ == std::this_thread::get_id(); } + +void Scheduler::AssertInLoopThread() const { + if (!IsLoopThread()) { + abort(); + } +} +} // namespace internal } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h index 51a6e6af..affa8b70 100644 --- a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h @@ -3,34 +3,34 @@ // - protoc-gen-cpp-tableau-loader v0.9.0 // - protoc v3.19.3 -#pragma once - -#include -#include -#include -#include - -namespace tableau { -namespace internal { -class Scheduler { - public: - typedef std::function Job; - - public: - Scheduler() : thread_id_(std::this_thread::get_id()) {} - static Scheduler& Current(); - // thread-safety - void Post(const Job& job); - void Dispatch(const Job& job); - int LoopOnce(); - bool IsLoopThread() const; - void AssertInLoopThread() const; - - private: - std::thread::id thread_id_; - std::mutex mutex_; - std::vector jobs_; -}; - -} // namespace internal +#pragma once + +#include +#include +#include +#include + +namespace tableau { +namespace internal { +class Scheduler { + public: + typedef std::function Job; + + public: + Scheduler() : thread_id_(std::this_thread::get_id()) {} + static Scheduler& Current(); + // thread-safety + void Post(const Job& job); + void Dispatch(const Job& job); + int LoopOnce(); + bool IsLoopThread() const; + void AssertInLoopThread() const; + + private: + std::thread::id thread_id_; + std::mutex mutex_; + std::vector jobs_; +}; + +} // namespace internal } // namespace tableau From 9cf8363702a5acb7bc79e1817ebaa91a591e7de5 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 22 Aug 2025 17:34:14 +0800 Subject: [PATCH 12/18] feat: update gen.bat for go-tableau-loader --- test/go-tableau-loader/gen.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/go-tableau-loader/gen.bat b/test/go-tableau-loader/gen.bat index 78520f73..d0204928 100644 --- a/test/go-tableau-loader/gen.bat +++ b/test/go-tableau-loader/gen.bat @@ -5,7 +5,7 @@ setlocal enabledelayedexpansion for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i cd /d "%repoRoot%" -set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\debug\protoc.exe" +set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\protoc.exe" set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" set "PLGUIN_DIR=%repoRoot%\cmd\protoc-gen-go-tableau-loader" From 6923b589fbd095805da08c253ef0c6334f7d2b86 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 22 Aug 2025 21:06:28 +0800 Subject: [PATCH 13/18] feat: simplify CMake --- README.md | 20 ++++----- init.bat | 11 +++-- init.sh | 16 ++++--- .../{src => }/CMakeLists.txt | 24 +++++----- test/cpp-tableau-loader/gen.bat | 2 +- test/cpp-tableau-loader/gen.sh | 4 +- test/cpp-tableau-loader/src/main.cpp | 44 ++++++++++--------- 7 files changed, 63 insertions(+), 58 deletions(-) rename test/cpp-tableau-loader/{src => }/CMakeLists.txt (69%) diff --git a/README.md b/README.md index d28acf27..f6d472e9 100644 --- a/README.md +++ b/README.md @@ -22,24 +22,22 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). - Change dir: `cd test/cpp-tableau-loader` - Generate protoconf: `bash ./gen.sh` -- Create build dir: `mkdir build` and `cd build` - CMake: - - C++17: `cmake ../src/` - - C++20: `cmake ../src/ -DCMAKE_CXX_STANDARD=20` - - clang: `cmake ../src/ -DCMAKE_CXX_COMPILER=clang++` -- Build: `cmake --build .` -- Run: `../bin/loader` + - C++17: `cmake -S . -B build` + - C++20: `cmake -S . -B build -DCMAKE_CXX_STANDARD=20` + - clang: `cmake -S . -B build -DCMAKE_CXX_COMPILER=clang++` +- Build: `cmake --build build --parallel 10` +- Run: `./bin/loader` ### Dev at Windows - Change dir: `cd test\cpp-tableau-loader` - Generate protoconf: `.\gen.bat` -- Create build dir: `mkdir build` and `cd build` - CMake: - - C++17: `cmake -G "NMake Makefiles" ..\src\` - - C++20: `cmake -G "NMake Makefiles" ..\src\ -DCMAKE_CXX_STANDARD=20` -- Build: `cmake --build .` -- Run: `..\bin\loader.exe` + - C++17: `cmake -S . -B build -G "NMake Makefiles"` + - C++20: `cmake -S . -B build -G "NMake Makefiles" -DCMAKE_CXX_STANDARD=20` +- Build: `cmake --build build --parallel 10` +- Run: `.\bin\loader.exe` ### References diff --git a/init.bat b/init.bat index 79dc55a6..d3e0b9ae 100644 --- a/init.bat +++ b/init.bat @@ -14,14 +14,13 @@ git submodule update --init --recursive REM Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) REM Refer: https://github.com/protocolbuffers/protobuf/blob/3.19.x/cmake/README.md#cmake-configuration cd cmake -mkdir build 2>nul -cd build -REM use Debug configuration -cmake -G "NMake Makefiles" ^ +REM use Debug version +cmake -S . -B build ^ + -G "NMake Makefiles" ^ -DCMAKE_BUILD_TYPE=Debug ^ -DCMAKE_POLICY_VERSION_MINIMUM="3.5" ^ - .. + REM Compile the code -cmake --build . +cmake --build build endlocal diff --git a/init.sh b/init.sh index 6b2c5ab1..87e0d1b2 100755 --- a/init.sh +++ b/init.sh @@ -17,10 +17,14 @@ git submodule update --init --recursive cd third_party/_submodules/protobuf git checkout v3.19.3 git submodule update --init --recursive -./autogen.sh + # Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) -./configure -make -j"$(nproc)" -make check -j"$(nproc)" -# sudo make install -# sudo ldconfig # refresh shared library cache. \ No newline at end of file +# Refer: https://github.com/protocolbuffers/protobuf/blob/3.19.x/cmake/README.md#cmake-configuration +cd cmake +# use Debug version +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_POLICY_VERSION_MINIMUM="3.5" + +# Compile the code +cmake --build build \ No newline at end of file diff --git a/test/cpp-tableau-loader/src/CMakeLists.txt b/test/cpp-tableau-loader/CMakeLists.txt similarity index 69% rename from test/cpp-tableau-loader/src/CMakeLists.txt rename to test/cpp-tableau-loader/CMakeLists.txt index e946a6b6..9c5b00f5 100644 --- a/test/cpp-tableau-loader/src/CMakeLists.txt +++ b/test/cpp-tableau-loader/CMakeLists.txt @@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.22) # set the project name project(loader) -file(GLOB_RECURSE PROTO_SOURCE *.cc) -file(GLOB_RECURSE SOURCE *.cpp) +file(GLOB_RECURSE PROTO_SOURCE ${PROJECT_SOURCE_DIR}/src/*.cc) +file(GLOB_RECURSE SOURCE ${PROJECT_SOURCE_DIR}/src/*.cpp) # check C++ standard requirement set(MIN_CXX_STANDARD 17) @@ -25,13 +25,14 @@ endif() # root dir define SET(CMAKE_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -SET(THRID_PARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../third_party) +SET(THRID_PARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../third_party) # google protobuf SET(PROTOBUF_ROOT_DIR ${THRID_PARTY_DIR}/_submodules/protobuf/) -SET(PROTOBUF_DIR ${PROTOBUF_ROOT_DIR}/src/) -# SET(PROTOBUF_INCLUDE ${PROTOBUF_DIR}/google/protobuf) -SET(PROTOBUF_LIB ${PROTOBUF_DIR}/.libs/) +SET(PROTOBUF_SRC_DIR ${PROTOBUF_ROOT_DIR}/src/) +SET(PROTOBUF_BUILD_DIR ${PROTOBUF_ROOT_DIR}/cmake/build/) +# loader +SET(LOADER_SRC_DIR ${PROJECT_SOURCE_DIR}/src/) # protoconf SET(PROTOCONF_DIR ${TOOLS_DIR}/protoconf) @@ -39,27 +40,28 @@ SET(PROTOCONF_TOOL ${PROTOCONF_DIR}/protoconf) # common include SET(COMMON_INCLUDE_DIR - ${PROTOBUF_DIR} + ${PROTOBUF_SRC_DIR} + ${LOADER_SRC_DIR} ) # common lib if(MSVC) SET(COMMON_LIB - ${PROTOBUF_ROOT_DIR}/cmake/build/libprotobufd.lib + ${PROTOBUF_BUILD_DIR}/libprotobufd.lib ) SET(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") else() SET(COMMON_LIB - ${PROTOBUF_LIB}/libprotobuf.a + ${PROTOBUF_BUILD_DIR}/libprotobufd.a pthread stdc++fs ) endif() # include -include_directories(${PROJECT_SOURCE_DIR} ${COMMON_INCLUDE_DIR} ${PROJECT_SOURCE_DIR}/protoconf) +include_directories(${COMMON_INCLUDE_DIR} ${LOADER_SRC_DIR}/protoconf) # add the executable add_executable(${PROJECT_NAME} ${PROTO_SOURCE} ${SOURCE}) -set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/../bin) +set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin) target_link_libraries(${PROJECT_NAME} ${COMMON_LIB}) \ No newline at end of file diff --git a/test/cpp-tableau-loader/gen.bat b/test/cpp-tableau-loader/gen.bat index dc4b7c62..31d5a196 100644 --- a/test/cpp-tableau-loader/gen.bat +++ b/test/cpp-tableau-loader/gen.bat @@ -17,7 +17,7 @@ REM remove old generated files rmdir /s /q "%PROTOCONF_OUT%" 2>nul mkdir "%PROTOCONF_OUT%" -REM build +REM build protoc plugin of loader pushd "%PLGUIN_DIR%" go build popd diff --git a/test/cpp-tableau-loader/gen.sh b/test/cpp-tableau-loader/gen.sh index 83cc14ac..12937cc7 100755 --- a/test/cpp-tableau-loader/gen.sh +++ b/test/cpp-tableau-loader/gen.sh @@ -7,7 +7,7 @@ set -o pipefail shopt -s globstar cd "$(git rev-parse --show-toplevel)" -PROTOC="./third_party/_submodules/protobuf/src/protoc" +PROTOC="./third_party/_submodules/protobuf/src/cmake/build/protoc" PROTOBUF_PROTO="./third_party/_submodules/protobuf/src" TABLEAU_PROTO="./third_party/_submodules/tableau/proto" ROOTDIR="./test/cpp-tableau-loader" @@ -19,7 +19,7 @@ PROTOCONF_OUT="${ROOTDIR}/src/protoconf" rm -rfv "$PROTOCONF_OUT" mkdir -p "$PROTOCONF_OUT" -# build +# build protoc plugin of loader cd "${PLGUIN_DIR}" && go build && cd - export PATH="${PATH}:${PLGUIN_DIR}" diff --git a/test/cpp-tableau-loader/src/main.cpp b/test/cpp-tableau-loader/src/main.cpp index 649aa714..08bce442 100644 --- a/test/cpp-tableau-loader/src/main.cpp +++ b/test/cpp-tableau-loader/src/main.cpp @@ -13,8 +13,10 @@ #include "protoconf/patch_conf.pc.h" #include "protoconf/test_conf.pc.h" +const std::string kTestdataDir = "../testdata"; + bool LoadWithPatch(std::shared_ptr options) { - return Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + return Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); } bool CustomReadFile(const std::filesystem::path& filename, std::string& content) { @@ -33,8 +35,8 @@ bool TestPatch() { // patchconf ATOM_DEBUG("-----TestPatch patchconf"); - options->patch_dirs = {"../../testdata/patchconf/"}; - bool ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + options->patch_dirs = {kTestdataDir + "/patchconf/"}; + bool ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { ATOM_ERROR("failed to load with patchconf"); return false; @@ -48,7 +50,7 @@ bool TestPatch() { } ATOM_DEBUG("RecursivePatchConf: %s", mgr->Data().ShortDebugString().c_str()); tableau::RecursivePatchConf result; - ok = result.Load("../../testdata/patchresult/", tableau::Format::kJSON); + ok = result.Load(kTestdataDir + "/patchresult/", tableau::Format::kJSON); if (!ok) { ATOM_ERROR("failed to load with patch result"); return false; @@ -61,8 +63,8 @@ bool TestPatch() { // patchconf2 ATOM_DEBUG("-----TestPatch patchconf2"); - options->patch_dirs = {"../../testdata/patchconf2/"}; - ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + options->patch_dirs = {kTestdataDir + "/patchconf2/"}; + ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { ATOM_ERROR("failed to load with patchconf2"); return false; @@ -70,11 +72,11 @@ bool TestPatch() { // patchconf2 different format ATOM_DEBUG("-----TestPatch patchconf2 different format"); - options->patch_dirs = {"../../testdata/patchconf2/"}; + options->patch_dirs = {kTestdataDir + "/patchconf2/"}; auto mopts = std::make_shared(); - mopts->patch_paths = {"../../testdata/patchconf2/PatchMergeConf.txt"}; + mopts->patch_paths = {kTestdataDir + "/patchconf2/PatchMergeConf.txt"}; options->messager_options["PatchMergeConf"] = mopts; - ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { ATOM_ERROR("failed to load with patchconf2"); return false; @@ -83,10 +85,10 @@ bool TestPatch() { // multiple patch files ATOM_DEBUG("-----TestPatch multiple patch files"); mopts = std::make_shared(); - mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", - "../../testdata/patchconf2/PatchMergeConf.json"}; + mopts->patch_paths = {kTestdataDir + "/patchconf/PatchMergeConf.json", + kTestdataDir + "/patchconf2/PatchMergeConf.json"}; options->messager_options["PatchMergeConf"] = mopts; - ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { ATOM_ERROR("failed to load with multiple patch files"); return false; @@ -95,11 +97,11 @@ bool TestPatch() { // mode only main ATOM_DEBUG("-----TestPatch ModeOnlyMain"); mopts = std::make_shared(); - mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", - "../../testdata/patchconf2/PatchMergeConf.json"}; + mopts->patch_paths = {kTestdataDir + "/patchconf/PatchMergeConf.json", + kTestdataDir + "/patchconf2/PatchMergeConf.json"}; options->messager_options["PatchMergeConf"] = mopts; options->mode = tableau::load::LoadMode::kOnlyMain; - ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { ATOM_ERROR("failed to load with mode only main"); return false; @@ -114,11 +116,11 @@ bool TestPatch() { // mode only patch ATOM_DEBUG("-----TestPatch ModeOnlyPatch"); mopts = std::make_shared(); - mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", - "../../testdata/patchconf2/PatchMergeConf.json"}; + mopts->patch_paths = {kTestdataDir + "/patchconf/PatchMergeConf.json", + kTestdataDir + "/patchconf2/PatchMergeConf.json"}; options->messager_options["PatchMergeConf"] = mopts; options->mode = tableau::load::LoadMode::kOnlyPatch; - ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { ATOM_ERROR("failed to load with mode only patch"); return false; @@ -130,12 +132,12 @@ int main() { Hub::Instance().InitOnce(); auto options = std::make_shared(); options->ignore_unknown_fields = true; - options->patch_dirs = {"../../testdata/patchconf/"}; + options->patch_dirs = {kTestdataDir + "/patchconf/"}; auto mopts = std::make_shared(); - mopts->path = "../../testdata/conf/ItemConf.json"; + mopts->path = kTestdataDir + "/conf/ItemConf.json"; options->messager_options["ItemConf"] = mopts; - bool ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + bool ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { ATOM_ERROR("protobuf hub load failed: %s", tableau::GetErrMsg().c_str()); return 1; From 7e6b569520081ce57c7ea0941c5334dcf31cc134 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 22 Aug 2025 21:24:31 +0800 Subject: [PATCH 14/18] feat: formats and cmake parallel --- init.bat | 4 ++-- init.sh | 2 +- test/cpp-tableau-loader/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/init.bat b/init.bat index d3e0b9ae..7d0c0858 100644 --- a/init.bat +++ b/init.bat @@ -18,9 +18,9 @@ REM use Debug version cmake -S . -B build ^ -G "NMake Makefiles" ^ -DCMAKE_BUILD_TYPE=Debug ^ - -DCMAKE_POLICY_VERSION_MINIMUM="3.5" ^ + -DCMAKE_POLICY_VERSION_MINIMUM="3.5" REM Compile the code -cmake --build build +cmake --build build --parallel 10 endlocal diff --git a/init.sh b/init.sh index 87e0d1b2..e294105d 100755 --- a/init.sh +++ b/init.sh @@ -27,4 +27,4 @@ cmake -S . -B build \ -DCMAKE_POLICY_VERSION_MINIMUM="3.5" # Compile the code -cmake --build build \ No newline at end of file +cmake --build build --parallel 10 diff --git a/test/cpp-tableau-loader/CMakeLists.txt b/test/cpp-tableau-loader/CMakeLists.txt index 9c5b00f5..7787aed9 100644 --- a/test/cpp-tableau-loader/CMakeLists.txt +++ b/test/cpp-tableau-loader/CMakeLists.txt @@ -64,4 +64,4 @@ include_directories(${COMMON_INCLUDE_DIR} ${LOADER_SRC_DIR}/protoconf) # add the executable add_executable(${PROJECT_NAME} ${PROTO_SOURCE} ${SOURCE}) set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin) -target_link_libraries(${PROJECT_NAME} ${COMMON_LIB}) \ No newline at end of file +target_link_libraries(${PROJECT_NAME} ${COMMON_LIB}) From 1c3e78b0e391df0c69ad18ad9e8dbd75d8962c1e Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 22 Aug 2025 21:43:05 +0800 Subject: [PATCH 15/18] fix: CRLF -> LF --- .../embed/logger.pc.cc | 2 +- .../src/protoconf/scheduler.pc.cc | 106 +++++++++--------- .../src/protoconf/scheduler.pc.h | 60 +++++----- test/go-tableau-loader/gen.sh | 70 ++++++------ 4 files changed, 119 insertions(+), 119 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc index 4ecc2908..c35ea606 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc @@ -98,7 +98,7 @@ std::ostream& operator<<(std::ostream& os, const Now&) { auto now = std::chrono::system_clock::now(); auto now_time_t = std::chrono::system_clock::to_time_t(now); auto now_us = std::chrono::duration_cast(now.time_since_epoch()) % 1000000; - return os << std::put_time(std::localtime(&now_time_t), "%F %T") << "." << std::setw(6) << std::setfill('0') + return os << std::put_time(std::localtime_s(&now_time_t), "%F %T") << "." << std::setw(6) << std::setfill('0') << now_us.count(); } diff --git a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc index 92f0aea6..4ab4a4a9 100644 --- a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc @@ -3,57 +3,57 @@ // - protoc-gen-cpp-tableau-loader v0.9.0 // - protoc v3.19.3 -#include "scheduler.pc.h" - -namespace tableau { -namespace internal { -// Thread-local storage (TLS) -thread_local Scheduler* tls_sched = nullptr; - -Scheduler& Scheduler::Current() { - if (tls_sched == nullptr) { - tls_sched = new Scheduler; - } - return *tls_sched; -} - -int Scheduler::LoopOnce() { - AssertInLoopThread(); - - int count = 0; - std::vector jobs; - { - // scoped for auto-release lock. - // wake up immediately when there are pending tasks. - std::unique_lock lock(mutex_); - jobs.swap(jobs_); - } - for (auto&& job : jobs) { - job(); - } - count += static_cast(jobs.size()); - return count; -} - -void Scheduler::Post(const Job& job) { - std::unique_lock lock(mutex_); - jobs_.push_back(job); -} - -void Scheduler::Dispatch(const Job& job) { - if (IsLoopThread()) { - job(); // run it immediately - } else { - Post(job); // post and run it at next loop - } -} - -bool Scheduler::IsLoopThread() const { return thread_id_ == std::this_thread::get_id(); } - -void Scheduler::AssertInLoopThread() const { - if (!IsLoopThread()) { - abort(); - } -} -} // namespace internal +#include "scheduler.pc.h" + +namespace tableau { +namespace internal { +// Thread-local storage (TLS) +thread_local Scheduler* tls_sched = nullptr; + +Scheduler& Scheduler::Current() { + if (tls_sched == nullptr) { + tls_sched = new Scheduler; + } + return *tls_sched; +} + +int Scheduler::LoopOnce() { + AssertInLoopThread(); + + int count = 0; + std::vector jobs; + { + // scoped for auto-release lock. + // wake up immediately when there are pending tasks. + std::unique_lock lock(mutex_); + jobs.swap(jobs_); + } + for (auto&& job : jobs) { + job(); + } + count += static_cast(jobs.size()); + return count; +} + +void Scheduler::Post(const Job& job) { + std::unique_lock lock(mutex_); + jobs_.push_back(job); +} + +void Scheduler::Dispatch(const Job& job) { + if (IsLoopThread()) { + job(); // run it immediately + } else { + Post(job); // post and run it at next loop + } +} + +bool Scheduler::IsLoopThread() const { return thread_id_ == std::this_thread::get_id(); } + +void Scheduler::AssertInLoopThread() const { + if (!IsLoopThread()) { + abort(); + } +} +} // namespace internal } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h index affa8b70..51a6e6af 100644 --- a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h @@ -3,34 +3,34 @@ // - protoc-gen-cpp-tableau-loader v0.9.0 // - protoc v3.19.3 -#pragma once - -#include -#include -#include -#include - -namespace tableau { -namespace internal { -class Scheduler { - public: - typedef std::function Job; - - public: - Scheduler() : thread_id_(std::this_thread::get_id()) {} - static Scheduler& Current(); - // thread-safety - void Post(const Job& job); - void Dispatch(const Job& job); - int LoopOnce(); - bool IsLoopThread() const; - void AssertInLoopThread() const; - - private: - std::thread::id thread_id_; - std::mutex mutex_; - std::vector jobs_; -}; - -} // namespace internal +#pragma once + +#include +#include +#include +#include + +namespace tableau { +namespace internal { +class Scheduler { + public: + typedef std::function Job; + + public: + Scheduler() : thread_id_(std::this_thread::get_id()) {} + static Scheduler& Current(); + // thread-safety + void Post(const Job& job); + void Dispatch(const Job& job); + int LoopOnce(); + bool IsLoopThread() const; + void AssertInLoopThread() const; + + private: + std::thread::id thread_id_; + std::mutex mutex_; + std::vector jobs_; +}; + +} // namespace internal } // namespace tableau diff --git a/test/go-tableau-loader/gen.sh b/test/go-tableau-loader/gen.sh index a5e9c8da..9cd5f66c 100755 --- a/test/go-tableau-loader/gen.sh +++ b/test/go-tableau-loader/gen.sh @@ -1,35 +1,35 @@ -#!/bin/bash - -# set -eux -set -e -set -o pipefail - -shopt -s globstar - -cd "$(git rev-parse --show-toplevel)" -PROTOC="./third_party/_submodules/protobuf/src/protoc" -PROTOBUF_PROTO="./third_party/_submodules/protobuf/src" -TABLEAU_PROTO="./third_party/_submodules/tableau/proto" -PLGUIN_DIR="./cmd/protoc-gen-go-tableau-loader" -PROTOCONF_IN="./test/proto" -PROTOCONF_OUT="./test/go-tableau-loader/protoconf" -LOADER_OUT="$PROTOCONF_OUT/loader" - -# remove old generated files -rm -rfv "$PROTOCONF_OUT" "$LOADER_OUT" -mkdir -p "$PROTOCONF_OUT" "$LOADER_OUT" - -# build -cd "${PLGUIN_DIR}" && go build && cd - - -export PATH="${PLGUIN_DIR}:${PATH}" - -${PROTOC} \ - --go-tableau-loader_out="$LOADER_OUT" \ - --go-tableau-loader_opt=paths=source_relative,pkg=loader \ - --go_out="$PROTOCONF_OUT" \ - --go_opt=paths=source_relative \ - --proto_path="$PROTOBUF_PROTO" \ - --proto_path="$TABLEAU_PROTO" \ - --proto_path="$PROTOCONF_IN" \ - "$PROTOCONF_IN"/**/*.proto +#!/bin/bash + +# set -eux +set -e +set -o pipefail + +shopt -s globstar + +cd "$(git rev-parse --show-toplevel)" +PROTOC="./third_party/_submodules/protobuf/src/cmake/build/protoc" +PROTOBUF_PROTO="./third_party/_submodules/protobuf/src" +TABLEAU_PROTO="./third_party/_submodules/tableau/proto" +PLGUIN_DIR="./cmd/protoc-gen-go-tableau-loader" +PROTOCONF_IN="./test/proto" +PROTOCONF_OUT="./test/go-tableau-loader/protoconf" +LOADER_OUT="$PROTOCONF_OUT/loader" + +# remove old generated files +rm -rfv "$PROTOCONF_OUT" "$LOADER_OUT" +mkdir -p "$PROTOCONF_OUT" "$LOADER_OUT" + +# build +cd "${PLGUIN_DIR}" && go build && cd - + +export PATH="${PLGUIN_DIR}:${PATH}" + +${PROTOC} \ + --go-tableau-loader_out="$LOADER_OUT" \ + --go-tableau-loader_opt=paths=source_relative,pkg=loader \ + --go_out="$PROTOCONF_OUT" \ + --go_opt=paths=source_relative \ + --proto_path="$PROTOBUF_PROTO" \ + --proto_path="$TABLEAU_PROTO" \ + --proto_path="$PROTOCONF_IN" \ + "$PROTOCONF_IN"/**/*.proto From 0cc00e06600bcc17a68955bcf31eee63b12f4535 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 22 Aug 2025 21:47:38 +0800 Subject: [PATCH 16/18] fix: gen.sh --- test/cpp-tableau-loader/gen.sh | 2 +- test/go-tableau-loader/gen.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cpp-tableau-loader/gen.sh b/test/cpp-tableau-loader/gen.sh index 12937cc7..32756cb3 100755 --- a/test/cpp-tableau-loader/gen.sh +++ b/test/cpp-tableau-loader/gen.sh @@ -7,7 +7,7 @@ set -o pipefail shopt -s globstar cd "$(git rev-parse --show-toplevel)" -PROTOC="./third_party/_submodules/protobuf/src/cmake/build/protoc" +PROTOC="./third_party/_submodules/protobuf/cmake/build/protoc" PROTOBUF_PROTO="./third_party/_submodules/protobuf/src" TABLEAU_PROTO="./third_party/_submodules/tableau/proto" ROOTDIR="./test/cpp-tableau-loader" diff --git a/test/go-tableau-loader/gen.sh b/test/go-tableau-loader/gen.sh index 9cd5f66c..36fd1cb0 100755 --- a/test/go-tableau-loader/gen.sh +++ b/test/go-tableau-loader/gen.sh @@ -7,7 +7,7 @@ set -o pipefail shopt -s globstar cd "$(git rev-parse --show-toplevel)" -PROTOC="./third_party/_submodules/protobuf/src/cmake/build/protoc" +PROTOC="./third_party/_submodules/protobuf/cmake/build/protoc" PROTOBUF_PROTO="./third_party/_submodules/protobuf/src" TABLEAU_PROTO="./third_party/_submodules/tableau/proto" PLGUIN_DIR="./cmd/protoc-gen-go-tableau-loader" From 29500f6d45b9efc09fa110d8d5ad3f35b0caa15b Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 22 Aug 2025 21:53:08 +0800 Subject: [PATCH 17/18] feat: update scripts --- .../embed/logger.pc.cc | 2 +- init.bat | 6 +- init.sh | 3 +- test/go-tableau-loader/gen.sh | 70 +++++++++---------- 4 files changed, 40 insertions(+), 41 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc index c35ea606..4ecc2908 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc @@ -98,7 +98,7 @@ std::ostream& operator<<(std::ostream& os, const Now&) { auto now = std::chrono::system_clock::now(); auto now_time_t = std::chrono::system_clock::to_time_t(now); auto now_us = std::chrono::duration_cast(now.time_since_epoch()) % 1000000; - return os << std::put_time(std::localtime_s(&now_time_t), "%F %T") << "." << std::setw(6) << std::setfill('0') + return os << std::put_time(std::localtime(&now_time_t), "%F %T") << "." << std::setw(6) << std::setfill('0') << now_us.count(); } diff --git a/init.bat b/init.bat index 7d0c0858..72a57cf7 100644 --- a/init.bat +++ b/init.bat @@ -16,9 +16,9 @@ REM Refer: https://github.com/protocolbuffers/protobuf/blob/3.19.x/cmake/README. cd cmake REM use Debug version cmake -S . -B build ^ - -G "NMake Makefiles" ^ - -DCMAKE_BUILD_TYPE=Debug ^ - -DCMAKE_POLICY_VERSION_MINIMUM="3.5" +-G "NMake Makefiles" ^ +-DCMAKE_BUILD_TYPE=Debug ^ +-DCMAKE_POLICY_VERSION_MINIMUM="3.5" REM Compile the code cmake --build build --parallel 10 diff --git a/init.sh b/init.sh index e294105d..092096f3 100755 --- a/init.sh +++ b/init.sh @@ -23,8 +23,7 @@ git submodule update --init --recursive cd cmake # use Debug version cmake -S . -B build \ - -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_POLICY_VERSION_MINIMUM="3.5" + -DCMAKE_BUILD_TYPE=Debug # Compile the code cmake --build build --parallel 10 diff --git a/test/go-tableau-loader/gen.sh b/test/go-tableau-loader/gen.sh index 36fd1cb0..e547ff5e 100755 --- a/test/go-tableau-loader/gen.sh +++ b/test/go-tableau-loader/gen.sh @@ -1,35 +1,35 @@ -#!/bin/bash - -# set -eux -set -e -set -o pipefail - -shopt -s globstar - -cd "$(git rev-parse --show-toplevel)" -PROTOC="./third_party/_submodules/protobuf/cmake/build/protoc" -PROTOBUF_PROTO="./third_party/_submodules/protobuf/src" -TABLEAU_PROTO="./third_party/_submodules/tableau/proto" -PLGUIN_DIR="./cmd/protoc-gen-go-tableau-loader" -PROTOCONF_IN="./test/proto" -PROTOCONF_OUT="./test/go-tableau-loader/protoconf" -LOADER_OUT="$PROTOCONF_OUT/loader" - -# remove old generated files -rm -rfv "$PROTOCONF_OUT" "$LOADER_OUT" -mkdir -p "$PROTOCONF_OUT" "$LOADER_OUT" - -# build -cd "${PLGUIN_DIR}" && go build && cd - - -export PATH="${PLGUIN_DIR}:${PATH}" - -${PROTOC} \ - --go-tableau-loader_out="$LOADER_OUT" \ - --go-tableau-loader_opt=paths=source_relative,pkg=loader \ - --go_out="$PROTOCONF_OUT" \ - --go_opt=paths=source_relative \ - --proto_path="$PROTOBUF_PROTO" \ - --proto_path="$TABLEAU_PROTO" \ - --proto_path="$PROTOCONF_IN" \ - "$PROTOCONF_IN"/**/*.proto +#!/bin/bash + +# set -eux +set -e +set -o pipefail + +shopt -s globstar + +cd "$(git rev-parse --show-toplevel)" +PROTOC="./third_party/_submodules/protobuf/cmake/build/protoc" +PROTOBUF_PROTO="./third_party/_submodules/protobuf/src" +TABLEAU_PROTO="./third_party/_submodules/tableau/proto" +PLGUIN_DIR="./cmd/protoc-gen-go-tableau-loader" +PROTOCONF_IN="./test/proto" +PROTOCONF_OUT="./test/go-tableau-loader/protoconf" +LOADER_OUT="$PROTOCONF_OUT/loader" + +# remove old generated files +rm -rfv "$PROTOCONF_OUT" "$LOADER_OUT" +mkdir -p "$PROTOCONF_OUT" "$LOADER_OUT" + +# build +cd "${PLGUIN_DIR}" && go build && cd - + +export PATH="${PLGUIN_DIR}:${PATH}" + +${PROTOC} \ + --go-tableau-loader_out="$LOADER_OUT" \ + --go-tableau-loader_opt=paths=source_relative,pkg=loader \ + --go_out="$PROTOCONF_OUT" \ + --go_opt=paths=source_relative \ + --proto_path="$PROTOBUF_PROTO" \ + --proto_path="$TABLEAU_PROTO" \ + --proto_path="$PROTOCONF_IN" \ + "$PROTOCONF_IN"/**/*.proto From be4c1fefeebf4203a764ba9808493dff0d441d50 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 22 Aug 2025 22:04:59 +0800 Subject: [PATCH 18/18] feat: use CRLF for bat script --- .gitattributes | 4 +- init.bat | 52 +++++++-------- test/cpp-tableau-loader/gen.bat | 110 ++++++++++++++++---------------- test/go-tableau-loader/gen.bat | 88 ++++++++++++------------- 4 files changed, 127 insertions(+), 127 deletions(-) diff --git a/.gitattributes b/.gitattributes index e6c41c5a..287ff3ef 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,8 +8,8 @@ # endings, and so they are checked in with CRLF endings, with a test # to catch problems. (See https://github.com/golang/go/issues/37791.) # -# We'll prevent accidental CRLF line endings from entering the repo -# via MR checks and tests. +# You can prevent accidental CRLF line endings from entering the repo +# via PR/MR checks. # # See https://github.com/golang/go/issues/9281. # See https://adaptivepatchwork.com/2012/03/01/mind-the-end-of-your-line. diff --git a/init.bat b/init.bat index 72a57cf7..6103990c 100644 --- a/init.bat +++ b/init.bat @@ -1,26 +1,26 @@ -@echo off -setlocal - -for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i -cd /d "%repoRoot%" - -git submodule update --init --recursive - -REM google protobuf -cd third_party\_submodules\protobuf -git checkout v3.19.3 -git submodule update --init --recursive - -REM Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) -REM Refer: https://github.com/protocolbuffers/protobuf/blob/3.19.x/cmake/README.md#cmake-configuration -cd cmake -REM use Debug version -cmake -S . -B build ^ --G "NMake Makefiles" ^ --DCMAKE_BUILD_TYPE=Debug ^ --DCMAKE_POLICY_VERSION_MINIMUM="3.5" - -REM Compile the code -cmake --build build --parallel 10 - -endlocal +@echo off +setlocal + +for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i +cd /d "%repoRoot%" + +git submodule update --init --recursive + +REM google protobuf +cd third_party\_submodules\protobuf +git checkout v3.19.3 +git submodule update --init --recursive + +REM Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) +REM Refer: https://github.com/protocolbuffers/protobuf/blob/3.19.x/cmake/README.md#cmake-configuration +cd cmake +REM use Debug version +cmake -S . -B build ^ +-G "NMake Makefiles" ^ +-DCMAKE_BUILD_TYPE=Debug ^ +-DCMAKE_POLICY_VERSION_MINIMUM="3.5" + +REM Compile the code +cmake --build build --parallel 10 + +endlocal diff --git a/test/cpp-tableau-loader/gen.bat b/test/cpp-tableau-loader/gen.bat index 31d5a196..c1a4ac96 100644 --- a/test/cpp-tableau-loader/gen.bat +++ b/test/cpp-tableau-loader/gen.bat @@ -1,55 +1,55 @@ -@echo off -setlocal -setlocal enabledelayedexpansion - -for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i -cd /d "%repoRoot%" - -set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\protoc.exe" -set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" -set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" -set "ROOTDIR=%repoRoot%\test\cpp-tableau-loader" -set "PLGUIN_DIR=%repoRoot%\cmd\protoc-gen-cpp-tableau-loader" -set "PROTOCONF_IN=%repoRoot%\test\proto" -set "PROTOCONF_OUT=%ROOTDIR%\src\protoconf" - -REM remove old generated files -rmdir /s /q "%PROTOCONF_OUT%" 2>nul -mkdir "%PROTOCONF_OUT%" - -REM build protoc plugin of loader -pushd "%PLGUIN_DIR%" -go build -popd - -set "PATH=%PATH%;%PLGUIN_DIR%" - -set protoFiles= -pushd "%PROTOCONF_IN%" -for /R %%f in (*.proto) do ( - set protoFiles=!protoFiles! "%%f" -) -popd -"%PROTOC%" ^ ---cpp-tableau-loader_out="%PROTOCONF_OUT%" ^ ---cpp-tableau-loader_opt=paths=source_relative,shards=2 ^ ---cpp_out="%PROTOCONF_OUT%" ^ ---proto_path="%PROTOBUF_PROTO%" ^ ---proto_path="%TABLEAU_PROTO%" ^ ---proto_path="%PROTOCONF_IN%" ^ -!protoFiles! - -set "TABLEAU_IN=%TABLEAU_PROTO%\tableau\protobuf" -set "TABLEAU_OUT=%ROOTDIR%\src" -REM remove old generated files -if exist "%TABLEAU_OUT%\tableau" rmdir /s /q "%TABLEAU_OUT%\tableau" -mkdir "%TABLEAU_OUT%\tableau" - -"%PROTOC%" ^ ---cpp_out="%TABLEAU_OUT%" ^ ---proto_path="%PROTOBUF_PROTO%" ^ ---proto_path="%TABLEAU_PROTO%" ^ -"%TABLEAU_IN%\tableau.proto" "%TABLEAU_IN%\wellknown.proto" - -endlocal -endlocal +@echo off +setlocal +setlocal enabledelayedexpansion + +for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i +cd /d "%repoRoot%" + +set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\protoc.exe" +set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" +set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" +set "ROOTDIR=%repoRoot%\test\cpp-tableau-loader" +set "PLGUIN_DIR=%repoRoot%\cmd\protoc-gen-cpp-tableau-loader" +set "PROTOCONF_IN=%repoRoot%\test\proto" +set "PROTOCONF_OUT=%ROOTDIR%\src\protoconf" + +REM remove old generated files +rmdir /s /q "%PROTOCONF_OUT%" 2>nul +mkdir "%PROTOCONF_OUT%" + +REM build protoc plugin of loader +pushd "%PLGUIN_DIR%" +go build +popd + +set "PATH=%PATH%;%PLGUIN_DIR%" + +set protoFiles= +pushd "%PROTOCONF_IN%" +for /R %%f in (*.proto) do ( + set protoFiles=!protoFiles! "%%f" +) +popd +"%PROTOC%" ^ +--cpp-tableau-loader_out="%PROTOCONF_OUT%" ^ +--cpp-tableau-loader_opt=paths=source_relative,shards=2 ^ +--cpp_out="%PROTOCONF_OUT%" ^ +--proto_path="%PROTOBUF_PROTO%" ^ +--proto_path="%TABLEAU_PROTO%" ^ +--proto_path="%PROTOCONF_IN%" ^ +!protoFiles! + +set "TABLEAU_IN=%TABLEAU_PROTO%\tableau\protobuf" +set "TABLEAU_OUT=%ROOTDIR%\src" +REM remove old generated files +if exist "%TABLEAU_OUT%\tableau" rmdir /s /q "%TABLEAU_OUT%\tableau" +mkdir "%TABLEAU_OUT%\tableau" + +"%PROTOC%" ^ +--cpp_out="%TABLEAU_OUT%" ^ +--proto_path="%PROTOBUF_PROTO%" ^ +--proto_path="%TABLEAU_PROTO%" ^ +"%TABLEAU_IN%\tableau.proto" "%TABLEAU_IN%\wellknown.proto" + +endlocal +endlocal diff --git a/test/go-tableau-loader/gen.bat b/test/go-tableau-loader/gen.bat index d0204928..7a9c2fb3 100644 --- a/test/go-tableau-loader/gen.bat +++ b/test/go-tableau-loader/gen.bat @@ -1,44 +1,44 @@ -@echo off -setlocal -setlocal enabledelayedexpansion - -for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i -cd /d "%repoRoot%" - -set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\protoc.exe" -set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" -set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" -set "PLGUIN_DIR=%repoRoot%\cmd\protoc-gen-go-tableau-loader" -set "PROTOCONF_IN=%repoRoot%\test\proto" -set "PROTOCONF_OUT=%repoRoot%\test\go-tableau-loader\protoconf" -set "LOADER_OUT=%PROTOCONF_OUT%\loader" - -REM remove old generated files -rmdir /s /q "%PROTOCONF_OUT%" "%LOADER_OUT%" 2>nul -mkdir "%PROTOCONF_OUT%" "%LOADER_OUT%" - -REM build -pushd "%PLGUIN_DIR%" -go build -popd - -set "PATH=%PATH%;%PLGUIN_DIR%" - -set protoFiles= -pushd "%PROTOCONF_IN%" -for /R %%f in (*.proto) do ( - set protoFiles=!protoFiles! "%%f" -) -popd -"%PROTOC%" ^ ---go-tableau-loader_out="%LOADER_OUT%" ^ ---go-tableau-loader_opt=paths=source_relative,pkg=loader ^ ---go_out="%PROTOCONF_OUT%" ^ ---go_opt=paths=source_relative ^ ---proto_path="%PROTOBUF_PROTO%" ^ ---proto_path="%TABLEAU_PROTO%" ^ ---proto_path="%PROTOCONF_IN%" ^ -!protoFiles! - -endlocal -endlocal +@echo off +setlocal +setlocal enabledelayedexpansion + +for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i +cd /d "%repoRoot%" + +set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\protoc.exe" +set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" +set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" +set "PLGUIN_DIR=%repoRoot%\cmd\protoc-gen-go-tableau-loader" +set "PROTOCONF_IN=%repoRoot%\test\proto" +set "PROTOCONF_OUT=%repoRoot%\test\go-tableau-loader\protoconf" +set "LOADER_OUT=%PROTOCONF_OUT%\loader" + +REM remove old generated files +rmdir /s /q "%PROTOCONF_OUT%" "%LOADER_OUT%" 2>nul +mkdir "%PROTOCONF_OUT%" "%LOADER_OUT%" + +REM build +pushd "%PLGUIN_DIR%" +go build +popd + +set "PATH=%PATH%;%PLGUIN_DIR%" + +set protoFiles= +pushd "%PROTOCONF_IN%" +for /R %%f in (*.proto) do ( + set protoFiles=!protoFiles! "%%f" +) +popd +"%PROTOC%" ^ +--go-tableau-loader_out="%LOADER_OUT%" ^ +--go-tableau-loader_opt=paths=source_relative,pkg=loader ^ +--go_out="%PROTOCONF_OUT%" ^ +--go_opt=paths=source_relative ^ +--proto_path="%PROTOBUF_PROTO%" ^ +--proto_path="%TABLEAU_PROTO%" ^ +--proto_path="%PROTOCONF_IN%" ^ +!protoFiles! + +endlocal +endlocal