From a8b8a54a764dec6ad807825cd23817ec764e6ba5 Mon Sep 17 00:00:00 2001 From: wenchy Date: Tue, 12 Aug 2025 11:34:29 +0800 Subject: [PATCH 1/9] refactor: support both global-level and messager-level load options --- README.md | 5 +- .../src/hub/custom/item/custom_item_conf.h | 2 +- test/cpp-tableau-loader/src/main.cpp | 30 ++++-- .../src/protoconf/hero_conf.pc.cc | 8 +- .../src/protoconf/hero_conf.pc.h | 6 +- .../src/protoconf/hub.pc.cc | 6 +- .../cpp-tableau-loader/src/protoconf/hub.pc.h | 1 - .../src/protoconf/item_conf.pc.cc | 4 +- .../src/protoconf/item_conf.pc.h | 4 +- .../src/protoconf/load.pc.cc | 102 +++++++++++++----- .../src/protoconf/load.pc.h | 98 +++++++++++++---- .../src/protoconf/messager.pc.h | 41 ------- .../src/protoconf/patch_conf.pc.cc | 12 +-- .../src/protoconf/patch_conf.pc.h | 8 +- .../src/protoconf/test_conf.pc.cc | 16 +-- .../src/protoconf/test_conf.pc.h | 10 +- .../src/protoconf/util.pc.cc | 5 +- .../src/protoconf/util.pc.h | 4 +- 18 files changed, 221 insertions(+), 141 deletions(-) delete mode 100644 test/cpp-tableau-loader/src/protoconf/messager.pc.h diff --git a/README.md b/README.md index af2d1f22..e7525f53 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,11 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). > [protobufjs: Reflection vs. static code](https://github.com/protobufjs/protobuf.js/blob/master/cli/README.md#reflection-vs-static-code) -If using reflection (`.proto` or `JSON`) but not static code, then [proto3-json-serializer](https://github.com/googleapis/proto3-json-serializer-nodejs) is a good option. +If using reflection (`.proto` or `JSON`) but not static code, and for well-known types support, then [proto3-json-serializer](https://github.com/googleapis/proto3-json-serializer-nodejs) is a good option. This library implements proto3 JSON serialization and deserialization for +[protobuf.js](https://www.npmjs.com/package/protobufjs) protobuf objects +according to the [spec](https://protobuf.dev/programming-guides/proto3/#json). ### References: - [How to Setup a TypeScript + Node.js Project](https://khalilstemmler.com/blogs/typescript/node-starter-project/) +- [proto3-json-serializer](https://github.com/googleapis/proto3-json-serializer-nodejs) diff --git a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h index 5ebb6820..a616641a 100644 --- a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h +++ b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h @@ -5,7 +5,7 @@ class CustomItemConf : public tableau::Messager { public: static const std::string& Name() { return kCustomName; }; virtual bool Load(const std::string& dir, tableau::Format fmt, - std::shared_ptr options = nullptr) override { + std::shared_ptr options = nullptr) override { return true; } virtual bool ProcessAfterLoadAll(const tableau::Hub& hub) override; diff --git a/test/cpp-tableau-loader/src/main.cpp b/test/cpp-tableau-loader/src/main.cpp index 71c3d5e8..7079edb5 100644 --- a/test/cpp-tableau-loader/src/main.cpp +++ b/test/cpp-tableau-loader/src/main.cpp @@ -73,7 +73,9 @@ bool TestPatch() { // patchconf2 different format std::cout << "-----TestPatch patchconf2 different format" << std::endl; options->patch_dirs = {"../../testdata/patchconf2/"}; - options->patch_paths["PatchMergeConf"] = {"../../testdata/patchconf2/PatchMergeConf.txt"}; + auto mopts = std::make_shared(); + mopts->patch_paths = {"../../testdata/patchconf2/PatchMergeConf.txt"}; + options->messager_options["PatchMergeConf"] = mopts; ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { std::cout << "failed to load with patchconf2" << std::endl; @@ -82,8 +84,10 @@ bool TestPatch() { // multiple patch files std::cout << "-----TestPatch multiple patch files" << std::endl; - options->patch_paths["PatchMergeConf"] = {"../../testdata/patchconf/PatchMergeConf.json", - "../../testdata/patchconf2/PatchMergeConf.json"}; + mopts = std::make_shared(); + mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", + "../../testdata/patchconf2/PatchMergeConf.json"}; + options->messager_options["PatchMergeConf"] = mopts; ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { std::cout << "failed to load with multiple patch files" << std::endl; @@ -92,9 +96,11 @@ bool TestPatch() { // mode only main std::cout << "-----TestPatch ModeOnlyMain" << std::endl; - options->patch_paths["PatchMergeConf"] = {"../../testdata/patchconf/PatchMergeConf.json", - "../../testdata/patchconf2/PatchMergeConf.json"}; - options->mode = tableau::LoadMode::kModeOnlyMain; + mopts = std::make_shared(); + mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", + "../../testdata/patchconf2/PatchMergeConf.json"}; + options->messager_options["PatchMergeConf"] = mopts; + options->mode = tableau::LoadMode::kOnlyMain; ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { std::cout << "failed to load with mode only main" << std::endl; @@ -109,9 +115,11 @@ bool TestPatch() { // mode only patch std::cout << "-----TestPatch ModeOnlyPatch" << std::endl; - options->patch_paths["PatchMergeConf"] = {"../../testdata/patchconf/PatchMergeConf.json", - "../../testdata/patchconf2/PatchMergeConf.json"}; - options->mode = tableau::LoadMode::kModeOnlyPatch; + mopts = std::make_shared(); + mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", + "../../testdata/patchconf2/PatchMergeConf.json"}; + options->messager_options["PatchMergeConf"] = mopts; + options->mode = tableau::LoadMode::kOnlyPatch; ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { std::cout << "failed to load with mode only patch" << std::endl; @@ -125,7 +133,9 @@ int main() { auto options = std::make_shared(); options->ignore_unknown_fields = true; options->patch_dirs = {"../../testdata/patchconf/"}; - options->paths["ItemConf"] = "../../testdata/conf/ItemConf.json"; + auto mopts = std::make_shared(); + mopts->path = "../../testdata/conf/ItemConf.json"; + options->messager_options["ItemConf"] = mopts; bool ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); if (!ok) { diff --git a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc index 16094a7e..2deaf59c 100644 --- a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc @@ -12,9 +12,9 @@ namespace tableau { const std::string HeroConf::kProtoName = "HeroConf"; -bool HeroConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool HeroConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessage(data_, dir, fmt, options); + bool loaded = LoadMessageInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; @@ -71,9 +71,9 @@ const HeroConf::Hero_Attr_OrderedMap* HeroConf::GetOrderedMap(const std::string& const std::string HeroBaseConf::kProtoName = "HeroBaseConf"; -bool HeroBaseConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool HeroBaseConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessage(data_, dir, fmt, options); + bool loaded = LoadMessageInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; diff --git a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h index 02698c01..e07b07c4 100644 --- a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h @@ -7,7 +7,7 @@ #pragma once #include -#include "messager.pc.h" +#include "load.pc.h" #include "util.pc.h" #include "hero_conf.pb.h" @@ -15,7 +15,7 @@ namespace tableau { class HeroConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::HeroConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -46,7 +46,7 @@ class HeroConf : public Messager { class HeroBaseConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::HeroBaseConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index 77e2d003..4953d5e1 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -5,8 +5,8 @@ #include "hub.pc.h" +#include "load.pc.h" #include "logger.pc.h" -#include "messager.pc.h" #include "util.pc.h" namespace tableau { @@ -58,11 +58,13 @@ std::shared_ptr Hub::InternalLoad(const std::string& dir, Format fm std::shared_ptr options /* = nullptr */) const { // intercept protobuf error logs auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); + auto opts = ParseLoadOptions(options); auto msger_map = NewMessagerMap(); for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); - bool ok = iter.second->Load(dir, fmt, options); + auto mopts = ParseMessagerOptions(opts, name); + bool ok = iter.second->Load(dir, fmt, mopts); if (!ok) { ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str()); // restore to old protobuf log handler diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index bde26d02..e80c9710 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -11,7 +11,6 @@ #include #include "load.pc.h" -#include "messager.pc.h" #include "scheduler.pc.h" namespace tableau { diff --git a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc index c4ca73ad..ff88dec8 100644 --- a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc @@ -12,9 +12,9 @@ namespace tableau { const std::string ItemConf::kProtoName = "ItemConf"; -bool ItemConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool ItemConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessage(data_, dir, fmt, options); + bool loaded = LoadMessageInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; diff --git a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h index e5afd07c..308b99e1 100644 --- a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h @@ -7,7 +7,7 @@ #pragma once #include -#include "messager.pc.h" +#include "load.pc.h" #include "util.pc.h" #include "item_conf.pb.h" @@ -15,7 +15,7 @@ namespace tableau { class ItemConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::ItemConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc index 46659083..ba7fe4cb 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -167,21 +167,68 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag return true; } -bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& path, Format fmt, tableau::Patch patch, - std::shared_ptr options /* = nullptr*/) { +std::shared_ptr ParseLoadOptions(std::shared_ptr opts) { + std::shared_ptr new_opts = std::make_shared(); + // set default values + new_opts->mode = LoadMode::kAll; + new_opts->read_func = util::ReadFile; + new_opts->load_func = LoadMessager; + + if (opts == nullptr) { + return new_opts; + } + if (opts->mode != LoadMode::kNone) { + new_opts->mode = opts->mode; + } + if (opts->read_func != nullptr) { + new_opts->read_func = opts->read_func; + } + if (opts->load_func != nullptr) { + new_opts->load_func = opts->load_func; + } + return new_opts; +} + +std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, + const std::string& name) { + std::shared_ptr mopts; + if (auto iter = opts->messager_options.find(name); iter != opts->messager_options.end()) { + mopts = iter->second; + } else { + mopts = std::make_shared(); + } + if (mopts->ignore_unknown_fields.has_value()) { + mopts->ignore_unknown_fields = opts->ignore_unknown_fields; + } + if (mopts->patch_dirs.empty()) { + mopts->patch_dirs = opts->patch_dirs; + } + if (mopts->mode == LoadMode::kNone) { + mopts->mode = opts->mode; + } + if (mopts->read_func == nullptr) { + mopts->read_func = opts->read_func; + } + if (mopts->load_func == nullptr) { + mopts->load_func = opts->load_func; + } + return mopts; +} + +bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::string& path, Format fmt, tableau::Patch patch, + std::shared_ptr options /* = nullptr*/) { if (options == nullptr) { - return LoadMessageByPath(msg, path, fmt, nullptr); + return LoadMessager(msg, path, fmt, nullptr); } - if (options->mode == LoadMode::kModeOnlyMain) { + if (options->mode == LoadMode::kOnlyMain) { // ignore patch files when LoadMode::kModeOnlyMain specified - return LoadMessageByPath(msg, path, fmt, nullptr); + return options->load_func(msg, path, fmt, nullptr); } std::string name = util::GetProtoName(msg); std::vector patch_paths; - auto iter = options->patch_paths.find(name); - if (iter != options->patch_paths.end()) { + if (!options->patch_paths.empty()) { // patch path specified in PatchPaths, then use it instead of PatchDirs. - patch_paths = iter->second; + patch_paths = options->patch_paths; } else { std::string filename = name + util::Format2Ext(fmt); for (auto&& patch_dir : options->patch_dirs) { @@ -200,27 +247,27 @@ bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& pat } } if (existed_patch_paths.empty()) { - if (options->mode == LoadMode::kModeOnlyPatch) { + if (options->mode == LoadMode::kOnlyPatch) { // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. return true; } // no valid patch path provided, then just load from the "main" file. - return LoadMessageByPath(msg, path, fmt, options); + return options->load_func(msg, path, fmt, options); } switch (patch) { case tableau::PATCH_REPLACE: { // just use the last "patch" file std::string& patch_path = existed_patch_paths.back(); - if (!LoadMessageByPath(msg, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { + if (!options->load_func(msg, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { return false; } break; } case tableau::PATCH_MERGE: { - if (options->mode != LoadMode::kModeOnlyPatch) { + if (options->mode != LoadMode::kOnlyPatch) { // load msg from the "main" file - if (!LoadMessageByPath(msg, path, fmt, options)) { + if (!options->load_func(msg, path, fmt, options)) { return false; } } @@ -229,7 +276,7 @@ bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& pat std::unique_ptr _auto_release(msg.New()); // load patch_msg from each "patch" file for (auto&& patch_path : existed_patch_paths) { - if (!LoadMessageByPath(*patch_msg_ptr, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { + if (!options->load_func(*patch_msg_ptr, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { return false; } if (!PatchMessage(msg, *patch_msg_ptr)) { @@ -248,8 +295,8 @@ bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& pat return true; } -bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt, - std::shared_ptr options /* = nullptr*/) { +bool LoadMessager(google::protobuf::Message& msg, const std::string& path, Format fmt, + std::shared_ptr options /* = nullptr*/) { std::string content; ReadFunc read_func = util::ReadFile; if (options != nullptr && options->read_func) { @@ -276,17 +323,14 @@ bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, } } -bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt, - std::shared_ptr options /* = nullptr*/) { +bool LoadMessageInDir(google::protobuf::Message& msg, const std::string& dir, Format fmt, + std::shared_ptr options /* = nullptr*/) { std::string name = util::GetProtoName(msg); std::string path; - if (options) { - auto iter = options->paths.find(name); - if (iter != options->paths.end()) { - // path specified in Paths, then use it instead of dir. - path = iter->second; - fmt = util::Ext2Format(util::GetExt(iter->second)); - } + if (options && !options->path.empty()) { + // path specified in Paths, then use it instead of dir. + path = options->path; + fmt = util::Ext2Format(util::GetExt(path)); } if (path.empty()) { std::string filename = name + util::Format2Ext(fmt); @@ -305,9 +349,11 @@ bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format // access the extension directly using the generated identifier const tableau::WorksheetOptions worksheet_options = descriptor->options().GetExtension(tableau::worksheet); if (worksheet_options.patch() != tableau::PATCH_NONE) { - return LoadMessageWithPatch(msg, path, fmt, worksheet_options.patch(), options); + return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); } - - return LoadMessageByPath(msg, path, fmt, options); + if (options) { + return options->load_func(msg, path, fmt, options); + } + return LoadMessager(msg, path, fmt); } } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.h b/test/cpp-tableau-loader/src/protoconf/load.pc.h index 4fc8a035..7b79bdbc 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.h @@ -6,43 +6,105 @@ #pragma once #include +#include #include +#include #include #include "util.pc.h" namespace tableau { enum class LoadMode { - kModeDefault, - kModeOnlyMain, - kModeOnlyPatch, + kNone, + kAll, // Load all related files + kOnlyMain, // Only load the main file + kOnlyPatch, // Only load the patch files }; +class MessagerOptions; +class Hub; + // ReadFunc reads the config file and returns its content. using ReadFunc = std::function; +// LoadFunc defines a func which can load message's content based on the given +// path, format, and options. +using LoadFunc = std::function options)>; -struct LoadOptions { - // read_func reads the config file and returns its content. - ReadFunc read_func; +// BaseOptions is the common struct for both global-level and messager-level +// options. +struct BaseOptions { // Whether to ignore unknown JSON fields during parsing. // // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. - bool ignore_unknown_fields = false; - // Paths maps each messager name to a corresponding config file path. + std::optional ignore_unknown_fields; + // Specify the directory paths for config patching. + std::vector patch_dirs; + // Specify the loading mode for config patching. + // - For LoadOptions, default is LoadMode::kModeAll. + // - For MessagerOptions, inherit from LoadOptions if not set. + LoadMode mode; + // You can specify custom read function to read a config file's content. + // - For LoadOptions, default is util::ReadFile. + // - For MessagerOptions, inherit from LoadOptions if not set. + ReadFunc read_func; + // You can specify custom load function to load a messager's content. + // - For LoadOptions, default is LoadMessage. + // - For MessagerOptions, inherit from LoadOptions if not set. + LoadFunc load_func; +}; + +// LoadOptionsOptions is the options struct, which contains both global-level and +// messager-level options. +struct LoadOptions : public BaseOptions { + // messager_options maps each messager name to a MessageOptions. + // If specified, then the messager will be parsed with the given options + // directly. + std::unordered_map> messager_options; +}; + +// MessagerOptions defines the options for loading a messager. +struct MessagerOptions : public BaseOptions { + // Path maps each messager name to a corresponding config file path. // If specified, then the main messager will be parsed from the file // directly, other than the specified load dir. - std::unordered_map paths; + std::string path; // Patch paths maps each messager name to one or multiple corresponding patch file paths. // If specified, then main messager will be patched. - std::unordered_map> patch_paths; - // Patch dirs specifies the directory paths for config patching. - std::vector patch_dirs; - // Mode specifies the loading mode for config patching. - LoadMode mode = LoadMode::kModeDefault; + std::vector patch_paths; +}; + +class Messager { + public: + struct Stats { + std::chrono::microseconds duration; // total load time consuming. + }; + + public: + virtual ~Messager() = default; + static const std::string& Name() { return kEmpty; } + const Stats& GetStats() { return stats_; } + // Load fills message from file in the specified directory and format. + virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) = 0; + // Message returns the inner message data. + virtual const google::protobuf::Message* Message() const { return nullptr; } + // callback after all messagers loaded. + virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } + + protected: + // callback after this messager loaded. + virtual bool ProcessAfterLoad() { return true; }; + Stats stats_; }; -bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); -bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); +// ParseLoadOptions parses load options with default global-level options. +std::shared_ptr ParseLoadOptions(std::shared_ptr opts); +// ParseMessagerOptions parses messager options with both global-level and +// messager-level options taken into consideration. +std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, + const std::string& name); +bool LoadMessager(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, + std::shared_ptr options = nullptr); +bool LoadMessageInDir(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, + std::shared_ptr options = nullptr); } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/messager.pc.h b/test/cpp-tableau-loader/src/protoconf/messager.pc.h deleted file mode 100644 index 3e8de4e7..00000000 --- a/test/cpp-tableau-loader/src/protoconf/messager.pc.h +++ /dev/null @@ -1,41 +0,0 @@ -// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. -// versions: -// - protoc-gen-cpp-tableau-loader v0.8.0 -// - protoc v3.19.3 - -#pragma once -#include - -#include -#include -#include - -#include "util.pc.h" - -namespace tableau { -class Hub; -struct LoadOptions; - -class Messager { - public: - struct Stats { - std::chrono::microseconds duration; // total load time consuming. - }; - - public: - virtual ~Messager() = default; - static const std::string& Name() { return kEmpty; } - const Stats& GetStats() { return stats_; } - // Load fills message from file in the specified directory and format. - virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) = 0; - // Message returns the inner message data. - virtual const google::protobuf::Message* Message() const { return nullptr; } - // callback after all messagers loaded. - virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } - - protected: - // callback after this messager loaded. - virtual bool ProcessAfterLoad() { return true; }; - Stats stats_; -}; -} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc index 22f8dc1d..47816b58 100644 --- a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc @@ -12,9 +12,9 @@ namespace tableau { const std::string PatchReplaceConf::kProtoName = "PatchReplaceConf"; -bool PatchReplaceConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool PatchReplaceConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessage(data_, dir, fmt, options); + bool loaded = LoadMessageInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; @@ -22,9 +22,9 @@ bool PatchReplaceConf::Load(const std::string& dir, Format fmt, std::shared_ptr< const std::string PatchMergeConf::kProtoName = "PatchMergeConf"; -bool PatchMergeConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool PatchMergeConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessage(data_, dir, fmt, options); + bool loaded = LoadMessageInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; @@ -40,9 +40,9 @@ const protoconf::Item* PatchMergeConf::Get(uint32_t id) const { const std::string RecursivePatchConf::kProtoName = "RecursivePatchConf"; -bool RecursivePatchConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool RecursivePatchConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessage(data_, dir, fmt, options); + bool loaded = LoadMessageInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; diff --git a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h index 6f0c366a..191fd685 100644 --- a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h @@ -7,7 +7,7 @@ #pragma once #include -#include "messager.pc.h" +#include "load.pc.h" #include "util.pc.h" #include "patch_conf.pb.h" @@ -15,7 +15,7 @@ namespace tableau { class PatchReplaceConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::PatchReplaceConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -28,7 +28,7 @@ class PatchReplaceConf : public Messager { class PatchMergeConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::PatchMergeConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -43,7 +43,7 @@ class PatchMergeConf : public Messager { class RecursivePatchConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::RecursivePatchConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } diff --git a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc index effa1bd9..6137f554 100644 --- a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc @@ -12,9 +12,9 @@ namespace tableau { const std::string ActivityConf::kProtoName = "ActivityConf"; -bool ActivityConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool ActivityConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessage(data_, dir, fmt, options); + bool loaded = LoadMessageInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; @@ -240,9 +240,9 @@ const protoconf::Section::SectionItem* ActivityConf::FindFirstAward(uint32_t id) const std::string ChapterConf::kProtoName = "ChapterConf"; -bool ChapterConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool ChapterConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessage(data_, dir, fmt, options); + bool loaded = LoadMessageInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; @@ -258,9 +258,9 @@ const protoconf::ChapterConf::Chapter* ChapterConf::Get(uint64_t id) const { const std::string ThemeConf::kProtoName = "ThemeConf"; -bool ThemeConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool ThemeConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessage(data_, dir, fmt, options); + bool loaded = LoadMessageInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; @@ -288,9 +288,9 @@ const std::string* ThemeConf::Get(const std::string& name, const std::string& pa const std::string TaskConf::kProtoName = "TaskConf"; -bool TaskConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool TaskConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessage(data_, dir, fmt, options); + bool loaded = LoadMessageInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; diff --git a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h index 09aeb632..2b12c0e0 100644 --- a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h @@ -7,7 +7,7 @@ #pragma once #include -#include "messager.pc.h" +#include "load.pc.h" #include "util.pc.h" #include "test_conf.pb.h" @@ -15,7 +15,7 @@ namespace tableau { class ActivityConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::ActivityConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -102,7 +102,7 @@ class ActivityConf : public Messager { class ChapterConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::ChapterConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -117,7 +117,7 @@ class ChapterConf : public Messager { class ThemeConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::ThemeConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -133,7 +133,7 @@ class ThemeConf : public Messager { class TaskConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::TaskConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.cc b/test/cpp-tableau-loader/src/protoconf/util.pc.cc index fb61f76e..05068ead 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.cc @@ -25,7 +25,6 @@ #include "load.pc.h" #include "logger.pc.h" -#include "messager.pc.h" namespace tableau { #ifdef _WIN32 @@ -139,11 +138,11 @@ const std::string& Format2Ext(Format fmt) { } bool JSON2Message(const std::string& json, google::protobuf::Message& msg, - std::shared_ptr options /* = nullptr */) { + std::shared_ptr options /* = nullptr */) { google::protobuf::util::Status status; if (options != nullptr) { google::protobuf::util::JsonParseOptions parse_options; - parse_options.ignore_unknown_fields = options->ignore_unknown_fields; + parse_options.ignore_unknown_fields = options->ignore_unknown_fields.value_or(false); status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); } else { status = google::protobuf::util::JsonStringToMessage(json, &msg); diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.h b/test/cpp-tableau-loader/src/protoconf/util.pc.h index e6b70f90..63027192 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.h @@ -38,7 +38,7 @@ extern const std::string kJSONExt; extern const std::string kTextExt; extern const std::string kBinExt; -struct LoadOptions; +struct MessagerOptions; namespace util { // Combine hash values @@ -88,7 +88,7 @@ Format Ext2Format(const std::string& ext); const std::string& Format2Ext(Format fmt); bool JSON2Message(const std::string& json, google::protobuf::Message& msg, - std::shared_ptr options = nullptr); + std::shared_ptr options = nullptr); bool Text2Message(const std::string& text, google::protobuf::Message& msg); bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); From cd4c0c023f58729b43a28fdab18efa4fe9e44ea9 Mon Sep 17 00:00:00 2001 From: wenchy Date: Thu, 14 Aug 2025 21:24:03 +0800 Subject: [PATCH 2/9] fix: test --- .../src/protoconf/hero_conf.pc.cc | 4 ++-- .../src/protoconf/item_conf.pc.cc | 2 +- test/cpp-tableau-loader/src/protoconf/load.pc.cc | 14 ++++++++++---- test/cpp-tableau-loader/src/protoconf/load.pc.h | 2 +- .../src/protoconf/patch_conf.pc.cc | 6 +++--- .../src/protoconf/test_conf.pc.cc | 8 ++++---- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc index 2deaf59c..d0182838 100644 --- a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc @@ -14,7 +14,7 @@ const std::string HeroConf::kProtoName = "HeroConf"; bool HeroConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessageInDir(data_, dir, fmt, options); + bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; @@ -73,7 +73,7 @@ const std::string HeroBaseConf::kProtoName = "HeroBaseConf"; bool HeroBaseConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessageInDir(data_, dir, fmt, options); + bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; diff --git a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc index ff88dec8..a1460ba4 100644 --- a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc @@ -14,7 +14,7 @@ const std::string ItemConf::kProtoName = "ItemConf"; bool ItemConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessageInDir(data_, dir, fmt, options); + bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc index ba7fe4cb..d3bdb231 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -177,6 +177,12 @@ std::shared_ptr ParseLoadOptions(std::shared_ptrignore_unknown_fields.has_value()) { + new_opts->ignore_unknown_fields = opts->ignore_unknown_fields; + } + if (!opts->patch_dirs.empty()) { + new_opts->patch_dirs = opts->patch_dirs; + } if (opts->mode != LoadMode::kNone) { new_opts->mode = opts->mode; } @@ -216,7 +222,7 @@ std::shared_ptr ParseMessagerOptions(std::shared_ptr options /* = nullptr*/) { + std::shared_ptr options /* = nullptr*/) { if (options == nullptr) { return LoadMessager(msg, path, fmt, nullptr); } @@ -296,7 +302,7 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::string& pa } bool LoadMessager(google::protobuf::Message& msg, const std::string& path, Format fmt, - std::shared_ptr options /* = nullptr*/) { + std::shared_ptr options /* = nullptr*/) { std::string content; ReadFunc read_func = util::ReadFile; if (options != nullptr && options->read_func) { @@ -323,8 +329,8 @@ bool LoadMessager(google::protobuf::Message& msg, const std::string& path, Forma } } -bool LoadMessageInDir(google::protobuf::Message& msg, const std::string& dir, Format fmt, - std::shared_ptr options /* = nullptr*/) { +bool LoadMessagerInDir(google::protobuf::Message& msg, const std::string& dir, Format fmt, + std::shared_ptr options /* = nullptr*/) { std::string name = util::GetProtoName(msg); std::string path; if (options && !options->path.empty()) { diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.h b/test/cpp-tableau-loader/src/protoconf/load.pc.h index 7b79bdbc..ebf0a527 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.h @@ -105,6 +105,6 @@ std::shared_ptr ParseMessagerOptions(std::shared_ptr options = nullptr); -bool LoadMessageInDir(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, +bool LoadMessagerInDir(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, std::shared_ptr options = nullptr); } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc index 47816b58..66a14069 100644 --- a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc @@ -14,7 +14,7 @@ const std::string PatchReplaceConf::kProtoName = "PatchReplaceConf"; bool PatchReplaceConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessageInDir(data_, dir, fmt, options); + bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; @@ -24,7 +24,7 @@ const std::string PatchMergeConf::kProtoName = "PatchMergeConf"; bool PatchMergeConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessageInDir(data_, dir, fmt, options); + bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; @@ -42,7 +42,7 @@ const std::string RecursivePatchConf::kProtoName = "RecursivePatchConf"; bool RecursivePatchConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessageInDir(data_, dir, fmt, options); + bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; diff --git a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc index 6137f554..c178d100 100644 --- a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc @@ -14,7 +14,7 @@ const std::string ActivityConf::kProtoName = "ActivityConf"; bool ActivityConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessageInDir(data_, dir, fmt, options); + bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; @@ -242,7 +242,7 @@ const std::string ChapterConf::kProtoName = "ChapterConf"; bool ChapterConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessageInDir(data_, dir, fmt, options); + bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; @@ -260,7 +260,7 @@ const std::string ThemeConf::kProtoName = "ThemeConf"; bool ThemeConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessageInDir(data_, dir, fmt, options); + bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; @@ -290,7 +290,7 @@ const std::string TaskConf::kProtoName = "TaskConf"; bool TaskConf::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; - bool loaded = LoadMessageInDir(data_, dir, fmt, options); + bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; stats_.duration = profiler.Elapse(); return ok; From 6dc0f1c9622ee4eeddbbbbacfab18de36b4bea93 Mon Sep 17 00:00:00 2001 From: wenchy Date: Thu, 14 Aug 2025 21:43:27 +0800 Subject: [PATCH 3/9] refactor: cpp loader codegen --- .../embed/load.pc.cc | 341 ++++++++++-------- .../embed/load.pc.h | 98 ++++- .../embed/messager.pc.h | 36 -- .../embed/util.pc.cc | 5 +- .../embed/util.pc.h | 4 +- cmd/protoc-gen-cpp-tableau-loader/hub.go | 7 +- cmd/protoc-gen-cpp-tableau-loader/messager.go | 8 +- .../src/protoconf/load.pc.cc | 307 ++++++++-------- .../src/protoconf/load.pc.h | 4 +- 9 files changed, 445 insertions(+), 365 deletions(-) delete mode 100644 cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc index 74467101..d8af6e32 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc @@ -8,6 +8,203 @@ #include "util.pc.h" namespace tableau { + +std::shared_ptr ParseLoadOptions(std::shared_ptr opts) { + std::shared_ptr new_opts = std::make_shared(); + // set default values + new_opts->mode = LoadMode::kAll; + new_opts->read_func = util::ReadFile; + new_opts->load_func = LoadMessager; + + if (opts == nullptr) { + return new_opts; + } + if (opts->ignore_unknown_fields.has_value()) { + new_opts->ignore_unknown_fields = opts->ignore_unknown_fields; + } + if (!opts->patch_dirs.empty()) { + new_opts->patch_dirs = opts->patch_dirs; + } + if (opts->mode != LoadMode::kNone) { + new_opts->mode = opts->mode; + } + if (opts->read_func != nullptr) { + new_opts->read_func = opts->read_func; + } + if (opts->load_func != nullptr) { + new_opts->load_func = opts->load_func; + } + return new_opts; +} + +std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, + const std::string& name) { + std::shared_ptr mopts; + if (auto iter = opts->messager_options.find(name); iter != opts->messager_options.end()) { + mopts = iter->second; + } else { + mopts = std::make_shared(); + } + if (mopts->ignore_unknown_fields.has_value()) { + mopts->ignore_unknown_fields = opts->ignore_unknown_fields; + } + if (mopts->patch_dirs.empty()) { + mopts->patch_dirs = opts->patch_dirs; + } + if (mopts->mode == LoadMode::kNone) { + mopts->mode = opts->mode; + } + if (mopts->read_func == nullptr) { + mopts->read_func = opts->read_func; + } + if (mopts->load_func == nullptr) { + mopts->load_func = opts->load_func; + } + return mopts; +} + +bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::string& path, Format fmt, tableau::Patch patch, + std::shared_ptr options /* = nullptr*/) { + if (options == nullptr) { + return LoadMessager(msg, path, fmt, nullptr); + } + if (options->mode == LoadMode::kOnlyMain) { + // ignore patch files when LoadMode::kModeOnlyMain specified + return options->load_func(msg, path, fmt, nullptr); + } + std::string name = util::GetProtoName(msg); + std::vector patch_paths; + if (!options->patch_paths.empty()) { + // patch path specified in PatchPaths, then use it instead of PatchDirs. + patch_paths = options->patch_paths; + } else { + std::string filename = name + util::Format2Ext(fmt); + for (auto&& patch_dir : options->patch_dirs) { +#if __cplusplus >= 201703L + patch_paths.emplace_back((std::filesystem::path(patch_dir) / filename).make_preferred().string()); +#else + patch_paths.emplace_back(patch_dir + kPathSeperator + filename); +#endif + } + } + + std::vector existed_patch_paths; + for (auto&& patch_path : patch_paths) { + if (util::ExistsFile(patch_path)) { + existed_patch_paths.emplace_back(patch_path); + } + } + if (existed_patch_paths.empty()) { + if (options->mode == LoadMode::kOnlyPatch) { + // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. + return true; + } + // no valid patch path provided, then just load from the "main" file. + return options->load_func(msg, path, fmt, options); + } + + switch (patch) { + case tableau::PATCH_REPLACE: { + // just use the last "patch" file + std::string& patch_path = existed_patch_paths.back(); + if (!options->load_func(msg, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { + return false; + } + break; + } + case tableau::PATCH_MERGE: { + if (options->mode != LoadMode::kOnlyPatch) { + // load msg from the "main" file + if (!options->load_func(msg, path, fmt, options)) { + return false; + } + } + // Create a new instance of the same type of the original message + google::protobuf::Message* patch_msg_ptr = msg.New(); + std::unique_ptr _auto_release(msg.New()); + // load patch_msg from each "patch" file + for (auto&& patch_path : existed_patch_paths) { + if (!options->load_func(*patch_msg_ptr, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { + return false; + } + if (!PatchMessage(msg, *patch_msg_ptr)) { + return false; + } + } + break; + } + default: { + SetErrMsg("unknown patch type: " + util::GetPatchName(patch)); + return false; + } + } + ATOM_DEBUG("patched(%s) %s by %s: %s", util::GetPatchName(patch).c_str(), name.c_str(), + ATOM_VECTOR_STR(existed_patch_paths).c_str(), msg.ShortDebugString().c_str()); + return true; +} + +bool LoadMessager(google::protobuf::Message& msg, const std::string& path, Format fmt, + std::shared_ptr options /* = nullptr*/) { + std::string content; + ReadFunc read_func = util::ReadFile; + if (options != nullptr && options->read_func) { + read_func = options->read_func; + } + bool ok = read_func(path, content); + if (!ok) { + return false; + } + switch (fmt) { + case Format::kJSON: { + return util::JSON2Message(content, msg, options); + } + case Format::kText: { + return util::Text2Message(content, msg); + } + case Format::kBin: { + return util::Bin2Message(content, msg); + } + default: { + SetErrMsg("unknown format: " + std::to_string(static_cast(fmt))); + return false; + } + } +} + +bool LoadMessagerInDir(google::protobuf::Message& msg, const std::string& dir, Format fmt, + std::shared_ptr options /* = nullptr*/) { + std::string name = util::GetProtoName(msg); + std::string path; + if (options && !options->path.empty()) { + // path specified in Paths, then use it instead of dir. + path = options->path; + fmt = util::Ext2Format(util::GetExt(path)); + } + if (path.empty()) { + std::string filename = name + util::Format2Ext(fmt); +#if __cplusplus >= 201703L + path = (std::filesystem::path(dir) / filename).make_preferred().string(); +#else + path = dir + kPathSeperator + filename; +#endif + } + + const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); + if (!descriptor) { + SetErrMsg("failed to get descriptor of message: " + name); + return false; + } + // access the extension directly using the generated identifier + const tableau::WorksheetOptions worksheet_options = descriptor->options().GetExtension(tableau::worksheet); + if (worksheet_options.patch() != tableau::PATCH_NONE) { + return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); + } + if (options) { + return options->load_func(msg, path, fmt, options); + } + return LoadMessager(msg, path, fmt); +} + #ifdef _WIN32 #undef GetMessage #endif @@ -161,148 +358,4 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); return true; } - -bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& path, Format fmt, tableau::Patch patch, - std::shared_ptr options /* = nullptr*/) { - if (options == nullptr) { - return LoadMessageByPath(msg, path, fmt, nullptr); - } - if (options->mode == LoadMode::kModeOnlyMain) { - // ignore patch files when LoadMode::kModeOnlyMain specified - return LoadMessageByPath(msg, path, fmt, nullptr); - } - std::string name = util::GetProtoName(msg); - std::vector patch_paths; - auto iter = options->patch_paths.find(name); - if (iter != options->patch_paths.end()) { - // patch path specified in PatchPaths, then use it instead of PatchDirs. - patch_paths = iter->second; - } else { - std::string filename = name + util::Format2Ext(fmt); - for (auto&& patch_dir : options->patch_dirs) { -#if __cplusplus >= 201703L - patch_paths.emplace_back((std::filesystem::path(patch_dir) / filename).make_preferred().string()); -#else - patch_paths.emplace_back(patch_dir + kPathSeperator + filename); -#endif - } - } - - std::vector existed_patch_paths; - for (auto&& patch_path : patch_paths) { - if (util::ExistsFile(patch_path)) { - existed_patch_paths.emplace_back(patch_path); - } - } - if (existed_patch_paths.empty()) { - if (options->mode == LoadMode::kModeOnlyPatch) { - // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. - return true; - } - // no valid patch path provided, then just load from the "main" file. - return LoadMessageByPath(msg, path, fmt, options); - } - - switch (patch) { - case tableau::PATCH_REPLACE: { - // just use the last "patch" file - std::string& patch_path = existed_patch_paths.back(); - if (!LoadMessageByPath(msg, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { - return false; - } - break; - } - case tableau::PATCH_MERGE: { - if (options->mode != LoadMode::kModeOnlyPatch) { - // load msg from the "main" file - if (!LoadMessageByPath(msg, path, fmt, options)) { - return false; - } - } - // Create a new instance of the same type of the original message - google::protobuf::Message* patch_msg_ptr = msg.New(); - std::unique_ptr _auto_release(msg.New()); - // load patch_msg from each "patch" file - for (auto&& patch_path : existed_patch_paths) { - if (!LoadMessageByPath(*patch_msg_ptr, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { - return false; - } - if (!PatchMessage(msg, *patch_msg_ptr)) { - return false; - } - } - break; - } - default: { - SetErrMsg("unknown patch type: " + util::GetPatchName(patch)); - return false; - } - } - ATOM_DEBUG("patched(%s) %s by %s: %s", util::GetPatchName(patch).c_str(), name.c_str(), - ATOM_VECTOR_STR(existed_patch_paths).c_str(), msg.ShortDebugString().c_str()); - return true; -} - -bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt, - std::shared_ptr options /* = nullptr*/) { - std::string content; - ReadFunc read_func = util::ReadFile; - if (options != nullptr && options->read_func) { - read_func = options->read_func; - } - bool ok = read_func(path, content); - if (!ok) { - return false; - } - switch (fmt) { - case Format::kJSON: { - return util::JSON2Message(content, msg, options); - } - case Format::kText: { - return util::Text2Message(content, msg); - } - case Format::kBin: { - return util::Bin2Message(content, msg); - } - default: { - SetErrMsg("unknown format: " + std::to_string(static_cast(fmt))); - return false; - } - } -} - -bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt, - std::shared_ptr options /* = nullptr*/) { - std::string name = util::GetProtoName(msg); - std::string path; - if (options) { - auto iter = options->paths.find(name); - if (iter != options->paths.end()) { - // path specified in Paths, then use it instead of dir. - path = iter->second; - fmt = util::Ext2Format(util::GetExt(iter->second)); - } - } - if (path.empty()) { - std::string filename = name + util::Format2Ext(fmt); -#if __cplusplus >= 201703L - path = (std::filesystem::path(dir) / filename).make_preferred().string(); -#else - path = dir + kPathSeperator + filename; -#endif - } - - const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); - if (!descriptor) { - SetErrMsg("failed to get descriptor of message: " + name); - return false; - } - // access the extension directly using the generated identifier - const tableau::WorksheetOptions worksheet_options = descriptor->options().GetExtension(tableau::worksheet); - if (worksheet_options.patch() != tableau::PATCH_NONE) { - return LoadMessageWithPatch(msg, path, fmt, worksheet_options.patch(), options); - } - - return LoadMessageByPath(msg, path, fmt, options); -} } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h index b64ca32b..41cd5d13 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h @@ -1,43 +1,105 @@ #pragma once #include +#include #include +#include #include #include "util.pc.h" namespace tableau { enum class LoadMode { - kModeDefault, - kModeOnlyMain, - kModeOnlyPatch, + kNone, + kAll, // Load all related files + kOnlyMain, // Only load the main file + kOnlyPatch, // Only load the patch files }; +class MessagerOptions; +class Hub; + // ReadFunc reads the config file and returns its content. using ReadFunc = std::function; +// LoadFunc defines a func which can load message's content based on the given +// path, format, and options. +using LoadFunc = std::function options)>; -struct LoadOptions { - // read_func reads the config file and returns its content. - ReadFunc read_func; +// BaseOptions is the common struct for both global-level and messager-level +// options. +struct BaseOptions { // Whether to ignore unknown JSON fields during parsing. // // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. - bool ignore_unknown_fields = false; - // Paths maps each messager name to a corresponding config file path. + std::optional ignore_unknown_fields; + // Specify the directory paths for config patching. + std::vector patch_dirs; + // Specify the loading mode for config patching. + // - For LoadOptions, default is LoadMode::kModeAll. + // - For MessagerOptions, inherit from LoadOptions if not set. + LoadMode mode; + // You can specify custom read function to read a config file's content. + // - For LoadOptions, default is util::ReadFile. + // - For MessagerOptions, inherit from LoadOptions if not set. + ReadFunc read_func; + // You can specify custom load function to load a messager's content. + // - For LoadOptions, default is LoadMessage. + // - For MessagerOptions, inherit from LoadOptions if not set. + LoadFunc load_func; +}; + +// LoadOptionsOptions is the options struct, which contains both global-level and +// messager-level options. +struct LoadOptions : public BaseOptions { + // messager_options maps each messager name to a MessageOptions. + // If specified, then the messager will be parsed with the given options + // directly. + std::unordered_map> messager_options; +}; + +// MessagerOptions defines the options for loading a messager. +struct MessagerOptions : public BaseOptions { + // Path maps each messager name to a corresponding config file path. // If specified, then the main messager will be parsed from the file // directly, other than the specified load dir. - std::unordered_map paths; + std::string path; // Patch paths maps each messager name to one or multiple corresponding patch file paths. // If specified, then main messager will be patched. - std::unordered_map> patch_paths; - // Patch dirs specifies the directory paths for config patching. - std::vector patch_dirs; - // Mode specifies the loading mode for config patching. - LoadMode mode = LoadMode::kModeDefault; + std::vector patch_paths; +}; + +class Messager { + public: + struct Stats { + std::chrono::microseconds duration; // total load time consuming. + }; + + public: + virtual ~Messager() = default; + static const std::string& Name() { return kEmpty; } + const Stats& GetStats() { return stats_; } + // Load fills message from file in the specified directory and format. + virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) = 0; + // Message returns the inner message data. + virtual const google::protobuf::Message* Message() const { return nullptr; } + // callback after all messagers loaded. + virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } + + protected: + // callback after this messager loaded. + virtual bool ProcessAfterLoad() { return true; }; + Stats stats_; }; -bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); -bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); +// ParseLoadOptions parses load options with default global-level options. +std::shared_ptr ParseLoadOptions(std::shared_ptr opts); +// ParseMessagerOptions parses messager options with both global-level and +// messager-level options taken into consideration. +std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, + const std::string& name); +bool LoadMessager(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, + std::shared_ptr options = nullptr); +bool LoadMessagerInDir(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, + std::shared_ptr options = nullptr); } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h deleted file mode 100644 index 7d5bda88..00000000 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once -#include - -#include -#include -#include - -#include "util.pc.h" - -namespace tableau { -class Hub; -struct LoadOptions; - -class Messager { - public: - struct Stats { - std::chrono::microseconds duration; // total load time consuming. - }; - - public: - virtual ~Messager() = default; - static const std::string& Name() { return kEmpty; } - const Stats& GetStats() { return stats_; } - // Load fills message from file in the specified directory and format. - virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) = 0; - // Message returns the inner message data. - virtual const google::protobuf::Message* Message() const { return nullptr; } - // callback after all messagers loaded. - virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } - - protected: - // callback after this messager loaded. - virtual bool ProcessAfterLoad() { return true; }; - Stats stats_; -}; -} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc index 5ecd56e3..aa306205 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc @@ -20,7 +20,6 @@ #include "load.pc.h" #include "logger.pc.h" -#include "messager.pc.h" namespace tableau { #ifdef _WIN32 @@ -134,11 +133,11 @@ const std::string& Format2Ext(Format fmt) { } bool JSON2Message(const std::string& json, google::protobuf::Message& msg, - std::shared_ptr options /* = nullptr */) { + std::shared_ptr options /* = nullptr */) { google::protobuf::util::Status status; if (options != nullptr) { google::protobuf::util::JsonParseOptions parse_options; - parse_options.ignore_unknown_fields = options->ignore_unknown_fields; + parse_options.ignore_unknown_fields = options->ignore_unknown_fields.value_or(false); status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); } else { status = google::protobuf::util::JsonStringToMessage(json, &msg); diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h index 60debed4..9c983c69 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h @@ -33,7 +33,7 @@ extern const std::string kJSONExt; extern const std::string kTextExt; extern const std::string kBinExt; -struct LoadOptions; +struct MessagerOptions; namespace util { // Combine hash values @@ -83,7 +83,7 @@ Format Ext2Format(const std::string& ext); const std::string& Format2Ext(Format fmt); bool JSON2Message(const std::string& json, google::protobuf::Message& msg, - std::shared_ptr options = nullptr); + std::shared_ptr options = nullptr); bool Text2Message(const std::string& text, google::protobuf::Message& msg); bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); diff --git a/cmd/protoc-gen-cpp-tableau-loader/hub.go b/cmd/protoc-gen-cpp-tableau-loader/hub.go index 5e0e468e..63453bed 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/hub.go +++ b/cmd/protoc-gen-cpp-tableau-loader/hub.go @@ -146,7 +146,6 @@ const hubHpp = `#pragma once #include #include "load.pc.h" -#include "messager.pc.h" #include "scheduler.pc.h" namespace tableau { @@ -294,8 +293,8 @@ void Registry::Register() { const hubCppHeader = `#include "hub.pc.h" +#include "load.pc.h" #include "logger.pc.h" -#include "messager.pc.h" #include "util.pc.h"` const hubCpp = ` @@ -348,11 +347,13 @@ std::shared_ptr Hub::InternalLoad(const std::string& dir, Format fm std::shared_ptr options /* = nullptr */) const { // intercept protobuf error logs auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); + auto opts = ParseLoadOptions(options); auto msger_map = NewMessagerMap(); for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); - bool ok = iter.second->Load(dir, fmt, options); + auto mopts = ParseMessagerOptions(opts, name); + bool ok = iter.second->Load(dir, fmt, mopts); if (!ok) { ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str()); // restore to old protobuf log handler diff --git a/cmd/protoc-gen-cpp-tableau-loader/messager.go b/cmd/protoc-gen-cpp-tableau-loader/messager.go index fef3c3a3..19f2e3a5 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/messager.go +++ b/cmd/protoc-gen-cpp-tableau-loader/messager.go @@ -45,7 +45,7 @@ func generateHppFileContent(file *protogen.File, g *protogen.GeneratedFile) { g.P("#pragma once") g.P("#include ") g.P() - g.P(`#include "`, "messager.", pcExt, `.h"`) + g.P(`#include "`, "load.", pcExt, `.h"`) g.P(`#include "`, "util.", pcExt, `.h"`) g.P(`#include "`, file.GeneratedFilenamePrefix, ".", pbExt, `.h"`) g.P() @@ -85,7 +85,7 @@ func genHppMessage(file *protogen.File, g *protogen.GeneratedFile, message *prot g.P("class ", message.Desc.Name(), " : public Messager {") g.P(" public:") g.P(helper.Indent(1), "static const std::string& Name() { return kProtoName; }") - g.P(helper.Indent(1), "virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override;") + g.P(helper.Indent(1), "virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr options = nullptr) override;") g.P(helper.Indent(1), "const ", cppFullName, "& Data() const { return data_; }") g.P(helper.Indent(1), "const google::protobuf::Message* Message() const override { return &data_; }") g.P() @@ -158,9 +158,9 @@ func genCppMessage(g *protogen.GeneratedFile, message *protogen.Message) { g.P("const std::string ", messagerName, "::kProtoName = ", `"`, messagerName, `";`) g.P() - g.P("bool ", messagerName, "::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) {") + g.P("bool ", messagerName, "::Load(const std::string& dir, Format fmt, std::shared_ptr options /* = nullptr */) {") g.P(helper.Indent(1), "tableau::util::TimeProfiler profiler;") - g.P(helper.Indent(1), "bool loaded = LoadMessage(data_, dir, fmt, options);") + g.P(helper.Indent(1), "bool loaded = LoadMessagerInDir(data_, dir, fmt, options);") g.P(helper.Indent(1), "bool ok = loaded ? ProcessAfterLoad() : false;") g.P(helper.Indent(1), "stats_.duration = profiler.Elapse();") g.P(helper.Indent(1), "return ok;") diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc index d3bdb231..c41e76dd 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -13,159 +13,6 @@ #include "util.pc.h" namespace tableau { -#ifdef _WIN32 -#undef GetMessage -#endif - -// PatchMessage patches src into dst, which must be a message with the same descriptor. -// -// # Default PatchMessage mechanism -// - scalar: Populated scalar fields in src are copied to dst. -// - message: Populated singular messages in src are merged into dst by -// recursively calling [xproto.PatchMessage], or replace dst message if -// "PATCH_REPLACE" is specified for this field. -// - list: The elements of every list field in src are appended to the -// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" -// is specified for this field. -// - map: The entries of every map field in src are MERGED (different from -// the behavior of proto.Merge) into the corresponding map field in dst, -// or replace dst map if "PATCH_REPLACE" is specified for this field. -// - unknown: The unknown fields of src are appended to the unknown -// fields of dst (TODO: untested). -// -// # References: -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details -bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { - const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); - const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); - // Ensure both messages are of the same type - if (dst_descriptor != src_descriptor) { - SetErrMsg("dst and src are not messages with the same descriptor"); - ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), - src_descriptor->name().c_str()); - return false; - } - - // Get the reflection and descriptor for the messages - const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); - const google::protobuf::Reflection* src_reflection = src.GetReflection(); - - // List all populated fields - std::vector fields; - src_reflection->ListFields(src, &fields); - - // Iterates over every populated field. - for (auto fd : fields) { - const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); - tableau::Patch patch = opts.prop().patch(); - if (patch == tableau::PATCH_REPLACE) { - dst_reflection->ClearField(&dst, fd); - } - if (fd->is_map()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 - auto key_fd = fd->message_type()->map_key(); - int src_count = src_reflection->FieldSize(src, fd); - int dst_count = dst_reflection->FieldSize(dst, fd); - switch (key_fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - std::unordered_map dst_key_index_map; \ - for (int i = 0; i < dst_count; i++) { \ - auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ - TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ - dst_key_index_map[key] = i; \ - } \ - for (int j = 0; j < src_count; j++) { \ - auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ - TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ - auto it = dst_key_index_map.find(key); \ - if (it != dst_key_index_map.end()) { \ - int index = it->second; \ - auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ - PatchMessage(dst_entry, src_entry); \ - } else { \ - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ - } \ - } \ - break; \ - } - - HANDLE_TYPE(INT32, Int32, int32_t); - HANDLE_TYPE(INT64, Int64, int64_t); - HANDLE_TYPE(UINT32, UInt32, uint32_t); - HANDLE_TYPE(UINT64, UInt64, uint64_t); - HANDLE_TYPE(BOOL, Bool, bool); - HANDLE_TYPE(STRING, String, std::string); - default: { - // other types are impossible to be protobuf map key - ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); - break; - } -#undef HANDLE_TYPE - } - } else if (fd->is_repeated()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 - int count = src_reflection->FieldSize(src, fd); - for (int j = 0; j < count; j++) { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ - break; \ - } - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { - const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); - break; - } - } - } - } else { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ - dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ - break; - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: - const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); - PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); - break; - } - } - } - - dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); - return true; -} std::shared_ptr ParseLoadOptions(std::shared_ptr opts) { std::shared_ptr new_opts = std::make_shared(); @@ -362,4 +209,158 @@ bool LoadMessagerInDir(google::protobuf::Message& msg, const std::string& dir, F } return LoadMessager(msg, path, fmt); } + +#ifdef _WIN32 +#undef GetMessage +#endif + +// PatchMessage patches src into dst, which must be a message with the same descriptor. +// +// # Default PatchMessage mechanism +// - scalar: Populated scalar fields in src are copied to dst. +// - message: Populated singular messages in src are merged into dst by +// recursively calling [xproto.PatchMessage], or replace dst message if +// "PATCH_REPLACE" is specified for this field. +// - list: The elements of every list field in src are appended to the +// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" +// is specified for this field. +// - map: The entries of every map field in src are MERGED (different from +// the behavior of proto.Merge) into the corresponding map field in dst, +// or replace dst map if "PATCH_REPLACE" is specified for this field. +// - unknown: The unknown fields of src are appended to the unknown +// fields of dst (TODO: untested). +// +// # References: +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { + const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); + const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); + // Ensure both messages are of the same type + if (dst_descriptor != src_descriptor) { + SetErrMsg("dst and src are not messages with the same descriptor"); + ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), + src_descriptor->name().c_str()); + return false; + } + + // Get the reflection and descriptor for the messages + const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); + const google::protobuf::Reflection* src_reflection = src.GetReflection(); + + // List all populated fields + std::vector fields; + src_reflection->ListFields(src, &fields); + + // Iterates over every populated field. + for (auto fd : fields) { + const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); + tableau::Patch patch = opts.prop().patch(); + if (patch == tableau::PATCH_REPLACE) { + dst_reflection->ClearField(&dst, fd); + } + if (fd->is_map()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 + auto key_fd = fd->message_type()->map_key(); + int src_count = src_reflection->FieldSize(src, fd); + int dst_count = dst_reflection->FieldSize(dst, fd); + switch (key_fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + std::unordered_map dst_key_index_map; \ + for (int i = 0; i < dst_count; i++) { \ + auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ + TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ + dst_key_index_map[key] = i; \ + } \ + for (int j = 0; j < src_count; j++) { \ + auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ + TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ + auto it = dst_key_index_map.find(key); \ + if (it != dst_key_index_map.end()) { \ + int index = it->second; \ + auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ + PatchMessage(dst_entry, src_entry); \ + } else { \ + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ + } \ + } \ + break; \ + } + + HANDLE_TYPE(INT32, Int32, int32_t); + HANDLE_TYPE(INT64, Int64, int64_t); + HANDLE_TYPE(UINT32, UInt32, uint32_t); + HANDLE_TYPE(UINT64, UInt64, uint64_t); + HANDLE_TYPE(BOOL, Bool, bool); + HANDLE_TYPE(STRING, String, std::string); + default: { + // other types are impossible to be protobuf map key + ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); + break; + } +#undef HANDLE_TYPE + } + } else if (fd->is_repeated()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 + int count = src_reflection->FieldSize(src, fd); + for (int j = 0; j < count; j++) { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ + break; \ + } + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); + break; + } + } + } + } else { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ + dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ + break; + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); + PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); + break; + } + } + } + + dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); + return true; +} } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.h b/test/cpp-tableau-loader/src/protoconf/load.pc.h index ebf0a527..3a475a81 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.h @@ -104,7 +104,7 @@ std::shared_ptr ParseLoadOptions(std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, const std::string& name); bool LoadMessager(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); + std::shared_ptr options = nullptr); bool LoadMessagerInDir(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); + std::shared_ptr options = nullptr); } // namespace tableau From 85acb4a8249b7c30534abce06fd36e7fd178a854 Mon Sep 17 00:00:00 2001 From: wenchy Date: Thu, 14 Aug 2025 21:48:28 +0800 Subject: [PATCH 4/9] fix: cpp loader codegen --- cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc | 3 +++ test/cpp-tableau-loader/src/protoconf/load.pc.cc | 3 +++ 2 files changed, 6 insertions(+) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc index d8af6e32..e8be38f4 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc @@ -9,6 +9,9 @@ namespace tableau { +// Forward declaration of the PatchMessage function +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); + std::shared_ptr ParseLoadOptions(std::shared_ptr opts) { std::shared_ptr new_opts = std::make_shared(); // set default values diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc index c41e76dd..d02c5e5b 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -14,6 +14,9 @@ namespace tableau { +// Forward declaration of the PatchMessage function +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); + std::shared_ptr ParseLoadOptions(std::shared_ptr opts) { std::shared_ptr new_opts = std::make_shared(); // set default values From c1ec130e41331d5dbbfef433736ff803c17b6a66 Mon Sep 17 00:00:00 2001 From: wenchy Date: Thu, 14 Aug 2025 22:25:56 +0800 Subject: [PATCH 5/9] feat: support at least c++17 --- README.md | 5 +- .../embed/load.pc.cc | 10 ---- .../embed/util.pc.cc | 56 +------------------ .../embed/util.pc.h | 9 --- test/cpp-tableau-loader/src/CMakeLists.txt | 12 ++-- .../src/protoconf/load.pc.cc | 10 ---- .../src/protoconf/util.pc.cc | 56 +------------------ .../src/protoconf/util.pc.h | 9 --- 8 files changed, 15 insertions(+), 152 deletions(-) diff --git a/README.md b/README.md index e7525f53..cc5989c9 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,9 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). > TODO: [devcontainer](https://code.visualstudio.com/docs/devcontainers/containers) -- Development OS: linux +- Supported OS: Windows, macOS, Linux - Init protobuf: `bash init.sh` +- C++ compiler: at least C++17 ## C++ @@ -15,7 +16,7 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). - Change dir: `cd test/cpp-tableau-loader` - Generate protoconf: `bash ./gen.sh` - Create build dir: `mkdir build && cd build` -- Run cmake: `cmake ../src/`(use c++11) or `cmake -DUSE_CPP17=ON ../src/`(use c++17) +- Run cmake: `cmake ../src/` (C++20) or `cmake -DUSE_CPP17=ON ../src/` (C++17) - Build: `make -j8`, then the **bin** dir will be generated at `test/cpp-tableau-loader/bin`. ### References diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc index e8be38f4..0cf2d6c3 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc @@ -1,8 +1,6 @@ #include "load.pc.h" -#if __cplusplus >= 201703L #include -#endif #include "logger.pc.h" #include "util.pc.h" @@ -83,11 +81,7 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::string& pa } else { std::string filename = name + util::Format2Ext(fmt); for (auto&& patch_dir : options->patch_dirs) { -#if __cplusplus >= 201703L patch_paths.emplace_back((std::filesystem::path(patch_dir) / filename).make_preferred().string()); -#else - patch_paths.emplace_back(patch_dir + kPathSeperator + filename); -#endif } } @@ -185,11 +179,7 @@ bool LoadMessagerInDir(google::protobuf::Message& msg, const std::string& dir, F } if (path.empty()) { std::string filename = name + util::Format2Ext(fmt); -#if __cplusplus >= 201703L path = (std::filesystem::path(dir) / filename).make_preferred().string(); -#else - path = dir + kPathSeperator + filename; -#endif } const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc index aa306205..67dc773e 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc @@ -3,29 +3,15 @@ #include #include +#include #include #include #include -#if __cplusplus >= 201703L -#include -#else -#ifdef _WIN32 -#include -#include -#else -#include -#endif -#endif - #include "load.pc.h" #include "logger.pc.h" namespace tableau { -#ifdef _WIN32 -#define mkdir(path, mode) _mkdir(path) -#endif - static thread_local std::string g_err_msg; const std::string& GetErrMsg() { return g_err_msg; } void SetErrMsg(const std::string& msg) { g_err_msg = msg; } @@ -37,7 +23,6 @@ const std::string kBinExt = ".bin"; namespace util { int Mkdir(const std::string& path) { -#if __cplusplus >= 201703L std::error_code ec; if (!std::filesystem::create_directories(path, ec)) { if (ec) { @@ -46,46 +31,11 @@ int Mkdir(const std::string& path) { } } return 0; -#else - std::string path_ = path + kPathSeperator; - struct stat info; - for (size_t pos = path_.find(kPathSeperator, 0); pos != std::string::npos; pos = path_.find(kPathSeperator, pos)) { - ++pos; - auto sub_dir = path_.substr(0, pos); - if (stat(sub_dir.c_str(), &info) == 0 && info.st_mode & S_IFDIR) { - continue; - } - int status = mkdir(sub_dir.c_str(), 0755); - if (status != 0) { - std::cerr << "system error: " << strerror(errno) << std::endl; - return -1; - } - } - return 0; -#endif } -std::string GetDir(const std::string& path) { -#if __cplusplus >= 201703L - return std::filesystem::path(path).parent_path().string(); -#else - size_t pos = path.find_last_of(kPathSeperator); - if (pos != std::string::npos) { - return path.substr(0, pos); - } - return kEmpty; -#endif -} +std::string GetDir(const std::string& path) { return std::filesystem::path(path).parent_path().string(); } -bool ExistsFile(const std::string& filename) { -#if __cplusplus >= 201703L - return std::filesystem::exists(filename); -#else - std::ifstream file(filename); - // returns true if the file exists and is accessible - return file.good(); -#endif -} +bool ExistsFile(const std::string& filename) { return std::filesystem::exists(filename); } bool ReadFile(const std::string& filename, std::string& content) { std::ifstream file(filename); diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h index 9c983c69..0ebb9121 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h @@ -11,15 +11,6 @@ namespace tableau { const std::string& GetErrMsg(); void SetErrMsg(const std::string& msg); -#if __cplusplus < 201703L -// Platform-specific path separator -#ifdef _WIN32 -constexpr char kPathSeperator = '\\'; -#else -constexpr char kPathSeperator = '/'; -#endif -#endif - enum class Format { kUnknown, kJSON, diff --git a/test/cpp-tableau-loader/src/CMakeLists.txt b/test/cpp-tableau-loader/src/CMakeLists.txt index b827e3d4..710cd763 100644 --- a/test/cpp-tableau-loader/src/CMakeLists.txt +++ b/test/cpp-tableau-loader/src/CMakeLists.txt @@ -5,16 +5,16 @@ project(loader) file(GLOB_RECURSE PROTO_SOURCE *.cc) file(GLOB_RECURSE SOURCE *.cpp) -# Add option to switch between C++11 and C++17 -option(USE_CPP17 "Use C++17 standard (otherwise use C++11)" OFF) +# Add option to switch between C++20 and C++17 +option(USE_CPP17 "Use C++17 standard (otherwise use C++20)" OFF) # specify the C++ standard if(USE_CPP17) set(CMAKE_CXX_STANDARD 17) message(STATUS "Using C++17 standard") else() - set(CMAKE_CXX_STANDARD 11) - message(STATUS "Using C++11 standard") + set(CMAKE_CXX_STANDARD 20) + message(STATUS "Using C++20 standard") endif() set(CMAKE_CXX_STANDARD_REQUIRED True) @@ -32,7 +32,7 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lstdc++fs") endif() else() - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -fPIC -std=c++11 -Wno-deprecated -Wno-unused-variable -Wno-sign-compare -Wno-strict-aliasing -fno-strict-aliasing -DNDEBUG") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -fPIC -std=c++20 -Wno-deprecated -Wno-unused-variable -Wno-sign-compare -Wno-strict-aliasing -fno-strict-aliasing -DNDEBUG") endif() endif() @@ -65,7 +65,7 @@ else() SET(COMMON_LIB ${PROTOBUF_LIB}/libprotobuf.a pthread - $<$:stdc++fs> + stdc++fs ) endif() diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc index d02c5e5b..d08d3f93 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -5,9 +5,7 @@ #include "load.pc.h" -#if __cplusplus >= 201703L #include -#endif #include "logger.pc.h" #include "util.pc.h" @@ -88,11 +86,7 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::string& pa } else { std::string filename = name + util::Format2Ext(fmt); for (auto&& patch_dir : options->patch_dirs) { -#if __cplusplus >= 201703L patch_paths.emplace_back((std::filesystem::path(patch_dir) / filename).make_preferred().string()); -#else - patch_paths.emplace_back(patch_dir + kPathSeperator + filename); -#endif } } @@ -190,11 +184,7 @@ bool LoadMessagerInDir(google::protobuf::Message& msg, const std::string& dir, F } if (path.empty()) { std::string filename = name + util::Format2Ext(fmt); -#if __cplusplus >= 201703L path = (std::filesystem::path(dir) / filename).make_preferred().string(); -#else - path = dir + kPathSeperator + filename; -#endif } const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.cc b/test/cpp-tableau-loader/src/protoconf/util.pc.cc index 05068ead..98ee62de 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.cc @@ -8,29 +8,15 @@ #include #include +#include #include #include #include -#if __cplusplus >= 201703L -#include -#else -#ifdef _WIN32 -#include -#include -#else -#include -#endif -#endif - #include "load.pc.h" #include "logger.pc.h" namespace tableau { -#ifdef _WIN32 -#define mkdir(path, mode) _mkdir(path) -#endif - static thread_local std::string g_err_msg; const std::string& GetErrMsg() { return g_err_msg; } void SetErrMsg(const std::string& msg) { g_err_msg = msg; } @@ -42,7 +28,6 @@ const std::string kBinExt = ".bin"; namespace util { int Mkdir(const std::string& path) { -#if __cplusplus >= 201703L std::error_code ec; if (!std::filesystem::create_directories(path, ec)) { if (ec) { @@ -51,46 +36,11 @@ int Mkdir(const std::string& path) { } } return 0; -#else - std::string path_ = path + kPathSeperator; - struct stat info; - for (size_t pos = path_.find(kPathSeperator, 0); pos != std::string::npos; pos = path_.find(kPathSeperator, pos)) { - ++pos; - auto sub_dir = path_.substr(0, pos); - if (stat(sub_dir.c_str(), &info) == 0 && info.st_mode & S_IFDIR) { - continue; - } - int status = mkdir(sub_dir.c_str(), 0755); - if (status != 0) { - std::cerr << "system error: " << strerror(errno) << std::endl; - return -1; - } - } - return 0; -#endif } -std::string GetDir(const std::string& path) { -#if __cplusplus >= 201703L - return std::filesystem::path(path).parent_path().string(); -#else - size_t pos = path.find_last_of(kPathSeperator); - if (pos != std::string::npos) { - return path.substr(0, pos); - } - return kEmpty; -#endif -} +std::string GetDir(const std::string& path) { return std::filesystem::path(path).parent_path().string(); } -bool ExistsFile(const std::string& filename) { -#if __cplusplus >= 201703L - return std::filesystem::exists(filename); -#else - std::ifstream file(filename); - // returns true if the file exists and is accessible - return file.good(); -#endif -} +bool ExistsFile(const std::string& filename) { return std::filesystem::exists(filename); } bool ReadFile(const std::string& filename, std::string& content) { std::ifstream file(filename); diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.h b/test/cpp-tableau-loader/src/protoconf/util.pc.h index 63027192..c80f03ad 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.h @@ -16,15 +16,6 @@ namespace tableau { const std::string& GetErrMsg(); void SetErrMsg(const std::string& msg); -#if __cplusplus < 201703L -// Platform-specific path separator -#ifdef _WIN32 -constexpr char kPathSeperator = '\\'; -#else -constexpr char kPathSeperator = '/'; -#endif -#endif - enum class Format { kUnknown, kJSON, From cf2fa80283cdfa86d798a3a6fc6be1643e10bd5e Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 15 Aug 2025 10:44:33 +0800 Subject: [PATCH 6/9] fix: gen.bat --- test/cpp-tableau-loader/gen.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp-tableau-loader/gen.bat b/test/cpp-tableau-loader/gen.bat index 0589b935..39b8aa34 100644 --- a/test/cpp-tableau-loader/gen.bat +++ b/test/cpp-tableau-loader/gen.bat @@ -32,7 +32,7 @@ for /R %%f in (*.proto) do ( popd "%PROTOC%" ^ --cpp-tableau-loader_out="%PROTOCONF_OUT%" ^ - --cpp-tableau-loader_opt=paths=source_relative,registry-shards=2 ^ + --cpp-tableau-loader_opt=paths=source_relative,shards=2 ^ --cpp_out="%PROTOCONF_OUT%" ^ --proto_path="%PROTOBUF_PROTO%" ^ --proto_path="%TABLEAU_PROTO%" ^ From e1be25decf867fcad5542a58320fbddb8a2423e1 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 15 Aug 2025 11:13:31 +0800 Subject: [PATCH 7/9] fix: update CMakeLists and fix compile warnings --- .../embed/load.pc.h | 2 +- test/cpp-tableau-loader/src/CMakeLists.txt | 16 ++-------------- test/cpp-tableau-loader/src/protoconf/load.pc.h | 2 +- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h index 41cd5d13..ada21043 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h @@ -16,7 +16,7 @@ enum class LoadMode { kOnlyPatch, // Only load the patch files }; -class MessagerOptions; +struct MessagerOptions; class Hub; // ReadFunc reads the config file and returns its content. diff --git a/test/cpp-tableau-loader/src/CMakeLists.txt b/test/cpp-tableau-loader/src/CMakeLists.txt index 710cd763..e5d5ad7d 100644 --- a/test/cpp-tableau-loader/src/CMakeLists.txt +++ b/test/cpp-tableau-loader/src/CMakeLists.txt @@ -19,21 +19,9 @@ endif() set(CMAKE_CXX_STANDARD_REQUIRED True) if (MSVC) - if(USE_CPP17) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /DNDEBUG /std:c++17") - else() - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /DNDEBUG") - endif() + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /DNDEBUG") else() - if(USE_CPP17) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -fPIC -std=c++17 -Wno-deprecated -Wno-unused-variable -Wno-sign-compare -Wno-strict-aliasing -fno-strict-aliasing -DNDEBUG") - # For older GCC versions that need explicit linking with stdc++fs - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9.0") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lstdc++fs") - endif() - else() - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -fPIC -std=c++20 -Wno-deprecated -Wno-unused-variable -Wno-sign-compare -Wno-strict-aliasing -fno-strict-aliasing -DNDEBUG") - endif() + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -fPIC -Wno-deprecated -Wno-unused-variable -Wno-sign-compare -Wno-strict-aliasing -fno-strict-aliasing -DNDEBUG") endif() # root dir define diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.h b/test/cpp-tableau-loader/src/protoconf/load.pc.h index 3a475a81..80c6ebf1 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.h @@ -21,7 +21,7 @@ enum class LoadMode { kOnlyPatch, // Only load the patch files }; -class MessagerOptions; +struct MessagerOptions; class Hub; // ReadFunc reads the config file and returns its content. From 9fc3148e07fba445017d1f546b0efed4b188e58f Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 15 Aug 2025 11:27:18 +0800 Subject: [PATCH 8/9] fix: some compile warnings in MSVC --- cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h | 2 +- cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.cc | 2 +- cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h | 8 ++++---- .../src/hub/custom/item/custom_item_conf.h | 2 +- test/cpp-tableau-loader/src/main.cpp | 8 ++++---- test/cpp-tableau-loader/src/protoconf/load.pc.h | 2 +- test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc | 2 +- test/cpp-tableau-loader/src/protoconf/util.pc.h | 8 ++++---- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h index ada21043..b37f05a8 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h @@ -84,7 +84,7 @@ class Messager { // Message returns the inner message data. virtual const google::protobuf::Message* Message() const { return nullptr; } // callback after all messagers loaded. - virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } + virtual bool ProcessAfterLoadAll(const Hub&) { return true; } protected: // callback after this messager loaded. diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.cc index 5a9aca0b..2280b974 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.cc @@ -26,7 +26,7 @@ int Scheduler::LoopOnce() { for (auto&& job : jobs) { job(); } - count += jobs.size(); + count += static_cast(jobs.size()); return count; } diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h index 0ebb9121..f2ed38c0 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h @@ -32,7 +32,7 @@ namespace util { // References: // - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x // - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key -inline void HashCombine(std::size_t& seed) {} +inline void HashCombine(std::size_t&) {} template inline void HashCombine(std::size_t& seed, const T& v, O... others) { @@ -65,9 +65,9 @@ bool ReadFile(const std::string& filename, std::string& content); std::string GetExt(const std::string& path); // Convert file extension to Format type. // NOTE: ext includes dot ".", such as: -// - kJSONExt：".json" -// - kTextExt".txt" -// - kBinExt".bin" +// - kJSONExt: ".json" +// - kTextExt: ".txt" +// - kBinExt: ".bin" Format Ext2Format(const std::string& ext); // Empty string will be returned if an unsupported enum value has been passed, // and the error message can be obtained by GetErrMsg(). diff --git a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h index a616641a..76835861 100644 --- a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h +++ b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h @@ -4,7 +4,7 @@ class CustomItemConf : public tableau::Messager { public: static const std::string& Name() { return kCustomName; }; - virtual bool Load(const std::string& dir, tableau::Format fmt, + virtual bool Load(const std::string&, tableau::Format, std::shared_ptr options = nullptr) override { return true; } diff --git a/test/cpp-tableau-loader/src/main.cpp b/test/cpp-tableau-loader/src/main.cpp index 7079edb5..d298ff2e 100644 --- a/test/cpp-tableau-loader/src/main.cpp +++ b/test/cpp-tableau-loader/src/main.cpp @@ -195,13 +195,13 @@ int main() { for (auto&& it : *chapter_ordered_map) { std::cout << "---" << it.first << "-----section_ordered_map" << std::endl; - for (auto&& item : it.second.first) { - std::cout << item.first << std::endl; + for (auto&& kv : it.second.first) { + std::cout << kv.first << std::endl; } std::cout << "---" << it.first << " -----section_map" << std::endl; - for (auto&& item : it.second.second->section_map()) { - std::cout << item.first << std::endl; + for (auto&& kv : it.second.second->section_map()) { + std::cout << kv.first << std::endl; } std::cout << "chapter_id: " << it.second.second->chapter_id() << std::endl; diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.h b/test/cpp-tableau-loader/src/protoconf/load.pc.h index 80c6ebf1..b9088234 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.h @@ -89,7 +89,7 @@ class Messager { // Message returns the inner message data. virtual const google::protobuf::Message* Message() const { return nullptr; } // callback after all messagers loaded. - virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } + virtual bool ProcessAfterLoadAll(const Hub&) { return true; } protected: // callback after this messager loaded. diff --git a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc index b375f26b..8fa92e30 100644 --- a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc @@ -31,7 +31,7 @@ int Scheduler::LoopOnce() { for (auto&& job : jobs) { job(); } - count += jobs.size(); + count += static_cast(jobs.size()); return count; } diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.h b/test/cpp-tableau-loader/src/protoconf/util.pc.h index c80f03ad..6d5a94fc 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.h @@ -37,7 +37,7 @@ namespace util { // References: // - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x // - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key -inline void HashCombine(std::size_t& seed) {} +inline void HashCombine(std::size_t&) {} template inline void HashCombine(std::size_t& seed, const T& v, O... others) { @@ -70,9 +70,9 @@ bool ReadFile(const std::string& filename, std::string& content); std::string GetExt(const std::string& path); // Convert file extension to Format type. // NOTE: ext includes dot ".", such as: -// - kJSONExt：".json" -// - kTextExt".txt" -// - kBinExt".bin" +// - kJSONExt: ".json" +// - kTextExt: ".txt" +// - kBinExt: ".bin" Format Ext2Format(const std::string& ext); // Empty string will be returned if an unsupported enum value has been passed, // and the error message can be obtained by GetErrMsg(). From a3897a073507bea8c71e2c7032f0cea3796162e0 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 15 Aug 2025 15:08:32 +0800 Subject: [PATCH 9/9] dev(linux): support at least c++17 --- README.md | 9 +++++++-- test/cpp-tableau-loader/src/CMakeLists.txt | 17 ++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index cc5989c9..23faf9df 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,20 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). - Supported OS: Windows, macOS, Linux - Init protobuf: `bash init.sh` -- C++ compiler: at least C++17 +- C++ standard: at least C++17 ## C++ +### Dev at Linux + - Install: **CMake 3.22** or above - Change dir: `cd test/cpp-tableau-loader` - Generate protoconf: `bash ./gen.sh` - Create build dir: `mkdir build && cd build` -- Run cmake: `cmake ../src/` (C++20) or `cmake -DUSE_CPP17=ON ../src/` (C++17) +- Run cmake: + - C++17: `cmake ../src/` + - C++20: `cmake ../src/ -DCMAKE_CXX_STANDARD=20` + - clang: `cmake ../src/ -DCMAKE_CXX_COMPILER=clang++` - Build: `make -j8`, then the **bin** dir will be generated at `test/cpp-tableau-loader/bin`. ### References diff --git a/test/cpp-tableau-loader/src/CMakeLists.txt b/test/cpp-tableau-loader/src/CMakeLists.txt index e5d5ad7d..81402304 100644 --- a/test/cpp-tableau-loader/src/CMakeLists.txt +++ b/test/cpp-tableau-loader/src/CMakeLists.txt @@ -5,18 +5,17 @@ project(loader) file(GLOB_RECURSE PROTO_SOURCE *.cc) file(GLOB_RECURSE SOURCE *.cpp) -# Add option to switch between C++20 and C++17 -option(USE_CPP17 "Use C++17 standard (otherwise use C++20)" OFF) - -# specify the C++ standard -if(USE_CPP17) - set(CMAKE_CXX_STANDARD 17) - message(STATUS "Using C++17 standard") +# check C++ standard requirement +set(MIN_CXX_STANDARD 17) +if (NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD ${MIN_CXX_STANDARD}) # default min C++ standard else() - set(CMAKE_CXX_STANDARD 20) - message(STATUS "Using C++20 standard") + if(CMAKE_CXX_STANDARD LESS ${MIN_CXX_STANDARD}) + message(FATAL_ERROR "CMAKE_CXX_STANDARD is less than the minimum required standard: C++${MIN_CXX_STANDARD}.") + endif() endif() set(CMAKE_CXX_STANDARD_REQUIRED True) +message(STATUS "Using C++${CMAKE_CXX_STANDARD} standard") if (MSVC) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /DNDEBUG")