From 08f7f1f8e46f0a037bed1593d7803a3f1a087c6a Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 24 Apr 2025 12:08:12 +0800 Subject: [PATCH 01/14] feat: update embed --- .../embed/hub.pc.cc | 70 ++++--- .../embed/hub.pc.h | 176 ++++-------------- .../embed/messager.pc.h | 129 +++++++++++++ cmd/protoc-gen-cpp-tableau-loader/messager.go | 2 +- 4 files changed, 200 insertions(+), 177 deletions(-) create mode 100644 cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc index d1834ccd..8de26a5b 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc @@ -17,6 +17,7 @@ #include #include "logger.pc.h" +#include "messager.pc.h" #include "registry.pc.h" namespace tableau { @@ -439,29 +440,29 @@ bool StoreMessage(google::protobuf::Message& msg, const std::string& dir, Format } bool Hub::Load(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { - auto msger_container = LoadNewMessagerContainer(dir, fmt, options); - if (!msger_container) { + auto msger_map = InternalLoad(dir, fmt, options); + if (!msger_map) { return false; } - bool ok = internal::Postprocess(options->postprocessor, msger_container); + bool ok = internal::Postprocess(options->postprocessor, msger_map); if (!ok) { return false; } - SetMessagerContainer(msger_container); + SetMessagerMap(msger_map); return true; } bool Hub::AsyncLoad(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { - auto msger_container = LoadNewMessagerContainer(dir, fmt, options); - if (!msger_container) { + auto msger_map = InternalLoad(dir, fmt, options); + if (!msger_map) { return false; } - bool ok = internal::Postprocess(options->postprocessor, msger_container); + bool ok = internal::Postprocess(options->postprocessor, msger_map); if (!ok) { return false; } - sched_->Dispatch(std::bind(&Hub::SetMessagerContainer, this, msger_container)); + sched_->Dispatch(std::bind(&Hub::SetMessagerMap, this, msger_map)); return true; } @@ -471,12 +472,12 @@ void Hub::InitScheduler() { sched_->Current(); } -MessagerContainer Hub::LoadNewMessagerContainer(const std::string& dir, Format fmt /* = Format::kJSON */, - const LoadOptions* options /* = nullptr */) { +std::shared_ptr Hub::InternalLoad(const std::string& dir, Format fmt /* = Format::kJSON */, + const LoadOptions* options /* = nullptr */) const { // intercept protobuf error logs auto old_handler = google::protobuf::SetLogHandler(ProtobufLogHandler); - auto msger_container = NewMessagerContainer(); - for (auto iter : *msger_container) { + auto msger_map = NewMessagerMap(); + for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); bool ok = iter.second->Load(dir, fmt, options); @@ -491,44 +492,40 @@ MessagerContainer Hub::LoadNewMessagerContainer(const std::string& dir, Format f // restore to old protobuf log handler google::protobuf::SetLogHandler(old_handler); - return msger_container; + return msger_map; } -MessagerContainer Hub::NewMessagerContainer() { - MessagerContainer msger_container = std::make_shared(); +std::shared_ptr Hub::NewMessagerMap() const { + std::shared_ptr msger_map = std::make_shared(); for (auto&& it : Registry::registrar) { if (!options_.filter || options_.filter(it.first)) { - (*msger_container)[it.first] = it.second(); + (*msger_map)[it.first] = it.second(); } } - return msger_container; + return msger_map; } -void Hub::SetMessagerContainer(MessagerContainer msger_container) { +std::shared_ptr Hub::GetMessagerMap() const { return GetMessagerContainer()->msger_map_; } + +void Hub::SetMessagerMap(std::shared_ptr msger_map) { // replace with thread-safe guarantee. std::unique_lock lock(mutex_); - msger_container_ = msger_container; - last_loaded_time_ = std::time(nullptr); -} - -MessagerContainer Hub::GetMessagerContainerWithProvider() const { - if (msger_container_provider_ != nullptr) { - return msger_container_provider_(); - } - return msger_container_; + msger_container_ = std::make_shared(msger_map); } const std::shared_ptr Hub::GetMessager(const std::string& name) const { - auto container = GetMessagerContainerWithProvider(); - if (container) { - auto it = container->find(name); - if (it != container->end()) { + auto msger_map = GetMessagerMap(); + if (msger_map) { + auto it = msger_map->find(name); + if (it != msger_map->end()) { return it->second; } } return nullptr; } +std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; } + namespace internal { // Thread-local storage (TLS) thread_local Scheduler* tls_sched = nullptr; @@ -577,14 +574,15 @@ void Scheduler::AssertInLoopThread() const { } } -bool Postprocess(Postprocessor postprocessor, MessagerContainer container) { +bool Postprocess(Postprocessor postprocessor, std::shared_ptr msger_map) { // create a temporary hub with messager container for post process - Hub hub(container); + Hub tmp_hub; + tmp_hub.SetMessagerMap(msger_map); // messager-level postprocess - for (auto iter : *container) { + for (auto iter : *msger_map) { auto msger = iter.second; - bool ok = msger->ProcessAfterLoadAll(hub); + bool ok = msger->ProcessAfterLoadAll(tmp_hub); if (!ok) { g_err_msg = "hub call ProcessAfterLoadAll failed, messager: " + msger->Name(); return false; @@ -593,7 +591,7 @@ bool Postprocess(Postprocessor postprocessor, MessagerContainer container) { // hub-level postprocess if (postprocessor != nullptr) { - bool ok = postprocessor(hub); + bool ok = postprocessor(tmp_hub); if (!ok) { g_err_msg = "hub call Postprocesser failed, you'd better check your custom 'postprocessor' load option"; return false; diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h index 6253b7e8..c8b27e35 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h @@ -15,67 +15,32 @@ #include #include -namespace tableau { -enum class Format { - kUnknown, - kJSON, - kText, - kBin, -}; - -enum class LoadMode { - kModeDefault, - kModeOnlyMain, - kModeOnlyPatch, -}; +#include "messager.pc.h" +namespace tableau { extern const std::string kUnknownExt; extern const std::string kJSONExt; extern const std::string kTextExt; extern const std::string kBinExt; -static const std::string kEmpty = ""; const std::string& GetErrMsg(); -class Messager; +class MessagerContainer; class Hub; using MessagerMap = std::unordered_map>; -using MessagerContainer = std::shared_ptr; // FilterFunc filter in messagers if returned value is true. // NOTE: name is the protobuf message name, e.g.: "message ItemConf{...}". using Filter = std::function; -using MessagerContainerProvider = std::function; -using Postprocessor = std::function; -// ReadFunc reads the config file and returns its content. -using ReadFunc = std::function; +using MessagerContainerProvider = std::function()>; struct HubOptions { // Filter can only filter in certain specific messagers based on the // condition that you provide. Filter filter; -}; - -struct LoadOptions { - // postprocessor is called after loading all configurations. - Postprocessor postprocessor; - // read_func reads the config file and returns its content. - ReadFunc read_func; - // Whether to ignore unknown JSON fields during parsing. - // - // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. - bool ignore_unknown_fields = false; - // Paths maps each messager name to a corresponding config file path. - // If specified, then the main messager will be parsed from the file - // directly, other than the specified load dir. - std::unordered_map paths; - // Patch paths maps each messager name to one or multiple corresponding patch file paths. - // If specified, then main messager will be patched. - std::unordered_map> patch_paths; - // Patch dirs specifies the directory paths for config patching. - std::vector patch_dirs; - // Mode specifies the loading mode for config patching. - LoadMode mode = LoadMode::kModeDefault; + // Provide custom MessagerContainer. For keeping configuration access + // consistent in a coroutine or a transaction. + MessagerContainerProvider provider; }; // Convert file extension to Format type. @@ -119,42 +84,14 @@ class Scheduler { std::vector jobs_; }; -bool Postprocess(Postprocessor postprocessor, MessagerContainer container); +bool Postprocess(Postprocessor postprocessor, std::shared_ptr msger_map); } // namespace internal -class Messager { - public: - struct Stats { - std::chrono::microseconds duration; // total load time consuming. - // TODO: crc32 of config file to decide whether changed or not - // std::string crc32; - // int64_t last_modified_time = 0; // unix timestamp - }; - - public: - virtual ~Messager() = default; - static const std::string& Name() { return kEmpty; } - const Stats& GetStats() { return stats_; } - // Load fills message from file in the specified directory and format. - virtual bool Load(const std::string& dir, Format fmt, const LoadOptions* options = nullptr) = 0; - // Message returns the inner message data. - virtual const google::protobuf::Message* Message() const { return nullptr; } - // callback after all messagers loaded. - virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } - - protected: - // callback after this messager loaded. - virtual bool ProcessAfterLoad() { return true; }; - Stats stats_; -}; - class Hub { public: - Hub(const HubOptions* options = nullptr) : options_(options ? *options : HubOptions{}) {} - Hub(MessagerContainer container, const HubOptions* options = nullptr) : options_(options ? *options : HubOptions{}) { - SetMessagerContainer(container); - } + Hub(const HubOptions* options = nullptr) + : msger_container_(std::make_shared()), options_(options ? *options : HubOptions{}) {} /***** Synchronous Loading *****/ // Load fills messages (in MessagerContainer) from files in the specified directory and format. bool Load(const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); @@ -167,9 +104,18 @@ class Hub { // You'd better initialize the scheduler in the main thread. void InitScheduler(); + /***** MessagerMap *****/ + std::shared_ptr GetMessagerMap() const; + void SetMessagerMap(std::shared_ptr msger_map); + /***** MessagerContainer *****/ - MessagerContainer GetMessagerContainer() const { return msger_container_; } - void SetMessagerContainerProvider(MessagerContainerProvider provider) { msger_container_provider_ = provider; } + // This function is exposed only for use in MessagerContainerProvider. + std::shared_ptr GetMessagerContainer() const { + if (options_.provider != nullptr) { + return options_.provider(); + } + return msger_container_; + } /***** Access APIs *****/ template @@ -182,28 +128,21 @@ class Hub { const U* GetOrderedMap(Args... args) const; // GetLastLoadedTime returns the time when hub's msger_container_ was last set. - inline std::time_t GetLastLoadedTime() const { return last_loaded_time_; } + inline std::time_t GetLastLoadedTime() const; private: - MessagerContainer LoadNewMessagerContainer(const std::string& dir, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr); - MessagerContainer NewMessagerContainer(); - void SetMessagerContainer(MessagerContainer msger_container); - MessagerContainer GetMessagerContainerWithProvider() const; + std::shared_ptr InternalLoad(const std::string& dir, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr) const; + std::shared_ptr NewMessagerMap() const; const std::shared_ptr GetMessager(const std::string& name) const; private: // For thread-safe guarantee during configuration updating. std::mutex mutex_; // All messagers' container. - MessagerContainer msger_container_; - // Provide custom MessagerContainer. For keeping configuration access - // consistent in a coroutine or a transaction. - MessagerContainerProvider msger_container_provider_; + std::shared_ptr msger_container_; // Loading scheduler. internal::Scheduler* sched_ = nullptr; - // Last loaded time - std::time_t last_loaded_time_ = 0; // Hub options const HubOptions options_; }; @@ -216,69 +155,26 @@ const std::shared_ptr Hub::Get() const { template const U* Hub::Get(Args... args) const { - auto msg = GetMessager(T::Name()); - auto msger = std::dynamic_pointer_cast(msg); + auto msger = Get(); return msger ? msger->Get(args...) : nullptr; } template const U* Hub::GetOrderedMap(Args... args) const { - auto msg = GetMessager(T::Name()); - auto msger = std::dynamic_pointer_cast(msg); + auto msger = Get(); return msger ? msger->GetOrderedMap(args...) : nullptr; } -namespace util { - -// Combine hash values -// -// References: -// - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x -// - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key -inline void HashCombine(std::size_t& seed) {} - -template -inline void HashCombine(std::size_t& seed, const T& v, O... others) { - std::hash hasher; - seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); - HashCombine(seed, others...); -} - -template -inline std::size_t SugaredHashCombine(const T& v, O... others) { - std::size_t seed = 0; // start with a hash value 0 - HashCombine(seed, v, others...); - return seed; -} - -// Mkdir makes dir recursively. -int Mkdir(const std::string& path); -// GetDir returns all but the last element of path, typically the path's -// directory. -std::string GetDir(const std::string& path); -// GetExt returns the file name extension used by path. -// The extension is the suffix beginning at the final dot -// in the final element of path; it is empty if there is -// no dot. -std::string GetExt(const std::string& path); - -class TimeProfiler { - protected: - std::chrono::time_point last_; +class MessagerContainer { + public: + MessagerContainer(std::shared_ptr msger_map = nullptr) + : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), + last_loaded_time_(std::time(nullptr)) {} public: - TimeProfiler() { Start(); } - void Start() { last_ = std::chrono::steady_clock::now(); } - // Calculate duration between the last time point and now, - // and update last time point to now. - std::chrono::microseconds Elapse() { - auto now = std::chrono::steady_clock::now(); - auto duration = now - last_; // This is of type std::chrono::duration - last_ = now; - return std::chrono::duration_cast(duration); - } + std::shared_ptr msger_map_; + std::time_t last_loaded_time_; + // Auto-generated fields below }; -} // namespace util - } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h new file mode 100644 index 00000000..df436eda --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h @@ -0,0 +1,129 @@ +#pragma once +#include + +#include +#include +#include + +namespace tableau { +enum class Format { + kUnknown, + kJSON, + kText, + kBin, +}; + +enum class LoadMode { + kModeDefault, + kModeOnlyMain, + kModeOnlyPatch, +}; + +static const std::string kEmpty = ""; + +class Hub; + +using Postprocessor = std::function; +// ReadFunc reads the config file and returns its content. +using ReadFunc = std::function; + +struct LoadOptions { + // postprocessor is called after loading all configurations. + Postprocessor postprocessor; + // read_func reads the config file and returns its content. + ReadFunc read_func; + // Whether to ignore unknown JSON fields during parsing. + // + // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. + bool ignore_unknown_fields = false; + // Paths maps each messager name to a corresponding config file path. + // If specified, then the main messager will be parsed from the file + // directly, other than the specified load dir. + std::unordered_map paths; + // Patch paths maps each messager name to one or multiple corresponding patch file paths. + // If specified, then main messager will be patched. + std::unordered_map> patch_paths; + // Patch dirs specifies the directory paths for config patching. + std::vector patch_dirs; + // Mode specifies the loading mode for config patching. + LoadMode mode = LoadMode::kModeDefault; +}; + +class Messager { + public: + struct Stats { + std::chrono::microseconds duration; // total load time consuming. + // TODO: crc32 of config file to decide whether changed or not + // std::string crc32; + // int64_t last_modified_time = 0; // unix timestamp + }; + + public: + virtual ~Messager() = default; + static const std::string& Name() { return kEmpty; } + const Stats& GetStats() { return stats_; } + // Load fills message from file in the specified directory and format. + virtual bool Load(const std::string& dir, Format fmt, const LoadOptions* options = nullptr) = 0; + // Message returns the inner message data. + virtual const google::protobuf::Message* Message() const { return nullptr; } + // callback after all messagers loaded. + virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } + + protected: + // callback after this messager loaded. + virtual bool ProcessAfterLoad() { return true; }; + Stats stats_; +}; + +namespace util { +// Combine hash values +// +// References: +// - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x +// - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key +inline void HashCombine(std::size_t& seed) {} + +template +inline void HashCombine(std::size_t& seed, const T& v, O... others) { + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + HashCombine(seed, others...); +} + +template +inline std::size_t SugaredHashCombine(const T& v, O... others) { + std::size_t seed = 0; // start with a hash value 0 + HashCombine(seed, v, others...); + return seed; +} + +// Mkdir makes dir recursively. +int Mkdir(const std::string& path); +// GetDir returns all but the last element of path, typically the path's +// directory. +std::string GetDir(const std::string& path); +// GetExt returns the file name extension used by path. +// The extension is the suffix beginning at the final dot +// in the final element of path; it is empty if there is +// no dot. +std::string GetExt(const std::string& path); + +class TimeProfiler { + protected: + std::chrono::time_point last_; + + public: + TimeProfiler() { Start(); } + void Start() { last_ = std::chrono::steady_clock::now(); } + // Calculate duration between the last time point and now, + // and update last time point to now. + std::chrono::microseconds Elapse() { + auto now = std::chrono::steady_clock::now(); + auto duration = now - last_; // This is of type std::chrono::duration + last_ = now; + return std::chrono::duration_cast(duration); + } +}; + +} // namespace util +} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/messager.go b/cmd/protoc-gen-cpp-tableau-loader/messager.go index 6c1fa88e..e846fd3e 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/messager.go +++ b/cmd/protoc-gen-cpp-tableau-loader/messager.go @@ -45,7 +45,7 @@ func generateHppFileContent(gen *protogen.Plugin, file *protogen.File, g *protog g.P("#pragma once") g.P("#include ") g.P() - g.P(`#include "`, "hub.", pcExt, `.h"`) + g.P(`#include "`, "messager.", pcExt, `.h"`) g.P(`#include "`, file.GeneratedFilenamePrefix, ".", pbExt, `.h"`) g.P() From 595239d3371f91d86b1fd708481010b2e7f0d5cc Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 24 Apr 2025 14:37:15 +0800 Subject: [PATCH 02/14] feat: impl --- cmd/protoc-gen-cpp-tableau-loader/embed.go | 61 ++++- .../embed/hub.pc.cc | 2 + .../embed/hub.pc.h | 8 +- test/cpp-tableau-loader/src/main.cpp | 4 +- .../src/protoconf/hero_conf.pc.h | 2 +- .../src/protoconf/hub.pc.cc | 117 +++++++--- .../cpp-tableau-loader/src/protoconf/hub.pc.h | 219 +++++++----------- .../src/protoconf/item_conf.pc.h | 2 +- .../src/protoconf/messager.pc.h | 134 +++++++++++ .../src/protoconf/patch_conf.pc.h | 2 +- .../src/protoconf/test_conf.pc.h | 2 +- 11 files changed, 375 insertions(+), 178 deletions(-) create mode 100644 test/cpp-tableau-loader/src/protoconf/messager.pc.h diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed.go b/cmd/protoc-gen-cpp-tableau-loader/embed.go index 1880d82a..59b2af3f 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed.go +++ b/cmd/protoc-gen-cpp-tableau-loader/embed.go @@ -3,7 +3,9 @@ package main import ( "embed" "path" + "strings" + "github.com/iancoleman/strcase" "github.com/tableauio/loader/cmd/protoc-gen-cpp-tableau-loader/helper" "google.golang.org/protobuf/compiler/protogen" ) @@ -11,12 +13,20 @@ import ( //go:embed embed/* var efs embed.FS +const ( + includeNotes = "// Auto-generated includes below\n" + specializationNotes = "// Auto-generated specializations below\n" + initializationNotes = "// Auto-generated initializations below\n" + fieldNotes = "// Auto-generated fields below\n" +) + // generateEmbed generates related registry files. func generateEmbed(gen *protogen.Plugin) { entries, err := efs.ReadDir("embed") if err != nil { panic(err) } + protofiles, fileMessagers := getAllOrderedFilesAndMessagers(gen) for _, entry := range entries { if entry.IsDir() { continue @@ -30,6 +40,55 @@ func generateEmbed(gen *protogen.Plugin) { if err != nil { panic(err) } - g.P(string(content)) + file := string(content) + switch entry.Name() { + case "hub.pc.cc": + // Auto-generated specializations below + impl := "" + for _, messagers := range fileMessagers { + for _, messager := range messagers { + impl += "template <>\n" + impl += "const std::shared_ptr<" + messager + "> Hub::Get<" + messager + ">() const {\n" + impl += " return GetMessagerContainer()->" + strcase.ToSnake(messager) + "_;\n" + impl += "}\n" + impl += "\n" + } + } + file = strings.ReplaceAll(file, specializationNotes, specializationNotes+impl) + case "hub.pc.h": + // Auto-generated includes below + impl := "" + for _, proto := range protofiles { + impl += `#include "` + proto + "." + pcExt + `.h"` + "\n" + } + file = strings.ReplaceAll(file, includeNotes, includeNotes+impl) + // Auto-generated specializations below + impl = "" + for _, messagers := range fileMessagers { + for _, messager := range messagers { + impl += "template <>\n" + impl += "const std::shared_ptr<" + messager + "> Hub::Get<" + messager + ">() const;\n" + impl += "\n" + } + } + file = strings.ReplaceAll(file, specializationNotes, specializationNotes+impl) + // Auto-generated initializations below + impl = "" + for _, messagers := range fileMessagers { + for _, messager := range messagers { + impl += " " + strcase.ToSnake(messager) + "_ = std::dynamic_pointer_cast<" + messager + `>((*msger_map_)["` + messager + `"]);` + "\n" + } + } + file = strings.ReplaceAll(file, initializationNotes, initializationNotes+impl) + // Auto-generated fields below + impl = "" + for _, messagers := range fileMessagers { + for _, messager := range messagers { + impl += " std::shared_ptr<" + messager + "> " + strcase.ToSnake(messager) + "_;\n" + } + } + file = strings.ReplaceAll(file, fieldNotes, fieldNotes+impl) + } + g.P(file) } } diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc index 8de26a5b..6a2e4350 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc @@ -526,6 +526,8 @@ const std::shared_ptr Hub::GetMessager(const std::string& name) const std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; } +// Auto-generated specializations below + namespace internal { // Thread-local storage (TLS) thread_local Scheduler* tls_sched = nullptr; diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h index c8b27e35..5f3b6604 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h @@ -17,6 +17,8 @@ #include "messager.pc.h" +// Auto-generated includes below + namespace tableau { extern const std::string kUnknownExt; extern const std::string kJSONExt; @@ -165,11 +167,15 @@ const U* Hub::GetOrderedMap(Args... args) const { return msger ? msger->GetOrderedMap(args...) : nullptr; } +// Auto-generated specializations below + class MessagerContainer { public: MessagerContainer(std::shared_ptr msger_map = nullptr) : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), - last_loaded_time_(std::time(nullptr)) {} + last_loaded_time_(std::time(nullptr)) { + // Auto-generated initializations below + } public: std::shared_ptr msger_map_; diff --git a/test/cpp-tableau-loader/src/main.cpp b/test/cpp-tableau-loader/src/main.cpp index b8b69754..ad22f611 100644 --- a/test/cpp-tableau-loader/src/main.cpp +++ b/test/cpp-tableau-loader/src/main.cpp @@ -136,8 +136,8 @@ int main() { std::cout << "protobuf hub load failed: " << tableau::GetErrMsg() << std::endl; return 1; } - auto msger_container = Hub::Instance().GetMessagerContainer(); - for (auto&& item : *msger_container) { + auto msger_map = Hub::Instance().GetMessagerMap(); + for (auto&& item : *msger_map) { auto&& stats = item.second->GetStats(); ATOM_DEBUG("%s: duration: %dus", item.first.c_str(), stats.duration.count()); } diff --git a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h index 279b1740..56df8472 100644 --- a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h @@ -7,7 +7,7 @@ #pragma once #include -#include "hub.pc.h" +#include "messager.pc.h" #include "hero_conf.pb.h" namespace tableau { diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index 303a00c5..dcf65ae1 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -22,6 +22,7 @@ #include #include "logger.pc.h" +#include "messager.pc.h" #include "registry.pc.h" namespace tableau { @@ -444,29 +445,29 @@ bool StoreMessage(google::protobuf::Message& msg, const std::string& dir, Format } bool Hub::Load(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { - auto msger_container = LoadNewMessagerContainer(dir, fmt, options); - if (!msger_container) { + auto msger_map = InternalLoad(dir, fmt, options); + if (!msger_map) { return false; } - bool ok = internal::Postprocess(options->postprocessor, msger_container); + bool ok = internal::Postprocess(options->postprocessor, msger_map); if (!ok) { return false; } - SetMessagerContainer(msger_container); + SetMessagerMap(msger_map); return true; } bool Hub::AsyncLoad(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { - auto msger_container = LoadNewMessagerContainer(dir, fmt, options); - if (!msger_container) { + auto msger_map = InternalLoad(dir, fmt, options); + if (!msger_map) { return false; } - bool ok = internal::Postprocess(options->postprocessor, msger_container); + bool ok = internal::Postprocess(options->postprocessor, msger_map); if (!ok) { return false; } - sched_->Dispatch(std::bind(&Hub::SetMessagerContainer, this, msger_container)); + sched_->Dispatch(std::bind(&Hub::SetMessagerMap, this, msger_map)); return true; } @@ -476,12 +477,12 @@ void Hub::InitScheduler() { sched_->Current(); } -MessagerContainer Hub::LoadNewMessagerContainer(const std::string& dir, Format fmt /* = Format::kJSON */, - const LoadOptions* options /* = nullptr */) { +std::shared_ptr Hub::InternalLoad(const std::string& dir, Format fmt /* = Format::kJSON */, + const LoadOptions* options /* = nullptr */) const { // intercept protobuf error logs auto old_handler = google::protobuf::SetLogHandler(ProtobufLogHandler); - auto msger_container = NewMessagerContainer(); - for (auto iter : *msger_container) { + auto msger_map = NewMessagerMap(); + for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); bool ok = iter.second->Load(dir, fmt, options); @@ -496,44 +497,87 @@ MessagerContainer Hub::LoadNewMessagerContainer(const std::string& dir, Format f // restore to old protobuf log handler google::protobuf::SetLogHandler(old_handler); - return msger_container; + return msger_map; } -MessagerContainer Hub::NewMessagerContainer() { - MessagerContainer msger_container = std::make_shared(); +std::shared_ptr Hub::NewMessagerMap() const { + std::shared_ptr msger_map = std::make_shared(); for (auto&& it : Registry::registrar) { if (!options_.filter || options_.filter(it.first)) { - (*msger_container)[it.first] = it.second(); + (*msger_map)[it.first] = it.second(); } } - return msger_container; + return msger_map; } -void Hub::SetMessagerContainer(MessagerContainer msger_container) { +std::shared_ptr Hub::GetMessagerMap() const { return GetMessagerContainer()->msger_map_; } + +void Hub::SetMessagerMap(std::shared_ptr msger_map) { // replace with thread-safe guarantee. std::unique_lock lock(mutex_); - msger_container_ = msger_container; - last_loaded_time_ = std::time(nullptr); -} - -MessagerContainer Hub::GetMessagerContainerWithProvider() const { - if (msger_container_provider_ != nullptr) { - return msger_container_provider_(); - } - return msger_container_; + msger_container_ = std::make_shared(msger_map); } const std::shared_ptr Hub::GetMessager(const std::string& name) const { - auto container = GetMessagerContainerWithProvider(); - if (container) { - auto it = container->find(name); - if (it != container->end()) { + auto msger_map = GetMessagerMap(); + if (msger_map) { + auto it = msger_map->find(name); + if (it != msger_map->end()) { return it->second; } } return nullptr; } +std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; } + +// Auto-generated specializations below +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->hero_base_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->hero_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->item_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->patch_merge_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->patch_replace_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->recursive_patch_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->activity_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->chapter_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->theme_conf_; +} + + namespace internal { // Thread-local storage (TLS) thread_local Scheduler* tls_sched = nullptr; @@ -582,14 +626,15 @@ void Scheduler::AssertInLoopThread() const { } } -bool Postprocess(Postprocessor postprocessor, MessagerContainer container) { +bool Postprocess(Postprocessor postprocessor, std::shared_ptr msger_map) { // create a temporary hub with messager container for post process - Hub hub(container); + Hub tmp_hub; + tmp_hub.SetMessagerMap(msger_map); // messager-level postprocess - for (auto iter : *container) { + for (auto iter : *msger_map) { auto msger = iter.second; - bool ok = msger->ProcessAfterLoadAll(hub); + bool ok = msger->ProcessAfterLoadAll(tmp_hub); if (!ok) { g_err_msg = "hub call ProcessAfterLoadAll failed, messager: " + msger->Name(); return false; @@ -598,7 +643,7 @@ bool Postprocess(Postprocessor postprocessor, MessagerContainer container) { // hub-level postprocess if (postprocessor != nullptr) { - bool ok = postprocessor(hub); + bool ok = postprocessor(tmp_hub); if (!ok) { g_err_msg = "hub call Postprocesser failed, you'd better check your custom 'postprocessor' load option"; return false; diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index c735ab36..9f73d9d4 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -20,67 +20,38 @@ #include #include -namespace tableau { -enum class Format { - kUnknown, - kJSON, - kText, - kBin, -}; +#include "messager.pc.h" -enum class LoadMode { - kModeDefault, - kModeOnlyMain, - kModeOnlyPatch, -}; +// Auto-generated includes below +#include "hero_conf.pc.h" +#include "item_conf.pc.h" +#include "patch_conf.pc.h" +#include "test_conf.pc.h" +namespace tableau { extern const std::string kUnknownExt; extern const std::string kJSONExt; extern const std::string kTextExt; extern const std::string kBinExt; -static const std::string kEmpty = ""; const std::string& GetErrMsg(); -class Messager; +class MessagerContainer; class Hub; using MessagerMap = std::unordered_map>; -using MessagerContainer = std::shared_ptr; // FilterFunc filter in messagers if returned value is true. // NOTE: name is the protobuf message name, e.g.: "message ItemConf{...}". using Filter = std::function; -using MessagerContainerProvider = std::function; -using Postprocessor = std::function; -// ReadFunc reads the config file and returns its content. -using ReadFunc = std::function; +using MessagerContainerProvider = std::function()>; struct HubOptions { // Filter can only filter in certain specific messagers based on the // condition that you provide. Filter filter; -}; - -struct LoadOptions { - // postprocessor is called after loading all configurations. - Postprocessor postprocessor; - // read_func reads the config file and returns its content. - ReadFunc read_func; - // Whether to ignore unknown JSON fields during parsing. - // - // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. - bool ignore_unknown_fields = false; - // Paths maps each messager name to a corresponding config file path. - // If specified, then the main messager will be parsed from the file - // directly, other than the specified load dir. - std::unordered_map paths; - // Patch paths maps each messager name to one or multiple corresponding patch file paths. - // If specified, then main messager will be patched. - std::unordered_map> patch_paths; - // Patch dirs specifies the directory paths for config patching. - std::vector patch_dirs; - // Mode specifies the loading mode for config patching. - LoadMode mode = LoadMode::kModeDefault; + // Provide custom MessagerContainer. For keeping configuration access + // consistent in a coroutine or a transaction. + MessagerContainerProvider provider; }; // Convert file extension to Format type. @@ -124,42 +95,14 @@ class Scheduler { std::vector jobs_; }; -bool Postprocess(Postprocessor postprocessor, MessagerContainer container); +bool Postprocess(Postprocessor postprocessor, std::shared_ptr msger_map); } // namespace internal -class Messager { - public: - struct Stats { - std::chrono::microseconds duration; // total load time consuming. - // TODO: crc32 of config file to decide whether changed or not - // std::string crc32; - // int64_t last_modified_time = 0; // unix timestamp - }; - - public: - virtual ~Messager() = default; - static const std::string& Name() { return kEmpty; } - const Stats& GetStats() { return stats_; } - // Load fills message from file in the specified directory and format. - virtual bool Load(const std::string& dir, Format fmt, const LoadOptions* options = nullptr) = 0; - // Message returns the inner message data. - virtual const google::protobuf::Message* Message() const { return nullptr; } - // callback after all messagers loaded. - virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } - - protected: - // callback after this messager loaded. - virtual bool ProcessAfterLoad() { return true; }; - Stats stats_; -}; - class Hub { public: - Hub(const HubOptions* options = nullptr) : options_(options ? *options : HubOptions{}) {} - Hub(MessagerContainer container, const HubOptions* options = nullptr) : options_(options ? *options : HubOptions{}) { - SetMessagerContainer(container); - } + Hub(const HubOptions* options = nullptr) + : msger_container_(std::make_shared()), options_(options ? *options : HubOptions{}) {} /***** Synchronous Loading *****/ // Load fills messages (in MessagerContainer) from files in the specified directory and format. bool Load(const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); @@ -172,9 +115,18 @@ class Hub { // You'd better initialize the scheduler in the main thread. void InitScheduler(); + /***** MessagerMap *****/ + std::shared_ptr GetMessagerMap() const; + void SetMessagerMap(std::shared_ptr msger_map); + /***** MessagerContainer *****/ - MessagerContainer GetMessagerContainer() const { return msger_container_; } - void SetMessagerContainerProvider(MessagerContainerProvider provider) { msger_container_provider_ = provider; } + // This function is exposed only for use in MessagerContainerProvider. + std::shared_ptr GetMessagerContainer() const { + if (options_.provider != nullptr) { + return options_.provider(); + } + return msger_container_; + } /***** Access APIs *****/ template @@ -187,28 +139,21 @@ class Hub { const U* GetOrderedMap(Args... args) const; // GetLastLoadedTime returns the time when hub's msger_container_ was last set. - inline std::time_t GetLastLoadedTime() const { return last_loaded_time_; } + inline std::time_t GetLastLoadedTime() const; private: - MessagerContainer LoadNewMessagerContainer(const std::string& dir, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr); - MessagerContainer NewMessagerContainer(); - void SetMessagerContainer(MessagerContainer msger_container); - MessagerContainer GetMessagerContainerWithProvider() const; + std::shared_ptr InternalLoad(const std::string& dir, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr) const; + std::shared_ptr NewMessagerMap() const; const std::shared_ptr GetMessager(const std::string& name) const; private: // For thread-safe guarantee during configuration updating. std::mutex mutex_; // All messagers' container. - MessagerContainer msger_container_; - // Provide custom MessagerContainer. For keeping configuration access - // consistent in a coroutine or a transaction. - MessagerContainerProvider msger_container_provider_; + std::shared_ptr msger_container_; // Loading scheduler. internal::Scheduler* sched_ = nullptr; - // Last loaded time - std::time_t last_loaded_time_ = 0; // Hub options const HubOptions options_; }; @@ -221,69 +166,75 @@ const std::shared_ptr Hub::Get() const { template const U* Hub::Get(Args... args) const { - auto msg = GetMessager(T::Name()); - auto msger = std::dynamic_pointer_cast(msg); + auto msger = Get(); return msger ? msger->Get(args...) : nullptr; } template const U* Hub::GetOrderedMap(Args... args) const { - auto msg = GetMessager(T::Name()); - auto msger = std::dynamic_pointer_cast(msg); + auto msger = Get(); return msger ? msger->GetOrderedMap(args...) : nullptr; } -namespace util { +// Auto-generated specializations below +template <> +const std::shared_ptr Hub::Get() const; -// Combine hash values -// -// References: -// - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x -// - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key -inline void HashCombine(std::size_t& seed) {} +template <> +const std::shared_ptr Hub::Get() const; -template -inline void HashCombine(std::size_t& seed, const T& v, O... others) { - std::hash hasher; - seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); - HashCombine(seed, others...); -} +template <> +const std::shared_ptr Hub::Get() const; -template -inline std::size_t SugaredHashCombine(const T& v, O... others) { - std::size_t seed = 0; // start with a hash value 0 - HashCombine(seed, v, others...); - return seed; -} +template <> +const std::shared_ptr Hub::Get() const; + +template <> +const std::shared_ptr Hub::Get() const; -// Mkdir makes dir recursively. -int Mkdir(const std::string& path); -// GetDir returns all but the last element of path, typically the path's -// directory. -std::string GetDir(const std::string& path); -// GetExt returns the file name extension used by path. -// The extension is the suffix beginning at the final dot -// in the final element of path; it is empty if there is -// no dot. -std::string GetExt(const std::string& path); - -class TimeProfiler { - protected: - std::chrono::time_point last_; +template <> +const std::shared_ptr Hub::Get() const; +template <> +const std::shared_ptr Hub::Get() const; + +template <> +const std::shared_ptr Hub::Get() const; + +template <> +const std::shared_ptr Hub::Get() const; + + +class MessagerContainer { public: - TimeProfiler() { Start(); } - void Start() { last_ = std::chrono::steady_clock::now(); } - // Calculate duration between the last time point and now, - // and update last time point to now. - std::chrono::microseconds Elapse() { - auto now = std::chrono::steady_clock::now(); - auto duration = now - last_; // This is of type std::chrono::duration - last_ = now; - return std::chrono::duration_cast(duration); + MessagerContainer(std::shared_ptr msger_map = nullptr) + : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), + last_loaded_time_(std::time(nullptr)) { + // Auto-generated initializations below + activity_conf_ = std::dynamic_pointer_cast((*msger_map_)["ActivityConf"]); + chapter_conf_ = std::dynamic_pointer_cast((*msger_map_)["ChapterConf"]); + theme_conf_ = std::dynamic_pointer_cast((*msger_map_)["ThemeConf"]); + hero_base_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroBaseConf"]); + hero_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroConf"]); + item_conf_ = std::dynamic_pointer_cast((*msger_map_)["ItemConf"]); + patch_merge_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchMergeConf"]); + patch_replace_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchReplaceConf"]); + recursive_patch_conf_ = std::dynamic_pointer_cast((*msger_map_)["RecursivePatchConf"]); } -}; -} // namespace util + public: + std::shared_ptr msger_map_; + std::time_t last_loaded_time_; + // Auto-generated fields below + std::shared_ptr activity_conf_; + std::shared_ptr chapter_conf_; + std::shared_ptr theme_conf_; + std::shared_ptr hero_base_conf_; + std::shared_ptr hero_conf_; + std::shared_ptr item_conf_; + std::shared_ptr patch_merge_conf_; + std::shared_ptr patch_replace_conf_; + std::shared_ptr recursive_patch_conf_; +}; } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h index 3787010a..32e6087d 100644 --- a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h @@ -7,7 +7,7 @@ #pragma once #include -#include "hub.pc.h" +#include "messager.pc.h" #include "item_conf.pb.h" namespace tableau { diff --git a/test/cpp-tableau-loader/src/protoconf/messager.pc.h b/test/cpp-tableau-loader/src/protoconf/messager.pc.h new file mode 100644 index 00000000..8c4f9293 --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/messager.pc.h @@ -0,0 +1,134 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#pragma once +#include + +#include +#include +#include + +namespace tableau { +enum class Format { + kUnknown, + kJSON, + kText, + kBin, +}; + +enum class LoadMode { + kModeDefault, + kModeOnlyMain, + kModeOnlyPatch, +}; + +static const std::string kEmpty = ""; + +class Hub; + +using Postprocessor = std::function; +// ReadFunc reads the config file and returns its content. +using ReadFunc = std::function; + +struct LoadOptions { + // postprocessor is called after loading all configurations. + Postprocessor postprocessor; + // read_func reads the config file and returns its content. + ReadFunc read_func; + // Whether to ignore unknown JSON fields during parsing. + // + // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. + bool ignore_unknown_fields = false; + // Paths maps each messager name to a corresponding config file path. + // If specified, then the main messager will be parsed from the file + // directly, other than the specified load dir. + std::unordered_map paths; + // Patch paths maps each messager name to one or multiple corresponding patch file paths. + // If specified, then main messager will be patched. + std::unordered_map> patch_paths; + // Patch dirs specifies the directory paths for config patching. + std::vector patch_dirs; + // Mode specifies the loading mode for config patching. + LoadMode mode = LoadMode::kModeDefault; +}; + +class Messager { + public: + struct Stats { + std::chrono::microseconds duration; // total load time consuming. + // TODO: crc32 of config file to decide whether changed or not + // std::string crc32; + // int64_t last_modified_time = 0; // unix timestamp + }; + + public: + virtual ~Messager() = default; + static const std::string& Name() { return kEmpty; } + const Stats& GetStats() { return stats_; } + // Load fills message from file in the specified directory and format. + virtual bool Load(const std::string& dir, Format fmt, const LoadOptions* options = nullptr) = 0; + // Message returns the inner message data. + virtual const google::protobuf::Message* Message() const { return nullptr; } + // callback after all messagers loaded. + virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } + + protected: + // callback after this messager loaded. + virtual bool ProcessAfterLoad() { return true; }; + Stats stats_; +}; + +namespace util { +// Combine hash values +// +// References: +// - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x +// - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key +inline void HashCombine(std::size_t& seed) {} + +template +inline void HashCombine(std::size_t& seed, const T& v, O... others) { + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + HashCombine(seed, others...); +} + +template +inline std::size_t SugaredHashCombine(const T& v, O... others) { + std::size_t seed = 0; // start with a hash value 0 + HashCombine(seed, v, others...); + return seed; +} + +// Mkdir makes dir recursively. +int Mkdir(const std::string& path); +// GetDir returns all but the last element of path, typically the path's +// directory. +std::string GetDir(const std::string& path); +// GetExt returns the file name extension used by path. +// The extension is the suffix beginning at the final dot +// in the final element of path; it is empty if there is +// no dot. +std::string GetExt(const std::string& path); + +class TimeProfiler { + protected: + std::chrono::time_point last_; + + public: + TimeProfiler() { Start(); } + void Start() { last_ = std::chrono::steady_clock::now(); } + // Calculate duration between the last time point and now, + // and update last time point to now. + std::chrono::microseconds Elapse() { + auto now = std::chrono::steady_clock::now(); + auto duration = now - last_; // This is of type std::chrono::duration + last_ = now; + return std::chrono::duration_cast(duration); + } +}; + +} // namespace util +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h index b4bbc6d0..6b4c3f75 100644 --- a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h @@ -7,7 +7,7 @@ #pragma once #include -#include "hub.pc.h" +#include "messager.pc.h" #include "patch_conf.pb.h" namespace tableau { diff --git a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h index d3165227..a0328515 100644 --- a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h @@ -7,7 +7,7 @@ #pragma once #include -#include "hub.pc.h" +#include "messager.pc.h" #include "test_conf.pb.h" namespace tableau { From 5331e1776b1ad2042f74b75653156f79fbe3bbd4 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 24 Apr 2025 16:54:45 +0800 Subject: [PATCH 03/14] feat: generated order --- cmd/protoc-gen-cpp-tableau-loader/embed.go | 16 ++++++++-------- cmd/protoc-gen-cpp-tableau-loader/registry.go | 2 +- test/cpp-tableau-loader/src/protoconf/hub.pc.h | 12 ++++++------ .../src/protoconf/registry.pc.h | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed.go b/cmd/protoc-gen-cpp-tableau-loader/embed.go index 59b2af3f..83afc287 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed.go +++ b/cmd/protoc-gen-cpp-tableau-loader/embed.go @@ -45,8 +45,8 @@ func generateEmbed(gen *protogen.Plugin) { case "hub.pc.cc": // Auto-generated specializations below impl := "" - for _, messagers := range fileMessagers { - for _, messager := range messagers { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { impl += "template <>\n" impl += "const std::shared_ptr<" + messager + "> Hub::Get<" + messager + ">() const {\n" impl += " return GetMessagerContainer()->" + strcase.ToSnake(messager) + "_;\n" @@ -64,8 +64,8 @@ func generateEmbed(gen *protogen.Plugin) { file = strings.ReplaceAll(file, includeNotes, includeNotes+impl) // Auto-generated specializations below impl = "" - for _, messagers := range fileMessagers { - for _, messager := range messagers { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { impl += "template <>\n" impl += "const std::shared_ptr<" + messager + "> Hub::Get<" + messager + ">() const;\n" impl += "\n" @@ -74,16 +74,16 @@ func generateEmbed(gen *protogen.Plugin) { file = strings.ReplaceAll(file, specializationNotes, specializationNotes+impl) // Auto-generated initializations below impl = "" - for _, messagers := range fileMessagers { - for _, messager := range messagers { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { impl += " " + strcase.ToSnake(messager) + "_ = std::dynamic_pointer_cast<" + messager + `>((*msger_map_)["` + messager + `"]);` + "\n" } } file = strings.ReplaceAll(file, initializationNotes, initializationNotes+impl) // Auto-generated fields below impl = "" - for _, messagers := range fileMessagers { - for _, messager := range messagers { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { impl += " std::shared_ptr<" + messager + "> " + strcase.ToSnake(messager) + "_;\n" } } diff --git a/cmd/protoc-gen-cpp-tableau-loader/registry.go b/cmd/protoc-gen-cpp-tableau-loader/registry.go index 28d7e009..ee99526d 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/registry.go +++ b/cmd/protoc-gen-cpp-tableau-loader/registry.go @@ -159,7 +159,7 @@ func generateShardedRegistryCppFileContent(gen *protogen.Plugin, g *protogen.Gen } const registryHppTop = `#pragma once -#include "hub.pc.h" +#include "messager.pc.h" namespace tableau { using MessagerGenerator = std::function()>; // messager name -> messager generator diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index 9f73d9d4..a6eea56d 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -211,30 +211,30 @@ class MessagerContainer { : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), last_loaded_time_(std::time(nullptr)) { // Auto-generated initializations below - activity_conf_ = std::dynamic_pointer_cast((*msger_map_)["ActivityConf"]); - chapter_conf_ = std::dynamic_pointer_cast((*msger_map_)["ChapterConf"]); - theme_conf_ = std::dynamic_pointer_cast((*msger_map_)["ThemeConf"]); hero_base_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroBaseConf"]); hero_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroConf"]); item_conf_ = std::dynamic_pointer_cast((*msger_map_)["ItemConf"]); patch_merge_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchMergeConf"]); patch_replace_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchReplaceConf"]); recursive_patch_conf_ = std::dynamic_pointer_cast((*msger_map_)["RecursivePatchConf"]); + activity_conf_ = std::dynamic_pointer_cast((*msger_map_)["ActivityConf"]); + chapter_conf_ = std::dynamic_pointer_cast((*msger_map_)["ChapterConf"]); + theme_conf_ = std::dynamic_pointer_cast((*msger_map_)["ThemeConf"]); } public: std::shared_ptr msger_map_; std::time_t last_loaded_time_; // Auto-generated fields below - std::shared_ptr activity_conf_; - std::shared_ptr chapter_conf_; - std::shared_ptr theme_conf_; std::shared_ptr hero_base_conf_; std::shared_ptr hero_conf_; std::shared_ptr item_conf_; std::shared_ptr patch_merge_conf_; std::shared_ptr patch_replace_conf_; std::shared_ptr recursive_patch_conf_; + std::shared_ptr activity_conf_; + std::shared_ptr chapter_conf_; + std::shared_ptr theme_conf_; }; } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/registry.pc.h b/test/cpp-tableau-loader/src/protoconf/registry.pc.h index 968a700b..59034f19 100644 --- a/test/cpp-tableau-loader/src/protoconf/registry.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/registry.pc.h @@ -4,7 +4,7 @@ // - protoc v3.19.3 #pragma once -#include "hub.pc.h" +#include "messager.pc.h" namespace tableau { using MessagerGenerator = std::function()>; // messager name -> messager generator From 52f016bcfbe3d5a3069e82608ad2d92d042e6be9 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 24 Apr 2025 17:25:50 +0800 Subject: [PATCH 04/14] feat: move includes to hub.pc.cc --- cmd/protoc-gen-cpp-tableau-loader/embed.go | 33 ++++++++++++------- .../embed/hub.pc.cc | 7 ++++ .../embed/hub.pc.h | 11 ++----- .../src/protoconf/hub.pc.cc | 20 +++++++++++ .../cpp-tableau-loader/src/protoconf/hub.pc.h | 33 +++++++------------ 5 files changed, 63 insertions(+), 41 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed.go b/cmd/protoc-gen-cpp-tableau-loader/embed.go index 83afc287..d6372a2a 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed.go +++ b/cmd/protoc-gen-cpp-tableau-loader/embed.go @@ -15,6 +15,7 @@ var efs embed.FS const ( includeNotes = "// Auto-generated includes below\n" + declarationNotes = "// Auto-generated declarations below\n" specializationNotes = "// Auto-generated specializations below\n" initializationNotes = "// Auto-generated initializations below\n" fieldNotes = "// Auto-generated fields below\n" @@ -43,8 +44,14 @@ func generateEmbed(gen *protogen.Plugin) { file := string(content) switch entry.Name() { case "hub.pc.cc": - // Auto-generated specializations below + // Auto-generated includes below impl := "" + for _, proto := range protofiles { + impl += `#include "` + proto + "." + pcExt + `.h"` + "\n" + } + file = strings.ReplaceAll(file, includeNotes, includeNotes+impl) + // Auto-generated specializations below + impl = "" for _, proto := range protofiles { for _, messager := range fileMessagers[proto] { impl += "template <>\n" @@ -55,13 +62,23 @@ func generateEmbed(gen *protogen.Plugin) { } } file = strings.ReplaceAll(file, specializationNotes, specializationNotes+impl) + // Auto-generated initializations below + impl = "" + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + impl += " " + strcase.ToSnake(messager) + "_ = std::dynamic_pointer_cast<" + messager + `>((*msger_map_)["` + messager + `"]);` + "\n" + } + } + file = strings.ReplaceAll(file, initializationNotes, initializationNotes+impl) case "hub.pc.h": - // Auto-generated includes below + // Auto-generated declarations below impl := "" for _, proto := range protofiles { - impl += `#include "` + proto + "." + pcExt + `.h"` + "\n" + for _, messager := range fileMessagers[proto] { + impl += "class " + messager + ";\n" + } } - file = strings.ReplaceAll(file, includeNotes, includeNotes+impl) + file = strings.ReplaceAll(file, declarationNotes, declarationNotes+impl) // Auto-generated specializations below impl = "" for _, proto := range protofiles { @@ -72,14 +89,6 @@ func generateEmbed(gen *protogen.Plugin) { } } file = strings.ReplaceAll(file, specializationNotes, specializationNotes+impl) - // Auto-generated initializations below - impl = "" - for _, proto := range protofiles { - for _, messager := range fileMessagers[proto] { - impl += " " + strcase.ToSnake(messager) + "_ = std::dynamic_pointer_cast<" + messager + `>((*msger_map_)["` + messager + `"]);` + "\n" - } - } - file = strings.ReplaceAll(file, initializationNotes, initializationNotes+impl) // Auto-generated fields below impl = "" for _, proto := range protofiles { diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc index 6a2e4350..14bfb19d 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc @@ -20,6 +20,8 @@ #include "messager.pc.h" #include "registry.pc.h" +// Auto-generated includes below + namespace tableau { #ifdef _WIN32 #undef GetMessage @@ -527,6 +529,11 @@ const std::shared_ptr Hub::GetMessager(const std::string& name) const std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; } // Auto-generated specializations below +MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) + : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), + last_loaded_time_(std::time(nullptr)) { + // Auto-generated initializations below +} namespace internal { // Thread-local storage (TLS) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h index 5f3b6604..5b90a7ec 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h @@ -17,8 +17,6 @@ #include "messager.pc.h" -// Auto-generated includes below - namespace tableau { extern const std::string kUnknownExt; extern const std::string kJSONExt; @@ -30,6 +28,8 @@ const std::string& GetErrMsg(); class MessagerContainer; class Hub; +// Auto-generated declarations below + using MessagerMap = std::unordered_map>; // FilterFunc filter in messagers if returned value is true. // NOTE: name is the protobuf message name, e.g.: "message ItemConf{...}". @@ -168,14 +168,9 @@ const U* Hub::GetOrderedMap(Args... args) const { } // Auto-generated specializations below - class MessagerContainer { public: - MessagerContainer(std::shared_ptr msger_map = nullptr) - : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), - last_loaded_time_(std::time(nullptr)) { - // Auto-generated initializations below - } + MessagerContainer(std::shared_ptr msger_map = nullptr); public: std::shared_ptr msger_map_; diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index dcf65ae1..d4b2c297 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -25,6 +25,12 @@ #include "messager.pc.h" #include "registry.pc.h" +// Auto-generated includes below +#include "hero_conf.pc.h" +#include "item_conf.pc.h" +#include "patch_conf.pc.h" +#include "test_conf.pc.h" + namespace tableau { #ifdef _WIN32 #undef GetMessage @@ -577,6 +583,20 @@ const std::shared_ptr Hub::Get() const { return GetMessagerContainer()->theme_conf_; } +MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) + : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), + last_loaded_time_(std::time(nullptr)) { + // Auto-generated initializations below + hero_base_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroBaseConf"]); + hero_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroConf"]); + item_conf_ = std::dynamic_pointer_cast((*msger_map_)["ItemConf"]); + patch_merge_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchMergeConf"]); + patch_replace_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchReplaceConf"]); + recursive_patch_conf_ = std::dynamic_pointer_cast((*msger_map_)["RecursivePatchConf"]); + activity_conf_ = std::dynamic_pointer_cast((*msger_map_)["ActivityConf"]); + chapter_conf_ = std::dynamic_pointer_cast((*msger_map_)["ChapterConf"]); + theme_conf_ = std::dynamic_pointer_cast((*msger_map_)["ThemeConf"]); +} namespace internal { // Thread-local storage (TLS) diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index a6eea56d..07b24f8c 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -22,12 +22,6 @@ #include "messager.pc.h" -// Auto-generated includes below -#include "hero_conf.pc.h" -#include "item_conf.pc.h" -#include "patch_conf.pc.h" -#include "test_conf.pc.h" - namespace tableau { extern const std::string kUnknownExt; extern const std::string kJSONExt; @@ -39,6 +33,17 @@ const std::string& GetErrMsg(); class MessagerContainer; class Hub; +// Auto-generated declarations below +class HeroBaseConf; +class HeroConf; +class ItemConf; +class PatchMergeConf; +class PatchReplaceConf; +class RecursivePatchConf; +class ActivityConf; +class ChapterConf; +class ThemeConf; + using MessagerMap = std::unordered_map>; // FilterFunc filter in messagers if returned value is true. // NOTE: name is the protobuf message name, e.g.: "message ItemConf{...}". @@ -204,23 +209,9 @@ const std::shared_ptr Hub::Get() const; template <> const std::shared_ptr Hub::Get() const; - class MessagerContainer { public: - MessagerContainer(std::shared_ptr msger_map = nullptr) - : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), - last_loaded_time_(std::time(nullptr)) { - // Auto-generated initializations below - hero_base_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroBaseConf"]); - hero_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroConf"]); - item_conf_ = std::dynamic_pointer_cast((*msger_map_)["ItemConf"]); - patch_merge_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchMergeConf"]); - patch_replace_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchReplaceConf"]); - recursive_patch_conf_ = std::dynamic_pointer_cast((*msger_map_)["RecursivePatchConf"]); - activity_conf_ = std::dynamic_pointer_cast((*msger_map_)["ActivityConf"]); - chapter_conf_ = std::dynamic_pointer_cast((*msger_map_)["ChapterConf"]); - theme_conf_ = std::dynamic_pointer_cast((*msger_map_)["ThemeConf"]); - } + MessagerContainer(std::shared_ptr msger_map = nullptr); public: std::shared_ptr msger_map_; From 71576151570d86b37727fbdfddf386fa0bc12af3 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 24 Apr 2025 18:01:05 +0800 Subject: [PATCH 05/14] feat: update code --- .../embed/hub.pc.cc | 97 ++++--------------- .../embed/hub.pc.h | 4 +- .../embed/logger.pc.cc | 2 +- .../embed/messager.pc.h | 55 ----------- .../embed/util.pc.cc | 56 +++++++++++ .../embed/util.pc.h | 59 +++++++++++ cmd/protoc-gen-cpp-tableau-loader/messager.go | 2 + test/cpp-tableau-loader/src/main.cpp | 4 - .../src/protoconf/hero_conf.pc.cc | 1 + .../src/protoconf/hero_conf.pc.h | 1 + .../src/protoconf/hub.pc.cc | 97 ++++--------------- .../cpp-tableau-loader/src/protoconf/hub.pc.h | 4 +- .../src/protoconf/item_conf.pc.cc | 1 + .../src/protoconf/item_conf.pc.h | 1 + .../src/protoconf/logger.pc.cc | 2 +- .../src/protoconf/messager.pc.h | 55 ----------- .../src/protoconf/patch_conf.pc.cc | 1 + .../src/protoconf/patch_conf.pc.h | 1 + .../src/protoconf/test_conf.pc.cc | 1 + .../src/protoconf/test_conf.pc.h | 1 + .../src/protoconf/util.pc.cc | 61 ++++++++++++ .../src/protoconf/util.pc.h | 64 ++++++++++++ 22 files changed, 296 insertions(+), 274 deletions(-) create mode 100644 cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc create mode 100644 cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h create mode 100644 test/cpp-tableau-loader/src/protoconf/util.pc.cc create mode 100644 test/cpp-tableau-loader/src/protoconf/util.pc.h diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc index 14bfb19d..149d8580 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc @@ -4,13 +4,6 @@ #include #include -#ifdef _WIN32 -#include -#include -#else -#include -#endif - #include #include #include @@ -19,16 +12,13 @@ #include "logger.pc.h" #include "messager.pc.h" #include "registry.pc.h" +#include "util.pc.h" // Auto-generated includes below namespace tableau { #ifdef _WIN32 #undef GetMessage -#define mkdir(path, mode) _mkdir(path) -static constexpr char kPathSeperator = '\\'; -#else -static constexpr char kPathSeperator = '/'; #endif static thread_local std::string g_err_msg; @@ -446,7 +436,7 @@ bool Hub::Load(const std::string& dir, Format fmt /* = Format::kJSON */, const L if (!msger_map) { return false; } - bool ok = internal::Postprocess(options->postprocessor, msger_map); + bool ok = Postprocess(msger_map); if (!ok) { return false; } @@ -460,7 +450,7 @@ bool Hub::AsyncLoad(const std::string& dir, Format fmt /* = Format::kJSON */, if (!msger_map) { return false; } - bool ok = internal::Postprocess(options->postprocessor, msger_map); + bool ok = Postprocess(msger_map); if (!ok) { return false; } @@ -526,6 +516,23 @@ const std::shared_ptr Hub::GetMessager(const std::string& name) const return nullptr; } +bool Hub::Postprocess(std::shared_ptr msger_map) { + // create a temporary hub with messager container for post process + Hub tmp_hub; + tmp_hub.SetMessagerMap(msger_map); + + // messager-level postprocess + for (auto iter : *msger_map) { + auto msger = iter.second; + bool ok = msger->ProcessAfterLoadAll(tmp_hub); + if (!ok) { + g_err_msg = "hub call ProcessAfterLoadAll failed, messager: " + msger->Name(); + return false; + } + } + return true; +} + std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; } // Auto-generated specializations below @@ -583,70 +590,6 @@ void Scheduler::AssertInLoopThread() const { } } -bool Postprocess(Postprocessor postprocessor, std::shared_ptr msger_map) { - // create a temporary hub with messager container for post process - Hub tmp_hub; - tmp_hub.SetMessagerMap(msger_map); - - // messager-level postprocess - for (auto iter : *msger_map) { - auto msger = iter.second; - bool ok = msger->ProcessAfterLoadAll(tmp_hub); - if (!ok) { - g_err_msg = "hub call ProcessAfterLoadAll failed, messager: " + msger->Name(); - return false; - } - } - - // hub-level postprocess - if (postprocessor != nullptr) { - bool ok = postprocessor(tmp_hub); - if (!ok) { - g_err_msg = "hub call Postprocesser failed, you'd better check your custom 'postprocessor' load option"; - return false; - } - } - - return true; -} - } // namespace internal -namespace util { -int Mkdir(const std::string& path) { - std::string path_ = path + kPathSeperator; - struct stat info; - for (size_t pos = path_.find(kPathSeperator, 0); pos != std::string::npos; pos = path_.find(kPathSeperator, pos)) { - ++pos; - auto sub_dir = path_.substr(0, pos); - if (stat(sub_dir.c_str(), &info) == 0 && info.st_mode & S_IFDIR) { - continue; - } - int status = mkdir(sub_dir.c_str(), 0755); - if (status != 0) { - std::cerr << "system error: " << strerror(errno) << std::endl; - return -1; - } - } - return 0; -} - -std::string GetDir(const std::string& path) { - size_t pos = path.find_last_of(kPathSeperator); - if (pos != std::string::npos) { - return path.substr(0, pos); - } - return kEmpty; -} - -std::string GetExt(const std::string& path) { - std::size_t pos = path.find_last_of("."); - if (pos != std::string::npos) { - return path.substr(pos); - } - return kEmpty; -} - -} // namespace util - } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h index 5b90a7ec..48455d72 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h @@ -86,8 +86,6 @@ class Scheduler { std::vector jobs_; }; -bool Postprocess(Postprocessor postprocessor, std::shared_ptr msger_map); - } // namespace internal class Hub { @@ -138,6 +136,8 @@ class Hub { std::shared_ptr NewMessagerMap() const; const std::shared_ptr GetMessager(const std::string& name) const; + bool Postprocess(std::shared_ptr msger_map); + private: // For thread-safe guarantee during configuration updating. std::mutex mutex_; diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc index 1cac4d2f..893b63a6 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc @@ -15,7 +15,7 @@ #include #include -#include "hub.pc.h" +#include "util.pc.h" #ifdef _WIN32 #define gettid() GetCurrentThreadId() diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h index df436eda..4e38ca70 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include @@ -23,13 +22,10 @@ static const std::string kEmpty = ""; class Hub; -using Postprocessor = std::function; // ReadFunc reads the config file and returns its content. using ReadFunc = std::function; struct LoadOptions { - // postprocessor is called after loading all configurations. - Postprocessor postprocessor; // read_func reads the config file and returns its content. ReadFunc read_func; // Whether to ignore unknown JSON fields during parsing. @@ -75,55 +71,4 @@ class Messager { Stats stats_; }; -namespace util { -// Combine hash values -// -// References: -// - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x -// - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key -inline void HashCombine(std::size_t& seed) {} - -template -inline void HashCombine(std::size_t& seed, const T& v, O... others) { - std::hash hasher; - seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); - HashCombine(seed, others...); -} - -template -inline std::size_t SugaredHashCombine(const T& v, O... others) { - std::size_t seed = 0; // start with a hash value 0 - HashCombine(seed, v, others...); - return seed; -} - -// Mkdir makes dir recursively. -int Mkdir(const std::string& path); -// GetDir returns all but the last element of path, typically the path's -// directory. -std::string GetDir(const std::string& path); -// GetExt returns the file name extension used by path. -// The extension is the suffix beginning at the final dot -// in the final element of path; it is empty if there is -// no dot. -std::string GetExt(const std::string& path); - -class TimeProfiler { - protected: - std::chrono::time_point last_; - - public: - TimeProfiler() { Start(); } - void Start() { last_ = std::chrono::steady_clock::now(); } - // Calculate duration between the last time point and now, - // and update last time point to now. - std::chrono::microseconds Elapse() { - auto now = std::chrono::steady_clock::now(); - auto duration = now - last_; // This is of type std::chrono::duration - last_ = now; - return std::chrono::duration_cast(duration); - } -}; - -} // namespace util } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc new file mode 100644 index 00000000..6b02848b --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc @@ -0,0 +1,56 @@ +#include "util.pc.h" + +#ifdef _WIN32 +#include +#include +#else +#include +#endif + +#include "messager.pc.h" + +namespace tableau { +#ifdef _WIN32 +#define mkdir(path, mode) _mkdir(path) +static constexpr char kPathSeperator = '\\'; +#else +static constexpr char kPathSeperator = '/'; +#endif + +namespace util { +int Mkdir(const std::string& path) { + std::string path_ = path + kPathSeperator; + struct stat info; + for (size_t pos = path_.find(kPathSeperator, 0); pos != std::string::npos; pos = path_.find(kPathSeperator, pos)) { + ++pos; + auto sub_dir = path_.substr(0, pos); + if (stat(sub_dir.c_str(), &info) == 0 && info.st_mode & S_IFDIR) { + continue; + } + int status = mkdir(sub_dir.c_str(), 0755); + if (status != 0) { + std::cerr << "system error: " << strerror(errno) << std::endl; + return -1; + } + } + return 0; +} + +std::string GetDir(const std::string& path) { + size_t pos = path.find_last_of(kPathSeperator); + if (pos != std::string::npos) { + return path.substr(0, pos); + } + return kEmpty; +} + +std::string GetExt(const std::string& path) { + std::size_t pos = path.find_last_of("."); + if (pos != std::string::npos) { + return path.substr(pos); + } + return kEmpty; +} + +} // namespace util +} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h new file mode 100644 index 00000000..5622380d --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include + +namespace tableau { +namespace util { +// Combine hash values +// +// References: +// - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x +// - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key +inline void HashCombine(std::size_t& seed) {} + +template +inline void HashCombine(std::size_t& seed, const T& v, O... others) { + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + HashCombine(seed, others...); +} + +template +inline std::size_t SugaredHashCombine(const T& v, O... others) { + std::size_t seed = 0; // start with a hash value 0 + HashCombine(seed, v, others...); + return seed; +} + +// Mkdir makes dir recursively. +int Mkdir(const std::string& path); +// GetDir returns all but the last element of path, typically the path's +// directory. +std::string GetDir(const std::string& path); +// GetExt returns the file name extension used by path. +// The extension is the suffix beginning at the final dot +// in the final element of path; it is empty if there is +// no dot. +std::string GetExt(const std::string& path); + +class TimeProfiler { + protected: + std::chrono::time_point last_; + + public: + TimeProfiler() { Start(); } + void Start() { last_ = std::chrono::steady_clock::now(); } + // Calculate duration between the last time point and now, + // and update last time point to now. + std::chrono::microseconds Elapse() { + auto now = std::chrono::steady_clock::now(); + auto duration = now - last_; // This is of type std::chrono::duration + last_ = now; + return std::chrono::duration_cast(duration); + } +}; + +} // namespace util +} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/messager.go b/cmd/protoc-gen-cpp-tableau-loader/messager.go index e846fd3e..2fdd8b13 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/messager.go +++ b/cmd/protoc-gen-cpp-tableau-loader/messager.go @@ -46,6 +46,7 @@ func generateHppFileContent(gen *protogen.Plugin, file *protogen.File, g *protog g.P("#include ") g.P() g.P(`#include "`, "messager.", pcExt, `.h"`) + g.P(`#include "`, "util.", pcExt, `.h"`) g.P(`#include "`, file.GeneratedFilenamePrefix, ".", pbExt, `.h"`) g.P() @@ -135,6 +136,7 @@ func generateCppFileContent(gen *protogen.Plugin, file *protogen.File, g *protog g.P(`#include "`, file.GeneratedFilenamePrefix, ".", pcExt, `.h"`) g.P() g.P(`#include "hub.pc.h"`) + g.P(`#include "util.pc.h"`) g.P() g.P("namespace ", *namespace, " {") diff --git a/test/cpp-tableau-loader/src/main.cpp b/test/cpp-tableau-loader/src/main.cpp index ad22f611..53fde80a 100644 --- a/test/cpp-tableau-loader/src/main.cpp +++ b/test/cpp-tableau-loader/src/main.cpp @@ -125,10 +125,6 @@ int main() { tableau::LoadOptions options; options.ignore_unknown_fields = true; options.patch_dirs = {"../../testdata/patchconf/"}; - options.postprocessor = [](const tableau::Hub& hub) { - std::cout << "post process done!" << std::endl; - return 1; - }; options.paths["ItemConf"] = "../../testdata/conf/ItemConf.json"; bool ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, &options); diff --git a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc index 793e9e26..60cff677 100644 --- a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc @@ -7,6 +7,7 @@ #include "hero_conf.pc.h" #include "hub.pc.h" +#include "util.pc.h" namespace tableau { const std::string HeroConf::kProtoName = "HeroConf"; diff --git a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h index 56df8472..53ec91a2 100644 --- a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h @@ -8,6 +8,7 @@ #include #include "messager.pc.h" +#include "util.pc.h" #include "hero_conf.pb.h" namespace tableau { diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index d4b2c297..3c3b8d28 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -9,13 +9,6 @@ #include #include -#ifdef _WIN32 -#include -#include -#else -#include -#endif - #include #include #include @@ -24,6 +17,7 @@ #include "logger.pc.h" #include "messager.pc.h" #include "registry.pc.h" +#include "util.pc.h" // Auto-generated includes below #include "hero_conf.pc.h" @@ -34,10 +28,6 @@ namespace tableau { #ifdef _WIN32 #undef GetMessage -#define mkdir(path, mode) _mkdir(path) -static constexpr char kPathSeperator = '\\'; -#else -static constexpr char kPathSeperator = '/'; #endif static thread_local std::string g_err_msg; @@ -455,7 +445,7 @@ bool Hub::Load(const std::string& dir, Format fmt /* = Format::kJSON */, const L if (!msger_map) { return false; } - bool ok = internal::Postprocess(options->postprocessor, msger_map); + bool ok = Postprocess(msger_map); if (!ok) { return false; } @@ -469,7 +459,7 @@ bool Hub::AsyncLoad(const std::string& dir, Format fmt /* = Format::kJSON */, if (!msger_map) { return false; } - bool ok = internal::Postprocess(options->postprocessor, msger_map); + bool ok = Postprocess(msger_map); if (!ok) { return false; } @@ -535,6 +525,23 @@ const std::shared_ptr Hub::GetMessager(const std::string& name) const return nullptr; } +bool Hub::Postprocess(std::shared_ptr msger_map) { + // create a temporary hub with messager container for post process + Hub tmp_hub; + tmp_hub.SetMessagerMap(msger_map); + + // messager-level postprocess + for (auto iter : *msger_map) { + auto msger = iter.second; + bool ok = msger->ProcessAfterLoadAll(tmp_hub); + if (!ok) { + g_err_msg = "hub call ProcessAfterLoadAll failed, messager: " + msger->Name(); + return false; + } + } + return true; +} + std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; } // Auto-generated specializations below @@ -646,70 +653,6 @@ void Scheduler::AssertInLoopThread() const { } } -bool Postprocess(Postprocessor postprocessor, std::shared_ptr msger_map) { - // create a temporary hub with messager container for post process - Hub tmp_hub; - tmp_hub.SetMessagerMap(msger_map); - - // messager-level postprocess - for (auto iter : *msger_map) { - auto msger = iter.second; - bool ok = msger->ProcessAfterLoadAll(tmp_hub); - if (!ok) { - g_err_msg = "hub call ProcessAfterLoadAll failed, messager: " + msger->Name(); - return false; - } - } - - // hub-level postprocess - if (postprocessor != nullptr) { - bool ok = postprocessor(tmp_hub); - if (!ok) { - g_err_msg = "hub call Postprocesser failed, you'd better check your custom 'postprocessor' load option"; - return false; - } - } - - return true; -} - } // namespace internal -namespace util { -int Mkdir(const std::string& path) { - std::string path_ = path + kPathSeperator; - struct stat info; - for (size_t pos = path_.find(kPathSeperator, 0); pos != std::string::npos; pos = path_.find(kPathSeperator, pos)) { - ++pos; - auto sub_dir = path_.substr(0, pos); - if (stat(sub_dir.c_str(), &info) == 0 && info.st_mode & S_IFDIR) { - continue; - } - int status = mkdir(sub_dir.c_str(), 0755); - if (status != 0) { - std::cerr << "system error: " << strerror(errno) << std::endl; - return -1; - } - } - return 0; -} - -std::string GetDir(const std::string& path) { - size_t pos = path.find_last_of(kPathSeperator); - if (pos != std::string::npos) { - return path.substr(0, pos); - } - return kEmpty; -} - -std::string GetExt(const std::string& path) { - std::size_t pos = path.find_last_of("."); - if (pos != std::string::npos) { - return path.substr(pos); - } - return kEmpty; -} - -} // namespace util - } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index 07b24f8c..c73dedf9 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -100,8 +100,6 @@ class Scheduler { std::vector jobs_; }; -bool Postprocess(Postprocessor postprocessor, std::shared_ptr msger_map); - } // namespace internal class Hub { @@ -152,6 +150,8 @@ class Hub { std::shared_ptr NewMessagerMap() const; const std::shared_ptr GetMessager(const std::string& name) const; + bool Postprocess(std::shared_ptr msger_map); + private: // For thread-safe guarantee during configuration updating. std::mutex mutex_; diff --git a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc index d1440a77..723e2599 100644 --- a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc @@ -7,6 +7,7 @@ #include "item_conf.pc.h" #include "hub.pc.h" +#include "util.pc.h" namespace tableau { const std::string ItemConf::kProtoName = "ItemConf"; diff --git a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h index 32e6087d..77442e63 100644 --- a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h @@ -8,6 +8,7 @@ #include #include "messager.pc.h" +#include "util.pc.h" #include "item_conf.pb.h" namespace tableau { diff --git a/test/cpp-tableau-loader/src/protoconf/logger.pc.cc b/test/cpp-tableau-loader/src/protoconf/logger.pc.cc index ba7401bf..99e62c6d 100644 --- a/test/cpp-tableau-loader/src/protoconf/logger.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/logger.pc.cc @@ -20,7 +20,7 @@ #include #include -#include "hub.pc.h" +#include "util.pc.h" #ifdef _WIN32 #define gettid() GetCurrentThreadId() diff --git a/test/cpp-tableau-loader/src/protoconf/messager.pc.h b/test/cpp-tableau-loader/src/protoconf/messager.pc.h index 8c4f9293..e5fb56cc 100644 --- a/test/cpp-tableau-loader/src/protoconf/messager.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/messager.pc.h @@ -6,7 +6,6 @@ #pragma once #include -#include #include #include @@ -28,13 +27,10 @@ static const std::string kEmpty = ""; class Hub; -using Postprocessor = std::function; // ReadFunc reads the config file and returns its content. using ReadFunc = std::function; struct LoadOptions { - // postprocessor is called after loading all configurations. - Postprocessor postprocessor; // read_func reads the config file and returns its content. ReadFunc read_func; // Whether to ignore unknown JSON fields during parsing. @@ -80,55 +76,4 @@ class Messager { Stats stats_; }; -namespace util { -// Combine hash values -// -// References: -// - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x -// - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key -inline void HashCombine(std::size_t& seed) {} - -template -inline void HashCombine(std::size_t& seed, const T& v, O... others) { - std::hash hasher; - seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); - HashCombine(seed, others...); -} - -template -inline std::size_t SugaredHashCombine(const T& v, O... others) { - std::size_t seed = 0; // start with a hash value 0 - HashCombine(seed, v, others...); - return seed; -} - -// Mkdir makes dir recursively. -int Mkdir(const std::string& path); -// GetDir returns all but the last element of path, typically the path's -// directory. -std::string GetDir(const std::string& path); -// GetExt returns the file name extension used by path. -// The extension is the suffix beginning at the final dot -// in the final element of path; it is empty if there is -// no dot. -std::string GetExt(const std::string& path); - -class TimeProfiler { - protected: - std::chrono::time_point last_; - - public: - TimeProfiler() { Start(); } - void Start() { last_ = std::chrono::steady_clock::now(); } - // Calculate duration between the last time point and now, - // and update last time point to now. - std::chrono::microseconds Elapse() { - auto now = std::chrono::steady_clock::now(); - auto duration = now - last_; // This is of type std::chrono::duration - last_ = now; - return std::chrono::duration_cast(duration); - } -}; - -} // namespace util } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc index a6c95051..de89e988 100644 --- a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc @@ -7,6 +7,7 @@ #include "patch_conf.pc.h" #include "hub.pc.h" +#include "util.pc.h" namespace tableau { const std::string PatchReplaceConf::kProtoName = "PatchReplaceConf"; diff --git a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h index 6b4c3f75..44ed81cb 100644 --- a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h @@ -8,6 +8,7 @@ #include #include "messager.pc.h" +#include "util.pc.h" #include "patch_conf.pb.h" namespace tableau { diff --git a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc index b004e6d7..014983bb 100644 --- a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc @@ -7,6 +7,7 @@ #include "test_conf.pc.h" #include "hub.pc.h" +#include "util.pc.h" namespace tableau { const std::string ActivityConf::kProtoName = "ActivityConf"; diff --git a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h index a0328515..ee167d33 100644 --- a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h @@ -8,6 +8,7 @@ #include #include "messager.pc.h" +#include "util.pc.h" #include "test_conf.pb.h" namespace tableau { diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.cc b/test/cpp-tableau-loader/src/protoconf/util.pc.cc new file mode 100644 index 00000000..c6ec22e2 --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.cc @@ -0,0 +1,61 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#include "util.pc.h" + +#ifdef _WIN32 +#include +#include +#else +#include +#endif + +#include "messager.pc.h" + +namespace tableau { +#ifdef _WIN32 +#define mkdir(path, mode) _mkdir(path) +static constexpr char kPathSeperator = '\\'; +#else +static constexpr char kPathSeperator = '/'; +#endif + +namespace util { +int Mkdir(const std::string& path) { + std::string path_ = path + kPathSeperator; + struct stat info; + for (size_t pos = path_.find(kPathSeperator, 0); pos != std::string::npos; pos = path_.find(kPathSeperator, pos)) { + ++pos; + auto sub_dir = path_.substr(0, pos); + if (stat(sub_dir.c_str(), &info) == 0 && info.st_mode & S_IFDIR) { + continue; + } + int status = mkdir(sub_dir.c_str(), 0755); + if (status != 0) { + std::cerr << "system error: " << strerror(errno) << std::endl; + return -1; + } + } + return 0; +} + +std::string GetDir(const std::string& path) { + size_t pos = path.find_last_of(kPathSeperator); + if (pos != std::string::npos) { + return path.substr(0, pos); + } + return kEmpty; +} + +std::string GetExt(const std::string& path) { + std::size_t pos = path.find_last_of("."); + if (pos != std::string::npos) { + return path.substr(pos); + } + return kEmpty; +} + +} // namespace util +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.h b/test/cpp-tableau-loader/src/protoconf/util.pc.h new file mode 100644 index 00000000..7c2fd947 --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.h @@ -0,0 +1,64 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#pragma once + +#include +#include +#include + +namespace tableau { +namespace util { +// Combine hash values +// +// References: +// - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x +// - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key +inline void HashCombine(std::size_t& seed) {} + +template +inline void HashCombine(std::size_t& seed, const T& v, O... others) { + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + HashCombine(seed, others...); +} + +template +inline std::size_t SugaredHashCombine(const T& v, O... others) { + std::size_t seed = 0; // start with a hash value 0 + HashCombine(seed, v, others...); + return seed; +} + +// Mkdir makes dir recursively. +int Mkdir(const std::string& path); +// GetDir returns all but the last element of path, typically the path's +// directory. +std::string GetDir(const std::string& path); +// GetExt returns the file name extension used by path. +// The extension is the suffix beginning at the final dot +// in the final element of path; it is empty if there is +// no dot. +std::string GetExt(const std::string& path); + +class TimeProfiler { + protected: + std::chrono::time_point last_; + + public: + TimeProfiler() { Start(); } + void Start() { last_ = std::chrono::steady_clock::now(); } + // Calculate duration between the last time point and now, + // and update last time point to now. + std::chrono::microseconds Elapse() { + auto now = std::chrono::steady_clock::now(); + auto duration = now - last_; // This is of type std::chrono::duration + last_ = now; + return std::chrono::duration_cast(duration); + } +}; + +} // namespace util +} // namespace tableau From c1d3f8e6817a37e9d9da4b30e0a5b41dfcba250c Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 24 Apr 2025 19:24:45 +0800 Subject: [PATCH 06/14] feat: notes --- cmd/protoc-gen-cpp-tableau-loader/embed.go | 20 +++++++++---------- .../embed/hub.pc.cc | 2 +- .../embed/hub.pc.h | 4 ++-- .../src/protoconf/hub.pc.cc | 2 +- .../cpp-tableau-loader/src/protoconf/hub.pc.h | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed.go b/cmd/protoc-gen-cpp-tableau-loader/embed.go index d6372a2a..bcde35ca 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed.go +++ b/cmd/protoc-gen-cpp-tableau-loader/embed.go @@ -14,11 +14,11 @@ import ( var efs embed.FS const ( - includeNotes = "// Auto-generated includes below\n" - declarationNotes = "// Auto-generated declarations below\n" - specializationNotes = "// Auto-generated specializations below\n" - initializationNotes = "// Auto-generated initializations below\n" - fieldNotes = "// Auto-generated fields below\n" + includeNotes = "// Auto-generated includes below\n" + declarationNotes = "// Auto-generated declarations below\n" + templateSpecializationNotes = "// Auto-generated template specializations below\n" + initializationNotes = "// Auto-generated initializations below\n" + fieldNotes = "// Auto-generated all messagers as fields for fast access below\n" ) // generateEmbed generates related registry files. @@ -50,7 +50,7 @@ func generateEmbed(gen *protogen.Plugin) { impl += `#include "` + proto + "." + pcExt + `.h"` + "\n" } file = strings.ReplaceAll(file, includeNotes, includeNotes+impl) - // Auto-generated specializations below + // Auto-generated template specializations below impl = "" for _, proto := range protofiles { for _, messager := range fileMessagers[proto] { @@ -61,7 +61,7 @@ func generateEmbed(gen *protogen.Plugin) { impl += "\n" } } - file = strings.ReplaceAll(file, specializationNotes, specializationNotes+impl) + file = strings.ReplaceAll(file, templateSpecializationNotes, templateSpecializationNotes+impl) // Auto-generated initializations below impl = "" for _, proto := range protofiles { @@ -79,7 +79,7 @@ func generateEmbed(gen *protogen.Plugin) { } } file = strings.ReplaceAll(file, declarationNotes, declarationNotes+impl) - // Auto-generated specializations below + // Auto-generated template specializations below impl = "" for _, proto := range protofiles { for _, messager := range fileMessagers[proto] { @@ -88,8 +88,8 @@ func generateEmbed(gen *protogen.Plugin) { impl += "\n" } } - file = strings.ReplaceAll(file, specializationNotes, specializationNotes+impl) - // Auto-generated fields below + file = strings.ReplaceAll(file, templateSpecializationNotes, templateSpecializationNotes+impl) + // Auto-generated all messagers as fields for fast access below impl = "" for _, proto := range protofiles { for _, messager := range fileMessagers[proto] { diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc index 149d8580..f03b6b84 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc @@ -535,7 +535,7 @@ bool Hub::Postprocess(std::shared_ptr msger_map) { std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; } -// Auto-generated specializations below +// Auto-generated template specializations below MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), last_loaded_time_(std::time(nullptr)) { diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h index 48455d72..cfb33f83 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h @@ -167,7 +167,7 @@ const U* Hub::GetOrderedMap(Args... args) const { return msger ? msger->GetOrderedMap(args...) : nullptr; } -// Auto-generated specializations below +// Auto-generated template specializations below class MessagerContainer { public: MessagerContainer(std::shared_ptr msger_map = nullptr); @@ -175,7 +175,7 @@ class MessagerContainer { public: std::shared_ptr msger_map_; std::time_t last_loaded_time_; - // Auto-generated fields below + // Auto-generated all messagers as fields for fast access below }; } // namespace tableau \ No newline at end of file diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index 3c3b8d28..784134ae 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -544,7 +544,7 @@ bool Hub::Postprocess(std::shared_ptr msger_map) { std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; } -// Auto-generated specializations below +// Auto-generated template specializations below template <> const std::shared_ptr Hub::Get() const { return GetMessagerContainer()->hero_base_conf_; diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index c73dedf9..dd265bc9 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -181,7 +181,7 @@ const U* Hub::GetOrderedMap(Args... args) const { return msger ? msger->GetOrderedMap(args...) : nullptr; } -// Auto-generated specializations below +// Auto-generated template specializations below template <> const std::shared_ptr Hub::Get() const; @@ -216,7 +216,7 @@ class MessagerContainer { public: std::shared_ptr msger_map_; std::time_t last_loaded_time_; - // Auto-generated fields below + // Auto-generated all messagers as fields for fast access below std::shared_ptr hero_base_conf_; std::shared_ptr hero_conf_; std::shared_ptr item_conf_; From e73094ec349b7caf332355bd59fe795dd290245d Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 24 Apr 2025 19:36:03 +0800 Subject: [PATCH 07/14] feat: scheduler --- .../embed/hub.pc.cc | 52 +--------------- .../embed/hub.pc.h | 28 +-------- .../embed/scheduler.pc.cc | 54 +++++++++++++++++ .../embed/scheduler.pc.h | 31 ++++++++++ .../src/protoconf/hub.pc.cc | 52 +--------------- .../cpp-tableau-loader/src/protoconf/hub.pc.h | 28 +-------- .../src/protoconf/scheduler.pc.cc | 59 +++++++++++++++++++ .../src/protoconf/scheduler.pc.h | 36 +++++++++++ 8 files changed, 184 insertions(+), 156 deletions(-) create mode 100644 cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.cc create mode 100644 cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.h create mode 100644 test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc create mode 100644 test/cpp-tableau-loader/src/protoconf/scheduler.pc.h diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc index f03b6b84..7f10d4f6 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc @@ -459,6 +459,7 @@ bool Hub::AsyncLoad(const std::string& dir, Format fmt /* = Format::kJSON */, } int Hub::LoopOnce() { return sched_->LoopOnce(); } + void Hub::InitScheduler() { sched_ = new internal::Scheduler(); sched_->Current(); @@ -541,55 +542,4 @@ MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = last_loaded_time_(std::time(nullptr)) { // Auto-generated initializations below } - -namespace internal { -// Thread-local storage (TLS) -thread_local Scheduler* tls_sched = nullptr; -Scheduler& Scheduler::Current() { - if (tls_sched == nullptr) { - tls_sched = new Scheduler; - } - return *tls_sched; -} - -int Scheduler::LoopOnce() { - AssertInLoopThread(); - - int count = 0; - std::vector jobs; - { - // scoped for auto-release lock. - // wake up immediately when there are pending tasks. - std::unique_lock lock(mutex_); - jobs.swap(jobs_); - } - for (auto&& job : jobs) { - job(); - } - count += jobs.size(); - return count; -} - -void Scheduler::Post(const Job& job) { - std::unique_lock lock(mutex_); - jobs_.push_back(job); -} - -void Scheduler::Dispatch(const Job& job) { - if (IsLoopThread()) { - job(); // run it immediately - } else { - Post(job); // post and run it at next loop - } -} - -bool Scheduler::IsLoopThread() const { return thread_id_ == std::this_thread::get_id(); } -void Scheduler::AssertInLoopThread() const { - if (!IsLoopThread()) { - abort(); - } -} - -} // namespace internal - } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h index cfb33f83..4b33e3ac 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h @@ -4,18 +4,15 @@ #include #include -#include #include #include #include -#include #include #include -#include #include -#include #include "messager.pc.h" +#include "scheduler.pc.h" namespace tableau { extern const std::string kUnknownExt; @@ -65,29 +62,6 @@ bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); -namespace internal { -class Scheduler { - public: - typedef std::function Job; - - public: - Scheduler() : thread_id_(std::this_thread::get_id()) {} - static Scheduler& Current(); - // thread-safety - void Post(const Job& job); - void Dispatch(const Job& job); - int LoopOnce(); - bool IsLoopThread() const; - void AssertInLoopThread() const; - - private: - std::thread::id thread_id_; - std::mutex mutex_; - std::vector jobs_; -}; - -} // namespace internal - class Hub { public: Hub(const HubOptions* options = nullptr) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.cc new file mode 100644 index 00000000..5a9aca0b --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.cc @@ -0,0 +1,54 @@ +#include "scheduler.pc.h" + +namespace tableau { +namespace internal { +// Thread-local storage (TLS) +thread_local Scheduler* tls_sched = nullptr; + +Scheduler& Scheduler::Current() { + if (tls_sched == nullptr) { + tls_sched = new Scheduler; + } + return *tls_sched; +} + +int Scheduler::LoopOnce() { + AssertInLoopThread(); + + int count = 0; + std::vector jobs; + { + // scoped for auto-release lock. + // wake up immediately when there are pending tasks. + std::unique_lock lock(mutex_); + jobs.swap(jobs_); + } + for (auto&& job : jobs) { + job(); + } + count += jobs.size(); + return count; +} + +void Scheduler::Post(const Job& job) { + std::unique_lock lock(mutex_); + jobs_.push_back(job); +} + +void Scheduler::Dispatch(const Job& job) { + if (IsLoopThread()) { + job(); // run it immediately + } else { + Post(job); // post and run it at next loop + } +} + +bool Scheduler::IsLoopThread() const { return thread_id_ == std::this_thread::get_id(); } + +void Scheduler::AssertInLoopThread() const { + if (!IsLoopThread()) { + abort(); + } +} +} // namespace internal +} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.h new file mode 100644 index 00000000..5d5380e8 --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include + +namespace tableau { +namespace internal { +class Scheduler { + public: + typedef std::function Job; + + public: + Scheduler() : thread_id_(std::this_thread::get_id()) {} + static Scheduler& Current(); + // thread-safety + void Post(const Job& job); + void Dispatch(const Job& job); + int LoopOnce(); + bool IsLoopThread() const; + void AssertInLoopThread() const; + + private: + std::thread::id thread_id_; + std::mutex mutex_; + std::vector jobs_; +}; + +} // namespace internal +} // namespace tableau \ No newline at end of file diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index 784134ae..ba69738a 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -468,6 +468,7 @@ bool Hub::AsyncLoad(const std::string& dir, Format fmt /* = Format::kJSON */, } int Hub::LoopOnce() { return sched_->LoopOnce(); } + void Hub::InitScheduler() { sched_ = new internal::Scheduler(); sched_->Current(); @@ -604,55 +605,4 @@ MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = chapter_conf_ = std::dynamic_pointer_cast((*msger_map_)["ChapterConf"]); theme_conf_ = std::dynamic_pointer_cast((*msger_map_)["ThemeConf"]); } - -namespace internal { -// Thread-local storage (TLS) -thread_local Scheduler* tls_sched = nullptr; -Scheduler& Scheduler::Current() { - if (tls_sched == nullptr) { - tls_sched = new Scheduler; - } - return *tls_sched; -} - -int Scheduler::LoopOnce() { - AssertInLoopThread(); - - int count = 0; - std::vector jobs; - { - // scoped for auto-release lock. - // wake up immediately when there are pending tasks. - std::unique_lock lock(mutex_); - jobs.swap(jobs_); - } - for (auto&& job : jobs) { - job(); - } - count += jobs.size(); - return count; -} - -void Scheduler::Post(const Job& job) { - std::unique_lock lock(mutex_); - jobs_.push_back(job); -} - -void Scheduler::Dispatch(const Job& job) { - if (IsLoopThread()) { - job(); // run it immediately - } else { - Post(job); // post and run it at next loop - } -} - -bool Scheduler::IsLoopThread() const { return thread_id_ == std::this_thread::get_id(); } -void Scheduler::AssertInLoopThread() const { - if (!IsLoopThread()) { - abort(); - } -} - -} // namespace internal - } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index dd265bc9..4e630ba6 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -9,18 +9,15 @@ #include #include -#include #include #include #include -#include #include #include -#include #include -#include #include "messager.pc.h" +#include "scheduler.pc.h" namespace tableau { extern const std::string kUnknownExt; @@ -79,29 +76,6 @@ bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); -namespace internal { -class Scheduler { - public: - typedef std::function Job; - - public: - Scheduler() : thread_id_(std::this_thread::get_id()) {} - static Scheduler& Current(); - // thread-safety - void Post(const Job& job); - void Dispatch(const Job& job); - int LoopOnce(); - bool IsLoopThread() const; - void AssertInLoopThread() const; - - private: - std::thread::id thread_id_; - std::mutex mutex_; - std::vector jobs_; -}; - -} // namespace internal - class Hub { public: Hub(const HubOptions* options = nullptr) diff --git a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc new file mode 100644 index 00000000..b375f26b --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc @@ -0,0 +1,59 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#include "scheduler.pc.h" + +namespace tableau { +namespace internal { +// Thread-local storage (TLS) +thread_local Scheduler* tls_sched = nullptr; + +Scheduler& Scheduler::Current() { + if (tls_sched == nullptr) { + tls_sched = new Scheduler; + } + return *tls_sched; +} + +int Scheduler::LoopOnce() { + AssertInLoopThread(); + + int count = 0; + std::vector jobs; + { + // scoped for auto-release lock. + // wake up immediately when there are pending tasks. + std::unique_lock lock(mutex_); + jobs.swap(jobs_); + } + for (auto&& job : jobs) { + job(); + } + count += jobs.size(); + return count; +} + +void Scheduler::Post(const Job& job) { + std::unique_lock lock(mutex_); + jobs_.push_back(job); +} + +void Scheduler::Dispatch(const Job& job) { + if (IsLoopThread()) { + job(); // run it immediately + } else { + Post(job); // post and run it at next loop + } +} + +bool Scheduler::IsLoopThread() const { return thread_id_ == std::this_thread::get_id(); } + +void Scheduler::AssertInLoopThread() const { + if (!IsLoopThread()) { + abort(); + } +} +} // namespace internal +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h new file mode 100644 index 00000000..636008d7 --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h @@ -0,0 +1,36 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#pragma once + +#include +#include +#include +#include + +namespace tableau { +namespace internal { +class Scheduler { + public: + typedef std::function Job; + + public: + Scheduler() : thread_id_(std::this_thread::get_id()) {} + static Scheduler& Current(); + // thread-safety + void Post(const Job& job); + void Dispatch(const Job& job); + int LoopOnce(); + bool IsLoopThread() const; + void AssertInLoopThread() const; + + private: + std::thread::id thread_id_; + std::mutex mutex_; + std::vector jobs_; +}; + +} // namespace internal +} // namespace tableau From d0de881dce2dc570804eb980cac1d48832dbfc43 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 24 Apr 2025 20:47:21 +0800 Subject: [PATCH 08/14] feat: add load.pc.cc --- .../embed/hub.pc.cc | 421 +----------------- .../embed/hub.pc.h | 31 +- .../embed/load.pc.cc | 295 ++++++++++++ .../embed/load.pc.h | 43 ++ .../embed/messager.pc.h | 42 +- .../embed/util.pc.cc | 129 ++++++ .../embed/util.pc.h | 46 +- .../src/protoconf/hub.pc.cc | 421 +----------------- .../cpp-tableau-loader/src/protoconf/hub.pc.h | 31 +- .../src/protoconf/load.pc.cc | 300 +++++++++++++ .../src/protoconf/load.pc.h | 48 ++ .../src/protoconf/messager.pc.h | 42 +- .../src/protoconf/util.pc.cc | 129 ++++++ .../src/protoconf/util.pc.h | 46 +- 14 files changed, 1046 insertions(+), 978 deletions(-) create mode 100644 cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc create mode 100644 cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h create mode 100644 test/cpp-tableau-loader/src/protoconf/load.pc.cc create mode 100644 test/cpp-tableau-loader/src/protoconf/load.pc.h diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc index 7f10d4f6..2f0506f9 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc @@ -4,10 +4,7 @@ #include #include -#include -#include #include -#include #include "logger.pc.h" #include "messager.pc.h" @@ -17,420 +14,6 @@ // Auto-generated includes below namespace tableau { -#ifdef _WIN32 -#undef GetMessage -#endif - -static thread_local std::string g_err_msg; -const std::string& GetErrMsg() { return g_err_msg; } - -const std::string kUnknownExt = ".unknown"; -const std::string kJSONExt = ".json"; -const std::string kTextExt = ".txt"; -const std::string kBinExt = ".bin"; - -Format Ext2Format(const std::string& ext) { - if (ext == kJSONExt) { - return Format::kJSON; - } else if (ext == kTextExt) { - return Format::kText; - } else if (ext == kBinExt) { - return Format::kBin; - } else { - return Format::kUnknown; - } -} - -const std::string& Format2Ext(Format fmt) { - switch (fmt) { - case Format::kJSON: - return kJSONExt; - case Format::kText: - return kTextExt; - case Format::kBin: - return kBinExt; - default: - return kUnknownExt; - } -} - -// refer: https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/stubs/logging.h -void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg) { - static const std::unordered_map kLevelMap = {{google::protobuf::LOGLEVEL_INFO, log::kInfo}, - {google::protobuf::LOGLEVEL_WARNING, log::kWarn}, - {google::protobuf::LOGLEVEL_ERROR, log::kError}, - {google::protobuf::LOGLEVEL_FATAL, log::kFatal}}; - log::Level lvl = log::kWarn; // default - auto iter = kLevelMap.find(level); - if (iter != kLevelMap.end()) { - lvl = iter->second; - } - ATOM_LOGGER_CALL(tableau::log::DefaultLogger(), lvl, "[libprotobuf %s:%d] %s", filename, line, msg.c_str()); -} - -const std::string& GetProtoName(const google::protobuf::Message& msg) { - const auto* md = msg.GetDescriptor(); - return md != nullptr ? md->name() : kEmpty; -} - -bool ExistsFile(const std::string& filename) { - std::ifstream file(filename); - // returns true if the file exists and is accessible - return file.good(); -} - -bool ReadFile(const std::string& filename, std::string& content) { - std::ifstream file(filename); - if (!file.is_open()) { - g_err_msg = "failed to open " + filename + ": " + strerror(errno); - return false; - } - std::stringstream ss; - ss << file.rdbuf(); - content = ss.str(); - return true; -} - -std::string GetPatchName(tableau::Patch patch) { - auto* descriptor = tableau::Patch_descriptor(); - if (descriptor) { - auto* value = descriptor->FindValueByNumber(patch); - if (value) { - return value->name(); - } - } - return std::to_string(static_cast(patch)); -} - -bool Message2JSON(const google::protobuf::Message& msg, std::string& json) { - google::protobuf::util::JsonPrintOptions options; - options.add_whitespace = true; - options.always_print_primitive_fields = true; - options.preserve_proto_field_names = true; - return google::protobuf::util::MessageToJsonString(msg, &json, options).ok(); -} - -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options /* = nullptr */) { - google::protobuf::util::Status status; - if (options != nullptr) { - google::protobuf::util::JsonParseOptions parse_options; - parse_options.ignore_unknown_fields = options->ignore_unknown_fields; - status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); - } else { - status = google::protobuf::util::JsonStringToMessage(json, &msg); - } - if (!status.ok()) { - g_err_msg = "failed to parse " + GetProtoName(msg) + kJSONExt + ": " + status.ToString(); - return false; - } - return true; -} - -bool Text2Message(const std::string& text, google::protobuf::Message& msg) { - if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) { - g_err_msg = "failed to parse " + GetProtoName(msg) + kTextExt; - return false; - } - return true; -} -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) { - if (!msg.ParseFromString(bin)) { - g_err_msg = "failed to parse " + GetProtoName(msg) + kBinExt; - return false; - } - return true; -} - -// PatchMessage patches src into dst, which must be a message with the same descriptor. -// -// # Default PatchMessage mechanism -// - scalar: Populated scalar fields in src are copied to dst. -// - message: Populated singular messages in src are merged into dst by -// recursively calling [xproto.PatchMessage], or replace dst message if -// "PATCH_REPLACE" is specified for this field. -// - list: The elements of every list field in src are appended to the -// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" -// is specified for this field. -// - map: The entries of every map field in src are MERGED (different from -// the behavior of proto.Merge) into the corresponding map field in dst, -// or replace dst map if "PATCH_REPLACE" is specified for this field. -// - unknown: The unknown fields of src are appended to the unknown -// fields of dst (TODO: untested). -// -// # References: -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details -bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { - const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); - const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); - // Ensure both messages are of the same type - if (dst_descriptor != src_descriptor) { - g_err_msg = "dst and src are not messages with the same descriptor"; - ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), - src_descriptor->name().c_str()); - return false; - } - - // Get the reflection and descriptor for the messages - const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); - const google::protobuf::Reflection* src_reflection = src.GetReflection(); - - // List all populated fields - std::vector fields; - src_reflection->ListFields(src, &fields); - - // Iterates over every populated field. - for (auto fd : fields) { - const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); - tableau::Patch patch = opts.prop().patch(); - if (patch == tableau::PATCH_REPLACE) { - dst_reflection->ClearField(&dst, fd); - } - if (fd->is_map()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 - auto key_fd = fd->message_type()->map_key(); - auto value_fd = fd->message_type()->map_value(); - int src_count = src_reflection->FieldSize(src, fd); - int dst_count = dst_reflection->FieldSize(dst, fd); - switch (key_fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - std::unordered_map dst_key_index_map; \ - for (int i = 0; i < dst_count; i++) { \ - auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ - TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ - dst_key_index_map[key] = i; \ - } \ - for (int j = 0; j < src_count; j++) { \ - auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ - TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ - auto it = dst_key_index_map.find(key); \ - if (it != dst_key_index_map.end()) { \ - int index = it->second; \ - auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ - PatchMessage(dst_entry, src_entry); \ - } else { \ - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ - } \ - } \ - break; \ - } - - HANDLE_TYPE(INT32, Int32, int32_t); - HANDLE_TYPE(INT64, Int64, int64_t); - HANDLE_TYPE(UINT32, UInt32, uint32_t); - HANDLE_TYPE(UINT64, UInt64, uint64_t); - HANDLE_TYPE(BOOL, Bool, bool); - HANDLE_TYPE(STRING, String, std::string); - default: { - // other types are impossible to be protobuf map key - ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); - break; - } -#undef HANDLE_TYPE - } - } else if (fd->is_repeated()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 - int count = src_reflection->FieldSize(src, fd); - for (int j = 0; j < count; j++) { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ - break; \ - } - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { - const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); - break; - } - } - } - } else { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ - dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ - break; - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: - const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); - PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); - break; - } - } - } - - dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); - return true; -} - -bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& path, Format fmt, tableau::Patch patch, - const LoadOptions* options /* = nullptr*/) { - if (options == nullptr) { - return LoadMessageByPath(msg, path, fmt, nullptr); - } - if (options->mode == LoadMode::kModeOnlyMain) { - // ignore patch files when LoadMode::kModeOnlyMain specified - return LoadMessageByPath(msg, path, fmt, nullptr); - } - std::string name = GetProtoName(msg); - std::vector patch_paths; - auto iter = options->patch_paths.find(name); - if (iter != options->patch_paths.end()) { - // patch path specified in PatchPaths, then use it instead of PatchDirs. - patch_paths = iter->second; - } else { - for (auto&& patch_dir : options->patch_dirs) { - patch_paths.emplace_back(patch_dir + name + Format2Ext(fmt)); - } - } - - std::vector existed_patch_paths; - for (auto&& patch_path : patch_paths) { - if (ExistsFile(patch_path)) { - existed_patch_paths.emplace_back(patch_path); - } - } - if (existed_patch_paths.empty()) { - if (options->mode == LoadMode::kModeOnlyPatch) { - // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. - return true; - } - // no valid patch path provided, then just load from the "main" file. - return LoadMessageByPath(msg, path, fmt, options); - } - - switch (patch) { - case tableau::PATCH_REPLACE: { - // just use the last "patch" file - std::string& patch_path = existed_patch_paths.back(); - if (!LoadMessageByPath(msg, patch_path, Ext2Format(util::GetExt(patch_path)), options)) { - return false; - } - break; - } - case tableau::PATCH_MERGE: { - if (options->mode != LoadMode::kModeOnlyPatch) { - // load msg from the "main" file - if (!LoadMessageByPath(msg, path, fmt, options)) { - return false; - } - } - // Create a new instance of the same type of the original message - google::protobuf::Message* patch_msg_ptr = msg.New(); - std::unique_ptr _auto_release(msg.New()); - // load patch_msg from each "patch" file - for (auto&& patch_path : existed_patch_paths) { - if (!LoadMessageByPath(*patch_msg_ptr, patch_path, Ext2Format(util::GetExt(patch_path)), options)) { - return false; - } - if (!PatchMessage(msg, *patch_msg_ptr)) { - return false; - } - } - break; - } - default: { - g_err_msg = "unknown patch type: " + GetPatchName(patch); - return false; - } - } - ATOM_DEBUG("patched(%s) %s by %s: %s", GetPatchName(patch).c_str(), name.c_str(), - ATOM_VECTOR_STR(existed_patch_paths).c_str(), msg.ShortDebugString().c_str()); - return true; -} - -bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt, - const LoadOptions* options /* = nullptr*/) { - std::string content; - ReadFunc read_func = ReadFile; - if (options != nullptr && options->read_func) { - read_func = options->read_func; - } - bool ok = read_func(path, content); - if (!ok) { - return false; - } - switch (fmt) { - case Format::kJSON: { - return JSON2Message(content, msg, options); - } - case Format::kText: { - return Text2Message(content, msg); - } - case Format::kBin: { - return Bin2Message(content, msg); - } - default: { - g_err_msg = "unknown format: " + std::to_string(static_cast(fmt)); - return false; - } - } -} - -bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt, - const LoadOptions* options /* = nullptr*/) { - std::string name = GetProtoName(msg); - std::string path; - if (options) { - auto iter = options->paths.find(name); - if (iter != options->paths.end()) { - // path specified in Paths, then use it instead of dir. - path = iter->second; - fmt = Ext2Format(util::GetExt(iter->second)); - } - } - if (path.empty()) { - path = dir + name + Format2Ext(fmt); - } - - const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); - if (!descriptor) { - g_err_msg = "failed to get descriptor of message: " + name; - return false; - } - // access the extension directly using the generated identifier - const tableau::WorksheetOptions worksheet_options = descriptor->options().GetExtension(tableau::worksheet); - if (worksheet_options.patch() != tableau::PATCH_NONE) { - return LoadMessageWithPatch(msg, path, fmt, worksheet_options.patch(), options); - } - - return LoadMessageByPath(msg, path, fmt, options); -} - -bool StoreMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt) { - // TODO: write protobuf message to file, support 3 formats: json, text, and bin. - return false; -} - bool Hub::Load(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { auto msger_map = InternalLoad(dir, fmt, options); if (!msger_map) { @@ -468,7 +51,7 @@ void Hub::InitScheduler() { std::shared_ptr Hub::InternalLoad(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) const { // intercept protobuf error logs - auto old_handler = google::protobuf::SetLogHandler(ProtobufLogHandler); + auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); auto msger_map = NewMessagerMap(); for (auto iter : *msger_map) { auto&& name = iter.first; @@ -527,7 +110,7 @@ bool Hub::Postprocess(std::shared_ptr msger_map) { auto msger = iter.second; bool ok = msger->ProcessAfterLoadAll(tmp_hub); if (!ok) { - g_err_msg = "hub call ProcessAfterLoadAll failed, messager: " + msger->Name(); + SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + msger->Name()); return false; } } diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h index 4b33e3ac..207c79a0 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h @@ -3,25 +3,17 @@ #include #include -#include -#include #include #include #include #include #include +#include "load.pc.h" #include "messager.pc.h" #include "scheduler.pc.h" namespace tableau { -extern const std::string kUnknownExt; -extern const std::string kJSONExt; -extern const std::string kTextExt; -extern const std::string kBinExt; - -const std::string& GetErrMsg(); - class MessagerContainer; class Hub; @@ -42,26 +34,6 @@ struct HubOptions { MessagerContainerProvider provider; }; -// Convert file extension to Format type. -// NOTE: ext includes dot ".", such as: -// - kJSONExt：".json" -// - kTextExt".txt" -// - kBinExt".bin" -Format Ext2Format(const std::string& ext); -// Empty string will be returned if an unsupported enum value has been passed, -// and the error message can be obtained by GetErrMsg(). -const std::string& Format2Ext(Format fmt); -bool Message2JSON(const google::protobuf::Message& msg, std::string& json); -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options = nullptr); -bool Text2Message(const std::string& text, google::protobuf::Message& msg); -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); -void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg); -const std::string& GetProtoName(const google::protobuf::Message& msg); -bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr); -bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr); - class Hub { public: Hub(const HubOptions* options = nullptr) @@ -151,5 +123,4 @@ class MessagerContainer { std::time_t last_loaded_time_; // Auto-generated all messagers as fields for fast access below }; - } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc new file mode 100644 index 00000000..5c60d98b --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc @@ -0,0 +1,295 @@ +#include "load.pc.h" + +#include "logger.pc.h" +#include "util.pc.h" + +namespace tableau { +#ifdef _WIN32 +#undef GetMessage +#endif + +// PatchMessage patches src into dst, which must be a message with the same descriptor. +// +// # Default PatchMessage mechanism +// - scalar: Populated scalar fields in src are copied to dst. +// - message: Populated singular messages in src are merged into dst by +// recursively calling [xproto.PatchMessage], or replace dst message if +// "PATCH_REPLACE" is specified for this field. +// - list: The elements of every list field in src are appended to the +// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" +// is specified for this field. +// - map: The entries of every map field in src are MERGED (different from +// the behavior of proto.Merge) into the corresponding map field in dst, +// or replace dst map if "PATCH_REPLACE" is specified for this field. +// - unknown: The unknown fields of src are appended to the unknown +// fields of dst (TODO: untested). +// +// # References: +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { + const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); + const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); + // Ensure both messages are of the same type + if (dst_descriptor != src_descriptor) { + SetErrMsg("dst and src are not messages with the same descriptor"); + ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), + src_descriptor->name().c_str()); + return false; + } + + // Get the reflection and descriptor for the messages + const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); + const google::protobuf::Reflection* src_reflection = src.GetReflection(); + + // List all populated fields + std::vector fields; + src_reflection->ListFields(src, &fields); + + // Iterates over every populated field. + for (auto fd : fields) { + const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); + tableau::Patch patch = opts.prop().patch(); + if (patch == tableau::PATCH_REPLACE) { + dst_reflection->ClearField(&dst, fd); + } + if (fd->is_map()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 + auto key_fd = fd->message_type()->map_key(); + auto value_fd = fd->message_type()->map_value(); + int src_count = src_reflection->FieldSize(src, fd); + int dst_count = dst_reflection->FieldSize(dst, fd); + switch (key_fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + std::unordered_map dst_key_index_map; \ + for (int i = 0; i < dst_count; i++) { \ + auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ + TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ + dst_key_index_map[key] = i; \ + } \ + for (int j = 0; j < src_count; j++) { \ + auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ + TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ + auto it = dst_key_index_map.find(key); \ + if (it != dst_key_index_map.end()) { \ + int index = it->second; \ + auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ + PatchMessage(dst_entry, src_entry); \ + } else { \ + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ + } \ + } \ + break; \ + } + + HANDLE_TYPE(INT32, Int32, int32_t); + HANDLE_TYPE(INT64, Int64, int64_t); + HANDLE_TYPE(UINT32, UInt32, uint32_t); + HANDLE_TYPE(UINT64, UInt64, uint64_t); + HANDLE_TYPE(BOOL, Bool, bool); + HANDLE_TYPE(STRING, String, std::string); + default: { + // other types are impossible to be protobuf map key + ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); + break; + } +#undef HANDLE_TYPE + } + } else if (fd->is_repeated()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 + int count = src_reflection->FieldSize(src, fd); + for (int j = 0; j < count; j++) { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ + break; \ + } + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); + break; + } + } + } + } else { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ + dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ + break; + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); + PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); + break; + } + } + } + + dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); + return true; +} + +bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& path, Format fmt, tableau::Patch patch, + const LoadOptions* options /* = nullptr*/) { + if (options == nullptr) { + return LoadMessageByPath(msg, path, fmt, nullptr); + } + if (options->mode == LoadMode::kModeOnlyMain) { + // ignore patch files when LoadMode::kModeOnlyMain specified + return LoadMessageByPath(msg, path, fmt, nullptr); + } + std::string name = util::GetProtoName(msg); + std::vector patch_paths; + auto iter = options->patch_paths.find(name); + if (iter != options->patch_paths.end()) { + // patch path specified in PatchPaths, then use it instead of PatchDirs. + patch_paths = iter->second; + } else { + for (auto&& patch_dir : options->patch_dirs) { + patch_paths.emplace_back(patch_dir + name + util::Format2Ext(fmt)); + } + } + + std::vector existed_patch_paths; + for (auto&& patch_path : patch_paths) { + if (util::ExistsFile(patch_path)) { + existed_patch_paths.emplace_back(patch_path); + } + } + if (existed_patch_paths.empty()) { + if (options->mode == LoadMode::kModeOnlyPatch) { + // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. + return true; + } + // no valid patch path provided, then just load from the "main" file. + return LoadMessageByPath(msg, path, fmt, options); + } + + switch (patch) { + case tableau::PATCH_REPLACE: { + // just use the last "patch" file + std::string& patch_path = existed_patch_paths.back(); + if (!LoadMessageByPath(msg, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { + return false; + } + break; + } + case tableau::PATCH_MERGE: { + if (options->mode != LoadMode::kModeOnlyPatch) { + // load msg from the "main" file + if (!LoadMessageByPath(msg, path, fmt, options)) { + return false; + } + } + // Create a new instance of the same type of the original message + google::protobuf::Message* patch_msg_ptr = msg.New(); + std::unique_ptr _auto_release(msg.New()); + // load patch_msg from each "patch" file + for (auto&& patch_path : existed_patch_paths) { + if (!LoadMessageByPath(*patch_msg_ptr, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { + return false; + } + if (!PatchMessage(msg, *patch_msg_ptr)) { + return false; + } + } + break; + } + default: { + SetErrMsg("unknown patch type: " + util::GetPatchName(patch)); + return false; + } + } + ATOM_DEBUG("patched(%s) %s by %s: %s", util::GetPatchName(patch).c_str(), name.c_str(), + ATOM_VECTOR_STR(existed_patch_paths).c_str(), msg.ShortDebugString().c_str()); + return true; +} + +bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt, + const LoadOptions* options /* = nullptr*/) { + std::string content; + ReadFunc read_func = util::ReadFile; + if (options != nullptr && options->read_func) { + read_func = options->read_func; + } + bool ok = read_func(path, content); + if (!ok) { + return false; + } + switch (fmt) { + case Format::kJSON: { + return util::JSON2Message(content, msg, options); + } + case Format::kText: { + return util::Text2Message(content, msg); + } + case Format::kBin: { + return util::Bin2Message(content, msg); + } + default: { + SetErrMsg("unknown format: " + std::to_string(static_cast(fmt))); + return false; + } + } +} + +bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt, + const LoadOptions* options /* = nullptr*/) { + std::string name = util::GetProtoName(msg); + std::string path; + if (options) { + auto iter = options->paths.find(name); + if (iter != options->paths.end()) { + // path specified in Paths, then use it instead of dir. + path = iter->second; + fmt = util::Ext2Format(util::GetExt(iter->second)); + } + } + if (path.empty()) { + path = dir + name + util::Format2Ext(fmt); + } + + const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); + if (!descriptor) { + SetErrMsg("failed to get descriptor of message: " + name); + return false; + } + // access the extension directly using the generated identifier + const tableau::WorksheetOptions worksheet_options = descriptor->options().GetExtension(tableau::worksheet); + if (worksheet_options.patch() != tableau::PATCH_NONE) { + return LoadMessageWithPatch(msg, path, fmt, worksheet_options.patch(), options); + } + + return LoadMessageByPath(msg, path, fmt, options); +} +} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h new file mode 100644 index 00000000..8f725973 --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h @@ -0,0 +1,43 @@ +#pragma once +#include + +#include +#include + +#include "util.pc.h" + +namespace tableau { +enum class LoadMode { + kModeDefault, + kModeOnlyMain, + kModeOnlyPatch, +}; + +// ReadFunc reads the config file and returns its content. +using ReadFunc = std::function; + +struct LoadOptions { + // read_func reads the config file and returns its content. + ReadFunc read_func; + // Whether to ignore unknown JSON fields during parsing. + // + // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. + bool ignore_unknown_fields = false; + // Paths maps each messager name to a corresponding config file path. + // If specified, then the main messager will be parsed from the file + // directly, other than the specified load dir. + std::unordered_map paths; + // Patch paths maps each messager name to one or multiple corresponding patch file paths. + // If specified, then main messager will be patched. + std::unordered_map> patch_paths; + // Patch dirs specifies the directory paths for config patching. + std::vector patch_dirs; + // Mode specifies the loading mode for config patching. + LoadMode mode = LoadMode::kModeDefault; +}; + +bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr); +bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr); +} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h index 4e38ca70..a021f59e 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h @@ -4,46 +4,11 @@ #include #include -namespace tableau { -enum class Format { - kUnknown, - kJSON, - kText, - kBin, -}; - -enum class LoadMode { - kModeDefault, - kModeOnlyMain, - kModeOnlyPatch, -}; - -static const std::string kEmpty = ""; +#include "util.pc.h" +namespace tableau { class Hub; - -// ReadFunc reads the config file and returns its content. -using ReadFunc = std::function; - -struct LoadOptions { - // read_func reads the config file and returns its content. - ReadFunc read_func; - // Whether to ignore unknown JSON fields during parsing. - // - // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. - bool ignore_unknown_fields = false; - // Paths maps each messager name to a corresponding config file path. - // If specified, then the main messager will be parsed from the file - // directly, other than the specified load dir. - std::unordered_map paths; - // Patch paths maps each messager name to one or multiple corresponding patch file paths. - // If specified, then main messager will be patched. - std::unordered_map> patch_paths; - // Patch dirs specifies the directory paths for config patching. - std::vector patch_dirs; - // Mode specifies the loading mode for config patching. - LoadMode mode = LoadMode::kModeDefault; -}; +struct LoadOptions; class Messager { public: @@ -70,5 +35,4 @@ class Messager { virtual bool ProcessAfterLoad() { return true; }; Stats stats_; }; - } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc index 6b02848b..e2abd361 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc @@ -1,5 +1,12 @@ #include "util.pc.h" +#include +#include + +#include +#include +#include + #ifdef _WIN32 #include #include @@ -7,6 +14,8 @@ #include #endif +#include "load.pc.h" +#include "logger.pc.h" #include "messager.pc.h" namespace tableau { @@ -17,6 +26,15 @@ static constexpr char kPathSeperator = '\\'; static constexpr char kPathSeperator = '/'; #endif +static thread_local std::string g_err_msg; +const std::string& GetErrMsg() { return g_err_msg; } +void SetErrMsg(const std::string& msg) { g_err_msg = msg; } + +const std::string kUnknownExt = ".unknown"; +const std::string kJSONExt = ".json"; +const std::string kTextExt = ".txt"; +const std::string kBinExt = ".bin"; + namespace util { int Mkdir(const std::string& path) { std::string path_ = path + kPathSeperator; @@ -44,6 +62,24 @@ std::string GetDir(const std::string& path) { return kEmpty; } +bool ExistsFile(const std::string& filename) { + std::ifstream file(filename); + // returns true if the file exists and is accessible + return file.good(); +} + +bool ReadFile(const std::string& filename, std::string& content) { + std::ifstream file(filename); + if (!file.is_open()) { + SetErrMsg("failed to open " + filename + ": " + strerror(errno)); + return false; + } + std::stringstream ss; + ss << file.rdbuf(); + content = ss.str(); + return true; +} + std::string GetExt(const std::string& path) { std::size_t pos = path.find_last_of("."); if (pos != std::string::npos) { @@ -52,5 +88,98 @@ std::string GetExt(const std::string& path) { return kEmpty; } +Format Ext2Format(const std::string& ext) { + if (ext == kJSONExt) { + return Format::kJSON; + } else if (ext == kTextExt) { + return Format::kText; + } else if (ext == kBinExt) { + return Format::kBin; + } else { + return Format::kUnknown; + } +} + +const std::string& Format2Ext(Format fmt) { + switch (fmt) { + case Format::kJSON: + return kJSONExt; + case Format::kText: + return kTextExt; + case Format::kBin: + return kBinExt; + default: + return kUnknownExt; + } +} + +bool Message2JSON(const google::protobuf::Message& msg, std::string& json) { + google::protobuf::util::JsonPrintOptions options; + options.add_whitespace = true; + options.always_print_primitive_fields = true; + options.preserve_proto_field_names = true; + return google::protobuf::util::MessageToJsonString(msg, &json, options).ok(); +} + +bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options /* = nullptr */) { + google::protobuf::util::Status status; + if (options != nullptr) { + google::protobuf::util::JsonParseOptions parse_options; + parse_options.ignore_unknown_fields = options->ignore_unknown_fields; + status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); + } else { + status = google::protobuf::util::JsonStringToMessage(json, &msg); + } + if (!status.ok()) { + SetErrMsg("failed to parse " + GetProtoName(msg) + kJSONExt + ": " + status.ToString()); + return false; + } + return true; +} + +bool Text2Message(const std::string& text, google::protobuf::Message& msg) { + if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) { + SetErrMsg("failed to parse " + GetProtoName(msg) + kTextExt); + return false; + } + return true; +} +bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) { + if (!msg.ParseFromString(bin)) { + SetErrMsg("failed to parse " + GetProtoName(msg) + kBinExt); + return false; + } + return true; +} + +const std::string& GetProtoName(const google::protobuf::Message& msg) { + const auto* md = msg.GetDescriptor(); + return md != nullptr ? md->name() : kEmpty; +} + +std::string GetPatchName(tableau::Patch patch) { + auto* descriptor = tableau::Patch_descriptor(); + if (descriptor) { + auto* value = descriptor->FindValueByNumber(patch); + if (value) { + return value->name(); + } + } + return std::to_string(static_cast(patch)); +} + +// refer: https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/stubs/logging.h +void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg) { + static const std::unordered_map kLevelMap = {{google::protobuf::LOGLEVEL_INFO, log::kInfo}, + {google::protobuf::LOGLEVEL_WARNING, log::kWarn}, + {google::protobuf::LOGLEVEL_ERROR, log::kError}, + {google::protobuf::LOGLEVEL_FATAL, log::kFatal}}; + log::Level lvl = log::kWarn; // default + auto iter = kLevelMap.find(level); + if (iter != kLevelMap.end()) { + lvl = iter->second; + } + ATOM_LOGGER_CALL(tableau::log::DefaultLogger(), lvl, "[libprotobuf %s:%d] %s", filename, line, msg.c_str()); +} } // namespace util } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h index 5622380d..ab8d9e08 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h @@ -1,10 +1,31 @@ #pragma once +#include #include #include #include +#include "tableau/protobuf/tableau.pb.h" + namespace tableau { +const std::string& GetErrMsg(); +void SetErrMsg(const std::string& msg); + +enum class Format { + kUnknown, + kJSON, + kText, + kBin, +}; + +static const std::string kEmpty = ""; +extern const std::string kUnknownExt; +extern const std::string kJSONExt; +extern const std::string kTextExt; +extern const std::string kBinExt; + +struct LoadOptions; + namespace util { // Combine hash values // @@ -32,11 +53,35 @@ int Mkdir(const std::string& path); // GetDir returns all but the last element of path, typically the path's // directory. std::string GetDir(const std::string& path); +// ExistsFile checks if a file exists. +bool ExistsFile(const std::string& filename); +// ReadFile reads the file named by filename and returns the contents. +bool ReadFile(const std::string& filename, std::string& content); + // GetExt returns the file name extension used by path. // The extension is the suffix beginning at the final dot // in the final element of path; it is empty if there is // no dot. std::string GetExt(const std::string& path); +// Convert file extension to Format type. +// NOTE: ext includes dot ".", such as: +// - kJSONExt：".json" +// - kTextExt".txt" +// - kBinExt".bin" +Format Ext2Format(const std::string& ext); +// Empty string will be returned if an unsupported enum value has been passed, +// and the error message can be obtained by GetErrMsg(). +const std::string& Format2Ext(Format fmt); + +bool Message2JSON(const google::protobuf::Message& msg, std::string& json); +bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options = nullptr); +bool Text2Message(const std::string& text, google::protobuf::Message& msg); +bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); + +const std::string& GetProtoName(const google::protobuf::Message& msg); +std::string GetPatchName(tableau::Patch patch); + +void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg); class TimeProfiler { protected: @@ -54,6 +99,5 @@ class TimeProfiler { return std::chrono::duration_cast(duration); } }; - } // namespace util } // namespace tableau \ No newline at end of file diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index ba69738a..7a449098 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -9,10 +9,7 @@ #include #include -#include -#include #include -#include #include "logger.pc.h" #include "messager.pc.h" @@ -26,420 +23,6 @@ #include "test_conf.pc.h" namespace tableau { -#ifdef _WIN32 -#undef GetMessage -#endif - -static thread_local std::string g_err_msg; -const std::string& GetErrMsg() { return g_err_msg; } - -const std::string kUnknownExt = ".unknown"; -const std::string kJSONExt = ".json"; -const std::string kTextExt = ".txt"; -const std::string kBinExt = ".bin"; - -Format Ext2Format(const std::string& ext) { - if (ext == kJSONExt) { - return Format::kJSON; - } else if (ext == kTextExt) { - return Format::kText; - } else if (ext == kBinExt) { - return Format::kBin; - } else { - return Format::kUnknown; - } -} - -const std::string& Format2Ext(Format fmt) { - switch (fmt) { - case Format::kJSON: - return kJSONExt; - case Format::kText: - return kTextExt; - case Format::kBin: - return kBinExt; - default: - return kUnknownExt; - } -} - -// refer: https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/stubs/logging.h -void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg) { - static const std::unordered_map kLevelMap = {{google::protobuf::LOGLEVEL_INFO, log::kInfo}, - {google::protobuf::LOGLEVEL_WARNING, log::kWarn}, - {google::protobuf::LOGLEVEL_ERROR, log::kError}, - {google::protobuf::LOGLEVEL_FATAL, log::kFatal}}; - log::Level lvl = log::kWarn; // default - auto iter = kLevelMap.find(level); - if (iter != kLevelMap.end()) { - lvl = iter->second; - } - ATOM_LOGGER_CALL(tableau::log::DefaultLogger(), lvl, "[libprotobuf %s:%d] %s", filename, line, msg.c_str()); -} - -const std::string& GetProtoName(const google::protobuf::Message& msg) { - const auto* md = msg.GetDescriptor(); - return md != nullptr ? md->name() : kEmpty; -} - -bool ExistsFile(const std::string& filename) { - std::ifstream file(filename); - // returns true if the file exists and is accessible - return file.good(); -} - -bool ReadFile(const std::string& filename, std::string& content) { - std::ifstream file(filename); - if (!file.is_open()) { - g_err_msg = "failed to open " + filename + ": " + strerror(errno); - return false; - } - std::stringstream ss; - ss << file.rdbuf(); - content = ss.str(); - return true; -} - -std::string GetPatchName(tableau::Patch patch) { - auto* descriptor = tableau::Patch_descriptor(); - if (descriptor) { - auto* value = descriptor->FindValueByNumber(patch); - if (value) { - return value->name(); - } - } - return std::to_string(static_cast(patch)); -} - -bool Message2JSON(const google::protobuf::Message& msg, std::string& json) { - google::protobuf::util::JsonPrintOptions options; - options.add_whitespace = true; - options.always_print_primitive_fields = true; - options.preserve_proto_field_names = true; - return google::protobuf::util::MessageToJsonString(msg, &json, options).ok(); -} - -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options /* = nullptr */) { - google::protobuf::util::Status status; - if (options != nullptr) { - google::protobuf::util::JsonParseOptions parse_options; - parse_options.ignore_unknown_fields = options->ignore_unknown_fields; - status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); - } else { - status = google::protobuf::util::JsonStringToMessage(json, &msg); - } - if (!status.ok()) { - g_err_msg = "failed to parse " + GetProtoName(msg) + kJSONExt + ": " + status.ToString(); - return false; - } - return true; -} - -bool Text2Message(const std::string& text, google::protobuf::Message& msg) { - if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) { - g_err_msg = "failed to parse " + GetProtoName(msg) + kTextExt; - return false; - } - return true; -} -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) { - if (!msg.ParseFromString(bin)) { - g_err_msg = "failed to parse " + GetProtoName(msg) + kBinExt; - return false; - } - return true; -} - -// PatchMessage patches src into dst, which must be a message with the same descriptor. -// -// # Default PatchMessage mechanism -// - scalar: Populated scalar fields in src are copied to dst. -// - message: Populated singular messages in src are merged into dst by -// recursively calling [xproto.PatchMessage], or replace dst message if -// "PATCH_REPLACE" is specified for this field. -// - list: The elements of every list field in src are appended to the -// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" -// is specified for this field. -// - map: The entries of every map field in src are MERGED (different from -// the behavior of proto.Merge) into the corresponding map field in dst, -// or replace dst map if "PATCH_REPLACE" is specified for this field. -// - unknown: The unknown fields of src are appended to the unknown -// fields of dst (TODO: untested). -// -// # References: -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details -bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { - const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); - const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); - // Ensure both messages are of the same type - if (dst_descriptor != src_descriptor) { - g_err_msg = "dst and src are not messages with the same descriptor"; - ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), - src_descriptor->name().c_str()); - return false; - } - - // Get the reflection and descriptor for the messages - const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); - const google::protobuf::Reflection* src_reflection = src.GetReflection(); - - // List all populated fields - std::vector fields; - src_reflection->ListFields(src, &fields); - - // Iterates over every populated field. - for (auto fd : fields) { - const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); - tableau::Patch patch = opts.prop().patch(); - if (patch == tableau::PATCH_REPLACE) { - dst_reflection->ClearField(&dst, fd); - } - if (fd->is_map()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 - auto key_fd = fd->message_type()->map_key(); - auto value_fd = fd->message_type()->map_value(); - int src_count = src_reflection->FieldSize(src, fd); - int dst_count = dst_reflection->FieldSize(dst, fd); - switch (key_fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - std::unordered_map dst_key_index_map; \ - for (int i = 0; i < dst_count; i++) { \ - auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ - TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ - dst_key_index_map[key] = i; \ - } \ - for (int j = 0; j < src_count; j++) { \ - auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ - TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ - auto it = dst_key_index_map.find(key); \ - if (it != dst_key_index_map.end()) { \ - int index = it->second; \ - auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ - PatchMessage(dst_entry, src_entry); \ - } else { \ - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ - } \ - } \ - break; \ - } - - HANDLE_TYPE(INT32, Int32, int32_t); - HANDLE_TYPE(INT64, Int64, int64_t); - HANDLE_TYPE(UINT32, UInt32, uint32_t); - HANDLE_TYPE(UINT64, UInt64, uint64_t); - HANDLE_TYPE(BOOL, Bool, bool); - HANDLE_TYPE(STRING, String, std::string); - default: { - // other types are impossible to be protobuf map key - ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); - break; - } -#undef HANDLE_TYPE - } - } else if (fd->is_repeated()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 - int count = src_reflection->FieldSize(src, fd); - for (int j = 0; j < count; j++) { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ - break; \ - } - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { - const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); - break; - } - } - } - } else { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ - dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ - break; - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: - const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); - PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); - break; - } - } - } - - dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); - return true; -} - -bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& path, Format fmt, tableau::Patch patch, - const LoadOptions* options /* = nullptr*/) { - if (options == nullptr) { - return LoadMessageByPath(msg, path, fmt, nullptr); - } - if (options->mode == LoadMode::kModeOnlyMain) { - // ignore patch files when LoadMode::kModeOnlyMain specified - return LoadMessageByPath(msg, path, fmt, nullptr); - } - std::string name = GetProtoName(msg); - std::vector patch_paths; - auto iter = options->patch_paths.find(name); - if (iter != options->patch_paths.end()) { - // patch path specified in PatchPaths, then use it instead of PatchDirs. - patch_paths = iter->second; - } else { - for (auto&& patch_dir : options->patch_dirs) { - patch_paths.emplace_back(patch_dir + name + Format2Ext(fmt)); - } - } - - std::vector existed_patch_paths; - for (auto&& patch_path : patch_paths) { - if (ExistsFile(patch_path)) { - existed_patch_paths.emplace_back(patch_path); - } - } - if (existed_patch_paths.empty()) { - if (options->mode == LoadMode::kModeOnlyPatch) { - // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. - return true; - } - // no valid patch path provided, then just load from the "main" file. - return LoadMessageByPath(msg, path, fmt, options); - } - - switch (patch) { - case tableau::PATCH_REPLACE: { - // just use the last "patch" file - std::string& patch_path = existed_patch_paths.back(); - if (!LoadMessageByPath(msg, patch_path, Ext2Format(util::GetExt(patch_path)), options)) { - return false; - } - break; - } - case tableau::PATCH_MERGE: { - if (options->mode != LoadMode::kModeOnlyPatch) { - // load msg from the "main" file - if (!LoadMessageByPath(msg, path, fmt, options)) { - return false; - } - } - // Create a new instance of the same type of the original message - google::protobuf::Message* patch_msg_ptr = msg.New(); - std::unique_ptr _auto_release(msg.New()); - // load patch_msg from each "patch" file - for (auto&& patch_path : existed_patch_paths) { - if (!LoadMessageByPath(*patch_msg_ptr, patch_path, Ext2Format(util::GetExt(patch_path)), options)) { - return false; - } - if (!PatchMessage(msg, *patch_msg_ptr)) { - return false; - } - } - break; - } - default: { - g_err_msg = "unknown patch type: " + GetPatchName(patch); - return false; - } - } - ATOM_DEBUG("patched(%s) %s by %s: %s", GetPatchName(patch).c_str(), name.c_str(), - ATOM_VECTOR_STR(existed_patch_paths).c_str(), msg.ShortDebugString().c_str()); - return true; -} - -bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt, - const LoadOptions* options /* = nullptr*/) { - std::string content; - ReadFunc read_func = ReadFile; - if (options != nullptr && options->read_func) { - read_func = options->read_func; - } - bool ok = read_func(path, content); - if (!ok) { - return false; - } - switch (fmt) { - case Format::kJSON: { - return JSON2Message(content, msg, options); - } - case Format::kText: { - return Text2Message(content, msg); - } - case Format::kBin: { - return Bin2Message(content, msg); - } - default: { - g_err_msg = "unknown format: " + std::to_string(static_cast(fmt)); - return false; - } - } -} - -bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt, - const LoadOptions* options /* = nullptr*/) { - std::string name = GetProtoName(msg); - std::string path; - if (options) { - auto iter = options->paths.find(name); - if (iter != options->paths.end()) { - // path specified in Paths, then use it instead of dir. - path = iter->second; - fmt = Ext2Format(util::GetExt(iter->second)); - } - } - if (path.empty()) { - path = dir + name + Format2Ext(fmt); - } - - const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); - if (!descriptor) { - g_err_msg = "failed to get descriptor of message: " + name; - return false; - } - // access the extension directly using the generated identifier - const tableau::WorksheetOptions worksheet_options = descriptor->options().GetExtension(tableau::worksheet); - if (worksheet_options.patch() != tableau::PATCH_NONE) { - return LoadMessageWithPatch(msg, path, fmt, worksheet_options.patch(), options); - } - - return LoadMessageByPath(msg, path, fmt, options); -} - -bool StoreMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt) { - // TODO: write protobuf message to file, support 3 formats: json, text, and bin. - return false; -} - bool Hub::Load(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { auto msger_map = InternalLoad(dir, fmt, options); if (!msger_map) { @@ -477,7 +60,7 @@ void Hub::InitScheduler() { std::shared_ptr Hub::InternalLoad(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) const { // intercept protobuf error logs - auto old_handler = google::protobuf::SetLogHandler(ProtobufLogHandler); + auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); auto msger_map = NewMessagerMap(); for (auto iter : *msger_map) { auto&& name = iter.first; @@ -536,7 +119,7 @@ bool Hub::Postprocess(std::shared_ptr msger_map) { auto msger = iter.second; bool ok = msger->ProcessAfterLoadAll(tmp_hub); if (!ok) { - g_err_msg = "hub call ProcessAfterLoadAll failed, messager: " + msger->Name(); + SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + msger->Name()); return false; } } diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index 4e630ba6..9c423e9e 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -8,25 +8,17 @@ #include #include -#include -#include #include #include #include #include #include +#include "load.pc.h" #include "messager.pc.h" #include "scheduler.pc.h" namespace tableau { -extern const std::string kUnknownExt; -extern const std::string kJSONExt; -extern const std::string kTextExt; -extern const std::string kBinExt; - -const std::string& GetErrMsg(); - class MessagerContainer; class Hub; @@ -56,26 +48,6 @@ struct HubOptions { MessagerContainerProvider provider; }; -// Convert file extension to Format type. -// NOTE: ext includes dot ".", such as: -// - kJSONExt：".json" -// - kTextExt".txt" -// - kBinExt".bin" -Format Ext2Format(const std::string& ext); -// Empty string will be returned if an unsupported enum value has been passed, -// and the error message can be obtained by GetErrMsg(). -const std::string& Format2Ext(Format fmt); -bool Message2JSON(const google::protobuf::Message& msg, std::string& json); -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options = nullptr); -bool Text2Message(const std::string& text, google::protobuf::Message& msg); -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); -void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg); -const std::string& GetProtoName(const google::protobuf::Message& msg); -bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr); -bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr); - class Hub { public: Hub(const HubOptions* options = nullptr) @@ -201,5 +173,4 @@ class MessagerContainer { std::shared_ptr chapter_conf_; std::shared_ptr theme_conf_; }; - } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc new file mode 100644 index 00000000..fa46ea5c --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -0,0 +1,300 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#include "load.pc.h" + +#include "logger.pc.h" +#include "util.pc.h" + +namespace tableau { +#ifdef _WIN32 +#undef GetMessage +#endif + +// PatchMessage patches src into dst, which must be a message with the same descriptor. +// +// # Default PatchMessage mechanism +// - scalar: Populated scalar fields in src are copied to dst. +// - message: Populated singular messages in src are merged into dst by +// recursively calling [xproto.PatchMessage], or replace dst message if +// "PATCH_REPLACE" is specified for this field. +// - list: The elements of every list field in src are appended to the +// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" +// is specified for this field. +// - map: The entries of every map field in src are MERGED (different from +// the behavior of proto.Merge) into the corresponding map field in dst, +// or replace dst map if "PATCH_REPLACE" is specified for this field. +// - unknown: The unknown fields of src are appended to the unknown +// fields of dst (TODO: untested). +// +// # References: +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { + const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); + const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); + // Ensure both messages are of the same type + if (dst_descriptor != src_descriptor) { + SetErrMsg("dst and src are not messages with the same descriptor"); + ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), + src_descriptor->name().c_str()); + return false; + } + + // Get the reflection and descriptor for the messages + const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); + const google::protobuf::Reflection* src_reflection = src.GetReflection(); + + // List all populated fields + std::vector fields; + src_reflection->ListFields(src, &fields); + + // Iterates over every populated field. + for (auto fd : fields) { + const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); + tableau::Patch patch = opts.prop().patch(); + if (patch == tableau::PATCH_REPLACE) { + dst_reflection->ClearField(&dst, fd); + } + if (fd->is_map()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 + auto key_fd = fd->message_type()->map_key(); + auto value_fd = fd->message_type()->map_value(); + int src_count = src_reflection->FieldSize(src, fd); + int dst_count = dst_reflection->FieldSize(dst, fd); + switch (key_fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + std::unordered_map dst_key_index_map; \ + for (int i = 0; i < dst_count; i++) { \ + auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ + TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ + dst_key_index_map[key] = i; \ + } \ + for (int j = 0; j < src_count; j++) { \ + auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ + TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ + auto it = dst_key_index_map.find(key); \ + if (it != dst_key_index_map.end()) { \ + int index = it->second; \ + auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ + PatchMessage(dst_entry, src_entry); \ + } else { \ + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ + } \ + } \ + break; \ + } + + HANDLE_TYPE(INT32, Int32, int32_t); + HANDLE_TYPE(INT64, Int64, int64_t); + HANDLE_TYPE(UINT32, UInt32, uint32_t); + HANDLE_TYPE(UINT64, UInt64, uint64_t); + HANDLE_TYPE(BOOL, Bool, bool); + HANDLE_TYPE(STRING, String, std::string); + default: { + // other types are impossible to be protobuf map key + ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); + break; + } +#undef HANDLE_TYPE + } + } else if (fd->is_repeated()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 + int count = src_reflection->FieldSize(src, fd); + for (int j = 0; j < count; j++) { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ + break; \ + } + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); + break; + } + } + } + } else { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ + dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ + break; + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); + PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); + break; + } + } + } + + dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); + return true; +} + +bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& path, Format fmt, tableau::Patch patch, + const LoadOptions* options /* = nullptr*/) { + if (options == nullptr) { + return LoadMessageByPath(msg, path, fmt, nullptr); + } + if (options->mode == LoadMode::kModeOnlyMain) { + // ignore patch files when LoadMode::kModeOnlyMain specified + return LoadMessageByPath(msg, path, fmt, nullptr); + } + std::string name = util::GetProtoName(msg); + std::vector patch_paths; + auto iter = options->patch_paths.find(name); + if (iter != options->patch_paths.end()) { + // patch path specified in PatchPaths, then use it instead of PatchDirs. + patch_paths = iter->second; + } else { + for (auto&& patch_dir : options->patch_dirs) { + patch_paths.emplace_back(patch_dir + name + util::Format2Ext(fmt)); + } + } + + std::vector existed_patch_paths; + for (auto&& patch_path : patch_paths) { + if (util::ExistsFile(patch_path)) { + existed_patch_paths.emplace_back(patch_path); + } + } + if (existed_patch_paths.empty()) { + if (options->mode == LoadMode::kModeOnlyPatch) { + // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. + return true; + } + // no valid patch path provided, then just load from the "main" file. + return LoadMessageByPath(msg, path, fmt, options); + } + + switch (patch) { + case tableau::PATCH_REPLACE: { + // just use the last "patch" file + std::string& patch_path = existed_patch_paths.back(); + if (!LoadMessageByPath(msg, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { + return false; + } + break; + } + case tableau::PATCH_MERGE: { + if (options->mode != LoadMode::kModeOnlyPatch) { + // load msg from the "main" file + if (!LoadMessageByPath(msg, path, fmt, options)) { + return false; + } + } + // Create a new instance of the same type of the original message + google::protobuf::Message* patch_msg_ptr = msg.New(); + std::unique_ptr _auto_release(msg.New()); + // load patch_msg from each "patch" file + for (auto&& patch_path : existed_patch_paths) { + if (!LoadMessageByPath(*patch_msg_ptr, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { + return false; + } + if (!PatchMessage(msg, *patch_msg_ptr)) { + return false; + } + } + break; + } + default: { + SetErrMsg("unknown patch type: " + util::GetPatchName(patch)); + return false; + } + } + ATOM_DEBUG("patched(%s) %s by %s: %s", util::GetPatchName(patch).c_str(), name.c_str(), + ATOM_VECTOR_STR(existed_patch_paths).c_str(), msg.ShortDebugString().c_str()); + return true; +} + +bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt, + const LoadOptions* options /* = nullptr*/) { + std::string content; + ReadFunc read_func = util::ReadFile; + if (options != nullptr && options->read_func) { + read_func = options->read_func; + } + bool ok = read_func(path, content); + if (!ok) { + return false; + } + switch (fmt) { + case Format::kJSON: { + return util::JSON2Message(content, msg, options); + } + case Format::kText: { + return util::Text2Message(content, msg); + } + case Format::kBin: { + return util::Bin2Message(content, msg); + } + default: { + SetErrMsg("unknown format: " + std::to_string(static_cast(fmt))); + return false; + } + } +} + +bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt, + const LoadOptions* options /* = nullptr*/) { + std::string name = util::GetProtoName(msg); + std::string path; + if (options) { + auto iter = options->paths.find(name); + if (iter != options->paths.end()) { + // path specified in Paths, then use it instead of dir. + path = iter->second; + fmt = util::Ext2Format(util::GetExt(iter->second)); + } + } + if (path.empty()) { + path = dir + name + util::Format2Ext(fmt); + } + + const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); + if (!descriptor) { + SetErrMsg("failed to get descriptor of message: " + name); + return false; + } + // access the extension directly using the generated identifier + const tableau::WorksheetOptions worksheet_options = descriptor->options().GetExtension(tableau::worksheet); + if (worksheet_options.patch() != tableau::PATCH_NONE) { + return LoadMessageWithPatch(msg, path, fmt, worksheet_options.patch(), options); + } + + return LoadMessageByPath(msg, path, fmt, options); +} +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.h b/test/cpp-tableau-loader/src/protoconf/load.pc.h new file mode 100644 index 00000000..e61f957e --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.h @@ -0,0 +1,48 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#pragma once +#include + +#include +#include + +#include "util.pc.h" + +namespace tableau { +enum class LoadMode { + kModeDefault, + kModeOnlyMain, + kModeOnlyPatch, +}; + +// ReadFunc reads the config file and returns its content. +using ReadFunc = std::function; + +struct LoadOptions { + // read_func reads the config file and returns its content. + ReadFunc read_func; + // Whether to ignore unknown JSON fields during parsing. + // + // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. + bool ignore_unknown_fields = false; + // Paths maps each messager name to a corresponding config file path. + // If specified, then the main messager will be parsed from the file + // directly, other than the specified load dir. + std::unordered_map paths; + // Patch paths maps each messager name to one or multiple corresponding patch file paths. + // If specified, then main messager will be patched. + std::unordered_map> patch_paths; + // Patch dirs specifies the directory paths for config patching. + std::vector patch_dirs; + // Mode specifies the loading mode for config patching. + LoadMode mode = LoadMode::kModeDefault; +}; + +bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr); +bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr); +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/messager.pc.h b/test/cpp-tableau-loader/src/protoconf/messager.pc.h index e5fb56cc..b7151709 100644 --- a/test/cpp-tableau-loader/src/protoconf/messager.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/messager.pc.h @@ -9,46 +9,11 @@ #include #include -namespace tableau { -enum class Format { - kUnknown, - kJSON, - kText, - kBin, -}; - -enum class LoadMode { - kModeDefault, - kModeOnlyMain, - kModeOnlyPatch, -}; - -static const std::string kEmpty = ""; +#include "util.pc.h" +namespace tableau { class Hub; - -// ReadFunc reads the config file and returns its content. -using ReadFunc = std::function; - -struct LoadOptions { - // read_func reads the config file and returns its content. - ReadFunc read_func; - // Whether to ignore unknown JSON fields during parsing. - // - // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. - bool ignore_unknown_fields = false; - // Paths maps each messager name to a corresponding config file path. - // If specified, then the main messager will be parsed from the file - // directly, other than the specified load dir. - std::unordered_map paths; - // Patch paths maps each messager name to one or multiple corresponding patch file paths. - // If specified, then main messager will be patched. - std::unordered_map> patch_paths; - // Patch dirs specifies the directory paths for config patching. - std::vector patch_dirs; - // Mode specifies the loading mode for config patching. - LoadMode mode = LoadMode::kModeDefault; -}; +struct LoadOptions; class Messager { public: @@ -75,5 +40,4 @@ class Messager { virtual bool ProcessAfterLoad() { return true; }; Stats stats_; }; - } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.cc b/test/cpp-tableau-loader/src/protoconf/util.pc.cc index c6ec22e2..b86442f9 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.cc @@ -5,6 +5,13 @@ #include "util.pc.h" +#include +#include + +#include +#include +#include + #ifdef _WIN32 #include #include @@ -12,6 +19,8 @@ #include #endif +#include "load.pc.h" +#include "logger.pc.h" #include "messager.pc.h" namespace tableau { @@ -22,6 +31,15 @@ static constexpr char kPathSeperator = '\\'; static constexpr char kPathSeperator = '/'; #endif +static thread_local std::string g_err_msg; +const std::string& GetErrMsg() { return g_err_msg; } +void SetErrMsg(const std::string& msg) { g_err_msg = msg; } + +const std::string kUnknownExt = ".unknown"; +const std::string kJSONExt = ".json"; +const std::string kTextExt = ".txt"; +const std::string kBinExt = ".bin"; + namespace util { int Mkdir(const std::string& path) { std::string path_ = path + kPathSeperator; @@ -49,6 +67,24 @@ std::string GetDir(const std::string& path) { return kEmpty; } +bool ExistsFile(const std::string& filename) { + std::ifstream file(filename); + // returns true if the file exists and is accessible + return file.good(); +} + +bool ReadFile(const std::string& filename, std::string& content) { + std::ifstream file(filename); + if (!file.is_open()) { + SetErrMsg("failed to open " + filename + ": " + strerror(errno)); + return false; + } + std::stringstream ss; + ss << file.rdbuf(); + content = ss.str(); + return true; +} + std::string GetExt(const std::string& path) { std::size_t pos = path.find_last_of("."); if (pos != std::string::npos) { @@ -57,5 +93,98 @@ std::string GetExt(const std::string& path) { return kEmpty; } +Format Ext2Format(const std::string& ext) { + if (ext == kJSONExt) { + return Format::kJSON; + } else if (ext == kTextExt) { + return Format::kText; + } else if (ext == kBinExt) { + return Format::kBin; + } else { + return Format::kUnknown; + } +} + +const std::string& Format2Ext(Format fmt) { + switch (fmt) { + case Format::kJSON: + return kJSONExt; + case Format::kText: + return kTextExt; + case Format::kBin: + return kBinExt; + default: + return kUnknownExt; + } +} + +bool Message2JSON(const google::protobuf::Message& msg, std::string& json) { + google::protobuf::util::JsonPrintOptions options; + options.add_whitespace = true; + options.always_print_primitive_fields = true; + options.preserve_proto_field_names = true; + return google::protobuf::util::MessageToJsonString(msg, &json, options).ok(); +} + +bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options /* = nullptr */) { + google::protobuf::util::Status status; + if (options != nullptr) { + google::protobuf::util::JsonParseOptions parse_options; + parse_options.ignore_unknown_fields = options->ignore_unknown_fields; + status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); + } else { + status = google::protobuf::util::JsonStringToMessage(json, &msg); + } + if (!status.ok()) { + SetErrMsg("failed to parse " + GetProtoName(msg) + kJSONExt + ": " + status.ToString()); + return false; + } + return true; +} + +bool Text2Message(const std::string& text, google::protobuf::Message& msg) { + if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) { + SetErrMsg("failed to parse " + GetProtoName(msg) + kTextExt); + return false; + } + return true; +} +bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) { + if (!msg.ParseFromString(bin)) { + SetErrMsg("failed to parse " + GetProtoName(msg) + kBinExt); + return false; + } + return true; +} + +const std::string& GetProtoName(const google::protobuf::Message& msg) { + const auto* md = msg.GetDescriptor(); + return md != nullptr ? md->name() : kEmpty; +} + +std::string GetPatchName(tableau::Patch patch) { + auto* descriptor = tableau::Patch_descriptor(); + if (descriptor) { + auto* value = descriptor->FindValueByNumber(patch); + if (value) { + return value->name(); + } + } + return std::to_string(static_cast(patch)); +} + +// refer: https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/stubs/logging.h +void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg) { + static const std::unordered_map kLevelMap = {{google::protobuf::LOGLEVEL_INFO, log::kInfo}, + {google::protobuf::LOGLEVEL_WARNING, log::kWarn}, + {google::protobuf::LOGLEVEL_ERROR, log::kError}, + {google::protobuf::LOGLEVEL_FATAL, log::kFatal}}; + log::Level lvl = log::kWarn; // default + auto iter = kLevelMap.find(level); + if (iter != kLevelMap.end()) { + lvl = iter->second; + } + ATOM_LOGGER_CALL(tableau::log::DefaultLogger(), lvl, "[libprotobuf %s:%d] %s", filename, line, msg.c_str()); +} } // namespace util } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.h b/test/cpp-tableau-loader/src/protoconf/util.pc.h index 7c2fd947..9d24149b 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.h @@ -4,12 +4,33 @@ // - protoc v3.19.3 #pragma once +#include #include #include #include +#include "tableau/protobuf/tableau.pb.h" + namespace tableau { +const std::string& GetErrMsg(); +void SetErrMsg(const std::string& msg); + +enum class Format { + kUnknown, + kJSON, + kText, + kBin, +}; + +static const std::string kEmpty = ""; +extern const std::string kUnknownExt; +extern const std::string kJSONExt; +extern const std::string kTextExt; +extern const std::string kBinExt; + +struct LoadOptions; + namespace util { // Combine hash values // @@ -37,11 +58,35 @@ int Mkdir(const std::string& path); // GetDir returns all but the last element of path, typically the path's // directory. std::string GetDir(const std::string& path); +// ExistsFile checks if a file exists. +bool ExistsFile(const std::string& filename); +// ReadFile reads the file named by filename and returns the contents. +bool ReadFile(const std::string& filename, std::string& content); + // GetExt returns the file name extension used by path. // The extension is the suffix beginning at the final dot // in the final element of path; it is empty if there is // no dot. std::string GetExt(const std::string& path); +// Convert file extension to Format type. +// NOTE: ext includes dot ".", such as: +// - kJSONExt：".json" +// - kTextExt".txt" +// - kBinExt".bin" +Format Ext2Format(const std::string& ext); +// Empty string will be returned if an unsupported enum value has been passed, +// and the error message can be obtained by GetErrMsg(). +const std::string& Format2Ext(Format fmt); + +bool Message2JSON(const google::protobuf::Message& msg, std::string& json); +bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options = nullptr); +bool Text2Message(const std::string& text, google::protobuf::Message& msg); +bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); + +const std::string& GetProtoName(const google::protobuf::Message& msg); +std::string GetPatchName(tableau::Patch patch); + +void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg); class TimeProfiler { protected: @@ -59,6 +104,5 @@ class TimeProfiler { return std::chrono::duration_cast(duration); } }; - } // namespace util } // namespace tableau From b53894eef89d05a5fa50db393e8a195dcac34ae7 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 24 Apr 2025 21:46:09 +0800 Subject: [PATCH 09/14] feat: private + friend class --- cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h | 4 +++- test/cpp-tableau-loader/src/protoconf/hub.pc.h | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h index 207c79a0..29bc676c 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h @@ -115,10 +115,12 @@ const U* Hub::GetOrderedMap(Args... args) const { // Auto-generated template specializations below class MessagerContainer { + friend class Hub; + public: MessagerContainer(std::shared_ptr msger_map = nullptr); - public: + private: std::shared_ptr msger_map_; std::time_t last_loaded_time_; // Auto-generated all messagers as fields for fast access below diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index 9c423e9e..1c81120e 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -156,10 +156,12 @@ template <> const std::shared_ptr Hub::Get() const; class MessagerContainer { + friend class Hub; + public: MessagerContainer(std::shared_ptr msger_map = nullptr); - public: + private: std::shared_ptr msger_map_; std::time_t last_loaded_time_; // Auto-generated all messagers as fields for fast access below From b350515b9b1b47be2d28d048af994fa025a1aa90 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 25 Apr 2025 16:04:25 +0800 Subject: [PATCH 10/14] feat: target code --- test/cpp-tableau-loader/src/hub/hub.cpp | 1 - .../src/protoconf/hub.pc.cc | 74 +++---------------- .../cpp-tableau-loader/src/protoconf/hub.pc.h | 31 ++++++++ .../src/protoconf/registry.pc.cc | 14 ---- .../src/protoconf/registry.pc.h | 29 -------- .../src/protoconf/registry_shard0.pc.cc | 23 +++++- .../src/protoconf/registry_shard1.pc.cc | 41 +++++++++- 7 files changed, 104 insertions(+), 109 deletions(-) delete mode 100644 test/cpp-tableau-loader/src/protoconf/registry.pc.cc delete mode 100644 test/cpp-tableau-loader/src/protoconf/registry.pc.h diff --git a/test/cpp-tableau-loader/src/hub/hub.cpp b/test/cpp-tableau-loader/src/hub/hub.cpp index bba703a3..4c973299 100644 --- a/test/cpp-tableau-loader/src/hub/hub.cpp +++ b/test/cpp-tableau-loader/src/hub/hub.cpp @@ -3,7 +3,6 @@ #include "hub.h" #include "hub/custom/item/custom_item_conf.h" #include "protoconf/logger.pc.h" -#include "protoconf/registry.pc.h" void LogWrite(std::ostream* os, const tableau::log::SourceLocation& loc, const tableau::log::LevelInfo& lvl, const std::string& content) { diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index 7a449098..20ccbb26 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -13,16 +13,11 @@ #include "logger.pc.h" #include "messager.pc.h" -#include "registry.pc.h" #include "util.pc.h" -// Auto-generated includes below -#include "hero_conf.pc.h" -#include "item_conf.pc.h" -#include "patch_conf.pc.h" -#include "test_conf.pc.h" - namespace tableau { +Registrar Registry::registrar = Registrar(); + bool Hub::Load(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { auto msger_map = InternalLoad(dir, fmt, options); if (!msger_map) { @@ -128,64 +123,17 @@ bool Hub::Postprocess(std::shared_ptr msger_map) { std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; } -// Auto-generated template specializations below -template <> -const std::shared_ptr Hub::Get() const { - return GetMessagerContainer()->hero_base_conf_; -} - -template <> -const std::shared_ptr Hub::Get() const { - return GetMessagerContainer()->hero_conf_; -} - -template <> -const std::shared_ptr Hub::Get() const { - return GetMessagerContainer()->item_conf_; -} - -template <> -const std::shared_ptr Hub::Get() const { - return GetMessagerContainer()->patch_merge_conf_; -} - -template <> -const std::shared_ptr Hub::Get() const { - return GetMessagerContainer()->patch_replace_conf_; -} - -template <> -const std::shared_ptr Hub::Get() const { - return GetMessagerContainer()->recursive_patch_conf_; -} - -template <> -const std::shared_ptr Hub::Get() const { - return GetMessagerContainer()->activity_conf_; -} - -template <> -const std::shared_ptr Hub::Get() const { - return GetMessagerContainer()->chapter_conf_; -} - -template <> -const std::shared_ptr Hub::Get() const { - return GetMessagerContainer()->theme_conf_; -} - MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), last_loaded_time_(std::time(nullptr)) { - // Auto-generated initializations below - hero_base_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroBaseConf"]); - hero_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroConf"]); - item_conf_ = std::dynamic_pointer_cast((*msger_map_)["ItemConf"]); - patch_merge_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchMergeConf"]); - patch_replace_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchReplaceConf"]); - recursive_patch_conf_ = std::dynamic_pointer_cast((*msger_map_)["RecursivePatchConf"]); - activity_conf_ = std::dynamic_pointer_cast((*msger_map_)["ActivityConf"]); - chapter_conf_ = std::dynamic_pointer_cast((*msger_map_)["ChapterConf"]); - theme_conf_ = std::dynamic_pointer_cast((*msger_map_)["ThemeConf"]); + // Auto-generated shards below + InitShard0(); + InitShard1(); +} + +void Registry::Init() { + // Auto-generated shards below + InitShard0(); + InitShard1(); } } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index 1c81120e..064d4b28 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -161,6 +161,11 @@ class MessagerContainer { public: MessagerContainer(std::shared_ptr msger_map = nullptr); + private: + // Auto-generated shards below + void InitShard0(); + void InitShard1(); + private: std::shared_ptr msger_map_; std::time_t last_loaded_time_; @@ -175,4 +180,30 @@ class MessagerContainer { std::shared_ptr chapter_conf_; std::shared_ptr theme_conf_; }; + +using MessagerGenerator = std::function()>; +// messager name -> messager generator +using Registrar = std::unordered_map; +class Registry { + friend class Hub; + + public: + static void Init(); + + template + static void Register(); + + private: + // Auto-generated shards below + static void InitShard0(); + static void InitShard1(); + + private: + static Registrar registrar; +}; + +template +void Registry::Register() { + registrar[T::Name()] = []() { return std::make_shared(); }; +} } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/registry.pc.cc b/test/cpp-tableau-loader/src/protoconf/registry.pc.cc deleted file mode 100644 index b52a609f..00000000 --- a/test/cpp-tableau-loader/src/protoconf/registry.pc.cc +++ /dev/null @@ -1,14 +0,0 @@ -// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. -// versions: -// - protoc-gen-cpp-tableau-loader v0.8.0 -// - protoc v3.19.3 - -#include "registry.pc.h" - -namespace tableau { -Registrar Registry::registrar = Registrar(); -void Registry::Init() { - InitShard0(); - InitShard1(); -} -} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/registry.pc.h b/test/cpp-tableau-loader/src/protoconf/registry.pc.h deleted file mode 100644 index 59034f19..00000000 --- a/test/cpp-tableau-loader/src/protoconf/registry.pc.h +++ /dev/null @@ -1,29 +0,0 @@ -// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. -// versions: -// - protoc-gen-cpp-tableau-loader v0.8.0 -// - protoc v3.19.3 - -#pragma once -#include "messager.pc.h" -namespace tableau { -using MessagerGenerator = std::function()>; -// messager name -> messager generator -using Registrar = std::unordered_map; -class Registry { - public: - static void Init(); - static void InitShard0(); - static void InitShard1(); - - template - static void Register(); - - static Registrar registrar; -}; - -template -void Registry::Register() { - registrar[T::Name()] = []() { return std::make_shared(); }; -} - -} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/registry_shard0.pc.cc b/test/cpp-tableau-loader/src/protoconf/registry_shard0.pc.cc index 8ea5e6fc..7fb9a3af 100644 --- a/test/cpp-tableau-loader/src/protoconf/registry_shard0.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/registry_shard0.pc.cc @@ -3,7 +3,7 @@ // - protoc-gen-cpp-tableau-loader v0.8.0 // - protoc v3.19.3 -#include "registry.pc.h" +#include "hub.pc.h" #include "hero_conf.pc.h" #include "item_conf.pc.h" @@ -14,4 +14,25 @@ void Registry::InitShard0() { Register(); Register(); } + +void MessagerContainer::InitShard0() { + hero_base_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroBaseConf"]); + hero_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroConf"]); + item_conf_ = std::dynamic_pointer_cast((*msger_map_)["ItemConf"]); +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->hero_base_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->hero_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->item_conf_; +} } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/registry_shard1.pc.cc b/test/cpp-tableau-loader/src/protoconf/registry_shard1.pc.cc index fbebb5f6..e0c1a479 100644 --- a/test/cpp-tableau-loader/src/protoconf/registry_shard1.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/registry_shard1.pc.cc @@ -3,7 +3,7 @@ // - protoc-gen-cpp-tableau-loader v0.8.0 // - protoc v3.19.3 -#include "registry.pc.h" +#include "hub.pc.h" #include "patch_conf.pc.h" #include "test_conf.pc.h" @@ -17,4 +17,43 @@ void Registry::InitShard1() { Register(); Register(); } + +void MessagerContainer::InitShard1() { + patch_merge_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchMergeConf"]); + patch_replace_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchReplaceConf"]); + recursive_patch_conf_ = std::dynamic_pointer_cast((*msger_map_)["RecursivePatchConf"]); + activity_conf_ = std::dynamic_pointer_cast((*msger_map_)["ActivityConf"]); + chapter_conf_ = std::dynamic_pointer_cast((*msger_map_)["ChapterConf"]); + theme_conf_ = std::dynamic_pointer_cast((*msger_map_)["ThemeConf"]); +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->patch_merge_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->patch_replace_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->recursive_patch_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->activity_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->chapter_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->theme_conf_; +} } // namespace tableau From ab43d44457a20fda8b8ae04dfdf60dd465fa7e72 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 25 Apr 2025 17:40:08 +0800 Subject: [PATCH 11/14] feat: move registry to hub --- cmd/protoc-gen-cpp-tableau-loader/embed.go | 70 +-- .../embed/hub.pc.cc | 128 ------ .../embed/hub.pc.h | 128 ------ cmd/protoc-gen-cpp-tableau-loader/hub.go | 432 ++++++++++++++++++ cmd/protoc-gen-cpp-tableau-loader/main.go | 16 +- cmd/protoc-gen-cpp-tableau-loader/registry.go | 183 -------- cmd/protoc-gen-cpp-tableau-loader/sharded.go | 127 +++++ .../src/protoconf/hub.pc.cc | 2 - .../cpp-tableau-loader/src/protoconf/hub.pc.h | 10 +- ...registry_shard0.pc.cc => hub_shard0.pc.cc} | 24 +- ...registry_shard1.pc.cc => hub_shard1.pc.cc} | 36 +- 11 files changed, 602 insertions(+), 554 deletions(-) delete mode 100644 cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc delete mode 100644 cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h create mode 100644 cmd/protoc-gen-cpp-tableau-loader/hub.go delete mode 100644 cmd/protoc-gen-cpp-tableau-loader/registry.go create mode 100644 cmd/protoc-gen-cpp-tableau-loader/sharded.go rename test/cpp-tableau-loader/src/protoconf/{registry_shard0.pc.cc => hub_shard0.pc.cc} (100%) rename test/cpp-tableau-loader/src/protoconf/{registry_shard1.pc.cc => hub_shard1.pc.cc} (100%) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed.go b/cmd/protoc-gen-cpp-tableau-loader/embed.go index bcde35ca..1880d82a 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed.go +++ b/cmd/protoc-gen-cpp-tableau-loader/embed.go @@ -3,9 +3,7 @@ package main import ( "embed" "path" - "strings" - "github.com/iancoleman/strcase" "github.com/tableauio/loader/cmd/protoc-gen-cpp-tableau-loader/helper" "google.golang.org/protobuf/compiler/protogen" ) @@ -13,21 +11,12 @@ import ( //go:embed embed/* var efs embed.FS -const ( - includeNotes = "// Auto-generated includes below\n" - declarationNotes = "// Auto-generated declarations below\n" - templateSpecializationNotes = "// Auto-generated template specializations below\n" - initializationNotes = "// Auto-generated initializations below\n" - fieldNotes = "// Auto-generated all messagers as fields for fast access below\n" -) - // generateEmbed generates related registry files. func generateEmbed(gen *protogen.Plugin) { entries, err := efs.ReadDir("embed") if err != nil { panic(err) } - protofiles, fileMessagers := getAllOrderedFilesAndMessagers(gen) for _, entry := range entries { if entry.IsDir() { continue @@ -41,63 +30,6 @@ func generateEmbed(gen *protogen.Plugin) { if err != nil { panic(err) } - file := string(content) - switch entry.Name() { - case "hub.pc.cc": - // Auto-generated includes below - impl := "" - for _, proto := range protofiles { - impl += `#include "` + proto + "." + pcExt + `.h"` + "\n" - } - file = strings.ReplaceAll(file, includeNotes, includeNotes+impl) - // Auto-generated template specializations below - impl = "" - for _, proto := range protofiles { - for _, messager := range fileMessagers[proto] { - impl += "template <>\n" - impl += "const std::shared_ptr<" + messager + "> Hub::Get<" + messager + ">() const {\n" - impl += " return GetMessagerContainer()->" + strcase.ToSnake(messager) + "_;\n" - impl += "}\n" - impl += "\n" - } - } - file = strings.ReplaceAll(file, templateSpecializationNotes, templateSpecializationNotes+impl) - // Auto-generated initializations below - impl = "" - for _, proto := range protofiles { - for _, messager := range fileMessagers[proto] { - impl += " " + strcase.ToSnake(messager) + "_ = std::dynamic_pointer_cast<" + messager + `>((*msger_map_)["` + messager + `"]);` + "\n" - } - } - file = strings.ReplaceAll(file, initializationNotes, initializationNotes+impl) - case "hub.pc.h": - // Auto-generated declarations below - impl := "" - for _, proto := range protofiles { - for _, messager := range fileMessagers[proto] { - impl += "class " + messager + ";\n" - } - } - file = strings.ReplaceAll(file, declarationNotes, declarationNotes+impl) - // Auto-generated template specializations below - impl = "" - for _, proto := range protofiles { - for _, messager := range fileMessagers[proto] { - impl += "template <>\n" - impl += "const std::shared_ptr<" + messager + "> Hub::Get<" + messager + ">() const;\n" - impl += "\n" - } - } - file = strings.ReplaceAll(file, templateSpecializationNotes, templateSpecializationNotes+impl) - // Auto-generated all messagers as fields for fast access below - impl = "" - for _, proto := range protofiles { - for _, messager := range fileMessagers[proto] { - impl += " std::shared_ptr<" + messager + "> " + strcase.ToSnake(messager) + "_;\n" - } - } - file = strings.ReplaceAll(file, fieldNotes, fieldNotes+impl) - } - g.P(file) + g.P(string(content)) } } diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc deleted file mode 100644 index 2f0506f9..00000000 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc +++ /dev/null @@ -1,128 +0,0 @@ -#include "hub.pc.h" - -#include -#include -#include - -#include - -#include "logger.pc.h" -#include "messager.pc.h" -#include "registry.pc.h" -#include "util.pc.h" - -// Auto-generated includes below - -namespace tableau { -bool Hub::Load(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { - auto msger_map = InternalLoad(dir, fmt, options); - if (!msger_map) { - return false; - } - bool ok = Postprocess(msger_map); - if (!ok) { - return false; - } - SetMessagerMap(msger_map); - return true; -} - -bool Hub::AsyncLoad(const std::string& dir, Format fmt /* = Format::kJSON */, - const LoadOptions* options /* = nullptr */) { - auto msger_map = InternalLoad(dir, fmt, options); - if (!msger_map) { - return false; - } - bool ok = Postprocess(msger_map); - if (!ok) { - return false; - } - sched_->Dispatch(std::bind(&Hub::SetMessagerMap, this, msger_map)); - return true; -} - -int Hub::LoopOnce() { return sched_->LoopOnce(); } - -void Hub::InitScheduler() { - sched_ = new internal::Scheduler(); - sched_->Current(); -} - -std::shared_ptr Hub::InternalLoad(const std::string& dir, Format fmt /* = Format::kJSON */, - const LoadOptions* options /* = nullptr */) const { - // intercept protobuf error logs - auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); - auto msger_map = NewMessagerMap(); - for (auto iter : *msger_map) { - auto&& name = iter.first; - ATOM_DEBUG("loading %s", name.c_str()); - bool ok = iter.second->Load(dir, fmt, options); - if (!ok) { - ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str()); - // restore to old protobuf log handler - google::protobuf::SetLogHandler(old_handler); - return nullptr; - } - ATOM_DEBUG("loaded %s", name.c_str()); - } - - // restore to old protobuf log handler - google::protobuf::SetLogHandler(old_handler); - return msger_map; -} - -std::shared_ptr Hub::NewMessagerMap() const { - std::shared_ptr msger_map = std::make_shared(); - for (auto&& it : Registry::registrar) { - if (!options_.filter || options_.filter(it.first)) { - (*msger_map)[it.first] = it.second(); - } - } - return msger_map; -} - -std::shared_ptr Hub::GetMessagerMap() const { return GetMessagerContainer()->msger_map_; } - -void Hub::SetMessagerMap(std::shared_ptr msger_map) { - // replace with thread-safe guarantee. - std::unique_lock lock(mutex_); - msger_container_ = std::make_shared(msger_map); -} - -const std::shared_ptr Hub::GetMessager(const std::string& name) const { - auto msger_map = GetMessagerMap(); - if (msger_map) { - auto it = msger_map->find(name); - if (it != msger_map->end()) { - return it->second; - } - } - return nullptr; -} - -bool Hub::Postprocess(std::shared_ptr msger_map) { - // create a temporary hub with messager container for post process - Hub tmp_hub; - tmp_hub.SetMessagerMap(msger_map); - - // messager-level postprocess - for (auto iter : *msger_map) { - auto msger = iter.second; - bool ok = msger->ProcessAfterLoadAll(tmp_hub); - if (!ok) { - SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + msger->Name()); - return false; - } - } - return true; -} - -std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; } - -// Auto-generated template specializations below -MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) - : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), - last_loaded_time_(std::time(nullptr)) { - // Auto-generated initializations below -} -} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h deleted file mode 100644 index 29bc676c..00000000 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h +++ /dev/null @@ -1,128 +0,0 @@ -#pragma once -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "load.pc.h" -#include "messager.pc.h" -#include "scheduler.pc.h" - -namespace tableau { -class MessagerContainer; -class Hub; - -// Auto-generated declarations below - -using MessagerMap = std::unordered_map>; -// FilterFunc filter in messagers if returned value is true. -// NOTE: name is the protobuf message name, e.g.: "message ItemConf{...}". -using Filter = std::function; -using MessagerContainerProvider = std::function()>; - -struct HubOptions { - // Filter can only filter in certain specific messagers based on the - // condition that you provide. - Filter filter; - // Provide custom MessagerContainer. For keeping configuration access - // consistent in a coroutine or a transaction. - MessagerContainerProvider provider; -}; - -class Hub { - public: - Hub(const HubOptions* options = nullptr) - : msger_container_(std::make_shared()), options_(options ? *options : HubOptions{}) {} - /***** Synchronous Loading *****/ - // Load fills messages (in MessagerContainer) from files in the specified directory and format. - bool Load(const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); - - /***** Asynchronous Loading *****/ - // Load configs into temp MessagerContainer, and you should call LoopOnce() in you app's main loop, - // in order to take the temp MessagerContainer into effect. - bool AsyncLoad(const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); - int LoopOnce(); - // You'd better initialize the scheduler in the main thread. - void InitScheduler(); - - /***** MessagerMap *****/ - std::shared_ptr GetMessagerMap() const; - void SetMessagerMap(std::shared_ptr msger_map); - - /***** MessagerContainer *****/ - // This function is exposed only for use in MessagerContainerProvider. - std::shared_ptr GetMessagerContainer() const { - if (options_.provider != nullptr) { - return options_.provider(); - } - return msger_container_; - } - - /***** Access APIs *****/ - template - const std::shared_ptr Get() const; - - template - const U* Get(Args... args) const; - - template - const U* GetOrderedMap(Args... args) const; - - // GetLastLoadedTime returns the time when hub's msger_container_ was last set. - inline std::time_t GetLastLoadedTime() const; - - private: - std::shared_ptr InternalLoad(const std::string& dir, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr) const; - std::shared_ptr NewMessagerMap() const; - const std::shared_ptr GetMessager(const std::string& name) const; - - bool Postprocess(std::shared_ptr msger_map); - - private: - // For thread-safe guarantee during configuration updating. - std::mutex mutex_; - // All messagers' container. - std::shared_ptr msger_container_; - // Loading scheduler. - internal::Scheduler* sched_ = nullptr; - // Hub options - const HubOptions options_; -}; - -template -const std::shared_ptr Hub::Get() const { - auto msg = GetMessager(T::Name()); - return std::dynamic_pointer_cast(msg); -} - -template -const U* Hub::Get(Args... args) const { - auto msger = Get(); - return msger ? msger->Get(args...) : nullptr; -} - -template -const U* Hub::GetOrderedMap(Args... args) const { - auto msger = Get(); - return msger ? msger->GetOrderedMap(args...) : nullptr; -} - -// Auto-generated template specializations below -class MessagerContainer { - friend class Hub; - - public: - MessagerContainer(std::shared_ptr msger_map = nullptr); - - private: - std::shared_ptr msger_map_; - std::time_t last_loaded_time_; - // Auto-generated all messagers as fields for fast access below -}; -} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/hub.go b/cmd/protoc-gen-cpp-tableau-loader/hub.go new file mode 100644 index 00000000..4838522f --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/hub.go @@ -0,0 +1,432 @@ +package main + +import ( + "errors" + "sort" + + "github.com/iancoleman/strcase" + "github.com/tableauio/loader/cmd/protoc-gen-cpp-tableau-loader/helper" + "github.com/tableauio/tableau/proto/tableaupb" + "google.golang.org/protobuf/compiler/protogen" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" +) + +func getAllOrderedFilesAndMessagers(gen *protogen.Plugin) (protofiles []string, fileMessagers map[string][]string) { + fileMessagers = map[string][]string{} + for _, f := range gen.Files { + if !f.Generate { + continue + } + opts := f.Desc.Options().(*descriptorpb.FileOptions) + workbook := proto.GetExtension(opts, tableaupb.E_Workbook).(*tableaupb.WorkbookOptions) + if workbook == nil { + continue + } + protofiles = append(protofiles, f.GeneratedFilenamePrefix) + var messagers []string + for _, message := range f.Messages { + opts, ok := message.Desc.Options().(*descriptorpb.MessageOptions) + if !ok { + gen.Error(errors.New("get message options failed")) + } + worksheet, ok := proto.GetExtension(opts, tableaupb.E_Worksheet).(*tableaupb.WorksheetOptions) + if !ok { + gen.Error(errors.New("get worksheet extension failed")) + } + if worksheet != nil { + messagerName := string(message.Desc.Name()) + messagers = append(messagers, messagerName) + } + } + // sort messagers in one file to keep in order + sort.Strings(messagers) + fileMessagers[f.GeneratedFilenamePrefix] = messagers + } + // sort all files to keep in order + sort.Strings(protofiles) + return +} + +// generateHub generates related hub files. +func generateHub(gen *protogen.Plugin) { + if *registryShards <= 1 { + protofiles, fileMessagers := getAllOrderedFilesAndMessagers(gen) + + hppFilename := "hub." + pcExt + ".h" + g1 := gen.NewGeneratedFile(hppFilename, "") + helper.GenerateCommonHeader(gen, g1, version) + g1.P() + g1.P(hubHpp) + generateHubHppTplSpec(gen, g1, protofiles, fileMessagers) + g1.P(msgContainerHpp) + generateHubHppMsgContainerMembers(gen, g1, protofiles, fileMessagers) + g1.P(registryHpp) + g1.P(bottomHpp) + + cppFilename := "hub." + pcExt + ".cc" + g2 := gen.NewGeneratedFile(cppFilename, "") + helper.GenerateCommonHeader(gen, g2, version) + g2.P() + g2.P(hubCppHeader) + generateHubCppHeader(gen, g2, protofiles, fileMessagers) + g2.P(hubCpp) + generateHubCppTplSpec(gen, g2, protofiles, fileMessagers) + g2.P(msgContainerCpp) + generateHubCppMsgContainerCtor(gen, g2, protofiles, fileMessagers) + g2.P(registryCpp) + generateHubCppRegistry(gen, g2, protofiles, fileMessagers) + g2.P(bottomCpp) + } else { + // sharding + generateShardedHub(gen) + } +} + +func generateHubHppTplSpec(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P("template <>") + g.P("const std::shared_ptr<", messager, "> Hub::Get<", messager, ">() const;") + g.P() + } + } +} + +func generateHubHppMsgContainerMembers(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P(" std::shared_ptr<", messager, "> ", strcase.ToSnake(messager), "_;") + } + } +} + +func generateHubCppHeader(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { + for _, proto := range protofiles { + g.P(`#include "`, proto, ".", pcExt, `.h"`) + } + g.P() +} + +func generateHubCppTplSpec(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P("template <>") + g.P("const std::shared_ptr<", messager, "> Hub::Get<", messager, ">() const {;") + g.P(" return GetMessagerContainer()->", strcase.ToSnake(messager), "_;") + g.P("}") + g.P() + } + } +} + +func generateHubCppMsgContainerCtor(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P(" ", strcase.ToSnake(messager), "_ = std::dynamic_pointer_cast<", messager, `>((*msger_map_)["`, messager, `"]);`) + } + } +} + +func generateHubCppRegistry(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P(" Register<", messager, ">();") + } + } +} + +const hubHpp = `#pragma once +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "load.pc.h" +#include "messager.pc.h" +#include "scheduler.pc.h" + +namespace tableau { +class MessagerContainer; +class Hub; + +// Auto-generated declarations below +class HeroBaseConf; +class HeroConf; +class ItemConf; +class PatchMergeConf; +class PatchReplaceConf; +class RecursivePatchConf; +class ActivityConf; +class ChapterConf; +class ThemeConf; + +using MessagerMap = std::unordered_map>; +// FilterFunc filter in messagers if returned value is true. +// NOTE: name is the protobuf message name, e.g.: "message ItemConf{...}". +using Filter = std::function; +using MessagerContainerProvider = std::function()>; + +struct HubOptions { + // Filter can only filter in certain specific messagers based on the + // condition that you provide. + Filter filter; + // Provide custom MessagerContainer. For keeping configuration access + // consistent in a coroutine or a transaction. + MessagerContainerProvider provider; +}; + +class Hub { + public: + Hub(const HubOptions* options = nullptr) + : msger_container_(std::make_shared()), options_(options ? *options : HubOptions{}) {} + /***** Synchronous Loading *****/ + // Load fills messages (in MessagerContainer) from files in the specified directory and format. + bool Load(const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); + + /***** Asynchronous Loading *****/ + // Load configs into temp MessagerContainer, and you should call LoopOnce() in you app's main loop, + // in order to take the temp MessagerContainer into effect. + bool AsyncLoad(const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); + int LoopOnce(); + // You'd better initialize the scheduler in the main thread. + void InitScheduler(); + + /***** MessagerMap *****/ + std::shared_ptr GetMessagerMap() const; + void SetMessagerMap(std::shared_ptr msger_map); + + /***** MessagerContainer *****/ + // This function is exposed only for use in MessagerContainerProvider. + std::shared_ptr GetMessagerContainer() const { + if (options_.provider != nullptr) { + return options_.provider(); + } + return msger_container_; + } + + /***** Access APIs *****/ + template + const std::shared_ptr Get() const; + + template + const U* Get(Args... args) const; + + template + const U* GetOrderedMap(Args... args) const; + + // GetLastLoadedTime returns the time when hub's msger_container_ was last set. + inline std::time_t GetLastLoadedTime() const; + + private: + std::shared_ptr InternalLoad(const std::string& dir, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr) const; + std::shared_ptr NewMessagerMap() const; + const std::shared_ptr GetMessager(const std::string& name) const; + + bool Postprocess(std::shared_ptr msger_map); + + private: + // For thread-safe guarantee during configuration updating. + std::mutex mutex_; + // All messagers' container. + std::shared_ptr msger_container_; + // Loading scheduler. + internal::Scheduler* sched_ = nullptr; + // Hub options + const HubOptions options_; +}; + +template +const std::shared_ptr Hub::Get() const { + auto msg = GetMessager(T::Name()); + return std::dynamic_pointer_cast(msg); +} + +template +const U* Hub::Get(Args... args) const { + auto msger = Get(); + return msger ? msger->Get(args...) : nullptr; +} + +template +const U* Hub::GetOrderedMap(Args... args) const { + auto msger = Get(); + return msger ? msger->GetOrderedMap(args...) : nullptr; +} +` + +const msgContainerHpp = `class MessagerContainer { + friend class Hub; + + public: + MessagerContainer(std::shared_ptr msger_map = nullptr); + + private: + std::shared_ptr msger_map_; + std::time_t last_loaded_time_; + + private:` + +const registryHpp = `}; + +using MessagerGenerator = std::function()>; +// messager name -> messager generator +using Registrar = std::unordered_map; +class Registry { + friend class Hub; + + public: + static void Init(); + + template + static void Register(); + + private:` + +const bottomHpp = ` static Registrar registrar; +}; + +template +void Registry::Register() { + registrar[T::Name()] = []() { return std::make_shared(); }; +} +} // namespace tableau` + +const hubCppHeader = `#include "hub.pc.h" + +#include +#include +#include + +#include + +#include "logger.pc.h" +#include "messager.pc.h" +#include "util.pc.h"` + +const hubCpp = ` +namespace tableau { +Registrar Registry::registrar = Registrar(); + +bool Hub::Load(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { + auto msger_map = InternalLoad(dir, fmt, options); + if (!msger_map) { + return false; + } + bool ok = Postprocess(msger_map); + if (!ok) { + return false; + } + SetMessagerMap(msger_map); + return true; +} + +bool Hub::AsyncLoad(const std::string& dir, Format fmt /* = Format::kJSON */, + const LoadOptions* options /* = nullptr */) { + auto msger_map = InternalLoad(dir, fmt, options); + if (!msger_map) { + return false; + } + bool ok = Postprocess(msger_map); + if (!ok) { + return false; + } + sched_->Dispatch(std::bind(&Hub::SetMessagerMap, this, msger_map)); + return true; +} + +int Hub::LoopOnce() { return sched_->LoopOnce(); } + +void Hub::InitScheduler() { + sched_ = new internal::Scheduler(); + sched_->Current(); +} + +std::shared_ptr Hub::InternalLoad(const std::string& dir, Format fmt /* = Format::kJSON */, + const LoadOptions* options /* = nullptr */) const { + // intercept protobuf error logs + auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); + auto msger_map = NewMessagerMap(); + for (auto iter : *msger_map) { + auto&& name = iter.first; + ATOM_DEBUG("loading %s", name.c_str()); + bool ok = iter.second->Load(dir, fmt, options); + if (!ok) { + ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str()); + // restore to old protobuf log handler + google::protobuf::SetLogHandler(old_handler); + return nullptr; + } + ATOM_DEBUG("loaded %s", name.c_str()); + } + + // restore to old protobuf log handler + google::protobuf::SetLogHandler(old_handler); + return msger_map; +} + +std::shared_ptr Hub::NewMessagerMap() const { + std::shared_ptr msger_map = std::make_shared(); + for (auto&& it : Registry::registrar) { + if (!options_.filter || options_.filter(it.first)) { + (*msger_map)[it.first] = it.second(); + } + } + return msger_map; +} + +std::shared_ptr Hub::GetMessagerMap() const { return GetMessagerContainer()->msger_map_; } + +void Hub::SetMessagerMap(std::shared_ptr msger_map) { + // replace with thread-safe guarantee. + std::unique_lock lock(mutex_); + msger_container_ = std::make_shared(msger_map); +} + +const std::shared_ptr Hub::GetMessager(const std::string& name) const { + auto msger_map = GetMessagerMap(); + if (msger_map) { + auto it = msger_map->find(name); + if (it != msger_map->end()) { + return it->second; + } + } + return nullptr; +} + +bool Hub::Postprocess(std::shared_ptr msger_map) { + // create a temporary hub with messager container for post process + Hub tmp_hub; + tmp_hub.SetMessagerMap(msger_map); + + // messager-level postprocess + for (auto iter : *msger_map) { + auto msger = iter.second; + bool ok = msger->ProcessAfterLoadAll(tmp_hub); + if (!ok) { + SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + msger->Name()); + return false; + } + } + return true; +} + +std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; }` + +const msgContainerCpp = ` +MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) + : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), + last_loaded_time_(std::time(nullptr)) {` + +const registryCpp = `} + +void Registry::Init() {` + +const bottomCpp = `} +} // namespace tableau` diff --git a/cmd/protoc-gen-cpp-tableau-loader/main.go b/cmd/protoc-gen-cpp-tableau-loader/main.go index e780de54..9b9641c4 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/main.go +++ b/cmd/protoc-gen-cpp-tableau-loader/main.go @@ -26,7 +26,7 @@ var registryShards *int const ( ModeDefault = "default" // generate all at once. - ModeRegistry = "registry" // only generate "registry.pc.h/cc" files. + ModeHub = "hub" // only generate "hub.pc.h/cc" files. ModeMessager = "messager" // only generate "*.pc.h/cc" for each .proto files. ) @@ -34,12 +34,12 @@ func main() { var flags flag.FlagSet namespace = flags.String("namespace", "tableau", "tableau namespace") messagerSuffix = flags.String("suffix", "Mgr", "tableau messager name suffix") - mode = flags.String("mode", "default", `available mode: default, registry, and messager. + mode = flags.String("mode", "default", `available mode: default, hub, and messager. - default: generate all at once. - - registry: only generate "registry.pc.h/cc" files. + - hub: only generate "hub.pc.h/cc" files. - messager: only generate "*.pc.h/cc" for each .proto files. `) - registryShards = flags.Int("registry-shards", 1, "count of generated registry files") + registryShards = flags.Int("registry-shards", 1, "count of generated hub & registry files") flag.Parse() protogen.Options{ @@ -54,7 +54,7 @@ func main() { switch *mode { case ModeMessager: generateMessager(gen, f) - case ModeRegistry: + case ModeHub: // pass case ModeDefault: generateMessager(gen, f) @@ -63,12 +63,12 @@ func main() { switch *mode { case ModeDefault: - generateRegistry(gen) + generateHub(gen) generateEmbed(gen) case ModeMessager: // pass - case ModeRegistry: - generateRegistry(gen) + case ModeHub: + generateHub(gen) } return nil }) diff --git a/cmd/protoc-gen-cpp-tableau-loader/registry.go b/cmd/protoc-gen-cpp-tableau-loader/registry.go deleted file mode 100644 index ee99526d..00000000 --- a/cmd/protoc-gen-cpp-tableau-loader/registry.go +++ /dev/null @@ -1,183 +0,0 @@ -package main - -import ( - "errors" - "math" - "sort" - "strconv" - - "github.com/tableauio/loader/cmd/protoc-gen-cpp-tableau-loader/helper" - "github.com/tableauio/tableau/proto/tableaupb" - "google.golang.org/protobuf/compiler/protogen" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -func getAllOrderedFilesAndMessagers(gen *protogen.Plugin) (protofiles []string, fileMessagers map[string][]string) { - fileMessagers = map[string][]string{} - for _, f := range gen.Files { - if !f.Generate { - continue - } - opts := f.Desc.Options().(*descriptorpb.FileOptions) - workbook := proto.GetExtension(opts, tableaupb.E_Workbook).(*tableaupb.WorkbookOptions) - if workbook == nil { - continue - } - protofiles = append(protofiles, f.GeneratedFilenamePrefix) - var messagers []string - for _, message := range f.Messages { - opts, ok := message.Desc.Options().(*descriptorpb.MessageOptions) - if !ok { - gen.Error(errors.New("get message options failed")) - } - worksheet, ok := proto.GetExtension(opts, tableaupb.E_Worksheet).(*tableaupb.WorksheetOptions) - if !ok { - gen.Error(errors.New("get worksheet extension failed")) - } - if worksheet != nil { - messagerName := string(message.Desc.Name()) - messagers = append(messagers, messagerName) - } - } - // sort messagers in one file to keep in order - sort.Strings(messagers) - fileMessagers[f.GeneratedFilenamePrefix] = messagers - } - // sort all files to keep in order - sort.Strings(protofiles) - return -} - -// generateRegistry generates related registry files. -func generateRegistry(gen *protogen.Plugin) { - if *registryShards <= 1 { - hppFilename := "registry." + pcExt + ".h" - g1 := gen.NewGeneratedFile(hppFilename, "") - helper.GenerateCommonHeader(gen, g1, version) - g1.P() - g1.P(registryHppTop) - g1.P(registryHppBottom) - - protofiles, fileMessagers := getAllOrderedFilesAndMessagers(gen) - cppFilename := "registry." + pcExt + ".cc" - g2 := gen.NewGeneratedFile(cppFilename, "") - helper.GenerateCommonHeader(gen, g2, version) - g2.P() - generateRegistryCppFileContent(gen, g2, protofiles, fileMessagers) - } else { - // sharding - generateShardedRegistry(gen) - } -} - -// generateRegistryCppFileContent generates the registry logic. -func generateRegistryCppFileContent(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { - g.P(`#include "`, "registry.", pcExt, `.h"`) - g.P() - for _, proto := range protofiles { - g.P(`#include "`, proto, ".", pcExt, `.h"`) - } - g.P() - - g.P("namespace ", *namespace, " {") - g.P("Registrar Registry::registrar = Registrar();") - g.P("void Registry::Init() {") - for _, messagers := range fileMessagers { - for _, messager := range messagers { - g.P(" Register<", messager, ">();") - } - } - g.P("}") - g.P("} // namespace ", *namespace) -} - -// generateShardedRegistry generates related registry files. -func generateShardedRegistry(gen *protogen.Plugin) { - protofiles, fileMessagers := getAllOrderedFilesAndMessagers(gen) - hppFilename := "registry." + pcExt + ".h" - g1 := gen.NewGeneratedFile(hppFilename, "") - helper.GenerateCommonHeader(gen, g1, version) - g1.P() - g1.P(registryHppTop) - - cppFilename := "registry." + pcExt + ".cc" - g2 := gen.NewGeneratedFile(cppFilename, "") - helper.GenerateCommonHeader(gen, g2, version) - g2.P() - g2.P(`#include "`, "registry.", pcExt, `.h"`) - g2.P() - - g2.P("namespace ", *namespace, " {") - g2.P("Registrar Registry::registrar = Registrar();") - g2.P("void Registry::Init() {") - - shardSize := int(math.Ceil(float64(len(protofiles)) / float64((*registryShards)))) - for i := 0; i < *registryShards; i++ { - cursor := (i + 1) * shardSize - if cursor > len(protofiles) { - cursor = len(protofiles) - } - if i*shardSize >= cursor { - break - } - g1.P(" static void InitShard", i, "();") - g2.P(" InitShard", i, "();") - cppFilename := "registry_shard" + strconv.Itoa(i) + "." + pcExt + ".cc" - g := gen.NewGeneratedFile(cppFilename, "") - helper.GenerateCommonHeader(gen, g, version) - g.P() - shardedProtofiles := protofiles[i*shardSize : cursor] - generateShardedRegistryCppFileContent(gen, g, i, shardedProtofiles, fileMessagers) - } - - g1.P(registryHppBottom) - g2.P("}") - g2.P("} // namespace ", *namespace) -} - -// generateShardedRegistryCppFileContent generates one registry shard logic. -func generateShardedRegistryCppFileContent(gen *protogen.Plugin, g *protogen.GeneratedFile, shardIndex int, protofiles []string, fileMessagers map[string][]string) { - g.P(`#include "`, "registry.", pcExt, `.h"`) - g.P() - for _, proto := range protofiles { - g.P(`#include "`, proto, ".", pcExt, `.h"`) - } - g.P() - - g.P("namespace ", *namespace, " {") - g.P("void Registry::InitShard", shardIndex, "() {") - for _, proto := range protofiles { - messagers := fileMessagers[proto] - for _, messager := range messagers { - g.P(" Register<", messager, ">();") - } - } - - g.P("}") - g.P("} // namespace ", *namespace) -} - -const registryHppTop = `#pragma once -#include "messager.pc.h" -namespace tableau { -using MessagerGenerator = std::function()>; -// messager name -> messager generator -using Registrar = std::unordered_map; -class Registry { - public: - static void Init();` - -const registryHppBottom = ` - template - static void Register(); - - static Registrar registrar; -}; - -template -void Registry::Register() { - registrar[T::Name()] = []() { return std::make_shared(); }; -} - -} // namespace tableau` diff --git a/cmd/protoc-gen-cpp-tableau-loader/sharded.go b/cmd/protoc-gen-cpp-tableau-loader/sharded.go new file mode 100644 index 00000000..3f3fd1cd --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/sharded.go @@ -0,0 +1,127 @@ +package main + +import ( + "math" + "strconv" + + "github.com/iancoleman/strcase" + "github.com/tableauio/loader/cmd/protoc-gen-cpp-tableau-loader/helper" + "google.golang.org/protobuf/compiler/protogen" +) + +// generateShardedHub generates related hub files. +func generateShardedHub(gen *protogen.Plugin) { + protofiles, fileMessagers := getAllOrderedFilesAndMessagers(gen) + + // detect real shard num + realShardNum := *registryShards + if realShardNum > len(protofiles) { + realShardNum = len(protofiles) + } + shardSize := int(math.Ceil(float64(len(protofiles)) / float64((realShardNum)))) + + hppFilename := "hub." + pcExt + ".h" + g1 := gen.NewGeneratedFile(hppFilename, "") + helper.GenerateCommonHeader(gen, g1, version) + g1.P() + g1.P(hubHpp) + generateHubHppTplSpec(gen, g1, protofiles, fileMessagers) + g1.P(msgContainerHpp) + generateShardedHubHppMsgContainerShards(gen, g1, realShardNum) + generateHubHppMsgContainerMembers(gen, g1, protofiles, fileMessagers) + g1.P(registryHpp) + generateShardedHubHppRegistryShards(gen, g1, realShardNum) + g1.P(bottomHpp) + + cppFilename := "hub." + pcExt + ".cc" + g2 := gen.NewGeneratedFile(cppFilename, "") + helper.GenerateCommonHeader(gen, g2, version) + g2.P() + g2.P(hubCppHeader) + g2.P(hubCpp) + g2.P(msgContainerCpp) + generateShardedHubCppMsgContainerShards(gen, g2, realShardNum) + g2.P(registryCpp) + generateShardedHubCppRegistryShards(gen, g2, realShardNum) + g2.P(bottomCpp) + + for i := 0; i < realShardNum; i++ { + cursor := (i + 1) * shardSize + if cursor > len(protofiles) { + cursor = len(protofiles) + } + cppFilename := "hub_shard" + strconv.Itoa(i) + "." + pcExt + ".cc" + g := gen.NewGeneratedFile(cppFilename, "") + helper.GenerateCommonHeader(gen, g, version) + g.P() + shardedProtofiles := protofiles[i*shardSize : cursor] + generateShardedHubCppFileContent(gen, g, i, shardedProtofiles, fileMessagers) + } +} + +func generateShardedHubHppMsgContainerShards(gen *protogen.Plugin, g *protogen.GeneratedFile, shardNum int) { + for i := 0; i < shardNum; i++ { + g.P(" void InitShard", i, "();") + } + g.P() + g.P(" private:") +} + +func generateShardedHubHppRegistryShards(gen *protogen.Plugin, g *protogen.GeneratedFile, shardNum int) { + for i := 0; i < shardNum; i++ { + g.P(" static void InitShard", i, "();") + } + g.P() + g.P(" private:") +} + +func generateShardedHubCppMsgContainerShards(gen *protogen.Plugin, g *protogen.GeneratedFile, shardNum int) { + for i := 0; i < shardNum; i++ { + g.P(" InitShard", i, "();") + } +} + +func generateShardedHubCppRegistryShards(gen *protogen.Plugin, g *protogen.GeneratedFile, shardNum int) { + for i := 0; i < shardNum; i++ { + g.P(" InitShard", i, "();") + } +} + +func generateShardedHubCppFileContent(gen *protogen.Plugin, g *protogen.GeneratedFile, shardIndex int, protofiles []string, fileMessagers map[string][]string) { + g.P(`#include "`, "hub.", pcExt, `.h"`) + g.P() + for _, proto := range protofiles { + g.P(`#include "`, proto, ".", pcExt, `.h"`) + } + g.P() + g.P("namespace ", *namespace, " {") + + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P("template <>") + g.P("const std::shared_ptr<" + messager + "> Hub::Get<" + messager + ">() const {") + g.P(" return GetMessagerContainer()->", strcase.ToSnake(messager), "_;") + g.P("}") + g.P() + } + } + + g.P("void MessagerContainer::InitShard", shardIndex, "() {") + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P(" ", strcase.ToSnake(messager), "_ = std::dynamic_pointer_cast<", messager, `>((*msger_map_)["`, messager, `"]);`) + } + } + g.P("}") + g.P() + + g.P("void Registry::InitShard", shardIndex, "() {") + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P(" Register<", messager, ">();") + } + } + g.P("}") + + g.P("} // namespace ", *namespace) +} diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index 20ccbb26..a39c0353 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -126,13 +126,11 @@ std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), last_loaded_time_(std::time(nullptr)) { - // Auto-generated shards below InitShard0(); InitShard1(); } void Registry::Init() { - // Auto-generated shards below InitShard0(); InitShard1(); } diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index 064d4b28..e5a7700d 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -127,7 +127,6 @@ const U* Hub::GetOrderedMap(Args... args) const { return msger ? msger->GetOrderedMap(args...) : nullptr; } -// Auto-generated template specializations below template <> const std::shared_ptr Hub::Get() const; @@ -162,14 +161,14 @@ class MessagerContainer { MessagerContainer(std::shared_ptr msger_map = nullptr); private: - // Auto-generated shards below + std::shared_ptr msger_map_; + std::time_t last_loaded_time_; + + private: void InitShard0(); void InitShard1(); private: - std::shared_ptr msger_map_; - std::time_t last_loaded_time_; - // Auto-generated all messagers as fields for fast access below std::shared_ptr hero_base_conf_; std::shared_ptr hero_conf_; std::shared_ptr item_conf_; @@ -194,7 +193,6 @@ class Registry { static void Register(); private: - // Auto-generated shards below static void InitShard0(); static void InitShard1(); diff --git a/test/cpp-tableau-loader/src/protoconf/registry_shard0.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub_shard0.pc.cc similarity index 100% rename from test/cpp-tableau-loader/src/protoconf/registry_shard0.pc.cc rename to test/cpp-tableau-loader/src/protoconf/hub_shard0.pc.cc index 7fb9a3af..157445b0 100644 --- a/test/cpp-tableau-loader/src/protoconf/registry_shard0.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub_shard0.pc.cc @@ -9,18 +9,6 @@ #include "item_conf.pc.h" namespace tableau { -void Registry::InitShard0() { - Register(); - Register(); - Register(); -} - -void MessagerContainer::InitShard0() { - hero_base_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroBaseConf"]); - hero_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroConf"]); - item_conf_ = std::dynamic_pointer_cast((*msger_map_)["ItemConf"]); -} - template <> const std::shared_ptr Hub::Get() const { return GetMessagerContainer()->hero_base_conf_; @@ -35,4 +23,16 @@ template <> const std::shared_ptr Hub::Get() const { return GetMessagerContainer()->item_conf_; } + +void MessagerContainer::InitShard0() { + hero_base_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroBaseConf"]); + hero_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroConf"]); + item_conf_ = std::dynamic_pointer_cast((*msger_map_)["ItemConf"]); +} + +void Registry::InitShard0() { + Register(); + Register(); + Register(); +} } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/registry_shard1.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub_shard1.pc.cc similarity index 100% rename from test/cpp-tableau-loader/src/protoconf/registry_shard1.pc.cc rename to test/cpp-tableau-loader/src/protoconf/hub_shard1.pc.cc index e0c1a479..fdaf74b6 100644 --- a/test/cpp-tableau-loader/src/protoconf/registry_shard1.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub_shard1.pc.cc @@ -9,24 +9,6 @@ #include "test_conf.pc.h" namespace tableau { -void Registry::InitShard1() { - Register(); - Register(); - Register(); - Register(); - Register(); - Register(); -} - -void MessagerContainer::InitShard1() { - patch_merge_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchMergeConf"]); - patch_replace_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchReplaceConf"]); - recursive_patch_conf_ = std::dynamic_pointer_cast((*msger_map_)["RecursivePatchConf"]); - activity_conf_ = std::dynamic_pointer_cast((*msger_map_)["ActivityConf"]); - chapter_conf_ = std::dynamic_pointer_cast((*msger_map_)["ChapterConf"]); - theme_conf_ = std::dynamic_pointer_cast((*msger_map_)["ThemeConf"]); -} - template <> const std::shared_ptr Hub::Get() const { return GetMessagerContainer()->patch_merge_conf_; @@ -56,4 +38,22 @@ template <> const std::shared_ptr Hub::Get() const { return GetMessagerContainer()->theme_conf_; } + +void MessagerContainer::InitShard1() { + patch_merge_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchMergeConf"]); + patch_replace_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchReplaceConf"]); + recursive_patch_conf_ = std::dynamic_pointer_cast((*msger_map_)["RecursivePatchConf"]); + activity_conf_ = std::dynamic_pointer_cast((*msger_map_)["ActivityConf"]); + chapter_conf_ = std::dynamic_pointer_cast((*msger_map_)["ChapterConf"]); + theme_conf_ = std::dynamic_pointer_cast((*msger_map_)["ThemeConf"]); +} + +void Registry::InitShard1() { + Register(); + Register(); + Register(); + Register(); + Register(); + Register(); +} } // namespace tableau From d64c0f7650f5a98d140f43dd2271849488128cf7 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 25 Apr 2025 17:48:23 +0800 Subject: [PATCH 12/14] fix: generated code --- cmd/protoc-gen-cpp-tableau-loader/hub.go | 12 +---------- .../cpp-tableau-loader/src/protoconf/hub.pc.h | 20 +++++++++---------- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/hub.go b/cmd/protoc-gen-cpp-tableau-loader/hub.go index 4838522f..666ebccb 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/hub.go +++ b/cmd/protoc-gen-cpp-tableau-loader/hub.go @@ -86,6 +86,7 @@ func generateHub(gen *protogen.Plugin) { func generateHubHppTplSpec(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { for _, proto := range protofiles { for _, messager := range fileMessagers[proto] { + g.P("class ", messager, ";") g.P("template <>") g.P("const std::shared_ptr<", messager, "> Hub::Get<", messager, ">() const;") g.P() @@ -155,17 +156,6 @@ namespace tableau { class MessagerContainer; class Hub; -// Auto-generated declarations below -class HeroBaseConf; -class HeroConf; -class ItemConf; -class PatchMergeConf; -class PatchReplaceConf; -class RecursivePatchConf; -class ActivityConf; -class ChapterConf; -class ThemeConf; - using MessagerMap = std::unordered_map>; // FilterFunc filter in messagers if returned value is true. // NOTE: name is the protobuf message name, e.g.: "message ItemConf{...}". diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index e5a7700d..62fa5779 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -22,17 +22,6 @@ namespace tableau { class MessagerContainer; class Hub; -// Auto-generated declarations below -class HeroBaseConf; -class HeroConf; -class ItemConf; -class PatchMergeConf; -class PatchReplaceConf; -class RecursivePatchConf; -class ActivityConf; -class ChapterConf; -class ThemeConf; - using MessagerMap = std::unordered_map>; // FilterFunc filter in messagers if returned value is true. // NOTE: name is the protobuf message name, e.g.: "message ItemConf{...}". @@ -127,30 +116,39 @@ const U* Hub::GetOrderedMap(Args... args) const { return msger ? msger->GetOrderedMap(args...) : nullptr; } +class HeroBaseConf; template <> const std::shared_ptr Hub::Get() const; +class HeroConf; template <> const std::shared_ptr Hub::Get() const; +class ItemConf; template <> const std::shared_ptr Hub::Get() const; +class PatchMergeConf; template <> const std::shared_ptr Hub::Get() const; +class PatchReplaceConf; template <> const std::shared_ptr Hub::Get() const; +class RecursivePatchConf; template <> const std::shared_ptr Hub::Get() const; +class ActivityConf; template <> const std::shared_ptr Hub::Get() const; +class ChapterConf; template <> const std::shared_ptr Hub::Get() const; +class ThemeConf; template <> const std::shared_ptr Hub::Get() const; From 8cc4765c30df66dd3f6c8cc9710e3dfcb7e7a52d Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 25 Apr 2025 19:56:50 +0800 Subject: [PATCH 13/14] feat: cr --- cmd/protoc-gen-cpp-tableau-loader/hub.go | 2 +- cmd/protoc-gen-cpp-tableau-loader/main.go | 6 +++--- cmd/protoc-gen-cpp-tableau-loader/{sharded.go => shard.go} | 2 +- test/cpp-tableau-loader/gen.sh | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) rename cmd/protoc-gen-cpp-tableau-loader/{sharded.go => shard.go} (99%) diff --git a/cmd/protoc-gen-cpp-tableau-loader/hub.go b/cmd/protoc-gen-cpp-tableau-loader/hub.go index 666ebccb..a181f686 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/hub.go +++ b/cmd/protoc-gen-cpp-tableau-loader/hub.go @@ -50,7 +50,7 @@ func getAllOrderedFilesAndMessagers(gen *protogen.Plugin) (protofiles []string, // generateHub generates related hub files. func generateHub(gen *protogen.Plugin) { - if *registryShards <= 1 { + if *shards <= 1 { protofiles, fileMessagers := getAllOrderedFilesAndMessagers(gen) hppFilename := "hub." + pcExt + ".h" diff --git a/cmd/protoc-gen-cpp-tableau-loader/main.go b/cmd/protoc-gen-cpp-tableau-loader/main.go index 9b9641c4..fcb20e5b 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/main.go +++ b/cmd/protoc-gen-cpp-tableau-loader/main.go @@ -21,8 +21,8 @@ var messagerSuffix *string // mode control generating rules for better dependency management. var mode *string -// count of generated registry files, aimed to boost compiling speed. -var registryShards *int +// count of generated hub cpp files, aimed to boost compiling speed. +var shards *int const ( ModeDefault = "default" // generate all at once. @@ -39,7 +39,7 @@ func main() { - hub: only generate "hub.pc.h/cc" files. - messager: only generate "*.pc.h/cc" for each .proto files. `) - registryShards = flags.Int("registry-shards", 1, "count of generated hub & registry files") + shards = flags.Int("shards", 1, "count of generated hub cpp files for distributed compiling speed-up") flag.Parse() protogen.Options{ diff --git a/cmd/protoc-gen-cpp-tableau-loader/sharded.go b/cmd/protoc-gen-cpp-tableau-loader/shard.go similarity index 99% rename from cmd/protoc-gen-cpp-tableau-loader/sharded.go rename to cmd/protoc-gen-cpp-tableau-loader/shard.go index 3f3fd1cd..84605516 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/sharded.go +++ b/cmd/protoc-gen-cpp-tableau-loader/shard.go @@ -14,7 +14,7 @@ func generateShardedHub(gen *protogen.Plugin) { protofiles, fileMessagers := getAllOrderedFilesAndMessagers(gen) // detect real shard num - realShardNum := *registryShards + realShardNum := *shards if realShardNum > len(protofiles) { realShardNum = len(protofiles) } diff --git a/test/cpp-tableau-loader/gen.sh b/test/cpp-tableau-loader/gen.sh index 97bb72b7..43d83360 100755 --- a/test/cpp-tableau-loader/gen.sh +++ b/test/cpp-tableau-loader/gen.sh @@ -26,7 +26,7 @@ export PATH="${PATH}:${PLGUIN_DIR}" ${PROTOC} \ --cpp-tableau-loader_out="$PROTOCONF_OUT" \ - --cpp-tableau-loader_opt=paths=source_relative,registry-shards=2 \ + --cpp-tableau-loader_opt=paths=source_relative,shards=2 \ --cpp_out="$PROTOCONF_OUT" \ --proto_path="$PROTOBUF_PROTO" \ --proto_path="$TABLEAU_PROTO" \ From c009dbfc807ff9db267d9c344b489ac702416cf3 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Tue, 29 Apr 2025 20:42:46 +0800 Subject: [PATCH 14/14] feat: support c++17 --- README.md | 2 +- .../embed/util.pc.cc | 24 +++++++++++++++ test/cpp-tableau-loader/src/CMakeLists.txt | 29 +++++++++++++++++-- .../src/protoconf/util.pc.cc | 24 +++++++++++++++ 4 files changed, 75 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 029e5879..af2d1f22 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). - Change dir: `cd test/cpp-tableau-loader` - Generate protoconf: `bash ./gen.sh` - Create build dir: `mkdir build && cd build` -- Run cmake: `cmake ../src/` +- Run cmake: `cmake ../src/`(use c++11) or `cmake -DUSE_CPP17=ON ../src/`(use c++17) - Build: `make -j8`, then the **bin** dir will be generated at `test/cpp-tableau-loader/bin`. ### References diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc index e2abd361..72cbc824 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc @@ -7,12 +7,17 @@ #include #include +#if __cplusplus >= 201703L +#include +namespace fs = std::filesystem; +#else #ifdef _WIN32 #include #include #else #include #endif +#endif #include "load.pc.h" #include "logger.pc.h" @@ -37,6 +42,16 @@ const std::string kBinExt = ".bin"; namespace util { int Mkdir(const std::string& path) { +#if __cplusplus >= 201703L + std::error_code ec; + if (!fs::create_directories(path, ec)) { + if (ec) { + std::cerr << "system error: " << ec.message() << std::endl; + return -1; + } + } + return 0; +#else std::string path_ = path + kPathSeperator; struct stat info; for (size_t pos = path_.find(kPathSeperator, 0); pos != std::string::npos; pos = path_.find(kPathSeperator, pos)) { @@ -52,20 +67,29 @@ int Mkdir(const std::string& path) { } } return 0; +#endif } std::string GetDir(const std::string& path) { +#if __cplusplus >= 201703L + return fs::path(path).parent_path().string(); +#else size_t pos = path.find_last_of(kPathSeperator); if (pos != std::string::npos) { return path.substr(0, pos); } return kEmpty; +#endif } bool ExistsFile(const std::string& filename) { +#if __cplusplus >= 201703L + return fs::exists(filename); +#else std::ifstream file(filename); // returns true if the file exists and is accessible return file.good(); +#endif } bool ReadFile(const std::string& filename, std::string& content) { diff --git a/test/cpp-tableau-loader/src/CMakeLists.txt b/test/cpp-tableau-loader/src/CMakeLists.txt index 3f054767..d8f72795 100644 --- a/test/cpp-tableau-loader/src/CMakeLists.txt +++ b/test/cpp-tableau-loader/src/CMakeLists.txt @@ -5,13 +5,35 @@ project(loader) file(GLOB_RECURSE PROTO_SOURCE *.cc) file(GLOB_RECURSE SOURCE *.cpp) +# Add option to switch between C++11 and C++17 +option(USE_CPP17 "Use C++17 standard (otherwise use C++11)" OFF) + # specify the C++ standard -set(CMAKE_CXX_STANDARD 11) +if(USE_CPP17) + set(CMAKE_CXX_STANDARD 17) + message(STATUS "Using C++17 standard") +else() + set(CMAKE_CXX_STANDARD 11) + message(STATUS "Using C++11 standard") +endif() set(CMAKE_CXX_STANDARD_REQUIRED True) + if (MSVC) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /DNDEBUG") + if(USE_CPP17) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /DNDEBUG /std:c++17") + else() + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /DNDEBUG /std:c++11") + endif() else() - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -fPIC -std=c++11 -Wno-deprecated -Wno-unused-variable -Wno-sign-compare -Wno-strict-aliasing -fno-strict-aliasing -DNDEBUG") + if(USE_CPP17) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -fPIC -std=c++17 -Wno-deprecated -Wno-unused-variable -Wno-sign-compare -Wno-strict-aliasing -fno-strict-aliasing -DNDEBUG") + # For older GCC versions that need explicit linking with stdc++fs + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9.0") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lstdc++fs") + endif() + else() + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -fPIC -std=c++11 -Wno-deprecated -Wno-unused-variable -Wno-sign-compare -Wno-strict-aliasing -fno-strict-aliasing -DNDEBUG") + endif() endif() # root dir define @@ -43,6 +65,7 @@ else() SET(COMMON_LIB ${PROTOBUF_LIB}/libprotobuf.a pthread + $<$:stdc++fs> ) endif() diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.cc b/test/cpp-tableau-loader/src/protoconf/util.pc.cc index b86442f9..9266d816 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.cc @@ -12,12 +12,17 @@ #include #include +#if __cplusplus >= 201703L +#include +namespace fs = std::filesystem; +#else #ifdef _WIN32 #include #include #else #include #endif +#endif #include "load.pc.h" #include "logger.pc.h" @@ -42,6 +47,16 @@ const std::string kBinExt = ".bin"; namespace util { int Mkdir(const std::string& path) { +#if __cplusplus >= 201703L + std::error_code ec; + if (!fs::create_directories(path, ec)) { + if (ec) { + std::cerr << "system error: " << ec.message() << std::endl; + return -1; + } + } + return 0; +#else std::string path_ = path + kPathSeperator; struct stat info; for (size_t pos = path_.find(kPathSeperator, 0); pos != std::string::npos; pos = path_.find(kPathSeperator, pos)) { @@ -57,20 +72,29 @@ int Mkdir(const std::string& path) { } } return 0; +#endif } std::string GetDir(const std::string& path) { +#if __cplusplus >= 201703L + return fs::path(path).parent_path().string(); +#else size_t pos = path.find_last_of(kPathSeperator); if (pos != std::string::npos) { return path.substr(0, pos); } return kEmpty; +#endif } bool ExistsFile(const std::string& filename) { +#if __cplusplus >= 201703L + return fs::exists(filename); +#else std::ifstream file(filename); // returns true if the file exists and is accessible return file.good(); +#endif } bool ReadFile(const std::string& filename, std::string& content) {