diff --git a/README.md b/README.md index 029e5879..af2d1f22 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). - Change dir: `cd test/cpp-tableau-loader` - Generate protoconf: `bash ./gen.sh` - Create build dir: `mkdir build && cd build` -- Run cmake: `cmake ../src/` +- Run cmake: `cmake ../src/`(use c++11) or `cmake -DUSE_CPP17=ON ../src/`(use c++17) - Build: `make -j8`, then the **bin** dir will be generated at `test/cpp-tableau-loader/bin`. ### References diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h deleted file mode 100644 index 6253b7e8..00000000 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h +++ /dev/null @@ -1,284 +0,0 @@ -#pragma once -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace tableau { -enum class Format { - kUnknown, - kJSON, - kText, - kBin, -}; - -enum class LoadMode { - kModeDefault, - kModeOnlyMain, - kModeOnlyPatch, -}; - -extern const std::string kUnknownExt; -extern const std::string kJSONExt; -extern const std::string kTextExt; -extern const std::string kBinExt; - -static const std::string kEmpty = ""; -const std::string& GetErrMsg(); - -class Messager; -class Hub; - -using MessagerMap = std::unordered_map>; -using MessagerContainer = std::shared_ptr; -// FilterFunc filter in messagers if returned value is true. -// NOTE: name is the protobuf message name, e.g.: "message ItemConf{...}". -using Filter = std::function; -using MessagerContainerProvider = std::function; -using Postprocessor = std::function; -// ReadFunc reads the config file and returns its content. -using ReadFunc = std::function; - -struct HubOptions { - // Filter can only filter in certain specific messagers based on the - // condition that you provide. - Filter filter; -}; - -struct LoadOptions { - // postprocessor is called after loading all configurations. - Postprocessor postprocessor; - // read_func reads the config file and returns its content. - ReadFunc read_func; - // Whether to ignore unknown JSON fields during parsing. - // - // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. - bool ignore_unknown_fields = false; - // Paths maps each messager name to a corresponding config file path. - // If specified, then the main messager will be parsed from the file - // directly, other than the specified load dir. - std::unordered_map paths; - // Patch paths maps each messager name to one or multiple corresponding patch file paths. - // If specified, then main messager will be patched. - std::unordered_map> patch_paths; - // Patch dirs specifies the directory paths for config patching. - std::vector patch_dirs; - // Mode specifies the loading mode for config patching. - LoadMode mode = LoadMode::kModeDefault; -}; - -// Convert file extension to Format type. -// NOTE: ext includes dot ".", such as: -// - kJSONExt：".json" -// - kTextExt".txt" -// - kBinExt".bin" -Format Ext2Format(const std::string& ext); -// Empty string will be returned if an unsupported enum value has been passed, -// and the error message can be obtained by GetErrMsg(). -const std::string& Format2Ext(Format fmt); -bool Message2JSON(const google::protobuf::Message& msg, std::string& json); -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options = nullptr); -bool Text2Message(const std::string& text, google::protobuf::Message& msg); -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); -void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg); -const std::string& GetProtoName(const google::protobuf::Message& msg); -bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr); -bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr); - -namespace internal { -class Scheduler { - public: - typedef std::function Job; - - public: - Scheduler() : thread_id_(std::this_thread::get_id()) {} - static Scheduler& Current(); - // thread-safety - void Post(const Job& job); - void Dispatch(const Job& job); - int LoopOnce(); - bool IsLoopThread() const; - void AssertInLoopThread() const; - - private: - std::thread::id thread_id_; - std::mutex mutex_; - std::vector jobs_; -}; - -bool Postprocess(Postprocessor postprocessor, MessagerContainer container); - -} // namespace internal - -class Messager { - public: - struct Stats { - std::chrono::microseconds duration; // total load time consuming. - // TODO: crc32 of config file to decide whether changed or not - // std::string crc32; - // int64_t last_modified_time = 0; // unix timestamp - }; - - public: - virtual ~Messager() = default; - static const std::string& Name() { return kEmpty; } - const Stats& GetStats() { return stats_; } - // Load fills message from file in the specified directory and format. - virtual bool Load(const std::string& dir, Format fmt, const LoadOptions* options = nullptr) = 0; - // Message returns the inner message data. - virtual const google::protobuf::Message* Message() const { return nullptr; } - // callback after all messagers loaded. - virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } - - protected: - // callback after this messager loaded. - virtual bool ProcessAfterLoad() { return true; }; - Stats stats_; -}; - -class Hub { - public: - Hub(const HubOptions* options = nullptr) : options_(options ? *options : HubOptions{}) {} - Hub(MessagerContainer container, const HubOptions* options = nullptr) : options_(options ? *options : HubOptions{}) { - SetMessagerContainer(container); - } - /***** Synchronous Loading *****/ - // Load fills messages (in MessagerContainer) from files in the specified directory and format. - bool Load(const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); - - /***** Asynchronous Loading *****/ - // Load configs into temp MessagerContainer, and you should call LoopOnce() in you app's main loop, - // in order to take the temp MessagerContainer into effect. - bool AsyncLoad(const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); - int LoopOnce(); - // You'd better initialize the scheduler in the main thread. - void InitScheduler(); - - /***** MessagerContainer *****/ - MessagerContainer GetMessagerContainer() const { return msger_container_; } - void SetMessagerContainerProvider(MessagerContainerProvider provider) { msger_container_provider_ = provider; } - - /***** Access APIs *****/ - template - const std::shared_ptr Get() const; - - template - const U* Get(Args... args) const; - - template - const U* GetOrderedMap(Args... args) const; - - // GetLastLoadedTime returns the time when hub's msger_container_ was last set. - inline std::time_t GetLastLoadedTime() const { return last_loaded_time_; } - - private: - MessagerContainer LoadNewMessagerContainer(const std::string& dir, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr); - MessagerContainer NewMessagerContainer(); - void SetMessagerContainer(MessagerContainer msger_container); - MessagerContainer GetMessagerContainerWithProvider() const; - const std::shared_ptr GetMessager(const std::string& name) const; - - private: - // For thread-safe guarantee during configuration updating. - std::mutex mutex_; - // All messagers' container. - MessagerContainer msger_container_; - // Provide custom MessagerContainer. For keeping configuration access - // consistent in a coroutine or a transaction. - MessagerContainerProvider msger_container_provider_; - // Loading scheduler. - internal::Scheduler* sched_ = nullptr; - // Last loaded time - std::time_t last_loaded_time_ = 0; - // Hub options - const HubOptions options_; -}; - -template -const std::shared_ptr Hub::Get() const { - auto msg = GetMessager(T::Name()); - return std::dynamic_pointer_cast(msg); -} - -template -const U* Hub::Get(Args... args) const { - auto msg = GetMessager(T::Name()); - auto msger = std::dynamic_pointer_cast(msg); - return msger ? msger->Get(args...) : nullptr; -} - -template -const U* Hub::GetOrderedMap(Args... args) const { - auto msg = GetMessager(T::Name()); - auto msger = std::dynamic_pointer_cast(msg); - return msger ? msger->GetOrderedMap(args...) : nullptr; -} - -namespace util { - -// Combine hash values -// -// References: -// - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x -// - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key -inline void HashCombine(std::size_t& seed) {} - -template -inline void HashCombine(std::size_t& seed, const T& v, O... others) { - std::hash hasher; - seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); - HashCombine(seed, others...); -} - -template -inline std::size_t SugaredHashCombine(const T& v, O... others) { - std::size_t seed = 0; // start with a hash value 0 - HashCombine(seed, v, others...); - return seed; -} - -// Mkdir makes dir recursively. -int Mkdir(const std::string& path); -// GetDir returns all but the last element of path, typically the path's -// directory. -std::string GetDir(const std::string& path); -// GetExt returns the file name extension used by path. -// The extension is the suffix beginning at the final dot -// in the final element of path; it is empty if there is -// no dot. -std::string GetExt(const std::string& path); - -class TimeProfiler { - protected: - std::chrono::time_point last_; - - public: - TimeProfiler() { Start(); } - void Start() { last_ = std::chrono::steady_clock::now(); } - // Calculate duration between the last time point and now, - // and update last time point to now. - std::chrono::microseconds Elapse() { - auto now = std::chrono::steady_clock::now(); - auto duration = now - last_; // This is of type std::chrono::duration - last_ = now; - return std::chrono::duration_cast(duration); - } -}; - -} // namespace util - -} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc similarity index 50% rename from cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc rename to cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc index d1834ccd..5c60d98b 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc @@ -1,153 +1,13 @@ -#include "hub.pc.h" - -#include -#include -#include - -#ifdef _WIN32 -#include -#include -#else -#include -#endif - -#include -#include -#include -#include +#include "load.pc.h" #include "logger.pc.h" -#include "registry.pc.h" +#include "util.pc.h" namespace tableau { #ifdef _WIN32 #undef GetMessage -#define mkdir(path, mode) _mkdir(path) -static constexpr char kPathSeperator = '\\'; -#else -static constexpr char kPathSeperator = '/'; #endif -static thread_local std::string g_err_msg; -const std::string& GetErrMsg() { return g_err_msg; } - -const std::string kUnknownExt = ".unknown"; -const std::string kJSONExt = ".json"; -const std::string kTextExt = ".txt"; -const std::string kBinExt = ".bin"; - -Format Ext2Format(const std::string& ext) { - if (ext == kJSONExt) { - return Format::kJSON; - } else if (ext == kTextExt) { - return Format::kText; - } else if (ext == kBinExt) { - return Format::kBin; - } else { - return Format::kUnknown; - } -} - -const std::string& Format2Ext(Format fmt) { - switch (fmt) { - case Format::kJSON: - return kJSONExt; - case Format::kText: - return kTextExt; - case Format::kBin: - return kBinExt; - default: - return kUnknownExt; - } -} - -// refer: https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/stubs/logging.h -void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg) { - static const std::unordered_map kLevelMap = {{google::protobuf::LOGLEVEL_INFO, log::kInfo}, - {google::protobuf::LOGLEVEL_WARNING, log::kWarn}, - {google::protobuf::LOGLEVEL_ERROR, log::kError}, - {google::protobuf::LOGLEVEL_FATAL, log::kFatal}}; - log::Level lvl = log::kWarn; // default - auto iter = kLevelMap.find(level); - if (iter != kLevelMap.end()) { - lvl = iter->second; - } - ATOM_LOGGER_CALL(tableau::log::DefaultLogger(), lvl, "[libprotobuf %s:%d] %s", filename, line, msg.c_str()); -} - -const std::string& GetProtoName(const google::protobuf::Message& msg) { - const auto* md = msg.GetDescriptor(); - return md != nullptr ? md->name() : kEmpty; -} - -bool ExistsFile(const std::string& filename) { - std::ifstream file(filename); - // returns true if the file exists and is accessible - return file.good(); -} - -bool ReadFile(const std::string& filename, std::string& content) { - std::ifstream file(filename); - if (!file.is_open()) { - g_err_msg = "failed to open " + filename + ": " + strerror(errno); - return false; - } - std::stringstream ss; - ss << file.rdbuf(); - content = ss.str(); - return true; -} - -std::string GetPatchName(tableau::Patch patch) { - auto* descriptor = tableau::Patch_descriptor(); - if (descriptor) { - auto* value = descriptor->FindValueByNumber(patch); - if (value) { - return value->name(); - } - } - return std::to_string(static_cast(patch)); -} - -bool Message2JSON(const google::protobuf::Message& msg, std::string& json) { - google::protobuf::util::JsonPrintOptions options; - options.add_whitespace = true; - options.always_print_primitive_fields = true; - options.preserve_proto_field_names = true; - return google::protobuf::util::MessageToJsonString(msg, &json, options).ok(); -} - -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options /* = nullptr */) { - google::protobuf::util::Status status; - if (options != nullptr) { - google::protobuf::util::JsonParseOptions parse_options; - parse_options.ignore_unknown_fields = options->ignore_unknown_fields; - status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); - } else { - status = google::protobuf::util::JsonStringToMessage(json, &msg); - } - if (!status.ok()) { - g_err_msg = "failed to parse " + GetProtoName(msg) + kJSONExt + ": " + status.ToString(); - return false; - } - return true; -} - -bool Text2Message(const std::string& text, google::protobuf::Message& msg) { - if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) { - g_err_msg = "failed to parse " + GetProtoName(msg) + kTextExt; - return false; - } - return true; -} -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) { - if (!msg.ParseFromString(bin)) { - g_err_msg = "failed to parse " + GetProtoName(msg) + kBinExt; - return false; - } - return true; -} - // PatchMessage patches src into dst, which must be a message with the same descriptor. // // # Default PatchMessage mechanism @@ -174,7 +34,7 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); // Ensure both messages are of the same type if (dst_descriptor != src_descriptor) { - g_err_msg = "dst and src are not messages with the same descriptor"; + SetErrMsg("dst and src are not messages with the same descriptor"); ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), src_descriptor->name().c_str()); return false; @@ -308,7 +168,7 @@ bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& pat // ignore patch files when LoadMode::kModeOnlyMain specified return LoadMessageByPath(msg, path, fmt, nullptr); } - std::string name = GetProtoName(msg); + std::string name = util::GetProtoName(msg); std::vector patch_paths; auto iter = options->patch_paths.find(name); if (iter != options->patch_paths.end()) { @@ -316,13 +176,13 @@ bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& pat patch_paths = iter->second; } else { for (auto&& patch_dir : options->patch_dirs) { - patch_paths.emplace_back(patch_dir + name + Format2Ext(fmt)); + patch_paths.emplace_back(patch_dir + name + util::Format2Ext(fmt)); } } std::vector existed_patch_paths; for (auto&& patch_path : patch_paths) { - if (ExistsFile(patch_path)) { + if (util::ExistsFile(patch_path)) { existed_patch_paths.emplace_back(patch_path); } } @@ -339,7 +199,7 @@ bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& pat case tableau::PATCH_REPLACE: { // just use the last "patch" file std::string& patch_path = existed_patch_paths.back(); - if (!LoadMessageByPath(msg, patch_path, Ext2Format(util::GetExt(patch_path)), options)) { + if (!LoadMessageByPath(msg, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { return false; } break; @@ -356,7 +216,7 @@ bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& pat std::unique_ptr _auto_release(msg.New()); // load patch_msg from each "patch" file for (auto&& patch_path : existed_patch_paths) { - if (!LoadMessageByPath(*patch_msg_ptr, patch_path, Ext2Format(util::GetExt(patch_path)), options)) { + if (!LoadMessageByPath(*patch_msg_ptr, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { return false; } if (!PatchMessage(msg, *patch_msg_ptr)) { @@ -366,11 +226,11 @@ bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& pat break; } default: { - g_err_msg = "unknown patch type: " + GetPatchName(patch); + SetErrMsg("unknown patch type: " + util::GetPatchName(patch)); return false; } } - ATOM_DEBUG("patched(%s) %s by %s: %s", GetPatchName(patch).c_str(), name.c_str(), + ATOM_DEBUG("patched(%s) %s by %s: %s", util::GetPatchName(patch).c_str(), name.c_str(), ATOM_VECTOR_STR(existed_patch_paths).c_str(), msg.ShortDebugString().c_str()); return true; } @@ -378,7 +238,7 @@ bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& pat bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt, const LoadOptions* options /* = nullptr*/) { std::string content; - ReadFunc read_func = ReadFile; + ReadFunc read_func = util::ReadFile; if (options != nullptr && options->read_func) { read_func = options->read_func; } @@ -388,16 +248,16 @@ bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, } switch (fmt) { case Format::kJSON: { - return JSON2Message(content, msg, options); + return util::JSON2Message(content, msg, options); } case Format::kText: { - return Text2Message(content, msg); + return util::Text2Message(content, msg); } case Format::kBin: { - return Bin2Message(content, msg); + return util::Bin2Message(content, msg); } default: { - g_err_msg = "unknown format: " + std::to_string(static_cast(fmt)); + SetErrMsg("unknown format: " + std::to_string(static_cast(fmt))); return false; } } @@ -405,23 +265,23 @@ bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt, const LoadOptions* options /* = nullptr*/) { - std::string name = GetProtoName(msg); + std::string name = util::GetProtoName(msg); std::string path; if (options) { auto iter = options->paths.find(name); if (iter != options->paths.end()) { // path specified in Paths, then use it instead of dir. path = iter->second; - fmt = Ext2Format(util::GetExt(iter->second)); + fmt = util::Ext2Format(util::GetExt(iter->second)); } } if (path.empty()) { - path = dir + name + Format2Ext(fmt); + path = dir + name + util::Format2Ext(fmt); } const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); if (!descriptor) { - g_err_msg = "failed to get descriptor of message: " + name; + SetErrMsg("failed to get descriptor of message: " + name); return false; } // access the extension directly using the generated identifier @@ -432,214 +292,4 @@ bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format return LoadMessageByPath(msg, path, fmt, options); } - -bool StoreMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt) { - // TODO: write protobuf message to file, support 3 formats: json, text, and bin. - return false; -} - -bool Hub::Load(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { - auto msger_container = LoadNewMessagerContainer(dir, fmt, options); - if (!msger_container) { - return false; - } - bool ok = internal::Postprocess(options->postprocessor, msger_container); - if (!ok) { - return false; - } - SetMessagerContainer(msger_container); - return true; -} - -bool Hub::AsyncLoad(const std::string& dir, Format fmt /* = Format::kJSON */, - const LoadOptions* options /* = nullptr */) { - auto msger_container = LoadNewMessagerContainer(dir, fmt, options); - if (!msger_container) { - return false; - } - bool ok = internal::Postprocess(options->postprocessor, msger_container); - if (!ok) { - return false; - } - sched_->Dispatch(std::bind(&Hub::SetMessagerContainer, this, msger_container)); - return true; -} - -int Hub::LoopOnce() { return sched_->LoopOnce(); } -void Hub::InitScheduler() { - sched_ = new internal::Scheduler(); - sched_->Current(); -} - -MessagerContainer Hub::LoadNewMessagerContainer(const std::string& dir, Format fmt /* = Format::kJSON */, - const LoadOptions* options /* = nullptr */) { - // intercept protobuf error logs - auto old_handler = google::protobuf::SetLogHandler(ProtobufLogHandler); - auto msger_container = NewMessagerContainer(); - for (auto iter : *msger_container) { - auto&& name = iter.first; - ATOM_DEBUG("loading %s", name.c_str()); - bool ok = iter.second->Load(dir, fmt, options); - if (!ok) { - ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str()); - // restore to old protobuf log handler - google::protobuf::SetLogHandler(old_handler); - return nullptr; - } - ATOM_DEBUG("loaded %s", name.c_str()); - } - - // restore to old protobuf log handler - google::protobuf::SetLogHandler(old_handler); - return msger_container; -} - -MessagerContainer Hub::NewMessagerContainer() { - MessagerContainer msger_container = std::make_shared(); - for (auto&& it : Registry::registrar) { - if (!options_.filter || options_.filter(it.first)) { - (*msger_container)[it.first] = it.second(); - } - } - return msger_container; -} - -void Hub::SetMessagerContainer(MessagerContainer msger_container) { - // replace with thread-safe guarantee. - std::unique_lock lock(mutex_); - msger_container_ = msger_container; - last_loaded_time_ = std::time(nullptr); -} - -MessagerContainer Hub::GetMessagerContainerWithProvider() const { - if (msger_container_provider_ != nullptr) { - return msger_container_provider_(); - } - return msger_container_; -} - -const std::shared_ptr Hub::GetMessager(const std::string& name) const { - auto container = GetMessagerContainerWithProvider(); - if (container) { - auto it = container->find(name); - if (it != container->end()) { - return it->second; - } - } - return nullptr; -} - -namespace internal { -// Thread-local storage (TLS) -thread_local Scheduler* tls_sched = nullptr; -Scheduler& Scheduler::Current() { - if (tls_sched == nullptr) { - tls_sched = new Scheduler; - } - return *tls_sched; -} - -int Scheduler::LoopOnce() { - AssertInLoopThread(); - - int count = 0; - std::vector jobs; - { - // scoped for auto-release lock. - // wake up immediately when there are pending tasks. - std::unique_lock lock(mutex_); - jobs.swap(jobs_); - } - for (auto&& job : jobs) { - job(); - } - count += jobs.size(); - return count; -} - -void Scheduler::Post(const Job& job) { - std::unique_lock lock(mutex_); - jobs_.push_back(job); -} - -void Scheduler::Dispatch(const Job& job) { - if (IsLoopThread()) { - job(); // run it immediately - } else { - Post(job); // post and run it at next loop - } -} - -bool Scheduler::IsLoopThread() const { return thread_id_ == std::this_thread::get_id(); } -void Scheduler::AssertInLoopThread() const { - if (!IsLoopThread()) { - abort(); - } -} - -bool Postprocess(Postprocessor postprocessor, MessagerContainer container) { - // create a temporary hub with messager container for post process - Hub hub(container); - - // messager-level postprocess - for (auto iter : *container) { - auto msger = iter.second; - bool ok = msger->ProcessAfterLoadAll(hub); - if (!ok) { - g_err_msg = "hub call ProcessAfterLoadAll failed, messager: " + msger->Name(); - return false; - } - } - - // hub-level postprocess - if (postprocessor != nullptr) { - bool ok = postprocessor(hub); - if (!ok) { - g_err_msg = "hub call Postprocesser failed, you'd better check your custom 'postprocessor' load option"; - return false; - } - } - - return true; -} - -} // namespace internal - -namespace util { -int Mkdir(const std::string& path) { - std::string path_ = path + kPathSeperator; - struct stat info; - for (size_t pos = path_.find(kPathSeperator, 0); pos != std::string::npos; pos = path_.find(kPathSeperator, pos)) { - ++pos; - auto sub_dir = path_.substr(0, pos); - if (stat(sub_dir.c_str(), &info) == 0 && info.st_mode & S_IFDIR) { - continue; - } - int status = mkdir(sub_dir.c_str(), 0755); - if (status != 0) { - std::cerr << "system error: " << strerror(errno) << std::endl; - return -1; - } - } - return 0; -} - -std::string GetDir(const std::string& path) { - size_t pos = path.find_last_of(kPathSeperator); - if (pos != std::string::npos) { - return path.substr(0, pos); - } - return kEmpty; -} - -std::string GetExt(const std::string& path) { - std::size_t pos = path.find_last_of("."); - if (pos != std::string::npos) { - return path.substr(pos); - } - return kEmpty; -} - -} // namespace util - } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h new file mode 100644 index 00000000..8f725973 --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h @@ -0,0 +1,43 @@ +#pragma once +#include + +#include +#include + +#include "util.pc.h" + +namespace tableau { +enum class LoadMode { + kModeDefault, + kModeOnlyMain, + kModeOnlyPatch, +}; + +// ReadFunc reads the config file and returns its content. +using ReadFunc = std::function; + +struct LoadOptions { + // read_func reads the config file and returns its content. + ReadFunc read_func; + // Whether to ignore unknown JSON fields during parsing. + // + // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. + bool ignore_unknown_fields = false; + // Paths maps each messager name to a corresponding config file path. + // If specified, then the main messager will be parsed from the file + // directly, other than the specified load dir. + std::unordered_map paths; + // Patch paths maps each messager name to one or multiple corresponding patch file paths. + // If specified, then main messager will be patched. + std::unordered_map> patch_paths; + // Patch dirs specifies the directory paths for config patching. + std::vector patch_dirs; + // Mode specifies the loading mode for config patching. + LoadMode mode = LoadMode::kModeDefault; +}; + +bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr); +bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr); +} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc index 1cac4d2f..893b63a6 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc @@ -15,7 +15,7 @@ #include #include -#include "hub.pc.h" +#include "util.pc.h" #ifdef _WIN32 #define gettid() GetCurrentThreadId() diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h new file mode 100644 index 00000000..a021f59e --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/messager.pc.h @@ -0,0 +1,38 @@ +#pragma once +#include + +#include +#include + +#include "util.pc.h" + +namespace tableau { +class Hub; +struct LoadOptions; + +class Messager { + public: + struct Stats { + std::chrono::microseconds duration; // total load time consuming. + // TODO: crc32 of config file to decide whether changed or not + // std::string crc32; + // int64_t last_modified_time = 0; // unix timestamp + }; + + public: + virtual ~Messager() = default; + static const std::string& Name() { return kEmpty; } + const Stats& GetStats() { return stats_; } + // Load fills message from file in the specified directory and format. + virtual bool Load(const std::string& dir, Format fmt, const LoadOptions* options = nullptr) = 0; + // Message returns the inner message data. + virtual const google::protobuf::Message* Message() const { return nullptr; } + // callback after all messagers loaded. + virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } + + protected: + // callback after this messager loaded. + virtual bool ProcessAfterLoad() { return true; }; + Stats stats_; +}; +} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.cc new file mode 100644 index 00000000..5a9aca0b --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.cc @@ -0,0 +1,54 @@ +#include "scheduler.pc.h" + +namespace tableau { +namespace internal { +// Thread-local storage (TLS) +thread_local Scheduler* tls_sched = nullptr; + +Scheduler& Scheduler::Current() { + if (tls_sched == nullptr) { + tls_sched = new Scheduler; + } + return *tls_sched; +} + +int Scheduler::LoopOnce() { + AssertInLoopThread(); + + int count = 0; + std::vector jobs; + { + // scoped for auto-release lock. + // wake up immediately when there are pending tasks. + std::unique_lock lock(mutex_); + jobs.swap(jobs_); + } + for (auto&& job : jobs) { + job(); + } + count += jobs.size(); + return count; +} + +void Scheduler::Post(const Job& job) { + std::unique_lock lock(mutex_); + jobs_.push_back(job); +} + +void Scheduler::Dispatch(const Job& job) { + if (IsLoopThread()) { + job(); // run it immediately + } else { + Post(job); // post and run it at next loop + } +} + +bool Scheduler::IsLoopThread() const { return thread_id_ == std::this_thread::get_id(); } + +void Scheduler::AssertInLoopThread() const { + if (!IsLoopThread()) { + abort(); + } +} +} // namespace internal +} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.h new file mode 100644 index 00000000..5d5380e8 --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/scheduler.pc.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include + +namespace tableau { +namespace internal { +class Scheduler { + public: + typedef std::function Job; + + public: + Scheduler() : thread_id_(std::this_thread::get_id()) {} + static Scheduler& Current(); + // thread-safety + void Post(const Job& job); + void Dispatch(const Job& job); + int LoopOnce(); + bool IsLoopThread() const; + void AssertInLoopThread() const; + + private: + std::thread::id thread_id_; + std::mutex mutex_; + std::vector jobs_; +}; + +} // namespace internal +} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc new file mode 100644 index 00000000..72cbc824 --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc @@ -0,0 +1,209 @@ +#include "util.pc.h" + +#include +#include + +#include +#include +#include + +#if __cplusplus >= 201703L +#include +namespace fs = std::filesystem; +#else +#ifdef _WIN32 +#include +#include +#else +#include +#endif +#endif + +#include "load.pc.h" +#include "logger.pc.h" +#include "messager.pc.h" + +namespace tableau { +#ifdef _WIN32 +#define mkdir(path, mode) _mkdir(path) +static constexpr char kPathSeperator = '\\'; +#else +static constexpr char kPathSeperator = '/'; +#endif + +static thread_local std::string g_err_msg; +const std::string& GetErrMsg() { return g_err_msg; } +void SetErrMsg(const std::string& msg) { g_err_msg = msg; } + +const std::string kUnknownExt = ".unknown"; +const std::string kJSONExt = ".json"; +const std::string kTextExt = ".txt"; +const std::string kBinExt = ".bin"; + +namespace util { +int Mkdir(const std::string& path) { +#if __cplusplus >= 201703L + std::error_code ec; + if (!fs::create_directories(path, ec)) { + if (ec) { + std::cerr << "system error: " << ec.message() << std::endl; + return -1; + } + } + return 0; +#else + std::string path_ = path + kPathSeperator; + struct stat info; + for (size_t pos = path_.find(kPathSeperator, 0); pos != std::string::npos; pos = path_.find(kPathSeperator, pos)) { + ++pos; + auto sub_dir = path_.substr(0, pos); + if (stat(sub_dir.c_str(), &info) == 0 && info.st_mode & S_IFDIR) { + continue; + } + int status = mkdir(sub_dir.c_str(), 0755); + if (status != 0) { + std::cerr << "system error: " << strerror(errno) << std::endl; + return -1; + } + } + return 0; +#endif +} + +std::string GetDir(const std::string& path) { +#if __cplusplus >= 201703L + return fs::path(path).parent_path().string(); +#else + size_t pos = path.find_last_of(kPathSeperator); + if (pos != std::string::npos) { + return path.substr(0, pos); + } + return kEmpty; +#endif +} + +bool ExistsFile(const std::string& filename) { +#if __cplusplus >= 201703L + return fs::exists(filename); +#else + std::ifstream file(filename); + // returns true if the file exists and is accessible + return file.good(); +#endif +} + +bool ReadFile(const std::string& filename, std::string& content) { + std::ifstream file(filename); + if (!file.is_open()) { + SetErrMsg("failed to open " + filename + ": " + strerror(errno)); + return false; + } + std::stringstream ss; + ss << file.rdbuf(); + content = ss.str(); + return true; +} + +std::string GetExt(const std::string& path) { + std::size_t pos = path.find_last_of("."); + if (pos != std::string::npos) { + return path.substr(pos); + } + return kEmpty; +} + +Format Ext2Format(const std::string& ext) { + if (ext == kJSONExt) { + return Format::kJSON; + } else if (ext == kTextExt) { + return Format::kText; + } else if (ext == kBinExt) { + return Format::kBin; + } else { + return Format::kUnknown; + } +} + +const std::string& Format2Ext(Format fmt) { + switch (fmt) { + case Format::kJSON: + return kJSONExt; + case Format::kText: + return kTextExt; + case Format::kBin: + return kBinExt; + default: + return kUnknownExt; + } +} + +bool Message2JSON(const google::protobuf::Message& msg, std::string& json) { + google::protobuf::util::JsonPrintOptions options; + options.add_whitespace = true; + options.always_print_primitive_fields = true; + options.preserve_proto_field_names = true; + return google::protobuf::util::MessageToJsonString(msg, &json, options).ok(); +} + +bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options /* = nullptr */) { + google::protobuf::util::Status status; + if (options != nullptr) { + google::protobuf::util::JsonParseOptions parse_options; + parse_options.ignore_unknown_fields = options->ignore_unknown_fields; + status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); + } else { + status = google::protobuf::util::JsonStringToMessage(json, &msg); + } + if (!status.ok()) { + SetErrMsg("failed to parse " + GetProtoName(msg) + kJSONExt + ": " + status.ToString()); + return false; + } + return true; +} + +bool Text2Message(const std::string& text, google::protobuf::Message& msg) { + if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) { + SetErrMsg("failed to parse " + GetProtoName(msg) + kTextExt); + return false; + } + return true; +} +bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) { + if (!msg.ParseFromString(bin)) { + SetErrMsg("failed to parse " + GetProtoName(msg) + kBinExt); + return false; + } + return true; +} + +const std::string& GetProtoName(const google::protobuf::Message& msg) { + const auto* md = msg.GetDescriptor(); + return md != nullptr ? md->name() : kEmpty; +} + +std::string GetPatchName(tableau::Patch patch) { + auto* descriptor = tableau::Patch_descriptor(); + if (descriptor) { + auto* value = descriptor->FindValueByNumber(patch); + if (value) { + return value->name(); + } + } + return std::to_string(static_cast(patch)); +} + +// refer: https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/stubs/logging.h +void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg) { + static const std::unordered_map kLevelMap = {{google::protobuf::LOGLEVEL_INFO, log::kInfo}, + {google::protobuf::LOGLEVEL_WARNING, log::kWarn}, + {google::protobuf::LOGLEVEL_ERROR, log::kError}, + {google::protobuf::LOGLEVEL_FATAL, log::kFatal}}; + log::Level lvl = log::kWarn; // default + auto iter = kLevelMap.find(level); + if (iter != kLevelMap.end()) { + lvl = iter->second; + } + ATOM_LOGGER_CALL(tableau::log::DefaultLogger(), lvl, "[libprotobuf %s:%d] %s", filename, line, msg.c_str()); +} +} // namespace util +} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h new file mode 100644 index 00000000..ab8d9e08 --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h @@ -0,0 +1,103 @@ +#pragma once +#include + +#include +#include +#include + +#include "tableau/protobuf/tableau.pb.h" + +namespace tableau { +const std::string& GetErrMsg(); +void SetErrMsg(const std::string& msg); + +enum class Format { + kUnknown, + kJSON, + kText, + kBin, +}; + +static const std::string kEmpty = ""; +extern const std::string kUnknownExt; +extern const std::string kJSONExt; +extern const std::string kTextExt; +extern const std::string kBinExt; + +struct LoadOptions; + +namespace util { +// Combine hash values +// +// References: +// - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x +// - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key +inline void HashCombine(std::size_t& seed) {} + +template +inline void HashCombine(std::size_t& seed, const T& v, O... others) { + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + HashCombine(seed, others...); +} + +template +inline std::size_t SugaredHashCombine(const T& v, O... others) { + std::size_t seed = 0; // start with a hash value 0 + HashCombine(seed, v, others...); + return seed; +} + +// Mkdir makes dir recursively. +int Mkdir(const std::string& path); +// GetDir returns all but the last element of path, typically the path's +// directory. +std::string GetDir(const std::string& path); +// ExistsFile checks if a file exists. +bool ExistsFile(const std::string& filename); +// ReadFile reads the file named by filename and returns the contents. +bool ReadFile(const std::string& filename, std::string& content); + +// GetExt returns the file name extension used by path. +// The extension is the suffix beginning at the final dot +// in the final element of path; it is empty if there is +// no dot. +std::string GetExt(const std::string& path); +// Convert file extension to Format type. +// NOTE: ext includes dot ".", such as: +// - kJSONExt：".json" +// - kTextExt".txt" +// - kBinExt".bin" +Format Ext2Format(const std::string& ext); +// Empty string will be returned if an unsupported enum value has been passed, +// and the error message can be obtained by GetErrMsg(). +const std::string& Format2Ext(Format fmt); + +bool Message2JSON(const google::protobuf::Message& msg, std::string& json); +bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options = nullptr); +bool Text2Message(const std::string& text, google::protobuf::Message& msg); +bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); + +const std::string& GetProtoName(const google::protobuf::Message& msg); +std::string GetPatchName(tableau::Patch patch); + +void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg); + +class TimeProfiler { + protected: + std::chrono::time_point last_; + + public: + TimeProfiler() { Start(); } + void Start() { last_ = std::chrono::steady_clock::now(); } + // Calculate duration between the last time point and now, + // and update last time point to now. + std::chrono::microseconds Elapse() { + auto now = std::chrono::steady_clock::now(); + auto duration = now - last_; // This is of type std::chrono::duration + last_ = now; + return std::chrono::duration_cast(duration); + } +}; +} // namespace util +} // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/hub.go b/cmd/protoc-gen-cpp-tableau-loader/hub.go new file mode 100644 index 00000000..a181f686 --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/hub.go @@ -0,0 +1,422 @@ +package main + +import ( + "errors" + "sort" + + "github.com/iancoleman/strcase" + "github.com/tableauio/loader/cmd/protoc-gen-cpp-tableau-loader/helper" + "github.com/tableauio/tableau/proto/tableaupb" + "google.golang.org/protobuf/compiler/protogen" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" +) + +func getAllOrderedFilesAndMessagers(gen *protogen.Plugin) (protofiles []string, fileMessagers map[string][]string) { + fileMessagers = map[string][]string{} + for _, f := range gen.Files { + if !f.Generate { + continue + } + opts := f.Desc.Options().(*descriptorpb.FileOptions) + workbook := proto.GetExtension(opts, tableaupb.E_Workbook).(*tableaupb.WorkbookOptions) + if workbook == nil { + continue + } + protofiles = append(protofiles, f.GeneratedFilenamePrefix) + var messagers []string + for _, message := range f.Messages { + opts, ok := message.Desc.Options().(*descriptorpb.MessageOptions) + if !ok { + gen.Error(errors.New("get message options failed")) + } + worksheet, ok := proto.GetExtension(opts, tableaupb.E_Worksheet).(*tableaupb.WorksheetOptions) + if !ok { + gen.Error(errors.New("get worksheet extension failed")) + } + if worksheet != nil { + messagerName := string(message.Desc.Name()) + messagers = append(messagers, messagerName) + } + } + // sort messagers in one file to keep in order + sort.Strings(messagers) + fileMessagers[f.GeneratedFilenamePrefix] = messagers + } + // sort all files to keep in order + sort.Strings(protofiles) + return +} + +// generateHub generates related hub files. +func generateHub(gen *protogen.Plugin) { + if *shards <= 1 { + protofiles, fileMessagers := getAllOrderedFilesAndMessagers(gen) + + hppFilename := "hub." + pcExt + ".h" + g1 := gen.NewGeneratedFile(hppFilename, "") + helper.GenerateCommonHeader(gen, g1, version) + g1.P() + g1.P(hubHpp) + generateHubHppTplSpec(gen, g1, protofiles, fileMessagers) + g1.P(msgContainerHpp) + generateHubHppMsgContainerMembers(gen, g1, protofiles, fileMessagers) + g1.P(registryHpp) + g1.P(bottomHpp) + + cppFilename := "hub." + pcExt + ".cc" + g2 := gen.NewGeneratedFile(cppFilename, "") + helper.GenerateCommonHeader(gen, g2, version) + g2.P() + g2.P(hubCppHeader) + generateHubCppHeader(gen, g2, protofiles, fileMessagers) + g2.P(hubCpp) + generateHubCppTplSpec(gen, g2, protofiles, fileMessagers) + g2.P(msgContainerCpp) + generateHubCppMsgContainerCtor(gen, g2, protofiles, fileMessagers) + g2.P(registryCpp) + generateHubCppRegistry(gen, g2, protofiles, fileMessagers) + g2.P(bottomCpp) + } else { + // sharding + generateShardedHub(gen) + } +} + +func generateHubHppTplSpec(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P("class ", messager, ";") + g.P("template <>") + g.P("const std::shared_ptr<", messager, "> Hub::Get<", messager, ">() const;") + g.P() + } + } +} + +func generateHubHppMsgContainerMembers(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P(" std::shared_ptr<", messager, "> ", strcase.ToSnake(messager), "_;") + } + } +} + +func generateHubCppHeader(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { + for _, proto := range protofiles { + g.P(`#include "`, proto, ".", pcExt, `.h"`) + } + g.P() +} + +func generateHubCppTplSpec(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P("template <>") + g.P("const std::shared_ptr<", messager, "> Hub::Get<", messager, ">() const {;") + g.P(" return GetMessagerContainer()->", strcase.ToSnake(messager), "_;") + g.P("}") + g.P() + } + } +} + +func generateHubCppMsgContainerCtor(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P(" ", strcase.ToSnake(messager), "_ = std::dynamic_pointer_cast<", messager, `>((*msger_map_)["`, messager, `"]);`) + } + } +} + +func generateHubCppRegistry(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P(" Register<", messager, ">();") + } + } +} + +const hubHpp = `#pragma once +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "load.pc.h" +#include "messager.pc.h" +#include "scheduler.pc.h" + +namespace tableau { +class MessagerContainer; +class Hub; + +using MessagerMap = std::unordered_map>; +// FilterFunc filter in messagers if returned value is true. +// NOTE: name is the protobuf message name, e.g.: "message ItemConf{...}". +using Filter = std::function; +using MessagerContainerProvider = std::function()>; + +struct HubOptions { + // Filter can only filter in certain specific messagers based on the + // condition that you provide. + Filter filter; + // Provide custom MessagerContainer. For keeping configuration access + // consistent in a coroutine or a transaction. + MessagerContainerProvider provider; +}; + +class Hub { + public: + Hub(const HubOptions* options = nullptr) + : msger_container_(std::make_shared()), options_(options ? *options : HubOptions{}) {} + /***** Synchronous Loading *****/ + // Load fills messages (in MessagerContainer) from files in the specified directory and format. + bool Load(const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); + + /***** Asynchronous Loading *****/ + // Load configs into temp MessagerContainer, and you should call LoopOnce() in you app's main loop, + // in order to take the temp MessagerContainer into effect. + bool AsyncLoad(const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); + int LoopOnce(); + // You'd better initialize the scheduler in the main thread. + void InitScheduler(); + + /***** MessagerMap *****/ + std::shared_ptr GetMessagerMap() const; + void SetMessagerMap(std::shared_ptr msger_map); + + /***** MessagerContainer *****/ + // This function is exposed only for use in MessagerContainerProvider. + std::shared_ptr GetMessagerContainer() const { + if (options_.provider != nullptr) { + return options_.provider(); + } + return msger_container_; + } + + /***** Access APIs *****/ + template + const std::shared_ptr Get() const; + + template + const U* Get(Args... args) const; + + template + const U* GetOrderedMap(Args... args) const; + + // GetLastLoadedTime returns the time when hub's msger_container_ was last set. + inline std::time_t GetLastLoadedTime() const; + + private: + std::shared_ptr InternalLoad(const std::string& dir, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr) const; + std::shared_ptr NewMessagerMap() const; + const std::shared_ptr GetMessager(const std::string& name) const; + + bool Postprocess(std::shared_ptr msger_map); + + private: + // For thread-safe guarantee during configuration updating. + std::mutex mutex_; + // All messagers' container. + std::shared_ptr msger_container_; + // Loading scheduler. + internal::Scheduler* sched_ = nullptr; + // Hub options + const HubOptions options_; +}; + +template +const std::shared_ptr Hub::Get() const { + auto msg = GetMessager(T::Name()); + return std::dynamic_pointer_cast(msg); +} + +template +const U* Hub::Get(Args... args) const { + auto msger = Get(); + return msger ? msger->Get(args...) : nullptr; +} + +template +const U* Hub::GetOrderedMap(Args... args) const { + auto msger = Get(); + return msger ? msger->GetOrderedMap(args...) : nullptr; +} +` + +const msgContainerHpp = `class MessagerContainer { + friend class Hub; + + public: + MessagerContainer(std::shared_ptr msger_map = nullptr); + + private: + std::shared_ptr msger_map_; + std::time_t last_loaded_time_; + + private:` + +const registryHpp = `}; + +using MessagerGenerator = std::function()>; +// messager name -> messager generator +using Registrar = std::unordered_map; +class Registry { + friend class Hub; + + public: + static void Init(); + + template + static void Register(); + + private:` + +const bottomHpp = ` static Registrar registrar; +}; + +template +void Registry::Register() { + registrar[T::Name()] = []() { return std::make_shared(); }; +} +} // namespace tableau` + +const hubCppHeader = `#include "hub.pc.h" + +#include +#include +#include + +#include + +#include "logger.pc.h" +#include "messager.pc.h" +#include "util.pc.h"` + +const hubCpp = ` +namespace tableau { +Registrar Registry::registrar = Registrar(); + +bool Hub::Load(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { + auto msger_map = InternalLoad(dir, fmt, options); + if (!msger_map) { + return false; + } + bool ok = Postprocess(msger_map); + if (!ok) { + return false; + } + SetMessagerMap(msger_map); + return true; +} + +bool Hub::AsyncLoad(const std::string& dir, Format fmt /* = Format::kJSON */, + const LoadOptions* options /* = nullptr */) { + auto msger_map = InternalLoad(dir, fmt, options); + if (!msger_map) { + return false; + } + bool ok = Postprocess(msger_map); + if (!ok) { + return false; + } + sched_->Dispatch(std::bind(&Hub::SetMessagerMap, this, msger_map)); + return true; +} + +int Hub::LoopOnce() { return sched_->LoopOnce(); } + +void Hub::InitScheduler() { + sched_ = new internal::Scheduler(); + sched_->Current(); +} + +std::shared_ptr Hub::InternalLoad(const std::string& dir, Format fmt /* = Format::kJSON */, + const LoadOptions* options /* = nullptr */) const { + // intercept protobuf error logs + auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); + auto msger_map = NewMessagerMap(); + for (auto iter : *msger_map) { + auto&& name = iter.first; + ATOM_DEBUG("loading %s", name.c_str()); + bool ok = iter.second->Load(dir, fmt, options); + if (!ok) { + ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str()); + // restore to old protobuf log handler + google::protobuf::SetLogHandler(old_handler); + return nullptr; + } + ATOM_DEBUG("loaded %s", name.c_str()); + } + + // restore to old protobuf log handler + google::protobuf::SetLogHandler(old_handler); + return msger_map; +} + +std::shared_ptr Hub::NewMessagerMap() const { + std::shared_ptr msger_map = std::make_shared(); + for (auto&& it : Registry::registrar) { + if (!options_.filter || options_.filter(it.first)) { + (*msger_map)[it.first] = it.second(); + } + } + return msger_map; +} + +std::shared_ptr Hub::GetMessagerMap() const { return GetMessagerContainer()->msger_map_; } + +void Hub::SetMessagerMap(std::shared_ptr msger_map) { + // replace with thread-safe guarantee. + std::unique_lock lock(mutex_); + msger_container_ = std::make_shared(msger_map); +} + +const std::shared_ptr Hub::GetMessager(const std::string& name) const { + auto msger_map = GetMessagerMap(); + if (msger_map) { + auto it = msger_map->find(name); + if (it != msger_map->end()) { + return it->second; + } + } + return nullptr; +} + +bool Hub::Postprocess(std::shared_ptr msger_map) { + // create a temporary hub with messager container for post process + Hub tmp_hub; + tmp_hub.SetMessagerMap(msger_map); + + // messager-level postprocess + for (auto iter : *msger_map) { + auto msger = iter.second; + bool ok = msger->ProcessAfterLoadAll(tmp_hub); + if (!ok) { + SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + msger->Name()); + return false; + } + } + return true; +} + +std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; }` + +const msgContainerCpp = ` +MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) + : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), + last_loaded_time_(std::time(nullptr)) {` + +const registryCpp = `} + +void Registry::Init() {` + +const bottomCpp = `} +} // namespace tableau` diff --git a/cmd/protoc-gen-cpp-tableau-loader/main.go b/cmd/protoc-gen-cpp-tableau-loader/main.go index e780de54..fcb20e5b 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/main.go +++ b/cmd/protoc-gen-cpp-tableau-loader/main.go @@ -21,12 +21,12 @@ var messagerSuffix *string // mode control generating rules for better dependency management. var mode *string -// count of generated registry files, aimed to boost compiling speed. -var registryShards *int +// count of generated hub cpp files, aimed to boost compiling speed. +var shards *int const ( ModeDefault = "default" // generate all at once. - ModeRegistry = "registry" // only generate "registry.pc.h/cc" files. + ModeHub = "hub" // only generate "hub.pc.h/cc" files. ModeMessager = "messager" // only generate "*.pc.h/cc" for each .proto files. ) @@ -34,12 +34,12 @@ func main() { var flags flag.FlagSet namespace = flags.String("namespace", "tableau", "tableau namespace") messagerSuffix = flags.String("suffix", "Mgr", "tableau messager name suffix") - mode = flags.String("mode", "default", `available mode: default, registry, and messager. + mode = flags.String("mode", "default", `available mode: default, hub, and messager. - default: generate all at once. - - registry: only generate "registry.pc.h/cc" files. + - hub: only generate "hub.pc.h/cc" files. - messager: only generate "*.pc.h/cc" for each .proto files. `) - registryShards = flags.Int("registry-shards", 1, "count of generated registry files") + shards = flags.Int("shards", 1, "count of generated hub cpp files for distributed compiling speed-up") flag.Parse() protogen.Options{ @@ -54,7 +54,7 @@ func main() { switch *mode { case ModeMessager: generateMessager(gen, f) - case ModeRegistry: + case ModeHub: // pass case ModeDefault: generateMessager(gen, f) @@ -63,12 +63,12 @@ func main() { switch *mode { case ModeDefault: - generateRegistry(gen) + generateHub(gen) generateEmbed(gen) case ModeMessager: // pass - case ModeRegistry: - generateRegistry(gen) + case ModeHub: + generateHub(gen) } return nil }) diff --git a/cmd/protoc-gen-cpp-tableau-loader/messager.go b/cmd/protoc-gen-cpp-tableau-loader/messager.go index 6c1fa88e..2fdd8b13 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/messager.go +++ b/cmd/protoc-gen-cpp-tableau-loader/messager.go @@ -45,7 +45,8 @@ func generateHppFileContent(gen *protogen.Plugin, file *protogen.File, g *protog g.P("#pragma once") g.P("#include ") g.P() - g.P(`#include "`, "hub.", pcExt, `.h"`) + g.P(`#include "`, "messager.", pcExt, `.h"`) + g.P(`#include "`, "util.", pcExt, `.h"`) g.P(`#include "`, file.GeneratedFilenamePrefix, ".", pbExt, `.h"`) g.P() @@ -135,6 +136,7 @@ func generateCppFileContent(gen *protogen.Plugin, file *protogen.File, g *protog g.P(`#include "`, file.GeneratedFilenamePrefix, ".", pcExt, `.h"`) g.P() g.P(`#include "hub.pc.h"`) + g.P(`#include "util.pc.h"`) g.P() g.P("namespace ", *namespace, " {") diff --git a/cmd/protoc-gen-cpp-tableau-loader/registry.go b/cmd/protoc-gen-cpp-tableau-loader/registry.go deleted file mode 100644 index 28d7e009..00000000 --- a/cmd/protoc-gen-cpp-tableau-loader/registry.go +++ /dev/null @@ -1,183 +0,0 @@ -package main - -import ( - "errors" - "math" - "sort" - "strconv" - - "github.com/tableauio/loader/cmd/protoc-gen-cpp-tableau-loader/helper" - "github.com/tableauio/tableau/proto/tableaupb" - "google.golang.org/protobuf/compiler/protogen" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -func getAllOrderedFilesAndMessagers(gen *protogen.Plugin) (protofiles []string, fileMessagers map[string][]string) { - fileMessagers = map[string][]string{} - for _, f := range gen.Files { - if !f.Generate { - continue - } - opts := f.Desc.Options().(*descriptorpb.FileOptions) - workbook := proto.GetExtension(opts, tableaupb.E_Workbook).(*tableaupb.WorkbookOptions) - if workbook == nil { - continue - } - protofiles = append(protofiles, f.GeneratedFilenamePrefix) - var messagers []string - for _, message := range f.Messages { - opts, ok := message.Desc.Options().(*descriptorpb.MessageOptions) - if !ok { - gen.Error(errors.New("get message options failed")) - } - worksheet, ok := proto.GetExtension(opts, tableaupb.E_Worksheet).(*tableaupb.WorksheetOptions) - if !ok { - gen.Error(errors.New("get worksheet extension failed")) - } - if worksheet != nil { - messagerName := string(message.Desc.Name()) - messagers = append(messagers, messagerName) - } - } - // sort messagers in one file to keep in order - sort.Strings(messagers) - fileMessagers[f.GeneratedFilenamePrefix] = messagers - } - // sort all files to keep in order - sort.Strings(protofiles) - return -} - -// generateRegistry generates related registry files. -func generateRegistry(gen *protogen.Plugin) { - if *registryShards <= 1 { - hppFilename := "registry." + pcExt + ".h" - g1 := gen.NewGeneratedFile(hppFilename, "") - helper.GenerateCommonHeader(gen, g1, version) - g1.P() - g1.P(registryHppTop) - g1.P(registryHppBottom) - - protofiles, fileMessagers := getAllOrderedFilesAndMessagers(gen) - cppFilename := "registry." + pcExt + ".cc" - g2 := gen.NewGeneratedFile(cppFilename, "") - helper.GenerateCommonHeader(gen, g2, version) - g2.P() - generateRegistryCppFileContent(gen, g2, protofiles, fileMessagers) - } else { - // sharding - generateShardedRegistry(gen) - } -} - -// generateRegistryCppFileContent generates the registry logic. -func generateRegistryCppFileContent(gen *protogen.Plugin, g *protogen.GeneratedFile, protofiles []string, fileMessagers map[string][]string) { - g.P(`#include "`, "registry.", pcExt, `.h"`) - g.P() - for _, proto := range protofiles { - g.P(`#include "`, proto, ".", pcExt, `.h"`) - } - g.P() - - g.P("namespace ", *namespace, " {") - g.P("Registrar Registry::registrar = Registrar();") - g.P("void Registry::Init() {") - for _, messagers := range fileMessagers { - for _, messager := range messagers { - g.P(" Register<", messager, ">();") - } - } - g.P("}") - g.P("} // namespace ", *namespace) -} - -// generateShardedRegistry generates related registry files. -func generateShardedRegistry(gen *protogen.Plugin) { - protofiles, fileMessagers := getAllOrderedFilesAndMessagers(gen) - hppFilename := "registry." + pcExt + ".h" - g1 := gen.NewGeneratedFile(hppFilename, "") - helper.GenerateCommonHeader(gen, g1, version) - g1.P() - g1.P(registryHppTop) - - cppFilename := "registry." + pcExt + ".cc" - g2 := gen.NewGeneratedFile(cppFilename, "") - helper.GenerateCommonHeader(gen, g2, version) - g2.P() - g2.P(`#include "`, "registry.", pcExt, `.h"`) - g2.P() - - g2.P("namespace ", *namespace, " {") - g2.P("Registrar Registry::registrar = Registrar();") - g2.P("void Registry::Init() {") - - shardSize := int(math.Ceil(float64(len(protofiles)) / float64((*registryShards)))) - for i := 0; i < *registryShards; i++ { - cursor := (i + 1) * shardSize - if cursor > len(protofiles) { - cursor = len(protofiles) - } - if i*shardSize >= cursor { - break - } - g1.P(" static void InitShard", i, "();") - g2.P(" InitShard", i, "();") - cppFilename := "registry_shard" + strconv.Itoa(i) + "." + pcExt + ".cc" - g := gen.NewGeneratedFile(cppFilename, "") - helper.GenerateCommonHeader(gen, g, version) - g.P() - shardedProtofiles := protofiles[i*shardSize : cursor] - generateShardedRegistryCppFileContent(gen, g, i, shardedProtofiles, fileMessagers) - } - - g1.P(registryHppBottom) - g2.P("}") - g2.P("} // namespace ", *namespace) -} - -// generateShardedRegistryCppFileContent generates one registry shard logic. -func generateShardedRegistryCppFileContent(gen *protogen.Plugin, g *protogen.GeneratedFile, shardIndex int, protofiles []string, fileMessagers map[string][]string) { - g.P(`#include "`, "registry.", pcExt, `.h"`) - g.P() - for _, proto := range protofiles { - g.P(`#include "`, proto, ".", pcExt, `.h"`) - } - g.P() - - g.P("namespace ", *namespace, " {") - g.P("void Registry::InitShard", shardIndex, "() {") - for _, proto := range protofiles { - messagers := fileMessagers[proto] - for _, messager := range messagers { - g.P(" Register<", messager, ">();") - } - } - - g.P("}") - g.P("} // namespace ", *namespace) -} - -const registryHppTop = `#pragma once -#include "hub.pc.h" -namespace tableau { -using MessagerGenerator = std::function()>; -// messager name -> messager generator -using Registrar = std::unordered_map; -class Registry { - public: - static void Init();` - -const registryHppBottom = ` - template - static void Register(); - - static Registrar registrar; -}; - -template -void Registry::Register() { - registrar[T::Name()] = []() { return std::make_shared(); }; -} - -} // namespace tableau` diff --git a/cmd/protoc-gen-cpp-tableau-loader/shard.go b/cmd/protoc-gen-cpp-tableau-loader/shard.go new file mode 100644 index 00000000..84605516 --- /dev/null +++ b/cmd/protoc-gen-cpp-tableau-loader/shard.go @@ -0,0 +1,127 @@ +package main + +import ( + "math" + "strconv" + + "github.com/iancoleman/strcase" + "github.com/tableauio/loader/cmd/protoc-gen-cpp-tableau-loader/helper" + "google.golang.org/protobuf/compiler/protogen" +) + +// generateShardedHub generates related hub files. +func generateShardedHub(gen *protogen.Plugin) { + protofiles, fileMessagers := getAllOrderedFilesAndMessagers(gen) + + // detect real shard num + realShardNum := *shards + if realShardNum > len(protofiles) { + realShardNum = len(protofiles) + } + shardSize := int(math.Ceil(float64(len(protofiles)) / float64((realShardNum)))) + + hppFilename := "hub." + pcExt + ".h" + g1 := gen.NewGeneratedFile(hppFilename, "") + helper.GenerateCommonHeader(gen, g1, version) + g1.P() + g1.P(hubHpp) + generateHubHppTplSpec(gen, g1, protofiles, fileMessagers) + g1.P(msgContainerHpp) + generateShardedHubHppMsgContainerShards(gen, g1, realShardNum) + generateHubHppMsgContainerMembers(gen, g1, protofiles, fileMessagers) + g1.P(registryHpp) + generateShardedHubHppRegistryShards(gen, g1, realShardNum) + g1.P(bottomHpp) + + cppFilename := "hub." + pcExt + ".cc" + g2 := gen.NewGeneratedFile(cppFilename, "") + helper.GenerateCommonHeader(gen, g2, version) + g2.P() + g2.P(hubCppHeader) + g2.P(hubCpp) + g2.P(msgContainerCpp) + generateShardedHubCppMsgContainerShards(gen, g2, realShardNum) + g2.P(registryCpp) + generateShardedHubCppRegistryShards(gen, g2, realShardNum) + g2.P(bottomCpp) + + for i := 0; i < realShardNum; i++ { + cursor := (i + 1) * shardSize + if cursor > len(protofiles) { + cursor = len(protofiles) + } + cppFilename := "hub_shard" + strconv.Itoa(i) + "." + pcExt + ".cc" + g := gen.NewGeneratedFile(cppFilename, "") + helper.GenerateCommonHeader(gen, g, version) + g.P() + shardedProtofiles := protofiles[i*shardSize : cursor] + generateShardedHubCppFileContent(gen, g, i, shardedProtofiles, fileMessagers) + } +} + +func generateShardedHubHppMsgContainerShards(gen *protogen.Plugin, g *protogen.GeneratedFile, shardNum int) { + for i := 0; i < shardNum; i++ { + g.P(" void InitShard", i, "();") + } + g.P() + g.P(" private:") +} + +func generateShardedHubHppRegistryShards(gen *protogen.Plugin, g *protogen.GeneratedFile, shardNum int) { + for i := 0; i < shardNum; i++ { + g.P(" static void InitShard", i, "();") + } + g.P() + g.P(" private:") +} + +func generateShardedHubCppMsgContainerShards(gen *protogen.Plugin, g *protogen.GeneratedFile, shardNum int) { + for i := 0; i < shardNum; i++ { + g.P(" InitShard", i, "();") + } +} + +func generateShardedHubCppRegistryShards(gen *protogen.Plugin, g *protogen.GeneratedFile, shardNum int) { + for i := 0; i < shardNum; i++ { + g.P(" InitShard", i, "();") + } +} + +func generateShardedHubCppFileContent(gen *protogen.Plugin, g *protogen.GeneratedFile, shardIndex int, protofiles []string, fileMessagers map[string][]string) { + g.P(`#include "`, "hub.", pcExt, `.h"`) + g.P() + for _, proto := range protofiles { + g.P(`#include "`, proto, ".", pcExt, `.h"`) + } + g.P() + g.P("namespace ", *namespace, " {") + + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P("template <>") + g.P("const std::shared_ptr<" + messager + "> Hub::Get<" + messager + ">() const {") + g.P(" return GetMessagerContainer()->", strcase.ToSnake(messager), "_;") + g.P("}") + g.P() + } + } + + g.P("void MessagerContainer::InitShard", shardIndex, "() {") + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P(" ", strcase.ToSnake(messager), "_ = std::dynamic_pointer_cast<", messager, `>((*msger_map_)["`, messager, `"]);`) + } + } + g.P("}") + g.P() + + g.P("void Registry::InitShard", shardIndex, "() {") + for _, proto := range protofiles { + for _, messager := range fileMessagers[proto] { + g.P(" Register<", messager, ">();") + } + } + g.P("}") + + g.P("} // namespace ", *namespace) +} diff --git a/test/cpp-tableau-loader/gen.sh b/test/cpp-tableau-loader/gen.sh index 97bb72b7..43d83360 100755 --- a/test/cpp-tableau-loader/gen.sh +++ b/test/cpp-tableau-loader/gen.sh @@ -26,7 +26,7 @@ export PATH="${PATH}:${PLGUIN_DIR}" ${PROTOC} \ --cpp-tableau-loader_out="$PROTOCONF_OUT" \ - --cpp-tableau-loader_opt=paths=source_relative,registry-shards=2 \ + --cpp-tableau-loader_opt=paths=source_relative,shards=2 \ --cpp_out="$PROTOCONF_OUT" \ --proto_path="$PROTOBUF_PROTO" \ --proto_path="$TABLEAU_PROTO" \ diff --git a/test/cpp-tableau-loader/src/CMakeLists.txt b/test/cpp-tableau-loader/src/CMakeLists.txt index 3f054767..d8f72795 100644 --- a/test/cpp-tableau-loader/src/CMakeLists.txt +++ b/test/cpp-tableau-loader/src/CMakeLists.txt @@ -5,13 +5,35 @@ project(loader) file(GLOB_RECURSE PROTO_SOURCE *.cc) file(GLOB_RECURSE SOURCE *.cpp) +# Add option to switch between C++11 and C++17 +option(USE_CPP17 "Use C++17 standard (otherwise use C++11)" OFF) + # specify the C++ standard -set(CMAKE_CXX_STANDARD 11) +if(USE_CPP17) + set(CMAKE_CXX_STANDARD 17) + message(STATUS "Using C++17 standard") +else() + set(CMAKE_CXX_STANDARD 11) + message(STATUS "Using C++11 standard") +endif() set(CMAKE_CXX_STANDARD_REQUIRED True) + if (MSVC) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /DNDEBUG") + if(USE_CPP17) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /DNDEBUG /std:c++17") + else() + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /DNDEBUG /std:c++11") + endif() else() - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -fPIC -std=c++11 -Wno-deprecated -Wno-unused-variable -Wno-sign-compare -Wno-strict-aliasing -fno-strict-aliasing -DNDEBUG") + if(USE_CPP17) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -fPIC -std=c++17 -Wno-deprecated -Wno-unused-variable -Wno-sign-compare -Wno-strict-aliasing -fno-strict-aliasing -DNDEBUG") + # For older GCC versions that need explicit linking with stdc++fs + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9.0") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lstdc++fs") + endif() + else() + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -fPIC -std=c++11 -Wno-deprecated -Wno-unused-variable -Wno-sign-compare -Wno-strict-aliasing -fno-strict-aliasing -DNDEBUG") + endif() endif() # root dir define @@ -43,6 +65,7 @@ else() SET(COMMON_LIB ${PROTOBUF_LIB}/libprotobuf.a pthread + $<$:stdc++fs> ) endif() diff --git a/test/cpp-tableau-loader/src/hub/hub.cpp b/test/cpp-tableau-loader/src/hub/hub.cpp index bba703a3..4c973299 100644 --- a/test/cpp-tableau-loader/src/hub/hub.cpp +++ b/test/cpp-tableau-loader/src/hub/hub.cpp @@ -3,7 +3,6 @@ #include "hub.h" #include "hub/custom/item/custom_item_conf.h" #include "protoconf/logger.pc.h" -#include "protoconf/registry.pc.h" void LogWrite(std::ostream* os, const tableau::log::SourceLocation& loc, const tableau::log::LevelInfo& lvl, const std::string& content) { diff --git a/test/cpp-tableau-loader/src/main.cpp b/test/cpp-tableau-loader/src/main.cpp index b8b69754..53fde80a 100644 --- a/test/cpp-tableau-loader/src/main.cpp +++ b/test/cpp-tableau-loader/src/main.cpp @@ -125,10 +125,6 @@ int main() { tableau::LoadOptions options; options.ignore_unknown_fields = true; options.patch_dirs = {"../../testdata/patchconf/"}; - options.postprocessor = [](const tableau::Hub& hub) { - std::cout << "post process done!" << std::endl; - return 1; - }; options.paths["ItemConf"] = "../../testdata/conf/ItemConf.json"; bool ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, &options); @@ -136,8 +132,8 @@ int main() { std::cout << "protobuf hub load failed: " << tableau::GetErrMsg() << std::endl; return 1; } - auto msger_container = Hub::Instance().GetMessagerContainer(); - for (auto&& item : *msger_container) { + auto msger_map = Hub::Instance().GetMessagerMap(); + for (auto&& item : *msger_map) { auto&& stats = item.second->GetStats(); ATOM_DEBUG("%s: duration: %dus", item.first.c_str(), stats.duration.count()); } diff --git a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc index 793e9e26..60cff677 100644 --- a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc @@ -7,6 +7,7 @@ #include "hero_conf.pc.h" #include "hub.pc.h" +#include "util.pc.h" namespace tableau { const std::string HeroConf::kProtoName = "HeroConf"; diff --git a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h index 279b1740..53ec91a2 100644 --- a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h @@ -7,7 +7,8 @@ #pragma once #include -#include "hub.pc.h" +#include "messager.pc.h" +#include "util.pc.h" #include "hero_conf.pb.h" namespace tableau { diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index 303a00c5..a39c0353 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -9,479 +9,55 @@ #include #include -#ifdef _WIN32 -#include -#include -#else -#include -#endif - -#include -#include #include -#include #include "logger.pc.h" -#include "registry.pc.h" +#include "messager.pc.h" +#include "util.pc.h" namespace tableau { -#ifdef _WIN32 -#undef GetMessage -#define mkdir(path, mode) _mkdir(path) -static constexpr char kPathSeperator = '\\'; -#else -static constexpr char kPathSeperator = '/'; -#endif - -static thread_local std::string g_err_msg; -const std::string& GetErrMsg() { return g_err_msg; } - -const std::string kUnknownExt = ".unknown"; -const std::string kJSONExt = ".json"; -const std::string kTextExt = ".txt"; -const std::string kBinExt = ".bin"; - -Format Ext2Format(const std::string& ext) { - if (ext == kJSONExt) { - return Format::kJSON; - } else if (ext == kTextExt) { - return Format::kText; - } else if (ext == kBinExt) { - return Format::kBin; - } else { - return Format::kUnknown; - } -} - -const std::string& Format2Ext(Format fmt) { - switch (fmt) { - case Format::kJSON: - return kJSONExt; - case Format::kText: - return kTextExt; - case Format::kBin: - return kBinExt; - default: - return kUnknownExt; - } -} - -// refer: https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/stubs/logging.h -void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg) { - static const std::unordered_map kLevelMap = {{google::protobuf::LOGLEVEL_INFO, log::kInfo}, - {google::protobuf::LOGLEVEL_WARNING, log::kWarn}, - {google::protobuf::LOGLEVEL_ERROR, log::kError}, - {google::protobuf::LOGLEVEL_FATAL, log::kFatal}}; - log::Level lvl = log::kWarn; // default - auto iter = kLevelMap.find(level); - if (iter != kLevelMap.end()) { - lvl = iter->second; - } - ATOM_LOGGER_CALL(tableau::log::DefaultLogger(), lvl, "[libprotobuf %s:%d] %s", filename, line, msg.c_str()); -} - -const std::string& GetProtoName(const google::protobuf::Message& msg) { - const auto* md = msg.GetDescriptor(); - return md != nullptr ? md->name() : kEmpty; -} - -bool ExistsFile(const std::string& filename) { - std::ifstream file(filename); - // returns true if the file exists and is accessible - return file.good(); -} - -bool ReadFile(const std::string& filename, std::string& content) { - std::ifstream file(filename); - if (!file.is_open()) { - g_err_msg = "failed to open " + filename + ": " + strerror(errno); - return false; - } - std::stringstream ss; - ss << file.rdbuf(); - content = ss.str(); - return true; -} - -std::string GetPatchName(tableau::Patch patch) { - auto* descriptor = tableau::Patch_descriptor(); - if (descriptor) { - auto* value = descriptor->FindValueByNumber(patch); - if (value) { - return value->name(); - } - } - return std::to_string(static_cast(patch)); -} - -bool Message2JSON(const google::protobuf::Message& msg, std::string& json) { - google::protobuf::util::JsonPrintOptions options; - options.add_whitespace = true; - options.always_print_primitive_fields = true; - options.preserve_proto_field_names = true; - return google::protobuf::util::MessageToJsonString(msg, &json, options).ok(); -} - -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options /* = nullptr */) { - google::protobuf::util::Status status; - if (options != nullptr) { - google::protobuf::util::JsonParseOptions parse_options; - parse_options.ignore_unknown_fields = options->ignore_unknown_fields; - status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); - } else { - status = google::protobuf::util::JsonStringToMessage(json, &msg); - } - if (!status.ok()) { - g_err_msg = "failed to parse " + GetProtoName(msg) + kJSONExt + ": " + status.ToString(); - return false; - } - return true; -} - -bool Text2Message(const std::string& text, google::protobuf::Message& msg) { - if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) { - g_err_msg = "failed to parse " + GetProtoName(msg) + kTextExt; - return false; - } - return true; -} -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) { - if (!msg.ParseFromString(bin)) { - g_err_msg = "failed to parse " + GetProtoName(msg) + kBinExt; - return false; - } - return true; -} - -// PatchMessage patches src into dst, which must be a message with the same descriptor. -// -// # Default PatchMessage mechanism -// - scalar: Populated scalar fields in src are copied to dst. -// - message: Populated singular messages in src are merged into dst by -// recursively calling [xproto.PatchMessage], or replace dst message if -// "PATCH_REPLACE" is specified for this field. -// - list: The elements of every list field in src are appended to the -// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" -// is specified for this field. -// - map: The entries of every map field in src are MERGED (different from -// the behavior of proto.Merge) into the corresponding map field in dst, -// or replace dst map if "PATCH_REPLACE" is specified for this field. -// - unknown: The unknown fields of src are appended to the unknown -// fields of dst (TODO: untested). -// -// # References: -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details -bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { - const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); - const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); - // Ensure both messages are of the same type - if (dst_descriptor != src_descriptor) { - g_err_msg = "dst and src are not messages with the same descriptor"; - ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), - src_descriptor->name().c_str()); - return false; - } - - // Get the reflection and descriptor for the messages - const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); - const google::protobuf::Reflection* src_reflection = src.GetReflection(); - - // List all populated fields - std::vector fields; - src_reflection->ListFields(src, &fields); - - // Iterates over every populated field. - for (auto fd : fields) { - const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); - tableau::Patch patch = opts.prop().patch(); - if (patch == tableau::PATCH_REPLACE) { - dst_reflection->ClearField(&dst, fd); - } - if (fd->is_map()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 - auto key_fd = fd->message_type()->map_key(); - auto value_fd = fd->message_type()->map_value(); - int src_count = src_reflection->FieldSize(src, fd); - int dst_count = dst_reflection->FieldSize(dst, fd); - switch (key_fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - std::unordered_map dst_key_index_map; \ - for (int i = 0; i < dst_count; i++) { \ - auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ - TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ - dst_key_index_map[key] = i; \ - } \ - for (int j = 0; j < src_count; j++) { \ - auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ - TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ - auto it = dst_key_index_map.find(key); \ - if (it != dst_key_index_map.end()) { \ - int index = it->second; \ - auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ - PatchMessage(dst_entry, src_entry); \ - } else { \ - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ - } \ - } \ - break; \ - } - - HANDLE_TYPE(INT32, Int32, int32_t); - HANDLE_TYPE(INT64, Int64, int64_t); - HANDLE_TYPE(UINT32, UInt32, uint32_t); - HANDLE_TYPE(UINT64, UInt64, uint64_t); - HANDLE_TYPE(BOOL, Bool, bool); - HANDLE_TYPE(STRING, String, std::string); - default: { - // other types are impossible to be protobuf map key - ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); - break; - } -#undef HANDLE_TYPE - } - } else if (fd->is_repeated()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 - int count = src_reflection->FieldSize(src, fd); - for (int j = 0; j < count; j++) { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ - break; \ - } - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { - const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); - break; - } - } - } - } else { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ - dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ - break; - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: - const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); - PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); - break; - } - } - } - - dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); - return true; -} - -bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& path, Format fmt, tableau::Patch patch, - const LoadOptions* options /* = nullptr*/) { - if (options == nullptr) { - return LoadMessageByPath(msg, path, fmt, nullptr); - } - if (options->mode == LoadMode::kModeOnlyMain) { - // ignore patch files when LoadMode::kModeOnlyMain specified - return LoadMessageByPath(msg, path, fmt, nullptr); - } - std::string name = GetProtoName(msg); - std::vector patch_paths; - auto iter = options->patch_paths.find(name); - if (iter != options->patch_paths.end()) { - // patch path specified in PatchPaths, then use it instead of PatchDirs. - patch_paths = iter->second; - } else { - for (auto&& patch_dir : options->patch_dirs) { - patch_paths.emplace_back(patch_dir + name + Format2Ext(fmt)); - } - } - - std::vector existed_patch_paths; - for (auto&& patch_path : patch_paths) { - if (ExistsFile(patch_path)) { - existed_patch_paths.emplace_back(patch_path); - } - } - if (existed_patch_paths.empty()) { - if (options->mode == LoadMode::kModeOnlyPatch) { - // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. - return true; - } - // no valid patch path provided, then just load from the "main" file. - return LoadMessageByPath(msg, path, fmt, options); - } - - switch (patch) { - case tableau::PATCH_REPLACE: { - // just use the last "patch" file - std::string& patch_path = existed_patch_paths.back(); - if (!LoadMessageByPath(msg, patch_path, Ext2Format(util::GetExt(patch_path)), options)) { - return false; - } - break; - } - case tableau::PATCH_MERGE: { - if (options->mode != LoadMode::kModeOnlyPatch) { - // load msg from the "main" file - if (!LoadMessageByPath(msg, path, fmt, options)) { - return false; - } - } - // Create a new instance of the same type of the original message - google::protobuf::Message* patch_msg_ptr = msg.New(); - std::unique_ptr _auto_release(msg.New()); - // load patch_msg from each "patch" file - for (auto&& patch_path : existed_patch_paths) { - if (!LoadMessageByPath(*patch_msg_ptr, patch_path, Ext2Format(util::GetExt(patch_path)), options)) { - return false; - } - if (!PatchMessage(msg, *patch_msg_ptr)) { - return false; - } - } - break; - } - default: { - g_err_msg = "unknown patch type: " + GetPatchName(patch); - return false; - } - } - ATOM_DEBUG("patched(%s) %s by %s: %s", GetPatchName(patch).c_str(), name.c_str(), - ATOM_VECTOR_STR(existed_patch_paths).c_str(), msg.ShortDebugString().c_str()); - return true; -} - -bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt, - const LoadOptions* options /* = nullptr*/) { - std::string content; - ReadFunc read_func = ReadFile; - if (options != nullptr && options->read_func) { - read_func = options->read_func; - } - bool ok = read_func(path, content); - if (!ok) { - return false; - } - switch (fmt) { - case Format::kJSON: { - return JSON2Message(content, msg, options); - } - case Format::kText: { - return Text2Message(content, msg); - } - case Format::kBin: { - return Bin2Message(content, msg); - } - default: { - g_err_msg = "unknown format: " + std::to_string(static_cast(fmt)); - return false; - } - } -} - -bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt, - const LoadOptions* options /* = nullptr*/) { - std::string name = GetProtoName(msg); - std::string path; - if (options) { - auto iter = options->paths.find(name); - if (iter != options->paths.end()) { - // path specified in Paths, then use it instead of dir. - path = iter->second; - fmt = Ext2Format(util::GetExt(iter->second)); - } - } - if (path.empty()) { - path = dir + name + Format2Ext(fmt); - } - - const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); - if (!descriptor) { - g_err_msg = "failed to get descriptor of message: " + name; - return false; - } - // access the extension directly using the generated identifier - const tableau::WorksheetOptions worksheet_options = descriptor->options().GetExtension(tableau::worksheet); - if (worksheet_options.patch() != tableau::PATCH_NONE) { - return LoadMessageWithPatch(msg, path, fmt, worksheet_options.patch(), options); - } - - return LoadMessageByPath(msg, path, fmt, options); -} - -bool StoreMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt) { - // TODO: write protobuf message to file, support 3 formats: json, text, and bin. - return false; -} +Registrar Registry::registrar = Registrar(); bool Hub::Load(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { - auto msger_container = LoadNewMessagerContainer(dir, fmt, options); - if (!msger_container) { + auto msger_map = InternalLoad(dir, fmt, options); + if (!msger_map) { return false; } - bool ok = internal::Postprocess(options->postprocessor, msger_container); + bool ok = Postprocess(msger_map); if (!ok) { return false; } - SetMessagerContainer(msger_container); + SetMessagerMap(msger_map); return true; } bool Hub::AsyncLoad(const std::string& dir, Format fmt /* = Format::kJSON */, const LoadOptions* options /* = nullptr */) { - auto msger_container = LoadNewMessagerContainer(dir, fmt, options); - if (!msger_container) { + auto msger_map = InternalLoad(dir, fmt, options); + if (!msger_map) { return false; } - bool ok = internal::Postprocess(options->postprocessor, msger_container); + bool ok = Postprocess(msger_map); if (!ok) { return false; } - sched_->Dispatch(std::bind(&Hub::SetMessagerContainer, this, msger_container)); + sched_->Dispatch(std::bind(&Hub::SetMessagerMap, this, msger_map)); return true; } int Hub::LoopOnce() { return sched_->LoopOnce(); } + void Hub::InitScheduler() { sched_ = new internal::Scheduler(); sched_->Current(); } -MessagerContainer Hub::LoadNewMessagerContainer(const std::string& dir, Format fmt /* = Format::kJSON */, - const LoadOptions* options /* = nullptr */) { +std::shared_ptr Hub::InternalLoad(const std::string& dir, Format fmt /* = Format::kJSON */, + const LoadOptions* options /* = nullptr */) const { // intercept protobuf error logs - auto old_handler = google::protobuf::SetLogHandler(ProtobufLogHandler); - auto msger_container = NewMessagerContainer(); - for (auto iter : *msger_container) { + auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); + auto msger_map = NewMessagerMap(); + for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); bool ok = iter.second->Load(dir, fmt, options); @@ -496,155 +72,66 @@ MessagerContainer Hub::LoadNewMessagerContainer(const std::string& dir, Format f // restore to old protobuf log handler google::protobuf::SetLogHandler(old_handler); - return msger_container; + return msger_map; } -MessagerContainer Hub::NewMessagerContainer() { - MessagerContainer msger_container = std::make_shared(); +std::shared_ptr Hub::NewMessagerMap() const { + std::shared_ptr msger_map = std::make_shared(); for (auto&& it : Registry::registrar) { if (!options_.filter || options_.filter(it.first)) { - (*msger_container)[it.first] = it.second(); + (*msger_map)[it.first] = it.second(); } } - return msger_container; + return msger_map; } -void Hub::SetMessagerContainer(MessagerContainer msger_container) { +std::shared_ptr Hub::GetMessagerMap() const { return GetMessagerContainer()->msger_map_; } + +void Hub::SetMessagerMap(std::shared_ptr msger_map) { // replace with thread-safe guarantee. std::unique_lock lock(mutex_); - msger_container_ = msger_container; - last_loaded_time_ = std::time(nullptr); -} - -MessagerContainer Hub::GetMessagerContainerWithProvider() const { - if (msger_container_provider_ != nullptr) { - return msger_container_provider_(); - } - return msger_container_; + msger_container_ = std::make_shared(msger_map); } const std::shared_ptr Hub::GetMessager(const std::string& name) const { - auto container = GetMessagerContainerWithProvider(); - if (container) { - auto it = container->find(name); - if (it != container->end()) { + auto msger_map = GetMessagerMap(); + if (msger_map) { + auto it = msger_map->find(name); + if (it != msger_map->end()) { return it->second; } } return nullptr; } -namespace internal { -// Thread-local storage (TLS) -thread_local Scheduler* tls_sched = nullptr; -Scheduler& Scheduler::Current() { - if (tls_sched == nullptr) { - tls_sched = new Scheduler; - } - return *tls_sched; -} - -int Scheduler::LoopOnce() { - AssertInLoopThread(); - - int count = 0; - std::vector jobs; - { - // scoped for auto-release lock. - // wake up immediately when there are pending tasks. - std::unique_lock lock(mutex_); - jobs.swap(jobs_); - } - for (auto&& job : jobs) { - job(); - } - count += jobs.size(); - return count; -} - -void Scheduler::Post(const Job& job) { - std::unique_lock lock(mutex_); - jobs_.push_back(job); -} - -void Scheduler::Dispatch(const Job& job) { - if (IsLoopThread()) { - job(); // run it immediately - } else { - Post(job); // post and run it at next loop - } -} - -bool Scheduler::IsLoopThread() const { return thread_id_ == std::this_thread::get_id(); } -void Scheduler::AssertInLoopThread() const { - if (!IsLoopThread()) { - abort(); - } -} - -bool Postprocess(Postprocessor postprocessor, MessagerContainer container) { +bool Hub::Postprocess(std::shared_ptr msger_map) { // create a temporary hub with messager container for post process - Hub hub(container); + Hub tmp_hub; + tmp_hub.SetMessagerMap(msger_map); // messager-level postprocess - for (auto iter : *container) { + for (auto iter : *msger_map) { auto msger = iter.second; - bool ok = msger->ProcessAfterLoadAll(hub); + bool ok = msger->ProcessAfterLoadAll(tmp_hub); if (!ok) { - g_err_msg = "hub call ProcessAfterLoadAll failed, messager: " + msger->Name(); + SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + msger->Name()); return false; } } - - // hub-level postprocess - if (postprocessor != nullptr) { - bool ok = postprocessor(hub); - if (!ok) { - g_err_msg = "hub call Postprocesser failed, you'd better check your custom 'postprocessor' load option"; - return false; - } - } - return true; } -} // namespace internal +std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainer()->last_loaded_time_; } -namespace util { -int Mkdir(const std::string& path) { - std::string path_ = path + kPathSeperator; - struct stat info; - for (size_t pos = path_.find(kPathSeperator, 0); pos != std::string::npos; pos = path_.find(kPathSeperator, pos)) { - ++pos; - auto sub_dir = path_.substr(0, pos); - if (stat(sub_dir.c_str(), &info) == 0 && info.st_mode & S_IFDIR) { - continue; - } - int status = mkdir(sub_dir.c_str(), 0755); - if (status != 0) { - std::cerr << "system error: " << strerror(errno) << std::endl; - return -1; - } - } - return 0; -} - -std::string GetDir(const std::string& path) { - size_t pos = path.find_last_of(kPathSeperator); - if (pos != std::string::npos) { - return path.substr(0, pos); - } - return kEmpty; +MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) + : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), + last_loaded_time_(std::time(nullptr)) { + InitShard0(); + InitShard1(); } -std::string GetExt(const std::string& path) { - std::size_t pos = path.find_last_of("."); - if (pos != std::string::npos) { - return path.substr(pos); - } - return kEmpty; +void Registry::Init() { + InitShard0(); + InitShard1(); } - -} // namespace util - } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index c735ab36..62fa5779 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -8,158 +8,39 @@ #include #include -#include -#include -#include #include #include -#include #include #include -#include #include -#include -namespace tableau { -enum class Format { - kUnknown, - kJSON, - kText, - kBin, -}; - -enum class LoadMode { - kModeDefault, - kModeOnlyMain, - kModeOnlyPatch, -}; +#include "load.pc.h" +#include "messager.pc.h" +#include "scheduler.pc.h" -extern const std::string kUnknownExt; -extern const std::string kJSONExt; -extern const std::string kTextExt; -extern const std::string kBinExt; - -static const std::string kEmpty = ""; -const std::string& GetErrMsg(); - -class Messager; +namespace tableau { +class MessagerContainer; class Hub; using MessagerMap = std::unordered_map>; -using MessagerContainer = std::shared_ptr; // FilterFunc filter in messagers if returned value is true. // NOTE: name is the protobuf message name, e.g.: "message ItemConf{...}". using Filter = std::function; -using MessagerContainerProvider = std::function; -using Postprocessor = std::function; -// ReadFunc reads the config file and returns its content. -using ReadFunc = std::function; +using MessagerContainerProvider = std::function()>; struct HubOptions { // Filter can only filter in certain specific messagers based on the // condition that you provide. Filter filter; -}; - -struct LoadOptions { - // postprocessor is called after loading all configurations. - Postprocessor postprocessor; - // read_func reads the config file and returns its content. - ReadFunc read_func; - // Whether to ignore unknown JSON fields during parsing. - // - // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. - bool ignore_unknown_fields = false; - // Paths maps each messager name to a corresponding config file path. - // If specified, then the main messager will be parsed from the file - // directly, other than the specified load dir. - std::unordered_map paths; - // Patch paths maps each messager name to one or multiple corresponding patch file paths. - // If specified, then main messager will be patched. - std::unordered_map> patch_paths; - // Patch dirs specifies the directory paths for config patching. - std::vector patch_dirs; - // Mode specifies the loading mode for config patching. - LoadMode mode = LoadMode::kModeDefault; -}; - -// Convert file extension to Format type. -// NOTE: ext includes dot ".", such as: -// - kJSONExt：".json" -// - kTextExt".txt" -// - kBinExt".bin" -Format Ext2Format(const std::string& ext); -// Empty string will be returned if an unsupported enum value has been passed, -// and the error message can be obtained by GetErrMsg(). -const std::string& Format2Ext(Format fmt); -bool Message2JSON(const google::protobuf::Message& msg, std::string& json); -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options = nullptr); -bool Text2Message(const std::string& text, google::protobuf::Message& msg); -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); -void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg); -const std::string& GetProtoName(const google::protobuf::Message& msg); -bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr); -bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr); - -namespace internal { -class Scheduler { - public: - typedef std::function Job; - - public: - Scheduler() : thread_id_(std::this_thread::get_id()) {} - static Scheduler& Current(); - // thread-safety - void Post(const Job& job); - void Dispatch(const Job& job); - int LoopOnce(); - bool IsLoopThread() const; - void AssertInLoopThread() const; - - private: - std::thread::id thread_id_; - std::mutex mutex_; - std::vector jobs_; -}; - -bool Postprocess(Postprocessor postprocessor, MessagerContainer container); - -} // namespace internal - -class Messager { - public: - struct Stats { - std::chrono::microseconds duration; // total load time consuming. - // TODO: crc32 of config file to decide whether changed or not - // std::string crc32; - // int64_t last_modified_time = 0; // unix timestamp - }; - - public: - virtual ~Messager() = default; - static const std::string& Name() { return kEmpty; } - const Stats& GetStats() { return stats_; } - // Load fills message from file in the specified directory and format. - virtual bool Load(const std::string& dir, Format fmt, const LoadOptions* options = nullptr) = 0; - // Message returns the inner message data. - virtual const google::protobuf::Message* Message() const { return nullptr; } - // callback after all messagers loaded. - virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } - - protected: - // callback after this messager loaded. - virtual bool ProcessAfterLoad() { return true; }; - Stats stats_; + // Provide custom MessagerContainer. For keeping configuration access + // consistent in a coroutine or a transaction. + MessagerContainerProvider provider; }; class Hub { public: - Hub(const HubOptions* options = nullptr) : options_(options ? *options : HubOptions{}) {} - Hub(MessagerContainer container, const HubOptions* options = nullptr) : options_(options ? *options : HubOptions{}) { - SetMessagerContainer(container); - } + Hub(const HubOptions* options = nullptr) + : msger_container_(std::make_shared()), options_(options ? *options : HubOptions{}) {} /***** Synchronous Loading *****/ // Load fills messages (in MessagerContainer) from files in the specified directory and format. bool Load(const std::string& dir, Format fmt = Format::kJSON, const LoadOptions* options = nullptr); @@ -172,9 +53,18 @@ class Hub { // You'd better initialize the scheduler in the main thread. void InitScheduler(); + /***** MessagerMap *****/ + std::shared_ptr GetMessagerMap() const; + void SetMessagerMap(std::shared_ptr msger_map); + /***** MessagerContainer *****/ - MessagerContainer GetMessagerContainer() const { return msger_container_; } - void SetMessagerContainerProvider(MessagerContainerProvider provider) { msger_container_provider_ = provider; } + // This function is exposed only for use in MessagerContainerProvider. + std::shared_ptr GetMessagerContainer() const { + if (options_.provider != nullptr) { + return options_.provider(); + } + return msger_container_; + } /***** Access APIs *****/ template @@ -187,28 +77,23 @@ class Hub { const U* GetOrderedMap(Args... args) const; // GetLastLoadedTime returns the time when hub's msger_container_ was last set. - inline std::time_t GetLastLoadedTime() const { return last_loaded_time_; } + inline std::time_t GetLastLoadedTime() const; private: - MessagerContainer LoadNewMessagerContainer(const std::string& dir, Format fmt = Format::kJSON, - const LoadOptions* options = nullptr); - MessagerContainer NewMessagerContainer(); - void SetMessagerContainer(MessagerContainer msger_container); - MessagerContainer GetMessagerContainerWithProvider() const; + std::shared_ptr InternalLoad(const std::string& dir, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr) const; + std::shared_ptr NewMessagerMap() const; const std::shared_ptr GetMessager(const std::string& name) const; + bool Postprocess(std::shared_ptr msger_map); + private: // For thread-safe guarantee during configuration updating. std::mutex mutex_; // All messagers' container. - MessagerContainer msger_container_; - // Provide custom MessagerContainer. For keeping configuration access - // consistent in a coroutine or a transaction. - MessagerContainerProvider msger_container_provider_; + std::shared_ptr msger_container_; // Loading scheduler. internal::Scheduler* sched_ = nullptr; - // Last loaded time - std::time_t last_loaded_time_ = 0; // Hub options const HubOptions options_; }; @@ -221,69 +106,100 @@ const std::shared_ptr Hub::Get() const { template const U* Hub::Get(Args... args) const { - auto msg = GetMessager(T::Name()); - auto msger = std::dynamic_pointer_cast(msg); + auto msger = Get(); return msger ? msger->Get(args...) : nullptr; } template const U* Hub::GetOrderedMap(Args... args) const { - auto msg = GetMessager(T::Name()); - auto msger = std::dynamic_pointer_cast(msg); + auto msger = Get(); return msger ? msger->GetOrderedMap(args...) : nullptr; } -namespace util { +class HeroBaseConf; +template <> +const std::shared_ptr Hub::Get() const; -// Combine hash values -// -// References: -// - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x -// - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key -inline void HashCombine(std::size_t& seed) {} +class HeroConf; +template <> +const std::shared_ptr Hub::Get() const; -template -inline void HashCombine(std::size_t& seed, const T& v, O... others) { - std::hash hasher; - seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); - HashCombine(seed, others...); -} +class ItemConf; +template <> +const std::shared_ptr Hub::Get() const; -template -inline std::size_t SugaredHashCombine(const T& v, O... others) { - std::size_t seed = 0; // start with a hash value 0 - HashCombine(seed, v, others...); - return seed; -} +class PatchMergeConf; +template <> +const std::shared_ptr Hub::Get() const; + +class PatchReplaceConf; +template <> +const std::shared_ptr Hub::Get() const; -// Mkdir makes dir recursively. -int Mkdir(const std::string& path); -// GetDir returns all but the last element of path, typically the path's -// directory. -std::string GetDir(const std::string& path); -// GetExt returns the file name extension used by path. -// The extension is the suffix beginning at the final dot -// in the final element of path; it is empty if there is -// no dot. -std::string GetExt(const std::string& path); - -class TimeProfiler { - protected: - std::chrono::time_point last_; +class RecursivePatchConf; +template <> +const std::shared_ptr Hub::Get() const; + +class ActivityConf; +template <> +const std::shared_ptr Hub::Get() const; + +class ChapterConf; +template <> +const std::shared_ptr Hub::Get() const; + +class ThemeConf; +template <> +const std::shared_ptr Hub::Get() const; + +class MessagerContainer { + friend class Hub; public: - TimeProfiler() { Start(); } - void Start() { last_ = std::chrono::steady_clock::now(); } - // Calculate duration between the last time point and now, - // and update last time point to now. - std::chrono::microseconds Elapse() { - auto now = std::chrono::steady_clock::now(); - auto duration = now - last_; // This is of type std::chrono::duration - last_ = now; - return std::chrono::duration_cast(duration); - } + MessagerContainer(std::shared_ptr msger_map = nullptr); + + private: + std::shared_ptr msger_map_; + std::time_t last_loaded_time_; + + private: + void InitShard0(); + void InitShard1(); + + private: + std::shared_ptr hero_base_conf_; + std::shared_ptr hero_conf_; + std::shared_ptr item_conf_; + std::shared_ptr patch_merge_conf_; + std::shared_ptr patch_replace_conf_; + std::shared_ptr recursive_patch_conf_; + std::shared_ptr activity_conf_; + std::shared_ptr chapter_conf_; + std::shared_ptr theme_conf_; }; -} // namespace util +using MessagerGenerator = std::function()>; +// messager name -> messager generator +using Registrar = std::unordered_map; +class Registry { + friend class Hub; + + public: + static void Init(); + + template + static void Register(); + private: + static void InitShard0(); + static void InitShard1(); + + private: + static Registrar registrar; +}; + +template +void Registry::Register() { + registrar[T::Name()] = []() { return std::make_shared(); }; +} } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/hub_shard0.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub_shard0.pc.cc new file mode 100644 index 00000000..157445b0 --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/hub_shard0.pc.cc @@ -0,0 +1,38 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#include "hub.pc.h" + +#include "hero_conf.pc.h" +#include "item_conf.pc.h" + +namespace tableau { +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->hero_base_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->hero_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->item_conf_; +} + +void MessagerContainer::InitShard0() { + hero_base_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroBaseConf"]); + hero_conf_ = std::dynamic_pointer_cast((*msger_map_)["HeroConf"]); + item_conf_ = std::dynamic_pointer_cast((*msger_map_)["ItemConf"]); +} + +void Registry::InitShard0() { + Register(); + Register(); + Register(); +} +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/hub_shard1.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub_shard1.pc.cc new file mode 100644 index 00000000..fdaf74b6 --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/hub_shard1.pc.cc @@ -0,0 +1,59 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#include "hub.pc.h" + +#include "patch_conf.pc.h" +#include "test_conf.pc.h" + +namespace tableau { +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->patch_merge_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->patch_replace_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->recursive_patch_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->activity_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->chapter_conf_; +} + +template <> +const std::shared_ptr Hub::Get() const { + return GetMessagerContainer()->theme_conf_; +} + +void MessagerContainer::InitShard1() { + patch_merge_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchMergeConf"]); + patch_replace_conf_ = std::dynamic_pointer_cast((*msger_map_)["PatchReplaceConf"]); + recursive_patch_conf_ = std::dynamic_pointer_cast((*msger_map_)["RecursivePatchConf"]); + activity_conf_ = std::dynamic_pointer_cast((*msger_map_)["ActivityConf"]); + chapter_conf_ = std::dynamic_pointer_cast((*msger_map_)["ChapterConf"]); + theme_conf_ = std::dynamic_pointer_cast((*msger_map_)["ThemeConf"]); +} + +void Registry::InitShard1() { + Register(); + Register(); + Register(); + Register(); + Register(); + Register(); +} +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc index d1440a77..723e2599 100644 --- a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc @@ -7,6 +7,7 @@ #include "item_conf.pc.h" #include "hub.pc.h" +#include "util.pc.h" namespace tableau { const std::string ItemConf::kProtoName = "ItemConf"; diff --git a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h index 3787010a..77442e63 100644 --- a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h @@ -7,7 +7,8 @@ #pragma once #include -#include "hub.pc.h" +#include "messager.pc.h" +#include "util.pc.h" #include "item_conf.pb.h" namespace tableau { diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc new file mode 100644 index 00000000..fa46ea5c --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -0,0 +1,300 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#include "load.pc.h" + +#include "logger.pc.h" +#include "util.pc.h" + +namespace tableau { +#ifdef _WIN32 +#undef GetMessage +#endif + +// PatchMessage patches src into dst, which must be a message with the same descriptor. +// +// # Default PatchMessage mechanism +// - scalar: Populated scalar fields in src are copied to dst. +// - message: Populated singular messages in src are merged into dst by +// recursively calling [xproto.PatchMessage], or replace dst message if +// "PATCH_REPLACE" is specified for this field. +// - list: The elements of every list field in src are appended to the +// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" +// is specified for this field. +// - map: The entries of every map field in src are MERGED (different from +// the behavior of proto.Merge) into the corresponding map field in dst, +// or replace dst map if "PATCH_REPLACE" is specified for this field. +// - unknown: The unknown fields of src are appended to the unknown +// fields of dst (TODO: untested). +// +// # References: +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { + const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); + const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); + // Ensure both messages are of the same type + if (dst_descriptor != src_descriptor) { + SetErrMsg("dst and src are not messages with the same descriptor"); + ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), + src_descriptor->name().c_str()); + return false; + } + + // Get the reflection and descriptor for the messages + const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); + const google::protobuf::Reflection* src_reflection = src.GetReflection(); + + // List all populated fields + std::vector fields; + src_reflection->ListFields(src, &fields); + + // Iterates over every populated field. + for (auto fd : fields) { + const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); + tableau::Patch patch = opts.prop().patch(); + if (patch == tableau::PATCH_REPLACE) { + dst_reflection->ClearField(&dst, fd); + } + if (fd->is_map()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 + auto key_fd = fd->message_type()->map_key(); + auto value_fd = fd->message_type()->map_value(); + int src_count = src_reflection->FieldSize(src, fd); + int dst_count = dst_reflection->FieldSize(dst, fd); + switch (key_fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + std::unordered_map dst_key_index_map; \ + for (int i = 0; i < dst_count; i++) { \ + auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ + TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ + dst_key_index_map[key] = i; \ + } \ + for (int j = 0; j < src_count; j++) { \ + auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ + TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ + auto it = dst_key_index_map.find(key); \ + if (it != dst_key_index_map.end()) { \ + int index = it->second; \ + auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ + PatchMessage(dst_entry, src_entry); \ + } else { \ + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ + } \ + } \ + break; \ + } + + HANDLE_TYPE(INT32, Int32, int32_t); + HANDLE_TYPE(INT64, Int64, int64_t); + HANDLE_TYPE(UINT32, UInt32, uint32_t); + HANDLE_TYPE(UINT64, UInt64, uint64_t); + HANDLE_TYPE(BOOL, Bool, bool); + HANDLE_TYPE(STRING, String, std::string); + default: { + // other types are impossible to be protobuf map key + ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); + break; + } +#undef HANDLE_TYPE + } + } else if (fd->is_repeated()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 + int count = src_reflection->FieldSize(src, fd); + for (int j = 0; j < count; j++) { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ + break; \ + } + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); + break; + } + } + } + } else { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ + dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ + break; + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); + PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); + break; + } + } + } + + dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); + return true; +} + +bool LoadMessageWithPatch(google::protobuf::Message& msg, const std::string& path, Format fmt, tableau::Patch patch, + const LoadOptions* options /* = nullptr*/) { + if (options == nullptr) { + return LoadMessageByPath(msg, path, fmt, nullptr); + } + if (options->mode == LoadMode::kModeOnlyMain) { + // ignore patch files when LoadMode::kModeOnlyMain specified + return LoadMessageByPath(msg, path, fmt, nullptr); + } + std::string name = util::GetProtoName(msg); + std::vector patch_paths; + auto iter = options->patch_paths.find(name); + if (iter != options->patch_paths.end()) { + // patch path specified in PatchPaths, then use it instead of PatchDirs. + patch_paths = iter->second; + } else { + for (auto&& patch_dir : options->patch_dirs) { + patch_paths.emplace_back(patch_dir + name + util::Format2Ext(fmt)); + } + } + + std::vector existed_patch_paths; + for (auto&& patch_path : patch_paths) { + if (util::ExistsFile(patch_path)) { + existed_patch_paths.emplace_back(patch_path); + } + } + if (existed_patch_paths.empty()) { + if (options->mode == LoadMode::kModeOnlyPatch) { + // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. + return true; + } + // no valid patch path provided, then just load from the "main" file. + return LoadMessageByPath(msg, path, fmt, options); + } + + switch (patch) { + case tableau::PATCH_REPLACE: { + // just use the last "patch" file + std::string& patch_path = existed_patch_paths.back(); + if (!LoadMessageByPath(msg, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { + return false; + } + break; + } + case tableau::PATCH_MERGE: { + if (options->mode != LoadMode::kModeOnlyPatch) { + // load msg from the "main" file + if (!LoadMessageByPath(msg, path, fmt, options)) { + return false; + } + } + // Create a new instance of the same type of the original message + google::protobuf::Message* patch_msg_ptr = msg.New(); + std::unique_ptr _auto_release(msg.New()); + // load patch_msg from each "patch" file + for (auto&& patch_path : existed_patch_paths) { + if (!LoadMessageByPath(*patch_msg_ptr, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) { + return false; + } + if (!PatchMessage(msg, *patch_msg_ptr)) { + return false; + } + } + break; + } + default: { + SetErrMsg("unknown patch type: " + util::GetPatchName(patch)); + return false; + } + } + ATOM_DEBUG("patched(%s) %s by %s: %s", util::GetPatchName(patch).c_str(), name.c_str(), + ATOM_VECTOR_STR(existed_patch_paths).c_str(), msg.ShortDebugString().c_str()); + return true; +} + +bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt, + const LoadOptions* options /* = nullptr*/) { + std::string content; + ReadFunc read_func = util::ReadFile; + if (options != nullptr && options->read_func) { + read_func = options->read_func; + } + bool ok = read_func(path, content); + if (!ok) { + return false; + } + switch (fmt) { + case Format::kJSON: { + return util::JSON2Message(content, msg, options); + } + case Format::kText: { + return util::Text2Message(content, msg); + } + case Format::kBin: { + return util::Bin2Message(content, msg); + } + default: { + SetErrMsg("unknown format: " + std::to_string(static_cast(fmt))); + return false; + } + } +} + +bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt, + const LoadOptions* options /* = nullptr*/) { + std::string name = util::GetProtoName(msg); + std::string path; + if (options) { + auto iter = options->paths.find(name); + if (iter != options->paths.end()) { + // path specified in Paths, then use it instead of dir. + path = iter->second; + fmt = util::Ext2Format(util::GetExt(iter->second)); + } + } + if (path.empty()) { + path = dir + name + util::Format2Ext(fmt); + } + + const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); + if (!descriptor) { + SetErrMsg("failed to get descriptor of message: " + name); + return false; + } + // access the extension directly using the generated identifier + const tableau::WorksheetOptions worksheet_options = descriptor->options().GetExtension(tableau::worksheet); + if (worksheet_options.patch() != tableau::PATCH_NONE) { + return LoadMessageWithPatch(msg, path, fmt, worksheet_options.patch(), options); + } + + return LoadMessageByPath(msg, path, fmt, options); +} +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.h b/test/cpp-tableau-loader/src/protoconf/load.pc.h new file mode 100644 index 00000000..e61f957e --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.h @@ -0,0 +1,48 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#pragma once +#include + +#include +#include + +#include "util.pc.h" + +namespace tableau { +enum class LoadMode { + kModeDefault, + kModeOnlyMain, + kModeOnlyPatch, +}; + +// ReadFunc reads the config file and returns its content. +using ReadFunc = std::function; + +struct LoadOptions { + // read_func reads the config file and returns its content. + ReadFunc read_func; + // Whether to ignore unknown JSON fields during parsing. + // + // Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions. + bool ignore_unknown_fields = false; + // Paths maps each messager name to a corresponding config file path. + // If specified, then the main messager will be parsed from the file + // directly, other than the specified load dir. + std::unordered_map paths; + // Patch paths maps each messager name to one or multiple corresponding patch file paths. + // If specified, then main messager will be patched. + std::unordered_map> patch_paths; + // Patch dirs specifies the directory paths for config patching. + std::vector patch_dirs; + // Mode specifies the loading mode for config patching. + LoadMode mode = LoadMode::kModeDefault; +}; + +bool LoadMessageByPath(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr); +bool LoadMessage(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON, + const LoadOptions* options = nullptr); +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/logger.pc.cc b/test/cpp-tableau-loader/src/protoconf/logger.pc.cc index ba7401bf..99e62c6d 100644 --- a/test/cpp-tableau-loader/src/protoconf/logger.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/logger.pc.cc @@ -20,7 +20,7 @@ #include #include -#include "hub.pc.h" +#include "util.pc.h" #ifdef _WIN32 #define gettid() GetCurrentThreadId() diff --git a/test/cpp-tableau-loader/src/protoconf/messager.pc.h b/test/cpp-tableau-loader/src/protoconf/messager.pc.h new file mode 100644 index 00000000..b7151709 --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/messager.pc.h @@ -0,0 +1,43 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#pragma once +#include + +#include +#include + +#include "util.pc.h" + +namespace tableau { +class Hub; +struct LoadOptions; + +class Messager { + public: + struct Stats { + std::chrono::microseconds duration; // total load time consuming. + // TODO: crc32 of config file to decide whether changed or not + // std::string crc32; + // int64_t last_modified_time = 0; // unix timestamp + }; + + public: + virtual ~Messager() = default; + static const std::string& Name() { return kEmpty; } + const Stats& GetStats() { return stats_; } + // Load fills message from file in the specified directory and format. + virtual bool Load(const std::string& dir, Format fmt, const LoadOptions* options = nullptr) = 0; + // Message returns the inner message data. + virtual const google::protobuf::Message* Message() const { return nullptr; } + // callback after all messagers loaded. + virtual bool ProcessAfterLoadAll(const Hub& hub) { return true; } + + protected: + // callback after this messager loaded. + virtual bool ProcessAfterLoad() { return true; }; + Stats stats_; +}; +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc index a6c95051..de89e988 100644 --- a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc @@ -7,6 +7,7 @@ #include "patch_conf.pc.h" #include "hub.pc.h" +#include "util.pc.h" namespace tableau { const std::string PatchReplaceConf::kProtoName = "PatchReplaceConf"; diff --git a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h index b4bbc6d0..44ed81cb 100644 --- a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h @@ -7,7 +7,8 @@ #pragma once #include -#include "hub.pc.h" +#include "messager.pc.h" +#include "util.pc.h" #include "patch_conf.pb.h" namespace tableau { diff --git a/test/cpp-tableau-loader/src/protoconf/registry.pc.cc b/test/cpp-tableau-loader/src/protoconf/registry.pc.cc deleted file mode 100644 index b52a609f..00000000 --- a/test/cpp-tableau-loader/src/protoconf/registry.pc.cc +++ /dev/null @@ -1,14 +0,0 @@ -// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. -// versions: -// - protoc-gen-cpp-tableau-loader v0.8.0 -// - protoc v3.19.3 - -#include "registry.pc.h" - -namespace tableau { -Registrar Registry::registrar = Registrar(); -void Registry::Init() { - InitShard0(); - InitShard1(); -} -} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/registry.pc.h b/test/cpp-tableau-loader/src/protoconf/registry.pc.h deleted file mode 100644 index 968a700b..00000000 --- a/test/cpp-tableau-loader/src/protoconf/registry.pc.h +++ /dev/null @@ -1,29 +0,0 @@ -// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. -// versions: -// - protoc-gen-cpp-tableau-loader v0.8.0 -// - protoc v3.19.3 - -#pragma once -#include "hub.pc.h" -namespace tableau { -using MessagerGenerator = std::function()>; -// messager name -> messager generator -using Registrar = std::unordered_map; -class Registry { - public: - static void Init(); - static void InitShard0(); - static void InitShard1(); - - template - static void Register(); - - static Registrar registrar; -}; - -template -void Registry::Register() { - registrar[T::Name()] = []() { return std::make_shared(); }; -} - -} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/registry_shard0.pc.cc b/test/cpp-tableau-loader/src/protoconf/registry_shard0.pc.cc deleted file mode 100644 index 8ea5e6fc..00000000 --- a/test/cpp-tableau-loader/src/protoconf/registry_shard0.pc.cc +++ /dev/null @@ -1,17 +0,0 @@ -// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. -// versions: -// - protoc-gen-cpp-tableau-loader v0.8.0 -// - protoc v3.19.3 - -#include "registry.pc.h" - -#include "hero_conf.pc.h" -#include "item_conf.pc.h" - -namespace tableau { -void Registry::InitShard0() { - Register(); - Register(); - Register(); -} -} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/registry_shard1.pc.cc b/test/cpp-tableau-loader/src/protoconf/registry_shard1.pc.cc deleted file mode 100644 index fbebb5f6..00000000 --- a/test/cpp-tableau-loader/src/protoconf/registry_shard1.pc.cc +++ /dev/null @@ -1,20 +0,0 @@ -// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. -// versions: -// - protoc-gen-cpp-tableau-loader v0.8.0 -// - protoc v3.19.3 - -#include "registry.pc.h" - -#include "patch_conf.pc.h" -#include "test_conf.pc.h" - -namespace tableau { -void Registry::InitShard1() { - Register(); - Register(); - Register(); - Register(); - Register(); - Register(); -} -} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc new file mode 100644 index 00000000..b375f26b --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.cc @@ -0,0 +1,59 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#include "scheduler.pc.h" + +namespace tableau { +namespace internal { +// Thread-local storage (TLS) +thread_local Scheduler* tls_sched = nullptr; + +Scheduler& Scheduler::Current() { + if (tls_sched == nullptr) { + tls_sched = new Scheduler; + } + return *tls_sched; +} + +int Scheduler::LoopOnce() { + AssertInLoopThread(); + + int count = 0; + std::vector jobs; + { + // scoped for auto-release lock. + // wake up immediately when there are pending tasks. + std::unique_lock lock(mutex_); + jobs.swap(jobs_); + } + for (auto&& job : jobs) { + job(); + } + count += jobs.size(); + return count; +} + +void Scheduler::Post(const Job& job) { + std::unique_lock lock(mutex_); + jobs_.push_back(job); +} + +void Scheduler::Dispatch(const Job& job) { + if (IsLoopThread()) { + job(); // run it immediately + } else { + Post(job); // post and run it at next loop + } +} + +bool Scheduler::IsLoopThread() const { return thread_id_ == std::this_thread::get_id(); } + +void Scheduler::AssertInLoopThread() const { + if (!IsLoopThread()) { + abort(); + } +} +} // namespace internal +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h new file mode 100644 index 00000000..636008d7 --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/scheduler.pc.h @@ -0,0 +1,36 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#pragma once + +#include +#include +#include +#include + +namespace tableau { +namespace internal { +class Scheduler { + public: + typedef std::function Job; + + public: + Scheduler() : thread_id_(std::this_thread::get_id()) {} + static Scheduler& Current(); + // thread-safety + void Post(const Job& job); + void Dispatch(const Job& job); + int LoopOnce(); + bool IsLoopThread() const; + void AssertInLoopThread() const; + + private: + std::thread::id thread_id_; + std::mutex mutex_; + std::vector jobs_; +}; + +} // namespace internal +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc index b004e6d7..014983bb 100644 --- a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc @@ -7,6 +7,7 @@ #include "test_conf.pc.h" #include "hub.pc.h" +#include "util.pc.h" namespace tableau { const std::string ActivityConf::kProtoName = "ActivityConf"; diff --git a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h index d3165227..ee167d33 100644 --- a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h @@ -7,7 +7,8 @@ #pragma once #include -#include "hub.pc.h" +#include "messager.pc.h" +#include "util.pc.h" #include "test_conf.pb.h" namespace tableau { diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.cc b/test/cpp-tableau-loader/src/protoconf/util.pc.cc new file mode 100644 index 00000000..9266d816 --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.cc @@ -0,0 +1,214 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#include "util.pc.h" + +#include +#include + +#include +#include +#include + +#if __cplusplus >= 201703L +#include +namespace fs = std::filesystem; +#else +#ifdef _WIN32 +#include +#include +#else +#include +#endif +#endif + +#include "load.pc.h" +#include "logger.pc.h" +#include "messager.pc.h" + +namespace tableau { +#ifdef _WIN32 +#define mkdir(path, mode) _mkdir(path) +static constexpr char kPathSeperator = '\\'; +#else +static constexpr char kPathSeperator = '/'; +#endif + +static thread_local std::string g_err_msg; +const std::string& GetErrMsg() { return g_err_msg; } +void SetErrMsg(const std::string& msg) { g_err_msg = msg; } + +const std::string kUnknownExt = ".unknown"; +const std::string kJSONExt = ".json"; +const std::string kTextExt = ".txt"; +const std::string kBinExt = ".bin"; + +namespace util { +int Mkdir(const std::string& path) { +#if __cplusplus >= 201703L + std::error_code ec; + if (!fs::create_directories(path, ec)) { + if (ec) { + std::cerr << "system error: " << ec.message() << std::endl; + return -1; + } + } + return 0; +#else + std::string path_ = path + kPathSeperator; + struct stat info; + for (size_t pos = path_.find(kPathSeperator, 0); pos != std::string::npos; pos = path_.find(kPathSeperator, pos)) { + ++pos; + auto sub_dir = path_.substr(0, pos); + if (stat(sub_dir.c_str(), &info) == 0 && info.st_mode & S_IFDIR) { + continue; + } + int status = mkdir(sub_dir.c_str(), 0755); + if (status != 0) { + std::cerr << "system error: " << strerror(errno) << std::endl; + return -1; + } + } + return 0; +#endif +} + +std::string GetDir(const std::string& path) { +#if __cplusplus >= 201703L + return fs::path(path).parent_path().string(); +#else + size_t pos = path.find_last_of(kPathSeperator); + if (pos != std::string::npos) { + return path.substr(0, pos); + } + return kEmpty; +#endif +} + +bool ExistsFile(const std::string& filename) { +#if __cplusplus >= 201703L + return fs::exists(filename); +#else + std::ifstream file(filename); + // returns true if the file exists and is accessible + return file.good(); +#endif +} + +bool ReadFile(const std::string& filename, std::string& content) { + std::ifstream file(filename); + if (!file.is_open()) { + SetErrMsg("failed to open " + filename + ": " + strerror(errno)); + return false; + } + std::stringstream ss; + ss << file.rdbuf(); + content = ss.str(); + return true; +} + +std::string GetExt(const std::string& path) { + std::size_t pos = path.find_last_of("."); + if (pos != std::string::npos) { + return path.substr(pos); + } + return kEmpty; +} + +Format Ext2Format(const std::string& ext) { + if (ext == kJSONExt) { + return Format::kJSON; + } else if (ext == kTextExt) { + return Format::kText; + } else if (ext == kBinExt) { + return Format::kBin; + } else { + return Format::kUnknown; + } +} + +const std::string& Format2Ext(Format fmt) { + switch (fmt) { + case Format::kJSON: + return kJSONExt; + case Format::kText: + return kTextExt; + case Format::kBin: + return kBinExt; + default: + return kUnknownExt; + } +} + +bool Message2JSON(const google::protobuf::Message& msg, std::string& json) { + google::protobuf::util::JsonPrintOptions options; + options.add_whitespace = true; + options.always_print_primitive_fields = true; + options.preserve_proto_field_names = true; + return google::protobuf::util::MessageToJsonString(msg, &json, options).ok(); +} + +bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options /* = nullptr */) { + google::protobuf::util::Status status; + if (options != nullptr) { + google::protobuf::util::JsonParseOptions parse_options; + parse_options.ignore_unknown_fields = options->ignore_unknown_fields; + status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); + } else { + status = google::protobuf::util::JsonStringToMessage(json, &msg); + } + if (!status.ok()) { + SetErrMsg("failed to parse " + GetProtoName(msg) + kJSONExt + ": " + status.ToString()); + return false; + } + return true; +} + +bool Text2Message(const std::string& text, google::protobuf::Message& msg) { + if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) { + SetErrMsg("failed to parse " + GetProtoName(msg) + kTextExt); + return false; + } + return true; +} +bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) { + if (!msg.ParseFromString(bin)) { + SetErrMsg("failed to parse " + GetProtoName(msg) + kBinExt); + return false; + } + return true; +} + +const std::string& GetProtoName(const google::protobuf::Message& msg) { + const auto* md = msg.GetDescriptor(); + return md != nullptr ? md->name() : kEmpty; +} + +std::string GetPatchName(tableau::Patch patch) { + auto* descriptor = tableau::Patch_descriptor(); + if (descriptor) { + auto* value = descriptor->FindValueByNumber(patch); + if (value) { + return value->name(); + } + } + return std::to_string(static_cast(patch)); +} + +// refer: https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/stubs/logging.h +void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg) { + static const std::unordered_map kLevelMap = {{google::protobuf::LOGLEVEL_INFO, log::kInfo}, + {google::protobuf::LOGLEVEL_WARNING, log::kWarn}, + {google::protobuf::LOGLEVEL_ERROR, log::kError}, + {google::protobuf::LOGLEVEL_FATAL, log::kFatal}}; + log::Level lvl = log::kWarn; // default + auto iter = kLevelMap.find(level); + if (iter != kLevelMap.end()) { + lvl = iter->second; + } + ATOM_LOGGER_CALL(tableau::log::DefaultLogger(), lvl, "[libprotobuf %s:%d] %s", filename, line, msg.c_str()); +} +} // namespace util +} // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.h b/test/cpp-tableau-loader/src/protoconf/util.pc.h new file mode 100644 index 00000000..9d24149b --- /dev/null +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.h @@ -0,0 +1,108 @@ +// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT. +// versions: +// - protoc-gen-cpp-tableau-loader v0.8.0 +// - protoc v3.19.3 + +#pragma once +#include + +#include +#include +#include + +#include "tableau/protobuf/tableau.pb.h" + +namespace tableau { +const std::string& GetErrMsg(); +void SetErrMsg(const std::string& msg); + +enum class Format { + kUnknown, + kJSON, + kText, + kBin, +}; + +static const std::string kEmpty = ""; +extern const std::string kUnknownExt; +extern const std::string kJSONExt; +extern const std::string kTextExt; +extern const std::string kBinExt; + +struct LoadOptions; + +namespace util { +// Combine hash values +// +// References: +// - https://stackoverflow.com/questions/2590677/how-do-i-combine-hash-values-in-c0x +// - https://stackoverflow.com/questions/17016175/c-unordered-map-using-a-custom-class-type-as-the-key +inline void HashCombine(std::size_t& seed) {} + +template +inline void HashCombine(std::size_t& seed, const T& v, O... others) { + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + HashCombine(seed, others...); +} + +template +inline std::size_t SugaredHashCombine(const T& v, O... others) { + std::size_t seed = 0; // start with a hash value 0 + HashCombine(seed, v, others...); + return seed; +} + +// Mkdir makes dir recursively. +int Mkdir(const std::string& path); +// GetDir returns all but the last element of path, typically the path's +// directory. +std::string GetDir(const std::string& path); +// ExistsFile checks if a file exists. +bool ExistsFile(const std::string& filename); +// ReadFile reads the file named by filename and returns the contents. +bool ReadFile(const std::string& filename, std::string& content); + +// GetExt returns the file name extension used by path. +// The extension is the suffix beginning at the final dot +// in the final element of path; it is empty if there is +// no dot. +std::string GetExt(const std::string& path); +// Convert file extension to Format type. +// NOTE: ext includes dot ".", such as: +// - kJSONExt：".json" +// - kTextExt".txt" +// - kBinExt".bin" +Format Ext2Format(const std::string& ext); +// Empty string will be returned if an unsupported enum value has been passed, +// and the error message can be obtained by GetErrMsg(). +const std::string& Format2Ext(Format fmt); + +bool Message2JSON(const google::protobuf::Message& msg, std::string& json); +bool JSON2Message(const std::string& json, google::protobuf::Message& msg, const LoadOptions* options = nullptr); +bool Text2Message(const std::string& text, google::protobuf::Message& msg); +bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); + +const std::string& GetProtoName(const google::protobuf::Message& msg); +std::string GetPatchName(tableau::Patch patch); + +void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg); + +class TimeProfiler { + protected: + std::chrono::time_point last_; + + public: + TimeProfiler() { Start(); } + void Start() { last_ = std::chrono::steady_clock::now(); } + // Calculate duration between the last time point and now, + // and update last time point to now. + std::chrono::microseconds Elapse() { + auto now = std::chrono::steady_clock::now(); + auto duration = now - last_; // This is of type std::chrono::duration + last_ = now; + return std::chrono::duration_cast(duration); + } +}; +} // namespace util +} // namespace tableau