diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..287ff3ef --- /dev/null +++ b/.gitattributes @@ -0,0 +1,16 @@ +# Treat all files in this repo as binary, with no git magic updating +# line endings. This produces predictable results in different environments. +# +# Windows users contributing to this repo will need to use a modern version +# of git and editors capable of LF line endings. +# +# Windows .bat files are known to have multiple bugs when run with LF +# endings, and so they are checked in with CRLF endings, with a test +# to catch problems. (See https://github.com/golang/go/issues/37791.) +# +# You can prevent accidental CRLF line endings from entering the repo +# via PR/MR checks. +# +# See https://github.com/golang/go/issues/9281. +# See https://adaptivepatchwork.com/2012/03/01/mind-the-end-of-your-line. +* -text \ No newline at end of file diff --git a/.gitignore b/.gitignore index 87c614f9..f686f554 100644 --- a/.gitignore +++ b/.gitignore @@ -28,7 +28,6 @@ *.app # custom files and directories -third_party/_submodules _out/ build/ bin/ diff --git a/.gitmodules b/.gitmodules index 5877cd25..d0be858b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,8 @@ [submodule "third_party/_submodules/protobuf"] path = third_party/_submodules/protobuf url = https://github.com/protocolbuffers/protobuf + ignore = dirty [submodule "third_party/_submodules/tableau"] path = third_party/_submodules/tableau url = https://github.com/tableauio/tableau + ignore = dirty diff --git a/README.md b/README.md index 23faf9df..f6d472e9 100644 --- a/README.md +++ b/README.md @@ -6,23 +6,38 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). > TODO: [devcontainer](https://code.visualstudio.com/docs/devcontainers/containers) -- Supported OS: Windows, macOS, Linux -- Init protobuf: `bash init.sh` - C++ standard: at least C++17 +- Install: [CMake 3.22](https://github.com/Kitware/CMake/releases/tag/v3.31.8) or above +- Init protobuf: + - macOS or Linux: `bash init.sh` + - Windows: + - Install [Visual Studio 2022](https://visualstudio.microsoft.com/downloads/) + - Environment Setup: Open the appropriate `Developer Command Prompt for VS 2022` from the *Start* menu to ensure `cl.exe` and other build tools are in your `PATH`. + - Change dir to **loader** repo + - Run: `.\init.bat` ## C++ ### Dev at Linux -- Install: **CMake 3.22** or above - Change dir: `cd test/cpp-tableau-loader` - Generate protoconf: `bash ./gen.sh` -- Create build dir: `mkdir build && cd build` -- Run cmake: - - C++17: `cmake ../src/` - - C++20: `cmake ../src/ -DCMAKE_CXX_STANDARD=20` - - clang: `cmake ../src/ -DCMAKE_CXX_COMPILER=clang++` -- Build: `make -j8`, then the **bin** dir will be generated at `test/cpp-tableau-loader/bin`. +- CMake: + - C++17: `cmake -S . -B build` + - C++20: `cmake -S . -B build -DCMAKE_CXX_STANDARD=20` + - clang: `cmake -S . -B build -DCMAKE_CXX_COMPILER=clang++` +- Build: `cmake --build build --parallel 10` +- Run: `./bin/loader` + +### Dev at Windows + +- Change dir: `cd test\cpp-tableau-loader` +- Generate protoconf: `.\gen.bat` +- CMake: + - C++17: `cmake -S . -B build -G "NMake Makefiles"` + - C++20: `cmake -S . -B build -G "NMake Makefiles" -DCMAKE_CXX_STANDARD=20` +- Build: `cmake --build build --parallel 10` +- Run: `.\bin\loader.exe` ### References diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc index 2f5b68f6..88209fee 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc @@ -1,77 +1,83 @@ #include "load.pc.h" -#include +#include +#include #include "logger.pc.h" #include "util.pc.h" namespace tableau { - -// Forward declaration of the PatchMessage function -bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); - -std::shared_ptr ParseLoadOptions(std::shared_ptr opts) { - std::shared_ptr new_opts = std::make_shared(); - // set default values - new_opts->mode = LoadMode::kAll; - new_opts->read_func = util::ReadFile; - new_opts->load_func = LoadMessager; - - if (opts == nullptr) { - return new_opts; +namespace load { +std::shared_ptr Options::ParseMessagerOptionsByName(const std::string& name) const { + std::shared_ptr mopts = std::make_shared(); + if (auto iter = messager_options.find(name); iter != messager_options.end() && iter->second) { + mopts = std::make_shared(*iter->second); } - if (opts->ignore_unknown_fields.has_value()) { - new_opts->ignore_unknown_fields = opts->ignore_unknown_fields; + if (!mopts->ignore_unknown_fields.has_value()) { + mopts->ignore_unknown_fields = ignore_unknown_fields; } - if (!opts->patch_dirs.empty()) { - new_opts->patch_dirs = opts->patch_dirs; + if (mopts->patch_dirs.empty()) { + mopts->patch_dirs = patch_dirs; } - if (opts->mode != LoadMode::kNone) { - new_opts->mode = opts->mode; + if (!mopts->mode.has_value()) { + mopts->mode = mode; } - if (opts->read_func != nullptr) { - new_opts->read_func = opts->read_func; + if (!mopts->read_func) { + mopts->read_func = read_func; } - if (opts->load_func != nullptr) { - new_opts->load_func = opts->load_func; + if (!mopts->load_func) { + mopts->load_func = load_func; } - return new_opts; + return mopts; } -std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, - const std::string& name) { - std::shared_ptr mopts; - if (auto iter = opts->messager_options.find(name); iter != opts->messager_options.end()) { - mopts = iter->second; - } else { - mopts = std::make_shared(); - } - if (mopts->ignore_unknown_fields.has_value()) { - mopts->ignore_unknown_fields = opts->ignore_unknown_fields; - } - if (mopts->patch_dirs.empty()) { - mopts->patch_dirs = opts->patch_dirs; +// Forward declaration +bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, + tableau::Patch patch, std::shared_ptr options = nullptr); + +bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, + std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); + std::string content; + bool ok = options->GetReadFunc()(path, content); + if (!ok) { + return false; } - if (mopts->mode == LoadMode::kNone) { - mopts->mode = opts->mode; + return Unmarshal(content, msg, fmt, options); +} + +bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt, + std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); + const std::string& name = msg.GetDescriptor()->name(); + std::filesystem::path path; + if (!options->path.empty()) { + // path specified in Paths, then use it instead of dir. + path = options->path; + fmt = util::GetFormat(path); } - if (mopts->read_func == nullptr) { - mopts->read_func = opts->read_func; + if (path.empty()) { + std::filesystem::path filename = name + util::Format2Ext(fmt); + path = dir / filename; } - if (mopts->load_func == nullptr) { - mopts->load_func = opts->load_func; + + const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); + // access the extension directly using the generated identifier + const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet); + if (worksheet_options.patch() != tableau::PATCH_NONE) { + return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); } - return mopts; + return options->GetLoadFunc()(msg, path, fmt, options); } bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, tableau::Patch patch, std::shared_ptr options /* = nullptr*/) { - if (options == nullptr) { - return LoadMessager(msg, path, fmt, nullptr); - } - if (options->mode == LoadMode::kOnlyMain) { + options = options ? options : std::make_shared(); + auto mode = options->GetMode(); + auto load_func = options->GetLoadFunc(); + if (mode == LoadMode::kOnlyMain) { // ignore patch files when LoadMode::kModeOnlyMain specified - return options->load_func(msg, path, fmt, nullptr); + return load_func(msg, path, fmt, nullptr); } const std::string& name = msg.GetDescriptor()->name(); std::vector patch_paths; @@ -92,27 +98,27 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem } } if (existed_patch_paths.empty()) { - if (options->mode == LoadMode::kOnlyPatch) { + if (mode == LoadMode::kOnlyPatch) { // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. return true; } // no valid patch path provided, then just load from the "main" file. - return options->load_func(msg, path, fmt, options); + return load_func(msg, path, fmt, options); } switch (patch) { case tableau::PATCH_REPLACE: { // just use the last "patch" file std::filesystem::path& patch_path = existed_patch_paths.back(); - if (!options->load_func(msg, patch_path, util::GetFormat(patch_path), options)) { + if (!load_func(msg, patch_path, util::GetFormat(patch_path), options)) { return false; } break; } case tableau::PATCH_MERGE: { - if (options->mode != LoadMode::kOnlyPatch) { + if (mode != LoadMode::kOnlyPatch) { // load msg from the "main" file - if (!options->load_func(msg, path, fmt, options)) { + if (!load_func(msg, path, fmt, options)) { return false; } } @@ -121,10 +127,10 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem std::unique_ptr _auto_release(msg.New()); // load patch_msg from each "patch" file for (auto&& patch_path : existed_patch_paths) { - if (!options->load_func(*patch_msg_ptr, patch_path, util::GetFormat(patch_path), options)) { + if (!load_func(*patch_msg_ptr, patch_path, util::GetFormat(patch_path), options)) { return false; } - if (!PatchMessage(msg, *patch_msg_ptr)) { + if (!util::PatchMessage(msg, *patch_msg_ptr)) { return false; } } @@ -140,26 +146,33 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem return true; } -bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, - std::shared_ptr options /* = nullptr*/) { - std::string content; - ReadFunc read_func = util::ReadFile; - if (options != nullptr && options->read_func) { - read_func = options->read_func; - } - bool ok = read_func(path, content); - if (!ok) { - return false; - } +bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Format fmt, + std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); switch (fmt) { case Format::kJSON: { - return util::JSON2Message(content, msg, options); + google::protobuf::util::JsonParseOptions parse_options; + parse_options.ignore_unknown_fields = options->GetIgnoreUnknownFields(); + auto status = google::protobuf::util::JsonStringToMessage(content, &msg, parse_options); + if (!status.ok()) { + SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kJSONExt + ": " + status.ToString()); + return false; + } + return true; } case Format::kText: { - return util::Text2Message(content, msg); + if (!google::protobuf::TextFormat::ParseFromString(content, &msg)) { + SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kTextExt); + return false; + } + return true; } case Format::kBin: { - return util::Bin2Message(content, msg); + if (!msg.ParseFromString(content)) { + SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kBinExt); + return false; + } + return true; } default: { SetErrMsg("unknown format: " + std::to_string(static_cast(fmt))); @@ -167,184 +180,5 @@ bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& p } } } - -bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt, - std::shared_ptr options /* = nullptr*/) { - const std::string& name = msg.GetDescriptor()->name(); - std::filesystem::path path; - if (options && !options->path.empty()) { - // path specified in Paths, then use it instead of dir. - path = options->path; - fmt = util::GetFormat(path); - } - if (path.empty()) { - std::filesystem::path filename = name + util::Format2Ext(fmt); - path = dir / filename; - } - - const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); - // access the extension directly using the generated identifier - const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet); - if (worksheet_options.patch() != tableau::PATCH_NONE) { - return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); - } - if (options) { - return options->load_func(msg, path, fmt, options); - } - return LoadMessager(msg, path, fmt); -} - -#ifdef _WIN32 -#undef GetMessage -#endif - -// PatchMessage patches src into dst, which must be a message with the same descriptor. -// -// # Default PatchMessage mechanism -// - scalar: Populated scalar fields in src are copied to dst. -// - message: Populated singular messages in src are merged into dst by -// recursively calling [xproto.PatchMessage], or replace dst message if -// "PATCH_REPLACE" is specified for this field. -// - list: The elements of every list field in src are appended to the -// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" -// is specified for this field. -// - map: The entries of every map field in src are MERGED (different from -// the behavior of proto.Merge) into the corresponding map field in dst, -// or replace dst map if "PATCH_REPLACE" is specified for this field. -// - unknown: The unknown fields of src are appended to the unknown -// fields of dst (TODO: untested). -// -// # References: -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details -bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { - const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); - const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); - // Ensure both messages are of the same type - if (dst_descriptor != src_descriptor) { - SetErrMsg("dst and src are not messages with the same descriptor"); - ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), - src_descriptor->name().c_str()); - return false; - } - - // Get the reflection and descriptor for the messages - const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); - const google::protobuf::Reflection* src_reflection = src.GetReflection(); - - // List all populated fields - std::vector fields; - src_reflection->ListFields(src, &fields); - - // Iterates over every populated field. - for (auto fd : fields) { - const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); - tableau::Patch patch = opts.prop().patch(); - if (patch == tableau::PATCH_REPLACE) { - dst_reflection->ClearField(&dst, fd); - } - if (fd->is_map()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 - auto key_fd = fd->message_type()->map_key(); - int src_count = src_reflection->FieldSize(src, fd); - int dst_count = dst_reflection->FieldSize(dst, fd); - switch (key_fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - std::unordered_map dst_key_index_map; \ - for (int i = 0; i < dst_count; i++) { \ - auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ - TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ - dst_key_index_map[key] = i; \ - } \ - for (int j = 0; j < src_count; j++) { \ - auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ - TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ - auto it = dst_key_index_map.find(key); \ - if (it != dst_key_index_map.end()) { \ - int index = it->second; \ - auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ - PatchMessage(dst_entry, src_entry); \ - } else { \ - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ - } \ - } \ - break; \ - } - - HANDLE_TYPE(INT32, Int32, int32_t); - HANDLE_TYPE(INT64, Int64, int64_t); - HANDLE_TYPE(UINT32, UInt32, uint32_t); - HANDLE_TYPE(UINT64, UInt64, uint64_t); - HANDLE_TYPE(BOOL, Bool, bool); - HANDLE_TYPE(STRING, String, std::string); - default: { - // other types are impossible to be protobuf map key - ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); - break; - } -#undef HANDLE_TYPE - } - } else if (fd->is_repeated()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 - int count = src_reflection->FieldSize(src, fd); - for (int j = 0; j < count; j++) { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ - break; \ - } - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { - const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); - break; - } - } - } - } else { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ - dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ - break; - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: - const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); - PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); - break; - } - } - } - - dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); - return true; -} +} // namespace load } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h index e8ba9118..8554a32c 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h @@ -4,21 +4,31 @@ #include #include #include +#include #include #include +#include +#include +#include "tableau/protobuf/tableau.pb.h" #include "util.pc.h" namespace tableau { +namespace load { enum class LoadMode { - kNone, kAll, // Load all related files kOnlyMain, // Only load the main file kOnlyPatch, // Only load the patch files }; struct MessagerOptions; -class Hub; + +bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt = Format::kJSON, + std::shared_ptr options = nullptr); +bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt = Format::kJSON, + std::shared_ptr options = nullptr); +bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Format fmt, + std::shared_ptr options = nullptr); // ReadFunc reads the config file and returns its content. using ReadFunc = std::function; @@ -37,26 +47,34 @@ struct BaseOptions { // Specify the directory paths for config patching. std::vector patch_dirs; // Specify the loading mode for config patching. - // - For LoadOptions, default is LoadMode::kModeAll. - // - For MessagerOptions, inherit from LoadOptions if not set. - LoadMode mode; + // Default is LoadMode::kModeAll. + std::optional mode; // You can specify custom read function to read a config file's content. - // - For LoadOptions, default is util::ReadFile. - // - For MessagerOptions, inherit from LoadOptions if not set. + // Default is util::ReadFile. ReadFunc read_func; // You can specify custom load function to load a messager's content. - // - For LoadOptions, default is LoadMessage. - // - For MessagerOptions, inherit from LoadOptions if not set. + // Default is LoadMessager. LoadFunc load_func; + + public: + inline bool GetIgnoreUnknownFields() const { return ignore_unknown_fields.value_or(false); } + inline LoadMode GetMode() const { return mode.value_or(LoadMode::kAll); } + inline ReadFunc GetReadFunc() const { return read_func ? read_func : util::ReadFile; } + inline LoadFunc GetLoadFunc() const { return load_func ? load_func : LoadMessager; } }; -// LoadOptionsOptions is the options struct, which contains both global-level and +// Options is the options struct, which contains both global-level and // messager-level options. -struct LoadOptions : public BaseOptions { +struct Options : public BaseOptions { // messager_options maps each messager name to a MessageOptions. // If specified, then the messager will be parsed with the given options // directly. - std::unordered_map> messager_options; + std::unordered_map> messager_options; + + public: + // ParseMessagerOptions parses messager options with both global-level and + // messager-level options taken into consideration. + std::shared_ptr ParseMessagerOptionsByName(const std::string& name) const; }; // MessagerOptions defines the options for loading a messager. @@ -69,6 +87,9 @@ struct MessagerOptions : public BaseOptions { // If specified, then main messager will be patched. std::vector patch_paths; }; +} // namespace load + +class Hub; class Messager { public: @@ -78,14 +99,11 @@ class Messager { public: virtual ~Messager() = default; - static const std::string& Name() { - static const std::string kEmpty = ""; - return kEmpty; - } + static const std::string& Name() = delete; const Stats& GetStats() { return stats_; } // Load fills message from file in the specified directory and format. virtual bool Load(const std::filesystem::path& dir, Format fmt, - std::shared_ptr options = nullptr) = 0; + std::shared_ptr options = nullptr) = 0; // Message returns the inner message data. virtual const google::protobuf::Message* Message() const { return nullptr; } // callback after all messagers loaded. @@ -96,15 +114,4 @@ class Messager { virtual bool ProcessAfterLoad() { return true; }; Stats stats_; }; - -// ParseLoadOptions parses load options with default global-level options. -std::shared_ptr ParseLoadOptions(std::shared_ptr opts); -// ParseMessagerOptions parses messager options with both global-level and -// messager-level options taken into consideration. -std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, - const std::string& name); -bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); -bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); } // namespace tableau \ No newline at end of file diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc index 8f2e199c..4ecc2908 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc @@ -1,18 +1,16 @@ #include "logger.pc.h" -#include -#include -#include #ifdef _WIN32 -#include #include #else #include -#include #include #endif +#include #include +#include +#include #include #include @@ -85,7 +83,7 @@ void Logger::Log(const SourceLocation& loc, Level level, const char* format, ... void DefaultWrite(std::ostream* os, const SourceLocation& loc, const LevelInfo& lvl, const std::string& content) { // clang-format off - *os << NowStr() << "|" + *os << Now() << "|" // << std::this_thread::get_id() << "|" << gettid() << "|" << lvl.name << "|" @@ -96,30 +94,12 @@ void DefaultWrite(std::ostream* os, const SourceLocation& loc, const LevelInfo& // clang-format on } -const char* NowStr() { - static char fmt[64], buf[64]; - struct tm tm; - -#ifdef _WIN32 - SYSTEMTIME wtm; - GetLocalTime(&wtm); - tm.tm_year = wtm.wYear - 1900; - tm.tm_mon = wtm.wMonth - 1; - tm.tm_mday = wtm.wDay; - tm.tm_hour = wtm.wHour; - tm.tm_min = wtm.wMinute; - tm.tm_sec = wtm.wSecond; - unsigned int usec = wtm.wMilliseconds * 1000; -#else - struct timeval tv; - gettimeofday(&tv, NULL); - localtime_r(&tv.tv_sec, &tm); - unsigned int usec = tv.tv_usec; -#endif - - strftime(fmt, sizeof fmt, "%Y-%m-%d %H:%M:%S.%%06u", &tm); - snprintf(buf, sizeof buf, fmt, usec); - return buf; +std::ostream& operator<<(std::ostream& os, const Now&) { + auto now = std::chrono::system_clock::now(); + auto now_time_t = std::chrono::system_clock::to_time_t(now); + auto now_us = std::chrono::duration_cast(now.time_since_epoch()) % 1000000; + return os << std::put_time(std::localtime(&now_time_t), "%F %T") << "." << std::setw(6) << std::setfill('0') + << now_us.count(); } } // namespace log diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h index ca212403..33a161b8 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h @@ -64,7 +64,9 @@ class Logger { Writer writer_; }; -const char* NowStr(); +class Now {}; +std::ostream& operator<<(std::ostream&, const Now&); + Logger* DefaultLogger(); void SetDefaultLogger(Logger* logger); diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc index 11236ea3..17c649e0 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc @@ -1,15 +1,9 @@ #include "util.pc.h" -#include -#include - -#include #include -#include -#include -#include "load.pc.h" #include "logger.pc.h" +#include "tableau/protobuf/tableau.pb.h" namespace tableau { static thread_local std::string g_err_msg; @@ -58,36 +52,157 @@ const std::string& Format2Ext(Format fmt) { } } -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, - std::shared_ptr options /* = nullptr */) { - google::protobuf::util::Status status; - if (options != nullptr) { - google::protobuf::util::JsonParseOptions parse_options; - parse_options.ignore_unknown_fields = options->ignore_unknown_fields.value_or(false); - status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); - } else { - status = google::protobuf::util::JsonStringToMessage(json, &msg); - } - if (!status.ok()) { - SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kJSONExt + ": " + status.ToString()); +#ifdef _WIN32 +#undef GetMessage +#endif + +// PatchMessage patches src into dst, which must be a message with the same descriptor. +// +// # Default PatchMessage mechanism +// - scalar: Populated scalar fields in src are copied to dst. +// - message: Populated singular messages in src are merged into dst by +// recursively calling [xproto.PatchMessage], or replace dst message if +// "PATCH_REPLACE" is specified for this field. +// - list: The elements of every list field in src are appended to the +// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" +// is specified for this field. +// - map: The entries of every map field in src are MERGED (different from +// the behavior of proto.Merge) into the corresponding map field in dst, +// or replace dst map if "PATCH_REPLACE" is specified for this field. +// - unknown: The unknown fields of src are appended to the unknown +// fields of dst (TODO: untested). +// +// # References: +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { + const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); + const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); + // Ensure both messages are of the same type + if (dst_descriptor != src_descriptor) { + SetErrMsg("dst and src are not messages with the same descriptor"); + ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), + src_descriptor->name().c_str()); return false; } - return true; -} -bool Text2Message(const std::string& text, google::protobuf::Message& msg) { - if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) { - SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kTextExt); - return false; + // Get the reflection and descriptor for the messages + const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); + const google::protobuf::Reflection* src_reflection = src.GetReflection(); + + // List all populated fields + std::vector fields; + src_reflection->ListFields(src, &fields); + + // Iterates over every populated field. + for (auto fd : fields) { + const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); + tableau::Patch patch = opts.prop().patch(); + if (patch == tableau::PATCH_REPLACE) { + dst_reflection->ClearField(&dst, fd); + } + if (fd->is_map()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 + auto key_fd = fd->message_type()->map_key(); + int src_count = src_reflection->FieldSize(src, fd); + int dst_count = dst_reflection->FieldSize(dst, fd); + switch (key_fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + std::unordered_map dst_key_index_map; \ + for (int i = 0; i < dst_count; i++) { \ + auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ + TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ + dst_key_index_map[key] = i; \ + } \ + for (int j = 0; j < src_count; j++) { \ + auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ + TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ + auto it = dst_key_index_map.find(key); \ + if (it != dst_key_index_map.end()) { \ + int index = it->second; \ + auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ + PatchMessage(dst_entry, src_entry); \ + } else { \ + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ + } \ + } \ + break; \ } - return true; -} -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) { - if (!msg.ParseFromString(bin)) { - SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kBinExt); - return false; + HANDLE_TYPE(INT32, Int32, int32_t); + HANDLE_TYPE(INT64, Int64, int64_t); + HANDLE_TYPE(UINT32, UInt32, uint32_t); + HANDLE_TYPE(UINT64, UInt64, uint64_t); + HANDLE_TYPE(BOOL, Bool, bool); + HANDLE_TYPE(STRING, String, std::string); + default: { + // other types are impossible to be protobuf map key + ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); + break; + } +#undef HANDLE_TYPE + } + } else if (fd->is_repeated()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 + int count = src_reflection->FieldSize(src, fd); + for (int j = 0; j < count; j++) { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ + break; \ } + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); + break; + } + } + } + } else { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ + dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ + break; + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); + PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); + break; + } + } + } + + dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); return true; } diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h index 72350710..a69b86a6 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h @@ -3,11 +3,8 @@ #include #include -#include #include -#include "tableau/protobuf/tableau.pb.h" - namespace tableau { const std::string& GetErrMsg(); void SetErrMsg(const std::string& msg); @@ -57,10 +54,8 @@ Format GetFormat(const std::filesystem::path& path); // and the error message can be obtained by GetErrMsg(). const std::string& Format2Ext(Format fmt); -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, - std::shared_ptr options = nullptr); -bool Text2Message(const std::string& text, google::protobuf::Message& msg); -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); +// PatchMessage patches src into dst, which must be a message with the same descriptor. +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg); diff --git a/cmd/protoc-gen-cpp-tableau-loader/hub.go b/cmd/protoc-gen-cpp-tableau-loader/hub.go index dee3a2cf..042c95e7 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/hub.go +++ b/cmd/protoc-gen-cpp-tableau-loader/hub.go @@ -141,6 +141,7 @@ func generateHubCppRegistry(g *protogen.GeneratedFile, protofiles []string, file const hubHpp = `#pragma once #include #include +#include #include #include #include @@ -180,13 +181,13 @@ class Hub { /***** Synchronous Loading *****/ // Load fills messages (in MessagerContainer) from files in the specified directory and format. bool Load(const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); + std::shared_ptr options = nullptr); /***** Asynchronous Loading *****/ // Load configs into temp MessagerContainer, and you should call LoopOnce() in you app's main loop, // in order to take the temp MessagerContainer into effect. bool AsyncLoad(const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); + std::shared_ptr options = nullptr); int LoopOnce(); // You'd better initialize the scheduler in the main thread. void InitScheduler(); @@ -214,7 +215,7 @@ class Hub { private: std::shared_ptr InternalLoad(const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr) const; + std::shared_ptr options = nullptr) const; std::shared_ptr NewMessagerMap() const; std::shared_ptr GetMessagerContainerWithProvider() const; const std::shared_ptr GetMessager(const std::string& name) const; @@ -310,7 +311,7 @@ void Hub::InitOnce(std::shared_ptr options) { } bool Hub::Load(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */, - std::shared_ptr options /* = nullptr */) { + std::shared_ptr options /* = nullptr */) { auto msger_map = InternalLoad(dir, fmt, options); if (!msger_map) { return false; @@ -324,7 +325,7 @@ bool Hub::Load(const std::filesystem::path& dir, Format fmt /* = Format::kJSON * } bool Hub::AsyncLoad(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */, - std::shared_ptr options /* = nullptr */) { + std::shared_ptr options /* = nullptr */) { auto msger_map = InternalLoad(dir, fmt, options); if (!msger_map) { return false; @@ -345,15 +346,15 @@ void Hub::InitScheduler() { } std::shared_ptr Hub::InternalLoad(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */, - std::shared_ptr options /* = nullptr */) const { + std::shared_ptr options /* = nullptr */) const { // intercept protobuf error logs auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); - auto opts = ParseLoadOptions(options); auto msger_map = NewMessagerMap(); + options = options ? options : std::make_shared(); for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); - auto mopts = ParseMessagerOptions(opts, name); + auto mopts = options->ParseMessagerOptionsByName(name); bool ok = iter.second->Load(dir, fmt, mopts); if (!ok) { ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str()); @@ -372,7 +373,7 @@ std::shared_ptr Hub::InternalLoad(const std::filesystem::path& dir, std::shared_ptr Hub::NewMessagerMap() const { std::shared_ptr msger_map = std::make_shared(); for (auto&& it : Registry::registrar) { - if (options_ == nullptr || options_->filter == nullptr || options_->filter(it.first)) { + if (!options_ || !options_->filter || options_->filter(it.first)) { (*msger_map)[it.first] = it.second(); } } @@ -392,7 +393,7 @@ const std::shared_ptr Hub::GetMessager(const std::string& name) const } std::shared_ptr Hub::GetMessagerContainerWithProvider() const { - if (options_ != nullptr && options_->provider != nullptr) { + if (options_ && options_->provider) { return options_->provider(*this); } return msger_container_; @@ -408,7 +409,7 @@ bool Hub::Postprocess(std::shared_ptr msger_map) { auto msger = iter.second; bool ok = msger->ProcessAfterLoadAll(tmp_hub); if (!ok) { - SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + msger->Name()); + SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + iter.first); return false; } } @@ -419,8 +420,7 @@ std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainerWithProv const msgContainerCpp = ` MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) - : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), - last_loaded_time_(std::time(nullptr)) {` + : msger_map_(msger_map ? msger_map : std::make_shared()), last_loaded_time_(std::time(nullptr)) {` const registryCpp = `} diff --git a/cmd/protoc-gen-cpp-tableau-loader/messager.go b/cmd/protoc-gen-cpp-tableau-loader/messager.go index 8734d486..a653c03e 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/messager.go +++ b/cmd/protoc-gen-cpp-tableau-loader/messager.go @@ -85,7 +85,7 @@ func genHppMessage(g *protogen.GeneratedFile, message *protogen.Message) { g.P("class ", message.Desc.Name(), " : public Messager {") g.P(" public:") g.P(helper.Indent(1), "static const std::string& Name() { return kProtoName; }") - g.P(helper.Indent(1), "virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override;") + g.P(helper.Indent(1), "virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override;") g.P(helper.Indent(1), "const ", cppFullName, "& Data() const { return data_; }") g.P(helper.Indent(1), "const google::protobuf::Message* Message() const override { return &data_; }") g.P() @@ -159,7 +159,7 @@ func genCppMessage(g *protogen.GeneratedFile, message *protogen.Message) { g.P("const std::string ", messagerName, "::kProtoName = ", cppFullName, `::GetDescriptor()->name();`) g.P() - g.P("bool ", messagerName, "::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) {") + g.P("bool ", messagerName, "::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) {") g.P(helper.Indent(1), "tableau::util::TimeProfiler profiler;") g.P(helper.Indent(1), "bool loaded = LoadMessagerInDir(data_, dir, fmt, options);") g.P(helper.Indent(1), "bool ok = loaded ? ProcessAfterLoad() : false;") diff --git a/cmd/protoc-gen-go-tableau-loader/hub.go b/cmd/protoc-gen-go-tableau-loader/hub.go index 1db4316d..7c178dc9 100644 --- a/cmd/protoc-gen-go-tableau-loader/hub.go +++ b/cmd/protoc-gen-go-tableau-loader/hub.go @@ -292,7 +292,7 @@ func (h *Hub) Load(dir string, format format.Format, options ...load.Option) err messagerMap := h.NewMessagerMap() opts := load.ParseOptions(options...) for name, msger := range messagerMap { - mopts := load.ParseMessagerOptionsFromOptions(opts, name) + mopts := opts.ParseMessagerOptionsByName(name) if err := msger.Load(dir, format, mopts); err != nil { return errors.WithMessagef(err, "failed to load: %v", name) } diff --git a/go.mod b/go.mod index 51acdf91..35e2b4d1 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 github.com/stretchr/testify v1.10.0 - github.com/tableauio/tableau v0.14.1 + github.com/tableauio/tableau v0.14.2-0.20250820113117-0b0be0ac122e golang.org/x/exp v0.0.0-20230418202329-0354be287a23 google.golang.org/protobuf v1.34.2 ) diff --git a/go.sum b/go.sum index b3c597dd..90ba32f1 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subchen/go-xmldom v1.1.2 h1:7evI2YqfYYOnuj+PBwyaOZZYjl3iWq35P6KfBUw9jeU= github.com/subchen/go-xmldom v1.1.2/go.mod h1:6Pg/HuX5/T4Jlj0IPJF1sRxKVoI/rrKP6LIMge9d5/8= -github.com/tableauio/tableau v0.14.1 h1:o6X9Tmjytb5GdB5XJdH9DYs8v7wURb5hxtX75VIoXP8= -github.com/tableauio/tableau v0.14.1/go.mod h1:EitY0lyJa8gP6tBkzWhS2WvYhC8hyWPGXoR4QPQ+Qmw= +github.com/tableauio/tableau v0.14.2-0.20250820113117-0b0be0ac122e h1:AJa5W5gNVcCPR+euRvN5NytR9CGE63NkEjG7OsOXzlY= +github.com/tableauio/tableau v0.14.2-0.20250820113117-0b0be0ac122e/go.mod h1:EitY0lyJa8gP6tBkzWhS2WvYhC8hyWPGXoR4QPQ+Qmw= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 h1:6932x8ltq1w4utjmfMPVj09jdMlkY0aiA6+Skbtl3/c= diff --git a/init.bat b/init.bat index fa2c85e5..6103990c 100644 --- a/init.bat +++ b/init.bat @@ -1,19 +1,26 @@ -@echo off -setlocal - -for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i -cd /d "%repoRoot%" - -git submodule update --init --recursive - -REM google protobuf -cd third_party\_submodules\protobuf -git checkout v3.19.3 -git submodule update --init --recursive - -REM Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) -cd cmake -cmake . -cmake --build . - -endlocal \ No newline at end of file +@echo off +setlocal + +for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i +cd /d "%repoRoot%" + +git submodule update --init --recursive + +REM google protobuf +cd third_party\_submodules\protobuf +git checkout v3.19.3 +git submodule update --init --recursive + +REM Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) +REM Refer: https://github.com/protocolbuffers/protobuf/blob/3.19.x/cmake/README.md#cmake-configuration +cd cmake +REM use Debug version +cmake -S . -B build ^ +-G "NMake Makefiles" ^ +-DCMAKE_BUILD_TYPE=Debug ^ +-DCMAKE_POLICY_VERSION_MINIMUM="3.5" + +REM Compile the code +cmake --build build --parallel 10 + +endlocal diff --git a/init.sh b/init.sh index 6b2c5ab1..092096f3 100755 --- a/init.sh +++ b/init.sh @@ -17,10 +17,13 @@ git submodule update --init --recursive cd third_party/_submodules/protobuf git checkout v3.19.3 git submodule update --init --recursive -./autogen.sh + # Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) -./configure -make -j"$(nproc)" -make check -j"$(nproc)" -# sudo make install -# sudo ldconfig # refresh shared library cache. \ No newline at end of file +# Refer: https://github.com/protocolbuffers/protobuf/blob/3.19.x/cmake/README.md#cmake-configuration +cd cmake +# use Debug version +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Debug + +# Compile the code +cmake --build build --parallel 10 diff --git a/test/cpp-tableau-loader/src/CMakeLists.txt b/test/cpp-tableau-loader/CMakeLists.txt similarity index 67% rename from test/cpp-tableau-loader/src/CMakeLists.txt rename to test/cpp-tableau-loader/CMakeLists.txt index 81402304..7787aed9 100644 --- a/test/cpp-tableau-loader/src/CMakeLists.txt +++ b/test/cpp-tableau-loader/CMakeLists.txt @@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.22) # set the project name project(loader) -file(GLOB_RECURSE PROTO_SOURCE *.cc) -file(GLOB_RECURSE SOURCE *.cpp) +file(GLOB_RECURSE PROTO_SOURCE ${PROJECT_SOURCE_DIR}/src/*.cc) +file(GLOB_RECURSE SOURCE ${PROJECT_SOURCE_DIR}/src/*.cpp) # check C++ standard requirement set(MIN_CXX_STANDARD 17) @@ -25,13 +25,14 @@ endif() # root dir define SET(CMAKE_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -SET(THRID_PARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../third_party) +SET(THRID_PARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../third_party) # google protobuf SET(PROTOBUF_ROOT_DIR ${THRID_PARTY_DIR}/_submodules/protobuf/) -SET(PROTOBUF_DIR ${PROTOBUF_ROOT_DIR}/src/) -# SET(PROTOBUF_INCLUDE ${PROTOBUF_DIR}/google/protobuf) -SET(PROTOBUF_LIB ${PROTOBUF_DIR}/.libs/) +SET(PROTOBUF_SRC_DIR ${PROTOBUF_ROOT_DIR}/src/) +SET(PROTOBUF_BUILD_DIR ${PROTOBUF_ROOT_DIR}/cmake/build/) +# loader +SET(LOADER_SRC_DIR ${PROJECT_SOURCE_DIR}/src/) # protoconf SET(PROTOCONF_DIR ${TOOLS_DIR}/protoconf) @@ -39,27 +40,28 @@ SET(PROTOCONF_TOOL ${PROTOCONF_DIR}/protoconf) # common include SET(COMMON_INCLUDE_DIR - ${PROTOBUF_DIR} + ${PROTOBUF_SRC_DIR} + ${LOADER_SRC_DIR} ) # common lib if(MSVC) SET(COMMON_LIB - ${PROTOBUF_ROOT_DIR}/cmake/Debug/libprotobufd.lib + ${PROTOBUF_BUILD_DIR}/libprotobufd.lib ) SET(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDebug") else() SET(COMMON_LIB - ${PROTOBUF_LIB}/libprotobuf.a + ${PROTOBUF_BUILD_DIR}/libprotobufd.a pthread stdc++fs ) endif() # include -include_directories(${PROJECT_SOURCE_DIR} ${COMMON_INCLUDE_DIR} ${PROJECT_SOURCE_DIR}/protoconf) +include_directories(${COMMON_INCLUDE_DIR} ${LOADER_SRC_DIR}/protoconf) # add the executable add_executable(${PROJECT_NAME} ${PROTO_SOURCE} ${SOURCE}) -set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/../bin) -target_link_libraries(${PROJECT_NAME} ${COMMON_LIB}) \ No newline at end of file +set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin) +target_link_libraries(${PROJECT_NAME} ${COMMON_LIB}) diff --git a/test/cpp-tableau-loader/gen.bat b/test/cpp-tableau-loader/gen.bat index 39b8aa34..c1a4ac96 100644 --- a/test/cpp-tableau-loader/gen.bat +++ b/test/cpp-tableau-loader/gen.bat @@ -1,56 +1,55 @@ -@echo off -setlocal -setlocal enabledelayedexpansion - -for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i -cd /d "%repoRoot%" - -set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\Debug\protoc.exe" -set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" -set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" -set "ROOTDIR=%repoRoot%\test\cpp-tableau-loader" -set "PLGUIN_DIR=%repoRoot%\cmd\protoc-gen-cpp-tableau-loader" -set "PROTOCONF_IN=%repoRoot%\test\proto" -set "PROTOCONF_OUT=%ROOTDIR%\src\protoconf" - -REM remove old generated files -if exist "%PROTOCONF_OUT%" rmdir /s /q "%PROTOCONF_OUT%" -mkdir "%PROTOCONF_OUT%" - -REM build -pushd "%PLGUIN_DIR%" -go build -popd - -set "PATH=%PATH%;%PLGUIN_DIR%" - -set protoFiles= -pushd "%PROTOCONF_IN%" -for /R %%f in (*.proto) do ( - set protoFiles=!protoFiles! "%%f" -) -popd -"%PROTOC%" ^ - --cpp-tableau-loader_out="%PROTOCONF_OUT%" ^ - --cpp-tableau-loader_opt=paths=source_relative,shards=2 ^ - --cpp_out="%PROTOCONF_OUT%" ^ - --proto_path="%PROTOBUF_PROTO%" ^ - --proto_path="%TABLEAU_PROTO%" ^ - --proto_path="%PROTOCONF_IN%" ^ - !protoFiles! - -set "TABLEAU_IN=%TABLEAU_PROTO%\tableau\protobuf" -set "TABLEAU_OUT=%ROOTDIR%\src" -REM remove old generated files -if exist "%TABLEAU_OUT%\tableau" rmdir /s /q "%TABLEAU_OUT%\tableau" -mkdir "%TABLEAU_OUT%\tableau" - -"%PROTOC%" ^ - --cpp_out="%TABLEAU_OUT%" ^ - --proto_path="%PROTOBUF_PROTO%" ^ - --proto_path="%TABLEAU_PROTO%" ^ - "%TABLEAU_IN%\tableau.proto" ^ - "%TABLEAU_IN%\wellknown.proto" - -endlocal -endlocal \ No newline at end of file +@echo off +setlocal +setlocal enabledelayedexpansion + +for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i +cd /d "%repoRoot%" + +set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\protoc.exe" +set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" +set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" +set "ROOTDIR=%repoRoot%\test\cpp-tableau-loader" +set "PLGUIN_DIR=%repoRoot%\cmd\protoc-gen-cpp-tableau-loader" +set "PROTOCONF_IN=%repoRoot%\test\proto" +set "PROTOCONF_OUT=%ROOTDIR%\src\protoconf" + +REM remove old generated files +rmdir /s /q "%PROTOCONF_OUT%" 2>nul +mkdir "%PROTOCONF_OUT%" + +REM build protoc plugin of loader +pushd "%PLGUIN_DIR%" +go build +popd + +set "PATH=%PATH%;%PLGUIN_DIR%" + +set protoFiles= +pushd "%PROTOCONF_IN%" +for /R %%f in (*.proto) do ( + set protoFiles=!protoFiles! "%%f" +) +popd +"%PROTOC%" ^ +--cpp-tableau-loader_out="%PROTOCONF_OUT%" ^ +--cpp-tableau-loader_opt=paths=source_relative,shards=2 ^ +--cpp_out="%PROTOCONF_OUT%" ^ +--proto_path="%PROTOBUF_PROTO%" ^ +--proto_path="%TABLEAU_PROTO%" ^ +--proto_path="%PROTOCONF_IN%" ^ +!protoFiles! + +set "TABLEAU_IN=%TABLEAU_PROTO%\tableau\protobuf" +set "TABLEAU_OUT=%ROOTDIR%\src" +REM remove old generated files +if exist "%TABLEAU_OUT%\tableau" rmdir /s /q "%TABLEAU_OUT%\tableau" +mkdir "%TABLEAU_OUT%\tableau" + +"%PROTOC%" ^ +--cpp_out="%TABLEAU_OUT%" ^ +--proto_path="%PROTOBUF_PROTO%" ^ +--proto_path="%TABLEAU_PROTO%" ^ +"%TABLEAU_IN%\tableau.proto" "%TABLEAU_IN%\wellknown.proto" + +endlocal +endlocal diff --git a/test/cpp-tableau-loader/gen.sh b/test/cpp-tableau-loader/gen.sh index 43d83360..32756cb3 100755 --- a/test/cpp-tableau-loader/gen.sh +++ b/test/cpp-tableau-loader/gen.sh @@ -7,7 +7,7 @@ set -o pipefail shopt -s globstar cd "$(git rev-parse --show-toplevel)" -PROTOC="./third_party/_submodules/protobuf/src/protoc" +PROTOC="./third_party/_submodules/protobuf/cmake/build/protoc" PROTOBUF_PROTO="./third_party/_submodules/protobuf/src" TABLEAU_PROTO="./third_party/_submodules/tableau/proto" ROOTDIR="./test/cpp-tableau-loader" @@ -19,7 +19,7 @@ PROTOCONF_OUT="${ROOTDIR}/src/protoconf" rm -rfv "$PROTOCONF_OUT" mkdir -p "$PROTOCONF_OUT" -# build +# build protoc plugin of loader cd "${PLGUIN_DIR}" && go build && cd - export PATH="${PATH}:${PLGUIN_DIR}" @@ -33,7 +33,7 @@ ${PROTOC} \ --proto_path="$PROTOCONF_IN" \ "$PROTOCONF_IN"/**/*.proto -TABLEAU_IN="./third_party/_submodules/tableau/proto/tableau/protobuf" +TABLEAU_IN="$TABLEAU_PROTO/tableau/protobuf" TABLEAU_OUT="${ROOTDIR}/src" # remove old generated files rm -rfv "$TABLEAU_OUT/tableau" diff --git a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.cpp b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.cpp index 86600409..c9bdd2d1 100644 --- a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.cpp +++ b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.cpp @@ -1,15 +1,17 @@ #include "hub/custom/item/custom_item_conf.h" +#include "logger.pc.h" + const std::string CustomItemConf::kCustomName = "CustomItemConf"; bool CustomItemConf::ProcessAfterLoadAll(const tableau::Hub& hub) { auto conf = hub.Get(1); if (!conf) { - std::cout << "hub get item 1 failed!" << std::endl; + ATOM_ERROR("hub get item 1 failed!"); return false; } special_item_conf_ = *conf; // value copy - std::cout << "custom item conf processed" << std::endl; + ATOM_DEBUG("custom item conf processed"); return true; } diff --git a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h index 637d9e41..e1f7527e 100644 --- a/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h +++ b/test/cpp-tableau-loader/src/hub/custom/item/custom_item_conf.h @@ -3,9 +3,9 @@ #include "protoconf/item_conf.pc.h" class CustomItemConf : public tableau::Messager { public: - static const std::string& Name() { return kCustomName; }; + static const std::string& Name() { return kCustomName; } virtual bool Load(const std::filesystem::path&, tableau::Format, - std::shared_ptr options = nullptr) override { + std::shared_ptr options = nullptr) override { return true; } virtual bool ProcessAfterLoadAll(const tableau::Hub& hub) override; diff --git a/test/cpp-tableau-loader/src/hub/hub.cpp b/test/cpp-tableau-loader/src/hub/hub.cpp index 1df497f7..03cbf33e 100644 --- a/test/cpp-tableau-loader/src/hub/hub.cpp +++ b/test/cpp-tableau-loader/src/hub/hub.cpp @@ -5,16 +5,9 @@ void LogWrite(std::ostream* os, const tableau::log::SourceLocation& loc, const tableau::log::LevelInfo& lvl, const std::string& content) { - // clang-format off - *os << tableau::log::NowStr() << " " - // << std::this_thread::get_id() << "|" - // << gettid() << " " - << lvl.name << " [" - << loc.filename << ":" << loc.line << "][" - << loc.funcname << "]" - << content - << std::endl << std::flush; - // clang-format on + *os << tableau::log::Now() << " " << lvl.name << " [" << loc.filename << ":" << loc.line << "][" << loc.funcname + << "]" << content << std::endl + << std::flush; } bool DefaultFilter(const std::string& name) { diff --git a/test/cpp-tableau-loader/src/main.cpp b/test/cpp-tableau-loader/src/main.cpp index f3a310ab..08bce442 100644 --- a/test/cpp-tableau-loader/src/main.cpp +++ b/test/cpp-tableau-loader/src/main.cpp @@ -13,8 +13,10 @@ #include "protoconf/patch_conf.pc.h" #include "protoconf/test_conf.pc.h" -bool LoadWithPatch(std::shared_ptr options) { - return Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); +const std::string kTestdataDir = "../testdata"; + +bool LoadWithPatch(std::shared_ptr options) { + return Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); } bool CustomReadFile(const std::filesystem::path& filename, std::string& content) { @@ -28,99 +30,99 @@ bool CustomReadFile(const std::filesystem::path& filename, std::string& content) } bool TestPatch() { - auto options = std::make_shared(); + auto options = std::make_shared(); options->read_func = CustomReadFile; // patchconf - std::cout << "-----TestPatch patchconf" << std::endl; - options->patch_dirs = {"../../testdata/patchconf/"}; - bool ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + ATOM_DEBUG("-----TestPatch patchconf"); + options->patch_dirs = {kTestdataDir + "/patchconf/"}; + bool ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "failed to load with patchconf" << std::endl; + ATOM_ERROR("failed to load with patchconf"); return false; } // print recursive patch conf auto mgr = Hub::Instance().Get(); if (!mgr) { - std::cout << "protobuf hub get RecursivePatchConf failed!" << std::endl; + ATOM_ERROR("protobuf hub get RecursivePatchConf failed!"); return false; } - std::cout << "RecursivePatchConf: " << std::endl << mgr->Data().ShortDebugString() << std::endl; + ATOM_DEBUG("RecursivePatchConf: %s", mgr->Data().ShortDebugString().c_str()); tableau::RecursivePatchConf result; - ok = result.Load("../../testdata/patchresult/", tableau::Format::kJSON); + ok = result.Load(kTestdataDir + "/patchresult/", tableau::Format::kJSON); if (!ok) { - std::cout << "failed to load with patch result" << std::endl; + ATOM_ERROR("failed to load with patch result"); return false; } - std::cout << "Expected patch result: " << std::endl << result.Data().ShortDebugString() << std::endl; + ATOM_DEBUG("Expected patch result: %s", result.Data().ShortDebugString().c_str()); if (!google::protobuf::util::MessageDifferencer::Equals(mgr->Data(), result.Data())) { - std::cout << "patch result not correct" << std::endl; + ATOM_ERROR("patch result not correct"); return false; } // patchconf2 - std::cout << "-----TestPatch patchconf2" << std::endl; - options->patch_dirs = {"../../testdata/patchconf2/"}; - ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + ATOM_DEBUG("-----TestPatch patchconf2"); + options->patch_dirs = {kTestdataDir + "/patchconf2/"}; + ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "failed to load with patchconf2" << std::endl; + ATOM_ERROR("failed to load with patchconf2"); return false; } // patchconf2 different format - std::cout << "-----TestPatch patchconf2 different format" << std::endl; - options->patch_dirs = {"../../testdata/patchconf2/"}; - auto mopts = std::make_shared(); - mopts->patch_paths = {"../../testdata/patchconf2/PatchMergeConf.txt"}; + ATOM_DEBUG("-----TestPatch patchconf2 different format"); + options->patch_dirs = {kTestdataDir + "/patchconf2/"}; + auto mopts = std::make_shared(); + mopts->patch_paths = {kTestdataDir + "/patchconf2/PatchMergeConf.txt"}; options->messager_options["PatchMergeConf"] = mopts; - ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "failed to load with patchconf2" << std::endl; + ATOM_ERROR("failed to load with patchconf2"); return false; } // multiple patch files - std::cout << "-----TestPatch multiple patch files" << std::endl; - mopts = std::make_shared(); - mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", - "../../testdata/patchconf2/PatchMergeConf.json"}; + ATOM_DEBUG("-----TestPatch multiple patch files"); + mopts = std::make_shared(); + mopts->patch_paths = {kTestdataDir + "/patchconf/PatchMergeConf.json", + kTestdataDir + "/patchconf2/PatchMergeConf.json"}; options->messager_options["PatchMergeConf"] = mopts; - ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "failed to load with multiple patch files" << std::endl; + ATOM_ERROR("failed to load with multiple patch files"); return false; } // mode only main - std::cout << "-----TestPatch ModeOnlyMain" << std::endl; - mopts = std::make_shared(); - mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", - "../../testdata/patchconf2/PatchMergeConf.json"}; + ATOM_DEBUG("-----TestPatch ModeOnlyMain"); + mopts = std::make_shared(); + mopts->patch_paths = {kTestdataDir + "/patchconf/PatchMergeConf.json", + kTestdataDir + "/patchconf2/PatchMergeConf.json"}; options->messager_options["PatchMergeConf"] = mopts; - options->mode = tableau::LoadMode::kOnlyMain; - ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + options->mode = tableau::load::LoadMode::kOnlyMain; + ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "failed to load with mode only main" << std::endl; + ATOM_ERROR("failed to load with mode only main"); return false; } auto patch_mgr = Hub::Instance().Get(); if (!patch_mgr) { - std::cout << "protobuf hub get PatchMergeConf failed!" << std::endl; - return 1; + ATOM_ERROR("protobuf hub get PatchMergeConf failed!"); + return false; } - std::cout << "PatchMergeConf: " << patch_mgr->Data().ShortDebugString() << std::endl; + ATOM_DEBUG("PatchMergeConf: %s", patch_mgr->Data().ShortDebugString().c_str()); // mode only patch - std::cout << "-----TestPatch ModeOnlyPatch" << std::endl; - mopts = std::make_shared(); - mopts->patch_paths = {"../../testdata/patchconf/PatchMergeConf.json", - "../../testdata/patchconf2/PatchMergeConf.json"}; + ATOM_DEBUG("-----TestPatch ModeOnlyPatch"); + mopts = std::make_shared(); + mopts->patch_paths = {kTestdataDir + "/patchconf/PatchMergeConf.json", + kTestdataDir + "/patchconf2/PatchMergeConf.json"}; options->messager_options["PatchMergeConf"] = mopts; - options->mode = tableau::LoadMode::kOnlyPatch; - ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + options->mode = tableau::load::LoadMode::kOnlyPatch; + ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "failed to load with mode only patch" << std::endl; + ATOM_ERROR("failed to load with mode only patch"); return false; } return true; @@ -128,16 +130,16 @@ bool TestPatch() { int main() { Hub::Instance().InitOnce(); - auto options = std::make_shared(); + auto options = std::make_shared(); options->ignore_unknown_fields = true; - options->patch_dirs = {"../../testdata/patchconf/"}; - auto mopts = std::make_shared(); - mopts->path = "../../testdata/conf/ItemConf.json"; + options->patch_dirs = {kTestdataDir + "/patchconf/"}; + auto mopts = std::make_shared(); + mopts->path = kTestdataDir + "/conf/ItemConf.json"; options->messager_options["ItemConf"] = mopts; - bool ok = Hub::Instance().Load("../../testdata/conf/", tableau::Format::kJSON, options); + bool ok = Hub::Instance().Load(kTestdataDir + "/conf/", tableau::Format::kJSON, options); if (!ok) { - std::cout << "protobuf hub load failed: " << tableau::GetErrMsg() << std::endl; + ATOM_ERROR("protobuf hub load failed: %s", tableau::GetErrMsg().c_str()); return 1; } auto msger_map = Hub::Instance().GetMessagerMap(); @@ -148,19 +150,19 @@ int main() { auto item_mgr = Hub::Instance().Get(); if (!item_mgr) { - std::cout << "protobuf hub get Item failed!" << std::endl; + ATOM_ERROR("protobuf hub get Item failed!"); return 1; } // std::cout << "item1: " << item_mgr->Data().DebugString() << std::endl; - std::cout << "-----Index: multi-column index test" << std::endl; + ATOM_DEBUG("-----Index: multi-column index test"); tableau::ItemConf::Index_AwardItemKey key{1, "apple"}; auto item = item_mgr->FindFirstAwardItem(key); if (!item) { - std::cout << "ItemConf FindFirstAwardItem failed!" << std::endl; + ATOM_ERROR("ItemConf FindFirstAwardItem failed!"); return 1; } - std::cout << "item: " << item->ShortDebugString() << std::endl; + ATOM_DEBUG("item: %s", item->ShortDebugString().c_str()); // auto activity_conf = Hub::Instance().Get(); // if (!activity_conf) { @@ -187,67 +189,67 @@ int main() { Hub::Instance().GetOrderedMap( 100001); if (!chapter_ordered_map) { - std::cout << "ActivityConf GetOrderedMap chapter failed!" << std::endl; + ATOM_ERROR("ActivityConf GetOrderedMap chapter failed!"); return 1; } for (auto&& it : *chapter_ordered_map) { - std::cout << "---" << it.first << "-----section_ordered_map" << std::endl; + ATOM_DEBUG("---%d-----section_ordered_map", it.first); for (auto&& kv : it.second.first) { - std::cout << kv.first << std::endl; + ATOM_DEBUG("%d", kv.first); } - std::cout << "---" << it.first << " -----section_map" << std::endl; + ATOM_DEBUG("---%d-----section_map", it.first); for (auto&& kv : it.second.second->section_map()) { - std::cout << kv.first << std::endl; + ATOM_DEBUG("%d", kv.first); } - std::cout << "chapter_id: " << it.second.second->chapter_id() << std::endl; - std::cout << "chapter_name: " << it.second.second->chapter_name() << std::endl; - std::cout << "award_id:" << it.second.second->award_id() << std::endl; + ATOM_DEBUG("chapter_id: %d", it.second.second->chapter_id()); + ATOM_DEBUG("chapter_name: %s", it.second.second->chapter_name().c_str()); + ATOM_DEBUG("award_id: %d", it.second.second->award_id()); } const auto* rank_ordered_map = Hub::Instance().GetOrderedMap(100001, 1, 2); if (!rank_ordered_map) { - std::cout << "ActivityConf GetOrderedMap rank failed!" << std::endl; + ATOM_ERROR("ActivityConf GetOrderedMap rank failed!"); return 1; } - std::cout << "-----rank_ordered_map" << std::endl; + ATOM_DEBUG("-----rank_ordered_map"); for (auto&& it : *rank_ordered_map) { - std::cout << it.first << std::endl; + ATOM_DEBUG("%d", it.first); } auto activity_conf = Hub::Instance().Get(); if (!activity_conf) { - std::cout << "protobuf hub get ActivityConf failed!" << std::endl; + ATOM_ERROR("protobuf hub get ActivityConf failed!"); return 1; } - std::cout << "-----Index accessers test" << std::endl; + ATOM_DEBUG("-----Index accessers test"); auto index_chapters = activity_conf->FindChapter(1); if (!index_chapters) { - std::cout << "ActivityConf FindChapter failed!" << std::endl; + ATOM_ERROR("ActivityConf FindChapter failed!"); return 1; } - std::cout << "-----FindChapter" << std::endl; + ATOM_DEBUG("-----FindChapter"); for (auto&& chapter : *index_chapters) { - std::cout << chapter->ShortDebugString() << std::endl; + ATOM_DEBUG("%s", chapter->ShortDebugString().c_str()); } auto index_first_chapter = activity_conf->FindFirstChapter(1); if (!index_first_chapter) { - std::cout << "ActivityConf FindFirstChapter failed!" << std::endl; + ATOM_ERROR("ActivityConf FindFirstChapter failed!"); return 1; } - std::cout << "-----FindFirstChapter" << std::endl; - std::cout << index_first_chapter->ShortDebugString() << std::endl; + ATOM_DEBUG("-----FindFirstChapter"); + ATOM_DEBUG("%s", index_first_chapter->ShortDebugString().c_str()); - std::cout << "specialItemName: " << Hub::Instance().Get()->GetSpecialItemName() << std::endl; + ATOM_DEBUG("specialItemName: %s", Hub::Instance().Get()->GetSpecialItemName().c_str()); if (!TestPatch()) { - std::cerr << "TestPatch failed!" << std::endl; + ATOM_ERROR("TestPatch failed!"); return 1; } return 0; diff --git a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc index 1c5a900e..fee254e7 100644 --- a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.cc @@ -12,7 +12,7 @@ namespace tableau { const std::string HeroConf::kProtoName = protoconf::HeroConf::GetDescriptor()->name(); -bool HeroConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool HeroConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; @@ -71,7 +71,7 @@ const HeroConf::Hero_Attr_OrderedMap* HeroConf::GetOrderedMap(const std::string& const std::string HeroBaseConf::kProtoName = protoconf::HeroBaseConf::GetDescriptor()->name(); -bool HeroBaseConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool HeroBaseConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; diff --git a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h index 17b5c180..325d1d7b 100644 --- a/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hero_conf.pc.h @@ -16,7 +16,7 @@ namespace tableau { class HeroConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::HeroConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -47,7 +47,7 @@ class HeroConf : public Messager { class HeroBaseConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::HeroBaseConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc index 51e591dc..6364fd43 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.cc @@ -20,7 +20,7 @@ void Hub::InitOnce(std::shared_ptr options) { } bool Hub::Load(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */, - std::shared_ptr options /* = nullptr */) { + std::shared_ptr options /* = nullptr */) { auto msger_map = InternalLoad(dir, fmt, options); if (!msger_map) { return false; @@ -34,7 +34,7 @@ bool Hub::Load(const std::filesystem::path& dir, Format fmt /* = Format::kJSON * } bool Hub::AsyncLoad(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */, - std::shared_ptr options /* = nullptr */) { + std::shared_ptr options /* = nullptr */) { auto msger_map = InternalLoad(dir, fmt, options); if (!msger_map) { return false; @@ -55,15 +55,15 @@ void Hub::InitScheduler() { } std::shared_ptr Hub::InternalLoad(const std::filesystem::path& dir, Format fmt /* = Format::kJSON */, - std::shared_ptr options /* = nullptr */) const { + std::shared_ptr options /* = nullptr */) const { // intercept protobuf error logs auto old_handler = google::protobuf::SetLogHandler(util::ProtobufLogHandler); - auto opts = ParseLoadOptions(options); auto msger_map = NewMessagerMap(); + options = options ? options : std::make_shared(); for (auto iter : *msger_map) { auto&& name = iter.first; ATOM_DEBUG("loading %s", name.c_str()); - auto mopts = ParseMessagerOptions(opts, name); + auto mopts = options->ParseMessagerOptionsByName(name); bool ok = iter.second->Load(dir, fmt, mopts); if (!ok) { ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str()); @@ -82,7 +82,7 @@ std::shared_ptr Hub::InternalLoad(const std::filesystem::path& dir, std::shared_ptr Hub::NewMessagerMap() const { std::shared_ptr msger_map = std::make_shared(); for (auto&& it : Registry::registrar) { - if (options_ == nullptr || options_->filter == nullptr || options_->filter(it.first)) { + if (!options_ || !options_->filter || options_->filter(it.first)) { (*msger_map)[it.first] = it.second(); } } @@ -102,7 +102,7 @@ const std::shared_ptr Hub::GetMessager(const std::string& name) const } std::shared_ptr Hub::GetMessagerContainerWithProvider() const { - if (options_ != nullptr && options_->provider != nullptr) { + if (options_ && options_->provider) { return options_->provider(*this); } return msger_container_; @@ -118,7 +118,7 @@ bool Hub::Postprocess(std::shared_ptr msger_map) { auto msger = iter.second; bool ok = msger->ProcessAfterLoadAll(tmp_hub); if (!ok) { - SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + msger->Name()); + SetErrMsg("hub call ProcessAfterLoadAll failed, messager: " + iter.first); return false; } } @@ -128,8 +128,7 @@ bool Hub::Postprocess(std::shared_ptr msger_map) { std::time_t Hub::GetLastLoadedTime() const { return GetMessagerContainerWithProvider()->last_loaded_time_; } MessagerContainer::MessagerContainer(std::shared_ptr msger_map /* = nullptr*/) - : msger_map_(msger_map != nullptr ? msger_map : std::make_shared()), - last_loaded_time_(std::time(nullptr)) { + : msger_map_(msger_map ? msger_map : std::make_shared()), last_loaded_time_(std::time(nullptr)) { InitShard0(); InitShard1(); } diff --git a/test/cpp-tableau-loader/src/protoconf/hub.pc.h b/test/cpp-tableau-loader/src/protoconf/hub.pc.h index da0d6771..e37e2ba1 100644 --- a/test/cpp-tableau-loader/src/protoconf/hub.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/hub.pc.h @@ -6,6 +6,7 @@ #pragma once #include #include +#include #include #include #include @@ -45,13 +46,13 @@ class Hub { /***** Synchronous Loading *****/ // Load fills messages (in MessagerContainer) from files in the specified directory and format. bool Load(const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); + std::shared_ptr options = nullptr); /***** Asynchronous Loading *****/ // Load configs into temp MessagerContainer, and you should call LoopOnce() in you app's main loop, // in order to take the temp MessagerContainer into effect. bool AsyncLoad(const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); + std::shared_ptr options = nullptr); int LoopOnce(); // You'd better initialize the scheduler in the main thread. void InitScheduler(); @@ -79,7 +80,7 @@ class Hub { private: std::shared_ptr InternalLoad(const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr) const; + std::shared_ptr options = nullptr) const; std::shared_ptr NewMessagerMap() const; std::shared_ptr GetMessagerContainerWithProvider() const; const std::shared_ptr GetMessager(const std::string& name) const; diff --git a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc index 596d6760..140568e3 100644 --- a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.cc @@ -12,7 +12,7 @@ namespace tableau { const std::string ItemConf::kProtoName = protoconf::ItemConf::GetDescriptor()->name(); -bool ItemConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool ItemConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; diff --git a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h index cd1f85b2..12a6ea69 100644 --- a/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/item_conf.pc.h @@ -16,7 +16,7 @@ namespace tableau { class ItemConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::ItemConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc index 6c3bc34e..852f73fc 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -5,78 +5,84 @@ #include "load.pc.h" -#include +#include +#include #include "logger.pc.h" #include "util.pc.h" namespace tableau { - -// Forward declaration of the PatchMessage function -bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); - -std::shared_ptr ParseLoadOptions(std::shared_ptr opts) { - std::shared_ptr new_opts = std::make_shared(); - // set default values - new_opts->mode = LoadMode::kAll; - new_opts->read_func = util::ReadFile; - new_opts->load_func = LoadMessager; - - if (opts == nullptr) { - return new_opts; +namespace load { +std::shared_ptr Options::ParseMessagerOptionsByName(const std::string& name) const { + std::shared_ptr mopts = std::make_shared(); + if (auto iter = messager_options.find(name); iter != messager_options.end() && iter->second) { + mopts = std::make_shared(*iter->second); } - if (opts->ignore_unknown_fields.has_value()) { - new_opts->ignore_unknown_fields = opts->ignore_unknown_fields; + if (!mopts->ignore_unknown_fields.has_value()) { + mopts->ignore_unknown_fields = ignore_unknown_fields; } - if (!opts->patch_dirs.empty()) { - new_opts->patch_dirs = opts->patch_dirs; + if (mopts->patch_dirs.empty()) { + mopts->patch_dirs = patch_dirs; } - if (opts->mode != LoadMode::kNone) { - new_opts->mode = opts->mode; + if (!mopts->mode.has_value()) { + mopts->mode = mode; } - if (opts->read_func != nullptr) { - new_opts->read_func = opts->read_func; + if (!mopts->read_func) { + mopts->read_func = read_func; } - if (opts->load_func != nullptr) { - new_opts->load_func = opts->load_func; + if (!mopts->load_func) { + mopts->load_func = load_func; } - return new_opts; + return mopts; } -std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, - const std::string& name) { - std::shared_ptr mopts; - if (auto iter = opts->messager_options.find(name); iter != opts->messager_options.end()) { - mopts = iter->second; - } else { - mopts = std::make_shared(); - } - if (mopts->ignore_unknown_fields.has_value()) { - mopts->ignore_unknown_fields = opts->ignore_unknown_fields; - } - if (mopts->patch_dirs.empty()) { - mopts->patch_dirs = opts->patch_dirs; +// Forward declaration +bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, + tableau::Patch patch, std::shared_ptr options = nullptr); + +bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, + std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); + std::string content; + bool ok = options->GetReadFunc()(path, content); + if (!ok) { + return false; } - if (mopts->mode == LoadMode::kNone) { - mopts->mode = opts->mode; + return Unmarshal(content, msg, fmt, options); +} + +bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt, + std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); + const std::string& name = msg.GetDescriptor()->name(); + std::filesystem::path path; + if (!options->path.empty()) { + // path specified in Paths, then use it instead of dir. + path = options->path; + fmt = util::GetFormat(path); } - if (mopts->read_func == nullptr) { - mopts->read_func = opts->read_func; + if (path.empty()) { + std::filesystem::path filename = name + util::Format2Ext(fmt); + path = dir / filename; } - if (mopts->load_func == nullptr) { - mopts->load_func = opts->load_func; + + const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); + // access the extension directly using the generated identifier + const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet); + if (worksheet_options.patch() != tableau::PATCH_NONE) { + return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); } - return mopts; + return options->GetLoadFunc()(msg, path, fmt, options); } bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, tableau::Patch patch, std::shared_ptr options /* = nullptr*/) { - if (options == nullptr) { - return LoadMessager(msg, path, fmt, nullptr); - } - if (options->mode == LoadMode::kOnlyMain) { + options = options ? options : std::make_shared(); + auto mode = options->GetMode(); + auto load_func = options->GetLoadFunc(); + if (mode == LoadMode::kOnlyMain) { // ignore patch files when LoadMode::kModeOnlyMain specified - return options->load_func(msg, path, fmt, nullptr); + return load_func(msg, path, fmt, nullptr); } const std::string& name = msg.GetDescriptor()->name(); std::vector patch_paths; @@ -97,27 +103,27 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem } } if (existed_patch_paths.empty()) { - if (options->mode == LoadMode::kOnlyPatch) { + if (mode == LoadMode::kOnlyPatch) { // just returns empty message when LoadMode::kModeOnlyPatch specified but no valid patch file provided. return true; } // no valid patch path provided, then just load from the "main" file. - return options->load_func(msg, path, fmt, options); + return load_func(msg, path, fmt, options); } switch (patch) { case tableau::PATCH_REPLACE: { // just use the last "patch" file std::filesystem::path& patch_path = existed_patch_paths.back(); - if (!options->load_func(msg, patch_path, util::GetFormat(patch_path), options)) { + if (!load_func(msg, patch_path, util::GetFormat(patch_path), options)) { return false; } break; } case tableau::PATCH_MERGE: { - if (options->mode != LoadMode::kOnlyPatch) { + if (mode != LoadMode::kOnlyPatch) { // load msg from the "main" file - if (!options->load_func(msg, path, fmt, options)) { + if (!load_func(msg, path, fmt, options)) { return false; } } @@ -126,10 +132,10 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem std::unique_ptr _auto_release(msg.New()); // load patch_msg from each "patch" file for (auto&& patch_path : existed_patch_paths) { - if (!options->load_func(*patch_msg_ptr, patch_path, util::GetFormat(patch_path), options)) { + if (!load_func(*patch_msg_ptr, patch_path, util::GetFormat(patch_path), options)) { return false; } - if (!PatchMessage(msg, *patch_msg_ptr)) { + if (!util::PatchMessage(msg, *patch_msg_ptr)) { return false; } } @@ -145,26 +151,33 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem return true; } -bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt, - std::shared_ptr options /* = nullptr*/) { - std::string content; - ReadFunc read_func = util::ReadFile; - if (options != nullptr && options->read_func) { - read_func = options->read_func; - } - bool ok = read_func(path, content); - if (!ok) { - return false; - } +bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Format fmt, + std::shared_ptr options /* = nullptr*/) { + options = options ? options : std::make_shared(); switch (fmt) { case Format::kJSON: { - return util::JSON2Message(content, msg, options); + google::protobuf::util::JsonParseOptions parse_options; + parse_options.ignore_unknown_fields = options->GetIgnoreUnknownFields(); + auto status = google::protobuf::util::JsonStringToMessage(content, &msg, parse_options); + if (!status.ok()) { + SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kJSONExt + ": " + status.ToString()); + return false; + } + return true; } case Format::kText: { - return util::Text2Message(content, msg); + if (!google::protobuf::TextFormat::ParseFromString(content, &msg)) { + SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kTextExt); + return false; + } + return true; } case Format::kBin: { - return util::Bin2Message(content, msg); + if (!msg.ParseFromString(content)) { + SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kBinExt); + return false; + } + return true; } default: { SetErrMsg("unknown format: " + std::to_string(static_cast(fmt))); @@ -172,184 +185,5 @@ bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& p } } } - -bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt, - std::shared_ptr options /* = nullptr*/) { - const std::string& name = msg.GetDescriptor()->name(); - std::filesystem::path path; - if (options && !options->path.empty()) { - // path specified in Paths, then use it instead of dir. - path = options->path; - fmt = util::GetFormat(path); - } - if (path.empty()) { - std::filesystem::path filename = name + util::Format2Ext(fmt); - path = dir / filename; - } - - const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); - // access the extension directly using the generated identifier - const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet); - if (worksheet_options.patch() != tableau::PATCH_NONE) { - return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); - } - if (options) { - return options->load_func(msg, path, fmt, options); - } - return LoadMessager(msg, path, fmt); -} - -#ifdef _WIN32 -#undef GetMessage -#endif - -// PatchMessage patches src into dst, which must be a message with the same descriptor. -// -// # Default PatchMessage mechanism -// - scalar: Populated scalar fields in src are copied to dst. -// - message: Populated singular messages in src are merged into dst by -// recursively calling [xproto.PatchMessage], or replace dst message if -// "PATCH_REPLACE" is specified for this field. -// - list: The elements of every list field in src are appended to the -// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" -// is specified for this field. -// - map: The entries of every map field in src are MERGED (different from -// the behavior of proto.Merge) into the corresponding map field in dst, -// or replace dst map if "PATCH_REPLACE" is specified for this field. -// - unknown: The unknown fields of src are appended to the unknown -// fields of dst (TODO: untested). -// -// # References: -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor -// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details -bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { - const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); - const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); - // Ensure both messages are of the same type - if (dst_descriptor != src_descriptor) { - SetErrMsg("dst and src are not messages with the same descriptor"); - ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), - src_descriptor->name().c_str()); - return false; - } - - // Get the reflection and descriptor for the messages - const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); - const google::protobuf::Reflection* src_reflection = src.GetReflection(); - - // List all populated fields - std::vector fields; - src_reflection->ListFields(src, &fields); - - // Iterates over every populated field. - for (auto fd : fields) { - const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); - tableau::Patch patch = opts.prop().patch(); - if (patch == tableau::PATCH_REPLACE) { - dst_reflection->ClearField(&dst, fd); - } - if (fd->is_map()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 - auto key_fd = fd->message_type()->map_key(); - int src_count = src_reflection->FieldSize(src, fd); - int dst_count = dst_reflection->FieldSize(dst, fd); - switch (key_fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - std::unordered_map dst_key_index_map; \ - for (int i = 0; i < dst_count; i++) { \ - auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ - TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ - dst_key_index_map[key] = i; \ - } \ - for (int j = 0; j < src_count; j++) { \ - auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ - TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ - auto it = dst_key_index_map.find(key); \ - if (it != dst_key_index_map.end()) { \ - int index = it->second; \ - auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ - PatchMessage(dst_entry, src_entry); \ - } else { \ - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ - } \ - } \ - break; \ - } - - HANDLE_TYPE(INT32, Int32, int32_t); - HANDLE_TYPE(INT64, Int64, int64_t); - HANDLE_TYPE(UINT32, UInt32, uint32_t); - HANDLE_TYPE(UINT64, UInt64, uint64_t); - HANDLE_TYPE(BOOL, Bool, bool); - HANDLE_TYPE(STRING, String, std::string); - default: { - // other types are impossible to be protobuf map key - ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); - break; - } -#undef HANDLE_TYPE - } - } else if (fd->is_repeated()) { - // Reference: - // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 - int count = src_reflection->FieldSize(src, fd); - for (int j = 0; j < count; j++) { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ - dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ - break; \ - } - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { - const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); - PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); - break; - } - } - } - } else { - switch (fd->cpp_type()) { -#define HANDLE_TYPE(CPPTYPE, METHOD) \ - case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ - dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ - break; - - HANDLE_TYPE(INT32, Int32); - HANDLE_TYPE(INT64, Int64); - HANDLE_TYPE(UINT32, UInt32); - HANDLE_TYPE(UINT64, UInt64); - HANDLE_TYPE(FLOAT, Float); - HANDLE_TYPE(DOUBLE, Double); - HANDLE_TYPE(BOOL, Bool); - HANDLE_TYPE(STRING, String); - HANDLE_TYPE(ENUM, Enum); -#undef HANDLE_TYPE - - case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: - const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); - PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); - break; - } - } - } - - dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); - return true; -} +} // namespace load } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.h b/test/cpp-tableau-loader/src/protoconf/load.pc.h index 8c1bbb83..2bd4b9ce 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.h @@ -9,21 +9,31 @@ #include #include #include +#include #include #include +#include +#include +#include "tableau/protobuf/tableau.pb.h" #include "util.pc.h" namespace tableau { +namespace load { enum class LoadMode { - kNone, kAll, // Load all related files kOnlyMain, // Only load the main file kOnlyPatch, // Only load the patch files }; struct MessagerOptions; -class Hub; + +bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt = Format::kJSON, + std::shared_ptr options = nullptr); +bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt = Format::kJSON, + std::shared_ptr options = nullptr); +bool Unmarshal(const std::string& content, google::protobuf::Message& msg, Format fmt, + std::shared_ptr options = nullptr); // ReadFunc reads the config file and returns its content. using ReadFunc = std::function; @@ -42,26 +52,34 @@ struct BaseOptions { // Specify the directory paths for config patching. std::vector patch_dirs; // Specify the loading mode for config patching. - // - For LoadOptions, default is LoadMode::kModeAll. - // - For MessagerOptions, inherit from LoadOptions if not set. - LoadMode mode; + // Default is LoadMode::kModeAll. + std::optional mode; // You can specify custom read function to read a config file's content. - // - For LoadOptions, default is util::ReadFile. - // - For MessagerOptions, inherit from LoadOptions if not set. + // Default is util::ReadFile. ReadFunc read_func; // You can specify custom load function to load a messager's content. - // - For LoadOptions, default is LoadMessage. - // - For MessagerOptions, inherit from LoadOptions if not set. + // Default is LoadMessager. LoadFunc load_func; + + public: + inline bool GetIgnoreUnknownFields() const { return ignore_unknown_fields.value_or(false); } + inline LoadMode GetMode() const { return mode.value_or(LoadMode::kAll); } + inline ReadFunc GetReadFunc() const { return read_func ? read_func : util::ReadFile; } + inline LoadFunc GetLoadFunc() const { return load_func ? load_func : LoadMessager; } }; -// LoadOptionsOptions is the options struct, which contains both global-level and +// Options is the options struct, which contains both global-level and // messager-level options. -struct LoadOptions : public BaseOptions { +struct Options : public BaseOptions { // messager_options maps each messager name to a MessageOptions. // If specified, then the messager will be parsed with the given options // directly. - std::unordered_map> messager_options; + std::unordered_map> messager_options; + + public: + // ParseMessagerOptions parses messager options with both global-level and + // messager-level options taken into consideration. + std::shared_ptr ParseMessagerOptionsByName(const std::string& name) const; }; // MessagerOptions defines the options for loading a messager. @@ -74,6 +92,9 @@ struct MessagerOptions : public BaseOptions { // If specified, then main messager will be patched. std::vector patch_paths; }; +} // namespace load + +class Hub; class Messager { public: @@ -83,14 +104,11 @@ class Messager { public: virtual ~Messager() = default; - static const std::string& Name() { - static const std::string kEmpty = ""; - return kEmpty; - } + static const std::string& Name() = delete; const Stats& GetStats() { return stats_; } // Load fills message from file in the specified directory and format. virtual bool Load(const std::filesystem::path& dir, Format fmt, - std::shared_ptr options = nullptr) = 0; + std::shared_ptr options = nullptr) = 0; // Message returns the inner message data. virtual const google::protobuf::Message* Message() const { return nullptr; } // callback after all messagers loaded. @@ -101,15 +119,4 @@ class Messager { virtual bool ProcessAfterLoad() { return true; }; Stats stats_; }; - -// ParseLoadOptions parses load options with default global-level options. -std::shared_ptr ParseLoadOptions(std::shared_ptr opts); -// ParseMessagerOptions parses messager options with both global-level and -// messager-level options taken into consideration. -std::shared_ptr ParseMessagerOptions(std::shared_ptr opts, - const std::string& name); -bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); -bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt = Format::kJSON, - std::shared_ptr options = nullptr); } // namespace tableau diff --git a/test/cpp-tableau-loader/src/protoconf/logger.pc.cc b/test/cpp-tableau-loader/src/protoconf/logger.pc.cc index ce98982c..747ce28d 100644 --- a/test/cpp-tableau-loader/src/protoconf/logger.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/logger.pc.cc @@ -5,19 +5,17 @@ #include "logger.pc.h" -#include -#include -#include #ifdef _WIN32 -#include #include #else #include -#include #include #endif +#include #include +#include +#include #include #include @@ -90,7 +88,7 @@ void Logger::Log(const SourceLocation& loc, Level level, const char* format, ... void DefaultWrite(std::ostream* os, const SourceLocation& loc, const LevelInfo& lvl, const std::string& content) { // clang-format off - *os << NowStr() << "|" + *os << Now() << "|" // << std::this_thread::get_id() << "|" << gettid() << "|" << lvl.name << "|" @@ -101,30 +99,12 @@ void DefaultWrite(std::ostream* os, const SourceLocation& loc, const LevelInfo& // clang-format on } -const char* NowStr() { - static char fmt[64], buf[64]; - struct tm tm; - -#ifdef _WIN32 - SYSTEMTIME wtm; - GetLocalTime(&wtm); - tm.tm_year = wtm.wYear - 1900; - tm.tm_mon = wtm.wMonth - 1; - tm.tm_mday = wtm.wDay; - tm.tm_hour = wtm.wHour; - tm.tm_min = wtm.wMinute; - tm.tm_sec = wtm.wSecond; - unsigned int usec = wtm.wMilliseconds * 1000; -#else - struct timeval tv; - gettimeofday(&tv, NULL); - localtime_r(&tv.tv_sec, &tm); - unsigned int usec = tv.tv_usec; -#endif - - strftime(fmt, sizeof fmt, "%Y-%m-%d %H:%M:%S.%%06u", &tm); - snprintf(buf, sizeof buf, fmt, usec); - return buf; +std::ostream& operator<<(std::ostream& os, const Now&) { + auto now = std::chrono::system_clock::now(); + auto now_time_t = std::chrono::system_clock::to_time_t(now); + auto now_us = std::chrono::duration_cast(now.time_since_epoch()) % 1000000; + return os << std::put_time(std::localtime(&now_time_t), "%F %T") << "." << std::setw(6) << std::setfill('0') + << now_us.count(); } } // namespace log diff --git a/test/cpp-tableau-loader/src/protoconf/logger.pc.h b/test/cpp-tableau-loader/src/protoconf/logger.pc.h index 434d4364..592c82b7 100644 --- a/test/cpp-tableau-loader/src/protoconf/logger.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/logger.pc.h @@ -69,7 +69,9 @@ class Logger { Writer writer_; }; -const char* NowStr(); +class Now {}; +std::ostream& operator<<(std::ostream&, const Now&); + Logger* DefaultLogger(); void SetDefaultLogger(Logger* logger); diff --git a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc index b08fa224..db80e10c 100644 --- a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.cc @@ -12,7 +12,7 @@ namespace tableau { const std::string PatchReplaceConf::kProtoName = protoconf::PatchReplaceConf::GetDescriptor()->name(); -bool PatchReplaceConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool PatchReplaceConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; @@ -22,7 +22,7 @@ bool PatchReplaceConf::Load(const std::filesystem::path& dir, Format fmt, std::s const std::string PatchMergeConf::kProtoName = protoconf::PatchMergeConf::GetDescriptor()->name(); -bool PatchMergeConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool PatchMergeConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; @@ -40,7 +40,7 @@ const protoconf::Item* PatchMergeConf::Get(uint32_t id) const { const std::string RecursivePatchConf::kProtoName = protoconf::RecursivePatchConf::GetDescriptor()->name(); -bool RecursivePatchConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool RecursivePatchConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; diff --git a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h index 2770cae2..3aaae6ef 100644 --- a/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/patch_conf.pc.h @@ -16,7 +16,7 @@ namespace tableau { class PatchReplaceConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::PatchReplaceConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -29,7 +29,7 @@ class PatchReplaceConf : public Messager { class PatchMergeConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::PatchMergeConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -44,7 +44,7 @@ class PatchMergeConf : public Messager { class RecursivePatchConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::RecursivePatchConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } diff --git a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc index fac05ce7..59b3e691 100644 --- a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.cc @@ -12,7 +12,7 @@ namespace tableau { const std::string ActivityConf::kProtoName = protoconf::ActivityConf::GetDescriptor()->name(); -bool ActivityConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool ActivityConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; @@ -240,7 +240,7 @@ const protoconf::Section::SectionItem* ActivityConf::FindFirstAward(uint32_t id) const std::string ChapterConf::kProtoName = protoconf::ChapterConf::GetDescriptor()->name(); -bool ChapterConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool ChapterConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; @@ -258,7 +258,7 @@ const protoconf::ChapterConf::Chapter* ChapterConf::Get(uint64_t id) const { const std::string ThemeConf::kProtoName = protoconf::ThemeConf::GetDescriptor()->name(); -bool ThemeConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool ThemeConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; @@ -288,7 +288,7 @@ const std::string* ThemeConf::Get(const std::string& name, const std::string& pa const std::string TaskConf::kProtoName = protoconf::TaskConf::GetDescriptor()->name(); -bool TaskConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { +bool TaskConf::Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options /* = nullptr */) { tableau::util::TimeProfiler profiler; bool loaded = LoadMessagerInDir(data_, dir, fmt, options); bool ok = loaded ? ProcessAfterLoad() : false; diff --git a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h index f3226a91..9036b8d5 100644 --- a/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/test_conf.pc.h @@ -16,7 +16,7 @@ namespace tableau { class ActivityConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::ActivityConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -103,7 +103,7 @@ class ActivityConf : public Messager { class ChapterConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::ChapterConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -118,7 +118,7 @@ class ChapterConf : public Messager { class ThemeConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::ThemeConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } @@ -134,7 +134,7 @@ class ThemeConf : public Messager { class TaskConf : public Messager { public: static const std::string& Name() { return kProtoName; } - virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; + virtual bool Load(const std::filesystem::path& dir, Format fmt, std::shared_ptr options = nullptr) override; const protoconf::TaskConf& Data() const { return data_; } const google::protobuf::Message* Message() const override { return &data_; } diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.cc b/test/cpp-tableau-loader/src/protoconf/util.pc.cc index 95dc95de..05da86b8 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.cc @@ -5,16 +5,10 @@ #include "util.pc.h" -#include -#include - -#include #include -#include -#include -#include "load.pc.h" #include "logger.pc.h" +#include "tableau/protobuf/tableau.pb.h" namespace tableau { static thread_local std::string g_err_msg; @@ -63,36 +57,157 @@ const std::string& Format2Ext(Format fmt) { } } -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, - std::shared_ptr options /* = nullptr */) { - google::protobuf::util::Status status; - if (options != nullptr) { - google::protobuf::util::JsonParseOptions parse_options; - parse_options.ignore_unknown_fields = options->ignore_unknown_fields.value_or(false); - status = google::protobuf::util::JsonStringToMessage(json, &msg, parse_options); - } else { - status = google::protobuf::util::JsonStringToMessage(json, &msg); - } - if (!status.ok()) { - SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kJSONExt + ": " + status.ToString()); +#ifdef _WIN32 +#undef GetMessage +#endif + +// PatchMessage patches src into dst, which must be a message with the same descriptor. +// +// # Default PatchMessage mechanism +// - scalar: Populated scalar fields in src are copied to dst. +// - message: Populated singular messages in src are merged into dst by +// recursively calling [xproto.PatchMessage], or replace dst message if +// "PATCH_REPLACE" is specified for this field. +// - list: The elements of every list field in src are appended to the +// corresponded list fields in dst, or replace dst list if "PATCH_REPLACE" +// is specified for this field. +// - map: The entries of every map field in src are MERGED (different from +// the behavior of proto.Merge) into the corresponding map field in dst, +// or replace dst map if "PATCH_REPLACE" is specified for this field. +// - unknown: The unknown fields of src are appended to the unknown +// fields of dst (TODO: untested). +// +// # References: +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Reflection +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#Descriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.descriptor/#FieldDescriptor +// - https://protobuf.dev/reference/cpp/api-docs/google.protobuf.message/#Message.MergeFrom.details +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src) { + const google::protobuf::Descriptor* dst_descriptor = dst.GetDescriptor(); + const google::protobuf::Descriptor* src_descriptor = src.GetDescriptor(); + // Ensure both messages are of the same type + if (dst_descriptor != src_descriptor) { + SetErrMsg("dst and src are not messages with the same descriptor"); + ATOM_ERROR("dst %s and src %s are not messages with the same descriptor", dst_descriptor->name().c_str(), + src_descriptor->name().c_str()); return false; } - return true; -} -bool Text2Message(const std::string& text, google::protobuf::Message& msg) { - if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) { - SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kTextExt); - return false; + // Get the reflection and descriptor for the messages + const google::protobuf::Reflection* dst_reflection = dst.GetReflection(); + const google::protobuf::Reflection* src_reflection = src.GetReflection(); + + // List all populated fields + std::vector fields; + src_reflection->ListFields(src, &fields); + + // Iterates over every populated field. + for (auto fd : fields) { + const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); + tableau::Patch patch = opts.prop().patch(); + if (patch == tableau::PATCH_REPLACE) { + dst_reflection->ClearField(&dst, fd); + } + if (fd->is_map()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 + auto key_fd = fd->message_type()->map_key(); + int src_count = src_reflection->FieldSize(src, fd); + int dst_count = dst_reflection->FieldSize(dst, fd); + switch (key_fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD, TYPENAME) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + std::unordered_map dst_key_index_map; \ + for (int i = 0; i < dst_count; i++) { \ + auto&& entry = dst_reflection->GetRepeatedMessage(dst, fd, i); \ + TYPENAME key = entry.GetReflection()->Get##METHOD(entry, key_fd); \ + dst_key_index_map[key] = i; \ + } \ + for (int j = 0; j < src_count; j++) { \ + auto&& src_entry = src_reflection->GetRepeatedMessage(src, fd, j); \ + TYPENAME key = src_entry.GetReflection()->Get##METHOD(src_entry, key_fd); \ + auto it = dst_key_index_map.find(key); \ + if (it != dst_key_index_map.end()) { \ + int index = it->second; \ + auto&& dst_entry = *dst_reflection->MutableRepeatedMessage(&dst, fd, index); \ + PatchMessage(dst_entry, src_entry); \ + } else { \ + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_entry); \ + } \ + } \ + break; \ } - return true; -} -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) { - if (!msg.ParseFromString(bin)) { - SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kBinExt); - return false; + HANDLE_TYPE(INT32, Int32, int32_t); + HANDLE_TYPE(INT64, Int64, int64_t); + HANDLE_TYPE(UINT32, UInt32, uint32_t); + HANDLE_TYPE(UINT64, UInt64, uint64_t); + HANDLE_TYPE(BOOL, Bool, bool); + HANDLE_TYPE(STRING, String, std::string); + default: { + // other types are impossible to be protobuf map key + ATOM_FATAL("invalid map key type: %d", key_fd->cpp_type()); + break; + } +#undef HANDLE_TYPE + } + } else if (fd->is_repeated()) { + // Reference: + // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/reflection_ops.cc#L68 + int count = src_reflection->FieldSize(src, fd); + for (int j = 0; j < count; j++) { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: { \ + dst_reflection->Add##METHOD(&dst, fd, src_reflection->GetRepeated##METHOD(src, fd, j)); \ + break; \ } + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + const google::protobuf::Message& src_child = src_reflection->GetRepeatedMessage(src, fd, j); + PatchMessage(*dst_reflection->AddMessage(&dst, fd), src_child); + break; + } + } + } + } else { + switch (fd->cpp_type()) { +#define HANDLE_TYPE(CPPTYPE, METHOD) \ + case google::protobuf::FieldDescriptor::CPPTYPE_##CPPTYPE: \ + dst_reflection->Set##METHOD(&dst, fd, src_reflection->Get##METHOD(src, fd)); \ + break; + + HANDLE_TYPE(INT32, Int32); + HANDLE_TYPE(INT64, Int64); + HANDLE_TYPE(UINT32, UInt32); + HANDLE_TYPE(UINT64, UInt64); + HANDLE_TYPE(FLOAT, Float); + HANDLE_TYPE(DOUBLE, Double); + HANDLE_TYPE(BOOL, Bool); + HANDLE_TYPE(STRING, String); + HANDLE_TYPE(ENUM, Enum); +#undef HANDLE_TYPE + + case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + const google::protobuf::Message& src_child = src_reflection->GetMessage(src, fd); + PatchMessage(*dst_reflection->MutableMessage(&dst, fd), src_child); + break; + } + } + } + + dst_reflection->MutableUnknownFields(&dst)->MergeFrom(src_reflection->GetUnknownFields(src)); return true; } diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.h b/test/cpp-tableau-loader/src/protoconf/util.pc.h index 2a5f7732..ab6544b3 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.h @@ -8,11 +8,8 @@ #include #include -#include #include -#include "tableau/protobuf/tableau.pb.h" - namespace tableau { const std::string& GetErrMsg(); void SetErrMsg(const std::string& msg); @@ -62,10 +59,8 @@ Format GetFormat(const std::filesystem::path& path); // and the error message can be obtained by GetErrMsg(). const std::string& Format2Ext(Format fmt); -bool JSON2Message(const std::string& json, google::protobuf::Message& msg, - std::shared_ptr options = nullptr); -bool Text2Message(const std::string& text, google::protobuf::Message& msg); -bool Bin2Message(const std::string& bin, google::protobuf::Message& msg); +// PatchMessage patches src into dst, which must be a message with the same descriptor. +bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg); diff --git a/test/go-tableau-loader/gen.bat b/test/go-tableau-loader/gen.bat new file mode 100644 index 00000000..7a9c2fb3 --- /dev/null +++ b/test/go-tableau-loader/gen.bat @@ -0,0 +1,44 @@ +@echo off +setlocal +setlocal enabledelayedexpansion + +for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i +cd /d "%repoRoot%" + +set "PROTOC=%repoRoot%\third_party\_submodules\protobuf\cmake\build\protoc.exe" +set "PROTOBUF_PROTO=%repoRoot%\third_party\_submodules\protobuf\src" +set "TABLEAU_PROTO=%repoRoot%\third_party\_submodules\tableau\proto" +set "PLGUIN_DIR=%repoRoot%\cmd\protoc-gen-go-tableau-loader" +set "PROTOCONF_IN=%repoRoot%\test\proto" +set "PROTOCONF_OUT=%repoRoot%\test\go-tableau-loader\protoconf" +set "LOADER_OUT=%PROTOCONF_OUT%\loader" + +REM remove old generated files +rmdir /s /q "%PROTOCONF_OUT%" "%LOADER_OUT%" 2>nul +mkdir "%PROTOCONF_OUT%" "%LOADER_OUT%" + +REM build +pushd "%PLGUIN_DIR%" +go build +popd + +set "PATH=%PATH%;%PLGUIN_DIR%" + +set protoFiles= +pushd "%PROTOCONF_IN%" +for /R %%f in (*.proto) do ( + set protoFiles=!protoFiles! "%%f" +) +popd +"%PROTOC%" ^ +--go-tableau-loader_out="%LOADER_OUT%" ^ +--go-tableau-loader_opt=paths=source_relative,pkg=loader ^ +--go_out="%PROTOCONF_OUT%" ^ +--go_opt=paths=source_relative ^ +--proto_path="%PROTOBUF_PROTO%" ^ +--proto_path="%TABLEAU_PROTO%" ^ +--proto_path="%PROTOCONF_IN%" ^ +!protoFiles! + +endlocal +endlocal diff --git a/test/go-tableau-loader/gen.sh b/test/go-tableau-loader/gen.sh index a5e9c8da..e547ff5e 100755 --- a/test/go-tableau-loader/gen.sh +++ b/test/go-tableau-loader/gen.sh @@ -7,7 +7,7 @@ set -o pipefail shopt -s globstar cd "$(git rev-parse --show-toplevel)" -PROTOC="./third_party/_submodules/protobuf/src/protoc" +PROTOC="./third_party/_submodules/protobuf/cmake/build/protoc" PROTOBUF_PROTO="./third_party/_submodules/protobuf/src" TABLEAU_PROTO="./third_party/_submodules/tableau/proto" PLGUIN_DIR="./cmd/protoc-gen-go-tableau-loader" diff --git a/test/go-tableau-loader/main.go b/test/go-tableau-loader/main.go index 329c6838..83bb9b7c 100644 --- a/test/go-tableau-loader/main.go +++ b/test/go-tableau-loader/main.go @@ -15,9 +15,12 @@ import ( func main() { err := hub.GetHub().Load("../testdata/conf/", format.JSON, load.IgnoreUnknownFields(), - load.Paths(map[string]string{ - "ItemConf": "../testdata/conf/ItemConf.json", - })) + load.WithMessagerOptions(map[string]*load.MessagerOptions{ + "ItemConf": { + Path: "../testdata/conf/ItemConf.json", + }, + }), + ) if err != nil { panic(err) } diff --git a/test/go-tableau-loader/protoconf/loader/hub.pc.go b/test/go-tableau-loader/protoconf/loader/hub.pc.go index 9c3c4f2c..899bdd5e 100644 --- a/test/go-tableau-loader/protoconf/loader/hub.pc.go +++ b/test/go-tableau-loader/protoconf/loader/hub.pc.go @@ -244,7 +244,7 @@ func (h *Hub) Load(dir string, format format.Format, options ...load.Option) err messagerMap := h.NewMessagerMap() opts := load.ParseOptions(options...) for name, msger := range messagerMap { - mopts := load.ParseMessagerOptionsFromOptions(opts, name) + mopts := opts.ParseMessagerOptionsByName(name) if err := msger.Load(dir, format, mopts); err != nil { return errors.WithMessagef(err, "failed to load: %v", name) }