Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 21 additions & 25 deletions cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc
Original file line number Diff line number Diff line change
Expand Up @@ -64,30 +64,30 @@ std::shared_ptr<const MessagerOptions> ParseMessagerOptions(std::shared_ptr<cons
return mopts;
}

bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::string& path, Format fmt, tableau::Patch patch,
std::shared_ptr<const MessagerOptions> options /* = nullptr*/) {
bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt,
tableau::Patch patch, std::shared_ptr<const MessagerOptions> options /* = nullptr*/) {
if (options == nullptr) {
return LoadMessager(msg, path, fmt, nullptr);
}
if (options->mode == LoadMode::kOnlyMain) {
// ignore patch files when LoadMode::kModeOnlyMain specified
return options->load_func(msg, path, fmt, nullptr);
}
std::string name = util::GetProtoName(msg);
std::vector<std::string> patch_paths;
const std::string& name = msg.GetDescriptor()->name();
std::vector<std::filesystem::path> patch_paths;
if (!options->patch_paths.empty()) {
// patch path specified in PatchPaths, then use it instead of PatchDirs.
patch_paths = options->patch_paths;
} else {
std::string filename = name + util::Format2Ext(fmt);
std::filesystem::path filename = name + util::Format2Ext(fmt);
for (auto&& patch_dir : options->patch_dirs) {
patch_paths.emplace_back((std::filesystem::path(patch_dir) / filename).make_preferred().string());
patch_paths.emplace_back(patch_dir / filename);
}
}

std::vector<std::string> existed_patch_paths;
std::vector<std::filesystem::path> existed_patch_paths;
for (auto&& patch_path : patch_paths) {
if (util::ExistsFile(patch_path)) {
if (std::filesystem::exists(patch_path)) {
existed_patch_paths.emplace_back(patch_path);
}
}
Expand All @@ -103,8 +103,8 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::string& pa
switch (patch) {
case tableau::PATCH_REPLACE: {
// just use the last "patch" file
std::string& patch_path = existed_patch_paths.back();
if (!options->load_func(msg, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) {
std::filesystem::path& patch_path = existed_patch_paths.back();
if (!options->load_func(msg, patch_path, util::GetFormat(patch_path), options)) {
return false;
}
break;
Expand All @@ -121,7 +121,7 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::string& pa
std::unique_ptr<google::protobuf::Message> _auto_release(msg.New());
// load patch_msg from each "patch" file
for (auto&& patch_path : existed_patch_paths) {
if (!options->load_func(*patch_msg_ptr, patch_path, util::Ext2Format(util::GetExt(patch_path)), options)) {
if (!options->load_func(*patch_msg_ptr, patch_path, util::GetFormat(patch_path), options)) {
return false;
}
if (!PatchMessage(msg, *patch_msg_ptr)) {
Expand All @@ -131,16 +131,16 @@ bool LoadMessagerWithPatch(google::protobuf::Message& msg, const std::string& pa
break;
}
default: {
SetErrMsg("unknown patch type: " + util::GetPatchName(patch));
SetErrMsg("unknown patch type: " + tableau::Patch_Name(patch));
return false;
}
}
ATOM_DEBUG("patched(%s) %s by %s: %s", util::GetPatchName(patch).c_str(), name.c_str(),
ATOM_DEBUG("patched(%s) %s by %s: %s", tableau::Patch_Name(patch).c_str(), name.c_str(),
ATOM_VECTOR_STR(existed_patch_paths).c_str(), msg.ShortDebugString().c_str());
return true;
}

bool LoadMessager(google::protobuf::Message& msg, const std::string& path, Format fmt,
bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt,
std::shared_ptr<const MessagerOptions> options /* = nullptr*/) {
std::string content;
ReadFunc read_func = util::ReadFile;
Expand Down Expand Up @@ -168,27 +168,23 @@ bool LoadMessager(google::protobuf::Message& msg, const std::string& path, Forma
}
}

bool LoadMessagerInDir(google::protobuf::Message& msg, const std::string& dir, Format fmt,
bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt,
std::shared_ptr<const MessagerOptions> options /* = nullptr*/) {
std::string name = util::GetProtoName(msg);
std::string path;
const std::string& name = msg.GetDescriptor()->name();
std::filesystem::path path;
if (options && !options->path.empty()) {
// path specified in Paths, then use it instead of dir.
path = options->path;
fmt = util::Ext2Format(util::GetExt(path));
fmt = util::GetFormat(path);
}
if (path.empty()) {
std::string filename = name + util::Format2Ext(fmt);
path = (std::filesystem::path(dir) / filename).make_preferred().string();
std::filesystem::path filename = name + util::Format2Ext(fmt);
path = dir / filename;
}

const google::protobuf::Descriptor* descriptor = msg.GetDescriptor();
if (!descriptor) {
SetErrMsg("failed to get descriptor of message: " + name);
return false;
}
// access the extension directly using the generated identifier
const tableau::WorksheetOptions worksheet_options = descriptor->options().GetExtension(tableau::worksheet);
const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet);
if (worksheet_options.patch() != tableau::PATCH_NONE) {
return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options);
}
Expand Down
23 changes: 14 additions & 9 deletions cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#include <google/protobuf/message.h>

#include <chrono>
#include <filesystem>
#include <functional>
#include <optional>
#include <string>
Expand All @@ -20,10 +21,10 @@ struct MessagerOptions;
class Hub;

// ReadFunc reads the config file and returns its content.
using ReadFunc = std::function<bool(const std::string& filename, std::string& content)>;
using ReadFunc = std::function<bool(const std::filesystem::path& filename, std::string& content)>;
// LoadFunc defines a func which can load message's content based on the given
// path, format, and options.
using LoadFunc = std::function<bool(google::protobuf::Message& msg, const std::string& path, Format fmt,
using LoadFunc = std::function<bool(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt,
std::shared_ptr<const MessagerOptions> options)>;

// BaseOptions is the common struct for both global-level and messager-level
Expand All @@ -34,7 +35,7 @@ struct BaseOptions {
// Refer https://protobuf.dev/reference/cpp/api-docs/google.protobuf.util.json_util/#JsonParseOptions.
std::optional<bool> ignore_unknown_fields;
// Specify the directory paths for config patching.
std::vector<std::string> patch_dirs;
std::vector<std::filesystem::path> patch_dirs;
// Specify the loading mode for config patching.
// - For LoadOptions, default is LoadMode::kModeAll.
// - For MessagerOptions, inherit from LoadOptions if not set.
Expand Down Expand Up @@ -63,10 +64,10 @@ struct MessagerOptions : public BaseOptions {
// Path maps each messager name to a corresponding config file path.
// If specified, then the main messager will be parsed from the file
// directly, other than the specified load dir.
std::string path;
std::filesystem::path path;
// Patch paths maps each messager name to one or multiple corresponding patch file paths.
// If specified, then main messager will be patched.
std::vector<std::string> patch_paths;
std::vector<std::filesystem::path> patch_paths;
};

class Messager {
Expand All @@ -77,10 +78,14 @@ class Messager {

public:
virtual ~Messager() = default;
static const std::string& Name() { return kEmpty; }
static const std::string& Name() {
static const std::string kEmpty = "";
return kEmpty;
}
const Stats& GetStats() { return stats_; }
// Load fills message from file in the specified directory and format.
virtual bool Load(const std::string& dir, Format fmt, std::shared_ptr<const MessagerOptions> options = nullptr) = 0;
virtual bool Load(const std::filesystem::path& dir, Format fmt,
std::shared_ptr<const MessagerOptions> options = nullptr) = 0;
// Message returns the inner message data.
virtual const google::protobuf::Message* Message() const { return nullptr; }
// callback after all messagers loaded.
Expand All @@ -98,8 +103,8 @@ std::shared_ptr<const LoadOptions> ParseLoadOptions(std::shared_ptr<const LoadOp
// messager-level options taken into consideration.
std::shared_ptr<const MessagerOptions> ParseMessagerOptions(std::shared_ptr<const LoadOptions> opts,
const std::string& name);
bool LoadMessager(google::protobuf::Message& msg, const std::string& path, Format fmt = Format::kJSON,
bool LoadMessager(google::protobuf::Message& msg, const std::filesystem::path& path, Format fmt = Format::kJSON,
std::shared_ptr<const MessagerOptions> options = nullptr);
bool LoadMessagerInDir(google::protobuf::Message& msg, const std::string& dir, Format fmt = Format::kJSON,
bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::path& dir, Format fmt = Format::kJSON,
std::shared_ptr<const MessagerOptions> options = nullptr);
} // namespace tableau
15 changes: 7 additions & 8 deletions cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.cc
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include <unistd.h>
#endif

#include <filesystem>
#include <thread>
#include <unordered_map>

Expand Down Expand Up @@ -53,14 +54,12 @@ void SetDefaultLogger(Logger* logger) {
g_default_logger = logger;
}

int Logger::Init(const std::string& path, Level level) {
std::string dir = util::GetDir(path);
if (!dir.empty()) {
// prepare the specified directory
int status = util::Mkdir(dir);
if (status == -1) {
return status;
}
int Logger::Init(const std::filesystem::path& path, Level level) {
std::error_code ec;
std::filesystem::create_directories(path.parent_path(), ec);
if (ec) {
SetErrMsg("failed to create log directory: " + ec.message());
return ec.value();
}
ofs_.open(path, std::ofstream::out | std::ofstream::app);
os_ = &ofs_; // use file stream as output stream
Expand Down
3 changes: 2 additions & 1 deletion cmd/protoc-gen-cpp-tableau-loader/embed/logger.pc.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#pragma once
#include <filesystem>
#include <fstream>
#include <functional>
#include <iostream>
Expand Down Expand Up @@ -49,7 +50,7 @@ class Logger {
~Logger() { ofs_.close(); }
// Init the logger with the specified path.
// NOTE: no guarantee of thread-safety.
int Init(const std::string& path, Level level);
int Init(const std::filesystem::path& path, Level level);
// Set the writer for writing log.
void SetWriter(Writer writer) { writer_ = writer; }
// Log with guarantee of thread-safety.
Expand Down
57 changes: 9 additions & 48 deletions cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,42 +22,18 @@ const std::string kTextExt = ".txt";
const std::string kBinExt = ".bin";

namespace util {
int Mkdir(const std::string& path) {
std::error_code ec;
if (!std::filesystem::create_directories(path, ec)) {
if (ec) {
std::cerr << "system error: " << ec.message() << std::endl;
return -1;
}
}
return 0;
}

std::string GetDir(const std::string& path) { return std::filesystem::path(path).parent_path().string(); }

bool ExistsFile(const std::string& filename) { return std::filesystem::exists(filename); }

bool ReadFile(const std::string& filename, std::string& content) {
bool ReadFile(const std::filesystem::path& filename, std::string& content) {
std::ifstream file(filename);
if (!file.is_open()) {
SetErrMsg("failed to open " + filename + ": " + strerror(errno));
SetErrMsg("failed to open " + filename.string() + ": " + strerror(errno));
return false;
}
std::stringstream ss;
ss << file.rdbuf();
content = ss.str();
content.assign(std::istreambuf_iterator<char>(file), {});
return true;
}

std::string GetExt(const std::string& path) {
std::size_t pos = path.find_last_of(".");
if (pos != std::string::npos) {
return path.substr(pos);
}
return kEmpty;
}

Format Ext2Format(const std::string& ext) {
Format GetFormat(const std::filesystem::path& path) {
auto ext = path.extension();
if (ext == kJSONExt) {
return Format::kJSON;
} else if (ext == kTextExt) {
Expand Down Expand Up @@ -93,43 +69,28 @@ bool JSON2Message(const std::string& json, google::protobuf::Message& msg,
status = google::protobuf::util::JsonStringToMessage(json, &msg);
}
if (!status.ok()) {
SetErrMsg("failed to parse " + GetProtoName(msg) + kJSONExt + ": " + status.ToString());
SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kJSONExt + ": " + status.ToString());
return false;
}
return true;
}

bool Text2Message(const std::string& text, google::protobuf::Message& msg) {
if (!google::protobuf::TextFormat::ParseFromString(text, &msg)) {
SetErrMsg("failed to parse " + GetProtoName(msg) + kTextExt);
SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kTextExt);
return false;
}
return true;
}

bool Bin2Message(const std::string& bin, google::protobuf::Message& msg) {
if (!msg.ParseFromString(bin)) {
SetErrMsg("failed to parse " + GetProtoName(msg) + kBinExt);
SetErrMsg("failed to parse " + msg.GetDescriptor()->name() + kBinExt);
return false;
}
return true;
}

const std::string& GetProtoName(const google::protobuf::Message& msg) {
const auto* md = msg.GetDescriptor();
return md != nullptr ? md->name() : kEmpty;
}

std::string GetPatchName(tableau::Patch patch) {
auto* descriptor = tableau::Patch_descriptor();
if (descriptor) {
auto* value = descriptor->FindValueByNumber(patch);
if (value) {
return value->name();
}
}
return std::to_string(static_cast<int>(patch));
}

// refer: https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/stubs/logging.h
void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg) {
static const std::unordered_map<int, log::Level> kLevelMap = {{google::protobuf::LOGLEVEL_INFO, log::kInfo},
Expand Down
27 changes: 4 additions & 23 deletions cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#include <google/protobuf/message.h>

#include <chrono>
#include <filesystem>
#include <functional>
#include <string>

Expand All @@ -18,7 +19,6 @@ enum class Format {
kBin,
};

static const std::string kEmpty = "";
extern const std::string kUnknownExt;
extern const std::string kJSONExt;
extern const std::string kTextExt;
Expand Down Expand Up @@ -48,27 +48,11 @@ inline std::size_t SugaredHashCombine(const T& v, O... others) {
return seed;
}

// Mkdir makes dir recursively.
int Mkdir(const std::string& path);
// GetDir returns all but the last element of path, typically the path's
// directory.
std::string GetDir(const std::string& path);
// ExistsFile checks if a file exists.
bool ExistsFile(const std::string& filename);
// ReadFile reads the file named by filename and returns the contents.
bool ReadFile(const std::string& filename, std::string& content);
bool ReadFile(const std::filesystem::path& filename, std::string& content);

// GetExt returns the file name extension used by path.
// The extension is the suffix beginning at the final dot
// in the final element of path; it is empty if there is
// no dot.
std::string GetExt(const std::string& path);
// Convert file extension to Format type.
// NOTE: ext includes dot ".", such as:
// - kJSONExt: ".json"
// - kTextExt: ".txt"
// - kBinExt: ".bin"
Format Ext2Format(const std::string& ext);
// GetFormat returns the Format type used by path.
Format GetFormat(const std::filesystem::path& path);
// Empty string will be returned if an unsupported enum value has been passed,
// and the error message can be obtained by GetErrMsg().
const std::string& Format2Ext(Format fmt);
Expand All @@ -78,9 +62,6 @@ bool JSON2Message(const std::string& json, google::protobuf::Message& msg,
bool Text2Message(const std::string& text, google::protobuf::Message& msg);
bool Bin2Message(const std::string& bin, google::protobuf::Message& msg);

const std::string& GetProtoName(const google::protobuf::Message& msg);
std::string GetPatchName(tableau::Patch patch);

void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& msg);

class TimeProfiler {
Expand Down
Loading