From 308541ca84d3e9bddc0ef319a3cafe1f024942d9 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Wed, 16 Dec 2020 13:41:48 -0500 Subject: [PATCH 01/17] new version Signed-off-by: Asra Ali --- bazel/external/json.BUILD | 24 + bazel/repositories.bzl | 11 + bazel/repository_locations.bzl | 14 + source/common/json/BUILD | 18 +- source/common/json/json_loader.cc | 280 ++++------ source/common/json/rapidjson_loader.cc | 704 +++++++++++++++++++++++++ source/common/json/rapidjson_loader.h | 22 + 7 files changed, 884 insertions(+), 189 deletions(-) create mode 100644 bazel/external/json.BUILD create mode 100644 source/common/json/rapidjson_loader.cc create mode 100644 source/common/json/rapidjson_loader.h diff --git a/bazel/external/json.BUILD b/bazel/external/json.BUILD new file mode 100644 index 0000000000000..5f673ca72dc60 --- /dev/null +++ b/bazel/external/json.BUILD @@ -0,0 +1,24 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +licenses(["notice"]) # Apache 2 + +cc_library( + name = "nlohmann_json_lib", + hdrs = glob([ + "include/nlohmann/*.hpp", + "include/nlohmann/**/*.hpp", + "include/nlohmann/*/*/*.hpp", + ]), + copts = [ + "-I external/nlohmann_json_lib", + ], + visibility = ["//visibility:public"], + alwayslink = 1, +) + +cc_library( + name = "json", + includes = ["include"], + visibility = ["//visibility:public"], + deps = [":nlohmann_json_lib"], +) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index db873d30c16fa..902c57e406543 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -144,6 +144,7 @@ def envoy_dependencies(skip_targets = []): _com_github_nghttp2_nghttp2() _com_github_nodejs_http_parser() _com_github_tencent_rapidjson() + _com_github_nlohmann_json() _com_google_absl() _com_google_googletest() _com_google_protobuf() @@ -437,6 +438,16 @@ def _com_github_tencent_rapidjson(): actual = "@com_github_tencent_rapidjson//:rapidjson", ) +def _com_github_nlohmann_json(): + external_http_archive( + name = "com_github_nlohmann_json", + build_file = "@envoy//bazel/external:json.BUILD", + ) + native.bind( + name = "json", + actual = "@com_github_nlohmann_json//:json", + ) + def _com_github_nodejs_http_parser(): external_http_archive( name = "com_github_nodejs_http_parser", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 458c69f1f82b7..1efb65077a7b0 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -445,6 +445,20 @@ REPOSITORY_LOCATIONS_SPEC = dict( release_date = "2019-12-03", cpe = "cpe:2.3:a:tencent:rapidjson:*", ), + com_github_nlohmann_json = dict( + project_name = "nlohmann JSON", + project_desc = "Fast JSON parser/generator for C++", + project_url = "https://nlohmann.github.io/json", + version = "3.9.1", + sha256 = "4cf0df69731494668bdd6460ed8cb269b68de9c19ad8c27abc24cd72605b2d5b", + strip_prefix = "json-{version}", + urls = ["https://github.com/nlohmann/json/archive/v{version}.tar.gz"], + # This will be a replacement for rapidJSON used in extensions and may also be a fast + # replacement for protobuf JSON. + use_category = ["controlplane", "dataplane_core"], + release_date = "2020-07-06", + cpe = "cpe:2.3:a:json_project:json:*", + ), com_github_twitter_common_lang = dict( project_name = "twitter.common.lang (Thrift)", project_desc = "twitter.common Python language and compatibility facilities", diff --git a/source/common/json/BUILD b/source/common/json/BUILD index edda394d6949d..7c2b342502971 100644 --- a/source/common/json/BUILD +++ b/source/common/json/BUILD @@ -8,12 +8,28 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_library( + name = "rapidjson_loader_lib", + srcs = ["rapidjson_loader.cc"], + hdrs = ["rapidjson_loader.h"], + external_deps = [ + "rapidjson", + ], + deps = [ + "//include/envoy/json:json_object_interface", + "//source/common/common:assert_lib", + "//source/common/common:hash_lib", + "//source/common/common:utility_lib", + "//source/common/protobuf:utility_lib", + ], +) + envoy_cc_library( name = "json_loader_lib", srcs = ["json_loader.cc"], hdrs = ["json_loader.h"], external_deps = [ - "rapidjson", + "json", ], deps = [ "//include/envoy/json:json_object_interface", diff --git a/source/common/json/json_loader.cc b/source/common/json/json_loader.cc index bb4ccf808662d..d87e8d20eb436 100644 --- a/source/common/json/json_loader.cc +++ b/source/common/json/json_loader.cc @@ -15,14 +15,8 @@ #include "common/common/utility.h" #include "common/protobuf/utility.h" -// Do not let RapidJson leak outside of this file. -#include "rapidjson/document.h" -#include "rapidjson/error/en.h" -#include "rapidjson/reader.h" -#include "rapidjson/schema.h" -#include "rapidjson/stream.h" -#include "rapidjson/stringbuffer.h" -#include "rapidjson/writer.h" +// Do not let nlohmann/json leak outside of this file. +#include "include/nlohmann/json.hpp" #include "absl/strings/match.h" @@ -38,9 +32,6 @@ using FieldSharedPtr = std::shared_ptr; class Field : public Object { public: - void setLineNumberStart(uint64_t line_number) { line_number_start_ = line_number; } - void setLineNumberEnd(uint64_t line_number) { line_number_end_ = line_number; } - // Container factories for handler. static FieldSharedPtr createObject() { return FieldSharedPtr{new Field(Type::Object)}; } static FieldSharedPtr createArray() { return FieldSharedPtr{new Field(Type::Array)}; } @@ -88,7 +79,7 @@ class Field : public Object { bool empty() const override; bool hasObject(const std::string& name) const override; void iterate(const ObjectCallback& callback) const override; - void validateSchema(const std::string& schema) const override; + void validateSchema(const std::string&) const override; private: enum class Type { @@ -139,9 +130,9 @@ class Field : public Object { bool isType(Type type) const { return type == type_; } void checkType(Type type) const { if (!isType(type)) { - throw Exception(fmt::format( - "JSON field from line {} accessed with type '{}' does not match actual type '{}'.", - line_number_start_, typeAsString(type), typeAsString(type_))); + throw Exception( + fmt::format("JSON field accessed with type '{}' does not match actual type '{}'.", + typeAsString(type), typeAsString(type_))); } } @@ -167,57 +158,50 @@ class Field : public Object { return value_.integer_value_; } - rapidjson::Document asRapidJsonDocument() const; - static void buildRapidJsonDocument(const Field& field, rapidjson::Value& value, - rapidjson::Document::AllocatorType& allocator); + nlohmann::json asJsonDocument() const; + static void buildJsonDocument(const Field& field, nlohmann::json& value); - uint64_t line_number_start_ = 0; - uint64_t line_number_end_ = 0; const Type type_; Value value_; }; /** - * Custom stream to allow access to the line number for each object. + * Consume events from SAX callbacks to build JSON Field. */ -class LineCountingStringStream : public rapidjson::StringStream { - // Ch is typedef in parent class to handle character encoding. +class ObjectHandler : public nlohmann::json_sax { public: - LineCountingStringStream(const Ch* src) : rapidjson::StringStream(src), line_number_(1) {} - Ch Take() { - Ch ret = rapidjson::StringStream::Take(); - if (ret == '\n') { - line_number_++; - } - return ret; + ObjectHandler() : state_(State::ExpectRoot) {} + + bool start_object(std::size_t) override; + bool end_object() override; + bool key(std::string& val) override; + bool start_array(std::size_t) override; + bool end_array() override; + bool boolean(bool value) override { return handleValueEvent(Field::createValue(value)); } + bool number_integer(int64_t value) override { + return handleValueEvent(Field::createValue(static_cast(value))); + } + bool number_unsigned(uint64_t value) override { + return handleValueEvent(Field::createValue(static_cast(value))); + } + bool number_float(double value, const std::string&) override { + return handleValueEvent(Field::createValue(value)); + } + bool null() override { return handleValueEvent(Field::createNull()); } + bool string(std::string& value) override { return handleValueEvent(Field::createValue(value)); } + bool binary(binary_t&) override { return false; } + bool parse_error(std::size_t position, const std::string& token, + const nlohmann::detail::exception& ex) override { + error_offset_ = position; + error_ = ex.what(); + error_token_ = token; + return true; } - uint64_t getLineNumber() const { return line_number_; } -private: - uint64_t line_number_; -}; - -/** - * Consume events from SAX callbacks to build JSON Field. - */ -class ObjectHandler : public rapidjson::BaseReaderHandler, ObjectHandler> { -public: - ObjectHandler(LineCountingStringStream& stream) : state_(State::ExpectRoot), stream_(stream){}; - - bool StartObject(); - bool EndObject(rapidjson::SizeType); - bool Key(const char* value, rapidjson::SizeType size, bool); - bool StartArray(); - bool EndArray(rapidjson::SizeType); - bool Bool(bool value); - bool Double(double value); - bool Int(int value); - bool Uint(unsigned value); - bool Int64(int64_t value); - bool Uint64(uint64_t value); - bool Null(); - bool String(const char* value, rapidjson::SizeType size, bool); - bool RawNumber(const char*, rapidjson::SizeType, bool); + bool hasParseError() { return !error_.empty(); } + std::size_t getErrorOffset() { return error_offset_; } + std::string getParseError() { return error_;} + std::string getErrorToken() { return error_token_; } ObjectSharedPtr getRoot() { return root_; } @@ -232,83 +216,79 @@ class ObjectHandler : public rapidjson::BaseReaderHandler, Obj ExpectFinished, }; State state_; - LineCountingStringStream& stream_; std::stack stack_; std::string key_; FieldSharedPtr root_; -}; -void Field::buildRapidJsonDocument(const Field& field, rapidjson::Value& value, - rapidjson::Document::AllocatorType& allocator) { + std::string error_; + std::string error_token_; + std::size_t error_offset_; +}; +void Field::buildJsonDocument(const Field& field, nlohmann::json& value) { switch (field.type_) { case Type::Array: { - value.SetArray(); - value.Reserve(field.value_.array_value_.size(), allocator); for (const auto& element : field.value_.array_value_) { switch (element->type_) { case Type::Array: case Type::Object: { - rapidjson::Value nested_value; - buildRapidJsonDocument(*element, nested_value, allocator); - value.PushBack(nested_value, allocator); + nlohmann::json nested_value; + buildJsonDocument(*element, nested_value); + value.push_back(nested_value); break; } case Type::Boolean: - value.PushBack(element->value_.boolean_value_, allocator); + value.push_back(element->value_.boolean_value_); break; case Type::Double: - value.PushBack(element->value_.double_value_, allocator); + value.push_back(element->value_.double_value_); break; case Type::Integer: - value.PushBack(element->value_.integer_value_, allocator); + value.push_back(element->value_.integer_value_); break; case Type::Null: - value.PushBack(rapidjson::Value(), allocator); + value.push_back(nlohmann::json::value_t::null); break; case Type::String: - value.PushBack(rapidjson::StringRef(element->value_.string_value_.c_str()), allocator); + value.push_back(element->value_.string_value_); } } break; } case Type::Object: { - value.SetObject(); for (const auto& item : field.value_.object_value_) { - auto name = rapidjson::StringRef(item.first.c_str()); + auto name = std::string(item.first.c_str()); switch (item.second->type_) { case Type::Array: case Type::Object: { - rapidjson::Value nested_value; - buildRapidJsonDocument(*item.second, nested_value, allocator); - value.AddMember(name, nested_value, allocator); + nlohmann::json nested_value; + buildJsonDocument(*item.second, nested_value); + value.emplace(name, nested_value); break; } case Type::Boolean: - value.AddMember(name, item.second->value_.boolean_value_, allocator); + value.emplace(name, item.second->value_.boolean_value_); break; case Type::Double: - value.AddMember(name, item.second->value_.double_value_, allocator); + value.emplace(name, item.second->value_.double_value_); break; case Type::Integer: - value.AddMember(name, item.second->value_.integer_value_, allocator); + value.emplace(name, item.second->value_.integer_value_); break; case Type::Null: - value.AddMember(name, rapidjson::Value(), allocator); + value.emplace(name, nlohmann::json::value_t::null); break; case Type::String: - value.AddMember(name, rapidjson::StringRef(item.second->value_.string_value_.c_str()), - allocator); + value.emplace(name, item.second->value_.string_value_); break; } } break; } case Type::Null: { - value.SetNull(); break; } default: @@ -316,26 +296,21 @@ void Field::buildRapidJsonDocument(const Field& field, rapidjson::Value& value, } } -rapidjson::Document Field::asRapidJsonDocument() const { - rapidjson::Document document; - rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); - buildRapidJsonDocument(*this, document, allocator); - return document; +nlohmann::json Field::asJsonDocument() const { + nlohmann::json j; + buildJsonDocument(*this, j); + return j; } uint64_t Field::hash() const { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - asRapidJsonDocument().Accept(writer); - return HashUtil::xxHash64(buffer.GetString()); + return HashUtil::xxHash64(asJsonString()); } bool Field::getBoolean(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Boolean)) { - throw Exception(fmt::format("key '{}' missing or not a boolean from lines {}-{}", name, - line_number_start_, line_number_end_)); + throw Exception(fmt::format("key '{}' missing or not a boolean", name)); } return value_itr->second->booleanValue(); } @@ -354,8 +329,7 @@ double Field::getDouble(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Double)) { - throw Exception(fmt::format("key '{}' missing or not a double from lines {}-{}", name, - line_number_start_, line_number_end_)); + throw Exception(fmt::format("key '{}' missing or not a double", name)); } return value_itr->second->doubleValue(); } @@ -374,8 +348,7 @@ int64_t Field::getInteger(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Integer)) { - throw Exception(fmt::format("key '{}' missing or not an integer from lines {}-{}", name, - line_number_start_, line_number_end_)); + throw Exception(fmt::format("key '{}' missing or not an integer", name)); } return value_itr->second->integerValue(); } @@ -397,12 +370,10 @@ ObjectSharedPtr Field::getObject(const std::string& name, bool allow_empty) cons if (allow_empty) { return createObject(); } else { - throw Exception(fmt::format("key '{}' missing from lines {}-{}", name, line_number_start_, - line_number_end_)); + throw Exception(fmt::format("key '{}' missing from lines", name)); } } else if (!value_itr->second->isType(Type::Object)) { - throw Exception(fmt::format("key '{}' not an object from line {}", name, - value_itr->second->line_number_start_)); + throw Exception(fmt::format("key '{}' not an object", name)); } else { return value_itr->second; } @@ -416,8 +387,7 @@ std::vector Field::getObjectArray(const std::string& name, if (allow_empty && value_itr == value_.object_value_.end()) { return std::vector(); } - throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, - line_number_start_, line_number_end_)); + throw Exception(fmt::format("key '{}' missing or not an array", name)); } std::vector array_value = value_itr->second->arrayValue(); @@ -428,8 +398,7 @@ std::string Field::getString(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::String)) { - throw Exception(fmt::format("key '{}' missing or not a string from lines {}-{}", name, - line_number_start_, line_number_end_)); + throw Exception(fmt::format("key '{}' missing or not a string", name)); } return value_itr->second->stringValue(); } @@ -452,16 +421,14 @@ std::vector Field::getStringArray(const std::string& name, bool all if (allow_empty && value_itr == value_.object_value_.end()) { return string_array; } - throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, - line_number_start_, line_number_end_)); + throw Exception(fmt::format("key '{}' missing or not an array", name)); } std::vector array = value_itr->second->arrayValue(); string_array.reserve(array.size()); for (const auto& element : array) { if (!element->isType(Type::String)) { - throw Exception(fmt::format("JSON array '{}' from line {} does not contain all strings", name, - line_number_start_)); + throw Exception(fmt::format("JSON array '{}' does not contain all strings", name)); } string_array.push_back(element->stringValue()); } @@ -475,11 +442,8 @@ std::vector Field::asObjectArray() const { } std::string Field::asJsonString() const { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - rapidjson::Document document = asRapidJsonDocument(); - document.Accept(writer); - return buffer.GetString(); + nlohmann::json j = asJsonDocument(); + return j.dump(); } bool Field::empty() const { @@ -509,36 +473,12 @@ void Field::iterate(const ObjectCallback& callback) const { } } -void Field::validateSchema(const std::string& schema) const { - rapidjson::Document schema_document; - if (schema_document.Parse<0>(schema.c_str()).HasParseError()) { - throw std::invalid_argument(fmt::format( - "Schema supplied to validateSchema is not valid JSON\n Error(offset {}) : {}\n", - schema_document.GetErrorOffset(), GetParseError_En(schema_document.GetParseError()))); - } - - rapidjson::SchemaDocument schema_document_for_validator(schema_document); - rapidjson::SchemaValidator schema_validator(schema_document_for_validator); - - if (!asRapidJsonDocument().Accept(schema_validator)) { - rapidjson::StringBuffer schema_string_buffer; - rapidjson::StringBuffer document_string_buffer; - - schema_validator.GetInvalidSchemaPointer().StringifyUriFragment(schema_string_buffer); - schema_validator.GetInvalidDocumentPointer().StringifyUriFragment(document_string_buffer); - - throw Exception(fmt::format( - "JSON at lines {}-{} does not conform to schema.\n Invalid schema: {}\n" - " Schema violation: {}\n" - " Offending document key: {}", - line_number_start_, line_number_end_, schema_string_buffer.GetString(), - schema_validator.GetInvalidSchemaKeyword(), document_string_buffer.GetString())); - } +void Field::validateSchema(const std::string&) const { + throw Exception("not implemented"); } -bool ObjectHandler::StartObject() { +bool ObjectHandler::start_object(std::size_t) { FieldSharedPtr object = Field::createObject(); - object->setLineNumberStart(stream_.getLineNumber()); switch (state_) { case State::ExpectValueOrStartObjectArray: @@ -561,10 +501,9 @@ bool ObjectHandler::StartObject() { } } -bool ObjectHandler::EndObject(rapidjson::SizeType) { +bool ObjectHandler::end_object() { switch (state_) { case State::ExpectKeyOrEndObject: - stack_.top()->setLineNumberEnd(stream_.getLineNumber()); stack_.pop(); if (stack_.empty()) { @@ -580,10 +519,10 @@ bool ObjectHandler::EndObject(rapidjson::SizeType) { } } -bool ObjectHandler::Key(const char* value, rapidjson::SizeType size, bool) { +bool ObjectHandler::key(std::string& val) { switch (state_) { case State::ExpectKeyOrEndObject: - key_ = std::string(value, size); + key_ = val; state_ = State::ExpectValueOrStartObjectArray; return true; default: @@ -591,9 +530,8 @@ bool ObjectHandler::Key(const char* value, rapidjson::SizeType size, bool) { } } -bool ObjectHandler::StartArray() { +bool ObjectHandler::start_array(std::size_t) { FieldSharedPtr array = Field::createArray(); - array->setLineNumberStart(stream_.getLineNumber()); switch (state_) { case State::ExpectValueOrStartObjectArray: @@ -615,10 +553,9 @@ bool ObjectHandler::StartArray() { } } -bool ObjectHandler::EndArray(rapidjson::SizeType) { +bool ObjectHandler::end_array() { switch (state_) { case State::ExpectArrayValueOrEndArray: - stack_.top()->setLineNumberEnd(stream_.getLineNumber()); stack_.pop(); if (stack_.empty()) { @@ -635,38 +572,7 @@ bool ObjectHandler::EndArray(rapidjson::SizeType) { } } -// Value handlers -bool ObjectHandler::Bool(bool value) { return handleValueEvent(Field::createValue(value)); } -bool ObjectHandler::Double(double value) { return handleValueEvent(Field::createValue(value)); } -bool ObjectHandler::Int(int value) { - return handleValueEvent(Field::createValue(static_cast(value))); -} -bool ObjectHandler::Uint(unsigned value) { - return handleValueEvent(Field::createValue(static_cast(value))); -} -bool ObjectHandler::Int64(int64_t value) { return handleValueEvent(Field::createValue(value)); } -bool ObjectHandler::Uint64(uint64_t value) { - if (value > static_cast(std::numeric_limits::max())) { - throw Exception(fmt::format("JSON value from line {} is larger than int64_t (not supported)", - stream_.getLineNumber())); - } - return handleValueEvent(Field::createValue(static_cast(value))); -} - -bool ObjectHandler::Null() { return handleValueEvent(Field::createNull()); } - -bool ObjectHandler::String(const char* value, rapidjson::SizeType size, bool) { - return handleValueEvent(Field::createValue(std::string(value, size))); -} - -bool ObjectHandler::RawNumber(const char*, rapidjson::SizeType, bool) { - // Only called if kParseNumbersAsStrings is set as a parse flag, which it is not. - NOT_REACHED_GCOVR_EXCL_LINE; -} - bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { - ptr->setLineNumberStart(stream_.getLineNumber()); - switch (state_) { case State::ExpectValueOrStartObjectArray: state_ = State::ExpectKeyOrEndObject; @@ -683,16 +589,14 @@ bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { } // namespace ObjectSharedPtr Factory::loadFromString(const std::string& json) { - LineCountingStringStream json_stream(json.c_str()); + ObjectHandler handler; - ObjectHandler handler(json_stream); - rapidjson::Reader reader; - reader.Parse(json_stream, handler); + nlohmann::json::sax_parse(json, &handler); - if (reader.HasParseError()) { - throw Exception(fmt::format("JSON supplied is not valid. Error(offset {}, line {}): {}\n", - reader.GetErrorOffset(), json_stream.getLineNumber(), - GetParseError_En(reader.GetParseErrorCode()))); + if (handler.hasParseError()) { + throw Exception(fmt::format("JSON supplied is not valid. Error(offset {}, token {}): {}\n", + handler.getErrorOffset(), handler.getErrorToken(), + handler.getParseError())); } return handler.getRoot(); diff --git a/source/common/json/rapidjson_loader.cc b/source/common/json/rapidjson_loader.cc new file mode 100644 index 0000000000000..dfc74458616a5 --- /dev/null +++ b/source/common/json/rapidjson_loader.cc @@ -0,0 +1,704 @@ +#include "common/json/json_loader.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/common/assert.h" +#include "common/common/fmt.h" +#include "common/common/hash.h" +#include "common/common/utility.h" +#include "common/protobuf/utility.h" + +// Do not let RapidJson leak outside of this file. +#include "rapidjson/document.h" +#include "rapidjson/error/en.h" +#include "rapidjson/reader.h" +#include "rapidjson/schema.h" +#include "rapidjson/stream.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +#include "absl/strings/match.h" + +namespace Envoy { +namespace Json { +namespace RapidJson { + +namespace { +/** + * Internal representation of Object. + */ +class Field; +using FieldSharedPtr = std::shared_ptr; + +class Field : public Object { +public: + void setLineNumberStart(uint64_t line_number) { line_number_start_ = line_number; } + void setLineNumberEnd(uint64_t line_number) { line_number_end_ = line_number; } + + // Container factories for handler. + static FieldSharedPtr createObject() { return FieldSharedPtr{new Field(Type::Object)}; } + static FieldSharedPtr createArray() { return FieldSharedPtr{new Field(Type::Array)}; } + static FieldSharedPtr createNull() { return FieldSharedPtr{new Field(Type::Null)}; } + + bool isNull() const override { return type_ == Type::Null; } + bool isArray() const override { return type_ == Type::Array; } + bool isObject() const override { return type_ == Type::Object; } + + // Value factory. + template static FieldSharedPtr createValue(T value) { + return FieldSharedPtr{new Field(value)}; // NOLINT(modernize-make-shared) + } + + void append(FieldSharedPtr field_ptr) { + checkType(Type::Array); + value_.array_value_.push_back(field_ptr); + } + void insert(const std::string& key, FieldSharedPtr field_ptr) { + checkType(Type::Object); + value_.object_value_[key] = field_ptr; + } + + uint64_t hash() const override; + + bool getBoolean(const std::string& name) const override; + bool getBoolean(const std::string& name, bool default_value) const override; + double getDouble(const std::string& name) const override; + double getDouble(const std::string& name, double default_value) const override; + int64_t getInteger(const std::string& name) const override; + int64_t getInteger(const std::string& name, int64_t default_value) const override; + ObjectSharedPtr getObject(const std::string& name, bool allow_empty) const override; + std::vector getObjectArray(const std::string& name, + bool allow_empty) const override; + std::string getString(const std::string& name) const override; + std::string getString(const std::string& name, const std::string& default_value) const override; + std::vector getStringArray(const std::string& name, bool allow_empty) const override; + std::vector asObjectArray() const override; + std::string asString() const override { return stringValue(); } + bool asBoolean() const override { return booleanValue(); } + double asDouble() const override { return doubleValue(); } + int64_t asInteger() const override { return integerValue(); } + std::string asJsonString() const override; + + bool empty() const override; + bool hasObject(const std::string& name) const override; + void iterate(const ObjectCallback& callback) const override; + void validateSchema(const std::string& schema) const override; + +private: + enum class Type { + Array, + Boolean, + Double, + Integer, + Null, + Object, + String, + }; + static const char* typeAsString(Type t) { + switch (t) { + case Type::Array: + return "Array"; + case Type::Boolean: + return "Boolean"; + case Type::Double: + return "Double"; + case Type::Integer: + return "Integer"; + case Type::Null: + return "Null"; + case Type::Object: + return "Object"; + case Type::String: + return "String"; + } + + NOT_REACHED_GCOVR_EXCL_LINE; + } + + struct Value { + std::vector array_value_; + bool boolean_value_; + double double_value_; + int64_t integer_value_; + std::map object_value_; + std::string string_value_; + }; + + explicit Field(Type type) : type_(type) {} + explicit Field(const std::string& value) : type_(Type::String) { value_.string_value_ = value; } + explicit Field(int64_t value) : type_(Type::Integer) { value_.integer_value_ = value; } + explicit Field(double value) : type_(Type::Double) { value_.double_value_ = value; } + explicit Field(bool value) : type_(Type::Boolean) { value_.boolean_value_ = value; } + + bool isType(Type type) const { return type == type_; } + void checkType(Type type) const { + if (!isType(type)) { + throw Exception(fmt::format( + "JSON field from line {} accessed with type '{}' does not match actual type '{}'.", + line_number_start_, typeAsString(type), typeAsString(type_))); + } + } + + // Value return type functions. + std::string stringValue() const { + checkType(Type::String); + return value_.string_value_; + } + std::vector arrayValue() const { + checkType(Type::Array); + return value_.array_value_; + } + bool booleanValue() const { + checkType(Type::Boolean); + return value_.boolean_value_; + } + double doubleValue() const { + checkType(Type::Double); + return value_.double_value_; + } + int64_t integerValue() const { + checkType(Type::Integer); + return value_.integer_value_; + } + + rapidjson::Document asRapidJsonDocument() const; + static void buildRapidJsonDocument(const Field& field, rapidjson::Value& value, + rapidjson::Document::AllocatorType& allocator); + + uint64_t line_number_start_ = 0; + uint64_t line_number_end_ = 0; + const Type type_; + Value value_; +}; + +/** + * Custom stream to allow access to the line number for each object. + */ +class LineCountingStringStream : public rapidjson::StringStream { + // Ch is typedef in parent class to handle character encoding. +public: + LineCountingStringStream(const Ch* src) : rapidjson::StringStream(src), line_number_(1) {} + Ch Take() { + Ch ret = rapidjson::StringStream::Take(); + if (ret == '\n') { + line_number_++; + } + return ret; + } + uint64_t getLineNumber() const { return line_number_; } + +private: + uint64_t line_number_; +}; + +/** + * Consume events from SAX callbacks to build JSON Field. + */ +class ObjectHandler : public rapidjson::BaseReaderHandler, ObjectHandler> { +public: + ObjectHandler(LineCountingStringStream& stream) : state_(State::ExpectRoot), stream_(stream){}; + + bool StartObject(); + bool EndObject(rapidjson::SizeType); + bool Key(const char* value, rapidjson::SizeType size, bool); + bool StartArray(); + bool EndArray(rapidjson::SizeType); + bool Bool(bool value); + bool Double(double value); + bool Int(int value); + bool Uint(unsigned value); + bool Int64(int64_t value); + bool Uint64(uint64_t value); + bool Null(); + bool String(const char* value, rapidjson::SizeType size, bool); + bool RawNumber(const char*, rapidjson::SizeType, bool); + + ObjectSharedPtr getRoot() { return root_; } + +private: + bool handleValueEvent(FieldSharedPtr ptr); + + enum class State { + ExpectRoot, + ExpectKeyOrEndObject, + ExpectValueOrStartObjectArray, + ExpectArrayValueOrEndArray, + ExpectFinished, + }; + State state_; + LineCountingStringStream& stream_; + + std::stack stack_; + std::string key_; + + FieldSharedPtr root_; +}; + +void Field::buildRapidJsonDocument(const Field& field, rapidjson::Value& value, + rapidjson::Document::AllocatorType& allocator) { + + switch (field.type_) { + case Type::Array: { + value.SetArray(); + value.Reserve(field.value_.array_value_.size(), allocator); + for (const auto& element : field.value_.array_value_) { + switch (element->type_) { + case Type::Array: + case Type::Object: { + rapidjson::Value nested_value; + buildRapidJsonDocument(*element, nested_value, allocator); + value.PushBack(nested_value, allocator); + break; + } + case Type::Boolean: + value.PushBack(element->value_.boolean_value_, allocator); + break; + case Type::Double: + value.PushBack(element->value_.double_value_, allocator); + break; + case Type::Integer: + value.PushBack(element->value_.integer_value_, allocator); + break; + case Type::Null: + value.PushBack(rapidjson::Value(), allocator); + break; + case Type::String: + value.PushBack(rapidjson::StringRef(element->value_.string_value_.c_str()), allocator); + } + } + break; + } + case Type::Object: { + value.SetObject(); + for (const auto& item : field.value_.object_value_) { + auto name = rapidjson::StringRef(item.first.c_str()); + + switch (item.second->type_) { + case Type::Array: + case Type::Object: { + rapidjson::Value nested_value; + buildRapidJsonDocument(*item.second, nested_value, allocator); + value.AddMember(name, nested_value, allocator); + break; + } + case Type::Boolean: + value.AddMember(name, item.second->value_.boolean_value_, allocator); + break; + case Type::Double: + value.AddMember(name, item.second->value_.double_value_, allocator); + break; + case Type::Integer: + value.AddMember(name, item.second->value_.integer_value_, allocator); + break; + case Type::Null: + value.AddMember(name, rapidjson::Value(), allocator); + break; + case Type::String: + value.AddMember(name, rapidjson::StringRef(item.second->value_.string_value_.c_str()), + allocator); + break; + } + } + break; + } + case Type::Null: { + value.SetNull(); + break; + } + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +rapidjson::Document Field::asRapidJsonDocument() const { + rapidjson::Document document; + rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); + buildRapidJsonDocument(*this, document, allocator); + return document; +} + +uint64_t Field::hash() const { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + asRapidJsonDocument().Accept(writer); + return HashUtil::xxHash64(buffer.GetString()); +} + +bool Field::getBoolean(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Boolean)) { + throw Exception(fmt::format("key '{}' missing or not a boolean from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->booleanValue(); +} + +bool Field::getBoolean(const std::string& name, bool default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getBoolean(name); + } else { + return default_value; + } +} + +double Field::getDouble(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Double)) { + throw Exception(fmt::format("key '{}' missing or not a double from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->doubleValue(); +} + +double Field::getDouble(const std::string& name, double default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getDouble(name); + } else { + return default_value; + } +} + +int64_t Field::getInteger(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Integer)) { + throw Exception(fmt::format("key '{}' missing or not an integer from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->integerValue(); +} + +int64_t Field::getInteger(const std::string& name, int64_t default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getInteger(name); + } else { + return default_value; + } +} + +ObjectSharedPtr Field::getObject(const std::string& name, bool allow_empty) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end()) { + if (allow_empty) { + return createObject(); + } else { + throw Exception(fmt::format("key '{}' missing from lines {}-{}", name, line_number_start_, + line_number_end_)); + } + } else if (!value_itr->second->isType(Type::Object)) { + throw Exception(fmt::format("key '{}' not an object from line {}", name, + value_itr->second->line_number_start_)); + } else { + return value_itr->second; + } +} + +std::vector Field::getObjectArray(const std::string& name, + bool allow_empty) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Array)) { + if (allow_empty && value_itr == value_.object_value_.end()) { + return std::vector(); + } + throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + + std::vector array_value = value_itr->second->arrayValue(); + return {array_value.begin(), array_value.end()}; +} + +std::string Field::getString(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::String)) { + throw Exception(fmt::format("key '{}' missing or not a string from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->stringValue(); +} + +std::string Field::getString(const std::string& name, const std::string& default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getString(name); + } else { + return default_value; + } +} + +std::vector Field::getStringArray(const std::string& name, bool allow_empty) const { + checkType(Type::Object); + std::vector string_array; + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Array)) { + if (allow_empty && value_itr == value_.object_value_.end()) { + return string_array; + } + throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + + std::vector array = value_itr->second->arrayValue(); + string_array.reserve(array.size()); + for (const auto& element : array) { + if (!element->isType(Type::String)) { + throw Exception(fmt::format("JSON array '{}' from line {} does not contain all strings", name, + line_number_start_)); + } + string_array.push_back(element->stringValue()); + } + + return string_array; +} + +std::vector Field::asObjectArray() const { + checkType(Type::Array); + return {value_.array_value_.begin(), value_.array_value_.end()}; +} + +std::string Field::asJsonString() const { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + rapidjson::Document document = asRapidJsonDocument(); + document.Accept(writer); + return buffer.GetString(); +} + +bool Field::empty() const { + if (isType(Type::Object)) { + return value_.object_value_.empty(); + } else if (isType(Type::Array)) { + return value_.array_value_.empty(); + } else { + throw Exception( + fmt::format("Json does not support empty() on types other than array and object")); + } +} + +bool Field::hasObject(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + return value_itr != value_.object_value_.end(); +} + +void Field::iterate(const ObjectCallback& callback) const { + checkType(Type::Object); + for (const auto& item : value_.object_value_) { + bool stop_iteration = !callback(item.first, *item.second); + if (stop_iteration) { + break; + } + } +} + +void Field::validateSchema(const std::string& schema) const { + rapidjson::Document schema_document; + if (schema_document.Parse<0>(schema.c_str()).HasParseError()) { + throw std::invalid_argument(fmt::format( + "Schema supplied to validateSchema is not valid JSON\n Error(offset {}) : {}\n", + schema_document.GetErrorOffset(), GetParseError_En(schema_document.GetParseError()))); + } + + rapidjson::SchemaDocument schema_document_for_validator(schema_document); + rapidjson::SchemaValidator schema_validator(schema_document_for_validator); + + if (!asRapidJsonDocument().Accept(schema_validator)) { + rapidjson::StringBuffer schema_string_buffer; + rapidjson::StringBuffer document_string_buffer; + + schema_validator.GetInvalidSchemaPointer().StringifyUriFragment(schema_string_buffer); + schema_validator.GetInvalidDocumentPointer().StringifyUriFragment(document_string_buffer); + + throw Exception(fmt::format( + "JSON at lines {}-{} does not conform to schema.\n Invalid schema: {}\n" + " Schema violation: {}\n" + " Offending document key: {}", + line_number_start_, line_number_end_, schema_string_buffer.GetString(), + schema_validator.GetInvalidSchemaKeyword(), document_string_buffer.GetString())); + } +} + +bool ObjectHandler::StartObject() { + FieldSharedPtr object = Field::createObject(); + object->setLineNumberStart(stream_.getLineNumber()); + + switch (state_) { + case State::ExpectValueOrStartObjectArray: + stack_.top()->insert(key_, object); + stack_.push(object); + state_ = State::ExpectKeyOrEndObject; + return true; + case State::ExpectArrayValueOrEndArray: + stack_.top()->append(object); + stack_.push(object); + state_ = State::ExpectKeyOrEndObject; + return true; + case State::ExpectRoot: + root_ = object; + stack_.push(object); + state_ = State::ExpectKeyOrEndObject; + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::EndObject(rapidjson::SizeType) { + switch (state_) { + case State::ExpectKeyOrEndObject: + stack_.top()->setLineNumberEnd(stream_.getLineNumber()); + stack_.pop(); + + if (stack_.empty()) { + state_ = State::ExpectFinished; + } else if (stack_.top()->isObject()) { + state_ = State::ExpectKeyOrEndObject; + } else if (stack_.top()->isArray()) { + state_ = State::ExpectArrayValueOrEndArray; + } + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::Key(const char* value, rapidjson::SizeType size, bool) { + switch (state_) { + case State::ExpectKeyOrEndObject: + key_ = std::string(value, size); + state_ = State::ExpectValueOrStartObjectArray; + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::StartArray() { + FieldSharedPtr array = Field::createArray(); + array->setLineNumberStart(stream_.getLineNumber()); + + switch (state_) { + case State::ExpectValueOrStartObjectArray: + stack_.top()->insert(key_, array); + stack_.push(array); + state_ = State::ExpectArrayValueOrEndArray; + return true; + case State::ExpectArrayValueOrEndArray: + stack_.top()->append(array); + stack_.push(array); + return true; + case State::ExpectRoot: + root_ = array; + stack_.push(array); + state_ = State::ExpectArrayValueOrEndArray; + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::EndArray(rapidjson::SizeType) { + switch (state_) { + case State::ExpectArrayValueOrEndArray: + stack_.top()->setLineNumberEnd(stream_.getLineNumber()); + stack_.pop(); + + if (stack_.empty()) { + state_ = State::ExpectFinished; + } else if (stack_.top()->isObject()) { + state_ = State::ExpectKeyOrEndObject; + } else if (stack_.top()->isArray()) { + state_ = State::ExpectArrayValueOrEndArray; + } + + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +// Value handlers +bool ObjectHandler::Bool(bool value) { return handleValueEvent(Field::createValue(value)); } +bool ObjectHandler::Double(double value) { return handleValueEvent(Field::createValue(value)); } +bool ObjectHandler::Int(int value) { + return handleValueEvent(Field::createValue(static_cast(value))); +} +bool ObjectHandler::Uint(unsigned value) { + return handleValueEvent(Field::createValue(static_cast(value))); +} +bool ObjectHandler::Int64(int64_t value) { return handleValueEvent(Field::createValue(value)); } +bool ObjectHandler::Uint64(uint64_t value) { + if (value > static_cast(std::numeric_limits::max())) { + throw Exception(fmt::format("JSON value from line {} is larger than int64_t (not supported)", + stream_.getLineNumber())); + } + return handleValueEvent(Field::createValue(static_cast(value))); +} + +bool ObjectHandler::Null() { return handleValueEvent(Field::createNull()); } + +bool ObjectHandler::String(const char* value, rapidjson::SizeType size, bool) { + return handleValueEvent(Field::createValue(std::string(value, size))); +} + +bool ObjectHandler::RawNumber(const char*, rapidjson::SizeType, bool) { + // Only called if kParseNumbersAsStrings is set as a parse flag, which it is not. + NOT_REACHED_GCOVR_EXCL_LINE; +} + +bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { + ptr->setLineNumberStart(stream_.getLineNumber()); + + switch (state_) { + case State::ExpectValueOrStartObjectArray: + state_ = State::ExpectKeyOrEndObject; + stack_.top()->insert(key_, ptr); + return true; + case State::ExpectArrayValueOrEndArray: + stack_.top()->append(ptr); + return true; + default: + return false; + } +} + +} // namespace + +ObjectSharedPtr Factory::loadFromString(const std::string& json) { + LineCountingStringStream json_stream(json.c_str()); + + ObjectHandler handler(json_stream); + rapidjson::Reader reader; + reader.Parse(json_stream, handler); + + if (reader.HasParseError()) { + throw Exception(fmt::format("JSON supplied is not valid. Error(offset {}, line {}): {}\n", + reader.GetErrorOffset(), json_stream.getLineNumber(), + GetParseError_En(reader.GetParseErrorCode()))); + } + + return handler.getRoot(); +} + +} // namespace RapidJson +} // namespace Json +} // namespace Envoy diff --git a/source/common/json/rapidjson_loader.h b/source/common/json/rapidjson_loader.h new file mode 100644 index 0000000000000..6b50ccd2b6268 --- /dev/null +++ b/source/common/json/rapidjson_loader.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include "envoy/json/json_object.h" + +namespace Envoy { +namespace Json { +namespace RapidJson { + +class Factory { +public: + /** + * Constructs a Json Object from a string. + */ + static ObjectSharedPtr loadFromString(const std::string& json); +}; + +} // namespace RapidJson +} // namespace Json +} // namespace Envoy From d1be96066ab43900a3c911b66f8e0ee26dd3f95d Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Thu, 17 Dec 2020 10:51:52 -0500 Subject: [PATCH 02/17] fix tests Signed-off-by: Asra Ali --- source/common/json/json_loader.cc | 99 +++++- source/common/json/rapidjson_loader.cc | 1 + test/common/json/json_loader_test.cc | 93 +----- test/common/json/rapidjson_loader_test.cc | 385 ++++++++++++++++++++++ 4 files changed, 480 insertions(+), 98 deletions(-) create mode 100644 test/common/json/rapidjson_loader_test.cc diff --git a/source/common/json/json_loader.cc b/source/common/json/json_loader.cc index d87e8d20eb436..5639092dc8f4c 100644 --- a/source/common/json/json_loader.cc +++ b/source/common/json/json_loader.cc @@ -32,6 +32,9 @@ using FieldSharedPtr = std::shared_ptr; class Field : public Object { public: + void setLineNumberStart(uint64_t line_number) { line_number_start_ = line_number; } + void setLineNumberEnd(uint64_t line_number) { line_number_end_ = line_number; } + // Container factories for handler. static FieldSharedPtr createObject() { return FieldSharedPtr{new Field(Type::Object)}; } static FieldSharedPtr createArray() { return FieldSharedPtr{new Field(Type::Array)}; } @@ -130,9 +133,9 @@ class Field : public Object { bool isType(Type type) const { return type == type_; } void checkType(Type type) const { if (!isType(type)) { - throw Exception( - fmt::format("JSON field accessed with type '{}' does not match actual type '{}'.", - typeAsString(type), typeAsString(type_))); + throw Exception(fmt::format( + "JSON field from line {} accessed with type '{}' does not match actual type '{}'.", + line_number_start_, typeAsString(type), typeAsString(type_))); } } @@ -161,10 +164,13 @@ class Field : public Object { nlohmann::json asJsonDocument() const; static void buildJsonDocument(const Field& field, nlohmann::json& value); + uint64_t line_number_start_ = 0; + uint64_t line_number_end_ = 0; const Type type_; Value value_; }; + /** * Consume events from SAX callbacks to build JSON Field. */ @@ -182,6 +188,10 @@ class ObjectHandler : public nlohmann::json_sax { return handleValueEvent(Field::createValue(static_cast(value))); } bool number_unsigned(uint64_t value) override { + if (value > static_cast(std::numeric_limits::max())) { + throw Exception(fmt::format("JSON value from line {} is larger than int64_t (not supported)", + line_number_)); + } return handleValueEvent(Field::createValue(static_cast(value))); } bool number_float(double value, const std::string&) override { @@ -195,7 +205,7 @@ class ObjectHandler : public nlohmann::json_sax { error_offset_ = position; error_ = ex.what(); error_token_ = token; - return true; + return false; } bool hasParseError() { return !error_.empty(); } @@ -205,6 +215,8 @@ class ObjectHandler : public nlohmann::json_sax { ObjectSharedPtr getRoot() { return root_; } + int line_number_{1}; + private: bool handleValueEvent(FieldSharedPtr ptr); @@ -227,6 +239,46 @@ class ObjectHandler : public nlohmann::json_sax { std::size_t error_offset_; }; +struct JsonContainer { + JsonContainer(const char* ch, ObjectHandler* handler) : data(ch), handler_(handler) {} + const char* data; + ObjectHandler* handler_; +}; + +struct JsonIterator { + using difference_type = std::ptrdiff_t; + using value_type = char; + using pointer = const char*; + using reference = const char&; + using iterator_category = std::input_iterator_tag; + + JsonIterator& operator++() { + ++ptr.data; + return *this; + } + + bool operator!=(const JsonIterator& rhs) const { return rhs.ptr.data != ptr.data; } + + reference operator*() { + const char& ch = *(ptr.data); + if (ch == '\n') { + ptr.handler_->line_number_++; + } + return ch; + } + + JsonContainer ptr; +}; + +JsonIterator begin(const JsonContainer& c) { + return JsonIterator{JsonContainer(c.data, c.handler_)}; +} + +JsonIterator end(const JsonContainer& c) { + return JsonIterator{JsonContainer(c.data + strlen(c.data), c.handler_)}; +} + + void Field::buildJsonDocument(const Field& field, nlohmann::json& value) { switch (field.type_) { case Type::Array: { @@ -310,7 +362,8 @@ bool Field::getBoolean(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Boolean)) { - throw Exception(fmt::format("key '{}' missing or not a boolean", name)); + throw Exception(fmt::format("key '{}' missing or not a boolean from lines {}-{}", name, + line_number_start_, line_number_end_)); } return value_itr->second->booleanValue(); } @@ -329,7 +382,8 @@ double Field::getDouble(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Double)) { - throw Exception(fmt::format("key '{}' missing or not a double", name)); + throw Exception(fmt::format("key '{}' missing or not a double from lines {}-{}", name, + line_number_start_, line_number_end_)); } return value_itr->second->doubleValue(); } @@ -348,7 +402,8 @@ int64_t Field::getInteger(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Integer)) { - throw Exception(fmt::format("key '{}' missing or not an integer", name)); + throw Exception(fmt::format("key '{}' missing or not an integer from lines {}-{}", name, + line_number_start_, line_number_end_)); } return value_itr->second->integerValue(); } @@ -370,10 +425,12 @@ ObjectSharedPtr Field::getObject(const std::string& name, bool allow_empty) cons if (allow_empty) { return createObject(); } else { - throw Exception(fmt::format("key '{}' missing from lines", name)); + throw Exception(fmt::format("key '{}' missing from lines {}-{}", name, line_number_start_, + line_number_end_)); } } else if (!value_itr->second->isType(Type::Object)) { - throw Exception(fmt::format("key '{}' not an object", name)); + throw Exception(fmt::format("key '{}' not an object from line {}", name, + value_itr->second->line_number_start_)); } else { return value_itr->second; } @@ -387,7 +444,8 @@ std::vector Field::getObjectArray(const std::string& name, if (allow_empty && value_itr == value_.object_value_.end()) { return std::vector(); } - throw Exception(fmt::format("key '{}' missing or not an array", name)); + throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, + line_number_start_, line_number_end_)); } std::vector array_value = value_itr->second->arrayValue(); @@ -398,7 +456,8 @@ std::string Field::getString(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::String)) { - throw Exception(fmt::format("key '{}' missing or not a string", name)); + throw Exception(fmt::format("key '{}' missing or not a string from lines {}-{}", name, + line_number_start_, line_number_end_)); } return value_itr->second->stringValue(); } @@ -421,14 +480,16 @@ std::vector Field::getStringArray(const std::string& name, bool all if (allow_empty && value_itr == value_.object_value_.end()) { return string_array; } - throw Exception(fmt::format("key '{}' missing or not an array", name)); + throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, + line_number_start_, line_number_end_)); } std::vector array = value_itr->second->arrayValue(); string_array.reserve(array.size()); for (const auto& element : array) { if (!element->isType(Type::String)) { - throw Exception(fmt::format("JSON array '{}' does not contain all strings", name)); + throw Exception(fmt::format("JSON array '{}' from line {} does not contain all strings", name, + line_number_start_)); } string_array.push_back(element->stringValue()); } @@ -479,6 +540,7 @@ void Field::validateSchema(const std::string&) const { bool ObjectHandler::start_object(std::size_t) { FieldSharedPtr object = Field::createObject(); + object->setLineNumberStart(line_number_); switch (state_) { case State::ExpectValueOrStartObjectArray: @@ -504,6 +566,7 @@ bool ObjectHandler::start_object(std::size_t) { bool ObjectHandler::end_object() { switch (state_) { case State::ExpectKeyOrEndObject: + stack_.top()->setLineNumberEnd(line_number_); stack_.pop(); if (stack_.empty()) { @@ -532,6 +595,7 @@ bool ObjectHandler::key(std::string& val) { bool ObjectHandler::start_array(std::size_t) { FieldSharedPtr array = Field::createArray(); + array->setLineNumberStart(line_number_); switch (state_) { case State::ExpectValueOrStartObjectArray: @@ -556,6 +620,7 @@ bool ObjectHandler::start_array(std::size_t) { bool ObjectHandler::end_array() { switch (state_) { case State::ExpectArrayValueOrEndArray: + stack_.top()->setLineNumberEnd(line_number_); stack_.pop(); if (stack_.empty()) { @@ -573,6 +638,8 @@ bool ObjectHandler::end_array() { } bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { + ptr->setLineNumberStart(line_number_); + switch (state_) { case State::ExpectValueOrStartObjectArray: state_ = State::ExpectKeyOrEndObject; @@ -582,7 +649,7 @@ bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { stack_.top()->append(ptr); return true; default: - return false; + return true; } } @@ -590,15 +657,15 @@ bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { ObjectSharedPtr Factory::loadFromString(const std::string& json) { ObjectHandler handler; + auto json_container = JsonContainer(json.data(), &handler); - nlohmann::json::sax_parse(json, &handler); + nlohmann::json::sax_parse(json_container, &handler); if (handler.hasParseError()) { throw Exception(fmt::format("JSON supplied is not valid. Error(offset {}, token {}): {}\n", handler.getErrorOffset(), handler.getErrorToken(), handler.getParseError())); } - return handler.getRoot(); } diff --git a/source/common/json/rapidjson_loader.cc b/source/common/json/rapidjson_loader.cc index dfc74458616a5..bcb5e01264c66 100644 --- a/source/common/json/rapidjson_loader.cc +++ b/source/common/json/rapidjson_loader.cc @@ -677,6 +677,7 @@ bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { stack_.top()->append(ptr); return true; default: + ENVOY_LOG_MISC(info, "RETURNING FALSE"); return false; } } diff --git a/test/common/json/json_loader_test.cc b/test/common/json/json_loader_test.cc index 884caf5b0d0c1..4359fe6ffa260 100644 --- a/test/common/json/json_loader_test.cc +++ b/test/common/json/json_loader_test.cc @@ -73,8 +73,9 @@ TEST_F(JsonLoaderTest, Basic) { { EXPECT_THROW_WITH_MESSAGE( Factory::loadFromString("{\"hello\": \n\n\"world\""), Exception, - "JSON supplied is not valid. Error(offset 19, line 3): Missing a comma or " - "'}' after an object member.\n"); + "JSON supplied is not valid. Error(offset 20, token \"world\"): " + "[json.exception.parse_error.101] parse error at line 3, column 8: syntax error while " + "parsing object - unexpected end of input; expected '}'\n"); } { @@ -257,14 +258,7 @@ TEST_F(JsonLoaderTest, Hash) { } TEST_F(JsonLoaderTest, Schema) { - { - std::string invalid_json_schema = R"EOF( - { - "properties": {"value1"} - } - )EOF"; - - std::string invalid_schema = R"EOF( + std::string invalid_schema = R"EOF( { "properties" : { "value1": {"type" : "faketype"} @@ -272,82 +266,15 @@ TEST_F(JsonLoaderTest, Schema) { } )EOF"; - std::string different_schema = R"EOF( - { - "properties" : { - "value1" : {"type" : "number"} - }, - "additionalProperties" : false - } - )EOF"; - - std::string valid_schema = R"EOF( - { - "properties": { - "value1": {"type" : "number"}, - "value2": {"type": "string"} - }, - "additionalProperties": false - } - )EOF"; - - std::string json_string = R"EOF( + std::string json_string = R"EOF( { "value1": 10, "value2" : "test" } )EOF"; - ObjectSharedPtr json = Factory::loadFromString(json_string); - EXPECT_THROW(json->validateSchema(invalid_json_schema), std::invalid_argument); - EXPECT_THROW(json->validateSchema(invalid_schema), Exception); - EXPECT_THROW(json->validateSchema(different_schema), Exception); - EXPECT_NO_THROW(json->validateSchema(valid_schema)); - } - - { - std::string json_string = R"EOF( - { - "value1": [false, 2.01, 3, null], - "value2" : "test" - } - )EOF"; - - std::string empty_schema = R"EOF({})EOF"; - - ObjectSharedPtr json = Factory::loadFromString(json_string); - EXPECT_NO_THROW(json->validateSchema(empty_schema)); - } -} - -TEST_F(JsonLoaderTest, NestedSchema) { - - std::string schema = R"EOF( - { - "properties": { - "value1": {"type" : "number"}, - "value2": {"type": "string"} - }, - "additionalProperties": false - } - )EOF"; - - std::string json_string = R"EOF( - { - "bar": "baz", - "foo": { - "value1": "should have been a number", - "value2" : "test" - } - } - )EOF"; - ObjectSharedPtr json = Factory::loadFromString(json_string); - - EXPECT_THROW_WITH_MESSAGE(json->getObject("foo")->validateSchema(schema), Exception, - "JSON at lines 4-7 does not conform to schema.\n Invalid schema: " - "#/properties/value1\n Schema violation: type\n Offending document " - "key: #/value1"); + EXPECT_THROW_WITH_MESSAGE(json->validateSchema(invalid_schema), Exception, "not implemented"); } TEST_F(JsonLoaderTest, MissingEnclosingDocument) { @@ -361,9 +288,11 @@ TEST_F(JsonLoaderTest, MissingEnclosingDocument) { ] )EOF"; - EXPECT_THROW_WITH_MESSAGE(Factory::loadFromString(json_string), Exception, - "JSON supplied is not valid. Error(offset 14, line 2): Terminate " - "parsing due to Handler error.\n"); + EXPECT_THROW_WITH_MESSAGE( + Factory::loadFromString(json_string), Exception, + "JSON supplied is not valid. Error(offset 16, token \"listeners\" :): " + "[json.exception.parse_error.101] parse error at line 2, column 15: syntax error while " + "parsing value - unexpected ':'; expected end of input\n"); } TEST_F(JsonLoaderTest, AsString) { diff --git a/test/common/json/rapidjson_loader_test.cc b/test/common/json/rapidjson_loader_test.cc new file mode 100644 index 0000000000000..884caf5b0d0c1 --- /dev/null +++ b/test/common/json/rapidjson_loader_test.cc @@ -0,0 +1,385 @@ +#include +#include + +#include "common/json/json_loader.h" +#include "common/stats/isolated_store_impl.h" + +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Json { +namespace { + +class JsonLoaderTest : public testing::Test { +protected: + JsonLoaderTest() : api_(Api::createApiForTest()) {} + + Api::ApiPtr api_; +}; + +TEST_F(JsonLoaderTest, Basic) { + EXPECT_THROW(Factory::loadFromString("{"), Exception); + + { + ObjectSharedPtr json = Factory::loadFromString("{\"hello\":123}"); + EXPECT_TRUE(json->hasObject("hello")); + EXPECT_FALSE(json->hasObject("world")); + EXPECT_FALSE(json->empty()); + EXPECT_THROW(json->getObject("world"), Exception); + EXPECT_THROW(json->getObject("hello"), Exception); + EXPECT_THROW(json->getBoolean("hello"), Exception); + EXPECT_THROW(json->getObjectArray("hello"), Exception); + EXPECT_THROW(json->getString("hello"), Exception); + + EXPECT_THROW_WITH_MESSAGE(json->getString("hello"), Exception, + "key 'hello' missing or not a string from lines 1-1"); + } + + { + ObjectSharedPtr json = Factory::loadFromString("{\"hello\":\"123\"\n}"); + EXPECT_THROW_WITH_MESSAGE(json->getInteger("hello"), Exception, + "key 'hello' missing or not an integer from lines 1-2"); + } + + { + ObjectSharedPtr json = Factory::loadFromString("{\"hello\":true}"); + EXPECT_TRUE(json->getBoolean("hello")); + EXPECT_TRUE(json->getBoolean("hello", false)); + EXPECT_FALSE(json->getBoolean("world", false)); + } + + { + ObjectSharedPtr json = Factory::loadFromString("{\"hello\": [\"a\", \"b\", 3]}"); + EXPECT_THROW(json->getStringArray("hello"), Exception); + EXPECT_THROW(json->getStringArray("world"), Exception); + } + + { + ObjectSharedPtr json = Factory::loadFromString("{\"hello\":123}"); + EXPECT_EQ(123, json->getInteger("hello", 456)); + EXPECT_EQ(456, json->getInteger("world", 456)); + } + + { + ObjectSharedPtr json = Factory::loadFromString("{\"hello\": \n[123]}"); + + EXPECT_THROW_WITH_MESSAGE( + json->getObjectArray("hello").at(0)->getString("hello"), Exception, + "JSON field from line 2 accessed with type 'Object' does not match actual type 'Integer'."); + } + + { + EXPECT_THROW_WITH_MESSAGE( + Factory::loadFromString("{\"hello\": \n\n\"world\""), Exception, + "JSON supplied is not valid. Error(offset 19, line 3): Missing a comma or " + "'}' after an object member.\n"); + } + + { + ObjectSharedPtr json_object = Factory::loadFromString("[\"foo\",\"bar\"]"); + EXPECT_FALSE(json_object->empty()); + } + + { + ObjectSharedPtr json_object = Factory::loadFromString("[]"); + EXPECT_TRUE(json_object->empty()); + } + + { + ObjectSharedPtr json = + Factory::loadFromString("{\"1\":{\"11\":\"111\"},\"2\":{\"22\":\"222\"}}"); + int pos = 0; + json->iterate([&pos](const std::string& key, const Json::Object& value) { + EXPECT_TRUE(key == "1" || key == "2"); + + if (key == "1") { + EXPECT_EQ("111", value.getString("11")); + } else { + EXPECT_EQ("222", value.getString("22")); + } + + pos++; + return true; + }); + + EXPECT_EQ(2, pos); + } + + { + ObjectSharedPtr json = + Factory::loadFromString("{\"1\":{\"11\":\"111\"},\"2\":{\"22\":\"222\"}}"); + int pos = 0; + json->iterate([&pos](const std::string& key, const Json::Object& value) { + EXPECT_TRUE(key == "1" || key == "2"); + + if (key == "1") { + EXPECT_EQ("111", value.getString("11")); + } else { + EXPECT_EQ("222", value.getString("22")); + } + + pos++; + return false; + }); + + EXPECT_EQ(1, pos); + } + + { + std::string json = R"EOF( + { + "descriptors": [ + [{"key": "hello", "value": "world"}, {"key": "foo", "value": "bar"}], + [{"key": "foo2", "value": "bar2"}] + ] + } + )EOF"; + + ObjectSharedPtr config = Factory::loadFromString(json); + EXPECT_EQ(2U, config->getObjectArray("descriptors")[0]->asObjectArray().size()); + EXPECT_EQ(1U, config->getObjectArray("descriptors")[1]->asObjectArray().size()); + } + + { + std::string json = R"EOF( + { + "descriptors": ["hello", "world"] + } + )EOF"; + + ObjectSharedPtr config = Factory::loadFromString(json); + std::vector array = config->getObjectArray("descriptors"); + EXPECT_THROW(array[0]->asObjectArray(), Exception); + } + + { + std::string json = R"EOF({})EOF"; + ObjectSharedPtr config = Factory::loadFromString(json); + ObjectSharedPtr object = config->getObject("foo", true); + EXPECT_EQ(2, object->getInteger("bar", 2)); + EXPECT_TRUE(object->empty()); + } + + { + std::string json = R"EOF({"foo": []})EOF"; + ObjectSharedPtr config = Factory::loadFromString(json); + EXPECT_TRUE(config->getStringArray("foo").empty()); + } + + { + std::string json = R"EOF({"foo": ["bar", "baz"]})EOF"; + ObjectSharedPtr config = Factory::loadFromString(json); + EXPECT_FALSE(config->getStringArray("foo").empty()); + } + + { + std::string json = R"EOF({})EOF"; + ObjectSharedPtr config = Factory::loadFromString(json); + EXPECT_THROW(config->getStringArray("foo"), EnvoyException); + } + + { + std::string json = R"EOF({})EOF"; + ObjectSharedPtr config = Factory::loadFromString(json); + EXPECT_TRUE(config->getStringArray("foo", true).empty()); + } + + { + ObjectSharedPtr json = Factory::loadFromString("{\"hello\": \n[2.0]}"); + EXPECT_THROW(json->getObjectArray("hello").at(0)->getDouble("foo"), Exception); + } + + { + ObjectSharedPtr json = Factory::loadFromString("{\"hello\": \n[null]}"); + EXPECT_THROW(json->getObjectArray("hello").at(0)->getDouble("foo"), Exception); + } + + { + ObjectSharedPtr json = Factory::loadFromString("{}"); + EXPECT_THROW((void)json->getObjectArray("hello").empty(), Exception); + } + + { + ObjectSharedPtr json = Factory::loadFromString("{}"); + EXPECT_TRUE(json->getObjectArray("hello", true).empty()); + } +} + +TEST_F(JsonLoaderTest, Integer) { + { + ObjectSharedPtr json = + Factory::loadFromString("{\"max\":9223372036854775807, \"min\":-9223372036854775808}"); + EXPECT_EQ(std::numeric_limits::max(), json->getInteger("max")); + EXPECT_EQ(std::numeric_limits::min(), json->getInteger("min")); + } + { + EXPECT_THROW(Factory::loadFromString("{\"val\":9223372036854775808}"), EnvoyException); + + // I believe this is a bug with rapidjson. + // It silently eats numbers below min int64_t with no exception. + // Fail when reading key instead of on parse. + ObjectSharedPtr json = Factory::loadFromString("{\"val\":-9223372036854775809}"); + EXPECT_THROW(json->getInteger("val"), EnvoyException); + } +} + +TEST_F(JsonLoaderTest, Double) { + { + ObjectSharedPtr json = Factory::loadFromString("{\"value1\": 10.5, \"value2\": -12.3}"); + EXPECT_EQ(10.5, json->getDouble("value1")); + EXPECT_EQ(-12.3, json->getDouble("value2")); + } + { + ObjectSharedPtr json = Factory::loadFromString("{\"foo\": 13.22}"); + EXPECT_EQ(13.22, json->getDouble("foo", 0)); + EXPECT_EQ(0, json->getDouble("bar", 0)); + } + { + ObjectSharedPtr json = Factory::loadFromString("{\"foo\": \"bar\"}"); + EXPECT_THROW(json->getDouble("foo"), Exception); + } +} + +TEST_F(JsonLoaderTest, Hash) { + ObjectSharedPtr json1 = Factory::loadFromString("{\"value1\": 10.5, \"value2\": -12.3}"); + ObjectSharedPtr json2 = Factory::loadFromString("{\"value2\": -12.3, \"value1\": 10.5}"); + ObjectSharedPtr json3 = Factory::loadFromString(" { \"value2\": -12.3, \"value1\": 10.5} "); + ObjectSharedPtr json4 = Factory::loadFromString("{\"value1\": 10.5}"); + + // Objects with keys in different orders should be the same + EXPECT_EQ(json1->hash(), json2->hash()); + // Whitespace is ignored + EXPECT_EQ(json2->hash(), json3->hash()); + // Ensure different hash is computed for different objects + EXPECT_NE(json1->hash(), json4->hash()); +} + +TEST_F(JsonLoaderTest, Schema) { + { + std::string invalid_json_schema = R"EOF( + { + "properties": {"value1"} + } + )EOF"; + + std::string invalid_schema = R"EOF( + { + "properties" : { + "value1": {"type" : "faketype"} + } + } + )EOF"; + + std::string different_schema = R"EOF( + { + "properties" : { + "value1" : {"type" : "number"} + }, + "additionalProperties" : false + } + )EOF"; + + std::string valid_schema = R"EOF( + { + "properties": { + "value1": {"type" : "number"}, + "value2": {"type": "string"} + }, + "additionalProperties": false + } + )EOF"; + + std::string json_string = R"EOF( + { + "value1": 10, + "value2" : "test" + } + )EOF"; + + ObjectSharedPtr json = Factory::loadFromString(json_string); + EXPECT_THROW(json->validateSchema(invalid_json_schema), std::invalid_argument); + EXPECT_THROW(json->validateSchema(invalid_schema), Exception); + EXPECT_THROW(json->validateSchema(different_schema), Exception); + EXPECT_NO_THROW(json->validateSchema(valid_schema)); + } + + { + std::string json_string = R"EOF( + { + "value1": [false, 2.01, 3, null], + "value2" : "test" + } + )EOF"; + + std::string empty_schema = R"EOF({})EOF"; + + ObjectSharedPtr json = Factory::loadFromString(json_string); + EXPECT_NO_THROW(json->validateSchema(empty_schema)); + } +} + +TEST_F(JsonLoaderTest, NestedSchema) { + + std::string schema = R"EOF( + { + "properties": { + "value1": {"type" : "number"}, + "value2": {"type": "string"} + }, + "additionalProperties": false + } + )EOF"; + + std::string json_string = R"EOF( + { + "bar": "baz", + "foo": { + "value1": "should have been a number", + "value2" : "test" + } + } + )EOF"; + + ObjectSharedPtr json = Factory::loadFromString(json_string); + + EXPECT_THROW_WITH_MESSAGE(json->getObject("foo")->validateSchema(schema), Exception, + "JSON at lines 4-7 does not conform to schema.\n Invalid schema: " + "#/properties/value1\n Schema violation: type\n Offending document " + "key: #/value1"); +} + +TEST_F(JsonLoaderTest, MissingEnclosingDocument) { + + std::string json_string = R"EOF( + "listeners" : [ + { + "address": "tcp://127.0.0.1:1234", + "filters": [] + } + ] + )EOF"; + + EXPECT_THROW_WITH_MESSAGE(Factory::loadFromString(json_string), Exception, + "JSON supplied is not valid. Error(offset 14, line 2): Terminate " + "parsing due to Handler error.\n"); +} + +TEST_F(JsonLoaderTest, AsString) { + ObjectSharedPtr json = Factory::loadFromString("{\"name1\": \"value1\", \"name2\": true}"); + json->iterate([&](const std::string& key, const Json::Object& value) { + EXPECT_TRUE(key == "name1" || key == "name2"); + + if (key == "name1") { + EXPECT_EQ("value1", value.asString()); + } else { + EXPECT_THROW(value.asString(), Exception); + } + return true; + }); +} + +} // namespace +} // namespace Json +} // namespace Envoy From e94571171af1046753b09b75c59381344643a609 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Thu, 17 Dec 2020 11:29:22 -0500 Subject: [PATCH 03/17] fix rapidjson Signed-off-by: Asra Ali --- source/common/json/rapidjson_loader.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/json/rapidjson_loader.cc b/source/common/json/rapidjson_loader.cc index bcb5e01264c66..41f770044e19f 100644 --- a/source/common/json/rapidjson_loader.cc +++ b/source/common/json/rapidjson_loader.cc @@ -1,4 +1,4 @@ -#include "common/json/json_loader.h" +#include "common/json/rapidjson_loader.h" #include #include From e4c25881ee8d97a5fc071fa063f396fa89269603 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Tue, 5 Jan 2021 10:17:34 -0500 Subject: [PATCH 04/17] wip Signed-off-by: Asra Ali --- source/common/json/json_loader.cc | 56 ++++++++++----------- source/common/json/rapidjson_loader.cc | 1 - test/common/json/BUILD | 20 ++++++++ test/common/json/json_loader_speed_test.cc | 52 +++++++++++++++++++ test/common/json/json_loader_test.cc | 8 ++- test/common/router/header_formatter_test.cc | 37 +++++++------- 6 files changed, 122 insertions(+), 52 deletions(-) create mode 100644 test/common/json/json_loader_speed_test.cc diff --git a/source/common/json/json_loader.cc b/source/common/json/json_loader.cc index 5639092dc8f4c..fa2410a8eb9d0 100644 --- a/source/common/json/json_loader.cc +++ b/source/common/json/json_loader.cc @@ -170,7 +170,6 @@ class Field : public Object { Value value_; }; - /** * Consume events from SAX callbacks to build JSON Field. */ @@ -200,18 +199,24 @@ class ObjectHandler : public nlohmann::json_sax { bool null() override { return handleValueEvent(Field::createNull()); } bool string(std::string& value) override { return handleValueEvent(Field::createValue(value)); } bool binary(binary_t&) override { return false; } - bool parse_error(std::size_t position, const std::string& token, + bool parse_error(std::size_t, const std::string& token, const nlohmann::detail::exception& ex) override { - error_offset_ = position; - error_ = ex.what(); - error_token_ = token; + // Errors are formatted like "[json.exception.parse_error.101] parse error at line x, column y: + // error string." + absl::string_view error = ex.what(); + // Extract position information (line, column, token) in the error. + auto start = error.find("line"); + auto npos = error.find(":"); + ENVOY_LOG_MISC(info, "ERROR {}", error); + error_position_ = absl::StrCat(error.substr(start, npos - start), ", token ", token); + // Extract portion after ":" to get error string. + error_ = error.substr(npos + 2); return false; } bool hasParseError() { return !error_.empty(); } - std::size_t getErrorOffset() { return error_offset_; } - std::string getParseError() { return error_;} - std::string getErrorToken() { return error_token_; } + std::string getParseError() { return error_; } + std::string getErrorPosition() { return error_position_; } ObjectSharedPtr getRoot() { return root_; } @@ -235,8 +240,7 @@ class ObjectHandler : public nlohmann::json_sax { FieldSharedPtr root_; std::string error_; - std::string error_token_; - std::size_t error_offset_; + std::string error_position_; }; struct JsonContainer { @@ -278,7 +282,6 @@ JsonIterator end(const JsonContainer& c) { return JsonIterator{JsonContainer(c.data + strlen(c.data), c.handler_)}; } - void Field::buildJsonDocument(const Field& field, nlohmann::json& value) { switch (field.type_) { case Type::Array: { @@ -354,15 +357,13 @@ nlohmann::json Field::asJsonDocument() const { return j; } -uint64_t Field::hash() const { - return HashUtil::xxHash64(asJsonString()); -} +uint64_t Field::hash() const { return HashUtil::xxHash64(asJsonString()); } bool Field::getBoolean(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Boolean)) { - throw Exception(fmt::format("key '{}' missing or not a boolean from lines {}-{}", name, + throw Exception(fmt::format("key '{}' missing or not a boolean from lines {}-{}", name, line_number_start_, line_number_end_)); } return value_itr->second->booleanValue(); @@ -382,7 +383,7 @@ double Field::getDouble(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Double)) { - throw Exception(fmt::format("key '{}' missing or not a double from lines {}-{}", name, + throw Exception(fmt::format("key '{}' missing or not a double from lines {}-{}", name, line_number_start_, line_number_end_)); } return value_itr->second->doubleValue(); @@ -402,7 +403,7 @@ int64_t Field::getInteger(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Integer)) { - throw Exception(fmt::format("key '{}' missing or not an integer from lines {}-{}", name, + throw Exception(fmt::format("key '{}' missing or not an integer from lines {}-{}", name, line_number_start_, line_number_end_)); } return value_itr->second->integerValue(); @@ -425,11 +426,11 @@ ObjectSharedPtr Field::getObject(const std::string& name, bool allow_empty) cons if (allow_empty) { return createObject(); } else { - throw Exception(fmt::format("key '{}' missing from lines {}-{}", name, line_number_start_, + throw Exception(fmt::format("key '{}' missing from lines {}-{}", name, line_number_start_, line_number_end_)); } } else if (!value_itr->second->isType(Type::Object)) { - throw Exception(fmt::format("key '{}' not an object from line {}", name, + throw Exception(fmt::format("key '{}' not an object from line {}", name, value_itr->second->line_number_start_)); } else { return value_itr->second; @@ -444,7 +445,7 @@ std::vector Field::getObjectArray(const std::string& name, if (allow_empty && value_itr == value_.object_value_.end()) { return std::vector(); } - throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, + throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, line_number_start_, line_number_end_)); } @@ -456,7 +457,7 @@ std::string Field::getString(const std::string& name) const { checkType(Type::Object); auto value_itr = value_.object_value_.find(name); if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::String)) { - throw Exception(fmt::format("key '{}' missing or not a string from lines {}-{}", name, + throw Exception(fmt::format("key '{}' missing or not a string from lines {}-{}", name, line_number_start_, line_number_end_)); } return value_itr->second->stringValue(); @@ -480,7 +481,7 @@ std::vector Field::getStringArray(const std::string& name, bool all if (allow_empty && value_itr == value_.object_value_.end()) { return string_array; } - throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, + throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, line_number_start_, line_number_end_)); } @@ -488,7 +489,7 @@ std::vector Field::getStringArray(const std::string& name, bool all string_array.reserve(array.size()); for (const auto& element : array) { if (!element->isType(Type::String)) { - throw Exception(fmt::format("JSON array '{}' from line {} does not contain all strings", name, + throw Exception(fmt::format("JSON array '{}' from line {} does not contain all strings", name, line_number_start_)); } string_array.push_back(element->stringValue()); @@ -534,9 +535,7 @@ void Field::iterate(const ObjectCallback& callback) const { } } -void Field::validateSchema(const std::string&) const { - throw Exception("not implemented"); -} +void Field::validateSchema(const std::string&) const { throw Exception("not implemented"); } bool ObjectHandler::start_object(std::size_t) { FieldSharedPtr object = Field::createObject(); @@ -662,9 +661,8 @@ ObjectSharedPtr Factory::loadFromString(const std::string& json) { nlohmann::json::sax_parse(json_container, &handler); if (handler.hasParseError()) { - throw Exception(fmt::format("JSON supplied is not valid. Error(offset {}, token {}): {}\n", - handler.getErrorOffset(), handler.getErrorToken(), - handler.getParseError())); + throw Exception(fmt::format("JSON supplied is not valid. Error({}): {}\n", + handler.getErrorPosition(), handler.getParseError())); } return handler.getRoot(); } diff --git a/source/common/json/rapidjson_loader.cc b/source/common/json/rapidjson_loader.cc index 41f770044e19f..04f8b63a15cfe 100644 --- a/source/common/json/rapidjson_loader.cc +++ b/source/common/json/rapidjson_loader.cc @@ -677,7 +677,6 @@ bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { stack_.top()->append(ptr); return true; default: - ENVOY_LOG_MISC(info, "RETURNING FALSE"); return false; } } diff --git a/test/common/json/BUILD b/test/common/json/BUILD index 803f2abca6af4..b93cba2b7abe3 100644 --- a/test/common/json/BUILD +++ b/test/common/json/BUILD @@ -1,5 +1,7 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_benchmark_test", + "envoy_cc_benchmark_binary", "envoy_cc_fuzz_test", "envoy_cc_test", "envoy_package", @@ -30,3 +32,21 @@ envoy_cc_test( "//test/test_common:utility_lib", ], ) + +envoy_cc_benchmark_binary( + name = "json_loader_speed_test", + srcs = ["json_loader_speed_test.cc"], + external_deps = [ + "benchmark", + "json", + ], + deps = [ + "//source/common/json:json_loader_lib", + "//source/common/json:rapidjson_loader_lib", + ], +) + +envoy_benchmark_test( + name = "json_loader_speed_test_benchmark_test", + benchmark_binary = "json_loader_speed_test", +) diff --git a/test/common/json/json_loader_speed_test.cc b/test/common/json/json_loader_speed_test.cc new file mode 100644 index 0000000000000..cad119e0b5e59 --- /dev/null +++ b/test/common/json/json_loader_speed_test.cc @@ -0,0 +1,52 @@ +#include + +#include "common/json/json_loader.h" +#include "common/json/rapidjson_loader.h" + +#include "benchmark/benchmark.h" +#include "include/nlohmann/json.hpp" + +namespace Envoy { + +// Creates a JSON string with a variable number of elements and array sizes. +std::string createJsonObject(const int elements) { + nlohmann::json j = nlohmann::json::object(); + for (int i = 0; i < elements; i++) { + j.emplace(std::to_string(i), i); + } + return j.dump(); +} + +std::string createJsonArray(const int elements) { + nlohmann::json j = nlohmann::json::array(); + for (int i = 0; i < elements; i++) { + j.push_back(std::to_string(i)); + } + return j.dump(); +} + +template void BM_JsonLoadObjects(benchmark::State& state) { + std::string json = createJsonObject(state.range(0)); + for (auto _ : state) { + Parser::loadFromString(json); + } + benchmark::DoNotOptimize(json); +} +BENCHMARK_TEMPLATE(BM_JsonLoadObjects, Json::Factory)->Arg(1)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK_TEMPLATE(BM_JsonLoadObjects, Json::RapidJson::Factory) + ->Arg(1) + ->Arg(10) + ->Arg(50) + ->Arg(100); + +template void BM_JsonLoadArray(benchmark::State& state) { + std::string json = createJsonArray(state.range(0)); + for (auto _ : state) { + Parser::loadFromString(json); + } + benchmark::DoNotOptimize(json); +} +BENCHMARK_TEMPLATE(BM_JsonLoadArray, Json::Factory)->Arg(1)->Arg(10)->Arg(50)->Arg(100); +BENCHMARK_TEMPLATE(BM_JsonLoadArray, Json::RapidJson::Factory)->Arg(1)->Arg(10)->Arg(50)->Arg(100); + +} // namespace Envoy diff --git a/test/common/json/json_loader_test.cc b/test/common/json/json_loader_test.cc index 4359fe6ffa260..fbab4467f80f9 100644 --- a/test/common/json/json_loader_test.cc +++ b/test/common/json/json_loader_test.cc @@ -73,8 +73,7 @@ TEST_F(JsonLoaderTest, Basic) { { EXPECT_THROW_WITH_MESSAGE( Factory::loadFromString("{\"hello\": \n\n\"world\""), Exception, - "JSON supplied is not valid. Error(offset 20, token \"world\"): " - "[json.exception.parse_error.101] parse error at line 3, column 8: syntax error while " + "JSON supplied is not valid. Error(line 3, column 8, token \"world\"): syntax error while " "parsing object - unexpected end of input; expected '}'\n"); } @@ -290,9 +289,8 @@ TEST_F(JsonLoaderTest, MissingEnclosingDocument) { EXPECT_THROW_WITH_MESSAGE( Factory::loadFromString(json_string), Exception, - "JSON supplied is not valid. Error(offset 16, token \"listeners\" :): " - "[json.exception.parse_error.101] parse error at line 2, column 15: syntax error while " - "parsing value - unexpected ':'; expected end of input\n"); + "JSON supplied is not valid. Error(line 2, column 15, token \"listeners\" :): syntax error " + "while parsing value - unexpected ':'; expected end of input\n"); } TEST_F(JsonLoaderTest, AsString) { diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index f045b12d4fda7..4d9c082eb0790 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -690,12 +690,12 @@ TEST_F(StreamInfoHeaderFormatterTest, UnknownVariable) { testInvalidFormat("INVA TEST_F(StreamInfoHeaderFormatterTest, WrongFormatOnUpstreamMetadataVariable) { // Invalid JSON. - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA(abcd)", false), - EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA(abcd), because JSON supplied is not valid. " - "Error(offset 0, line 1): Invalid value.\n"); + EXPECT_THROW_WITH_MESSAGE( + StreamInfoHeaderFormatter("UPSTREAM_METADATA(abcd)", false), EnvoyException, + "Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " + "...]), actual format UPSTREAM_METADATA(abcd), because JSON supplied is not valid. " + "Error(line 1, column 1, token a): syntax error while parsing value - invalid literal; last " + "read: 'a'\n"); // No parameters. EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA", false), EnvoyException, @@ -703,11 +703,12 @@ TEST_F(StreamInfoHeaderFormatterTest, WrongFormatOnUpstreamMetadataVariable) { "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " "UPSTREAM_METADATA"); - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA()", false), EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA(), because JSON supplied is not valid. " - "Error(offset 0, line 1): The document is empty.\n"); + EXPECT_THROW_WITH_MESSAGE( + StreamInfoHeaderFormatter("UPSTREAM_METADATA()", false), EnvoyException, + "Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " + "...]), actual format UPSTREAM_METADATA(), because JSON supplied is not valid. Error(line 1, " + "column 1, token ): syntax error while parsing value - unexpected end of input; expected " + "'[', '{', or a literal\n"); // One parameter. EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA([\"ns\"])", false), @@ -745,10 +746,10 @@ TEST_F(StreamInfoHeaderFormatterTest, WrongFormatOnUpstreamMetadataVariable) { // Invalid string elements. EXPECT_THROW_WITH_MESSAGE( StreamInfoHeaderFormatter("UPSTREAM_METADATA([\"a\", \"\\unothex\"])", false), EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA([\"a\", \"\\unothex\"]), because JSON supplied is not valid. " - "Error(offset 7, line 1): Incorrect hex digit after \\u escape in string.\n"); + "Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " + "...]), actual format UPSTREAM_METADATA([\"a\", \"\\unothex\"]), because JSON supplied is " + "not valid. Error(line 1, column 10, token \"\\un): syntax error while parsing value - " + "invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\un'\n"); // Non-array parameters. EXPECT_THROW_WITH_MESSAGE( @@ -829,12 +830,14 @@ TEST(HeaderParserTest, TestParseInternal) { {}, {"Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " "...]), actual format UPSTREAM_METADATA(no array), because JSON supplied is not valid. " - "Error(offset 1, line 1): Invalid value.\n"}}, + "Error(line 1, column 2, token no): syntax error while parsing value - invalid literal; " + "last read: 'no'\n"}}, {"%UPSTREAM_METADATA( no array)%", {}, {"Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " "...]), actual format UPSTREAM_METADATA( no array), because JSON supplied is not valid. " - "Error(offset 2, line 1): Invalid value.\n"}}, + "Error(line 1, column 3, token no): syntax error while parsing value - invalid literal; " + "last read: ' no'\n"}}, {"%UPSTREAM_METADATA([\"unterminated array\")%", {}, {"Invalid header configuration. Expecting ',', ']', or whitespace after " From e84f7f668735fc41998bc56f7ff7ce14ebc5790f Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Thu, 7 Jan 2021 13:08:55 -0500 Subject: [PATCH 05/17] fix format Signed-off-by: Asra Ali --- docs/root/version_history/current.rst | 1 + source/common/json/BUILD | 24 +- source/common/json/json_internal.cc | 673 ++++++++++++++++++ source/common/json/json_internal.h | 22 + ...json_loader.cc => json_internal_legacy.cc} | 2 +- source/common/json/json_internal_legacy.h | 22 + source/common/json/json_loader.cc | 665 +---------------- source/common/runtime/runtime_features.cc | 1 + test/common/json/BUILD | 19 +- test/common/json/json_loader_test.cc | 38 +- 10 files changed, 786 insertions(+), 681 deletions(-) create mode 100644 source/common/json/json_internal.cc create mode 100644 source/common/json/json_internal.h rename source/common/json/{rapidjson_loader.cc => json_internal_legacy.cc} (99%) create mode 100644 source/common/json/json_internal_legacy.h diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 53537d8eed950..a5d16fca66327 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -78,6 +78,7 @@ New Features * http: added frame flood and abuse checks to the upstream HTTP/2 codec. This check is off by default and can be enabled by setting the `envoy.reloadable_features.upstream_http2_flood_checks` runtime key to true. * http: added :ref:`stripping any port from host header ` support. * http: clusters now support selecting HTTP/1 or HTTP/2 based on ALPN, configurable via :ref:`alpn_config ` in the :ref:`http_protocol_options ` message. +* json: introduced new JSON parser (https://github.com/nlohmann/json) to replace RapidJSON. The new parser is enabled by default. To revert to the legacy RapidJSON parser, enable the runtime feature `envoy.reloadable_features.legacy_json`. * jwt_authn: added support for :ref:`per-route config `. * jwt_authn: changed config field :ref:`issuer ` to be optional to comply with JWT `RFC `_ requirements. * kill_request: added new :ref:`HTTP kill request filter `. diff --git a/source/common/json/BUILD b/source/common/json/BUILD index 7c2b342502971..4cc634bc0d697 100644 --- a/source/common/json/BUILD +++ b/source/common/json/BUILD @@ -9,9 +9,9 @@ licenses(["notice"]) # Apache 2 envoy_package() envoy_cc_library( - name = "rapidjson_loader_lib", - srcs = ["rapidjson_loader.cc"], - hdrs = ["rapidjson_loader.h"], + name = "json_internal_legacy_lib", + srcs = ["json_internal_legacy.cc"], + hdrs = ["json_internal_legacy.h"], external_deps = [ "rapidjson", ], @@ -25,9 +25,9 @@ envoy_cc_library( ) envoy_cc_library( - name = "json_loader_lib", - srcs = ["json_loader.cc"], - hdrs = ["json_loader.h"], + name = "json_internal_lib", + srcs = ["json_internal.cc"], + hdrs = ["json_internal.h"], external_deps = [ "json", ], @@ -39,3 +39,15 @@ envoy_cc_library( "//source/common/protobuf:utility_lib", ], ) + +envoy_cc_library( + name = "json_loader_lib", + srcs = ["json_loader.cc"], + hdrs = ["json_loader.h"], + deps = [ + ":json_internal_legacy_lib", + ":json_internal_lib", + "//include/envoy/json:json_object_interface", + "//source/common/runtime:runtime_features_lib", + ], +) diff --git a/source/common/json/json_internal.cc b/source/common/json/json_internal.cc new file mode 100644 index 0000000000000..18c674ef57e3a --- /dev/null +++ b/source/common/json/json_internal.cc @@ -0,0 +1,673 @@ +#include "common/json/json_internal.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/common/assert.h" +#include "common/common/fmt.h" +#include "common/common/hash.h" +#include "common/common/utility.h" +#include "common/protobuf/utility.h" + +// Do not let nlohmann/json leak outside of this file. +#include "include/nlohmann/json.hpp" + +#include "absl/strings/match.h" + +namespace Envoy { +namespace Json { +namespace Nlohmann { + +namespace { +/** + * Internal representation of Object. + */ +class Field; +using FieldSharedPtr = std::shared_ptr; + +class Field : public Object { +public: + void setLineNumberStart(uint64_t line_number) { line_number_start_ = line_number; } + void setLineNumberEnd(uint64_t line_number) { line_number_end_ = line_number; } + + // Container factories for handler. + static FieldSharedPtr createObject() { return FieldSharedPtr{new Field(Type::Object)}; } + static FieldSharedPtr createArray() { return FieldSharedPtr{new Field(Type::Array)}; } + static FieldSharedPtr createNull() { return FieldSharedPtr{new Field(Type::Null)}; } + + bool isNull() const override { return type_ == Type::Null; } + bool isArray() const override { return type_ == Type::Array; } + bool isObject() const override { return type_ == Type::Object; } + + // Value factory. + template static FieldSharedPtr createValue(T value) { + return FieldSharedPtr{new Field(value)}; // NOLINT(modernize-make-shared) + } + + void append(FieldSharedPtr field_ptr) { + checkType(Type::Array); + value_.array_value_.push_back(field_ptr); + } + void insert(const std::string& key, FieldSharedPtr field_ptr) { + checkType(Type::Object); + value_.object_value_[key] = field_ptr; + } + + uint64_t hash() const override; + + bool getBoolean(const std::string& name) const override; + bool getBoolean(const std::string& name, bool default_value) const override; + double getDouble(const std::string& name) const override; + double getDouble(const std::string& name, double default_value) const override; + int64_t getInteger(const std::string& name) const override; + int64_t getInteger(const std::string& name, int64_t default_value) const override; + ObjectSharedPtr getObject(const std::string& name, bool allow_empty) const override; + std::vector getObjectArray(const std::string& name, + bool allow_empty) const override; + std::string getString(const std::string& name) const override; + std::string getString(const std::string& name, const std::string& default_value) const override; + std::vector getStringArray(const std::string& name, bool allow_empty) const override; + std::vector asObjectArray() const override; + std::string asString() const override { return stringValue(); } + bool asBoolean() const override { return booleanValue(); } + double asDouble() const override { return doubleValue(); } + int64_t asInteger() const override { return integerValue(); } + std::string asJsonString() const override; + + bool empty() const override; + bool hasObject(const std::string& name) const override; + void iterate(const ObjectCallback& callback) const override; + void validateSchema(const std::string&) const override; + +private: + enum class Type { + Array, + Boolean, + Double, + Integer, + Null, + Object, + String, + }; + static const char* typeAsString(Type t) { + switch (t) { + case Type::Array: + return "Array"; + case Type::Boolean: + return "Boolean"; + case Type::Double: + return "Double"; + case Type::Integer: + return "Integer"; + case Type::Null: + return "Null"; + case Type::Object: + return "Object"; + case Type::String: + return "String"; + } + + NOT_REACHED_GCOVR_EXCL_LINE; + } + + struct Value { + std::vector array_value_; + bool boolean_value_; + double double_value_; + int64_t integer_value_; + std::map object_value_; + std::string string_value_; + }; + + explicit Field(Type type) : type_(type) {} + explicit Field(const std::string& value) : type_(Type::String) { value_.string_value_ = value; } + explicit Field(int64_t value) : type_(Type::Integer) { value_.integer_value_ = value; } + explicit Field(double value) : type_(Type::Double) { value_.double_value_ = value; } + explicit Field(bool value) : type_(Type::Boolean) { value_.boolean_value_ = value; } + + bool isType(Type type) const { return type == type_; } + void checkType(Type type) const { + if (!isType(type)) { + throw Exception(fmt::format( + "JSON field from line {} accessed with type '{}' does not match actual type '{}'.", + line_number_start_, typeAsString(type), typeAsString(type_))); + } + } + + // Value return type functions. + std::string stringValue() const { + checkType(Type::String); + return value_.string_value_; + } + std::vector arrayValue() const { + checkType(Type::Array); + return value_.array_value_; + } + bool booleanValue() const { + checkType(Type::Boolean); + return value_.boolean_value_; + } + double doubleValue() const { + checkType(Type::Double); + return value_.double_value_; + } + int64_t integerValue() const { + checkType(Type::Integer); + return value_.integer_value_; + } + + nlohmann::json asJsonDocument() const; + static void buildJsonDocument(const Field& field, nlohmann::json& value); + + uint64_t line_number_start_ = 0; + uint64_t line_number_end_ = 0; + const Type type_; + Value value_; +}; + +/** + * Consume events from SAX callbacks to build JSON Field. + */ +class ObjectHandler : public nlohmann::json_sax { +public: + ObjectHandler() : state_(State::ExpectRoot) {} + + bool start_object(std::size_t) override; + bool end_object() override; + bool key(std::string& val) override; + bool start_array(std::size_t) override; + bool end_array() override; + bool boolean(bool value) override { return handleValueEvent(Field::createValue(value)); } + bool number_integer(int64_t value) override { + return handleValueEvent(Field::createValue(static_cast(value))); + } + bool number_unsigned(uint64_t value) override { + if (value > static_cast(std::numeric_limits::max())) { + throw Exception(fmt::format("JSON value from line {} is larger than int64_t (not supported)", + line_number_)); + } + return handleValueEvent(Field::createValue(static_cast(value))); + } + bool number_float(double value, const std::string&) override { + return handleValueEvent(Field::createValue(value)); + } + bool null() override { return handleValueEvent(Field::createNull()); } + bool string(std::string& value) override { return handleValueEvent(Field::createValue(value)); } + bool binary(binary_t&) override { return false; } + bool parse_error(std::size_t, const std::string& token, + const nlohmann::detail::exception& ex) override { + // Errors are formatted like "[json.exception.parse_error.101] parse error at line x, column y: + // error string." + absl::string_view error = ex.what(); + // Extract position information (line, column, token) in the error. + auto start = error.find("line"); + auto npos = error.find(":"); + ENVOY_LOG_MISC(info, "ERROR {}", error); + error_position_ = absl::StrCat(error.substr(start, npos - start), ", token ", token); + // Extract portion after ":" to get error string. + error_ = error.substr(npos + 2); + return false; + } + + bool hasParseError() { return !error_.empty(); } + std::string getParseError() { return error_; } + std::string getErrorPosition() { return error_position_; } + + ObjectSharedPtr getRoot() { return root_; } + + int line_number_{1}; + +private: + bool handleValueEvent(FieldSharedPtr ptr); + + enum class State { + ExpectRoot, + ExpectKeyOrEndObject, + ExpectValueOrStartObjectArray, + ExpectArrayValueOrEndArray, + ExpectFinished, + }; + State state_; + + std::stack stack_; + std::string key_; + + FieldSharedPtr root_; + + std::string error_; + std::string error_position_; +}; + +struct JsonContainer { + JsonContainer(const char* ch, ObjectHandler* handler) : data(ch), handler_(handler) {} + const char* data; + ObjectHandler* handler_; +}; + +struct JsonIterator { + using difference_type = std::ptrdiff_t; + using value_type = char; + using pointer = const char*; + using reference = const char&; + using iterator_category = std::input_iterator_tag; + + JsonIterator& operator++() { + ++ptr.data; + return *this; + } + + bool operator!=(const JsonIterator& rhs) const { return rhs.ptr.data != ptr.data; } + + reference operator*() { + const char& ch = *(ptr.data); + if (ch == '\n') { + ptr.handler_->line_number_++; + } + return ch; + } + + JsonContainer ptr; +}; + +JsonIterator begin(const JsonContainer& c) { + return JsonIterator{JsonContainer(c.data, c.handler_)}; +} + +JsonIterator end(const JsonContainer& c) { + return JsonIterator{JsonContainer(c.data + strlen(c.data), c.handler_)}; +} + +void Field::buildJsonDocument(const Field& field, nlohmann::json& value) { + switch (field.type_) { + case Type::Array: { + for (const auto& element : field.value_.array_value_) { + switch (element->type_) { + case Type::Array: + case Type::Object: { + nlohmann::json nested_value; + buildJsonDocument(*element, nested_value); + value.push_back(nested_value); + break; + } + case Type::Boolean: + value.push_back(element->value_.boolean_value_); + break; + case Type::Double: + value.push_back(element->value_.double_value_); + break; + case Type::Integer: + value.push_back(element->value_.integer_value_); + break; + case Type::Null: + value.push_back(nlohmann::json::value_t::null); + break; + case Type::String: + value.push_back(element->value_.string_value_); + } + } + break; + } + case Type::Object: { + for (const auto& item : field.value_.object_value_) { + auto name = std::string(item.first.c_str()); + + switch (item.second->type_) { + case Type::Array: + case Type::Object: { + nlohmann::json nested_value; + buildJsonDocument(*item.second, nested_value); + value.emplace(name, nested_value); + break; + } + case Type::Boolean: + value.emplace(name, item.second->value_.boolean_value_); + break; + case Type::Double: + value.emplace(name, item.second->value_.double_value_); + break; + case Type::Integer: + value.emplace(name, item.second->value_.integer_value_); + break; + case Type::Null: + value.emplace(name, nlohmann::json::value_t::null); + break; + case Type::String: + value.emplace(name, item.second->value_.string_value_); + break; + } + } + break; + } + case Type::Null: { + break; + } + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +nlohmann::json Field::asJsonDocument() const { + nlohmann::json j; + buildJsonDocument(*this, j); + return j; +} + +uint64_t Field::hash() const { return HashUtil::xxHash64(asJsonString()); } + +bool Field::getBoolean(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Boolean)) { + throw Exception(fmt::format("key '{}' missing or not a boolean from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->booleanValue(); +} + +bool Field::getBoolean(const std::string& name, bool default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getBoolean(name); + } else { + return default_value; + } +} + +double Field::getDouble(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Double)) { + throw Exception(fmt::format("key '{}' missing or not a double from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->doubleValue(); +} + +double Field::getDouble(const std::string& name, double default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getDouble(name); + } else { + return default_value; + } +} + +int64_t Field::getInteger(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Integer)) { + throw Exception(fmt::format("key '{}' missing or not an integer from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->integerValue(); +} + +int64_t Field::getInteger(const std::string& name, int64_t default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getInteger(name); + } else { + return default_value; + } +} + +ObjectSharedPtr Field::getObject(const std::string& name, bool allow_empty) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end()) { + if (allow_empty) { + return createObject(); + } else { + throw Exception(fmt::format("key '{}' missing from lines {}-{}", name, line_number_start_, + line_number_end_)); + } + } else if (!value_itr->second->isType(Type::Object)) { + throw Exception(fmt::format("key '{}' not an object from line {}", name, + value_itr->second->line_number_start_)); + } else { + return value_itr->second; + } +} + +std::vector Field::getObjectArray(const std::string& name, + bool allow_empty) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Array)) { + if (allow_empty && value_itr == value_.object_value_.end()) { + return std::vector(); + } + throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + + std::vector array_value = value_itr->second->arrayValue(); + return {array_value.begin(), array_value.end()}; +} + +std::string Field::getString(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::String)) { + throw Exception(fmt::format("key '{}' missing or not a string from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->stringValue(); +} + +std::string Field::getString(const std::string& name, const std::string& default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getString(name); + } else { + return default_value; + } +} + +std::vector Field::getStringArray(const std::string& name, bool allow_empty) const { + checkType(Type::Object); + std::vector string_array; + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Array)) { + if (allow_empty && value_itr == value_.object_value_.end()) { + return string_array; + } + throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + + std::vector array = value_itr->second->arrayValue(); + string_array.reserve(array.size()); + for (const auto& element : array) { + if (!element->isType(Type::String)) { + throw Exception(fmt::format("JSON array '{}' from line {} does not contain all strings", name, + line_number_start_)); + } + string_array.push_back(element->stringValue()); + } + + return string_array; +} + +std::vector Field::asObjectArray() const { + checkType(Type::Array); + return {value_.array_value_.begin(), value_.array_value_.end()}; +} + +std::string Field::asJsonString() const { + nlohmann::json j = asJsonDocument(); + return j.dump(); +} + +bool Field::empty() const { + if (isType(Type::Object)) { + return value_.object_value_.empty(); + } else if (isType(Type::Array)) { + return value_.array_value_.empty(); + } else { + throw Exception( + fmt::format("Json does not support empty() on types other than array and object")); + } +} + +bool Field::hasObject(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + return value_itr != value_.object_value_.end(); +} + +void Field::iterate(const ObjectCallback& callback) const { + checkType(Type::Object); + for (const auto& item : value_.object_value_) { + bool stop_iteration = !callback(item.first, *item.second); + if (stop_iteration) { + break; + } + } +} + +void Field::validateSchema(const std::string&) const { throw Exception("not implemented"); } + +bool ObjectHandler::start_object(std::size_t) { + FieldSharedPtr object = Field::createObject(); + object->setLineNumberStart(line_number_); + + switch (state_) { + case State::ExpectValueOrStartObjectArray: + stack_.top()->insert(key_, object); + stack_.push(object); + state_ = State::ExpectKeyOrEndObject; + return true; + case State::ExpectArrayValueOrEndArray: + stack_.top()->append(object); + stack_.push(object); + state_ = State::ExpectKeyOrEndObject; + return true; + case State::ExpectRoot: + root_ = object; + stack_.push(object); + state_ = State::ExpectKeyOrEndObject; + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::end_object() { + switch (state_) { + case State::ExpectKeyOrEndObject: + stack_.top()->setLineNumberEnd(line_number_); + stack_.pop(); + + if (stack_.empty()) { + state_ = State::ExpectFinished; + } else if (stack_.top()->isObject()) { + state_ = State::ExpectKeyOrEndObject; + } else if (stack_.top()->isArray()) { + state_ = State::ExpectArrayValueOrEndArray; + } + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::key(std::string& val) { + switch (state_) { + case State::ExpectKeyOrEndObject: + key_ = val; + state_ = State::ExpectValueOrStartObjectArray; + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::start_array(std::size_t) { + FieldSharedPtr array = Field::createArray(); + array->setLineNumberStart(line_number_); + + switch (state_) { + case State::ExpectValueOrStartObjectArray: + stack_.top()->insert(key_, array); + stack_.push(array); + state_ = State::ExpectArrayValueOrEndArray; + return true; + case State::ExpectArrayValueOrEndArray: + stack_.top()->append(array); + stack_.push(array); + return true; + case State::ExpectRoot: + root_ = array; + stack_.push(array); + state_ = State::ExpectArrayValueOrEndArray; + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::end_array() { + switch (state_) { + case State::ExpectArrayValueOrEndArray: + stack_.top()->setLineNumberEnd(line_number_); + stack_.pop(); + + if (stack_.empty()) { + state_ = State::ExpectFinished; + } else if (stack_.top()->isObject()) { + state_ = State::ExpectKeyOrEndObject; + } else if (stack_.top()->isArray()) { + state_ = State::ExpectArrayValueOrEndArray; + } + + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { + ptr->setLineNumberStart(line_number_); + + switch (state_) { + case State::ExpectValueOrStartObjectArray: + state_ = State::ExpectKeyOrEndObject; + stack_.top()->insert(key_, ptr); + return true; + case State::ExpectArrayValueOrEndArray: + stack_.top()->append(ptr); + return true; + default: + return true; + } +} + +} // namespace + +ObjectSharedPtr Factory::loadFromString(const std::string& json) { + ObjectHandler handler; + auto json_container = JsonContainer(json.data(), &handler); + + nlohmann::json::sax_parse(json_container, &handler); + + if (handler.hasParseError()) { + throw Exception(fmt::format("JSON supplied is not valid. Error({}): {}\n", + handler.getErrorPosition(), handler.getParseError())); + } + return handler.getRoot(); +} + +} // namespace Nlohmann +} // namespace Json +} // namespace Envoy diff --git a/source/common/json/json_internal.h b/source/common/json/json_internal.h new file mode 100644 index 0000000000000..de665333a1c09 --- /dev/null +++ b/source/common/json/json_internal.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include "envoy/json/json_object.h" + +namespace Envoy { +namespace Json { +namespace Nlohmann { + +class Factory { +public: + /** + * Constructs a Json Object from a string. + */ + static ObjectSharedPtr loadFromString(const std::string& json); +}; + +} // namespace Nlohmann +} // namespace Json +} // namespace Envoy diff --git a/source/common/json/rapidjson_loader.cc b/source/common/json/json_internal_legacy.cc similarity index 99% rename from source/common/json/rapidjson_loader.cc rename to source/common/json/json_internal_legacy.cc index 04f8b63a15cfe..e5b284b1004fc 100644 --- a/source/common/json/rapidjson_loader.cc +++ b/source/common/json/json_internal_legacy.cc @@ -1,4 +1,4 @@ -#include "common/json/rapidjson_loader.h" +#include "common/json/json_internal_legacy.h" #include #include diff --git a/source/common/json/json_internal_legacy.h b/source/common/json/json_internal_legacy.h new file mode 100644 index 0000000000000..6b50ccd2b6268 --- /dev/null +++ b/source/common/json/json_internal_legacy.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include "envoy/json/json_object.h" + +namespace Envoy { +namespace Json { +namespace RapidJson { + +class Factory { +public: + /** + * Constructs a Json Object from a string. + */ + static ObjectSharedPtr loadFromString(const std::string& json); +}; + +} // namespace RapidJson +} // namespace Json +} // namespace Envoy diff --git a/source/common/json/json_loader.cc b/source/common/json/json_loader.cc index fa2410a8eb9d0..c0a1add871ae7 100644 --- a/source/common/json/json_loader.cc +++ b/source/common/json/json_loader.cc @@ -1,670 +1,17 @@ #include "common/json/json_loader.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/common/assert.h" -#include "common/common/fmt.h" -#include "common/common/hash.h" -#include "common/common/utility.h" -#include "common/protobuf/utility.h" - -// Do not let nlohmann/json leak outside of this file. -#include "include/nlohmann/json.hpp" - -#include "absl/strings/match.h" +#include "common/json/json_internal.h" +#include "common/json/json_internal_legacy.h" +#include "common/runtime/runtime_features.h" namespace Envoy { namespace Json { -namespace { -/** - * Internal representation of Object. - */ -class Field; -using FieldSharedPtr = std::shared_ptr; - -class Field : public Object { -public: - void setLineNumberStart(uint64_t line_number) { line_number_start_ = line_number; } - void setLineNumberEnd(uint64_t line_number) { line_number_end_ = line_number; } - - // Container factories for handler. - static FieldSharedPtr createObject() { return FieldSharedPtr{new Field(Type::Object)}; } - static FieldSharedPtr createArray() { return FieldSharedPtr{new Field(Type::Array)}; } - static FieldSharedPtr createNull() { return FieldSharedPtr{new Field(Type::Null)}; } - - bool isNull() const override { return type_ == Type::Null; } - bool isArray() const override { return type_ == Type::Array; } - bool isObject() const override { return type_ == Type::Object; } - - // Value factory. - template static FieldSharedPtr createValue(T value) { - return FieldSharedPtr{new Field(value)}; // NOLINT(modernize-make-shared) - } - - void append(FieldSharedPtr field_ptr) { - checkType(Type::Array); - value_.array_value_.push_back(field_ptr); - } - void insert(const std::string& key, FieldSharedPtr field_ptr) { - checkType(Type::Object); - value_.object_value_[key] = field_ptr; - } - - uint64_t hash() const override; - - bool getBoolean(const std::string& name) const override; - bool getBoolean(const std::string& name, bool default_value) const override; - double getDouble(const std::string& name) const override; - double getDouble(const std::string& name, double default_value) const override; - int64_t getInteger(const std::string& name) const override; - int64_t getInteger(const std::string& name, int64_t default_value) const override; - ObjectSharedPtr getObject(const std::string& name, bool allow_empty) const override; - std::vector getObjectArray(const std::string& name, - bool allow_empty) const override; - std::string getString(const std::string& name) const override; - std::string getString(const std::string& name, const std::string& default_value) const override; - std::vector getStringArray(const std::string& name, bool allow_empty) const override; - std::vector asObjectArray() const override; - std::string asString() const override { return stringValue(); } - bool asBoolean() const override { return booleanValue(); } - double asDouble() const override { return doubleValue(); } - int64_t asInteger() const override { return integerValue(); } - std::string asJsonString() const override; - - bool empty() const override; - bool hasObject(const std::string& name) const override; - void iterate(const ObjectCallback& callback) const override; - void validateSchema(const std::string&) const override; - -private: - enum class Type { - Array, - Boolean, - Double, - Integer, - Null, - Object, - String, - }; - static const char* typeAsString(Type t) { - switch (t) { - case Type::Array: - return "Array"; - case Type::Boolean: - return "Boolean"; - case Type::Double: - return "Double"; - case Type::Integer: - return "Integer"; - case Type::Null: - return "Null"; - case Type::Object: - return "Object"; - case Type::String: - return "String"; - } - - NOT_REACHED_GCOVR_EXCL_LINE; - } - - struct Value { - std::vector array_value_; - bool boolean_value_; - double double_value_; - int64_t integer_value_; - std::map object_value_; - std::string string_value_; - }; - - explicit Field(Type type) : type_(type) {} - explicit Field(const std::string& value) : type_(Type::String) { value_.string_value_ = value; } - explicit Field(int64_t value) : type_(Type::Integer) { value_.integer_value_ = value; } - explicit Field(double value) : type_(Type::Double) { value_.double_value_ = value; } - explicit Field(bool value) : type_(Type::Boolean) { value_.boolean_value_ = value; } - - bool isType(Type type) const { return type == type_; } - void checkType(Type type) const { - if (!isType(type)) { - throw Exception(fmt::format( - "JSON field from line {} accessed with type '{}' does not match actual type '{}'.", - line_number_start_, typeAsString(type), typeAsString(type_))); - } - } - - // Value return type functions. - std::string stringValue() const { - checkType(Type::String); - return value_.string_value_; - } - std::vector arrayValue() const { - checkType(Type::Array); - return value_.array_value_; - } - bool booleanValue() const { - checkType(Type::Boolean); - return value_.boolean_value_; - } - double doubleValue() const { - checkType(Type::Double); - return value_.double_value_; - } - int64_t integerValue() const { - checkType(Type::Integer); - return value_.integer_value_; - } - - nlohmann::json asJsonDocument() const; - static void buildJsonDocument(const Field& field, nlohmann::json& value); - - uint64_t line_number_start_ = 0; - uint64_t line_number_end_ = 0; - const Type type_; - Value value_; -}; - -/** - * Consume events from SAX callbacks to build JSON Field. - */ -class ObjectHandler : public nlohmann::json_sax { -public: - ObjectHandler() : state_(State::ExpectRoot) {} - - bool start_object(std::size_t) override; - bool end_object() override; - bool key(std::string& val) override; - bool start_array(std::size_t) override; - bool end_array() override; - bool boolean(bool value) override { return handleValueEvent(Field::createValue(value)); } - bool number_integer(int64_t value) override { - return handleValueEvent(Field::createValue(static_cast(value))); - } - bool number_unsigned(uint64_t value) override { - if (value > static_cast(std::numeric_limits::max())) { - throw Exception(fmt::format("JSON value from line {} is larger than int64_t (not supported)", - line_number_)); - } - return handleValueEvent(Field::createValue(static_cast(value))); - } - bool number_float(double value, const std::string&) override { - return handleValueEvent(Field::createValue(value)); - } - bool null() override { return handleValueEvent(Field::createNull()); } - bool string(std::string& value) override { return handleValueEvent(Field::createValue(value)); } - bool binary(binary_t&) override { return false; } - bool parse_error(std::size_t, const std::string& token, - const nlohmann::detail::exception& ex) override { - // Errors are formatted like "[json.exception.parse_error.101] parse error at line x, column y: - // error string." - absl::string_view error = ex.what(); - // Extract position information (line, column, token) in the error. - auto start = error.find("line"); - auto npos = error.find(":"); - ENVOY_LOG_MISC(info, "ERROR {}", error); - error_position_ = absl::StrCat(error.substr(start, npos - start), ", token ", token); - // Extract portion after ":" to get error string. - error_ = error.substr(npos + 2); - return false; - } - - bool hasParseError() { return !error_.empty(); } - std::string getParseError() { return error_; } - std::string getErrorPosition() { return error_position_; } - - ObjectSharedPtr getRoot() { return root_; } - - int line_number_{1}; - -private: - bool handleValueEvent(FieldSharedPtr ptr); - - enum class State { - ExpectRoot, - ExpectKeyOrEndObject, - ExpectValueOrStartObjectArray, - ExpectArrayValueOrEndArray, - ExpectFinished, - }; - State state_; - - std::stack stack_; - std::string key_; - - FieldSharedPtr root_; - - std::string error_; - std::string error_position_; -}; - -struct JsonContainer { - JsonContainer(const char* ch, ObjectHandler* handler) : data(ch), handler_(handler) {} - const char* data; - ObjectHandler* handler_; -}; - -struct JsonIterator { - using difference_type = std::ptrdiff_t; - using value_type = char; - using pointer = const char*; - using reference = const char&; - using iterator_category = std::input_iterator_tag; - - JsonIterator& operator++() { - ++ptr.data; - return *this; - } - - bool operator!=(const JsonIterator& rhs) const { return rhs.ptr.data != ptr.data; } - - reference operator*() { - const char& ch = *(ptr.data); - if (ch == '\n') { - ptr.handler_->line_number_++; - } - return ch; - } - - JsonContainer ptr; -}; - -JsonIterator begin(const JsonContainer& c) { - return JsonIterator{JsonContainer(c.data, c.handler_)}; -} - -JsonIterator end(const JsonContainer& c) { - return JsonIterator{JsonContainer(c.data + strlen(c.data), c.handler_)}; -} - -void Field::buildJsonDocument(const Field& field, nlohmann::json& value) { - switch (field.type_) { - case Type::Array: { - for (const auto& element : field.value_.array_value_) { - switch (element->type_) { - case Type::Array: - case Type::Object: { - nlohmann::json nested_value; - buildJsonDocument(*element, nested_value); - value.push_back(nested_value); - break; - } - case Type::Boolean: - value.push_back(element->value_.boolean_value_); - break; - case Type::Double: - value.push_back(element->value_.double_value_); - break; - case Type::Integer: - value.push_back(element->value_.integer_value_); - break; - case Type::Null: - value.push_back(nlohmann::json::value_t::null); - break; - case Type::String: - value.push_back(element->value_.string_value_); - } - } - break; - } - case Type::Object: { - for (const auto& item : field.value_.object_value_) { - auto name = std::string(item.first.c_str()); - - switch (item.second->type_) { - case Type::Array: - case Type::Object: { - nlohmann::json nested_value; - buildJsonDocument(*item.second, nested_value); - value.emplace(name, nested_value); - break; - } - case Type::Boolean: - value.emplace(name, item.second->value_.boolean_value_); - break; - case Type::Double: - value.emplace(name, item.second->value_.double_value_); - break; - case Type::Integer: - value.emplace(name, item.second->value_.integer_value_); - break; - case Type::Null: - value.emplace(name, nlohmann::json::value_t::null); - break; - case Type::String: - value.emplace(name, item.second->value_.string_value_); - break; - } - } - break; - } - case Type::Null: { - break; - } - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -nlohmann::json Field::asJsonDocument() const { - nlohmann::json j; - buildJsonDocument(*this, j); - return j; -} - -uint64_t Field::hash() const { return HashUtil::xxHash64(asJsonString()); } - -bool Field::getBoolean(const std::string& name) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Boolean)) { - throw Exception(fmt::format("key '{}' missing or not a boolean from lines {}-{}", name, - line_number_start_, line_number_end_)); - } - return value_itr->second->booleanValue(); -} - -bool Field::getBoolean(const std::string& name, bool default_value) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr != value_.object_value_.end()) { - return getBoolean(name); - } else { - return default_value; - } -} - -double Field::getDouble(const std::string& name) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Double)) { - throw Exception(fmt::format("key '{}' missing or not a double from lines {}-{}", name, - line_number_start_, line_number_end_)); - } - return value_itr->second->doubleValue(); -} - -double Field::getDouble(const std::string& name, double default_value) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr != value_.object_value_.end()) { - return getDouble(name); - } else { - return default_value; - } -} - -int64_t Field::getInteger(const std::string& name) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Integer)) { - throw Exception(fmt::format("key '{}' missing or not an integer from lines {}-{}", name, - line_number_start_, line_number_end_)); - } - return value_itr->second->integerValue(); -} - -int64_t Field::getInteger(const std::string& name, int64_t default_value) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr != value_.object_value_.end()) { - return getInteger(name); - } else { - return default_value; - } -} - -ObjectSharedPtr Field::getObject(const std::string& name, bool allow_empty) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end()) { - if (allow_empty) { - return createObject(); - } else { - throw Exception(fmt::format("key '{}' missing from lines {}-{}", name, line_number_start_, - line_number_end_)); - } - } else if (!value_itr->second->isType(Type::Object)) { - throw Exception(fmt::format("key '{}' not an object from line {}", name, - value_itr->second->line_number_start_)); - } else { - return value_itr->second; - } -} - -std::vector Field::getObjectArray(const std::string& name, - bool allow_empty) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Array)) { - if (allow_empty && value_itr == value_.object_value_.end()) { - return std::vector(); - } - throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, - line_number_start_, line_number_end_)); - } - - std::vector array_value = value_itr->second->arrayValue(); - return {array_value.begin(), array_value.end()}; -} - -std::string Field::getString(const std::string& name) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::String)) { - throw Exception(fmt::format("key '{}' missing or not a string from lines {}-{}", name, - line_number_start_, line_number_end_)); - } - return value_itr->second->stringValue(); -} - -std::string Field::getString(const std::string& name, const std::string& default_value) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr != value_.object_value_.end()) { - return getString(name); - } else { - return default_value; - } -} - -std::vector Field::getStringArray(const std::string& name, bool allow_empty) const { - checkType(Type::Object); - std::vector string_array; - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Array)) { - if (allow_empty && value_itr == value_.object_value_.end()) { - return string_array; - } - throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, - line_number_start_, line_number_end_)); - } - - std::vector array = value_itr->second->arrayValue(); - string_array.reserve(array.size()); - for (const auto& element : array) { - if (!element->isType(Type::String)) { - throw Exception(fmt::format("JSON array '{}' from line {} does not contain all strings", name, - line_number_start_)); - } - string_array.push_back(element->stringValue()); - } - - return string_array; -} - -std::vector Field::asObjectArray() const { - checkType(Type::Array); - return {value_.array_value_.begin(), value_.array_value_.end()}; -} - -std::string Field::asJsonString() const { - nlohmann::json j = asJsonDocument(); - return j.dump(); -} - -bool Field::empty() const { - if (isType(Type::Object)) { - return value_.object_value_.empty(); - } else if (isType(Type::Array)) { - return value_.array_value_.empty(); - } else { - throw Exception( - fmt::format("Json does not support empty() on types other than array and object")); - } -} - -bool Field::hasObject(const std::string& name) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - return value_itr != value_.object_value_.end(); -} - -void Field::iterate(const ObjectCallback& callback) const { - checkType(Type::Object); - for (const auto& item : value_.object_value_) { - bool stop_iteration = !callback(item.first, *item.second); - if (stop_iteration) { - break; - } - } -} - -void Field::validateSchema(const std::string&) const { throw Exception("not implemented"); } - -bool ObjectHandler::start_object(std::size_t) { - FieldSharedPtr object = Field::createObject(); - object->setLineNumberStart(line_number_); - - switch (state_) { - case State::ExpectValueOrStartObjectArray: - stack_.top()->insert(key_, object); - stack_.push(object); - state_ = State::ExpectKeyOrEndObject; - return true; - case State::ExpectArrayValueOrEndArray: - stack_.top()->append(object); - stack_.push(object); - state_ = State::ExpectKeyOrEndObject; - return true; - case State::ExpectRoot: - root_ = object; - stack_.push(object); - state_ = State::ExpectKeyOrEndObject; - return true; - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -bool ObjectHandler::end_object() { - switch (state_) { - case State::ExpectKeyOrEndObject: - stack_.top()->setLineNumberEnd(line_number_); - stack_.pop(); - - if (stack_.empty()) { - state_ = State::ExpectFinished; - } else if (stack_.top()->isObject()) { - state_ = State::ExpectKeyOrEndObject; - } else if (stack_.top()->isArray()) { - state_ = State::ExpectArrayValueOrEndArray; - } - return true; - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -bool ObjectHandler::key(std::string& val) { - switch (state_) { - case State::ExpectKeyOrEndObject: - key_ = val; - state_ = State::ExpectValueOrStartObjectArray; - return true; - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -bool ObjectHandler::start_array(std::size_t) { - FieldSharedPtr array = Field::createArray(); - array->setLineNumberStart(line_number_); - - switch (state_) { - case State::ExpectValueOrStartObjectArray: - stack_.top()->insert(key_, array); - stack_.push(array); - state_ = State::ExpectArrayValueOrEndArray; - return true; - case State::ExpectArrayValueOrEndArray: - stack_.top()->append(array); - stack_.push(array); - return true; - case State::ExpectRoot: - root_ = array; - stack_.push(array); - state_ = State::ExpectArrayValueOrEndArray; - return true; - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -bool ObjectHandler::end_array() { - switch (state_) { - case State::ExpectArrayValueOrEndArray: - stack_.top()->setLineNumberEnd(line_number_); - stack_.pop(); - - if (stack_.empty()) { - state_ = State::ExpectFinished; - } else if (stack_.top()->isObject()) { - state_ = State::ExpectKeyOrEndObject; - } else if (stack_.top()->isArray()) { - state_ = State::ExpectArrayValueOrEndArray; - } - - return true; - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { - ptr->setLineNumberStart(line_number_); - - switch (state_) { - case State::ExpectValueOrStartObjectArray: - state_ = State::ExpectKeyOrEndObject; - stack_.top()->insert(key_, ptr); - return true; - case State::ExpectArrayValueOrEndArray: - stack_.top()->append(ptr); - return true; - default: - return true; - } -} - -} // namespace - ObjectSharedPtr Factory::loadFromString(const std::string& json) { - ObjectHandler handler; - auto json_container = JsonContainer(json.data(), &handler); - - nlohmann::json::sax_parse(json_container, &handler); - - if (handler.hasParseError()) { - throw Exception(fmt::format("JSON supplied is not valid. Error({}): {}\n", - handler.getErrorPosition(), handler.getParseError())); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_rapidjson")) { + return Nlohmann::Factory::loadFromString(json); } - return handler.getRoot(); + return RapidJson::Factory::loadFromString(json); } } // namespace Json diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 34ba8d3a013e8..9b36c77d9e9c2 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -81,6 +81,7 @@ constexpr const char* runtime_features[] = { "envoy.reloadable_features.prefer_quic_kernel_bpf_packet_routing", "envoy.reloadable_features.preserve_query_string_in_path_redirects", "envoy.reloadable_features.preserve_upstream_date", + "envoy.reloadable_features.remove_rapidjson", "envoy.reloadable_features.require_ocsp_response_for_must_staple_certs", "envoy.reloadable_features.stop_faking_paths", "envoy.reloadable_features.strict_1xx_and_204_response_headers", diff --git a/test/common/json/BUILD b/test/common/json/BUILD index b93cba2b7abe3..feb4b04534805 100644 --- a/test/common/json/BUILD +++ b/test/common/json/BUILD @@ -23,14 +23,23 @@ envoy_cc_fuzz_test( ], ) +JSON_TEST_DEPS = [ + "//source/common/json:json_loader_lib", + "//source/common/stats:isolated_store_lib", + "//test/test_common:utility_lib", +] + envoy_cc_test( name = "json_loader_test", srcs = ["json_loader_test.cc"], - deps = [ - "//source/common/json:json_loader_lib", - "//source/common/stats:isolated_store_lib", - "//test/test_common:utility_lib", - ], + deps = JSON_TEST_DEPS, +) + +envoy_cc_test( + name = "json_loader_legacy_test", + srcs = ["json_loader_test.cc"], + args = ["--runtime-feature-disable-for-tests=envoy.reloadable_features.remove_rapidjson"], + deps = JSON_TEST_DEPS, ) envoy_cc_benchmark_binary( diff --git a/test/common/json/json_loader_test.cc b/test/common/json/json_loader_test.cc index fbab4467f80f9..c33c5b0b45632 100644 --- a/test/common/json/json_loader_test.cc +++ b/test/common/json/json_loader_test.cc @@ -71,10 +71,16 @@ TEST_F(JsonLoaderTest, Basic) { } { - EXPECT_THROW_WITH_MESSAGE( - Factory::loadFromString("{\"hello\": \n\n\"world\""), Exception, - "JSON supplied is not valid. Error(line 3, column 8, token \"world\"): syntax error while " - "parsing object - unexpected end of input; expected '}'\n"); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_rapidjson")) { + EXPECT_THROW_WITH_MESSAGE(Factory::loadFromString("{\"hello\": \n\n\"world\""), Exception, + "JSON supplied is not valid. Error(line 3, column 8, token " + "\"world\"): syntax error while " + "parsing object - unexpected end of input; expected '}'\n"); + } else { + EXPECT_THROW_WITH_MESSAGE(Factory::loadFromString("{\"hello\": \n\n\"world\""), Exception, + "JSON supplied is not valid. Error(offset 19, line 3): Missing a " + "comma or '}' after an object member.\n"); + } } { @@ -273,7 +279,14 @@ TEST_F(JsonLoaderTest, Schema) { )EOF"; ObjectSharedPtr json = Factory::loadFromString(json_string); - EXPECT_THROW_WITH_MESSAGE(json->validateSchema(invalid_schema), Exception, "not implemented"); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_rapidjson")) { + EXPECT_THROW_WITH_MESSAGE(json->validateSchema(invalid_schema), Exception, "not implemented"); + } else { + EXPECT_THROW_WITH_MESSAGE( + json->validateSchema(invalid_schema), Exception, + "JSON at lines 2-5 does not conform to schema.\n Invalid schema: #/properties/value1\n " + "Schema violation: type\n Offending document key: #/value1"); + } } TEST_F(JsonLoaderTest, MissingEnclosingDocument) { @@ -286,11 +299,16 @@ TEST_F(JsonLoaderTest, MissingEnclosingDocument) { } ] )EOF"; - - EXPECT_THROW_WITH_MESSAGE( - Factory::loadFromString(json_string), Exception, - "JSON supplied is not valid. Error(line 2, column 15, token \"listeners\" :): syntax error " - "while parsing value - unexpected ':'; expected end of input\n"); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_rapidjson")) { + EXPECT_THROW_WITH_MESSAGE( + Factory::loadFromString(json_string), Exception, + "JSON supplied is not valid. Error(line 2, column 15, token \"listeners\" :): syntax error " + "while parsing value - unexpected ':'; expected end of input\n"); + } else { + EXPECT_THROW_WITH_MESSAGE(Factory::loadFromString(json_string), Exception, + "JSON supplied is not valid. Error(offset 14, line 2): Terminate " + "parsing due to Handler error.\n"); + } } TEST_F(JsonLoaderTest, AsString) { From 388096dbed12fb1fcde4f4070b67471abb0a79d9 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Thu, 7 Jan 2021 13:28:11 -0500 Subject: [PATCH 06/17] fix spelling Signed-off-by: Asra Ali --- tools/spelling/spelling_dictionary.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 8d87a26db7254..33a63f76a6fda 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -196,6 +196,7 @@ NBSP NDEBUG NEXTHDR NGHTTP +NLOHMANN NOAUTH NOCHECKRESP NODELAY From 57a72d44b1771bdeff3184327b318273f8969e52 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Thu, 7 Jan 2021 14:37:13 -0500 Subject: [PATCH 07/17] fix build Signed-off-by: Asra Ali --- bazel/external/json.BUILD | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bazel/external/json.BUILD b/bazel/external/json.BUILD index 5f673ca72dc60..8a0b359d0fc87 100644 --- a/bazel/external/json.BUILD +++ b/bazel/external/json.BUILD @@ -9,11 +9,8 @@ cc_library( "include/nlohmann/**/*.hpp", "include/nlohmann/*/*/*.hpp", ]), - copts = [ - "-I external/nlohmann_json_lib", - ], + includes = ["external/nlohmann_json_lib"], visibility = ["//visibility:public"], - alwayslink = 1, ) cc_library( From f1af1e6024362020c36ed3eb2ae394a3191e70d2 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Thu, 7 Jan 2021 14:41:05 -0500 Subject: [PATCH 08/17] fix tests Signed-off-by: Asra Ali --- test/common/json/BUILD | 20 -- test/common/json/json_loader_speed_test.cc | 52 --- test/common/json/rapidjson_loader_test.cc | 385 --------------------- 3 files changed, 457 deletions(-) delete mode 100644 test/common/json/json_loader_speed_test.cc delete mode 100644 test/common/json/rapidjson_loader_test.cc diff --git a/test/common/json/BUILD b/test/common/json/BUILD index feb4b04534805..f3187a37c606e 100644 --- a/test/common/json/BUILD +++ b/test/common/json/BUILD @@ -1,7 +1,5 @@ load( "//bazel:envoy_build_system.bzl", - "envoy_benchmark_test", - "envoy_cc_benchmark_binary", "envoy_cc_fuzz_test", "envoy_cc_test", "envoy_package", @@ -41,21 +39,3 @@ envoy_cc_test( args = ["--runtime-feature-disable-for-tests=envoy.reloadable_features.remove_rapidjson"], deps = JSON_TEST_DEPS, ) - -envoy_cc_benchmark_binary( - name = "json_loader_speed_test", - srcs = ["json_loader_speed_test.cc"], - external_deps = [ - "benchmark", - "json", - ], - deps = [ - "//source/common/json:json_loader_lib", - "//source/common/json:rapidjson_loader_lib", - ], -) - -envoy_benchmark_test( - name = "json_loader_speed_test_benchmark_test", - benchmark_binary = "json_loader_speed_test", -) diff --git a/test/common/json/json_loader_speed_test.cc b/test/common/json/json_loader_speed_test.cc deleted file mode 100644 index cad119e0b5e59..0000000000000 --- a/test/common/json/json_loader_speed_test.cc +++ /dev/null @@ -1,52 +0,0 @@ -#include - -#include "common/json/json_loader.h" -#include "common/json/rapidjson_loader.h" - -#include "benchmark/benchmark.h" -#include "include/nlohmann/json.hpp" - -namespace Envoy { - -// Creates a JSON string with a variable number of elements and array sizes. -std::string createJsonObject(const int elements) { - nlohmann::json j = nlohmann::json::object(); - for (int i = 0; i < elements; i++) { - j.emplace(std::to_string(i), i); - } - return j.dump(); -} - -std::string createJsonArray(const int elements) { - nlohmann::json j = nlohmann::json::array(); - for (int i = 0; i < elements; i++) { - j.push_back(std::to_string(i)); - } - return j.dump(); -} - -template void BM_JsonLoadObjects(benchmark::State& state) { - std::string json = createJsonObject(state.range(0)); - for (auto _ : state) { - Parser::loadFromString(json); - } - benchmark::DoNotOptimize(json); -} -BENCHMARK_TEMPLATE(BM_JsonLoadObjects, Json::Factory)->Arg(1)->Arg(10)->Arg(50)->Arg(100); -BENCHMARK_TEMPLATE(BM_JsonLoadObjects, Json::RapidJson::Factory) - ->Arg(1) - ->Arg(10) - ->Arg(50) - ->Arg(100); - -template void BM_JsonLoadArray(benchmark::State& state) { - std::string json = createJsonArray(state.range(0)); - for (auto _ : state) { - Parser::loadFromString(json); - } - benchmark::DoNotOptimize(json); -} -BENCHMARK_TEMPLATE(BM_JsonLoadArray, Json::Factory)->Arg(1)->Arg(10)->Arg(50)->Arg(100); -BENCHMARK_TEMPLATE(BM_JsonLoadArray, Json::RapidJson::Factory)->Arg(1)->Arg(10)->Arg(50)->Arg(100); - -} // namespace Envoy diff --git a/test/common/json/rapidjson_loader_test.cc b/test/common/json/rapidjson_loader_test.cc deleted file mode 100644 index 884caf5b0d0c1..0000000000000 --- a/test/common/json/rapidjson_loader_test.cc +++ /dev/null @@ -1,385 +0,0 @@ -#include -#include - -#include "common/json/json_loader.h" -#include "common/stats/isolated_store_impl.h" - -#include "test/test_common/utility.h" - -#include "gtest/gtest.h" - -namespace Envoy { -namespace Json { -namespace { - -class JsonLoaderTest : public testing::Test { -protected: - JsonLoaderTest() : api_(Api::createApiForTest()) {} - - Api::ApiPtr api_; -}; - -TEST_F(JsonLoaderTest, Basic) { - EXPECT_THROW(Factory::loadFromString("{"), Exception); - - { - ObjectSharedPtr json = Factory::loadFromString("{\"hello\":123}"); - EXPECT_TRUE(json->hasObject("hello")); - EXPECT_FALSE(json->hasObject("world")); - EXPECT_FALSE(json->empty()); - EXPECT_THROW(json->getObject("world"), Exception); - EXPECT_THROW(json->getObject("hello"), Exception); - EXPECT_THROW(json->getBoolean("hello"), Exception); - EXPECT_THROW(json->getObjectArray("hello"), Exception); - EXPECT_THROW(json->getString("hello"), Exception); - - EXPECT_THROW_WITH_MESSAGE(json->getString("hello"), Exception, - "key 'hello' missing or not a string from lines 1-1"); - } - - { - ObjectSharedPtr json = Factory::loadFromString("{\"hello\":\"123\"\n}"); - EXPECT_THROW_WITH_MESSAGE(json->getInteger("hello"), Exception, - "key 'hello' missing or not an integer from lines 1-2"); - } - - { - ObjectSharedPtr json = Factory::loadFromString("{\"hello\":true}"); - EXPECT_TRUE(json->getBoolean("hello")); - EXPECT_TRUE(json->getBoolean("hello", false)); - EXPECT_FALSE(json->getBoolean("world", false)); - } - - { - ObjectSharedPtr json = Factory::loadFromString("{\"hello\": [\"a\", \"b\", 3]}"); - EXPECT_THROW(json->getStringArray("hello"), Exception); - EXPECT_THROW(json->getStringArray("world"), Exception); - } - - { - ObjectSharedPtr json = Factory::loadFromString("{\"hello\":123}"); - EXPECT_EQ(123, json->getInteger("hello", 456)); - EXPECT_EQ(456, json->getInteger("world", 456)); - } - - { - ObjectSharedPtr json = Factory::loadFromString("{\"hello\": \n[123]}"); - - EXPECT_THROW_WITH_MESSAGE( - json->getObjectArray("hello").at(0)->getString("hello"), Exception, - "JSON field from line 2 accessed with type 'Object' does not match actual type 'Integer'."); - } - - { - EXPECT_THROW_WITH_MESSAGE( - Factory::loadFromString("{\"hello\": \n\n\"world\""), Exception, - "JSON supplied is not valid. Error(offset 19, line 3): Missing a comma or " - "'}' after an object member.\n"); - } - - { - ObjectSharedPtr json_object = Factory::loadFromString("[\"foo\",\"bar\"]"); - EXPECT_FALSE(json_object->empty()); - } - - { - ObjectSharedPtr json_object = Factory::loadFromString("[]"); - EXPECT_TRUE(json_object->empty()); - } - - { - ObjectSharedPtr json = - Factory::loadFromString("{\"1\":{\"11\":\"111\"},\"2\":{\"22\":\"222\"}}"); - int pos = 0; - json->iterate([&pos](const std::string& key, const Json::Object& value) { - EXPECT_TRUE(key == "1" || key == "2"); - - if (key == "1") { - EXPECT_EQ("111", value.getString("11")); - } else { - EXPECT_EQ("222", value.getString("22")); - } - - pos++; - return true; - }); - - EXPECT_EQ(2, pos); - } - - { - ObjectSharedPtr json = - Factory::loadFromString("{\"1\":{\"11\":\"111\"},\"2\":{\"22\":\"222\"}}"); - int pos = 0; - json->iterate([&pos](const std::string& key, const Json::Object& value) { - EXPECT_TRUE(key == "1" || key == "2"); - - if (key == "1") { - EXPECT_EQ("111", value.getString("11")); - } else { - EXPECT_EQ("222", value.getString("22")); - } - - pos++; - return false; - }); - - EXPECT_EQ(1, pos); - } - - { - std::string json = R"EOF( - { - "descriptors": [ - [{"key": "hello", "value": "world"}, {"key": "foo", "value": "bar"}], - [{"key": "foo2", "value": "bar2"}] - ] - } - )EOF"; - - ObjectSharedPtr config = Factory::loadFromString(json); - EXPECT_EQ(2U, config->getObjectArray("descriptors")[0]->asObjectArray().size()); - EXPECT_EQ(1U, config->getObjectArray("descriptors")[1]->asObjectArray().size()); - } - - { - std::string json = R"EOF( - { - "descriptors": ["hello", "world"] - } - )EOF"; - - ObjectSharedPtr config = Factory::loadFromString(json); - std::vector array = config->getObjectArray("descriptors"); - EXPECT_THROW(array[0]->asObjectArray(), Exception); - } - - { - std::string json = R"EOF({})EOF"; - ObjectSharedPtr config = Factory::loadFromString(json); - ObjectSharedPtr object = config->getObject("foo", true); - EXPECT_EQ(2, object->getInteger("bar", 2)); - EXPECT_TRUE(object->empty()); - } - - { - std::string json = R"EOF({"foo": []})EOF"; - ObjectSharedPtr config = Factory::loadFromString(json); - EXPECT_TRUE(config->getStringArray("foo").empty()); - } - - { - std::string json = R"EOF({"foo": ["bar", "baz"]})EOF"; - ObjectSharedPtr config = Factory::loadFromString(json); - EXPECT_FALSE(config->getStringArray("foo").empty()); - } - - { - std::string json = R"EOF({})EOF"; - ObjectSharedPtr config = Factory::loadFromString(json); - EXPECT_THROW(config->getStringArray("foo"), EnvoyException); - } - - { - std::string json = R"EOF({})EOF"; - ObjectSharedPtr config = Factory::loadFromString(json); - EXPECT_TRUE(config->getStringArray("foo", true).empty()); - } - - { - ObjectSharedPtr json = Factory::loadFromString("{\"hello\": \n[2.0]}"); - EXPECT_THROW(json->getObjectArray("hello").at(0)->getDouble("foo"), Exception); - } - - { - ObjectSharedPtr json = Factory::loadFromString("{\"hello\": \n[null]}"); - EXPECT_THROW(json->getObjectArray("hello").at(0)->getDouble("foo"), Exception); - } - - { - ObjectSharedPtr json = Factory::loadFromString("{}"); - EXPECT_THROW((void)json->getObjectArray("hello").empty(), Exception); - } - - { - ObjectSharedPtr json = Factory::loadFromString("{}"); - EXPECT_TRUE(json->getObjectArray("hello", true).empty()); - } -} - -TEST_F(JsonLoaderTest, Integer) { - { - ObjectSharedPtr json = - Factory::loadFromString("{\"max\":9223372036854775807, \"min\":-9223372036854775808}"); - EXPECT_EQ(std::numeric_limits::max(), json->getInteger("max")); - EXPECT_EQ(std::numeric_limits::min(), json->getInteger("min")); - } - { - EXPECT_THROW(Factory::loadFromString("{\"val\":9223372036854775808}"), EnvoyException); - - // I believe this is a bug with rapidjson. - // It silently eats numbers below min int64_t with no exception. - // Fail when reading key instead of on parse. - ObjectSharedPtr json = Factory::loadFromString("{\"val\":-9223372036854775809}"); - EXPECT_THROW(json->getInteger("val"), EnvoyException); - } -} - -TEST_F(JsonLoaderTest, Double) { - { - ObjectSharedPtr json = Factory::loadFromString("{\"value1\": 10.5, \"value2\": -12.3}"); - EXPECT_EQ(10.5, json->getDouble("value1")); - EXPECT_EQ(-12.3, json->getDouble("value2")); - } - { - ObjectSharedPtr json = Factory::loadFromString("{\"foo\": 13.22}"); - EXPECT_EQ(13.22, json->getDouble("foo", 0)); - EXPECT_EQ(0, json->getDouble("bar", 0)); - } - { - ObjectSharedPtr json = Factory::loadFromString("{\"foo\": \"bar\"}"); - EXPECT_THROW(json->getDouble("foo"), Exception); - } -} - -TEST_F(JsonLoaderTest, Hash) { - ObjectSharedPtr json1 = Factory::loadFromString("{\"value1\": 10.5, \"value2\": -12.3}"); - ObjectSharedPtr json2 = Factory::loadFromString("{\"value2\": -12.3, \"value1\": 10.5}"); - ObjectSharedPtr json3 = Factory::loadFromString(" { \"value2\": -12.3, \"value1\": 10.5} "); - ObjectSharedPtr json4 = Factory::loadFromString("{\"value1\": 10.5}"); - - // Objects with keys in different orders should be the same - EXPECT_EQ(json1->hash(), json2->hash()); - // Whitespace is ignored - EXPECT_EQ(json2->hash(), json3->hash()); - // Ensure different hash is computed for different objects - EXPECT_NE(json1->hash(), json4->hash()); -} - -TEST_F(JsonLoaderTest, Schema) { - { - std::string invalid_json_schema = R"EOF( - { - "properties": {"value1"} - } - )EOF"; - - std::string invalid_schema = R"EOF( - { - "properties" : { - "value1": {"type" : "faketype"} - } - } - )EOF"; - - std::string different_schema = R"EOF( - { - "properties" : { - "value1" : {"type" : "number"} - }, - "additionalProperties" : false - } - )EOF"; - - std::string valid_schema = R"EOF( - { - "properties": { - "value1": {"type" : "number"}, - "value2": {"type": "string"} - }, - "additionalProperties": false - } - )EOF"; - - std::string json_string = R"EOF( - { - "value1": 10, - "value2" : "test" - } - )EOF"; - - ObjectSharedPtr json = Factory::loadFromString(json_string); - EXPECT_THROW(json->validateSchema(invalid_json_schema), std::invalid_argument); - EXPECT_THROW(json->validateSchema(invalid_schema), Exception); - EXPECT_THROW(json->validateSchema(different_schema), Exception); - EXPECT_NO_THROW(json->validateSchema(valid_schema)); - } - - { - std::string json_string = R"EOF( - { - "value1": [false, 2.01, 3, null], - "value2" : "test" - } - )EOF"; - - std::string empty_schema = R"EOF({})EOF"; - - ObjectSharedPtr json = Factory::loadFromString(json_string); - EXPECT_NO_THROW(json->validateSchema(empty_schema)); - } -} - -TEST_F(JsonLoaderTest, NestedSchema) { - - std::string schema = R"EOF( - { - "properties": { - "value1": {"type" : "number"}, - "value2": {"type": "string"} - }, - "additionalProperties": false - } - )EOF"; - - std::string json_string = R"EOF( - { - "bar": "baz", - "foo": { - "value1": "should have been a number", - "value2" : "test" - } - } - )EOF"; - - ObjectSharedPtr json = Factory::loadFromString(json_string); - - EXPECT_THROW_WITH_MESSAGE(json->getObject("foo")->validateSchema(schema), Exception, - "JSON at lines 4-7 does not conform to schema.\n Invalid schema: " - "#/properties/value1\n Schema violation: type\n Offending document " - "key: #/value1"); -} - -TEST_F(JsonLoaderTest, MissingEnclosingDocument) { - - std::string json_string = R"EOF( - "listeners" : [ - { - "address": "tcp://127.0.0.1:1234", - "filters": [] - } - ] - )EOF"; - - EXPECT_THROW_WITH_MESSAGE(Factory::loadFromString(json_string), Exception, - "JSON supplied is not valid. Error(offset 14, line 2): Terminate " - "parsing due to Handler error.\n"); -} - -TEST_F(JsonLoaderTest, AsString) { - ObjectSharedPtr json = Factory::loadFromString("{\"name1\": \"value1\", \"name2\": true}"); - json->iterate([&](const std::string& key, const Json::Object& value) { - EXPECT_TRUE(key == "name1" || key == "name2"); - - if (key == "name1") { - EXPECT_EQ("value1", value.asString()); - } else { - EXPECT_THROW(value.asString(), Exception); - } - return true; - }); -} - -} // namespace -} // namespace Json -} // namespace Envoy From 9b74414df374dc48071c3278444887da6ed4aa22 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Thu, 7 Jan 2021 15:07:20 -0500 Subject: [PATCH 09/17] fix release date Signed-off-by: Asra Ali --- bazel/repository_locations.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 5f71d3ffad029..4ac46caaf3562 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -454,7 +454,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( # This will be a replacement for rapidJSON used in extensions and may also be a fast # replacement for protobuf JSON. use_category = ["controlplane", "dataplane_core"], - release_date = "2020-07-06", + release_date = "2020-08-06", cpe = "cpe:2.3:a:json_project:json:*", ), com_github_twitter_common_lang = dict( From 73c8f4d00c593a8da0376480d583174d30c902ff Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Mon, 11 Jan 2021 10:15:29 -0500 Subject: [PATCH 10/17] fix coverage and clang tidy Signed-off-by: Asra Ali --- include/envoy/json/json_object.h | 21 --------------------- source/common/json/json_internal.cc | 20 ++++++++------------ source/common/json/json_internal_legacy.cc | 7 ++----- source/common/json/rapidjson_loader.h | 22 ---------------------- 4 files changed, 10 insertions(+), 60 deletions(-) delete mode 100644 source/common/json/rapidjson_loader.h diff --git a/include/envoy/json/json_object.h b/include/envoy/json/json_object.h index 7df162540ff58..79ce0d9c97655 100644 --- a/include/envoy/json/json_object.h +++ b/include/envoy/json/json_object.h @@ -79,12 +79,6 @@ class Object { */ virtual ObjectSharedPtr getObject(const std::string& name, bool allow_empty = false) const PURE; - /** - * Determine if an object is null. - * @return bool is the object null? - */ - virtual bool isNull() const PURE; - /** * Determine if an object has type Object. * @return bool is the object an Object? @@ -181,21 +175,6 @@ class Object { */ virtual std::string asString() const PURE; - /** - * @return the value of the object as a boolean (where the object is a boolean). - */ - virtual bool asBoolean() const PURE; - - /** - * @return the value of the object as a double (where the object is a double). - */ - virtual double asDouble() const PURE; - - /** - * @return the value of the object as an integer (where the object is an integer). - */ - virtual int64_t asInteger() const PURE; - /** * @return the JSON string representation of the object. */ diff --git a/source/common/json/json_internal.cc b/source/common/json/json_internal.cc index 18c674ef57e3a..2857b84606efc 100644 --- a/source/common/json/json_internal.cc +++ b/source/common/json/json_internal.cc @@ -41,7 +41,6 @@ class Field : public Object { static FieldSharedPtr createArray() { return FieldSharedPtr{new Field(Type::Array)}; } static FieldSharedPtr createNull() { return FieldSharedPtr{new Field(Type::Null)}; } - bool isNull() const override { return type_ == Type::Null; } bool isArray() const override { return type_ == Type::Array; } bool isObject() const override { return type_ == Type::Object; } @@ -75,9 +74,6 @@ class Field : public Object { std::vector getStringArray(const std::string& name, bool allow_empty) const override; std::vector asObjectArray() const override; std::string asString() const override { return stringValue(); } - bool asBoolean() const override { return booleanValue(); } - double asDouble() const override { return doubleValue(); } - int64_t asInteger() const override { return integerValue(); } std::string asJsonString() const override; bool empty() const override; @@ -176,7 +172,7 @@ class Field : public Object { */ class ObjectHandler : public nlohmann::json_sax { public: - ObjectHandler() : state_(State::ExpectRoot) {} + ObjectHandler() = default; bool start_object(std::size_t) override; bool end_object() override; @@ -233,7 +229,7 @@ class ObjectHandler : public nlohmann::json_sax { ExpectArrayValueOrEndArray, ExpectFinished, }; - State state_; + State state_{State::ExpectRoot}; std::stack stack_; std::string key_; @@ -251,11 +247,11 @@ struct JsonContainer { }; struct JsonIterator { - using difference_type = std::ptrdiff_t; - using value_type = char; - using pointer = const char*; - using reference = const char&; - using iterator_category = std::input_iterator_tag; + using difference_type = std::ptrdiff_t; // NOLINT(readability-identifier-naming) + using value_type = char; // NOLINT(readability-identifier-naming) + using pointer = const char*; // NOLINT(readability-identifier-naming) + using reference = const char&; // NOLINT(readability-identifier-naming) + using iterator_category = std::input_iterator_tag; // NOLINT(readability-identifier-naming) JsonIterator& operator++() { ++ptr.data; @@ -315,7 +311,7 @@ void Field::buildJsonDocument(const Field& field, nlohmann::json& value) { } case Type::Object: { for (const auto& item : field.value_.object_value_) { - auto name = std::string(item.first.c_str()); + auto name = std::string(item.first); switch (item.second->type_) { case Type::Array: diff --git a/source/common/json/json_internal_legacy.cc b/source/common/json/json_internal_legacy.cc index e5b284b1004fc..21af6280e5276 100644 --- a/source/common/json/json_internal_legacy.cc +++ b/source/common/json/json_internal_legacy.cc @@ -47,7 +47,6 @@ class Field : public Object { static FieldSharedPtr createArray() { return FieldSharedPtr{new Field(Type::Array)}; } static FieldSharedPtr createNull() { return FieldSharedPtr{new Field(Type::Null)}; } - bool isNull() const override { return type_ == Type::Null; } bool isArray() const override { return type_ == Type::Array; } bool isObject() const override { return type_ == Type::Object; } @@ -81,9 +80,6 @@ class Field : public Object { std::vector getStringArray(const std::string& name, bool allow_empty) const override; std::vector asObjectArray() const override; std::string asString() const override { return stringValue(); } - bool asBoolean() const override { return booleanValue(); } - double asDouble() const override { return doubleValue(); } - int64_t asInteger() const override { return integerValue(); } std::string asJsonString() const override; bool empty() const override; @@ -519,7 +515,8 @@ void Field::validateSchema(const std::string& schema) const { } rapidjson::SchemaDocument schema_document_for_validator(schema_document); - rapidjson::SchemaValidator schema_validator(schema_document_for_validator); + rapidjson::GenericSchemaValidator + schema_validator(schema_document_for_validator); if (!asRapidJsonDocument().Accept(schema_validator)) { rapidjson::StringBuffer schema_string_buffer; diff --git a/source/common/json/rapidjson_loader.h b/source/common/json/rapidjson_loader.h deleted file mode 100644 index 6b50ccd2b6268..0000000000000 --- a/source/common/json/rapidjson_loader.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include -#include - -#include "envoy/json/json_object.h" - -namespace Envoy { -namespace Json { -namespace RapidJson { - -class Factory { -public: - /** - * Constructs a Json Object from a string. - */ - static ObjectSharedPtr loadFromString(const std::string& json); -}; - -} // namespace RapidJson -} // namespace Json -} // namespace Envoy From 6aed98c225bed36b43252e8b52ff83546cea8739 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Tue, 12 Jan 2021 10:03:36 -0500 Subject: [PATCH 11/17] add more test coverage Signed-off-by: Asra Ali --- source/common/json/json_internal_legacy.cc | 7 +------ test/common/json/json_loader_test.cc | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/source/common/json/json_internal_legacy.cc b/source/common/json/json_internal_legacy.cc index 21af6280e5276..6aac6a7c71ce8 100644 --- a/source/common/json/json_internal_legacy.cc +++ b/source/common/json/json_internal_legacy.cc @@ -320,12 +320,7 @@ rapidjson::Document Field::asRapidJsonDocument() const { return document; } -uint64_t Field::hash() const { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - asRapidJsonDocument().Accept(writer); - return HashUtil::xxHash64(buffer.GetString()); -} +uint64_t Field::hash() const { return HashUtil::xxHash64(asJsonString()); } bool Field::getBoolean(const std::string& name) const { checkType(Type::Object); diff --git a/test/common/json/json_loader_test.cc b/test/common/json/json_loader_test.cc index c33c5b0b45632..ee0c7e49787c7 100644 --- a/test/common/json/json_loader_test.cc +++ b/test/common/json/json_loader_test.cc @@ -248,6 +248,15 @@ TEST_F(JsonLoaderTest, Double) { } } +TEST_F(JsonLoaderTest, LoadArray) { + ObjectSharedPtr json1 = Factory::loadFromString("[1.11, 22, \"cat\"]"); + ObjectSharedPtr json2 = Factory::loadFromString("[22, \"cat\", 1.11]"); + + // Array values in different orders will not be the same. + EXPECT_NE(json1->asJsonString(), json2->asJsonString()); + EXPECT_NE(json1->hash(), json2->hash()); +} + TEST_F(JsonLoaderTest, Hash) { ObjectSharedPtr json1 = Factory::loadFromString("{\"value1\": 10.5, \"value2\": -12.3}"); ObjectSharedPtr json2 = Factory::loadFromString("{\"value2\": -12.3, \"value1\": 10.5}"); @@ -260,6 +269,11 @@ TEST_F(JsonLoaderTest, Hash) { EXPECT_EQ(json2->hash(), json3->hash()); // Ensure different hash is computed for different objects EXPECT_NE(json1->hash(), json4->hash()); + + // Nested objects with keys in different orders should be the same. + ObjectSharedPtr json5 = Factory::loadFromString("{\"value1\": {\"a\": true, \"b\": null}}"); + ObjectSharedPtr json6 = Factory::loadFromString("{\"value1\": {\"b\": null, \"a\": true}}"); + EXPECT_EQ(json5->hash(), json6->hash()); } TEST_F(JsonLoaderTest, Schema) { From 000de4fdb192a4ec5fa8a33dcfa42df53a9f71b0 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Tue, 12 Jan 2021 13:24:59 -0500 Subject: [PATCH 12/17] fix release notes Signed-off-by: Asra Ali --- docs/root/version_history/current.rst | 41 --------------------------- 1 file changed, 41 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 23573f284c488..175d326dd05e1 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -19,48 +19,7 @@ Removed Config or Runtime New Features ------------ -* compression: the :ref:`compressor ` filter adds support for compressing request payloads. Its configuration is unified with the :ref:`decompressor ` filter with two new fields for different directions - :ref:`requests ` and :ref:`responses `. The latter deprecates the old response-specific fields and, if used, roots the response-specific stats in `.compressor...response.*` instead of `.compressor...*`. -* config: added ability to flush stats when the admin's :ref:`/stats endpoint ` is hit instead of on a timer via :ref:`stats_flush_on_admin `. -* config: added new runtime feature `envoy.features.enable_all_deprecated_features` that allows the use of all deprecated features. -* crash support: added the ability to dump L4 connection data on crash. -* formatter: added new :ref:`text_format_source ` field to support format strings both inline and from a file. -* formatter: added support for custom date formatting to :ref:`%DOWNSTREAM_PEER_CERT_V_START% ` and :ref:`%DOWNSTREAM_PEER_CERT_V_END% `, similar to :ref:`%START_TIME% `. -* grpc: implemented header value syntax support when defining :ref:`initial metadata ` for gRPC-based `ext_authz` :ref:`HTTP ` and :ref:`network ` filters, and :ref:`ratelimit ` filters. -* grpc-json: added support for configuring :ref:`unescaping behavior ` for path components. -* hds: added support for delta updates in the :ref:`HealthCheckSpecifier `, making only the Endpoints and Health Checkers that changed be reconstructed on receiving a new message, rather than the entire HDS. -* health_check: added option to use :ref:`no_traffic_healthy_interval ` which allows a different no traffic interval when the host is healthy. -* http: added HCM :ref:`timeout config field ` to control how long a downstream has to finish sending headers before the stream is cancelled. -* http: added frame flood and abuse checks to the upstream HTTP/2 codec. This check is off by default and can be enabled by setting the `envoy.reloadable_features.upstream_http2_flood_checks` runtime key to true. -* http: added :ref:`stripping any port from host header ` support. -* http: clusters now support selecting HTTP/1 or HTTP/2 based on ALPN, configurable via :ref:`alpn_config ` in the :ref:`http_protocol_options ` message. * json: introduced new JSON parser (https://github.com/nlohmann/json) to replace RapidJSON. The new parser is enabled by default. To revert to the legacy RapidJSON parser, enable the runtime feature `envoy.reloadable_features.legacy_json`. -* jwt_authn: added support for :ref:`per-route config `. -* jwt_authn: changed config field :ref:`issuer ` to be optional to comply with JWT `RFC `_ requirements. -* kill_request: added new :ref:`HTTP kill request filter `. -* listener: added an optional :ref:`default filter chain `. If this field is supplied, and none of the :ref:`filter_chains ` matches, this default filter chain is used to serve the connection. -* listener: added back the :ref:`use_original_dst field `. -* listener: added the :ref:`Listener.bind_to_port field `. -* log: added a new custom flag ``%_`` to the log pattern to print the actual message to log, but with escaped newlines. -* lua: added `downstreamDirectRemoteAddress()` and `downstreamLocalAddress()` APIs to :ref:`streamInfo() `. -* mongo_proxy: the list of commands to produce metrics for is now :ref:`configurable `. -* network: added a :ref:`timeout ` for incoming connections completing transport-level negotiation, including TLS and ALTS hanshakes. -* overload: add :ref:`envoy.overload_actions.reduce_timeouts ` overload action to enable scaling timeouts down with load. Scaling support :ref:`is limited ` to the HTTP connection and stream idle timeouts. -* ratelimit: added support for use of various :ref:`metadata ` as a ratelimit action. -* ratelimit: added :ref:`disable_x_envoy_ratelimited_header ` option to disable `X-Envoy-RateLimited` header. -* ratelimit: added :ref:`body ` field to support custom response bodies for non-OK responses from the external ratelimit service. -* router: added support for regex rewrites during HTTP redirects using :ref:`regex_rewrite `. -* sds: improved support for atomic :ref:`key rotations ` and added configurable rotation triggers for - :ref:`TlsCertificate ` and - :ref:`CertificateValidationContext `. -* signal: added an extension point for custom actions to run on the thread that has encountered a fatal error. Actions are configurable via :ref:`fatal_actions `. -* start_tls: :ref:`transport socket` which starts in clear-text but may programatically be converted to use tls. -* tcp: added a new :ref:`envoy.overload_actions.reject_incoming_connections ` action to reject incoming TCP connections. -* thrift_proxy: added a new :ref: `payload_passthrough ` option to skip decoding body in the Thrift message. -* tls: added support for RSA certificates with 4096-bit keys in FIPS mode. -* tracing: added SkyWalking tracer. -* tracing: added support for setting the hostname used when sending spans to a Zipkin collector using the :ref:`collector_hostname ` field. -* xds: added support for resource TTLs. A TTL is specified on the :ref:`Resource `. For SotW, a :ref:`Resource ` can be embedded - in the list of resources to specify the TTL. * tcp_proxy: add support for converting raw TCP streams into HTTP/1.1 CONNECT requests. See :ref:`upgrade documentation ` for details. Deprecated From 644c35f93585177cbf8ea9b46ce4fed973d00558 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Wed, 13 Jan 2021 11:12:27 -0500 Subject: [PATCH 13/17] fix release notes merge Signed-off-by: Asra Ali --- docs/root/version_history/current.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 6270c1ecc4298..9c9c3a4ac2187 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -30,8 +30,9 @@ Removed Config or Runtime New Features ------------ -* json: introduced new JSON parser (https://github.com/nlohmann/json) to replace RapidJSON. The new parser is enabled by default. To revert to the legacy RapidJSON parser, enable the runtime feature `envoy.reloadable_features.legacy_json`. + * access log: added the :ref:`formatters ` extension point for custom formatters (command operators). +* json: introduced new JSON parser (https://github.com/nlohmann/json) to replace RapidJSON. The new parser is enabled by default. To revert to the legacy RapidJSON parser, enable the runtime feature `envoy.reloadable_features.legacy_json`. * tcp_proxy: add support for converting raw TCP streams into HTTP/1.1 CONNECT requests. See :ref:`upgrade documentation ` for details. Deprecated From a6e0c41c7113ffc9c9bbd2dd214801bdd6b55e5c Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Wed, 20 Jan 2021 10:20:50 -0500 Subject: [PATCH 14/17] merge rel notes Signed-off-by: Asra Ali --- docs/root/version_history/current.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 2188f5353729a..9b4eb63918332 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -40,11 +40,11 @@ New Features ------------ * access log: added the :ref:`formatters ` extension point for custom formatters (command operators). -* json: introduced new JSON parser (https://github.com/nlohmann/json) to replace RapidJSON. The new parser is enabled by default. To revert to the legacy RapidJSON parser, enable the runtime feature `envoy.reloadable_features.legacy_json`. * access log: support command operator: %REQUEST_HEADERS_BYTES%, %RESPONSE_HEADERS_BYTES% and %RESPONSE_TRAILERS_BYTES%. * dispatcher: supports a stack of `Envoy::ScopeTrackedObject` instead of a single tracked object. This will allow Envoy to dump more debug information on crash. * http: added support for :ref:`:ref:`preconnecting `. Preconnecting is off by default, but recommended for clusters serving latency-sensitive traffic, especially if using HTTP/1.1. * http: change frame flood and abuse checks to the upstream HTTP/2 codec to ON by default. It can be disabled by setting the `envoy.reloadable_features.upstream_http2_flood_checks` runtime key to false. +* json: introduced new JSON parser (https://github.com/nlohmann/json) to replace RapidJSON. The new parser is enabled by default. To revert to the legacy RapidJSON parser, enable the runtime feature `envoy.reloadable_features.legacy_json`. * tcp_proxy: add support for converting raw TCP streams into HTTP/1.1 CONNECT requests. See :ref:`upgrade documentation ` for details. Deprecated From 2813ba43076d471489afc6ba08f18f6507e57568 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Wed, 20 Jan 2021 10:30:23 -0500 Subject: [PATCH 15/17] remove log Signed-off-by: Asra Ali --- source/common/json/json_internal.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/json/json_internal.cc b/source/common/json/json_internal.cc index 2857b84606efc..39bfe5d07e655 100644 --- a/source/common/json/json_internal.cc +++ b/source/common/json/json_internal.cc @@ -204,7 +204,6 @@ class ObjectHandler : public nlohmann::json_sax { // Extract position information (line, column, token) in the error. auto start = error.find("line"); auto npos = error.find(":"); - ENVOY_LOG_MISC(info, "ERROR {}", error); error_position_ = absl::StrCat(error.substr(start, npos - start), ", token ", token); // Extract portion after ":" to get error string. error_ = error.substr(npos + 2); From a914a3b66c2562f7da1637fdf3ca7f5ffe602108 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Mon, 1 Feb 2021 12:47:35 -0500 Subject: [PATCH 16/17] address comments Signed-off-by: Asra Ali --- docs/root/version_history/current.rst | 2 +- source/common/json/json_internal.cc | 38 +++++++++++++--------- source/common/json/json_internal_legacy.cc | 12 +++---- source/common/json/json_loader.cc | 2 +- source/common/runtime/runtime_features.cc | 3 +- test/common/json/BUILD | 2 +- test/common/json/json_loader_test.cc | 6 ++-- 7 files changed, 34 insertions(+), 31 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 73ad76f92d154..97304135f9ec3 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -60,7 +60,7 @@ New Features * grpc_json_transcoder: filter can now be configured on per-route/per-vhost level as well. Leaving empty list of services in the filter configuration disables transcoding on the specific route. * http: added support for :ref:`:ref:`preconnecting `. Preconnecting is off by default, but recommended for clusters serving latency-sensitive traffic, especially if using HTTP/1.1. * http: change frame flood and abuse checks to the upstream HTTP/2 codec to ON by default. It can be disabled by setting the `envoy.reloadable_features.upstream_http2_flood_checks` runtime key to false. -* json: introduced new JSON parser (https://github.com/nlohmann/json) to replace RapidJSON. The new parser is enabled by default. To revert to the legacy RapidJSON parser, enable the runtime feature `envoy.reloadable_features.legacy_json`. +* json: introduced new JSON parser (https://github.com/nlohmann/json) to replace RapidJSON. The new parser is disabled by default. To test the new RapidJSON parser, enable the runtime feature `envoy.reloadable_features.remove_legacy_json`. * overload: add support for scaling :ref:`transport connection timeouts`. This can be used to reduce the TLS handshake timeout in response to overload. * server: added :ref:`fips_mode ` statistic. * tcp_proxy: add support for converting raw TCP streams into HTTP/1.1 CONNECT requests. See :ref:`upgrade documentation ` for details. diff --git a/source/common/json/json_internal.cc b/source/common/json/json_internal.cc index 39bfe5d07e655..d26c62d1e163c 100644 --- a/source/common/json/json_internal.cc +++ b/source/common/json/json_internal.cc @@ -198,15 +198,25 @@ class ObjectHandler : public nlohmann::json_sax { bool binary(binary_t&) override { return false; } bool parse_error(std::size_t, const std::string& token, const nlohmann::detail::exception& ex) override { - // Errors are formatted like "[json.exception.parse_error.101] parse error at line x, column y: - // error string." + // Errors are formatted like "[json.exception.parse_error.101] parse error: explanatory string." + // or "[json.exception.parse_error.101] parser error at (position): explanatory string.". + // https://json.nlohmann.me/home/exceptions/#parse-errors absl::string_view error = ex.what(); - // Extract position information (line, column, token) in the error. - auto start = error.find("line"); - auto npos = error.find(":"); - error_position_ = absl::StrCat(error.substr(start, npos - start), ", token ", token); - // Extract portion after ":" to get error string. - error_ = error.substr(npos + 2); + // Colon will always exist in the parse error. + auto end = error.find(": "); + if (end == std::string::npos) { + ENVOY_BUG(false, "Error string not present. Check nlohmann/json " + "documentation in case error string changed."); + } else { + // Extract portion after ": " to get error string. + error_ = error.substr(end + 2); + // Extract position information if present. + auto start = error.find("at "); + if (start != std::string::npos && (start + 3) < end) { + start += 3; + error_position_ = absl::StrCat(error.substr(start, end - start), ", token ", token); + } + } return false; } @@ -370,9 +380,8 @@ bool Field::getBoolean(const std::string& name, bool default_value) const { auto value_itr = value_.object_value_.find(name); if (value_itr != value_.object_value_.end()) { return getBoolean(name); - } else { - return default_value; } + return default_value; } double Field::getDouble(const std::string& name) const { @@ -390,9 +399,8 @@ double Field::getDouble(const std::string& name, double default_value) const { auto value_itr = value_.object_value_.find(name); if (value_itr != value_.object_value_.end()) { return getDouble(name); - } else { - return default_value; } + return default_value; } int64_t Field::getInteger(const std::string& name) const { @@ -410,9 +418,8 @@ int64_t Field::getInteger(const std::string& name, int64_t default_value) const auto value_itr = value_.object_value_.find(name); if (value_itr != value_.object_value_.end()) { return getInteger(name); - } else { - return default_value; } + return default_value; } ObjectSharedPtr Field::getObject(const std::string& name, bool allow_empty) const { @@ -464,9 +471,8 @@ std::string Field::getString(const std::string& name, const std::string& default auto value_itr = value_.object_value_.find(name); if (value_itr != value_.object_value_.end()) { return getString(name); - } else { - return default_value; } + return default_value; } std::vector Field::getStringArray(const std::string& name, bool allow_empty) const { diff --git a/source/common/json/json_internal_legacy.cc b/source/common/json/json_internal_legacy.cc index 6aac6a7c71ce8..c323cf75810c7 100644 --- a/source/common/json/json_internal_legacy.cc +++ b/source/common/json/json_internal_legacy.cc @@ -337,9 +337,8 @@ bool Field::getBoolean(const std::string& name, bool default_value) const { auto value_itr = value_.object_value_.find(name); if (value_itr != value_.object_value_.end()) { return getBoolean(name); - } else { - return default_value; } + return default_value; } double Field::getDouble(const std::string& name) const { @@ -357,9 +356,8 @@ double Field::getDouble(const std::string& name, double default_value) const { auto value_itr = value_.object_value_.find(name); if (value_itr != value_.object_value_.end()) { return getDouble(name); - } else { - return default_value; } + return default_value; } int64_t Field::getInteger(const std::string& name) const { @@ -377,9 +375,8 @@ int64_t Field::getInteger(const std::string& name, int64_t default_value) const auto value_itr = value_.object_value_.find(name); if (value_itr != value_.object_value_.end()) { return getInteger(name); - } else { - return default_value; } + return default_value; } ObjectSharedPtr Field::getObject(const std::string& name, bool allow_empty) const { @@ -431,9 +428,8 @@ std::string Field::getString(const std::string& name, const std::string& default auto value_itr = value_.object_value_.find(name); if (value_itr != value_.object_value_.end()) { return getString(name); - } else { - return default_value; } + return default_value; } std::vector Field::getStringArray(const std::string& name, bool allow_empty) const { diff --git a/source/common/json/json_loader.cc b/source/common/json/json_loader.cc index c0a1add871ae7..0e4f3669a1b85 100644 --- a/source/common/json/json_loader.cc +++ b/source/common/json/json_loader.cc @@ -8,7 +8,7 @@ namespace Envoy { namespace Json { ObjectSharedPtr Factory::loadFromString(const std::string& json) { - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_rapidjson")) { + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_legacy_json")) { return Nlohmann::Factory::loadFromString(json); } return RapidJson::Factory::loadFromString(json); diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 63988196702e5..c165f1322e76a 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -79,7 +79,6 @@ constexpr const char* runtime_features[] = { "envoy.reloadable_features.preserve_query_string_in_path_redirects", "envoy.reloadable_features.preserve_upstream_date", "envoy.reloadable_features.remove_forked_chromium_url", - "envoy.reloadable_features.remove_rapidjson", "envoy.reloadable_features.require_ocsp_response_for_must_staple_certs", "envoy.reloadable_features.stop_faking_paths", "envoy.reloadable_features.strict_1xx_and_204_response_headers", @@ -109,6 +108,8 @@ constexpr const char* disabled_runtime_features[] = { "envoy.reloadable_features.enable_type_url_downgrade_and_upgrade", // TODO(alyssawilk) flip true after the release. "envoy.reloadable_features.new_tcp_connection_pool", + // TODO(asraa) flip to true in a separate PR to enable the new JSON by default. + "envoy.reloadable_features.remove_legacy_json", // Sentinel and test flag. "envoy.reloadable_features.test_feature_false", }; diff --git a/test/common/json/BUILD b/test/common/json/BUILD index f3187a37c606e..83bf67d96ad05 100644 --- a/test/common/json/BUILD +++ b/test/common/json/BUILD @@ -30,12 +30,12 @@ JSON_TEST_DEPS = [ envoy_cc_test( name = "json_loader_test", srcs = ["json_loader_test.cc"], + args = ["--runtime-feature-override-for-tests=envoy.reloadable_features.remove_legacy_json"], deps = JSON_TEST_DEPS, ) envoy_cc_test( name = "json_loader_legacy_test", srcs = ["json_loader_test.cc"], - args = ["--runtime-feature-disable-for-tests=envoy.reloadable_features.remove_rapidjson"], deps = JSON_TEST_DEPS, ) diff --git a/test/common/json/json_loader_test.cc b/test/common/json/json_loader_test.cc index ee0c7e49787c7..e26fa010c8353 100644 --- a/test/common/json/json_loader_test.cc +++ b/test/common/json/json_loader_test.cc @@ -71,7 +71,7 @@ TEST_F(JsonLoaderTest, Basic) { } { - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_rapidjson")) { + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_legacy_json")) { EXPECT_THROW_WITH_MESSAGE(Factory::loadFromString("{\"hello\": \n\n\"world\""), Exception, "JSON supplied is not valid. Error(line 3, column 8, token " "\"world\"): syntax error while " @@ -293,7 +293,7 @@ TEST_F(JsonLoaderTest, Schema) { )EOF"; ObjectSharedPtr json = Factory::loadFromString(json_string); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_rapidjson")) { + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_legacy_json")) { EXPECT_THROW_WITH_MESSAGE(json->validateSchema(invalid_schema), Exception, "not implemented"); } else { EXPECT_THROW_WITH_MESSAGE( @@ -313,7 +313,7 @@ TEST_F(JsonLoaderTest, MissingEnclosingDocument) { } ] )EOF"; - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_rapidjson")) { + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_legacy_json")) { EXPECT_THROW_WITH_MESSAGE( Factory::loadFromString(json_string), Exception, "JSON supplied is not valid. Error(line 2, column 15, token \"listeners\" :): syntax error " From c717f234f18e6c31eb3af252eed1d61f71002758 Mon Sep 17 00:00:00 2001 From: Asra Ali Date: Thu, 4 Feb 2021 15:00:27 -0500 Subject: [PATCH 17/17] revert header formatter expected errors Signed-off-by: Asra Ali --- test/common/router/header_formatter_test.cc | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index f37b1ce288bc4..52c5b56286247 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -694,8 +694,7 @@ TEST_F(StreamInfoHeaderFormatterTest, WrongFormatOnUpstreamMetadataVariable) { StreamInfoHeaderFormatter("UPSTREAM_METADATA(abcd)", false), EnvoyException, "Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " "...]), actual format UPSTREAM_METADATA(abcd), because JSON supplied is not valid. " - "Error(line 1, column 1, token a): syntax error while parsing value - invalid literal; last " - "read: 'a'\n"); + "Error(offset 0, line 1): Invalid value.\n"); // No parameters. EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA", false), EnvoyException, @@ -706,9 +705,8 @@ TEST_F(StreamInfoHeaderFormatterTest, WrongFormatOnUpstreamMetadataVariable) { EXPECT_THROW_WITH_MESSAGE( StreamInfoHeaderFormatter("UPSTREAM_METADATA()", false), EnvoyException, "Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " - "...]), actual format UPSTREAM_METADATA(), because JSON supplied is not valid. Error(line 1, " - "column 1, token ): syntax error while parsing value - unexpected end of input; expected " - "'[', '{', or a literal\n"); + "...]), actual format UPSTREAM_METADATA(), because JSON supplied is not valid. Error(offset " + "0, line 1): The document is empty.\n"); // One parameter. EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA([\"ns\"])", false), @@ -748,8 +746,7 @@ TEST_F(StreamInfoHeaderFormatterTest, WrongFormatOnUpstreamMetadataVariable) { StreamInfoHeaderFormatter("UPSTREAM_METADATA([\"a\", \"\\unothex\"])", false), EnvoyException, "Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " "...]), actual format UPSTREAM_METADATA([\"a\", \"\\unothex\"]), because JSON supplied is " - "not valid. Error(line 1, column 10, token \"\\un): syntax error while parsing value - " - "invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\un'\n"); + "not valid. Error(offset 7, line 1): Incorrect hex digit after \\u escape in string.\n"); // Non-array parameters. EXPECT_THROW_WITH_MESSAGE( @@ -830,14 +827,12 @@ TEST(HeaderParserTest, TestParseInternal) { {}, {"Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " "...]), actual format UPSTREAM_METADATA(no array), because JSON supplied is not valid. " - "Error(line 1, column 2, token no): syntax error while parsing value - invalid literal; " - "last read: 'no'\n"}}, + "Error(offset 1, line 1): Invalid value.\n"}}, {"%UPSTREAM_METADATA( no array)%", {}, {"Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " "...]), actual format UPSTREAM_METADATA( no array), because JSON supplied is not valid. " - "Error(line 1, column 3, token no): syntax error while parsing value - invalid literal; " - "last read: ' no'\n"}}, + "Error(offset 2, line 1): Invalid value.\n"}}, {"%UPSTREAM_METADATA([\"unterminated array\")%", {}, {"Invalid header configuration. Expecting ',', ']', or whitespace after "