diff --git a/bazel/external/json.BUILD b/bazel/external/json.BUILD new file mode 100644 index 0000000000000..8a0b359d0fc87 --- /dev/null +++ b/bazel/external/json.BUILD @@ -0,0 +1,21 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +licenses(["notice"]) # Apache 2 + +cc_library( + name = "nlohmann_json_lib", + hdrs = glob([ + "include/nlohmann/*.hpp", + "include/nlohmann/**/*.hpp", + "include/nlohmann/*/*/*.hpp", + ]), + includes = ["external/nlohmann_json_lib"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "json", + includes = ["include"], + visibility = ["//visibility:public"], + deps = [":nlohmann_json_lib"], +) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index fbee3008700c3..9b13ab3ac4692 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -144,6 +144,7 @@ def envoy_dependencies(skip_targets = []): _com_github_nghttp2_nghttp2() _com_github_nodejs_http_parser() _com_github_tencent_rapidjson() + _com_github_nlohmann_json() _com_google_absl() _com_google_googletest() _com_google_protobuf() @@ -428,6 +429,16 @@ def _com_github_tencent_rapidjson(): actual = "@com_github_tencent_rapidjson//:rapidjson", ) +def _com_github_nlohmann_json(): + external_http_archive( + name = "com_github_nlohmann_json", + build_file = "@envoy//bazel/external:json.BUILD", + ) + native.bind( + name = "json", + actual = "@com_github_nlohmann_json//:json", + ) + def _com_github_nodejs_http_parser(): external_http_archive( name = "com_github_nodejs_http_parser", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index b19b24a48c6b2..05fd9b8f7bdf3 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -443,6 +443,20 @@ REPOSITORY_LOCATIONS_SPEC = dict( release_date = "2019-12-03", cpe = "cpe:2.3:a:tencent:rapidjson:*", ), + com_github_nlohmann_json = dict( + project_name = "nlohmann JSON", + project_desc = "Fast JSON parser/generator for C++", + project_url = "https://nlohmann.github.io/json", + version = "3.9.1", + sha256 = "4cf0df69731494668bdd6460ed8cb269b68de9c19ad8c27abc24cd72605b2d5b", + strip_prefix = "json-{version}", + urls = ["https://github.com/nlohmann/json/archive/v{version}.tar.gz"], + # This will be a replacement for rapidJSON used in extensions and may also be a fast + # replacement for protobuf JSON. + use_category = ["controlplane", "dataplane_core"], + release_date = "2020-08-06", + cpe = "cpe:2.3:a:json_project:json:*", + ), com_github_twitter_common_lang = dict( project_name = "twitter.common.lang (Thrift)", project_desc = "twitter.common Python language and compatibility facilities", diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 1fae2645bce88..97304135f9ec3 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -52,6 +52,7 @@ Removed Config or Runtime New Features ------------ + * access log: added the :ref:`formatters ` extension point for custom formatters (command operators). * access log: support command operator: %REQUEST_HEADERS_BYTES%, %RESPONSE_HEADERS_BYTES% and %RESPONSE_TRAILERS_BYTES%. * dispatcher: supports a stack of `Envoy::ScopeTrackedObject` instead of a single tracked object. This will allow Envoy to dump more debug information on crash. @@ -59,6 +60,7 @@ New Features * grpc_json_transcoder: filter can now be configured on per-route/per-vhost level as well. Leaving empty list of services in the filter configuration disables transcoding on the specific route. * http: added support for :ref:`:ref:`preconnecting `. Preconnecting is off by default, but recommended for clusters serving latency-sensitive traffic, especially if using HTTP/1.1. * http: change frame flood and abuse checks to the upstream HTTP/2 codec to ON by default. It can be disabled by setting the `envoy.reloadable_features.upstream_http2_flood_checks` runtime key to false. +* json: introduced new JSON parser (https://github.com/nlohmann/json) to replace RapidJSON. The new parser is disabled by default. To test the new RapidJSON parser, enable the runtime feature `envoy.reloadable_features.remove_legacy_json`. * overload: add support for scaling :ref:`transport connection timeouts`. This can be used to reduce the TLS handshake timeout in response to overload. * server: added :ref:`fips_mode ` statistic. * tcp_proxy: add support for converting raw TCP streams into HTTP/1.1 CONNECT requests. See :ref:`upgrade documentation ` for details. diff --git a/include/envoy/json/json_object.h b/include/envoy/json/json_object.h index 7df162540ff58..79ce0d9c97655 100644 --- a/include/envoy/json/json_object.h +++ b/include/envoy/json/json_object.h @@ -79,12 +79,6 @@ class Object { */ virtual ObjectSharedPtr getObject(const std::string& name, bool allow_empty = false) const PURE; - /** - * Determine if an object is null. - * @return bool is the object null? - */ - virtual bool isNull() const PURE; - /** * Determine if an object has type Object. * @return bool is the object an Object? @@ -181,21 +175,6 @@ class Object { */ virtual std::string asString() const PURE; - /** - * @return the value of the object as a boolean (where the object is a boolean). - */ - virtual bool asBoolean() const PURE; - - /** - * @return the value of the object as a double (where the object is a double). - */ - virtual double asDouble() const PURE; - - /** - * @return the value of the object as an integer (where the object is an integer). - */ - virtual int64_t asInteger() const PURE; - /** * @return the JSON string representation of the object. */ diff --git a/source/common/json/BUILD b/source/common/json/BUILD index edda394d6949d..4cc634bc0d697 100644 --- a/source/common/json/BUILD +++ b/source/common/json/BUILD @@ -9,9 +9,9 @@ licenses(["notice"]) # Apache 2 envoy_package() envoy_cc_library( - name = "json_loader_lib", - srcs = ["json_loader.cc"], - hdrs = ["json_loader.h"], + name = "json_internal_legacy_lib", + srcs = ["json_internal_legacy.cc"], + hdrs = ["json_internal_legacy.h"], external_deps = [ "rapidjson", ], @@ -23,3 +23,31 @@ envoy_cc_library( "//source/common/protobuf:utility_lib", ], ) + +envoy_cc_library( + name = "json_internal_lib", + srcs = ["json_internal.cc"], + hdrs = ["json_internal.h"], + external_deps = [ + "json", + ], + deps = [ + "//include/envoy/json:json_object_interface", + "//source/common/common:assert_lib", + "//source/common/common:hash_lib", + "//source/common/common:utility_lib", + "//source/common/protobuf:utility_lib", + ], +) + +envoy_cc_library( + name = "json_loader_lib", + srcs = ["json_loader.cc"], + hdrs = ["json_loader.h"], + deps = [ + ":json_internal_legacy_lib", + ":json_internal_lib", + "//include/envoy/json:json_object_interface", + "//source/common/runtime:runtime_features_lib", + ], +) diff --git a/source/common/json/json_internal.cc b/source/common/json/json_internal.cc new file mode 100644 index 0000000000000..d26c62d1e163c --- /dev/null +++ b/source/common/json/json_internal.cc @@ -0,0 +1,674 @@ +#include "common/json/json_internal.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/common/assert.h" +#include "common/common/fmt.h" +#include "common/common/hash.h" +#include "common/common/utility.h" +#include "common/protobuf/utility.h" + +// Do not let nlohmann/json leak outside of this file. +#include "include/nlohmann/json.hpp" + +#include "absl/strings/match.h" + +namespace Envoy { +namespace Json { +namespace Nlohmann { + +namespace { +/** + * Internal representation of Object. + */ +class Field; +using FieldSharedPtr = std::shared_ptr; + +class Field : public Object { +public: + void setLineNumberStart(uint64_t line_number) { line_number_start_ = line_number; } + void setLineNumberEnd(uint64_t line_number) { line_number_end_ = line_number; } + + // Container factories for handler. + static FieldSharedPtr createObject() { return FieldSharedPtr{new Field(Type::Object)}; } + static FieldSharedPtr createArray() { return FieldSharedPtr{new Field(Type::Array)}; } + static FieldSharedPtr createNull() { return FieldSharedPtr{new Field(Type::Null)}; } + + bool isArray() const override { return type_ == Type::Array; } + bool isObject() const override { return type_ == Type::Object; } + + // Value factory. + template static FieldSharedPtr createValue(T value) { + return FieldSharedPtr{new Field(value)}; // NOLINT(modernize-make-shared) + } + + void append(FieldSharedPtr field_ptr) { + checkType(Type::Array); + value_.array_value_.push_back(field_ptr); + } + void insert(const std::string& key, FieldSharedPtr field_ptr) { + checkType(Type::Object); + value_.object_value_[key] = field_ptr; + } + + uint64_t hash() const override; + + bool getBoolean(const std::string& name) const override; + bool getBoolean(const std::string& name, bool default_value) const override; + double getDouble(const std::string& name) const override; + double getDouble(const std::string& name, double default_value) const override; + int64_t getInteger(const std::string& name) const override; + int64_t getInteger(const std::string& name, int64_t default_value) const override; + ObjectSharedPtr getObject(const std::string& name, bool allow_empty) const override; + std::vector getObjectArray(const std::string& name, + bool allow_empty) const override; + std::string getString(const std::string& name) const override; + std::string getString(const std::string& name, const std::string& default_value) const override; + std::vector getStringArray(const std::string& name, bool allow_empty) const override; + std::vector asObjectArray() const override; + std::string asString() const override { return stringValue(); } + std::string asJsonString() const override; + + bool empty() const override; + bool hasObject(const std::string& name) const override; + void iterate(const ObjectCallback& callback) const override; + void validateSchema(const std::string&) const override; + +private: + enum class Type { + Array, + Boolean, + Double, + Integer, + Null, + Object, + String, + }; + static const char* typeAsString(Type t) { + switch (t) { + case Type::Array: + return "Array"; + case Type::Boolean: + return "Boolean"; + case Type::Double: + return "Double"; + case Type::Integer: + return "Integer"; + case Type::Null: + return "Null"; + case Type::Object: + return "Object"; + case Type::String: + return "String"; + } + + NOT_REACHED_GCOVR_EXCL_LINE; + } + + struct Value { + std::vector array_value_; + bool boolean_value_; + double double_value_; + int64_t integer_value_; + std::map object_value_; + std::string string_value_; + }; + + explicit Field(Type type) : type_(type) {} + explicit Field(const std::string& value) : type_(Type::String) { value_.string_value_ = value; } + explicit Field(int64_t value) : type_(Type::Integer) { value_.integer_value_ = value; } + explicit Field(double value) : type_(Type::Double) { value_.double_value_ = value; } + explicit Field(bool value) : type_(Type::Boolean) { value_.boolean_value_ = value; } + + bool isType(Type type) const { return type == type_; } + void checkType(Type type) const { + if (!isType(type)) { + throw Exception(fmt::format( + "JSON field from line {} accessed with type '{}' does not match actual type '{}'.", + line_number_start_, typeAsString(type), typeAsString(type_))); + } + } + + // Value return type functions. + std::string stringValue() const { + checkType(Type::String); + return value_.string_value_; + } + std::vector arrayValue() const { + checkType(Type::Array); + return value_.array_value_; + } + bool booleanValue() const { + checkType(Type::Boolean); + return value_.boolean_value_; + } + double doubleValue() const { + checkType(Type::Double); + return value_.double_value_; + } + int64_t integerValue() const { + checkType(Type::Integer); + return value_.integer_value_; + } + + nlohmann::json asJsonDocument() const; + static void buildJsonDocument(const Field& field, nlohmann::json& value); + + uint64_t line_number_start_ = 0; + uint64_t line_number_end_ = 0; + const Type type_; + Value value_; +}; + +/** + * Consume events from SAX callbacks to build JSON Field. + */ +class ObjectHandler : public nlohmann::json_sax { +public: + ObjectHandler() = default; + + bool start_object(std::size_t) override; + bool end_object() override; + bool key(std::string& val) override; + bool start_array(std::size_t) override; + bool end_array() override; + bool boolean(bool value) override { return handleValueEvent(Field::createValue(value)); } + bool number_integer(int64_t value) override { + return handleValueEvent(Field::createValue(static_cast(value))); + } + bool number_unsigned(uint64_t value) override { + if (value > static_cast(std::numeric_limits::max())) { + throw Exception(fmt::format("JSON value from line {} is larger than int64_t (not supported)", + line_number_)); + } + return handleValueEvent(Field::createValue(static_cast(value))); + } + bool number_float(double value, const std::string&) override { + return handleValueEvent(Field::createValue(value)); + } + bool null() override { return handleValueEvent(Field::createNull()); } + bool string(std::string& value) override { return handleValueEvent(Field::createValue(value)); } + bool binary(binary_t&) override { return false; } + bool parse_error(std::size_t, const std::string& token, + const nlohmann::detail::exception& ex) override { + // Errors are formatted like "[json.exception.parse_error.101] parse error: explanatory string." + // or "[json.exception.parse_error.101] parser error at (position): explanatory string.". + // https://json.nlohmann.me/home/exceptions/#parse-errors + absl::string_view error = ex.what(); + // Colon will always exist in the parse error. + auto end = error.find(": "); + if (end == std::string::npos) { + ENVOY_BUG(false, "Error string not present. Check nlohmann/json " + "documentation in case error string changed."); + } else { + // Extract portion after ": " to get error string. + error_ = error.substr(end + 2); + // Extract position information if present. + auto start = error.find("at "); + if (start != std::string::npos && (start + 3) < end) { + start += 3; + error_position_ = absl::StrCat(error.substr(start, end - start), ", token ", token); + } + } + return false; + } + + bool hasParseError() { return !error_.empty(); } + std::string getParseError() { return error_; } + std::string getErrorPosition() { return error_position_; } + + ObjectSharedPtr getRoot() { return root_; } + + int line_number_{1}; + +private: + bool handleValueEvent(FieldSharedPtr ptr); + + enum class State { + ExpectRoot, + ExpectKeyOrEndObject, + ExpectValueOrStartObjectArray, + ExpectArrayValueOrEndArray, + ExpectFinished, + }; + State state_{State::ExpectRoot}; + + std::stack stack_; + std::string key_; + + FieldSharedPtr root_; + + std::string error_; + std::string error_position_; +}; + +struct JsonContainer { + JsonContainer(const char* ch, ObjectHandler* handler) : data(ch), handler_(handler) {} + const char* data; + ObjectHandler* handler_; +}; + +struct JsonIterator { + using difference_type = std::ptrdiff_t; // NOLINT(readability-identifier-naming) + using value_type = char; // NOLINT(readability-identifier-naming) + using pointer = const char*; // NOLINT(readability-identifier-naming) + using reference = const char&; // NOLINT(readability-identifier-naming) + using iterator_category = std::input_iterator_tag; // NOLINT(readability-identifier-naming) + + JsonIterator& operator++() { + ++ptr.data; + return *this; + } + + bool operator!=(const JsonIterator& rhs) const { return rhs.ptr.data != ptr.data; } + + reference operator*() { + const char& ch = *(ptr.data); + if (ch == '\n') { + ptr.handler_->line_number_++; + } + return ch; + } + + JsonContainer ptr; +}; + +JsonIterator begin(const JsonContainer& c) { + return JsonIterator{JsonContainer(c.data, c.handler_)}; +} + +JsonIterator end(const JsonContainer& c) { + return JsonIterator{JsonContainer(c.data + strlen(c.data), c.handler_)}; +} + +void Field::buildJsonDocument(const Field& field, nlohmann::json& value) { + switch (field.type_) { + case Type::Array: { + for (const auto& element : field.value_.array_value_) { + switch (element->type_) { + case Type::Array: + case Type::Object: { + nlohmann::json nested_value; + buildJsonDocument(*element, nested_value); + value.push_back(nested_value); + break; + } + case Type::Boolean: + value.push_back(element->value_.boolean_value_); + break; + case Type::Double: + value.push_back(element->value_.double_value_); + break; + case Type::Integer: + value.push_back(element->value_.integer_value_); + break; + case Type::Null: + value.push_back(nlohmann::json::value_t::null); + break; + case Type::String: + value.push_back(element->value_.string_value_); + } + } + break; + } + case Type::Object: { + for (const auto& item : field.value_.object_value_) { + auto name = std::string(item.first); + + switch (item.second->type_) { + case Type::Array: + case Type::Object: { + nlohmann::json nested_value; + buildJsonDocument(*item.second, nested_value); + value.emplace(name, nested_value); + break; + } + case Type::Boolean: + value.emplace(name, item.second->value_.boolean_value_); + break; + case Type::Double: + value.emplace(name, item.second->value_.double_value_); + break; + case Type::Integer: + value.emplace(name, item.second->value_.integer_value_); + break; + case Type::Null: + value.emplace(name, nlohmann::json::value_t::null); + break; + case Type::String: + value.emplace(name, item.second->value_.string_value_); + break; + } + } + break; + } + case Type::Null: { + break; + } + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +nlohmann::json Field::asJsonDocument() const { + nlohmann::json j; + buildJsonDocument(*this, j); + return j; +} + +uint64_t Field::hash() const { return HashUtil::xxHash64(asJsonString()); } + +bool Field::getBoolean(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Boolean)) { + throw Exception(fmt::format("key '{}' missing or not a boolean from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->booleanValue(); +} + +bool Field::getBoolean(const std::string& name, bool default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getBoolean(name); + } + return default_value; +} + +double Field::getDouble(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Double)) { + throw Exception(fmt::format("key '{}' missing or not a double from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->doubleValue(); +} + +double Field::getDouble(const std::string& name, double default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getDouble(name); + } + return default_value; +} + +int64_t Field::getInteger(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Integer)) { + throw Exception(fmt::format("key '{}' missing or not an integer from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->integerValue(); +} + +int64_t Field::getInteger(const std::string& name, int64_t default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getInteger(name); + } + return default_value; +} + +ObjectSharedPtr Field::getObject(const std::string& name, bool allow_empty) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end()) { + if (allow_empty) { + return createObject(); + } else { + throw Exception(fmt::format("key '{}' missing from lines {}-{}", name, line_number_start_, + line_number_end_)); + } + } else if (!value_itr->second->isType(Type::Object)) { + throw Exception(fmt::format("key '{}' not an object from line {}", name, + value_itr->second->line_number_start_)); + } else { + return value_itr->second; + } +} + +std::vector Field::getObjectArray(const std::string& name, + bool allow_empty) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Array)) { + if (allow_empty && value_itr == value_.object_value_.end()) { + return std::vector(); + } + throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + + std::vector array_value = value_itr->second->arrayValue(); + return {array_value.begin(), array_value.end()}; +} + +std::string Field::getString(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::String)) { + throw Exception(fmt::format("key '{}' missing or not a string from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->stringValue(); +} + +std::string Field::getString(const std::string& name, const std::string& default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getString(name); + } + return default_value; +} + +std::vector Field::getStringArray(const std::string& name, bool allow_empty) const { + checkType(Type::Object); + std::vector string_array; + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Array)) { + if (allow_empty && value_itr == value_.object_value_.end()) { + return string_array; + } + throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + + std::vector array = value_itr->second->arrayValue(); + string_array.reserve(array.size()); + for (const auto& element : array) { + if (!element->isType(Type::String)) { + throw Exception(fmt::format("JSON array '{}' from line {} does not contain all strings", name, + line_number_start_)); + } + string_array.push_back(element->stringValue()); + } + + return string_array; +} + +std::vector Field::asObjectArray() const { + checkType(Type::Array); + return {value_.array_value_.begin(), value_.array_value_.end()}; +} + +std::string Field::asJsonString() const { + nlohmann::json j = asJsonDocument(); + return j.dump(); +} + +bool Field::empty() const { + if (isType(Type::Object)) { + return value_.object_value_.empty(); + } else if (isType(Type::Array)) { + return value_.array_value_.empty(); + } else { + throw Exception( + fmt::format("Json does not support empty() on types other than array and object")); + } +} + +bool Field::hasObject(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + return value_itr != value_.object_value_.end(); +} + +void Field::iterate(const ObjectCallback& callback) const { + checkType(Type::Object); + for (const auto& item : value_.object_value_) { + bool stop_iteration = !callback(item.first, *item.second); + if (stop_iteration) { + break; + } + } +} + +void Field::validateSchema(const std::string&) const { throw Exception("not implemented"); } + +bool ObjectHandler::start_object(std::size_t) { + FieldSharedPtr object = Field::createObject(); + object->setLineNumberStart(line_number_); + + switch (state_) { + case State::ExpectValueOrStartObjectArray: + stack_.top()->insert(key_, object); + stack_.push(object); + state_ = State::ExpectKeyOrEndObject; + return true; + case State::ExpectArrayValueOrEndArray: + stack_.top()->append(object); + stack_.push(object); + state_ = State::ExpectKeyOrEndObject; + return true; + case State::ExpectRoot: + root_ = object; + stack_.push(object); + state_ = State::ExpectKeyOrEndObject; + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::end_object() { + switch (state_) { + case State::ExpectKeyOrEndObject: + stack_.top()->setLineNumberEnd(line_number_); + stack_.pop(); + + if (stack_.empty()) { + state_ = State::ExpectFinished; + } else if (stack_.top()->isObject()) { + state_ = State::ExpectKeyOrEndObject; + } else if (stack_.top()->isArray()) { + state_ = State::ExpectArrayValueOrEndArray; + } + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::key(std::string& val) { + switch (state_) { + case State::ExpectKeyOrEndObject: + key_ = val; + state_ = State::ExpectValueOrStartObjectArray; + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::start_array(std::size_t) { + FieldSharedPtr array = Field::createArray(); + array->setLineNumberStart(line_number_); + + switch (state_) { + case State::ExpectValueOrStartObjectArray: + stack_.top()->insert(key_, array); + stack_.push(array); + state_ = State::ExpectArrayValueOrEndArray; + return true; + case State::ExpectArrayValueOrEndArray: + stack_.top()->append(array); + stack_.push(array); + return true; + case State::ExpectRoot: + root_ = array; + stack_.push(array); + state_ = State::ExpectArrayValueOrEndArray; + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::end_array() { + switch (state_) { + case State::ExpectArrayValueOrEndArray: + stack_.top()->setLineNumberEnd(line_number_); + stack_.pop(); + + if (stack_.empty()) { + state_ = State::ExpectFinished; + } else if (stack_.top()->isObject()) { + state_ = State::ExpectKeyOrEndObject; + } else if (stack_.top()->isArray()) { + state_ = State::ExpectArrayValueOrEndArray; + } + + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { + ptr->setLineNumberStart(line_number_); + + switch (state_) { + case State::ExpectValueOrStartObjectArray: + state_ = State::ExpectKeyOrEndObject; + stack_.top()->insert(key_, ptr); + return true; + case State::ExpectArrayValueOrEndArray: + stack_.top()->append(ptr); + return true; + default: + return true; + } +} + +} // namespace + +ObjectSharedPtr Factory::loadFromString(const std::string& json) { + ObjectHandler handler; + auto json_container = JsonContainer(json.data(), &handler); + + nlohmann::json::sax_parse(json_container, &handler); + + if (handler.hasParseError()) { + throw Exception(fmt::format("JSON supplied is not valid. Error({}): {}\n", + handler.getErrorPosition(), handler.getParseError())); + } + return handler.getRoot(); +} + +} // namespace Nlohmann +} // namespace Json +} // namespace Envoy diff --git a/source/common/json/json_internal.h b/source/common/json/json_internal.h new file mode 100644 index 0000000000000..de665333a1c09 --- /dev/null +++ b/source/common/json/json_internal.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include "envoy/json/json_object.h" + +namespace Envoy { +namespace Json { +namespace Nlohmann { + +class Factory { +public: + /** + * Constructs a Json Object from a string. + */ + static ObjectSharedPtr loadFromString(const std::string& json); +}; + +} // namespace Nlohmann +} // namespace Json +} // namespace Envoy diff --git a/source/common/json/json_internal_legacy.cc b/source/common/json/json_internal_legacy.cc new file mode 100644 index 0000000000000..c323cf75810c7 --- /dev/null +++ b/source/common/json/json_internal_legacy.cc @@ -0,0 +1,692 @@ +#include "common/json/json_internal_legacy.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/common/assert.h" +#include "common/common/fmt.h" +#include "common/common/hash.h" +#include "common/common/utility.h" +#include "common/protobuf/utility.h" + +// Do not let RapidJson leak outside of this file. +#include "rapidjson/document.h" +#include "rapidjson/error/en.h" +#include "rapidjson/reader.h" +#include "rapidjson/schema.h" +#include "rapidjson/stream.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +#include "absl/strings/match.h" + +namespace Envoy { +namespace Json { +namespace RapidJson { + +namespace { +/** + * Internal representation of Object. + */ +class Field; +using FieldSharedPtr = std::shared_ptr; + +class Field : public Object { +public: + void setLineNumberStart(uint64_t line_number) { line_number_start_ = line_number; } + void setLineNumberEnd(uint64_t line_number) { line_number_end_ = line_number; } + + // Container factories for handler. + static FieldSharedPtr createObject() { return FieldSharedPtr{new Field(Type::Object)}; } + static FieldSharedPtr createArray() { return FieldSharedPtr{new Field(Type::Array)}; } + static FieldSharedPtr createNull() { return FieldSharedPtr{new Field(Type::Null)}; } + + bool isArray() const override { return type_ == Type::Array; } + bool isObject() const override { return type_ == Type::Object; } + + // Value factory. + template static FieldSharedPtr createValue(T value) { + return FieldSharedPtr{new Field(value)}; // NOLINT(modernize-make-shared) + } + + void append(FieldSharedPtr field_ptr) { + checkType(Type::Array); + value_.array_value_.push_back(field_ptr); + } + void insert(const std::string& key, FieldSharedPtr field_ptr) { + checkType(Type::Object); + value_.object_value_[key] = field_ptr; + } + + uint64_t hash() const override; + + bool getBoolean(const std::string& name) const override; + bool getBoolean(const std::string& name, bool default_value) const override; + double getDouble(const std::string& name) const override; + double getDouble(const std::string& name, double default_value) const override; + int64_t getInteger(const std::string& name) const override; + int64_t getInteger(const std::string& name, int64_t default_value) const override; + ObjectSharedPtr getObject(const std::string& name, bool allow_empty) const override; + std::vector getObjectArray(const std::string& name, + bool allow_empty) const override; + std::string getString(const std::string& name) const override; + std::string getString(const std::string& name, const std::string& default_value) const override; + std::vector getStringArray(const std::string& name, bool allow_empty) const override; + std::vector asObjectArray() const override; + std::string asString() const override { return stringValue(); } + std::string asJsonString() const override; + + bool empty() const override; + bool hasObject(const std::string& name) const override; + void iterate(const ObjectCallback& callback) const override; + void validateSchema(const std::string& schema) const override; + +private: + enum class Type { + Array, + Boolean, + Double, + Integer, + Null, + Object, + String, + }; + static const char* typeAsString(Type t) { + switch (t) { + case Type::Array: + return "Array"; + case Type::Boolean: + return "Boolean"; + case Type::Double: + return "Double"; + case Type::Integer: + return "Integer"; + case Type::Null: + return "Null"; + case Type::Object: + return "Object"; + case Type::String: + return "String"; + } + + NOT_REACHED_GCOVR_EXCL_LINE; + } + + struct Value { + std::vector array_value_; + bool boolean_value_; + double double_value_; + int64_t integer_value_; + std::map object_value_; + std::string string_value_; + }; + + explicit Field(Type type) : type_(type) {} + explicit Field(const std::string& value) : type_(Type::String) { value_.string_value_ = value; } + explicit Field(int64_t value) : type_(Type::Integer) { value_.integer_value_ = value; } + explicit Field(double value) : type_(Type::Double) { value_.double_value_ = value; } + explicit Field(bool value) : type_(Type::Boolean) { value_.boolean_value_ = value; } + + bool isType(Type type) const { return type == type_; } + void checkType(Type type) const { + if (!isType(type)) { + throw Exception(fmt::format( + "JSON field from line {} accessed with type '{}' does not match actual type '{}'.", + line_number_start_, typeAsString(type), typeAsString(type_))); + } + } + + // Value return type functions. + std::string stringValue() const { + checkType(Type::String); + return value_.string_value_; + } + std::vector arrayValue() const { + checkType(Type::Array); + return value_.array_value_; + } + bool booleanValue() const { + checkType(Type::Boolean); + return value_.boolean_value_; + } + double doubleValue() const { + checkType(Type::Double); + return value_.double_value_; + } + int64_t integerValue() const { + checkType(Type::Integer); + return value_.integer_value_; + } + + rapidjson::Document asRapidJsonDocument() const; + static void buildRapidJsonDocument(const Field& field, rapidjson::Value& value, + rapidjson::Document::AllocatorType& allocator); + + uint64_t line_number_start_ = 0; + uint64_t line_number_end_ = 0; + const Type type_; + Value value_; +}; + +/** + * Custom stream to allow access to the line number for each object. + */ +class LineCountingStringStream : public rapidjson::StringStream { + // Ch is typedef in parent class to handle character encoding. +public: + LineCountingStringStream(const Ch* src) : rapidjson::StringStream(src), line_number_(1) {} + Ch Take() { + Ch ret = rapidjson::StringStream::Take(); + if (ret == '\n') { + line_number_++; + } + return ret; + } + uint64_t getLineNumber() const { return line_number_; } + +private: + uint64_t line_number_; +}; + +/** + * Consume events from SAX callbacks to build JSON Field. + */ +class ObjectHandler : public rapidjson::BaseReaderHandler, ObjectHandler> { +public: + ObjectHandler(LineCountingStringStream& stream) : state_(State::ExpectRoot), stream_(stream){}; + + bool StartObject(); + bool EndObject(rapidjson::SizeType); + bool Key(const char* value, rapidjson::SizeType size, bool); + bool StartArray(); + bool EndArray(rapidjson::SizeType); + bool Bool(bool value); + bool Double(double value); + bool Int(int value); + bool Uint(unsigned value); + bool Int64(int64_t value); + bool Uint64(uint64_t value); + bool Null(); + bool String(const char* value, rapidjson::SizeType size, bool); + bool RawNumber(const char*, rapidjson::SizeType, bool); + + ObjectSharedPtr getRoot() { return root_; } + +private: + bool handleValueEvent(FieldSharedPtr ptr); + + enum class State { + ExpectRoot, + ExpectKeyOrEndObject, + ExpectValueOrStartObjectArray, + ExpectArrayValueOrEndArray, + ExpectFinished, + }; + State state_; + LineCountingStringStream& stream_; + + std::stack stack_; + std::string key_; + + FieldSharedPtr root_; +}; + +void Field::buildRapidJsonDocument(const Field& field, rapidjson::Value& value, + rapidjson::Document::AllocatorType& allocator) { + + switch (field.type_) { + case Type::Array: { + value.SetArray(); + value.Reserve(field.value_.array_value_.size(), allocator); + for (const auto& element : field.value_.array_value_) { + switch (element->type_) { + case Type::Array: + case Type::Object: { + rapidjson::Value nested_value; + buildRapidJsonDocument(*element, nested_value, allocator); + value.PushBack(nested_value, allocator); + break; + } + case Type::Boolean: + value.PushBack(element->value_.boolean_value_, allocator); + break; + case Type::Double: + value.PushBack(element->value_.double_value_, allocator); + break; + case Type::Integer: + value.PushBack(element->value_.integer_value_, allocator); + break; + case Type::Null: + value.PushBack(rapidjson::Value(), allocator); + break; + case Type::String: + value.PushBack(rapidjson::StringRef(element->value_.string_value_.c_str()), allocator); + } + } + break; + } + case Type::Object: { + value.SetObject(); + for (const auto& item : field.value_.object_value_) { + auto name = rapidjson::StringRef(item.first.c_str()); + + switch (item.second->type_) { + case Type::Array: + case Type::Object: { + rapidjson::Value nested_value; + buildRapidJsonDocument(*item.second, nested_value, allocator); + value.AddMember(name, nested_value, allocator); + break; + } + case Type::Boolean: + value.AddMember(name, item.second->value_.boolean_value_, allocator); + break; + case Type::Double: + value.AddMember(name, item.second->value_.double_value_, allocator); + break; + case Type::Integer: + value.AddMember(name, item.second->value_.integer_value_, allocator); + break; + case Type::Null: + value.AddMember(name, rapidjson::Value(), allocator); + break; + case Type::String: + value.AddMember(name, rapidjson::StringRef(item.second->value_.string_value_.c_str()), + allocator); + break; + } + } + break; + } + case Type::Null: { + value.SetNull(); + break; + } + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +rapidjson::Document Field::asRapidJsonDocument() const { + rapidjson::Document document; + rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); + buildRapidJsonDocument(*this, document, allocator); + return document; +} + +uint64_t Field::hash() const { return HashUtil::xxHash64(asJsonString()); } + +bool Field::getBoolean(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Boolean)) { + throw Exception(fmt::format("key '{}' missing or not a boolean from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->booleanValue(); +} + +bool Field::getBoolean(const std::string& name, bool default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getBoolean(name); + } + return default_value; +} + +double Field::getDouble(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Double)) { + throw Exception(fmt::format("key '{}' missing or not a double from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->doubleValue(); +} + +double Field::getDouble(const std::string& name, double default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getDouble(name); + } + return default_value; +} + +int64_t Field::getInteger(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Integer)) { + throw Exception(fmt::format("key '{}' missing or not an integer from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->integerValue(); +} + +int64_t Field::getInteger(const std::string& name, int64_t default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getInteger(name); + } + return default_value; +} + +ObjectSharedPtr Field::getObject(const std::string& name, bool allow_empty) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end()) { + if (allow_empty) { + return createObject(); + } else { + throw Exception(fmt::format("key '{}' missing from lines {}-{}", name, line_number_start_, + line_number_end_)); + } + } else if (!value_itr->second->isType(Type::Object)) { + throw Exception(fmt::format("key '{}' not an object from line {}", name, + value_itr->second->line_number_start_)); + } else { + return value_itr->second; + } +} + +std::vector Field::getObjectArray(const std::string& name, + bool allow_empty) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Array)) { + if (allow_empty && value_itr == value_.object_value_.end()) { + return std::vector(); + } + throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + + std::vector array_value = value_itr->second->arrayValue(); + return {array_value.begin(), array_value.end()}; +} + +std::string Field::getString(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::String)) { + throw Exception(fmt::format("key '{}' missing or not a string from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + return value_itr->second->stringValue(); +} + +std::string Field::getString(const std::string& name, const std::string& default_value) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + if (value_itr != value_.object_value_.end()) { + return getString(name); + } + return default_value; +} + +std::vector Field::getStringArray(const std::string& name, bool allow_empty) const { + checkType(Type::Object); + std::vector string_array; + auto value_itr = value_.object_value_.find(name); + if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Array)) { + if (allow_empty && value_itr == value_.object_value_.end()) { + return string_array; + } + throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, + line_number_start_, line_number_end_)); + } + + std::vector array = value_itr->second->arrayValue(); + string_array.reserve(array.size()); + for (const auto& element : array) { + if (!element->isType(Type::String)) { + throw Exception(fmt::format("JSON array '{}' from line {} does not contain all strings", name, + line_number_start_)); + } + string_array.push_back(element->stringValue()); + } + + return string_array; +} + +std::vector Field::asObjectArray() const { + checkType(Type::Array); + return {value_.array_value_.begin(), value_.array_value_.end()}; +} + +std::string Field::asJsonString() const { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + rapidjson::Document document = asRapidJsonDocument(); + document.Accept(writer); + return buffer.GetString(); +} + +bool Field::empty() const { + if (isType(Type::Object)) { + return value_.object_value_.empty(); + } else if (isType(Type::Array)) { + return value_.array_value_.empty(); + } else { + throw Exception( + fmt::format("Json does not support empty() on types other than array and object")); + } +} + +bool Field::hasObject(const std::string& name) const { + checkType(Type::Object); + auto value_itr = value_.object_value_.find(name); + return value_itr != value_.object_value_.end(); +} + +void Field::iterate(const ObjectCallback& callback) const { + checkType(Type::Object); + for (const auto& item : value_.object_value_) { + bool stop_iteration = !callback(item.first, *item.second); + if (stop_iteration) { + break; + } + } +} + +void Field::validateSchema(const std::string& schema) const { + rapidjson::Document schema_document; + if (schema_document.Parse<0>(schema.c_str()).HasParseError()) { + throw std::invalid_argument(fmt::format( + "Schema supplied to validateSchema is not valid JSON\n Error(offset {}) : {}\n", + schema_document.GetErrorOffset(), GetParseError_En(schema_document.GetParseError()))); + } + + rapidjson::SchemaDocument schema_document_for_validator(schema_document); + rapidjson::GenericSchemaValidator + schema_validator(schema_document_for_validator); + + if (!asRapidJsonDocument().Accept(schema_validator)) { + rapidjson::StringBuffer schema_string_buffer; + rapidjson::StringBuffer document_string_buffer; + + schema_validator.GetInvalidSchemaPointer().StringifyUriFragment(schema_string_buffer); + schema_validator.GetInvalidDocumentPointer().StringifyUriFragment(document_string_buffer); + + throw Exception(fmt::format( + "JSON at lines {}-{} does not conform to schema.\n Invalid schema: {}\n" + " Schema violation: {}\n" + " Offending document key: {}", + line_number_start_, line_number_end_, schema_string_buffer.GetString(), + schema_validator.GetInvalidSchemaKeyword(), document_string_buffer.GetString())); + } +} + +bool ObjectHandler::StartObject() { + FieldSharedPtr object = Field::createObject(); + object->setLineNumberStart(stream_.getLineNumber()); + + switch (state_) { + case State::ExpectValueOrStartObjectArray: + stack_.top()->insert(key_, object); + stack_.push(object); + state_ = State::ExpectKeyOrEndObject; + return true; + case State::ExpectArrayValueOrEndArray: + stack_.top()->append(object); + stack_.push(object); + state_ = State::ExpectKeyOrEndObject; + return true; + case State::ExpectRoot: + root_ = object; + stack_.push(object); + state_ = State::ExpectKeyOrEndObject; + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::EndObject(rapidjson::SizeType) { + switch (state_) { + case State::ExpectKeyOrEndObject: + stack_.top()->setLineNumberEnd(stream_.getLineNumber()); + stack_.pop(); + + if (stack_.empty()) { + state_ = State::ExpectFinished; + } else if (stack_.top()->isObject()) { + state_ = State::ExpectKeyOrEndObject; + } else if (stack_.top()->isArray()) { + state_ = State::ExpectArrayValueOrEndArray; + } + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::Key(const char* value, rapidjson::SizeType size, bool) { + switch (state_) { + case State::ExpectKeyOrEndObject: + key_ = std::string(value, size); + state_ = State::ExpectValueOrStartObjectArray; + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::StartArray() { + FieldSharedPtr array = Field::createArray(); + array->setLineNumberStart(stream_.getLineNumber()); + + switch (state_) { + case State::ExpectValueOrStartObjectArray: + stack_.top()->insert(key_, array); + stack_.push(array); + state_ = State::ExpectArrayValueOrEndArray; + return true; + case State::ExpectArrayValueOrEndArray: + stack_.top()->append(array); + stack_.push(array); + return true; + case State::ExpectRoot: + root_ = array; + stack_.push(array); + state_ = State::ExpectArrayValueOrEndArray; + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool ObjectHandler::EndArray(rapidjson::SizeType) { + switch (state_) { + case State::ExpectArrayValueOrEndArray: + stack_.top()->setLineNumberEnd(stream_.getLineNumber()); + stack_.pop(); + + if (stack_.empty()) { + state_ = State::ExpectFinished; + } else if (stack_.top()->isObject()) { + state_ = State::ExpectKeyOrEndObject; + } else if (stack_.top()->isArray()) { + state_ = State::ExpectArrayValueOrEndArray; + } + + return true; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +// Value handlers +bool ObjectHandler::Bool(bool value) { return handleValueEvent(Field::createValue(value)); } +bool ObjectHandler::Double(double value) { return handleValueEvent(Field::createValue(value)); } +bool ObjectHandler::Int(int value) { + return handleValueEvent(Field::createValue(static_cast(value))); +} +bool ObjectHandler::Uint(unsigned value) { + return handleValueEvent(Field::createValue(static_cast(value))); +} +bool ObjectHandler::Int64(int64_t value) { return handleValueEvent(Field::createValue(value)); } +bool ObjectHandler::Uint64(uint64_t value) { + if (value > static_cast(std::numeric_limits::max())) { + throw Exception(fmt::format("JSON value from line {} is larger than int64_t (not supported)", + stream_.getLineNumber())); + } + return handleValueEvent(Field::createValue(static_cast(value))); +} + +bool ObjectHandler::Null() { return handleValueEvent(Field::createNull()); } + +bool ObjectHandler::String(const char* value, rapidjson::SizeType size, bool) { + return handleValueEvent(Field::createValue(std::string(value, size))); +} + +bool ObjectHandler::RawNumber(const char*, rapidjson::SizeType, bool) { + // Only called if kParseNumbersAsStrings is set as a parse flag, which it is not. + NOT_REACHED_GCOVR_EXCL_LINE; +} + +bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { + ptr->setLineNumberStart(stream_.getLineNumber()); + + switch (state_) { + case State::ExpectValueOrStartObjectArray: + state_ = State::ExpectKeyOrEndObject; + stack_.top()->insert(key_, ptr); + return true; + case State::ExpectArrayValueOrEndArray: + stack_.top()->append(ptr); + return true; + default: + return false; + } +} + +} // namespace + +ObjectSharedPtr Factory::loadFromString(const std::string& json) { + LineCountingStringStream json_stream(json.c_str()); + + ObjectHandler handler(json_stream); + rapidjson::Reader reader; + reader.Parse(json_stream, handler); + + if (reader.HasParseError()) { + throw Exception(fmt::format("JSON supplied is not valid. Error(offset {}, line {}): {}\n", + reader.GetErrorOffset(), json_stream.getLineNumber(), + GetParseError_En(reader.GetParseErrorCode()))); + } + + return handler.getRoot(); +} + +} // namespace RapidJson +} // namespace Json +} // namespace Envoy diff --git a/source/common/json/json_internal_legacy.h b/source/common/json/json_internal_legacy.h new file mode 100644 index 0000000000000..6b50ccd2b6268 --- /dev/null +++ b/source/common/json/json_internal_legacy.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include "envoy/json/json_object.h" + +namespace Envoy { +namespace Json { +namespace RapidJson { + +class Factory { +public: + /** + * Constructs a Json Object from a string. + */ + static ObjectSharedPtr loadFromString(const std::string& json); +}; + +} // namespace RapidJson +} // namespace Json +} // namespace Envoy diff --git a/source/common/json/json_loader.cc b/source/common/json/json_loader.cc index bb4ccf808662d..0e4f3669a1b85 100644 --- a/source/common/json/json_loader.cc +++ b/source/common/json/json_loader.cc @@ -1,701 +1,17 @@ #include "common/json/json_loader.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/common/assert.h" -#include "common/common/fmt.h" -#include "common/common/hash.h" -#include "common/common/utility.h" -#include "common/protobuf/utility.h" - -// Do not let RapidJson leak outside of this file. -#include "rapidjson/document.h" -#include "rapidjson/error/en.h" -#include "rapidjson/reader.h" -#include "rapidjson/schema.h" -#include "rapidjson/stream.h" -#include "rapidjson/stringbuffer.h" -#include "rapidjson/writer.h" - -#include "absl/strings/match.h" +#include "common/json/json_internal.h" +#include "common/json/json_internal_legacy.h" +#include "common/runtime/runtime_features.h" namespace Envoy { namespace Json { -namespace { -/** - * Internal representation of Object. - */ -class Field; -using FieldSharedPtr = std::shared_ptr; - -class Field : public Object { -public: - void setLineNumberStart(uint64_t line_number) { line_number_start_ = line_number; } - void setLineNumberEnd(uint64_t line_number) { line_number_end_ = line_number; } - - // Container factories for handler. - static FieldSharedPtr createObject() { return FieldSharedPtr{new Field(Type::Object)}; } - static FieldSharedPtr createArray() { return FieldSharedPtr{new Field(Type::Array)}; } - static FieldSharedPtr createNull() { return FieldSharedPtr{new Field(Type::Null)}; } - - bool isNull() const override { return type_ == Type::Null; } - bool isArray() const override { return type_ == Type::Array; } - bool isObject() const override { return type_ == Type::Object; } - - // Value factory. - template static FieldSharedPtr createValue(T value) { - return FieldSharedPtr{new Field(value)}; // NOLINT(modernize-make-shared) - } - - void append(FieldSharedPtr field_ptr) { - checkType(Type::Array); - value_.array_value_.push_back(field_ptr); - } - void insert(const std::string& key, FieldSharedPtr field_ptr) { - checkType(Type::Object); - value_.object_value_[key] = field_ptr; - } - - uint64_t hash() const override; - - bool getBoolean(const std::string& name) const override; - bool getBoolean(const std::string& name, bool default_value) const override; - double getDouble(const std::string& name) const override; - double getDouble(const std::string& name, double default_value) const override; - int64_t getInteger(const std::string& name) const override; - int64_t getInteger(const std::string& name, int64_t default_value) const override; - ObjectSharedPtr getObject(const std::string& name, bool allow_empty) const override; - std::vector getObjectArray(const std::string& name, - bool allow_empty) const override; - std::string getString(const std::string& name) const override; - std::string getString(const std::string& name, const std::string& default_value) const override; - std::vector getStringArray(const std::string& name, bool allow_empty) const override; - std::vector asObjectArray() const override; - std::string asString() const override { return stringValue(); } - bool asBoolean() const override { return booleanValue(); } - double asDouble() const override { return doubleValue(); } - int64_t asInteger() const override { return integerValue(); } - std::string asJsonString() const override; - - bool empty() const override; - bool hasObject(const std::string& name) const override; - void iterate(const ObjectCallback& callback) const override; - void validateSchema(const std::string& schema) const override; - -private: - enum class Type { - Array, - Boolean, - Double, - Integer, - Null, - Object, - String, - }; - static const char* typeAsString(Type t) { - switch (t) { - case Type::Array: - return "Array"; - case Type::Boolean: - return "Boolean"; - case Type::Double: - return "Double"; - case Type::Integer: - return "Integer"; - case Type::Null: - return "Null"; - case Type::Object: - return "Object"; - case Type::String: - return "String"; - } - - NOT_REACHED_GCOVR_EXCL_LINE; - } - - struct Value { - std::vector array_value_; - bool boolean_value_; - double double_value_; - int64_t integer_value_; - std::map object_value_; - std::string string_value_; - }; - - explicit Field(Type type) : type_(type) {} - explicit Field(const std::string& value) : type_(Type::String) { value_.string_value_ = value; } - explicit Field(int64_t value) : type_(Type::Integer) { value_.integer_value_ = value; } - explicit Field(double value) : type_(Type::Double) { value_.double_value_ = value; } - explicit Field(bool value) : type_(Type::Boolean) { value_.boolean_value_ = value; } - - bool isType(Type type) const { return type == type_; } - void checkType(Type type) const { - if (!isType(type)) { - throw Exception(fmt::format( - "JSON field from line {} accessed with type '{}' does not match actual type '{}'.", - line_number_start_, typeAsString(type), typeAsString(type_))); - } - } - - // Value return type functions. - std::string stringValue() const { - checkType(Type::String); - return value_.string_value_; - } - std::vector arrayValue() const { - checkType(Type::Array); - return value_.array_value_; - } - bool booleanValue() const { - checkType(Type::Boolean); - return value_.boolean_value_; - } - double doubleValue() const { - checkType(Type::Double); - return value_.double_value_; - } - int64_t integerValue() const { - checkType(Type::Integer); - return value_.integer_value_; - } - - rapidjson::Document asRapidJsonDocument() const; - static void buildRapidJsonDocument(const Field& field, rapidjson::Value& value, - rapidjson::Document::AllocatorType& allocator); - - uint64_t line_number_start_ = 0; - uint64_t line_number_end_ = 0; - const Type type_; - Value value_; -}; - -/** - * Custom stream to allow access to the line number for each object. - */ -class LineCountingStringStream : public rapidjson::StringStream { - // Ch is typedef in parent class to handle character encoding. -public: - LineCountingStringStream(const Ch* src) : rapidjson::StringStream(src), line_number_(1) {} - Ch Take() { - Ch ret = rapidjson::StringStream::Take(); - if (ret == '\n') { - line_number_++; - } - return ret; - } - uint64_t getLineNumber() const { return line_number_; } - -private: - uint64_t line_number_; -}; - -/** - * Consume events from SAX callbacks to build JSON Field. - */ -class ObjectHandler : public rapidjson::BaseReaderHandler, ObjectHandler> { -public: - ObjectHandler(LineCountingStringStream& stream) : state_(State::ExpectRoot), stream_(stream){}; - - bool StartObject(); - bool EndObject(rapidjson::SizeType); - bool Key(const char* value, rapidjson::SizeType size, bool); - bool StartArray(); - bool EndArray(rapidjson::SizeType); - bool Bool(bool value); - bool Double(double value); - bool Int(int value); - bool Uint(unsigned value); - bool Int64(int64_t value); - bool Uint64(uint64_t value); - bool Null(); - bool String(const char* value, rapidjson::SizeType size, bool); - bool RawNumber(const char*, rapidjson::SizeType, bool); - - ObjectSharedPtr getRoot() { return root_; } - -private: - bool handleValueEvent(FieldSharedPtr ptr); - - enum class State { - ExpectRoot, - ExpectKeyOrEndObject, - ExpectValueOrStartObjectArray, - ExpectArrayValueOrEndArray, - ExpectFinished, - }; - State state_; - LineCountingStringStream& stream_; - - std::stack stack_; - std::string key_; - - FieldSharedPtr root_; -}; - -void Field::buildRapidJsonDocument(const Field& field, rapidjson::Value& value, - rapidjson::Document::AllocatorType& allocator) { - - switch (field.type_) { - case Type::Array: { - value.SetArray(); - value.Reserve(field.value_.array_value_.size(), allocator); - for (const auto& element : field.value_.array_value_) { - switch (element->type_) { - case Type::Array: - case Type::Object: { - rapidjson::Value nested_value; - buildRapidJsonDocument(*element, nested_value, allocator); - value.PushBack(nested_value, allocator); - break; - } - case Type::Boolean: - value.PushBack(element->value_.boolean_value_, allocator); - break; - case Type::Double: - value.PushBack(element->value_.double_value_, allocator); - break; - case Type::Integer: - value.PushBack(element->value_.integer_value_, allocator); - break; - case Type::Null: - value.PushBack(rapidjson::Value(), allocator); - break; - case Type::String: - value.PushBack(rapidjson::StringRef(element->value_.string_value_.c_str()), allocator); - } - } - break; - } - case Type::Object: { - value.SetObject(); - for (const auto& item : field.value_.object_value_) { - auto name = rapidjson::StringRef(item.first.c_str()); - - switch (item.second->type_) { - case Type::Array: - case Type::Object: { - rapidjson::Value nested_value; - buildRapidJsonDocument(*item.second, nested_value, allocator); - value.AddMember(name, nested_value, allocator); - break; - } - case Type::Boolean: - value.AddMember(name, item.second->value_.boolean_value_, allocator); - break; - case Type::Double: - value.AddMember(name, item.second->value_.double_value_, allocator); - break; - case Type::Integer: - value.AddMember(name, item.second->value_.integer_value_, allocator); - break; - case Type::Null: - value.AddMember(name, rapidjson::Value(), allocator); - break; - case Type::String: - value.AddMember(name, rapidjson::StringRef(item.second->value_.string_value_.c_str()), - allocator); - break; - } - } - break; - } - case Type::Null: { - value.SetNull(); - break; - } - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -rapidjson::Document Field::asRapidJsonDocument() const { - rapidjson::Document document; - rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); - buildRapidJsonDocument(*this, document, allocator); - return document; -} - -uint64_t Field::hash() const { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - asRapidJsonDocument().Accept(writer); - return HashUtil::xxHash64(buffer.GetString()); -} - -bool Field::getBoolean(const std::string& name) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Boolean)) { - throw Exception(fmt::format("key '{}' missing or not a boolean from lines {}-{}", name, - line_number_start_, line_number_end_)); - } - return value_itr->second->booleanValue(); -} - -bool Field::getBoolean(const std::string& name, bool default_value) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr != value_.object_value_.end()) { - return getBoolean(name); - } else { - return default_value; - } -} - -double Field::getDouble(const std::string& name) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Double)) { - throw Exception(fmt::format("key '{}' missing or not a double from lines {}-{}", name, - line_number_start_, line_number_end_)); - } - return value_itr->second->doubleValue(); -} - -double Field::getDouble(const std::string& name, double default_value) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr != value_.object_value_.end()) { - return getDouble(name); - } else { - return default_value; - } -} - -int64_t Field::getInteger(const std::string& name) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Integer)) { - throw Exception(fmt::format("key '{}' missing or not an integer from lines {}-{}", name, - line_number_start_, line_number_end_)); - } - return value_itr->second->integerValue(); -} - -int64_t Field::getInteger(const std::string& name, int64_t default_value) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr != value_.object_value_.end()) { - return getInteger(name); - } else { - return default_value; - } -} - -ObjectSharedPtr Field::getObject(const std::string& name, bool allow_empty) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end()) { - if (allow_empty) { - return createObject(); - } else { - throw Exception(fmt::format("key '{}' missing from lines {}-{}", name, line_number_start_, - line_number_end_)); - } - } else if (!value_itr->second->isType(Type::Object)) { - throw Exception(fmt::format("key '{}' not an object from line {}", name, - value_itr->second->line_number_start_)); - } else { - return value_itr->second; - } -} - -std::vector Field::getObjectArray(const std::string& name, - bool allow_empty) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Array)) { - if (allow_empty && value_itr == value_.object_value_.end()) { - return std::vector(); - } - throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, - line_number_start_, line_number_end_)); - } - - std::vector array_value = value_itr->second->arrayValue(); - return {array_value.begin(), array_value.end()}; -} - -std::string Field::getString(const std::string& name) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::String)) { - throw Exception(fmt::format("key '{}' missing or not a string from lines {}-{}", name, - line_number_start_, line_number_end_)); - } - return value_itr->second->stringValue(); -} - -std::string Field::getString(const std::string& name, const std::string& default_value) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - if (value_itr != value_.object_value_.end()) { - return getString(name); - } else { - return default_value; - } -} - -std::vector Field::getStringArray(const std::string& name, bool allow_empty) const { - checkType(Type::Object); - std::vector string_array; - auto value_itr = value_.object_value_.find(name); - if (value_itr == value_.object_value_.end() || !value_itr->second->isType(Type::Array)) { - if (allow_empty && value_itr == value_.object_value_.end()) { - return string_array; - } - throw Exception(fmt::format("key '{}' missing or not an array from lines {}-{}", name, - line_number_start_, line_number_end_)); - } - - std::vector array = value_itr->second->arrayValue(); - string_array.reserve(array.size()); - for (const auto& element : array) { - if (!element->isType(Type::String)) { - throw Exception(fmt::format("JSON array '{}' from line {} does not contain all strings", name, - line_number_start_)); - } - string_array.push_back(element->stringValue()); - } - - return string_array; -} - -std::vector Field::asObjectArray() const { - checkType(Type::Array); - return {value_.array_value_.begin(), value_.array_value_.end()}; -} - -std::string Field::asJsonString() const { - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - rapidjson::Document document = asRapidJsonDocument(); - document.Accept(writer); - return buffer.GetString(); -} - -bool Field::empty() const { - if (isType(Type::Object)) { - return value_.object_value_.empty(); - } else if (isType(Type::Array)) { - return value_.array_value_.empty(); - } else { - throw Exception( - fmt::format("Json does not support empty() on types other than array and object")); - } -} - -bool Field::hasObject(const std::string& name) const { - checkType(Type::Object); - auto value_itr = value_.object_value_.find(name); - return value_itr != value_.object_value_.end(); -} - -void Field::iterate(const ObjectCallback& callback) const { - checkType(Type::Object); - for (const auto& item : value_.object_value_) { - bool stop_iteration = !callback(item.first, *item.second); - if (stop_iteration) { - break; - } - } -} - -void Field::validateSchema(const std::string& schema) const { - rapidjson::Document schema_document; - if (schema_document.Parse<0>(schema.c_str()).HasParseError()) { - throw std::invalid_argument(fmt::format( - "Schema supplied to validateSchema is not valid JSON\n Error(offset {}) : {}\n", - schema_document.GetErrorOffset(), GetParseError_En(schema_document.GetParseError()))); - } - - rapidjson::SchemaDocument schema_document_for_validator(schema_document); - rapidjson::SchemaValidator schema_validator(schema_document_for_validator); - - if (!asRapidJsonDocument().Accept(schema_validator)) { - rapidjson::StringBuffer schema_string_buffer; - rapidjson::StringBuffer document_string_buffer; - - schema_validator.GetInvalidSchemaPointer().StringifyUriFragment(schema_string_buffer); - schema_validator.GetInvalidDocumentPointer().StringifyUriFragment(document_string_buffer); - - throw Exception(fmt::format( - "JSON at lines {}-{} does not conform to schema.\n Invalid schema: {}\n" - " Schema violation: {}\n" - " Offending document key: {}", - line_number_start_, line_number_end_, schema_string_buffer.GetString(), - schema_validator.GetInvalidSchemaKeyword(), document_string_buffer.GetString())); - } -} - -bool ObjectHandler::StartObject() { - FieldSharedPtr object = Field::createObject(); - object->setLineNumberStart(stream_.getLineNumber()); - - switch (state_) { - case State::ExpectValueOrStartObjectArray: - stack_.top()->insert(key_, object); - stack_.push(object); - state_ = State::ExpectKeyOrEndObject; - return true; - case State::ExpectArrayValueOrEndArray: - stack_.top()->append(object); - stack_.push(object); - state_ = State::ExpectKeyOrEndObject; - return true; - case State::ExpectRoot: - root_ = object; - stack_.push(object); - state_ = State::ExpectKeyOrEndObject; - return true; - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -bool ObjectHandler::EndObject(rapidjson::SizeType) { - switch (state_) { - case State::ExpectKeyOrEndObject: - stack_.top()->setLineNumberEnd(stream_.getLineNumber()); - stack_.pop(); - - if (stack_.empty()) { - state_ = State::ExpectFinished; - } else if (stack_.top()->isObject()) { - state_ = State::ExpectKeyOrEndObject; - } else if (stack_.top()->isArray()) { - state_ = State::ExpectArrayValueOrEndArray; - } - return true; - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -bool ObjectHandler::Key(const char* value, rapidjson::SizeType size, bool) { - switch (state_) { - case State::ExpectKeyOrEndObject: - key_ = std::string(value, size); - state_ = State::ExpectValueOrStartObjectArray; - return true; - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -bool ObjectHandler::StartArray() { - FieldSharedPtr array = Field::createArray(); - array->setLineNumberStart(stream_.getLineNumber()); - - switch (state_) { - case State::ExpectValueOrStartObjectArray: - stack_.top()->insert(key_, array); - stack_.push(array); - state_ = State::ExpectArrayValueOrEndArray; - return true; - case State::ExpectArrayValueOrEndArray: - stack_.top()->append(array); - stack_.push(array); - return true; - case State::ExpectRoot: - root_ = array; - stack_.push(array); - state_ = State::ExpectArrayValueOrEndArray; - return true; - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -bool ObjectHandler::EndArray(rapidjson::SizeType) { - switch (state_) { - case State::ExpectArrayValueOrEndArray: - stack_.top()->setLineNumberEnd(stream_.getLineNumber()); - stack_.pop(); - - if (stack_.empty()) { - state_ = State::ExpectFinished; - } else if (stack_.top()->isObject()) { - state_ = State::ExpectKeyOrEndObject; - } else if (stack_.top()->isArray()) { - state_ = State::ExpectArrayValueOrEndArray; - } - - return true; - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -// Value handlers -bool ObjectHandler::Bool(bool value) { return handleValueEvent(Field::createValue(value)); } -bool ObjectHandler::Double(double value) { return handleValueEvent(Field::createValue(value)); } -bool ObjectHandler::Int(int value) { - return handleValueEvent(Field::createValue(static_cast(value))); -} -bool ObjectHandler::Uint(unsigned value) { - return handleValueEvent(Field::createValue(static_cast(value))); -} -bool ObjectHandler::Int64(int64_t value) { return handleValueEvent(Field::createValue(value)); } -bool ObjectHandler::Uint64(uint64_t value) { - if (value > static_cast(std::numeric_limits::max())) { - throw Exception(fmt::format("JSON value from line {} is larger than int64_t (not supported)", - stream_.getLineNumber())); - } - return handleValueEvent(Field::createValue(static_cast(value))); -} - -bool ObjectHandler::Null() { return handleValueEvent(Field::createNull()); } - -bool ObjectHandler::String(const char* value, rapidjson::SizeType size, bool) { - return handleValueEvent(Field::createValue(std::string(value, size))); -} - -bool ObjectHandler::RawNumber(const char*, rapidjson::SizeType, bool) { - // Only called if kParseNumbersAsStrings is set as a parse flag, which it is not. - NOT_REACHED_GCOVR_EXCL_LINE; -} - -bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) { - ptr->setLineNumberStart(stream_.getLineNumber()); - - switch (state_) { - case State::ExpectValueOrStartObjectArray: - state_ = State::ExpectKeyOrEndObject; - stack_.top()->insert(key_, ptr); - return true; - case State::ExpectArrayValueOrEndArray: - stack_.top()->append(ptr); - return true; - default: - return false; - } -} - -} // namespace - ObjectSharedPtr Factory::loadFromString(const std::string& json) { - LineCountingStringStream json_stream(json.c_str()); - - ObjectHandler handler(json_stream); - rapidjson::Reader reader; - reader.Parse(json_stream, handler); - - if (reader.HasParseError()) { - throw Exception(fmt::format("JSON supplied is not valid. Error(offset {}, line {}): {}\n", - reader.GetErrorOffset(), json_stream.getLineNumber(), - GetParseError_En(reader.GetParseErrorCode()))); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_legacy_json")) { + return Nlohmann::Factory::loadFromString(json); } - - return handler.getRoot(); + return RapidJson::Factory::loadFromString(json); } } // namespace Json diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 0f5a701919124..c165f1322e76a 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -108,6 +108,8 @@ constexpr const char* disabled_runtime_features[] = { "envoy.reloadable_features.enable_type_url_downgrade_and_upgrade", // TODO(alyssawilk) flip true after the release. "envoy.reloadable_features.new_tcp_connection_pool", + // TODO(asraa) flip to true in a separate PR to enable the new JSON by default. + "envoy.reloadable_features.remove_legacy_json", // Sentinel and test flag. "envoy.reloadable_features.test_feature_false", }; diff --git a/test/common/json/BUILD b/test/common/json/BUILD index 803f2abca6af4..83bf67d96ad05 100644 --- a/test/common/json/BUILD +++ b/test/common/json/BUILD @@ -21,12 +21,21 @@ envoy_cc_fuzz_test( ], ) +JSON_TEST_DEPS = [ + "//source/common/json:json_loader_lib", + "//source/common/stats:isolated_store_lib", + "//test/test_common:utility_lib", +] + envoy_cc_test( name = "json_loader_test", srcs = ["json_loader_test.cc"], - deps = [ - "//source/common/json:json_loader_lib", - "//source/common/stats:isolated_store_lib", - "//test/test_common:utility_lib", - ], + args = ["--runtime-feature-override-for-tests=envoy.reloadable_features.remove_legacy_json"], + deps = JSON_TEST_DEPS, +) + +envoy_cc_test( + name = "json_loader_legacy_test", + srcs = ["json_loader_test.cc"], + deps = JSON_TEST_DEPS, ) diff --git a/test/common/json/json_loader_test.cc b/test/common/json/json_loader_test.cc index 884caf5b0d0c1..e26fa010c8353 100644 --- a/test/common/json/json_loader_test.cc +++ b/test/common/json/json_loader_test.cc @@ -71,10 +71,16 @@ TEST_F(JsonLoaderTest, Basic) { } { - EXPECT_THROW_WITH_MESSAGE( - Factory::loadFromString("{\"hello\": \n\n\"world\""), Exception, - "JSON supplied is not valid. Error(offset 19, line 3): Missing a comma or " - "'}' after an object member.\n"); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_legacy_json")) { + EXPECT_THROW_WITH_MESSAGE(Factory::loadFromString("{\"hello\": \n\n\"world\""), Exception, + "JSON supplied is not valid. Error(line 3, column 8, token " + "\"world\"): syntax error while " + "parsing object - unexpected end of input; expected '}'\n"); + } else { + EXPECT_THROW_WITH_MESSAGE(Factory::loadFromString("{\"hello\": \n\n\"world\""), Exception, + "JSON supplied is not valid. Error(offset 19, line 3): Missing a " + "comma or '}' after an object member.\n"); + } } { @@ -242,6 +248,15 @@ TEST_F(JsonLoaderTest, Double) { } } +TEST_F(JsonLoaderTest, LoadArray) { + ObjectSharedPtr json1 = Factory::loadFromString("[1.11, 22, \"cat\"]"); + ObjectSharedPtr json2 = Factory::loadFromString("[22, \"cat\", 1.11]"); + + // Array values in different orders will not be the same. + EXPECT_NE(json1->asJsonString(), json2->asJsonString()); + EXPECT_NE(json1->hash(), json2->hash()); +} + TEST_F(JsonLoaderTest, Hash) { ObjectSharedPtr json1 = Factory::loadFromString("{\"value1\": 10.5, \"value2\": -12.3}"); ObjectSharedPtr json2 = Factory::loadFromString("{\"value2\": -12.3, \"value1\": 10.5}"); @@ -254,17 +269,15 @@ TEST_F(JsonLoaderTest, Hash) { EXPECT_EQ(json2->hash(), json3->hash()); // Ensure different hash is computed for different objects EXPECT_NE(json1->hash(), json4->hash()); + + // Nested objects with keys in different orders should be the same. + ObjectSharedPtr json5 = Factory::loadFromString("{\"value1\": {\"a\": true, \"b\": null}}"); + ObjectSharedPtr json6 = Factory::loadFromString("{\"value1\": {\"b\": null, \"a\": true}}"); + EXPECT_EQ(json5->hash(), json6->hash()); } TEST_F(JsonLoaderTest, Schema) { - { - std::string invalid_json_schema = R"EOF( - { - "properties": {"value1"} - } - )EOF"; - - std::string invalid_schema = R"EOF( + std::string invalid_schema = R"EOF( { "properties" : { "value1": {"type" : "faketype"} @@ -272,82 +285,22 @@ TEST_F(JsonLoaderTest, Schema) { } )EOF"; - std::string different_schema = R"EOF( - { - "properties" : { - "value1" : {"type" : "number"} - }, - "additionalProperties" : false - } - )EOF"; - - std::string valid_schema = R"EOF( - { - "properties": { - "value1": {"type" : "number"}, - "value2": {"type": "string"} - }, - "additionalProperties": false - } - )EOF"; - - std::string json_string = R"EOF( + std::string json_string = R"EOF( { "value1": 10, "value2" : "test" } )EOF"; - ObjectSharedPtr json = Factory::loadFromString(json_string); - EXPECT_THROW(json->validateSchema(invalid_json_schema), std::invalid_argument); - EXPECT_THROW(json->validateSchema(invalid_schema), Exception); - EXPECT_THROW(json->validateSchema(different_schema), Exception); - EXPECT_NO_THROW(json->validateSchema(valid_schema)); - } - - { - std::string json_string = R"EOF( - { - "value1": [false, 2.01, 3, null], - "value2" : "test" - } - )EOF"; - - std::string empty_schema = R"EOF({})EOF"; - - ObjectSharedPtr json = Factory::loadFromString(json_string); - EXPECT_NO_THROW(json->validateSchema(empty_schema)); - } -} - -TEST_F(JsonLoaderTest, NestedSchema) { - - std::string schema = R"EOF( - { - "properties": { - "value1": {"type" : "number"}, - "value2": {"type": "string"} - }, - "additionalProperties": false - } - )EOF"; - - std::string json_string = R"EOF( - { - "bar": "baz", - "foo": { - "value1": "should have been a number", - "value2" : "test" - } - } - )EOF"; - ObjectSharedPtr json = Factory::loadFromString(json_string); - - EXPECT_THROW_WITH_MESSAGE(json->getObject("foo")->validateSchema(schema), Exception, - "JSON at lines 4-7 does not conform to schema.\n Invalid schema: " - "#/properties/value1\n Schema violation: type\n Offending document " - "key: #/value1"); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_legacy_json")) { + EXPECT_THROW_WITH_MESSAGE(json->validateSchema(invalid_schema), Exception, "not implemented"); + } else { + EXPECT_THROW_WITH_MESSAGE( + json->validateSchema(invalid_schema), Exception, + "JSON at lines 2-5 does not conform to schema.\n Invalid schema: #/properties/value1\n " + "Schema violation: type\n Offending document key: #/value1"); + } } TEST_F(JsonLoaderTest, MissingEnclosingDocument) { @@ -360,10 +313,16 @@ TEST_F(JsonLoaderTest, MissingEnclosingDocument) { } ] )EOF"; - - EXPECT_THROW_WITH_MESSAGE(Factory::loadFromString(json_string), Exception, - "JSON supplied is not valid. Error(offset 14, line 2): Terminate " - "parsing due to Handler error.\n"); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.remove_legacy_json")) { + EXPECT_THROW_WITH_MESSAGE( + Factory::loadFromString(json_string), Exception, + "JSON supplied is not valid. Error(line 2, column 15, token \"listeners\" :): syntax error " + "while parsing value - unexpected ':'; expected end of input\n"); + } else { + EXPECT_THROW_WITH_MESSAGE(Factory::loadFromString(json_string), Exception, + "JSON supplied is not valid. Error(offset 14, line 2): Terminate " + "parsing due to Handler error.\n"); + } } TEST_F(JsonLoaderTest, AsString) { diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index a99d7fbfe5587..52c5b56286247 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -690,12 +690,11 @@ TEST_F(StreamInfoHeaderFormatterTest, UnknownVariable) { testInvalidFormat("INVA TEST_F(StreamInfoHeaderFormatterTest, WrongFormatOnUpstreamMetadataVariable) { // Invalid JSON. - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA(abcd)", false), - EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA(abcd), because JSON supplied is not valid. " - "Error(offset 0, line 1): Invalid value.\n"); + EXPECT_THROW_WITH_MESSAGE( + StreamInfoHeaderFormatter("UPSTREAM_METADATA(abcd)", false), EnvoyException, + "Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " + "...]), actual format UPSTREAM_METADATA(abcd), because JSON supplied is not valid. " + "Error(offset 0, line 1): Invalid value.\n"); // No parameters. EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA", false), EnvoyException, @@ -703,11 +702,11 @@ TEST_F(StreamInfoHeaderFormatterTest, WrongFormatOnUpstreamMetadataVariable) { "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " "UPSTREAM_METADATA"); - EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA()", false), EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA(), because JSON supplied is not valid. " - "Error(offset 0, line 1): The document is empty.\n"); + EXPECT_THROW_WITH_MESSAGE( + StreamInfoHeaderFormatter("UPSTREAM_METADATA()", false), EnvoyException, + "Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " + "...]), actual format UPSTREAM_METADATA(), because JSON supplied is not valid. Error(offset " + "0, line 1): The document is empty.\n"); // One parameter. EXPECT_THROW_WITH_MESSAGE(StreamInfoHeaderFormatter("UPSTREAM_METADATA([\"ns\"])", false), @@ -745,10 +744,9 @@ TEST_F(StreamInfoHeaderFormatterTest, WrongFormatOnUpstreamMetadataVariable) { // Invalid string elements. EXPECT_THROW_WITH_MESSAGE( StreamInfoHeaderFormatter("UPSTREAM_METADATA([\"a\", \"\\unothex\"])", false), EnvoyException, - "Invalid header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA([\"a\", \"\\unothex\"]), because JSON supplied is not valid. " - "Error(offset 7, line 1): Incorrect hex digit after \\u escape in string.\n"); + "Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " + "...]), actual format UPSTREAM_METADATA([\"a\", \"\\unothex\"]), because JSON supplied is " + "not valid. Error(offset 7, line 1): Incorrect hex digit after \\u escape in string.\n"); // Non-array parameters. EXPECT_THROW_WITH_MESSAGE( diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index ecf2e82a8c2a5..33c47a987f99a 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -197,6 +197,7 @@ NBSP NDEBUG NEXTHDR NGHTTP +NLOHMANN NOAUTH NOCHECKRESP NODELAY