diff --git a/src/json.cc b/src/json.cc index daa81f1d..7053788e 100644 --- a/src/json.cc +++ b/src/json.cc @@ -262,10 +262,23 @@ class ObjectContext : public Context { std::unique_ptr field_name_; }; +class CallbackContext : public Context { + public: + CallbackContext(std::function)> callback) + : Context(nullptr), callback_(callback) {} + void AddValue(std::unique_ptr value) override { + callback_(std::move(value)); + } + private: + std::function)> callback_; +}; + // A builder context that allows building up a JSON object. class JSONBuilder { public: JSONBuilder() : context_(new TopLevelContext()) {} + JSONBuilder(std::function)> callback) + : context_(new CallbackContext(callback)) {} ~JSONBuilder() { delete context_; } void AddValue(std::unique_ptr value) { @@ -415,7 +428,7 @@ yajl_callbacks callbacks = { .yajl_end_array = &handle_end_array, }; -} +} // namespace std::vector> Parser::AllFromStream(std::istream& stream) throw(Exception) @@ -430,10 +443,7 @@ std::vector> Parser::AllFromStream(std::istream& stream) //yajl_config(handle, yajl_allow_trailing_garbage, 1); //yajl_config(handle, yajl_dont_validate_strings, 1); - for (;;) { - if (stream.eof()) { - break; - } + while (!stream.eof()) { stream.read(reinterpret_cast(&data[0]), kMax); size_t count = stream.gcount(); yajl_parse(handle, data, count); @@ -481,4 +491,63 @@ std::unique_ptr Parser::FromString(const std::string& input) return FromStream(stream); } +class Parser::ParseState { + public: + ParseState(std::function)> callback) + : builder_(callback), + handle_(yajl_alloc(&callbacks, NULL, (void*) &builder_)) { + yajl_config(handle_, yajl_allow_comments, 1); + yajl_config(handle_, yajl_allow_multiple_values, 1); + //yajl_config(handle_, yajl_allow_partial_values, 1); + //yajl_config(handle_, yajl_allow_trailing_garbage, 1); + //yajl_config(handle_, yajl_dont_validate_strings, 1); + } + + ~ParseState() { + yajl_status stat = yajl_complete_parse(handle_); + if (stat != yajl_status_ok) { + unsigned char* str = yajl_get_error(handle_, 0, nullptr, 0); + std::string error_str((const char*)str); + yajl_free_error(handle_, str); + throw Exception(error_str); + } + yajl_free(handle_); + } + + yajl_handle& handle() { return handle_; } + + private: + JSONBuilder builder_; + yajl_handle handle_; +}; + +Parser::Parser(std::function)> callback) + : state_(new ParseState(callback)) {} + +Parser::~Parser() {} + +std::size_t Parser::ParseStream(std::istream& stream) throw(Exception) { + const int kMax = 65536; + unsigned char data[kMax]; + size_t total_bytes_consumed = 0; + yajl_handle& handle = state_->handle(); + + while (!stream.eof()) { + stream.read(reinterpret_cast(&data[0]), kMax); + size_t count = stream.gcount(); + + yajl_status stat = yajl_parse(handle, data, count); + if (stat != yajl_status_ok) { + unsigned char* str = yajl_get_error(handle, 1, data, kMax); + std::string error_str((const char*)str); + yajl_free_error(handle, str); + throw Exception(error_str); + } + + total_bytes_consumed += yajl_get_bytes_consumed(handle); + } + + return total_bytes_consumed; +} + } // json diff --git a/src/json.h b/src/json.h index f1a5050e..b55d2a51 100644 --- a/src/json.h +++ b/src/json.h @@ -16,6 +16,7 @@ #ifndef JSON_H_ #define JSON_H_ +#include #include #include #include @@ -331,6 +332,10 @@ inline std::unique_ptr object( class Parser { public: + class ParseState; + + Parser(std::function)> callback); + ~Parser(); static std::vector> AllFromStream( std::istream& stream) throw(Exception); static std::vector> AllFromString( @@ -339,6 +344,15 @@ class Parser { throw(Exception); static std::unique_ptr FromString(const std::string& input) throw(Exception); + + size_t ParseStream(std::istream& stream) throw(Exception); + // Used to accept inline construction of streams. + size_t ParseStream(std::istream&& stream) throw(Exception) { + return ParseStream(stream); + } + + private: + std::unique_ptr state_; }; } diff --git a/test/json_unittest.cc b/test/json_unittest.cc index 6ddd153f..90030f15 100644 --- a/test/json_unittest.cc +++ b/test/json_unittest.cc @@ -424,4 +424,136 @@ TEST(ParseError, ObjectNoValue) { ASSERT_THROW(json::Parser::FromString("{\"x\":}"), json::Exception); } +// Streaming parsing test. + +TEST(StreamingTest, CompleteStream) { + GuardJsonException([](){ + json::value v; + json::Parser p([&v](json::value r) { v = std::move(r); }); + p.ParseStream(std::istringstream( + "{\n" + " \"foo\": [1, 2, 3],\n" + " \"bar\": {\"x\": 0, \"y\": null},\n" + " \"baz\": true,\n" + " \"str\": \"asdfasdf\"\n" + "}\n" + )); + EXPECT_TOSTRING_EQ( + "{" + "\"bar\":{\"x\":0.0,\"y\":null}," + "\"baz\":true," + "\"foo\":[1.0,2.0,3.0]," + "\"str\":\"asdfasdf\"" + "}", + v); + }); +} + +TEST(StreamingTest, SplitStream) { + GuardJsonException([](){ + json::value v; + json::Parser p([&v](json::value r) { v = std::move(r); }); + p.ParseStream(std::istringstream( + "{\n" + " \"foo\": [1, 2, 3],\n" + )); + p.ParseStream(std::istringstream( + " \"bar\": {\"x\": 0, \"y\": null},\n" + " \"baz\": true,\n" + )); + p.ParseStream(std::istringstream( + " \"str\": \"asdfasdf\"\n" + "}\n" + )); + EXPECT_TOSTRING_EQ( + "{" + "\"bar\":{\"x\":0.0,\"y\":null}," + "\"baz\":true," + "\"foo\":[1.0,2.0,3.0]," + "\"str\":\"asdfasdf\"" + "}", + v); + }); +} + +TEST(StreamingTest, BrokenStream) { + GuardJsonException([](){ + json::value v; + json::Parser p([&v](json::value r) { v = std::move(r); }); + p.ParseStream(std::istringstream( + "{\n" + " \"foo\": [1, 2, 3],\n" + " \"ba" + )); + p.ParseStream(std::istringstream( + "r\": {\"x\": 0, \"y\": nu" + )); + p.ParseStream(std::istringstream( + "ll},\n" + " \"baz\": true,\n" + " \"str\"" + )); + p.ParseStream(std::istringstream( + ": \"asdfasdf\"\n" + "}\n" + )); + EXPECT_TOSTRING_EQ( + "{" + "\"bar\":{\"x\":0.0,\"y\":null}," + "\"baz\":true," + "\"foo\":[1.0,2.0,3.0]," + "\"str\":\"asdfasdf\"" + "}", + v); + }); +} + +TEST(StreamingTest, MultipleObjectsStream) { + GuardJsonException([](){ + std::vector v; + json::Parser p([&v](json::value r) { v.emplace_back(std::move(r)); }); + p.ParseStream(std::istringstream( + "{\n" + " \"foo\": [1, 2, 3],\n" + " \"bar\": {\"x\": 0, \"y\": null},\n" + " \"baz\": true,\n" + " \"str\": \"asdfasdf\"\n" + "}\n" + "{\n" + " \"foo1\": [1, 2, 3],\n" + " \"bar1\": {\"x\": 0, \"y\": null},\n" + " \"baz1\": true,\n" + " \"str1\": \"asdfasdf\"\n" + "}\n" + )); + EXPECT_EQ(2, v.size()); + EXPECT_TOSTRING_EQ( + "{" + "\"bar\":{\"x\":0.0,\"y\":null}," + "\"baz\":true," + "\"foo\":[1.0,2.0,3.0]," + "\"str\":\"asdfasdf\"" + "}", + v[0]); + EXPECT_TOSTRING_EQ( + "{" + "\"bar1\":{\"x\":0.0,\"y\":null}," + "\"baz1\":true," + "\"foo1\":[1.0,2.0,3.0]," + "\"str1\":\"asdfasdf\"" + "}", + v[1]); + }); +} + +TEST(StreamingTest, ParseStreamReturnsByteCount) { + GuardJsonException([](){ + json::value v; + json::Parser p([&v](json::value r) { v = std::move(r); }); + size_t n = p.ParseStream(std::istringstream("123")); + EXPECT_TOSTRING_EQ("123", v); + EXPECT_EQ(3, n); + }); +} + } // namespace