Skip to content
This repository was archived by the owner on Aug 19, 2019. It is now read-only.

Add streaming JSON parsing. #106

Merged
igorpeshansky merged 3 commits into from
Mar 30, 2018
Merged

Add streaming JSON parsing. #106

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
197a84b
Add streaming JSON parsing.
igorpeshansky Mar 30, 2018
eef7638
Only complete parsing when the parser object goes away.
igorpeshansky Mar 30, 2018
044d9b4
Change loops; remove extra debugging printouts.
igorpeshansky Mar 30, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 74 additions & 5 deletions src/json.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,10 +262,23 @@ class ObjectContext : public Context {
std::unique_ptr<std::string> field_name_;
};

class CallbackContext : public Context {
public:
CallbackContext(std::function<void(std::unique_ptr<Value>)> callback)
: Context(nullptr), callback_(callback) {}
void AddValue(std::unique_ptr<Value> value) override {
callback_(std::move(value));
}
private:
std::function<void(std::unique_ptr<Value>)> callback_;
};

// A builder context that allows building up a JSON object.
class JSONBuilder {
public:
JSONBuilder() : context_(new TopLevelContext()) {}
JSONBuilder(std::function<void(std::unique_ptr<Value>)> callback)
: context_(new CallbackContext(callback)) {}
~JSONBuilder() { delete context_; }

void AddValue(std::unique_ptr<Value> value) {
Expand Down Expand Up @@ -415,7 +428,7 @@ yajl_callbacks callbacks = {
.yajl_end_array = &handle_end_array,
};

}
} // namespace

std::vector<std::unique_ptr<Value>> Parser::AllFromStream(std::istream& stream)
throw(Exception)
Expand All @@ -430,10 +443,7 @@ std::vector<std::unique_ptr<Value>> Parser::AllFromStream(std::istream& stream)
//yajl_config(handle, yajl_allow_trailing_garbage, 1);
//yajl_config(handle, yajl_dont_validate_strings, 1);

for (;;) {
if (stream.eof()) {
break;
}
while (!stream.eof()) {
stream.read(reinterpret_cast<char*>(&data[0]), kMax);
size_t count = stream.gcount();
yajl_parse(handle, data, count);
Expand Down Expand Up @@ -481,4 +491,63 @@ std::unique_ptr<Value> Parser::FromString(const std::string& input)
return FromStream(stream);
}

class Parser::ParseState {
public:
ParseState(std::function<void(std::unique_ptr<Value>)> callback)
: builder_(callback),
handle_(yajl_alloc(&callbacks, NULL, (void*) &builder_)) {
yajl_config(handle_, yajl_allow_comments, 1);
yajl_config(handle_, yajl_allow_multiple_values, 1);
//yajl_config(handle_, yajl_allow_partial_values, 1);
//yajl_config(handle_, yajl_allow_trailing_garbage, 1);
//yajl_config(handle_, yajl_dont_validate_strings, 1);
}

~ParseState() {
yajl_status stat = yajl_complete_parse(handle_);
if (stat != yajl_status_ok) {
unsigned char* str = yajl_get_error(handle_, 0, nullptr, 0);
std::string error_str((const char*)str);
yajl_free_error(handle_, str);
throw Exception(error_str);
}
yajl_free(handle_);
}

yajl_handle& handle() { return handle_; }

private:
JSONBuilder builder_;
yajl_handle handle_;
};

Parser::Parser(std::function<void(std::unique_ptr<Value>)> callback)
: state_(new ParseState(callback)) {}

Parser::~Parser() {}

std::size_t Parser::ParseStream(std::istream& stream) throw(Exception) {
const int kMax = 65536;
unsigned char data[kMax];
size_t total_bytes_consumed = 0;
yajl_handle& handle = state_->handle();

while (!stream.eof()) {
stream.read(reinterpret_cast<char*>(&data[0]), kMax);
size_t count = stream.gcount();

yajl_status stat = yajl_parse(handle, data, count);
if (stat != yajl_status_ok) {
unsigned char* str = yajl_get_error(handle, 1, data, kMax);
std::string error_str((const char*)str);
yajl_free_error(handle, str);
throw Exception(error_str);
}

total_bytes_consumed += yajl_get_bytes_consumed(handle);
}

return total_bytes_consumed;
}

} // json
14 changes: 14 additions & 0 deletions src/json.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#ifndef JSON_H_
#define JSON_H_

#include <functional>
#include <istream>
#include <ostream>
#include <map>
Expand Down Expand Up @@ -331,6 +332,10 @@ inline std::unique_ptr<Value> object(

class Parser {
public:
class ParseState;

Parser(std::function<void(std::unique_ptr<Value>)> callback);
~Parser();
static std::vector<std::unique_ptr<Value>> AllFromStream(
std::istream& stream) throw(Exception);
static std::vector<std::unique_ptr<Value>> AllFromString(
Expand All @@ -339,6 +344,15 @@ class Parser {
throw(Exception);
static std::unique_ptr<Value> FromString(const std::string& input)
throw(Exception);

size_t ParseStream(std::istream& stream) throw(Exception);
// Used to accept inline construction of streams.
size_t ParseStream(std::istream&& stream) throw(Exception) {
return ParseStream(stream);
}

private:
std::unique_ptr<ParseState> state_;
};

}
Expand Down
132 changes: 132 additions & 0 deletions test/json_unittest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,4 +424,136 @@ TEST(ParseError, ObjectNoValue) {
ASSERT_THROW(json::Parser::FromString("{\"x\":}"), json::Exception);
}

// Streaming parsing test.

TEST(StreamingTest, CompleteStream) {
GuardJsonException([](){
json::value v;
json::Parser p([&v](json::value r) { v = std::move(r); });
p.ParseStream(std::istringstream(
"{\n"
" \"foo\": [1, 2, 3],\n"
" \"bar\": {\"x\": 0, \"y\": null},\n"
" \"baz\": true,\n"
" \"str\": \"asdfasdf\"\n"
"}\n"
));
EXPECT_TOSTRING_EQ(
"{"
"\"bar\":{\"x\":0.0,\"y\":null},"
"\"baz\":true,"
"\"foo\":[1.0,2.0,3.0],"
"\"str\":\"asdfasdf\""
"}",
v);
});
}

TEST(StreamingTest, SplitStream) {
GuardJsonException([](){
json::value v;
json::Parser p([&v](json::value r) { v = std::move(r); });
p.ParseStream(std::istringstream(
"{\n"
" \"foo\": [1, 2, 3],\n"
));
p.ParseStream(std::istringstream(
" \"bar\": {\"x\": 0, \"y\": null},\n"
" \"baz\": true,\n"
));
p.ParseStream(std::istringstream(
" \"str\": \"asdfasdf\"\n"
"}\n"
));
EXPECT_TOSTRING_EQ(
"{"
"\"bar\":{\"x\":0.0,\"y\":null},"
"\"baz\":true,"
"\"foo\":[1.0,2.0,3.0],"
"\"str\":\"asdfasdf\""
"}",
v);
});
}

TEST(StreamingTest, BrokenStream) {
GuardJsonException([](){
json::value v;
json::Parser p([&v](json::value r) { v = std::move(r); });
p.ParseStream(std::istringstream(
"{\n"
" \"foo\": [1, 2, 3],\n"
" \"ba"
));
p.ParseStream(std::istringstream(
"r\": {\"x\": 0, \"y\": nu"
Copy link
Contributor

@bmoyles0117 bmoyles0117 Mar 30, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had to spend a minute or two to reason about the indentation you've chosen. I don't think it's incredibly useful to preserve the location based on where you left off on the previous line. If you feel strongly about it, I'd say keep it, but otherwise, consistent indentations would have been easier for my brain to parse as you have the final json encoded check that makes it clear what the entire json blob represents.

Copy link
Contributor Author

@igorpeshansky igorpeshansky Mar 30, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wanted to make it clear that the stream literally stops in the middle of a token, which may not be obvious from the test name. I think it's important enough to call out, and indentation works better than comments, IMO. Let's stick with this for now — we can always do a cleanup pass later.

));
p.ParseStream(std::istringstream(
"ll},\n"
" \"baz\": true,\n"
" \"str\""
));
p.ParseStream(std::istringstream(
": \"asdfasdf\"\n"
"}\n"
));
EXPECT_TOSTRING_EQ(
"{"
"\"bar\":{\"x\":0.0,\"y\":null},"
"\"baz\":true,"
"\"foo\":[1.0,2.0,3.0],"
"\"str\":\"asdfasdf\""
"}",
v);
});
}

TEST(StreamingTest, MultipleObjectsStream) {
GuardJsonException([](){
std::vector<json::value> v;
json::Parser p([&v](json::value r) { v.emplace_back(std::move(r)); });
p.ParseStream(std::istringstream(
"{\n"
" \"foo\": [1, 2, 3],\n"
" \"bar\": {\"x\": 0, \"y\": null},\n"
" \"baz\": true,\n"
" \"str\": \"asdfasdf\"\n"
"}\n"
"{\n"
" \"foo1\": [1, 2, 3],\n"
" \"bar1\": {\"x\": 0, \"y\": null},\n"
" \"baz1\": true,\n"
" \"str1\": \"asdfasdf\"\n"
"}\n"
));
EXPECT_EQ(2, v.size());
EXPECT_TOSTRING_EQ(
"{"
"\"bar\":{\"x\":0.0,\"y\":null},"
"\"baz\":true,"
"\"foo\":[1.0,2.0,3.0],"
"\"str\":\"asdfasdf\""
"}",
v[0]);
EXPECT_TOSTRING_EQ(
"{"
"\"bar1\":{\"x\":0.0,\"y\":null},"
"\"baz1\":true,"
"\"foo1\":[1.0,2.0,3.0],"
"\"str1\":\"asdfasdf\""
"}",
v[1]);
});
}

TEST(StreamingTest, ParseStreamReturnsByteCount) {
GuardJsonException([](){
json::value v;
json::Parser p([&v](json::value r) { v = std::move(r); });
size_t n = p.ParseStream(std::istringstream("123"));
EXPECT_TOSTRING_EQ("123", v);
EXPECT_EQ(3, n);
});
}

} // namespace