Streaming decode

benchmark/poison.patch

@@ -63,9 +63,10 @@ index c07cf0d..8426e22 100644
        {:jason, "~> 1.1", only: [:test, :bench]},
        {:exjsx, "~> 4.0", only: :bench},
        {:tiny, "~> 1.0", only: :bench},
-       {:jsone, "~> 1.4", only: :bench},
+-      {:jsone, "~> 1.4", only: :bench},
 -      {:jiffy, "~> 0.15", only: :bench},
 -      {:json, "~> 1.2", only: :bench}
++      {:jsone, path: "../..", only: :bench},
 +      {:jiffy, "~> 1.0", only: :bench},
 +      {:json, "~> 1.4", only: :bench}
      ]

doc/jsone.md

@@ -120,7 +120,7 @@ the last such instance.
 
 
 <pre><code>
-encode_option() = native_utf8 | native_forward_slash | canonical_form | {float_format, [<a href="#type-float_format_option">float_format_option()</a>]} | {datetime_format, <a href="#type-datetime_encode_format">datetime_encode_format()</a>} | {object_key_type, string | scalar | value} | {space, non_neg_integer()} | {indent, non_neg_integer()} | {map_unknown_value, fun((term()) -&gt; {ok, <a href="#type-json_value">json_value()</a>} | error)} | <a href="#type-common_option">common_option()</a>
+encode_option() = native_utf8 | native_forward_slash | canonical_form | {float_format, [<a href="#type-float_format_option">float_format_option()</a>]} | {datetime_format, <a href="#type-datetime_encode_format">datetime_encode_format()</a>} | {object_key_type, string | scalar | value} | {space, non_neg_integer()} | {indent, non_neg_integer()} | {map_unknown_value, undefined | fun((term()) -&gt; {ok, <a href="#type-json_value">json_value()</a>} | error)} | skip_undefined | <a href="#type-common_option">common_option()</a>
 </code></pre>
 
 `native_utf8`: <br />
@@ -156,8 +156,13 @@ encode_option() = native_utf8 | native_forward_slash | canonical_form | {float_f
 - Inserts a newline and `N` spaces for each level of indentation <br />
 - default: `0` <br />
 
+`skip_undefined`: <br />
+- If specified, each entry having `undefined` value in a object isn't included in the result JSON <br />
+
 `{map_unknown_value, Fun}`: <br />
-- If specified, unknown values encountered during an encoding process are converted to `json_value()` by applying `Fun`.
+- If `Fun` is a function, unknown values encountered during an encoding process are converted to `json_value()` by applying `Fun`. <br />
+- If `Fun` is `undefined`, the encoding results in an error if there are unknown values. <br />
+- default: `term_to_json_string/1` <br />
 
 
 
@@ -175,7 +180,7 @@ float_format_option() = {scientific, Decimals::0..249} | {decimals, Decimals::0.
 - The encoded string will contain at most `Decimals` number of digits past the decimal point. <br />
 - If `compact` is provided the trailing zeros at the end of the string are truncated. <br />
 
-For more details, see [erlang:float_to_list/2](http://erlang.org/doc/man/erlang.html#float_to_list-2).
+For more details, see [erlang:float_to_list/2](http://erlang.org/doc/man/erlang.md#float_to_list-2).
 
 ```
   > jsone:encode(1.23).
@@ -192,6 +197,46 @@ For more details, see [erlang:float_to_list/2](http://erlang.org/doc/man/erlang.
 
 
 
+### <a name="type-incomplete">incomplete()</a> ###
+
+
+<pre><code>
+incomplete() = {incomplete, <a href="#type-incomplete_fun">incomplete_fun()</a>}
+</code></pre>
+
+
+
+
+### <a name="type-incomplete_fun">incomplete_fun()</a> ###
+
+
+<pre><code>
+incomplete_fun() = fun((binary() | end_stream) -&gt; <a href="#type-incomplete">incomplete()</a> | <a href="#type-json_value">json_value()</a>)
+</code></pre>
+
+
+
+
+### <a name="type-incomplete_try">incomplete_try()</a> ###
+
+
+<pre><code>
+incomplete_try() = {incomplete, <a href="#type-incomplete_try_fun">incomplete_try_fun()</a>}
+</code></pre>
+
+
+
+
+### <a name="type-incomplete_try_fun">incomplete_try_fun()</a> ###
+
+
+<pre><code>
+incomplete_try_fun() = fun((binary() | end_stream) -&gt; <a href="#type-incomplete_try">incomplete_try()</a> | {ok, <a href="#type-json_value">json_value()</a>, Remainings::binary()} | {error, {Reason::term(), [<a href="#type-stack_item">stack_item()</a>]}})
+</code></pre>
+
+
+
+
 ### <a name="type-json_array">json_array()</a> ###
 
 
@@ -236,7 +281,7 @@ json_object() = <a href="#type-json_object_format_tuple">json_object_format_tupl
 
 
 <pre><code>
-json_object_format_map() = #{}
+json_object_format_map() = map()
 </code></pre>
 
 
@@ -392,7 +437,7 @@ utc_offset_seconds() = -86399..86399
 ## Function Index ##
 
 
-<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#decode-1">decode/1</a></td><td>Equivalent to <a href="#decode-2"><tt>decode(Json, [])</tt></a>.</td></tr><tr><td valign="top"><a href="#decode-2">decode/2</a></td><td>Decodes an erlang term from json text (a utf8 encoded binary).</td></tr><tr><td valign="top"><a href="#encode-1">encode/1</a></td><td>Equivalent to <a href="#encode-2"><tt>encode(JsonValue, [])</tt></a>.</td></tr><tr><td valign="top"><a href="#encode-2">encode/2</a></td><td>Encodes an erlang term into json text (a utf8 encoded binary).</td></tr><tr><td valign="top"><a href="#try_decode-1">try_decode/1</a></td><td>Equivalent to <a href="#try_decode-2"><tt>try_decode(Json, [])</tt></a>.</td></tr><tr><td valign="top"><a href="#try_decode-2">try_decode/2</a></td><td>Decodes an erlang term from json text (a utf8 encoded binary).</td></tr><tr><td valign="top"><a href="#try_encode-1">try_encode/1</a></td><td>Equivalent to <a href="#try_encode-2"><tt>try_encode(JsonValue, [])</tt></a>.</td></tr><tr><td valign="top"><a href="#try_encode-2">try_encode/2</a></td><td>Encodes an erlang term into json text (a utf8 encoded binary).</td></tr></table>
+<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#decode-1">decode/1</a></td><td>Equivalent to <a href="#decode-2"><tt>decode(Json, [])</tt></a>.</td></tr><tr><td valign="top"><a href="#decode-2">decode/2</a></td><td>Decodes an erlang term from json text (a utf8 encoded binary).</td></tr><tr><td valign="top"><a href="#decode_stream-1">decode_stream/1</a></td><td>Equivalent to <a href="#decode_stream-2"><tt>decode_stream(Json, [])</tt></a>.</td></tr><tr><td valign="top"><a href="#decode_stream-2">decode_stream/2</a></td><td>Decodes an Erlang from JSON text in chunks.</td></tr><tr><td valign="top"><a href="#encode-1">encode/1</a></td><td>Equivalent to <a href="#encode-2"><tt>encode(JsonValue, [])</tt></a>.</td></tr><tr><td valign="top"><a href="#encode-2">encode/2</a></td><td>Encodes an erlang term into json text (a utf8 encoded binary).</td></tr><tr><td valign="top"><a href="#ip_address_to_json_string-1">ip_address_to_json_string/1</a></td><td>Convert an IP address into a text representation.</td></tr><tr><td valign="top"><a href="#term_to_json_string-1">term_to_json_string/1</a></td><td>Converts the given term <code>X</code> to its string representation (i.e., the result of <code>io_lib:format("~p", [X])</code>).</td></tr><tr><td valign="top"><a href="#try_decode-1">try_decode/1</a></td><td>Equivalent to <a href="#try_decode-2"><tt>try_decode(Json, [])</tt></a>.</td></tr><tr><td valign="top"><a href="#try_decode-2">try_decode/2</a></td><td>Decodes an erlang term from json text (a utf8 encoded binary).</td></tr><tr><td valign="top"><a href="#try_decode_stream-1">try_decode_stream/1</a></td><td>Equivalent to <a href="#try_decode_stream-2"><tt>try_decode_stream(Json, [])</tt></a>.</td></tr><tr><td valign="top"><a href="#try_decode_stream-2">try_decode_stream/2</a></td><td>Decodes an Erlang from JSON text in chunks.</td></tr><tr><td valign="top"><a href="#try_encode-1">try_encode/1</a></td><td>Equivalent to <a href="#try_encode-2"><tt>try_encode(JsonValue, [])</tt></a>.</td></tr><tr><td valign="top"><a href="#try_encode-2">try_encode/2</a></td><td>Encodes an erlang term into json text (a utf8 encoded binary).</td></tr></table>
 
 
 <a name="functions"></a>
@@ -433,6 +478,43 @@ Raises an error exception if input is not valid json
       in call from jsone:decode/1 (src/jsone.erl, line 71)
 ```
 
+<a name="decode_stream-1"></a>
+
+### decode_stream/1 ###
+
+<pre><code>
+decode_stream(Json::binary()) -&gt; <a href="#type-incomplete">incomplete()</a>
+</code></pre>
+<br />
+
+Equivalent to [`decode_stream(Json, [])`](#decode_stream-2).
+
+<a name="decode_stream-2"></a>
+
+### decode_stream/2 ###
+
+<pre><code>
+decode_stream(Json::binary(), Options::[<a href="#type-decode_option">decode_option()</a>]) -&gt; <a href="#type-incomplete">incomplete()</a>
+</code></pre>
+<br />
+
+Decodes an Erlang from JSON text in chunks.
+
+Instead of returning a result, `{incomplete, fun()}` is returned. The
+returned fun takes a single argument and it should called to continue the
+decoding. When all the input has been provided, the fun should be called with
+`end_stream` to signal the end of input and then the fun returns a result or
+an error, as `decode/2` would do.
+
+```
+  1> {incomplete, F1} = jsone:decode(<<"[1,2,">>, []).
+  {incomplete, #Fun<jsone.44.79398840>}
+  2> {incomplete, F2} = F1(<<"3]>>).
+  {incomplete, #Fun<jsone.45.79398840>}
+  3> F2(end_stream).
+  [1,2,3]
+```
+
 <a name="encode-1"></a>
 
 ### encode/1 ###
@@ -467,6 +549,47 @@ Raises an error exception if input is not an instance of type `json_value()`
        in call from jsone:encode/1 (src/jsone.erl, line 97)
 ```
 
+<a name="ip_address_to_json_string-1"></a>
+
+### ip_address_to_json_string/1 ###
+
+<pre><code>
+ip_address_to_json_string(X::<a href="inet.md#type-ip_address">inet:ip_address()</a> | any()) -&gt; {ok, <a href="#type-json_string">json_string()</a>} | error
+</code></pre>
+<br />
+
+Convert an IP address into a text representation.
+
+This function can be specified as the value of the `map_unknown_value` encoding option.
+
+This function formats IPv6 addresses by following the recommendation defined in RFC 5952.
+Note that the trailing 32 bytes of special IPv6 addresses such as IPv4-Compatible (::X.X.X.X),
+IPv4-Mapped (::ffff:X.X.X.X), IPv4-Translated (::ffff:0:X.X.X.X) and IPv4/IPv6 translation
+(64:ff9b::X.X.X.X and 64:ff9b:1::X.X.X.X ~ 64:ff9b:1:ffff:ffff:ffff:X.X.X.X) are formatted
+using the IPv4 format.
+
+```
+  > EncodeOpt = [{map_unknown_value, fun jsone:ip_address_to_json_string/1}].
+  > jsone:encode(#{ip => {127, 0, 0, 1}}, EncodeOpt).
+  <<"{\"ip\":\"127.0.0.1\"}">>
+  > {ok, Addr} = inet:parse_address("2001:DB8:0000:0000:0001:0000:0000:0001").
+  > jsone:encode(Addr, EncodeOpt).
+  <<"\"2001:db8::1:0:0:1\"">>
+  > jsone:encode([foo, {0, 0, 0, 0, 0, 16#FFFF, 16#7F00, 16#0001}], EncodeOpt).
+  <<"[\"foo\",\"::ffff:127.0.0.1\"]">>
+```
+
+<a name="term_to_json_string-1"></a>
+
+### term_to_json_string/1 ###
+
+<pre><code>
+term_to_json_string(X::term()) -&gt; {ok, <a href="#type-json_string">json_string()</a>} | error
+</code></pre>
+<br />
+
+Converts the given term `X` to its string representation (i.e., the result of `io_lib:format("~p", [X])`).
+
 <a name="try_decode-1"></a>
 
 ### try_decode/1 ###
@@ -498,6 +621,37 @@ Decodes an erlang term from json text (a utf8 encoded binary)
                                 [{line,208}]}]}}
 ```
 
+<a name="try_decode_stream-1"></a>
+
+### try_decode_stream/1 ###
+
+<pre><code>
+try_decode_stream(Json::binary()) -&gt; <a href="#type-incomplete_try">incomplete_try()</a>
+</code></pre>
+<br />
+
+Equivalent to [`try_decode_stream(Json, [])`](#try_decode_stream-2).
+
+<a name="try_decode_stream-2"></a>
+
+### try_decode_stream/2 ###
+
+<pre><code>
+try_decode_stream(Json::binary(), Options::[<a href="#type-decode_option">decode_option()</a>]) -&gt; <a href="#type-incomplete_try">incomplete_try()</a>
+</code></pre>
+<br />
+
+Decodes an Erlang from JSON text in chunks.
+
+```
+  1> {incomplete, F1} = jsone:try_decode(<<"[1,2,">>, []).
+  {incomplete, #Fun<jsone.46.79398840>}
+  2> {incomplete, F2} = F1(<<"3] \"next value\"">>).
+  {incomplete, #Fun<jsone.47.79398840>}
+  2> F2(end_stream).
+  {ok,[1,2,3],<<" \"next value\"">>}
+```
+
 <a name="try_encode-1"></a>
 
 ### try_encode/1 ###
@@ -528,3 +682,4 @@ Encodes an erlang term into json text (a utf8 encoded binary)
                                 [hoge,[{array_values,[2]}],<<"[1,">>],
                                 [{line,86}]}]}}
 ```
+

src/jsone.erl

@@ -31,6 +31,8 @@
 %%--------------------------------------------------------------------------------
 -export([decode/1, decode/2,
          try_decode/1, try_decode/2,
+         decode_stream/1, decode_stream/2,
+         try_decode_stream/1, try_decode_stream/2,
          encode/1, encode/2,
          try_encode/1, try_encode/2,
          term_to_json_string/1,
@@ -49,6 +51,8 @@
               json_object_format_map/0,
               json_scalar/0,
 
+              incomplete/0,
+              incomplete_try/0,
               encode_option/0,
               decode_option/0,
               float_format_option/0,
@@ -130,6 +134,17 @@
 
 -type json_scalar() :: json_boolean() | json_number() | json_string().
 
+-type incomplete_fun() :: fun((binary() | end_stream) ->
+                                     incomplete() | json_value()).
+-type incomplete() :: {incomplete, incomplete_fun()}.
+
+-type incomplete_try_fun() :: fun((binary() | end_stream) ->
+                                         incomplete_try() |
+                                         {ok, json_value(), Remainings :: binary()} |
+                                         {error, {Reason :: term(), [stack_item()]}}).
+
+-type incomplete_try() :: {incomplete, incomplete_try_fun()}.
+
 -type float_format_option() :: {scientific, Decimals :: 0 .. 249} | {decimals, Decimals :: 0 .. 253} | compact.
 %% `scientific': <br />
 %% - The float will be formatted using scientific notation with `Decimals' digits of precision. <br />
@@ -287,7 +302,6 @@
 %% - If the value is `first' then the first duplicate key/value is returned.  <br />
 %% - If the value is `last' then the last duplicate key/value is returned.
 %% - default: `first'<br />
-%%
 
 -type stack_item() :: {Module :: module(),
                        Function :: atom(),
@@ -346,7 +360,9 @@ decode(Json, Options) ->
 
 
 %% @equiv try_decode(Json, [])
--spec try_decode(binary()) -> {ok, json_value(), Remainings :: binary()} | {error, {Reason :: term(), [stack_item()]}}.
+-spec try_decode(binary()) ->
+          {ok, json_value(), Remainings :: binary()} |
+          {error, {Reason :: term(), [stack_item()]}}.
 try_decode(Json) ->
     try_decode(Json, []).
 
@@ -363,11 +379,67 @@ try_decode(Json) ->
 %%                               [{line,208}]}]}}
 %% '''
 -spec try_decode(binary(), [decode_option()]) ->
-          {ok, json_value(), Remainings :: binary()} | {error, {Reason :: term(), [stack_item()]}}.
+          {ok, json_value(), Remainings :: binary()} |
+          {error, {Reason :: term(), [stack_item()]}}.
 try_decode(Json, Options) ->
     jsone_decode:decode(Json, Options).
 
 
+%% @equiv decode_stream(Json, [])
+-spec decode_stream(binary()) -> incomplete().
+decode_stream(Json) ->
+    decode_stream(Json, []).
+
+
+%% @doc Decodes an Erlang from JSON text in chunks.
+%%
+%% Instead of returning a result, `{incomplete, fun()}' is returned. The
+%% returned fun takes a single argument and it should called to continue the
+%% decoding. When all the input has been provided, the fun should be called with
+%% `end_stream' to signal the end of input and then the fun returns a result or
+%% an error, as `decode/2' would do.
+%%
+%% ```
+%% 1> {incomplete, F1} = jsone:decode(<<"[1,2,">>, []).
+%% {incomplete, #Fun<jsone.44.79398840>}
+%% 2> {incomplete, F2} = F1(<<"3]>>).
+%% {incomplete, #Fun<jsone.45.79398840>}
+%% 3> F2(end_stream).
+%% [1,2,3]
+%% '''
+
+-spec decode_stream(binary(), [decode_option()]) -> incomplete().
+decode_stream(Json, Options) ->
+    try
+        {incomplete, ContinueFun} = try_decode_stream(Json, Options),
+        {incomplete, replace_incomplete_fun(ContinueFun)}
+    catch
+        error:{badmatch, {error, {Reason, [StackItem]}}} ?CAPTURE_STACKTRACE->
+            erlang:raise(error, Reason, [StackItem | ?GET_STACKTRACE])
+    end.
+
+
+%% @equiv try_decode_stream(Json, [])
+-spec try_decode_stream(binary()) -> incomplete_try().
+try_decode_stream(Json) ->
+    try_decode_stream(Json, []).
+
+
+%% @doc Decodes an Erlang from JSON text in chunks.
+%%
+%% ```
+%% 1> {incomplete, F1} = jsone:try_decode(<<"[1,2,">>, []).
+%% {incomplete, #Fun<jsone.46.79398840>}
+%% 2> {incomplete, F2} = F1(<<"3] \"next value\"">>).
+%% {incomplete, #Fun<jsone.47.79398840>}
+%% 2> F2(end_stream).
+%% {ok,[1,2,3],<<" \"next value\"">>}
+%% '''
+-spec try_decode_stream(binary(), [decode_option()]) -> incomplete_try().
+try_decode_stream(Json, Options) ->
+    jsone_decode:decode_stream(Json, Options).
+
+
 %% @equiv encode(JsonValue, [])
 -spec encode(json_value()) -> binary().
 encode(JsonValue) ->
@@ -471,3 +543,25 @@ check_decode_remainings(<<$\n, Bin/binary>>) ->
     check_decode_remainings(Bin);
 check_decode_remainings(<<Bin/binary>>) ->
     erlang:error(badarg, [Bin]).
+
+%% Replace the fun in `{incomplete, Fun}' from `jsone_decode:decode_steram/2'
+%% with a fun that returns the same result as `jsone:decode/1,2'.
+replace_incomplete_fun(Fun) ->
+    fun (end_stream) ->
+            try
+                {ok, Value, Remainings} = Fun(end_stream),
+                check_decode_remainings(Remainings),
+                Value
+            catch
+                error:{badmatch, {error, {Reason, [StackItem]}}} ?CAPTURE_STACKTRACE->
+                    erlang:raise(error, Reason, [StackItem | ?GET_STACKTRACE])
+            end;
+        (Input) ->
+            try
+                {incomplete, ContinueFun} = Fun(Input),
+                {incomplete, replace_incomplete_fun(ContinueFun)}
+            catch
+                error:{badmatch, {error, {Reason, [StackItem]}}} ?CAPTURE_STACKTRACE->
+                    erlang:raise(error, Reason, [StackItem | ?GET_STACKTRACE])
+            end
+    end.