added bytes_to_utf8_lossy fn; updated README

Author: iliabylich

README.md

@@ -6,9 +6,31 @@ Each node has its own JavaScript class, so
 1. it's possible to dinstinguish them by checking `instanceof`
 2. they can be extended in pure JavaScript
 
-Here's how it looks like:
+Basic usage:
 
 ```js
+const inspect = require('util').inspect
+const { parse, Send } = require('lib-ruby-parser');
+
+function print_parse_result(parser_result) {
+    console.log(inspect(parser_result, { showHidden: false, depth: null }))
+}
+
+// This function must be defined by you.
+// It takes a string and return an array of bytes.
+// The following code is just an example:
+function bytes(str) {
+    const bytes = unescape(encodeURIComponent(str)).split('').map(c => c.charCodeAt(0))
+    return new Uint8Array(bytes)
+}
+
+const input = bytes("2 + 3 # x");
+const options = { record_tokens: true };
+const result = parse(input, options);
+print_parser_result(result);
+
+// prints:
+
 ParserResult {
   ast: Send {
     recv: Int {
@@ -34,37 +56,58 @@ ParserResult {
   tokens: [
     Token {
       name: 'tINTEGER',
-      value: '2',
+      value: Uint8Array(1) [ 50 ],
       loc: Loc { begin: 0, end: 1 }
     },
-    Token { name: 'tPLUS', value: '+', loc: Loc { begin: 2, end: 3 } },
+    Token {
+      name: 'tPLUS',
+      value: Uint8Array(1) [ 43 ],
+      loc: Loc { begin: 2, end: 3 }
+    },
     Token {
       name: 'tINTEGER',
-      value: '3',
+      value: Uint8Array(1) [ 51 ],
       loc: Loc { begin: 4, end: 5 }
     },
-    Token { name: 'EOF', value: '', loc: Loc { begin: 5, end: 5 } }
+    Token {
+      name: 'tNL',
+      value: Uint8Array(1) [ 10 ],
+      loc: Loc { begin: 8, end: 9 }
+    },
+    Token {
+      name: 'EOF',
+      value: Uint8Array(0) [],
+      loc: Loc { begin: 8, end: 8 }
+    }
   ],
   diagnostics: [],
-  comments: [],
+  comments: [
+    Comment {
+      kind: 'inline',
+      location: Range { begin_pos: 6, end_pos: 9 }
+    }
+  ],
   magic_comments: [],
-  input: '2 + 3'
+  input: Uint8Array(9) [
+    50, 32, 43,  32, 51,
+    32, 35, 32, 120
+  ]
 }
 ```
 
 ## API
 
 tldr; all classes mirror Rust implementation.
 
-TypeScript definition:
+Rough TypeScript definition:
 
 ```ts
 interface Loc { begin: number, end: number }
 interface Range { begin_pos: number, end_pos: number }
 
 interface Token {
     name: string,
-    value: string,
+    value: Uint8Array,
     loc: Loc
 }
 
@@ -93,3 +136,55 @@ type Node = Args | Class | ... /* other nodes */;
 
 function parse(code: String): ParserResult
 ```
+
+`String` and `Symbol` nodes are slightly exceptional as they contain `StringValue` Rust structure that is a `Uint8Array` here.
+
+This structure can be converted into JS `String` by using `bytes_to_utf8_lossy` function (keep in mind that it replaces unknown chars with a special `Unicode Replacement Character U+FFFD`, if you want some other strategy you are free to define your own converting function):
+
+```js
+const { parse, bytes_to_utf8_lossy } = require(path_to_require)
+
+const result = parse(bytes('"a\\xFFb"'), { record_tokens: true });
+console.log(result.ast)
+console.log(bytes_to_utf8_lossy(result.ast.value))
+
+// prints
+
+Str {
+  value: Uint8Array(3) [ 97, 255, 98 ], // "a" = 97, "\xFF" = 255, "b" = 98
+  begin_l: Range { begin_pos: 0, end_pos: 1 },
+  end_l: Range { begin_pos: 7, end_pos: 8 },
+  expression_l: Range { begin_pos: 0, end_pos: 8 }
+}
+
+a�b
+```
+
+## Encodings
+
+If you want to support encodings other than UTF-8/ASCII-8BIT/BINARY you need a custom decoder:
+
+```js
+const custom_decoder = (encoding: String, input: Uint8Array) => {
+  // Do some **real** decoding into UTF-8 here
+  //
+  // Here for simplicity we convert all "2" into "3"
+  //
+  assert(encoding === "US-ASCII");
+  assert(input === bytes("# encoding: us-ascii\n2 + 2"));
+
+  return bytes("# encoding: us-ascii\n3 + 3");
+}
+
+const result = parse(
+  bytes("# encoding: us-ascii\n2 + 2"),
+  { custom_decoder }
+);
+assert(result.ast.recv.value === "3");
+```
+
+## Platform support
+
+Currently NPM packages include pre-compiled `.node` files for Mac and Linux.
+
+If you need Windows support first go to [C++ bindings](https://github.com/lib-ruby-parser/cpp-bindings) and create an issue there. This repo is just a wrapper around its builds.

build-convert/build.rs

@@ -115,7 +115,6 @@ fn main() {
 #define LIB_RUBY_PARSER_CONVERT_GEN_H
 
 #include <napi.h>
-#include <iostream>
 #include \"lib-ruby-parser.h\"
 
 template<class> inline constexpr bool always_false_v = false;

bytes.h

@@ -11,17 +11,16 @@ namespace lib_ruby_parser_node
     class Bytes : public lib_ruby_parser::Bytes
     {
     public:
-        explicit Bytes(lib_ruby_parser::Bytes bytes)
+        explicit Bytes(lib_ruby_parser::Bytes bytes) : lib_ruby_parser::Bytes(bytes.into_ptr())
         {
-            lib_ruby_parser::Bytes(bytes.into_ptr());
         }
 
-        Napi::Value to_v8_value(Napi::Env env) const
+        Napi::Value ToV8(Napi::Env env) const
         {
-            Napi::TypedArray array = Napi::Uint8Array::New(env, this->size());
+            Napi::Uint8Array array = Napi::Uint8Array::New(env, this->size());
             for (size_t i = 0; i < this->size(); i++)
             {
-                array.Set(i, this->at(i));
+                array[i] = this->at(i);
             }
             return array;
         }

convert.h

@@ -4,6 +4,7 @@
 #include <napi.h>
 #include "lib-ruby-parser.h"
 #include "convert_gen.h"
+#include "bytes.h"
 
 namespace lib_ruby_parser_node
 {
@@ -111,7 +112,7 @@ namespace lib_ruby_parser_node
     {
         return TokenCtor.New({
             Napi::Value::From(env, token.name()),
-            Napi::Value::From(env, token.token_value.to_string_lossy()),
+            Napi::Value::From(env, Bytes(std::move(token.token_value)).ToV8(env)),
             convert(std::move(token.loc), env),
         });
     }
@@ -233,7 +234,7 @@ namespace lib_ruby_parser_node
             convert(std::move(result->diagnostics), env),
             convert(std::move(result->comments), env),
             convert(std::move(result->magic_comments), env),
-            Napi::String::New(env, result->input.to_string_lossy()),
+            Bytes(std::move(result->input)).ToV8(env),
         });
     }
 

custom_decoder.h

@@ -4,7 +4,6 @@
 #include <napi.h>
 #include "lib-ruby-parser.h"
 #include "bytes.h"
-#include <iostream>
 
 namespace lib_ruby_parser_node
 {

node_bindings.cc

@@ -4,7 +4,6 @@
 #include "custom_decoder.h"
 #include "bytes.h"
 #include "result.h"
-#include <iostream>
 #include <variant>
 #include <tuple>
 
@@ -69,8 +68,6 @@ namespace lib_ruby_parser_node
 
     Result<std::unique_ptr<lib_ruby_parser::ParserResult>> parse(const Napi::CallbackInfo &info)
     {
-        Napi::Env env = info.Env();
-
         if (info.Length() != 2)
         {
             return "Wrong number of arguments (expected 2)";
@@ -102,11 +99,39 @@ namespace lib_ruby_parser_node
         return convert(result.get(), env);
     }
 
+    Result<lib_ruby_parser::Bytes> bytes_to_utf8_lossy(const Napi::CallbackInfo &info)
+    {
+        if (info.Length() != 1)
+        {
+            return "Wrong number of arguments (expected 1)";
+        }
+
+        UNWRAP_RESULT(bytes, Bytes::FromV8(info[0]));
+
+        return std::move(bytes);
+    }
+
+    Napi::Value js_bytes_to_utf8_lossy(const Napi::CallbackInfo &info)
+    {
+        auto env = info.Env();
+
+        auto result = bytes_to_utf8_lossy(info);
+        if (result.is_err())
+        {
+            return JsThrow(env, result.get_err());
+        }
+        auto bytes = result.get();
+        return Napi::String::New(info.Env(), bytes.to_string_lossy());
+    }
+
     Napi::Object Init(Napi::Env env, Napi::Object exports)
     {
         exports.Set(Napi::String::New(env, "parse"),
                     Napi::Function::New(env, js_parse));
 
+        exports.Set(Napi::String::New(env, "bytes_to_utf8_lossy"),
+                    Napi::Function::New(env, js_bytes_to_utf8_lossy));
+
         InitCustomTypes(env, exports);
         InitNodeTypes(env, exports);
 

test.js

@@ -8,7 +8,7 @@ if (!path_to_require) {
 }
 
 console.log(`requiring ${path_to_require}`)
-const { parse, Range, Send, Self_, Int, Token, Loc, Diagnostic } = require(path_to_require)
+const { parse, Range, Send, Self_, Int, Token, Loc, Diagnostic, bytes_to_utf8_lossy } = require(path_to_require)
 const assert = require('assert').strict
 const inspect = require('util').inspect
 
@@ -47,13 +47,13 @@ function assert_range(range, begin_pos, end_pos, prefix) {
 function assert_token(token, name, value, loc, prefix) {
     assert(token instanceof Token, `[${prefix}] expected ${token} to be an instance of Token`)
     assert_eq(token.name, name, `[${prefix}].name`)
-    assert_eq(token.value, value, `[${prefix}].value`)
+    assert_eq(bytes_to_utf8_lossy(token.value), value, `[${prefix}].value`)
     assert_eq(token.loc.begin, loc.begin, `[${prefix}].loc.begin`)
     assert_eq(token.loc.end, loc.end, `[${prefix}].loc.end`)
 }
 
-function print_parse_result(parse_result) {
-    console.log(inspect(result, { showHidden: false, depth: null }))
+function print_parser_result(parser_result) {
+    console.log(inspect(parser_result, { showHidden: false, depth: null }))
 }
 
 function bytes(str) {