[WIP] Libdeflate

binding.gyp

@@ -8,14 +8,14 @@
     ['AR', '<(module_root_dir)/mason_packages/.link/bin/llvm-ar'],
     ['NM', '<(module_root_dir)/mason_packages/.link/bin/llvm-nm']
   ],
-  'includes': [ 'common.gypi' ], # brings in a default set of options that are inherited from gyp
+  'includes': [ 'common.gypi'], # brings in a default set of options that are inherited from gyp
   'variables': { # custom variables we use specific to this file
       'error_on_warnings%':'true', # can be overriden by a command line variable because of the % sign using "WERROR" (defined in Makefile)
-      # Use this variable to silence warnings from mason dependencies and from NAN
+      # Use this variable to silence warnings from mason dependencies
       # It's a variable to make easy to pass to
       # cflags (linux) and xcode (mac)
       'system_includes': [
-        "-isystem <(module_root_dir)/<!(node -e \"require('nan')\")",
+        "-isystem <!@(node -p \"require('node-addon-api').include.slice(1,-1)\")",
         "-isystem <(module_root_dir)/mason_packages/.link/include/"
       ],
       # Flags we pass to the compiler to ensure the compiler
@@ -77,8 +77,11 @@
         './src/module.cpp',
         './src/vtcomposite.cpp'
       ],
+      'libraries':[
+          '<(module_root_dir)/mason_packages/.link/lib/libdeflate.a'
+      ],
       'ldflags': [
-        '-Wl,-z,now',
+        '-Wl,-z,now'
       ],
       'conditions': [
         ['error_on_warnings == "true"', {
@@ -88,6 +91,7 @@
             }
         }]
       ],
+      #'include_dirs' : ["<!@(node -p \"require('node-addon-api').include\")"],
       'cflags_cc': [
           '<@(system_includes)',
           '<@(compiler_checks)'

mason-versions.ini

@@ -1,14 +1,14 @@
 [headers]
 boost=1.66.0
 geometry=96d3505
-gzip-hpp=0.1.0
 protozero=1.6.2
 spatial-algorithms=0.1.0
 variant=1.1.5
 vtzero=2915725
 [compiled]
+libdeflate=1.0
 clang++=5.0.1
 clang-tidy=5.0.1
 clang-format=5.0.1
 llvm-cov=5.0.1
-binutils=2.30
+binutils=2.30

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mapbox/vtcomposite",
-  "version": "0.1.1",
-  "description": "Skeleton for bindings to C++ libraries for Node.js using NAN",
+  "version": "0.2.1",
+  "description": "Compositing operations on Vector Tiles (c++ bindings using N-API)",
   "url": "http://github.com/mapbox/vtcomposite",
   "main": "./lib/index.js",
   "repository": {
@@ -18,7 +18,7 @@
   "license": "ISC",
   "dependencies": {
     "mason-js-sdk": "^0.1.4",
-    "nan": "~2.10.0",
+    "node-addon-api": "^1.5.0",
     "node-pre-gyp": "~0.10.2"
   },
   "devDependencies": {

scripts/sanitize.sh

@@ -51,4 +51,3 @@ else
     LD_PRELOAD=${MASON_LLVM_RT_PRELOAD} \
       npm test
 fi
-

src/deflate.hpp

@@ -0,0 +1,166 @@
+#pragma once
+
+// libdeflate
+#include <libdeflate.h>
+// std
+#include <limits>
+#include <stdexcept>
+#include <memory>
+
+namespace deflate {
+
+inline bool is_zlib(char const* data) noexcept
+{
+    return (static_cast<uint8_t>(data[0]) == 0x78 &&
+            (static_cast<uint8_t>(data[1]) == 0x9C ||
+             static_cast<uint8_t>(data[1]) == 0x01 ||
+             static_cast<uint8_t>(data[1]) == 0xDA ||
+             static_cast<uint8_t>(data[1]) == 0x5E));
+}
+
+inline bool is_gzip(char const* data) noexcept
+{
+    return (static_cast<uint8_t>(data[0]) == 0x1F && static_cast<uint8_t>(data[1]) == 0x8B);
+}
+
+inline bool is_compressed(char const* data, std::size_t size) noexcept
+{
+    return size > 2 && (is_gzip(data) || is_zlib(data));
+}
+
+class Compressor
+{
+    struct cleanup_compressor
+    {
+        void operator()(libdeflate_compressor* ptr) const
+        {
+            if (ptr) libdeflate_free_compressor(ptr);
+        }
+    };
+    std::size_t max_;
+    int level_;
+    std::unique_ptr<libdeflate_compressor, cleanup_compressor> compressor_;
+    // make noncopyable
+    Compressor(Compressor const&) = delete;
+    Compressor& operator=(Compressor const&) = delete;
+
+  public:
+    Compressor(int level = 6,
+               std::size_t max_bytes = 2000000000) // by default refuse operation if uncompressed data is > 2GB
+        : max_{max_bytes},
+          level_{level},
+          compressor_{libdeflate_alloc_compressor(level_)}
+    {
+        if (!compressor_)
+        {
+            throw std::runtime_error("libdeflate_alloc_compressor failed");
+        }
+    }
+
+    template <typename OutputType>
+    void operator()(OutputType& output,
+                    char const* data,
+                    std::size_t size) const
+    {
+        if (size > max_)
+        {
+            throw std::runtime_error("size may use more memory than intended when decompressing");
+        }
+
+        std::size_t max_compressed_size = libdeflate_gzip_compress_bound(compressor_.get(), size);
+        // TODO: sanity check this before allocating
+        if (max_compressed_size > output.size())
+        {
+            output.resize(max_compressed_size);
+        }
+
+        std::size_t actual_compressed_size = libdeflate_gzip_compress(compressor_.get(),
+                                                                      data,
+                                                                      size,
+                                                                      const_cast<char*>(output.data()),
+                                                                      max_compressed_size);
+        if (actual_compressed_size == 0)
+        {
+            throw std::runtime_error("actual_compressed_size 0");
+        }
+        output.resize(actual_compressed_size);
+    }
+};
+
+class Decompressor
+{
+    struct cleanup_decompressor
+    {
+        void operator()(libdeflate_decompressor* ptr) const
+        {
+            if (ptr) libdeflate_free_decompressor(ptr);
+        }
+    };
+
+    std::size_t const max_;
+    std::unique_ptr<libdeflate_decompressor, cleanup_decompressor> decompressor_;
+    // noncopyable
+    Decompressor(Decompressor const&) = delete;
+    Decompressor& operator=(Decompressor const&) = delete;
+
+  public:
+    Decompressor(std::size_t max_bytes = 2147483648u) // by default refuse operation if required uutput buffer is > 2GB
+        : max_{max_bytes},
+          decompressor_{libdeflate_alloc_decompressor(), cleanup_decompressor()}
+    {
+        if (!decompressor_)
+        {
+            throw std::runtime_error("libdeflate_alloc_decompressor failed");
+        }
+    }
+
+    template <typename OutputType>
+    void operator()(OutputType& output,
+                    char const* data,
+                    std::size_t size) const
+    {
+        if (is_gzip(data))
+            apply(output, libdeflate_gzip_decompress, data, size);
+        else if (is_zlib(data))
+            apply(output, libdeflate_zlib_decompress, data, size);
+    }
+
+    template <typename OutputType, typename Fun>
+    void apply(OutputType& output, Fun fun,
+               char const* data,
+               std::size_t size) const
+    {
+        std::size_t actual_size;
+        std::size_t uncompressed_size_guess = std::min(size * 4, max_);
+        output.resize(uncompressed_size_guess);
+        libdeflate_result result;
+        for (;;)
+        {
+            result = fun(decompressor_.get(),
+                         data,
+                         size,
+                         const_cast<char*>(output.data()),
+                         output.size(), &actual_size);
+            if (result == LIBDEFLATE_SUCCESS)
+            {
+                output.resize(actual_size);
+                break;
+            }
+            else if (result == LIBDEFLATE_INSUFFICIENT_SPACE)
+            {
+                if (output.size() == max_)
+                {
+                    throw std::runtime_error("request to resize output buffer can't exceed maximum limit");
+                }
+                std::size_t new_size = std::min(output.size() << 1, max_);
+                output.resize(new_size);
+            }
+            else //LIBDEFLATE_BAD_DATA
+            {
+                throw std::runtime_error("bad data: did not succeed");
+            }
+        }
+    }
+};
+
+} // namespace deflate

src/module.cpp

@@ -1,9 +1,10 @@
 #include "vtcomposite.hpp"
-#include <nan.h>
+#include <napi.h>
 
-NAN_MODULE_INIT(init)
+Napi::Object init(Napi::Env env, Napi::Object exports)
 {
-    Nan::SetMethod(target, "composite", vtile::composite);
+    exports.Set(Napi::String::New(env, "composite"), Napi::Function::New(env, vtile::composite));
+    return exports;
 }
 
-NODE_MODULE(module, init) // NOLINT
+NODE_API_MODULE(module, init) // NOLINT

src/module_utils.hpp

@@ -1,28 +1,13 @@
 #pragma once
-#include <nan.h>
+#include <napi.h>
 
 namespace utils {
 
-/*
-* This is an internal function used to return callback error messages instead of
-* throwing errors.
-* Usage:
-*
-* v8::Local<v8::Function> callback;
-* return CallbackError("error message", callback);  // "return" is important to
-* prevent duplicate callbacks from being fired!
-*
-*
-* "inline" is important here as well. See for more contex:
-* - https://github.com/mapbox/cpp/blob/master/glossary.md#inline-keyword
-* - https://github.com/mapbox/node-cpp-skel/pull/52#discussion_r126847394 for
-* context
-*
-*/
-inline void CallbackError(std::string message, v8::Local<v8::Function> func)
+inline Napi::Value CallbackError(std::string const& message, Napi::CallbackInfo const& info)
 {
-    Nan::Callback cb(func);
-    v8::Local<v8::Value> argv[1] = {Nan::Error(message.c_str())};
-    Nan::Call(cb, 1, argv);
+    Napi::Object obj = Napi::Object::New(info.Env());
+    obj.Set("message", message);
+    auto func = info[info.Length() - 1].As<Napi::Function>();
+    return func.Call({obj});
 }
-} // namespace utils
+} // namespace utils