diff --git a/bench/vtquery.bench.js b/bench/vtquery.bench.js index bd7f87e8..38f3968c 100644 --- a/bench/vtquery.bench.js +++ b/bench/vtquery.bench.js @@ -3,7 +3,7 @@ const argv = require('minimist')(process.argv.slice(2)); if (!argv.iterations || !argv.concurrency) { console.error('Please provide desired iterations, concurrency'); - console.error('Example: \nnode bench/vtquery.bench.js --iterations 50 --concurrency 10'); + console.error('Example: \nnode bench/vtquery.bench.js --gzipjs --iterations 50 --concurrency 10'); process.exit(1); } @@ -19,6 +19,7 @@ const assert = require('assert'); const Queue = require('d3-queue').queue; const vtquery = require('../lib/index.js'); const rules = require('./rules'); +const zlib = require('zlib'); let ruleCount = 1; // run each rule synchronously @@ -32,21 +33,51 @@ ruleQueue.awaitAll(function(err, res) { process.stdout.write('\n'); }); +function decompress(tile,cb) { + zlib.gunzip(tile.buffer,function(err, buf) { + tile.buffer = buf; + return cb(err); + }); +} + + function runRule(rule, ruleCallback) { process.stdout.write(`\n${ruleCount}: ${rule.description} ... `); let runs = 0; let runsQueue = Queue(); + // compress tiles such that the + // benchmark has to decompress on the fly + rule.tiles.forEach(function(t) { + t.buffer = zlib.gzipSync(t.buffer); + }); function run(cb) { - vtquery(rule.tiles, rule.queryPoint, rule.options, function(err, result) { - if (err) { - return cb(err); - } - ++runs; - return cb(); - }); + if (argv.gzipjs) { + const gzipQueue = Queue(); + rule.tiles.forEach(function(t) { + gzipQueue.defer(decompress,t); + }); + gzipQueue.awaitAll(function(error) { + if (error) return cb(error); + vtquery(rule.tiles, rule.queryPoint, rule.options, function(err, result) { + if (err) { + return cb(err); + } + ++runs; + return cb(); + }); + }); + } else { + vtquery(rule.tiles, rule.queryPoint, rule.options, function(err, result) { + if (err) { + return cb(err); + } + ++runs; + return cb(); + }); + } } // Start monitoring time before async work begins within the defer iterator below. diff --git a/binding.gyp b/binding.gyp index 6540cb7c..b41ee74f 100644 --- a/binding.gyp +++ b/binding.gyp @@ -63,6 +63,9 @@ './src/module.cpp', './src/vtquery.cpp' ], + "link_settings": { + "libraries": [ "-lz" ] + }, 'ldflags': [ '-Wl,-z,now', ], diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh index 2850e997..e3e37340 100755 --- a/scripts/install_deps.sh +++ b/scripts/install_deps.sh @@ -20,3 +20,4 @@ install spatial-algorithms 0.1.0 install boost 1.65.1 install cheap-ruler 2.5.3 install vector-tile f4728da +install gzip bb80aac \ No newline at end of file diff --git a/scripts/setup.sh b/scripts/setup.sh index 734164ae..9814fdc1 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -4,7 +4,7 @@ set -eu set -o pipefail export MASON_RELEASE="${MASON_RELEASE:-7c4f95d}" -export MASON_LLVM_RELEASE="${MASON_LLVM_RELEASE:-5.0.0}" +export MASON_LLVM_RELEASE="${MASON_LLVM_RELEASE:-5.0.1}" PLATFORM=$(uname | tr A-Z a-z) if [[ ${PLATFORM} == 'darwin' ]]; then @@ -89,8 +89,8 @@ function run() { echo "export MSAN_SYMBOLIZER_PATH=${llvm_toolchain_dir}/bin/llvm-symbolizer" >> ${config} echo "export UBSAN_OPTIONS=print_stacktrace=1" >> ${config} echo "export LSAN_OPTIONS=suppressions=${SUPPRESSION_FILE}" >> ${config} - echo "export ASAN_OPTIONS=symbolize=1:abort_on_error=1:detect_container_overflow=1:check_initialization_order=1:detect_stack_use_after_return=1" >> ${config} - echo 'export MASON_SANITIZE="-fsanitize=address,undefined,integer -fno-sanitize=vptr,function"' >> ${config} + echo "export ASAN_OPTIONS=detect_leaks=1:symbolize=1:abort_on_error=1:detect_container_overflow=1:check_initialization_order=1:detect_stack_use_after_return=1" >> ${config} + echo 'export MASON_SANITIZE="-fsanitize=address,undefined,integer,leak -fno-sanitize=vptr,function"' >> ${config} echo 'export MASON_SANITIZE_CXXFLAGS="${MASON_SANITIZE} -fno-sanitize=vptr,function -fsanitize-address-use-after-scope -fno-omit-frame-pointer -fno-common"' >> ${config} echo 'export MASON_SANITIZE_LDFLAGS="${MASON_SANITIZE}"' >> ${config} diff --git a/src/vtquery.cpp b/src/vtquery.cpp index da8ab9d9..3316b4ba 100644 --- a/src/vtquery.cpp +++ b/src/vtquery.cpp @@ -3,6 +3,8 @@ #include #include +#include +#include #include #include #include @@ -28,9 +30,12 @@ const char* getGeomTypeString(int enumVal) { return GeomTypeStrings[enumVal]; } +using materialized_prop_type = std::pair; + /// main storage item for returning to the user struct ResultObject { std::vector properties_vector; + std::vector properties_vector_materialized; std::string layer_name; mapbox::geometry::point coordinates; double distance; @@ -39,6 +44,7 @@ struct ResultObject { uint64_t id; ResultObject() : properties_vector(), + properties_vector_materialized(), layer_name(), coordinates(0.0, 0.0), distance(std::numeric_limits::max()), @@ -150,11 +156,9 @@ struct property_value_visitor { }; /// used to create the final v8 (JSON) object to return to the user -void set_property(vtzero::property const& property, +void set_property(materialized_prop_type const& property, v8::Local& properties_obj) { - - auto val = vtzero::convert_property_value(property.value()); - mapbox::util::apply_visitor(property_value_visitor{properties_obj, std::string(property.key())}, val); + mapbox::util::apply_visitor(property_value_visitor{properties_obj, property.first}, property.second); } GeomType get_geometry_type(vtzero::feature const& f) { @@ -268,11 +272,25 @@ struct Worker : Nan::AsyncWorker { // query point lng/lat geometry.hpp point (used for distance calculation later on) mapbox::geometry::point query_lnglat{data.longitude, data.latitude}; - // for each tile + gzip::Decompressor decompressor; + std::string uncompressed; + std::vector buffers; + std::vector> tiles; + tiles.reserve(data.tiles.size()); for (auto const& tile_ptr : data.tiles) { TileObject const& tile_obj = *tile_ptr; - vtzero::vector_tile tile{tile_obj.data}; + if (gzip::is_compressed(tile_obj.data.data(), tile_obj.data.size())) { + decompressor.decompress(uncompressed, tile_obj.data.data(), tile_obj.data.size()); + buffers.emplace_back(std::move(uncompressed)); + tiles.emplace_back(vtzero::vector_tile(buffers.back()), tile_obj.z, tile_obj.x, tile_obj.y); + } else { + tiles.emplace_back(vtzero::vector_tile(tile_obj.data), tile_obj.z, tile_obj.x, tile_obj.y); + } + } + // for each tile + for (auto& tile_obj : tiles) { + vtzero::vector_tile& tile = std::get<0>(tile_obj); while (auto layer = tile.next_layer()) { // check if this is a layer we should query @@ -282,8 +300,11 @@ struct Worker : Nan::AsyncWorker { } std::uint32_t extent = layer.extent(); + std::uint32_t tile_obj_z = std::get<1>(tile_obj); + std::uint32_t tile_obj_x = std::get<2>(tile_obj); + std::uint32_t tile_obj_y = std::get<3>(tile_obj); // query point in relation to the current tile the layer extent - mapbox::geometry::point query_point = utils::create_query_point(data.longitude, data.latitude, extent, tile_obj.z, tile_obj.x, tile_obj.y); + mapbox::geometry::point query_point = utils::create_query_point(data.longitude, data.latitude, extent, tile_obj_z, tile_obj_x, tile_obj_y); while (auto feature = layer.next_feature()) { auto original_geometry_type = get_geometry_type(feature); @@ -306,7 +327,7 @@ struct Worker : Nan::AsyncWorker { // if distance from the query point is greater than 0.0 (not a direct hit) so recalculate the latlng if (cp_info.distance > 0.0) { - ll = utils::convert_vt_to_ll(extent, tile_obj.z, tile_obj.x, tile_obj.y, cp_info); + ll = utils::convert_vt_to_ll(extent, tile_obj_z, tile_obj_x, tile_obj_y, cp_info); meters = utils::distance_in_meters(query_lnglat, ll); } @@ -352,15 +373,26 @@ struct Worker : Nan::AsyncWorker { } // end tile.layer.feature loop } // end tile.layer loop } // end tile loop + // Here we create "materialized" properties. We do this because, when reading from a compressed + // buffer, it is unsafe to touch `feature.properties_vector` once we've left this loop. + // That is because the buffer may represent uncompressed data that is not in scope outside of Execute() + for (auto& feature : results_queue_) { + feature.properties_vector_materialized.reserve(feature.properties_vector.size()); + for (auto const& property : feature.properties_vector) { + auto val = vtzero::convert_property_value(property.value()); + feature.properties_vector_materialized.emplace_back(std::string(property.key()), std::move(val)); + } + // we are now done with the feature.properties_vector + //feature.properties_vector.clear(); + } } catch (const std::exception& e) { SetErrorMessage(e.what()); } } void HandleOKCallback() override { + Nan::HandleScope scope; try { - Nan::HandleScope scope; - v8::Local results_object = Nan::New(); v8::Local features_array = Nan::New(); results_object->Set(Nan::New("type").ToLocalChecked(), Nan::New("FeatureCollection").ToLocalChecked()); @@ -384,7 +416,7 @@ struct Worker : Nan::AsyncWorker { // create properties object v8::Local properties_obj = Nan::New(); - for (auto const& prop : feature.properties_vector) { + for (auto const& prop : feature.properties_vector_materialized) { set_property(prop, properties_obj); }