From ffb3b8111fa51f433a00a22fedc06c0fa6eefbd5 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Sat, 21 Jul 2018 18:30:05 -0700 Subject: [PATCH 1/3] make node-mapnik addData async --- bench/bench.js | 74 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/bench/bench.js b/bench/bench.js index 087193e..95e094b 100644 --- a/bench/bench.js +++ b/bench/bench.js @@ -3,7 +3,7 @@ const zlib = require('zlib'); const argv = require('minimist')(process.argv.slice(2)); if (!argv.iterations || !argv.concurrency || !argv.package) { console.error('Please provide desired iterations, concurrency'); - console.error('Example: \nnode bench/bench.js --iterations 50 --concurrency 10 --package vtcomposite\nPackage options: vtcomposite or node-mapnik\nPass --compressed to bench compress tiles.'); + console.error('Example: \nnode bench/bench.js --iterations 50 --concurrency 10 --package vtcomposite\nPackage options: vtcomposite or node-mapnik\nPass --compress to bench compress tiles.'); process.exit(1); } @@ -71,48 +71,64 @@ function runRule(rule, ruleCallback) { } } ++runs; + + if (track_mem && runs % 1000) { + var mem = process.memoryUsage(); + if (mem.rss > memstats.max_rss) memstats.max_rss = mem.rss; + if (mem.heapTotal > memstats.max_heap_total) memstats.max_heap_total = mem.heapTotal; + if (mem.heapUsed > memstats.max_heap) memstats.max_heap = mem.heapUsed; + } return cb(); }); break; case 'node-mapnik': var target_vt = new mapnik.VectorTile(rule.zxy.z, rule.zxy.x, rule.zxy.y); - var source_tiles = new Array(rule.tiles.length); + let addDataQueue = Queue(); + function addData(tile,done) { + var vt = new mapnik.VectorTile(tile.z,tile.x,tile.y); + vt.addData(tile.buffer,function(err) { + if (err) throw err; + return done(null,vt); + }); + } for (var i = 0; i < rule.tiles.length; ++i) { - var vt = new mapnik.VectorTile(rule.tiles[i].z,rule.tiles[i].x,rule.tiles[i].y); - vt.addDataSync(rule.tiles[i].buffer); - source_tiles[i] = vt; + addDataQueue.defer(addData,rule.tiles[i]); } - // http://mapnik.org/documentation/node-mapnik/3.6/#VectorTile.composite - target_vt.composite(source_tiles, rule.options, function(err, result) { - if (err) { - return cb(err); - } - - let options = {compression:'none'} - if (rule.options.compress){ - options.compression = 'gzip'; - } - - result.getData(options, function(err, data) { + addDataQueue.awaitAll(function(error,source_tiles) { + if (error) throw error; +// console.log(source_tiles) + // http://mapnik.org/documentation/node-mapnik/3.6/#VectorTile.composite + target_vt.composite(source_tiles, rule.options, function(err, result) { if (err) { - throw err; + return cb(err); } + let options = {compression:'none'} if (rule.options.compress){ - if(data[0] !== 0x1F && data[1] !== 0x8B){ - throw new Error('resulting buffer is not compressed!'); - } + options.compression = 'gzip'; } - ++runs; - if (track_mem && runs % 1000) { - var mem = process.memoryUsage(); - if (mem.rss > memstats.max_rss) memstats.max_rss = mem.rss; - if (mem.heapTotal > memstats.max_heap_total) memstats.max_heap_total = mem.heapTotal; - if (mem.heapUsed > memstats.max_heap) memstats.max_heap = mem.heapUsed; - } - return cb(); + result.getData(options, function(err, data) { + if (err) { + throw err; + } + + if (rule.options.compress){ + if(data[0] !== 0x1F && data[1] !== 0x8B){ + throw new Error('resulting buffer is not compressed!'); + } + } + ++runs; + + if (track_mem && runs % 1000) { + var mem = process.memoryUsage(); + if (mem.rss > memstats.max_rss) memstats.max_rss = mem.rss; + if (mem.heapTotal > memstats.max_heap_total) memstats.max_heap_total = mem.heapTotal; + if (mem.heapUsed > memstats.max_heap) memstats.max_heap = mem.heapUsed; + } + return cb(); + }); }); }); break; From dd4c4d2647b7b1df55c75c99ca4e676cf5cf2ba7 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Sat, 21 Jul 2018 18:44:42 -0700 Subject: [PATCH 2/3] output memory stats --- bench/bench.js | 14 ++++++++++++-- package.json | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/bench/bench.js b/bench/bench.js index 95e094b..96d012a 100644 --- a/bench/bench.js +++ b/bench/bench.js @@ -1,5 +1,4 @@ "use strict"; -const zlib = require('zlib'); const argv = require('minimist')(process.argv.slice(2)); if (!argv.iterations || !argv.concurrency || !argv.package) { console.error('Please provide desired iterations, concurrency'); @@ -14,8 +13,10 @@ if (!argv.iterations || !argv.concurrency || !argv.package) { process.env.UV_THREADPOOL_SIZE = argv.concurrency; const fs = require('fs'); +const zlib = require('zlib'); const path = require('path'); const assert = require('assert'); +const bytes = require('bytes'); const Queue = require('d3-queue').queue; const composite = require('../lib/index.js'); const rules = require('./rules'); @@ -157,7 +158,7 @@ function runRule(rule, ruleCallback) { if (time == 0) { console.log("Warning: ms timer not high enough resolution to reliably track rate. Try more iterations"); } else { - // number of milliseconds per iteration + // number of milliseconds per iteration var rate = runs/(time/1000); process.stdout.write(rate.toFixed(0) + ' runs/s (' + time + 'ms)'); } @@ -176,3 +177,12 @@ function log(message) { process.stdout.write(message); } } + +process.on('exit',function() { + if (track_mem) { + console.log('Benchmark peak mem (max_rss, max_heap, max_heap_total): ', bytes(memstats.max_rss), bytes(memstats.max_heap), bytes(memstats.max_heap_total)); + } else { + console.log('Note: pass --mem to track memory usage'); + } + console.log('Benchmark iterations:',argv.iterations,'concurrency:',argv.concurrency); +}) diff --git a/package.json b/package.json index 7690fb4..96fd874 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "@mapbox/tilebelt": "^1.0.1", "@mapbox/vector-tile": "^1.3.1", "aws-sdk": "^2.245.1", - "bytes": "^2.4.0", + "bytes": "^2.5.0", "d3-queue": "^3.0.1", "mapnik": "3.6.1", "minimist": "~1.2.0", From 87c801f628e2e8bdfd6c282e5552d3aa2b17b587 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Sat, 21 Jul 2018 18:54:54 -0700 Subject: [PATCH 3/3] compress tiles if --compress is passed + dry up the code around memory checking and gzip compress assert --- bench/bench.js | 65 +++++++++++++++++++++----------------------------- bench/rules.js | 20 +--------------- 2 files changed, 28 insertions(+), 57 deletions(-) diff --git a/bench/bench.js b/bench/bench.js index 96d012a..fb28498 100644 --- a/bench/bench.js +++ b/bench/bench.js @@ -2,7 +2,7 @@ const argv = require('minimist')(process.argv.slice(2)); if (!argv.iterations || !argv.concurrency || !argv.package) { console.error('Please provide desired iterations, concurrency'); - console.error('Example: \nnode bench/bench.js --iterations 50 --concurrency 10 --package vtcomposite\nPackage options: vtcomposite or node-mapnik\nPass --compress to bench compress tiles.'); + console.error('Example: \nnode bench/bench.js --iterations 50 --concurrency 10 --package vtcomposite\nPackage options: vtcomposite or node-mapnik\nPass --compress to bench decompressing and compressing tiles.'); process.exit(1); } @@ -58,28 +58,38 @@ function runRule(rule, ruleCallback) { let runs = 0; let runsQueue = Queue(); + // If --compress force all benchmarks to compress final buffer + if(argv.compress){ + rule.options.compress = true; + } + function run(cb) { + + function done(err,result,callback) { + if (rule.options.compress){ + if(result[0] !== 0x1F && result[1] !== 0x8B){ + throw new Error('resulting buffer is not compressed!'); + } + } + ++runs; + + if (track_mem && runs % 1000) { + var mem = process.memoryUsage(); + if (mem.rss > memstats.max_rss) memstats.max_rss = mem.rss; + if (mem.heapTotal > memstats.max_heap_total) memstats.max_heap_total = mem.heapTotal; + if (mem.heapUsed > memstats.max_heap) memstats.max_heap = mem.heapUsed; + } + + return cb(); + } + switch(argv.package){ case 'vtcomposite': composite(rule.tiles, rule.zxy, rule.options, function(err, result) { if (err) { throw err; } - - if (rule.options.compress){ - if(result[0] !== 0x1F && result[1] !== 0x8B){ - throw new Error('resulting buffer is not compressed!'); - } - } - ++runs; - - if (track_mem && runs % 1000) { - var mem = process.memoryUsage(); - if (mem.rss > memstats.max_rss) memstats.max_rss = mem.rss; - if (mem.heapTotal > memstats.max_heap_total) memstats.max_heap_total = mem.heapTotal; - if (mem.heapUsed > memstats.max_heap) memstats.max_heap = mem.heapUsed; - } - return cb(); + return done(null,result); }); break; case 'node-mapnik': @@ -98,7 +108,6 @@ function runRule(rule, ruleCallback) { } addDataQueue.awaitAll(function(error,source_tiles) { if (error) throw error; -// console.log(source_tiles) // http://mapnik.org/documentation/node-mapnik/3.6/#VectorTile.composite target_vt.composite(source_tiles, rule.options, function(err, result) { if (err) { @@ -109,27 +118,7 @@ function runRule(rule, ruleCallback) { if (rule.options.compress){ options.compression = 'gzip'; } - - result.getData(options, function(err, data) { - if (err) { - throw err; - } - - if (rule.options.compress){ - if(data[0] !== 0x1F && data[1] !== 0x8B){ - throw new Error('resulting buffer is not compressed!'); - } - } - ++runs; - - if (track_mem && runs % 1000) { - var mem = process.memoryUsage(); - if (mem.rss > memstats.max_rss) memstats.max_rss = mem.rss; - if (mem.heapTotal > memstats.max_heap_total) memstats.max_heap_total = mem.heapTotal; - if (mem.heapUsed > memstats.max_heap) memstats.max_heap = mem.heapUsed; - } - return cb(); - }); + result.getData(options, done); }); }); break; diff --git a/bench/rules.js b/bench/rules.js index c7862bb..5c1e26f 100644 --- a/bench/rules.js +++ b/bench/rules.js @@ -105,15 +105,6 @@ module.exports = [ ], zxy: { z: 16, x: 10478, y: 25332} }, - { - description: 'returns compressed buffer - tiles completely made of points and linestrings, overzooming and lots of properties', - options: { compress:true, buffer_size: 128 }, - tiles: [ - { z: 15, x: 5239, y: 12666, buffer: fs.readFileSync('./test/fixtures/points-poi-sf-15-5239-12666.mvt')}, - { z: 15, x: 5239, y: 12666, buffer: fs.readFileSync('./test/fixtures/linestrings-sf-15-5239-12666.mvt')} - ], - zxy: { z: 16, x: 10478, y: 25332} - }, { description: 'buffer_size 128 - tiles completely made of points and linestrings, overzooming and lots of properties', options: { buffer_size: 128 }, @@ -150,15 +141,6 @@ module.exports = [ ], zxy: { z: 17, x: 20956, y: 50664} }, - { - description: 'return compressed buffer - tiles completely made of polygons, overzooming (2x) and lots of properties', - options: {compress:true, buffer_size: 128}, - tiles: [ - { z: 15, x: 5239, y: 12666, buffer: fs.readFileSync('./test/fixtures/polygons-buildings-sf-15-5239-12666.mvt')}, - { z: 15, x: 5239, y: 12666, buffer: fs.readFileSync('./test/fixtures/polygons-hillshade-sf-15-5239-12666.mvt')} - ], - zxy: { z: 17, x: 20956, y: 50664} - }, { description: 'buffer_size 4096 - tiles completely made of polygons, overzooming (2x) and lots of properties', options: {buffer_size: 4096}, @@ -186,4 +168,4 @@ function getTiles(name) { tiles.push({ buffer: buffer, z: parseInt(zxy[0]), x: parseInt(zxy[1]), y: parseInt(zxy[2]) }); }); return tiles; -} \ No newline at end of file +}