diff --git a/.travis.yml b/.travis.yml index 256a149..be25e13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,6 @@ language: node_js sudo: false -# build options and AWS keys for publishing binaries -# note: you'll need to generate your own keys for publishing a separate project -env: - global: - - secure: bLTdbWwf3v8X7yJlLLqPc9BRhFquwma4oYL9T2C38FXzSuqhXJZF+kNXjHd85E4lJNnuLXDjrPJLLkigd70J9Tclu2XQvy3cGzXI+EMsO6rDvQN24Eoi6uJ/qHhTrAoud526ouxO8qk9YYLjD6UWX1EOtqcEdwPMose8L1wsMZI= - - secure: lmfuNtK0/ubV4ZZobgfeKY6DzG41ZciS4a00yCe29jHLxAg+EEmB/qpsW5d1//cC94OKsAySW08xp2uX5wBVvtryaaoiwU7fdKF9DOxHRHieGw/3+m8NNjELkd5hKMKiqDj7l4QPMchzBQR2PQkoG0UMCGUFe+RmzZ2/iCdGHXU= - # enable c++11/14 builds addons: apt: @@ -42,14 +35,6 @@ script: # run your tests and build binaries matrix: include: - # linux publishable node v4/release - - os: linux - env: BUILDTYPE=release - node_js: 4 - # linux publishable node v4/debug - - os: linux - env: BUILDTYPE=debug - node_js: 4 # linux publishable node v6 - os: linux env: BUILDTYPE=release @@ -58,11 +43,6 @@ matrix: - os: linux env: BUILDTYPE=debug node_js: 6 - # osx publishable node v4 - - os: osx - osx_image: xcode8.2 - env: BUILDTYPE=release - node_js: 4 # osx publishable node v6 - os: osx osx_image: xcode8.2 @@ -71,7 +51,7 @@ matrix: # Sanitizer build node v4/Debug - os: linux env: BUILDTYPE=debug TOOLSET=asan - node_js: 4 + node_js: 6 sudo: required # Overrides `install` to set up custom asan flags install: @@ -94,7 +74,7 @@ matrix: # g++ build (default builds all use clang++) - os: linux env: BUILDTYPE=debug CXX="g++-6" CC="gcc-6" - node_js: 4 + node_js: 6 addons: apt: sources: @@ -111,7 +91,7 @@ matrix: # Coverage build - os: linux env: BUILDTYPE=debug CXXFLAGS="--coverage" LDFLAGS="--coverage" - node_js: 4 + node_js: 6 # Overrides `script` to publish coverage data to codecov before_script: - npm test @@ -137,7 +117,7 @@ matrix: # Clang tidy build - os: linux env: CLANG_TIDY - node_js: 4 + node_js: 6 # Overrides `install` to avoid initializing clang toolchain install: # First run the clang-tidy target diff --git a/API.md b/API.md new file mode 100644 index 0000000..4ae64b2 --- /dev/null +++ b/API.md @@ -0,0 +1,473 @@ + + +### Table of Contents + +- [coalesceCallback](#coalescecallback) +- [CoalesceResult](#coalesceresult) +- [coalesce](#coalesce) +- [PhrasematchSubqObject](#phrasematchsubqobject) +- [MemoryCache](#memorycache) + - [list](#list) + - [set](#set) + - [get](#get) + - [MemoryCache](#memorycache-1) + - [getmatching](#getmatching) + - [pack](#pack) +- [NormalizationCache](#normalizationcache) + - [get](#get-1) + - [getPrefixRange](#getprefixrange) + - [getAll](#getall) + - [writeBatch](#writebatch) + - [NormalizationCache](#normalizationcache-1) +- [RocksDBCache](#rocksdbcache) + - [pack](#pack-1) + - [get](#get-2) + - [get](#get-3) + - [list](#list-1) + - [merge](#merge) + - [RocksDBCache](#rocksdbcache-1) + +## coalesceCallback + +Type: [Function](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function) + +**Parameters** + +- `err` error if any, or null if not +- `results` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[CoalesceResult](#coalesceresult)>** the results of the coalesce operation + +## CoalesceResult + +A member of the result set from a coalesce operation. + +**Properties** + +- `x` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the X tile coordinate of the result +- `y` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the Y tile coordinate of the result +- `relev` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the computed relevance of the result +- `score` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the computed score of the result +- `id` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the feature ID of the result +- `idx` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the index ID (preserved from the inbound subquery) of the index the result came from +- `tmpid` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** a composite ID used for uniquely identifying features across indexes that incorporates the ID and IDX +- `distance` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the distance metric computed using the feature and proximity, if supplied; 0 otherwise +- `scoredist` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the composite score incorporating the feature's score with the distance (or the score if distance is 0) +- `matches_language` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** whether or not the match is valid for one of the languages in the inbound languages array + +## coalesce + +The coalesce function determines whether or not phrase matches in different +carmen indexes align spatially, and computes information about successful matches +such as combined relevance and score. The computation is done on the thread pool, +and exposed asynchronously to JS via a callback argument. + +**Parameters** + +- `phrasematches` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[PhrasematchSubqObject](#phrasematchsubqobject)>** an array of PhrasematchSubqObject objects, each of which describes a match candidate +- `options` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** options for how to perform the coalesce operation that aren't specific to a particular subquery + - `options.radius` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)?** the fall-off radius for determining how wide-reaching the effect of proximity bias is + - `options.centerzxy` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)>?** a 3-number array representing the ZXY of the tile on which the proximity point can be found + - `options.bboxzxy` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)>?** a 5-number array representing the zoom, minX, minY, maxX, and maxY values of the tile cover of the requested bbox, if any +- `callback` **[coalesceCallback](#coalescecallback)** the callback function + +## PhrasematchSubqObject + +The PhrasematchSubqObject type describes the metadata known about possible matches to be assessed for stacking by +coalesce as seen from Javascript. Note: it is of similar purpose to the PhrasematchSubq C++ struct type, but differs +slightly in specific field names and types. + +**Properties** + +- `phrase` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The matched string +- `weight` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** A float between 0 and 1 representing how much of the query this string covers +- `prefix` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** whether or not to do a prefix scan (as opposed to an exact match scan); used for autocomplete +- `idx` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** an identifier of the index the match came from; opaque to carmen-cache but returned in results +- `zoom` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** the configured tile zoom level for the index +- `mask` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** a bitmask representing which tokens in the original query the subquery covers +- `languages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)>** a list of the language IDs to be considered matching +- `cache` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** the carmen-cache from the index in which the match was found + +## MemoryCache + +### list + +Lists the keys in the store of the MemoryCache Object + +**Parameters** + +- `id` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const MemoryCache = new cache.MemoryCache('a'); + +cache.list('a', (err, result) => { + if (err) throw err; + console.log(result); +}); +``` + +Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** + +### set + +Replaces or appends the data for a given key + +**Parameters** + +- `id` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** +- `null` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** , data; an array of numbers where each number represents a grid +- `an` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** array of relevant languages +- `T` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** : append to data, F: replace data + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const MemoryCache = new cache.MemoryCache('a'); + +cache.set('a', [1,2,3], (err, result) => { + if (err) throw err; + console.log(result) +}); +``` + +Returns **any** undefined + +### get + +Retrieves data exactly matching phrase and language settings by id + +**Parameters** + +- `id` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** +- `matches_prefixes` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** : T if it matches exactly, F: if it does not +- `optional` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** ; array of languages + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const MemoryCache = new cache.MemoryCache('a'); + +MemoryCache.get(id, languages); + // => [grid, grid, grid, grid... ] +``` + +Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** integers referring to grids + +### MemoryCache + +Creates an in-memory key-value store mapping phrases and language IDs +to lists of corresponding grids (grids ie are integer representations of occurrences of the phrase within an index) + +**Parameters** + +- `id` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const MemoryCache = new cache.MemoryCache(id, languages); +``` + +Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** grid of integers + +### getmatching + +Retrieves data matching phrase and/or language settings by id. +If match_prefixes is true, anything that starts with that phrase, +and also for any entry regardless of language, +and with a relevance penalty applied to languages that don't match those requested. + +**Parameters** + +- `id` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** +- `matches_prefixes` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** : T if it matches exactly, F: if it does not +- `optional` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** ; array of languages + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const MemoryCache = new cache.MemoryCache('a'); + +MemoryCache.getmatching(id, languages); + // => [grid, grid, grid, grid... ] +``` + +Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** integers referring to grids + +### pack + +creates database from filename +optimize a memory cache and write to disc as rocksdbcache + +**Parameters** + +- `null` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** , filename + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const MemoryCache = new cache.MemoryCache('a'); + +cache.pack('filename'); +``` + +Returns **any** undefined + +## NormalizationCache + +NormalizationCache represents an one-to-many integer-to-integer +mapping where each integer is assumed to be the position of a given term +in a lexicographically-sorted vocabulary. The purpose of the cache is to capture +equivalencies between different elements in the dictionary, such that further metadata +(e.g., in a RocksDBCache) can be stored only for the canonical form of a given name. +The structure is stored using a RocksDB database on disk. + +### get + +retrieve the indices of the canonical labels for the index of a given non-canonical label + +**Parameters** + +- `id` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const nc = new cache.NormalizationCache('file.norm.rocksdb', true); + +// for a normalization cache for the dictionary ['main st', 'main street'] +// where 'main st' is canonical +const canonical = nc.get(1); // returns [0] +``` + +Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** + +### getPrefixRange + +given that in a lexicographically sorted list, all terms that share a prefix +are grouped together, this function retrieves the indices of all canonical forms +of all terms that share a given prefix as indicated by the index of the first term +in the shared prefix list and the number of terms that share a prefix, for which the canonical +form does not also share that same prefix + +**Parameters** + +- `start_id` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** +- `count` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** +- `scan_max` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)?** the maximum number of entries to scan +- `return_max` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)?** the maximum number of indices to return + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const nc = new cache.NormalizationCache('file.norm.rocksdb', true); + +// for a normalization cache for the dictionary +// ['saint marks ave', 'saint peters ave', 'st marks ave', 'st peters ave'] +// where the 'st ...' forms are canonical +const canonical = nc.getPrefixRange(0, 2); // looks up all the canonical + // forms for things that begin with + // 'saint' +``` + +Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** + +### getAll + +retrieve the entire contents of a NormalizationCache, as an array of arrays + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const nc = new cache.NormalizationCache('file.norm.rocksdb', true); + +// for a normalization cache for the dictionary +// ['saint marks ave', 'saint peters ave', 'st marks ave', 'st peters ave'] +// where the 'st ...' forms are canonical +const canonical = nc.getAll() // returns [[0, [2]], [1, [3]]] +``` + +Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** + +### writeBatch + +bulk-set the contents of a NormalizationCache to an array of arrays + +**Parameters** + +- `data` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** the values to be written to the cache, in the form \[\[from, [to, to, ...]], ...] + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const nc = new cache.NormalizationCache('file.norm.rocksdb', true); + +// for a normalization cache for the dictionary +// ['saint marks ave', 'saint peters ave', 'st marks ave', 'st peters ave'] +// where the 'st ...' forms are canonical +nc.writeBatch([[0, [2]], [1, [3]]]); +``` + +Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** + +### NormalizationCache + +Constructor for NormalizationCache pointing to an on-disk RocksDB database +to be used for reading or writing. + +**Parameters** + +- `filename` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** +- `read-only` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const nc = new cache.NormalizationCache('file.norm.rocksdb', false); +``` + +Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** + +## RocksDBCache + +### pack + +Writes an identical copy RocksDBCache from another RocksDBCache; not really used + +**Parameters** + +- `null` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** , filename + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const RocksDBCache = new cache.RocksDBCache('a'); + +cache.pack('filename'); +``` + +Returns **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** + +### get + +Retrieves data exactly matching phrase and language settings by id + +**Parameters** + +- `id` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** +- `matches_prefixes` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** : T if it matches exactly, F: if it does not +- `optional` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** ; array of languages + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const RocksDBCache = new cache.RocksDBCache('a'); + +RocksDBCache.get(id, languages); + // => [grid, grid, grid, grid... ] +``` + +Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** integers referring to grids + +### get + +Retrieves grid that at least partially matches phrase and/or language inputs + +**Parameters** + +- `id` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** +- `matches_prefixes` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** : T if it matches exactly, F: if it does not +- `optional` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** ; array of languages + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const RocksDBCache = new cache.RocksDBCache('a'); + +RocksDBCache.get(id, languages); + // => [grid, grid, grid, grid... ] +``` + +Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** integers referring to grids + +### list + +lists the keys in the RocksDBCache object + +**Parameters** + +- `id` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const RocksDBCache = new cache.RocksDBCache('a'); + +cache.list('a', (err, result) => { + if (err) throw err; + console.log(result); +}); +``` + +Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)** Set of keys/ids + +### merge + +merges the contents from 2 RocksDBCaches + +**Parameters** + +- `RocksDBCache` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** file 1 +- `RocksDBCache` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** file 2 +- `result` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** RocksDBCache file +- `method` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** which is either concat or freq +- `callback` **[Function](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function)** called from the mergeAfter method + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const RocksDBCache = new cache.RocksDBCache('a'); + +cache.merge('file1', 'file2', 'resultFile', 'method', (err, result) => { + if (err) throw err; + console.log(result); +}); +``` + +Returns **[Set](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Set)** Set of ids + +### RocksDBCache + +Creates an in-memory key-value store mapping phrases and language IDs +to lists of corresponding grids (grids ie are integer representations of occurrences of the phrase within an index) + +**Parameters** + +- `id` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** +- `filename` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** + +**Examples** + +```javascript +const cache = require('@mapbox/carmen-cache'); +const RocksDBCache = new cache.RocksDBCache('a', 'filename'); +``` + +Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 42d24be..fd09632 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,10 @@ +Running benchmarks +------------------ +`carmen-cache` includes benchmarks which are run during testing to ensure that they work, but not to ensure that they're fast. Developers proposing substantial changes should manually run the benchmarks and compare the results with master to be confident that any changes don't negatively affect performance. Benchmarks can be run by running `yarn bench`. + Publishing node-pre-gyp binaries -------------------------------- -Users who are installing `carmen-cache` and not actively doing development usually download prebuilt static binaries at `npm install` time automatically through `node-pre-gyp`. +Users who are installing `carmen-cache` and not actively doing development usually download prebuilt static binaries at `yarn install` time automatically through `node-pre-gyp`. - Binaries are published by travis to S3 and triggered by including `[publish binary]` in a commit message. - Binaries are published using the `version` key of `package.json` on the commit with this message. diff --git a/Makefile b/Makefile index b3a6ff3..d17db66 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ default: release node_modules: # install deps but for now ignore our own install script # so that we can run it directly in either debug or release - npm install --ignore-scripts + yarn install --ignore-scripts release: node_modules V=1 ./node_modules/.bin/node-pre-gyp configure build --loglevel=error diff --git a/test/coalesce.bench.test.js b/bench/coalesce.bench.test.js similarity index 66% rename from test/coalesce.bench.test.js rename to bench/coalesce.bench.test.js index 28f4544..3b6394a 100644 --- a/test/coalesce.bench.test.js +++ b/bench/coalesce.bench.test.js @@ -4,9 +4,6 @@ const coalesce = require('../index.js').coalesce; const test = require('tape'); (function() { - // asan makes everything slow, so skip benching - if (process.env.ASAN_OPTIONS) return; - const runs = 50; const b = new Cache('b'); b._set('3848571113', require('./fixtures/coalesce-bench-single-3848571113.json')); @@ -20,17 +17,14 @@ const test = require('tape'); prefix: false, mask: 1 << 0 }]; - test('coalesceSingle', (assert) => { + test('coalesceSingle', (t) => { const time = +new Date; function run(remaining) { if (!remaining) { const ops = (+new Date - time) / runs; - let expected_ops = 30; - if (process.env.BUILDTYPE === 'debug') { - expected_ops = 500; - } - assert.equal(ops < expected_ops, true, 'coalesceSingle @ ' + ops + 'ms < ' + expected_ops + 'ms'); - assert.end(); + const expected_ops = 30; + t.pass('coalesceSingle @ ' + ops + 'ms should be less than ' + expected_ops + 'ms'); + t.end(); return; } coalesce(stacks, {}, (err, res) => { @@ -38,8 +32,8 @@ const test = require('tape'); checks = checks && res.length === 37; checks = checks && res[0][0].tmpid === 129900; if (!checks) { - assert.fail('Failed checks'); - assert.end(); + t.fail('Failed checks'); + t.end(); } else { run(--remaining); } @@ -47,17 +41,16 @@ const test = require('tape'); } run(runs); }); - test('coalesceSingle proximity', (assert) => { + // Coalesce can optionally take a proximity parameter, and use this as a parameter. + // This can be more computationally expensive which is why it is a part of the benchmark test. + test('coalesceSingle proximity', (t) => { const time = +new Date; function run(remaining) { if (!remaining) { const ops = (+new Date - time) / runs; - let expected_ops = 30; - if (process.env.BUILDTYPE === 'debug') { - expected_ops = 500; - } - assert.equal(ops < expected_ops, true, 'coalesceSingle + proximity @ ' + ops + 'ms < ' + expected_ops + 'ms'); - assert.end(); + const expected_ops = 30; + t.pass('coalesceSingle + proximity @ ' + ops + 'ms should be less than ' + expected_ops + 'ms'); + t.end(); return; } coalesce(stacks, { centerzxy: [14,4893,6001] }, (err, res) => { @@ -67,8 +60,8 @@ const test = require('tape'); checks = checks && res[0][0].y === 6001; checks = checks && res[0][0].tmpid === 446213; if (!checks) { - assert.fail('Failed checks'); - assert.end(); + t.fail('Failed checks'); + t.end(); } else { run(--remaining); } @@ -79,9 +72,6 @@ const test = require('tape'); })(); (function() { - // asan makes everything slow, so skip benching - if (process.env.ASAN_OPTIONS) return; - const runs = 50; const a = new Cache('a', 0); const b = new Cache('b', 0); @@ -104,17 +94,14 @@ const test = require('tape'); phrase: '3848571113', prefix: false }]; - test('coalesceMulti', (assert) => { + test('coalesceMulti', (t) => { const time = +new Date; function run(remaining) { if (!remaining) { const ops = (+new Date - time) / runs; - let expected_ops = 60; - if (process.env.BUILDTYPE === 'debug') { - expected_ops = 1000; - } - assert.equal(ops < expected_ops, true, 'coalesceMulti @ ' + ops + 'ms < ' + expected_ops + 'ms'); - assert.end(); + const expected_ops = 60; + t.pass('coalesceMulti @ ' + ops + 'ms should be less than ' + expected_ops + 'ms'); + t.end(); return; } coalesce(stacks, {}, (err, res) => { @@ -123,8 +110,8 @@ const test = require('tape'); checks = checks && res[0][0].tmpid === 33593999; checks = checks && res[0][1].tmpid === 514584; if (!checks) { - assert.fail('Failed checks'); - assert.end(); + t.fail('Failed checks'); + t.end(); } else { run(--remaining); } @@ -132,17 +119,14 @@ const test = require('tape'); } run(runs); }); - test('coalesceMulti proximity', (assert) => { + test('coalesceMulti proximity', (t) => { const time = +new Date; function run(remaining) { if (!remaining) { const ops = (+new Date - time) / runs; - let expected_ops = 60; - if (process.env.BUILDTYPE === 'debug') { - expected_ops = 1000; - } - assert.equal(ops < expected_ops, true, 'coalesceMulti + proximity @' + ops + 'ms < ' + expected_ops + 'ms'); - assert.end(); + const expected_ops = 60; + t.pass('coalesceMulti + proximity @' + ops + 'ms should be less than ' + expected_ops + 'ms'); + t.end(); return; } coalesce(stacks, { centerzxy: [14,4893,6001] }, (err, res) => { @@ -153,8 +137,8 @@ const test = require('tape'); checks = checks && res[0][0].tmpid === 34000645; checks = checks && res[0][1].tmpid === 5156; if (!checks) { - assert.fail('Failed checks'); - assert.end(); + t.fail('Failed checks'); + t.end(); } else { run(--remaining); } diff --git a/test/fixtures/bench.pbf b/bench/fixtures/bench.pbf similarity index 100% rename from test/fixtures/bench.pbf rename to bench/fixtures/bench.pbf diff --git a/test/fixtures/coalesce-bench-multi-1965155344.json b/bench/fixtures/coalesce-bench-multi-1965155344.json similarity index 100% rename from test/fixtures/coalesce-bench-multi-1965155344.json rename to bench/fixtures/coalesce-bench-multi-1965155344.json diff --git a/test/fixtures/coalesce-bench-multi-3848571113.json b/bench/fixtures/coalesce-bench-multi-3848571113.json similarity index 100% rename from test/fixtures/coalesce-bench-multi-3848571113.json rename to bench/fixtures/coalesce-bench-multi-3848571113.json diff --git a/test/fixtures/coalesce-bench-single-3848571113.json b/bench/fixtures/coalesce-bench-single-3848571113.json similarity index 100% rename from test/fixtures/coalesce-bench-single-3848571113.json rename to bench/fixtures/coalesce-bench-single-3848571113.json diff --git a/binding.gyp b/binding.gyp index 86d5c73..1b73c74 100644 --- a/binding.gyp +++ b/binding.gyp @@ -1,9 +1,5 @@ { 'includes': [ 'common.gypi' ], - 'make_global_settings': [ - ['CXX', '<(module_root_dir)/mason_packages/.link/bin/clang++-3.9'], - ['LINK', '<(module_root_dir)/mason_packages/.link/bin/clang++-3.9'] - ], "variables": { # Flags we pass to the compiler to ensure the compiler # warns us about potentially buggy or dangerous code @@ -42,6 +38,12 @@ 'dependencies': [ 'action_before_build' ], 'product_dir': '<(module_path)', 'sources': [ + "./src/cpp_util.cpp", + "./src/node_util.cpp", + "./src/normalizationcache.cpp", + "./src/memorycache.cpp", + "./src/rocksdbcache.cpp", + "./src/coalesce.cpp", "./src/binding.cpp" ], "include_dirs" : [ diff --git a/cloudformation/ci.template.js b/cloudformation/ci.template.js new file mode 100644 index 0000000..719e800 --- /dev/null +++ b/cloudformation/ci.template.js @@ -0,0 +1,59 @@ +var cf = require('@mapbox/cloudfriend'); +var package_json = require('../package.json') + +module.exports = { + AWSTemplateFormatVersion: '2010-09-09', + Description: 'user for publishing to s3://mapbox-node-binary/' + package_json.name, + Resources: { + User: { + Type: 'AWS::IAM::User', + Properties: { + Policies: [ + { + PolicyName: 'list', + PolicyDocument: { + Statement: [ + { + Action: ['s3:ListBucket'], + Effect: 'Allow', + Resource: 'arn:aws:s3:::mapbox-node-binary', + Condition : { + StringLike : { + "s3:prefix": [ package_json.name + "/*"] + } + } + } + ] + } + }, + { + PolicyName: 'publish', + PolicyDocument: { + Statement: [ + { + Action: ['s3:DeleteObject', 's3:GetObject', 's3:GetObjectAcl', 's3:PutObject', 's3:PutObjectAcl'], + Effect: 'Allow', + Resource: 'arn:aws:s3:::mapbox-node-binary/' + package_json.name + '/*' + } + ] + } + } + ] + } + }, + AccessKey: { + Type: 'AWS::IAM::AccessKey', + Properties: { + UserName: cf.ref('User') + } + } + }, + Outputs: { + AccessKeyId: { + Value: cf.ref('AccessKey') + }, + SecretAccessKey: { + Value: cf.getAtt('AccessKey', 'SecretAccessKey') + } + } +}; diff --git a/common.gypi b/common.gypi index eeba6f5..61a26af 100644 --- a/common.gypi +++ b/common.gypi @@ -34,11 +34,15 @@ 'defines': [ 'NDEBUG' ], + 'cflags': ['-flto'], + 'ldflags': ['-flto'], 'xcode_settings': { 'OTHER_CPLUSPLUSFLAGS!': [ '-Os', '-O2' ], + 'OTHER_LDFLAGS':[ '-flto' ], + 'OTHER_CPLUSPLUSFLAGS!': [ '-flto' ], 'GCC_OPTIMIZATION_LEVEL': '3', 'GCC_GENERATE_DEBUGGING_SYMBOLS': 'NO', 'DEAD_CODE_STRIPPING': 'YES', @@ -47,4 +51,4 @@ } } } -} \ No newline at end of file +} diff --git a/package.json b/package.json index 571ddaa..eb7fc06 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "url": "git://github.com/mapbox/carmen-cache.git", "type": "git" }, - "version": "0.20.0", + "version": "0.20.0-cache-split-2", "dependencies": { "nan": "~2.5.1", "node-pre-gyp": "~0.6.32", @@ -17,18 +17,30 @@ "node-pre-gyp" ], "scripts": { - "test": "tape test/*.test.js && eslint *.js test/*.js", - "lint": "eslint *.js test/*.js", - "install": "node-pre-gyp install --fallback-to-build" + "test": "tape test/*.test.js bench/*.js && eslint *.js test/*.js bench/*.js", + "lint": "eslint *.js test/*.js bench/*.js", + "bench": "tape bench/*.js", + "install": "node-pre-gyp install --fallback-to-build", + "docs": "documentation build src/*.cpp --polyglot -f md -o API.md" }, "eslintConfig": { - "extends": "@mapbox/eslint-config-geocoding" + "plugins": ["tape"], + "extends": [ + "@mapbox/eslint-config-geocoding" + ], + "rules": { + "tape/assertion-message": ["error", "always"], + "tape/use-t": "error", + "tape/use-test": "error" + } }, "devDependencies": { "@mapbox/eslint-config-geocoding": "^1.0.1", + "eslint-plugin-tape": "1.1.0", + "aws-sdk": "^2.55.0", + "documentation": "^4.0.0-beta5", "eslint": "^4.17.0", "eslint-plugin-node": "^5.2.1", - "aws-sdk": "^2.55.0", "tape": "^4.6.3" }, "main": "./index.js", diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh index 2f63779..c2a50cd 100755 --- a/scripts/install_deps.sh +++ b/scripts/install_deps.sh @@ -3,16 +3,29 @@ set -eu set -o pipefail -function install() { - mason install $1 $2 - mason link $1 $2 -} +# gyp will put "MAKEFLAGS=r -- BUILDTYPE=Release" into the makefiles +# which breaks the rocksdb build +unset MAKEFLAGS # setup mason ./scripts/setup.sh --config local.env source local.env -install bzip2 1.0.6 -install rocksdb 4.13 -install clang++ 3.9.1 -install protozero 1.5.1 \ No newline at end of file +# avoid mis-reporting of CPU due to docker +# from resulting in OOM killer knocking out g++ +export MASON_CONCURRENCY=2 + +# only build from source if it does not exist +if [[ ! -f mason_packages/.link/lib/libbz2.a ]]; then + mason build bzip2 1.0.6 +fi + +# only build from source if it does not exist +if [[ ! -f mason_packages/.link/lib/librocksdb.a ]]; then + mason build rocksdb 4.13 +fi + +mason link bzip2 1.0.6 +mason link rocksdb 4.13 +mason install protozero 1.5.1 +mason link protozero 1.5.1 diff --git a/scripts/setup.sh b/scripts/setup.sh index e83cbd7..726853e 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -3,8 +3,10 @@ set -eu set -o pipefail -export MASON_RELEASE="${MASON_RELEASE:-eeba3b5}" -export MASON_LLVM_RELEASE="${MASON_LLVM_RELEASE:-5.0.0}" +# TODO(springmeyer) currently depends on mason branch: https://github.com/mapbox/mason/issues/566 +export MASON_RELEASE="${MASON_RELEASE:-cxx-override}" +export MASON_LLVM_RELEASE="${MASON_LLVM_RELEASE:-5.0.1}" +export BINUTILS_VERSION="${BINUTILS_VERSION:-2.30}" PLATFORM=$(uname | tr A-Z a-z) if [[ ${PLATFORM} == 'darwin' ]]; then @@ -64,12 +66,21 @@ function run() { setup_mason $(pwd)/.mason ${MASON_RELEASE} + + # install binutils for LTO on linux + if [[ $(uname -s) == 'Linux' ]] && [[ ${CXX:-} =~ 'clang++' ]]; then + $(pwd)/.mason/mason install binutils ${BINUTILS_VERSION} + $(pwd)/.mason/mason link binutils ${BINUTILS_VERSION} + fi # # ENV SETTINGS # echo "export PATH=${llvm_toolchain_dir}/bin:$(pwd)/.mason:$(pwd)/mason_packages/.link/bin:"'${PATH}' > ${config} echo "export CXX=${CXX:-${llvm_toolchain_dir}/bin/clang++}" >> ${config} + echo "export CC=${CC:-${llvm_toolchain_dir}/bin/clang}" >> ${config} + echo 'export MASON_CXX=${CXX}' >> ${config} + echo 'export MASON_CC=${CC}' >> ${config} echo "export MASON_RELEASE=${MASON_RELEASE}" >> ${config} echo "export MASON_LLVM_RELEASE=${MASON_LLVM_RELEASE}" >> ${config} # https://github.com/google/sanitizers/wiki/AddressSanitizerAsDso diff --git a/src/binding.cpp b/src/binding.cpp index 2b6ddeb..56d0ee9 100644 --- a/src/binding.cpp +++ b/src/binding.cpp @@ -1,2317 +1,9 @@ - #include "binding.hpp" -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -typedef std::chrono::high_resolution_clock Clock; - namespace carmen { using namespace v8; -Nan::Persistent MemoryCache::constructor; -Nan::Persistent RocksDBCache::constructor; -Nan::Persistent NormalizationCache::constructor; - -rocksdb::Status OpenDB(const rocksdb::Options& options, const std::string& name, std::unique_ptr& dbptr) { - rocksdb::DB* db; - rocksdb::Status status = rocksdb::DB::Open(options, name, &db); - dbptr = std::unique_ptr(db); - return status; -} - -rocksdb::Status OpenForReadOnlyDB(const rocksdb::Options& options, const std::string& name, std::unique_ptr& dbptr) { - rocksdb::DB* db; - rocksdb::Status status = rocksdb::DB::OpenForReadOnly(options, name, &db); - dbptr = std::unique_ptr(db); - return status; -} - -constexpr uint64_t LANGUAGE_MATCH_BOOST = (const uint64_t)(1) << 63; - -// in general, the key format including language field is <8-byte langfield> -// but as a size optimization for language-less indexes, we omit the langfield -// if it would otherwise have been ALL_LANGUAGES (all 1's), so if the first occurence -// of the separator character is also the last character in the string, we just retur ALL_LANGUAGES -// otherwise we extract it from the string -// -// we centralize both the adding of the field and extracting of the field here to keep from having -// to handle that optimization everywhere -inline langfield_type extract_langfield(std::string const& s) { - size_t length = s.length(); - size_t langfield_start = s.find(LANGFIELD_SEPARATOR) + 1; - size_t distance_from_end = length - langfield_start; - - if (distance_from_end == 0) { - return ALL_LANGUAGES; - } else { - langfield_type result(0); - memcpy(&result, s.data() + langfield_start, distance_from_end); - return result; - } -} - -inline void add_langfield(std::string& s, langfield_type langfield) { - if (langfield != ALL_LANGUAGES) { - char* lf_as_char = reinterpret_cast(&langfield); - - // we only want to copy over as many bytes as we're using - // so find the last byte that's not zero, and copy until there - // NOTE: this assumes little-endianness, where the bytes - // in use will be first rather than last - size_t highest(0); - for (size_t i = 0; i < sizeof(langfield_type); i++) { - if (lf_as_char[i] != 0) highest = i; - } - size_t field_length = highest + 1; - - s.reserve(sizeof(LANGFIELD_SEPARATOR) + field_length); - s.push_back(LANGFIELD_SEPARATOR); - s.append(lf_as_char, field_length); - } else { - s.push_back(LANGFIELD_SEPARATOR); - } -} - -intarray __get(MemoryCache const* c, std::string phrase, langfield_type langfield) { - arraycache const& cache = c->cache_; - intarray array; - - add_langfield(phrase, langfield); - arraycache::const_iterator aitr = cache.find(phrase); - if (aitr != cache.end()) { - array = aitr->second; - } - std::sort(array.begin(), array.end(), std::greater()); - return array; -} - -inline void decodeMessage(std::string const& message, intarray& array) { - protozero::pbf_reader item(message); - item.next(CACHE_ITEM); - auto vals = item.get_packed_uint64(); - uint64_t lastval = 0; - // delta decode values. - for (auto it = vals.first; it != vals.second; ++it) { - if (lastval == 0) { - lastval = *it; - array.emplace_back(lastval); - } else { - lastval = lastval - *it; - array.emplace_back(lastval); - } - } -} - -inline void decodeAndBoostMessage(std::string const& message, intarray& array) { - protozero::pbf_reader item(message); - item.next(CACHE_ITEM); - auto vals = item.get_packed_uint64(); - uint64_t lastval = 0; - // delta decode values. - for (auto it = vals.first; it != vals.second; ++it) { - if (lastval == 0) { - lastval = *it; - array.emplace_back(lastval | LANGUAGE_MATCH_BOOST); - } else { - lastval = lastval - *it; - array.emplace_back(lastval | LANGUAGE_MATCH_BOOST); - } - } -} - -intarray __get(RocksDBCache const* c, std::string phrase, langfield_type langfield) { - std::shared_ptr db = c->db; - intarray array; - - add_langfield(phrase, langfield); - std::string message; - rocksdb::Status s = db->Get(rocksdb::ReadOptions(), phrase, &message); - if (s.ok()) { - decodeMessage(message, array); - } - - return array; -} - -struct sortableGrid { - protozero::const_varint_iterator it; - protozero::const_varint_iterator end; - value_type unadjusted_lastval; - bool matches_language; -}; - -intarray __getmatching(MemoryCache const* c, std::string phrase, bool match_prefixes, langfield_type langfield) { - intarray array; - - if (!match_prefixes) phrase.push_back(LANGFIELD_SEPARATOR); - size_t phrase_length = phrase.length(); - const char* phrase_data = phrase.data(); - - // Load values from memory cache - - for (auto const& item : c->cache_) { - const char* item_data = item.first.data(); - size_t item_length = item.first.length(); - - if (item_length < phrase_length) continue; - - if (memcmp(phrase_data, item_data, phrase_length) == 0) { - langfield_type message_langfield = extract_langfield(item.first); - - if (message_langfield & langfield) { - array.reserve(array.size() + item.second.size()); - for (auto const& grid : item.second) { - array.emplace_back(grid | LANGUAGE_MATCH_BOOST); - } - } else { - array.insert(array.end(), item.second.begin(), item.second.end()); - } - } - } - std::sort(array.begin(), array.end(), std::greater()); - return array; -} - -intarray __getmatching(RocksDBCache const* c, std::string phrase, bool match_prefixes, langfield_type langfield) { - intarray array; - - if (!match_prefixes) phrase.push_back(LANGFIELD_SEPARATOR); - size_t phrase_length = phrase.length(); - - // Load values from message cache - std::vector> messages; - std::vector grids; - - if (match_prefixes) { - // if this is an autocomplete scan, use the prefix cache - if (phrase_length <= MEMO_PREFIX_LENGTH_T1) { - phrase = "=1" + phrase.substr(0, MEMO_PREFIX_LENGTH_T1); - } else if (phrase_length <= MEMO_PREFIX_LENGTH_T2) { - phrase = "=2" + phrase.substr(0, MEMO_PREFIX_LENGTH_T2); - } - } - - radix_max_heap::pair_radix_max_heap rh; - - std::shared_ptr db = c->db; - - std::unique_ptr rit(db->NewIterator(rocksdb::ReadOptions())); - for (rit->Seek(phrase); rit->Valid() && rit->key().ToString().compare(0, phrase.size(), phrase) == 0; rit->Next()) { - std::string key = rit->key().ToString(); - - // grab the langfield from the end of the key - langfield_type message_langfield = extract_langfield(key); - bool matches_language = (bool)(message_langfield & langfield); - - messages.emplace_back(std::make_tuple(rit->value().ToString(), matches_language)); - } - - // short-circuit the priority queue merging logic if we only found one message - // as will be the norm for exact matches in translationless indexes - if (messages.size() == 1) { - if (std::get<1>(messages[0])) { - decodeAndBoostMessage(std::get<0>(messages[0]), array); - } else { - decodeMessage(std::get<0>(messages[0]), array); - } - return array; - } - - for (std::tuple& message : messages) { - protozero::pbf_reader item(std::get<0>(message)); - bool matches_language = std::get<1>(message); - - item.next(CACHE_ITEM); - auto vals = item.get_packed_uint64(); - - if (vals.first != vals.second) { - value_type unadjusted_lastval = *(vals.first); - grids.emplace_back(sortableGrid{ - vals.first, - vals.second, - unadjusted_lastval, - matches_language}); - rh.push(matches_language ? unadjusted_lastval | LANGUAGE_MATCH_BOOST : unadjusted_lastval, grids.size() - 1); - } - } - - while (!rh.empty() && array.size() < PREFIX_MAX_GRID_LENGTH) { - size_t gridIdx = rh.top_value(); - uint64_t gridId = rh.top_key(); - rh.pop(); - - array.emplace_back(gridId); - sortableGrid* sg = &(grids[gridIdx]); - sg->it++; - if (sg->it != sg->end) { - sg->unadjusted_lastval -= *(grids[gridIdx].it); - rh.push( - sg->matches_language ? sg->unadjusted_lastval | LANGUAGE_MATCH_BOOST : sg->unadjusted_lastval, - gridIdx); - } - } - - return array; -} - -constexpr unsigned MAX_LANG = (sizeof(langfield_type) * 8) - 1; -inline langfield_type langarrayToLangfield(Local const& array) { - size_t array_size = array->Length(); - langfield_type out = 0; - for (unsigned i = 0; i < array_size; i++) { - unsigned int val = static_cast(array->Get(i)->NumberValue()); - if (val > MAX_LANG) { - // this should probably throw something - continue; - } - out = out | (static_cast(1) << val); - } - return out; -} - -inline Local langfieldToLangarray(langfield_type langfield) { - Local langs = Nan::New(); - - unsigned idx = 0; - for (unsigned i = 0; i <= MAX_LANG; i++) { - if (langfield & (static_cast(1) << i)) { - langs->Set(idx++, Nan::New(i)); - } - } - return langs; -} - -void MemoryCache::Initialize(Handle target) { - Nan::HandleScope scope; - Local t = Nan::New(MemoryCache::New); - t->InstanceTemplate()->SetInternalFieldCount(1); - t->SetClassName(Nan::New("MemoryCache").ToLocalChecked()); - Nan::SetPrototypeMethod(t, "pack", MemoryCache::pack); - Nan::SetPrototypeMethod(t, "list", MemoryCache::list); - Nan::SetPrototypeMethod(t, "_set", _set); - Nan::SetPrototypeMethod(t, "_get", _get); - Nan::SetPrototypeMethod(t, "_getMatching", _getmatching); - target->Set(Nan::New("MemoryCache").ToLocalChecked(), t->GetFunction()); - constructor.Reset(t); -} - -MemoryCache::MemoryCache() - : ObjectWrap(), - cache_() {} - -MemoryCache::~MemoryCache() {} - -void RocksDBCache::Initialize(Handle target) { - Nan::HandleScope scope; - Local t = Nan::New(RocksDBCache::New); - t->InstanceTemplate()->SetInternalFieldCount(1); - t->SetClassName(Nan::New("RocksDBCache").ToLocalChecked()); - Nan::SetPrototypeMethod(t, "pack", RocksDBCache::pack); - Nan::SetPrototypeMethod(t, "list", RocksDBCache::list); - Nan::SetPrototypeMethod(t, "_get", _get); - Nan::SetPrototypeMethod(t, "_getMatching", _getmatching); - Nan::SetMethod(t, "merge", merge); - target->Set(Nan::New("RocksDBCache").ToLocalChecked(), t->GetFunction()); - constructor.Reset(t); -} - -RocksDBCache::RocksDBCache() - : ObjectWrap(), - db() {} - -RocksDBCache::~RocksDBCache() {} - -inline void packVec(intarray const& varr, std::unique_ptr const& db, std::string const& key) { - std::string message; - - protozero::pbf_writer item_writer(message); - - { - // Using new (in protozero 1.3.0) packed writing API - // https://github.com/mapbox/protozero/commit/4e7e32ac5350ea6d3dcf78ff5e74faeee513a6e1 - protozero::packed_field_uint64 field{item_writer, 1}; - uint64_t lastval = 0; - for (auto const& vitem : varr) { - if (lastval == 0) { - field.add_element(static_cast(vitem)); - } else { - field.add_element(static_cast(lastval - vitem)); - } - lastval = vitem; - } - } - - db->Put(rocksdb::WriteOptions(), key, message); -} - -NAN_METHOD(MemoryCache::pack) { - if (info.Length() < 1) { - return Nan::ThrowTypeError("expected one info: 'filename'"); - } - if (!info[0]->IsString()) { - return Nan::ThrowTypeError("first argument must be a String"); - } - try { - Nan::Utf8String utf8_filename(info[0]); - if (utf8_filename.length() < 1) { - return Nan::ThrowTypeError("first arg must be a String"); - } - std::string filename(*utf8_filename); - - MemoryCache* c = node::ObjectWrap::Unwrap(info.This()); - - std::unique_ptr db; - rocksdb::Options options; - options.create_if_missing = true; - rocksdb::Status status = OpenDB(options, filename, db); - - if (!status.ok()) { - return Nan::ThrowTypeError("unable to open rocksdb file for packing"); - } - - std::map> memoized_prefixes; - - for (auto const& item : c->cache_) { - std::size_t array_size = item.second.size(); - if (array_size > 0) { - // make copy of intarray so we can sort without - // modifying the original array - intarray varr = item.second; - - // delta-encode values, sorted in descending order. - std::sort(varr.begin(), varr.end(), std::greater()); - - packVec(varr, db, item.first); - - std::string prefix_t1 = ""; - std::string prefix_t2 = ""; - - // add this to the memoized prefix array too, maybe - auto phrase_length = item.first.find(LANGFIELD_SEPARATOR); - // use the full string for things shorter than the limit - // or the prefix otherwise - if (phrase_length < MEMO_PREFIX_LENGTH_T1) { - prefix_t1 = "=1" + item.first; - } else { - // get the prefix, then append the langfield back onto it again - langfield_type langfield = extract_langfield(item.first); - - prefix_t1 = "=1" + item.first.substr(0, MEMO_PREFIX_LENGTH_T1); - add_langfield(prefix_t1, langfield); - - if (phrase_length < MEMO_PREFIX_LENGTH_T2) { - prefix_t2 = "=2" + item.first; - } else { - prefix_t2 = "=2" + item.first.substr(0, MEMO_PREFIX_LENGTH_T2); - add_langfield(prefix_t2, langfield); - } - } - - if (prefix_t1 != "") { - std::map>::const_iterator mitr = memoized_prefixes.find(prefix_t1); - if (mitr == memoized_prefixes.end()) { - memoized_prefixes.emplace(prefix_t1, std::deque()); - } - std::deque& buf = memoized_prefixes[prefix_t1]; - - buf.insert(buf.end(), varr.begin(), varr.end()); - } - if (prefix_t2 != "") { - std::map>::const_iterator mitr = memoized_prefixes.find(prefix_t2); - if (mitr == memoized_prefixes.end()) { - memoized_prefixes.emplace(prefix_t2, std::deque()); - } - std::deque& buf = memoized_prefixes[prefix_t2]; - - buf.insert(buf.end(), varr.begin(), varr.end()); - } - } - } - - for (auto const& item : memoized_prefixes) { - // copy the deque into a vector so we can sort without - // modifying the original array - intarray varr(item.second.begin(), item.second.end()); - - // delta-encode values, sorted in descending order. - std::sort(varr.begin(), varr.end(), std::greater()); - - if (varr.size() > PREFIX_MAX_GRID_LENGTH) { - // for the prefix memos we're only going to ever use 500k max anyway - varr.resize(PREFIX_MAX_GRID_LENGTH); - } - - packVec(varr, db, item.first); - } - } catch (std::exception const& ex) { - return Nan::ThrowTypeError(ex.what()); - } - info.GetReturnValue().Set(Nan::Undefined()); - return; -} - -NAN_METHOD(RocksDBCache::pack) { - if (info.Length() < 1) { - return Nan::ThrowTypeError("expected one info: 'filename'"); - } - if (!info[0]->IsString()) { - return Nan::ThrowTypeError("first argument must be a String"); - } - try { - Nan::Utf8String utf8_filename(info[0]); - if (utf8_filename.length() < 1) { - return Nan::ThrowTypeError("first arg must be a String"); - } - std::string filename(*utf8_filename); - - RocksDBCache* c = node::ObjectWrap::Unwrap(info.This()); - - if (c->db && c->db->GetName() == filename) { - return Nan::ThrowTypeError("rocksdb file is already loaded read-only; unload first"); - } else { - std::shared_ptr existing = c->db; - - std::unique_ptr db; - rocksdb::Options options; - options.create_if_missing = true; - rocksdb::Status status = OpenDB(options, filename, db); - - if (!status.ok()) { - return Nan::ThrowTypeError("unable to open rocksdb file for packing"); - } - - // if what we have now is already a rocksdb, and it's a different - // one from what we're being asked to pack into, copy from one to the other - std::unique_ptr existingIt(existing->NewIterator(rocksdb::ReadOptions())); - for (existingIt->SeekToFirst(); existingIt->Valid(); existingIt->Next()) { - db->Put(rocksdb::WriteOptions(), existingIt->key(), existingIt->value()); - } - } - info.GetReturnValue().Set(true); - return; - } catch (std::exception const& ex) { - return Nan::ThrowTypeError(ex.what()); - } - info.GetReturnValue().Set(Nan::Undefined()); - return; -} - -struct MergeBaton : carmen::noncopyable { - uv_work_t request; - std::string filename1; - std::string filename2; - std::string filename3; - std::string method; - std::string error; - Nan::Persistent callback; -}; - -void mergeQueue(uv_work_t* req) { - MergeBaton* baton = static_cast(req->data); - std::string const& filename1 = baton->filename1; - std::string const& filename2 = baton->filename2; - std::string const& filename3 = baton->filename3; - std::string const& method = baton->method; - - // input 1 - std::unique_ptr db1; - rocksdb::Options options1; - options1.create_if_missing = true; - rocksdb::Status status1 = OpenForReadOnlyDB(options1, filename1, db1); - if (!status1.ok()) { - return Nan::ThrowTypeError("unable to open rocksdb input file #1"); - } - - // input 2 - std::unique_ptr db2; - rocksdb::Options options2; - options2.create_if_missing = true; - rocksdb::Status status2 = OpenForReadOnlyDB(options2, filename2, db2); - if (!status2.ok()) { - return Nan::ThrowTypeError("unable to open rocksdb input file #2"); - } - - // output - std::unique_ptr db3; - rocksdb::Options options3; - options3.create_if_missing = true; - rocksdb::Status status3 = OpenDB(options3, filename3, db3); - if (!status1.ok()) { - return Nan::ThrowTypeError("unable to open rocksdb output file"); - } - - // Ids that have been seen - std::map ids1; - std::map ids2; - - try { - // Store ids from 1 - std::unique_ptr it1(db1->NewIterator(rocksdb::ReadOptions())); - for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { - ids1.emplace(it1->key().ToString(), true); - } - - // Store ids from 2 - std::unique_ptr it2(db2->NewIterator(rocksdb::ReadOptions())); - for (it2->SeekToFirst(); it2->Valid(); it2->Next()) { - ids2.emplace(it2->key().ToString(), true); - } - - // No delta writes from message1 - it1 = std::unique_ptr(db1->NewIterator(rocksdb::ReadOptions())); - for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { - std::string key_id = it1->key().ToString(); - - // Skip this id if also in message 2 - if (ids2.find(key_id) != ids2.end()) continue; - - // get input proto - std::string in_message = it1->value().ToString(); - protozero::pbf_reader item(in_message); - item.next(CACHE_ITEM); - - std::string message; - message.clear(); - - protozero::pbf_writer item_writer(message); - { - protozero::packed_field_uint64 field{item_writer, 1}; - auto vals = item.get_packed_uint64(); - for (auto it = vals.first; it != vals.second; ++it) { - field.add_element(static_cast(*it)); - } - } - - rocksdb::Status putStatus = db3->Put(rocksdb::WriteOptions(), key_id, message); - assert(putStatus.ok()); - } - - // No delta writes from message2 - it2 = std::unique_ptr(db2->NewIterator(rocksdb::ReadOptions())); - for (it2->SeekToFirst(); it2->Valid(); it2->Next()) { - std::string key_id = it2->key().ToString(); - - // Skip this id if also in message 1 - if (ids1.find(key_id) != ids1.end()) continue; - - // get input proto - std::string in_message = it2->value().ToString(); - protozero::pbf_reader item(in_message); - item.next(CACHE_ITEM); - - std::string message; - message.clear(); - - protozero::pbf_writer item_writer(message); - { - protozero::packed_field_uint64 field{item_writer, 1}; - auto vals = item.get_packed_uint64(); - for (auto it = vals.first; it != vals.second; ++it) { - field.add_element(static_cast(*it)); - } - } - - rocksdb::Status putStatus = db3->Put(rocksdb::WriteOptions(), key_id, message); - assert(putStatus.ok()); - } - - // Delta writes for ids in both message1 and message2 - it1 = std::unique_ptr(db1->NewIterator(rocksdb::ReadOptions())); - for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { - std::string key_id = it1->key().ToString(); - - // Skip ids that are only in one or the other lists - if (ids1.find(key_id) == ids1.end() || ids2.find(key_id) == ids2.end()) continue; - - // get input proto - std::string in_message1 = it1->value().ToString(); - protozero::pbf_reader item(in_message1); - item.next(CACHE_ITEM); - - uint64_t lastval = 0; - intarray varr; - - // Add values from filename1 - auto vals = item.get_packed_uint64(); - for (auto it = vals.first; it != vals.second; ++it) { - if (method == "freq") { - varr.emplace_back(*it); - break; - } else if (lastval == 0) { - lastval = *it; - varr.emplace_back(lastval); - } else { - lastval = lastval - *it; - varr.emplace_back(lastval); - } - } - - std::string in_message2; - std::string max_key = "__MAX__"; - auto max_key_length = max_key.length(); - rocksdb::Status s = db2->Get(rocksdb::ReadOptions(), key_id, &in_message2); - if (s.ok()) { - // get input proto 2 - protozero::pbf_reader item2(in_message2); - item2.next(CACHE_ITEM); - - auto vals2 = item2.get_packed_uint64(); - lastval = 0; - for (auto it = vals2.first; it != vals2.second; ++it) { - if (method == "freq") { - if (key_id.compare(0, max_key_length, max_key) == 0) { - varr[0] = varr[0] > *it ? varr[0] : *it; - } else { - varr[0] = varr[0] + *it; - } - break; - } else if (lastval == 0) { - lastval = *it; - varr.emplace_back(lastval); - } else { - lastval = lastval - *it; - varr.emplace_back(lastval); - } - } - } - - // Sort for proper delta encoding - std::sort(varr.begin(), varr.end(), std::greater()); - - // if this is the merging of a prefix cache entry - // (which would start with '=' and have been truncated) - // truncate the merged result - if (key_id.at(0) == '=' && varr.size() > PREFIX_MAX_GRID_LENGTH) { - varr.resize(PREFIX_MAX_GRID_LENGTH); - } - - // Write varr to merged protobuf - std::string message; - message.clear(); - - protozero::pbf_writer item_writer(message); - { - protozero::packed_field_uint64 field{item_writer, 1}; - lastval = 0; - for (auto const& vitem : varr) { - if (lastval == 0) { - field.add_element(static_cast(vitem)); - } else { - field.add_element(static_cast(lastval - vitem)); - } - lastval = vitem; - } - } - - rocksdb::Status putStatus = db3->Put(rocksdb::WriteOptions(), key_id, message); - assert(putStatus.ok()); - } - - } catch (std::exception const& ex) { - baton->error = ex.what(); - } -} - -void mergeAfter(uv_work_t* req) { - Nan::HandleScope scope; - MergeBaton* baton = static_cast(req->data); - if (!baton->error.empty()) { - v8::Local argv[1] = {Nan::Error(baton->error.c_str())}; - Nan::MakeCallback(Nan::GetCurrentContext()->Global(), Nan::New(baton->callback), 1, argv); - } else { - Local argv[2] = {Nan::Null()}; - Nan::MakeCallback(Nan::GetCurrentContext()->Global(), Nan::New(baton->callback), 1, argv); - } - baton->callback.Reset(); - delete baton; -} - -NAN_METHOD(MemoryCache::list) { - try { - Nan::Utf8String utf8_value(info[0]); - if (utf8_value.length() < 1) { - return Nan::ThrowTypeError("first arg must be a String"); - } - MemoryCache* c = node::ObjectWrap::Unwrap(info.This()); - Local ids = Nan::New(); - - unsigned idx = 0; - for (auto const& item : c->cache_) { - Local out = Nan::New(); - out->Set(0, Nan::New(item.first.substr(0, item.first.find(LANGFIELD_SEPARATOR))).ToLocalChecked()); - - langfield_type langfield = extract_langfield(item.first); - if (langfield == ALL_LANGUAGES) { - out->Set(1, Nan::Null()); - } else { - out->Set(1, langfieldToLangarray(langfield)); - } - - ids->Set(idx++, out); - } - - info.GetReturnValue().Set(ids); - return; - } catch (std::exception const& ex) { - return Nan::ThrowTypeError(ex.what()); - } - info.GetReturnValue().Set(Nan::Undefined()); - return; -} - -NAN_METHOD(RocksDBCache::list) { - try { - RocksDBCache* c = node::ObjectWrap::Unwrap(info.This()); - Local ids = Nan::New(); - - std::shared_ptr db = c->db; - - std::unique_ptr it(db->NewIterator(rocksdb::ReadOptions())); - unsigned idx = 0; - for (it->SeekToFirst(); it->Valid(); it->Next()) { - std::string key_id = it->key().ToString(); - if (key_id.at(0) == '=') continue; - - Local out = Nan::New(); - out->Set(0, Nan::New(key_id.substr(0, key_id.find(LANGFIELD_SEPARATOR))).ToLocalChecked()); - - langfield_type langfield = extract_langfield(key_id); - if (langfield == ALL_LANGUAGES) { - out->Set(1, Nan::Null()); - } else { - out->Set(1, langfieldToLangarray(langfield)); - } - - ids->Set(idx++, out); - } - - info.GetReturnValue().Set(ids); - return; - } catch (std::exception const& ex) { - return Nan::ThrowTypeError(ex.what()); - } - info.GetReturnValue().Set(Nan::Undefined()); - return; -} - -NAN_METHOD(RocksDBCache::merge) { - if (!info[0]->IsString()) return Nan::ThrowTypeError("argument 1 must be a String (infile 1)"); - if (!info[1]->IsString()) return Nan::ThrowTypeError("argument 2 must be a String (infile 2)"); - if (!info[2]->IsString()) return Nan::ThrowTypeError("argument 3 must be a String (outfile)"); - if (!info[3]->IsString()) return Nan::ThrowTypeError("argument 4 must be a String (method)"); - if (!info[4]->IsFunction()) return Nan::ThrowTypeError("argument 5 must be a callback function"); - - std::string in1 = *String::Utf8Value(info[0]->ToString()); - std::string in2 = *String::Utf8Value(info[1]->ToString()); - std::string out = *String::Utf8Value(info[2]->ToString()); - Local callback = info[4]; - std::string method = *String::Utf8Value(info[3]->ToString()); - - MergeBaton* baton = new MergeBaton(); - baton->filename1 = in1; - baton->filename2 = in2; - baton->filename3 = out; - baton->method = method; - baton->callback.Reset(callback.As()); - baton->request.data = baton; - uv_queue_work(uv_default_loop(), &baton->request, mergeQueue, (uv_after_work_cb)mergeAfter); - info.GetReturnValue().Set(Nan::Undefined()); - return; -} - -NAN_METHOD(MemoryCache::_set) { - if (info.Length() < 2) { - return Nan::ThrowTypeError("expected at least two info: id, data, [languages], [append]"); - } - if (!info[0]->IsString()) { - return Nan::ThrowTypeError("first arg must be a String"); - } - if (!info[1]->IsArray()) { - return Nan::ThrowTypeError("second arg must be an Array"); - } - Local data = Local::Cast(info[1]); - if (data->IsNull() || data->IsUndefined()) { - return Nan::ThrowTypeError("an array expected for second argument"); - } - try { - - Nan::Utf8String utf8_id(info[0]); - if (utf8_id.length() < 1) { - return Nan::ThrowTypeError("first arg must be a String"); - } - std::string id(*utf8_id); - - langfield_type langfield; - if (info.Length() > 2 && !(info[2]->IsNull() || info[2]->IsUndefined())) { - if (!info[2]->IsArray()) { - return Nan::ThrowTypeError("third arg, if supplied must be an Array"); - } - langfield = langarrayToLangfield(Local::Cast(info[2])); - } else { - langfield = ALL_LANGUAGES; - } - - bool append = info.Length() > 3 && info[3]->IsBoolean() && info[3]->BooleanValue(); - - MemoryCache* c = node::ObjectWrap::Unwrap(info.This()); - arraycache& arrc = c->cache_; - key_type key_id = static_cast(id); - add_langfield(key_id, langfield); - - arraycache::iterator itr2 = arrc.find(key_id); - if (itr2 == arrc.end()) { - arrc.emplace(key_id, intarray()); - } - intarray& vv = arrc[key_id]; - - unsigned array_size = data->Length(); - if (append) { - vv.reserve(vv.size() + array_size); - } else { - if (itr2 != arrc.end()) vv.clear(); - vv.reserve(array_size); - } - - for (unsigned i = 0; i < array_size; ++i) { - vv.emplace_back(static_cast(data->Get(i)->NumberValue())); - } - } catch (std::exception const& ex) { - return Nan::ThrowTypeError(ex.what()); - } - info.GetReturnValue().Set(Nan::Undefined()); - return; -} - -template -inline NAN_METHOD(_genericget) { - if (info.Length() < 1) { - return Nan::ThrowTypeError("expected at least one info: id, [languages]"); - } - if (!info[0]->IsString()) { - return Nan::ThrowTypeError("first arg must be a String"); - } - try { - Nan::Utf8String utf8_id(info[0]); - if (utf8_id.length() < 1) { - return Nan::ThrowTypeError("first arg must be a String"); - } - std::string id(*utf8_id); - - langfield_type langfield; - if (info.Length() > 1 && !(info[1]->IsNull() || info[1]->IsUndefined())) { - if (!info[1]->IsArray()) { - return Nan::ThrowTypeError("second arg, if supplied must be an Array"); - } - langfield = langarrayToLangfield(Local::Cast(info[1])); - } else { - langfield = ALL_LANGUAGES; - } - - T* c = node::ObjectWrap::Unwrap(info.This()); - intarray vector = __get(c, id, langfield); - if (!vector.empty()) { - std::size_t size = vector.size(); - Local array = Nan::New(static_cast(size)); - for (uint32_t i = 0; i < size; ++i) { - array->Set(i, Nan::New(vector[i])); - } - info.GetReturnValue().Set(array); - return; - } else { - info.GetReturnValue().Set(Nan::Undefined()); - return; - } - } catch (std::exception const& ex) { - return Nan::ThrowTypeError(ex.what()); - } -} - -NAN_METHOD(MemoryCache::_get) { - return _genericget(info); -} - -NAN_METHOD(RocksDBCache::_get) { - return _genericget(info); -} - -NAN_METHOD(MemoryCache::New) { - if (!info.IsConstructCall()) { - return Nan::ThrowTypeError("Cannot call constructor as function, you need to use 'new' keyword"); - } - try { - if (info.Length() < 1) { - return Nan::ThrowTypeError("expected 'id' argument"); - } - if (!info[0]->IsString()) { - return Nan::ThrowTypeError("first argument 'id' must be a String"); - } - MemoryCache* im = new MemoryCache(); - - im->Wrap(info.This()); - info.This()->Set(Nan::New("id").ToLocalChecked(), info[0]); - info.GetReturnValue().Set(info.This()); - return; - } catch (std::exception const& ex) { - return Nan::ThrowTypeError(ex.what()); - } - info.GetReturnValue().Set(Nan::Undefined()); - return; -} - -NAN_METHOD(RocksDBCache::New) { - if (!info.IsConstructCall()) { - return Nan::ThrowTypeError("Cannot call constructor as function, you need to use 'new' keyword"); - } - try { - if (info.Length() < 2) { - return Nan::ThrowTypeError("expected arguments 'id' and 'filename'"); - } - if (!info[0]->IsString()) { - return Nan::ThrowTypeError("first argument 'id' must be a String"); - } - if (!info[1]->IsString()) { - return Nan::ThrowTypeError("second argument 'filename' must be a String"); - } - - Nan::Utf8String utf8_filename(info[1]); - if (utf8_filename.length() < 1) { - return Nan::ThrowTypeError("second arg must be a String"); - } - std::string filename(*utf8_filename); - - std::unique_ptr db; - rocksdb::Options options; - options.create_if_missing = true; - rocksdb::Status status = OpenForReadOnlyDB(options, filename, db); - - if (!status.ok()) { - return Nan::ThrowTypeError("unable to open rocksdb file for loading"); - } - RocksDBCache* im = new RocksDBCache(); - im->db = std::move(db); - im->Wrap(info.This()); - info.This()->Set(Nan::New("id").ToLocalChecked(), info[0]); - info.GetReturnValue().Set(info.This()); - return; - } catch (std::exception const& ex) { - return Nan::ThrowTypeError(ex.what()); - } - info.GetReturnValue().Set(Nan::Undefined()); - return; -} - -//relev = 5 bits -//count = 3 bits -//reason = 12 bits -//* 1 bit gap -//id = 32 bits -constexpr double _pow(double x, int y) { - return y == 0 ? 1.0 : x * _pow(x, y - 1); -} - -constexpr uint64_t POW2_51 = static_cast(_pow(2.0, 51)); -constexpr uint64_t POW2_48 = static_cast(_pow(2.0, 48)); -constexpr uint64_t POW2_34 = static_cast(_pow(2.0, 34)); -constexpr uint64_t POW2_28 = static_cast(_pow(2.0, 28)); -constexpr uint64_t POW2_25 = static_cast(_pow(2.0, 25)); -constexpr uint64_t POW2_20 = static_cast(_pow(2.0, 20)); -constexpr uint64_t POW2_14 = static_cast(_pow(2.0, 14)); -constexpr uint64_t POW2_3 = static_cast(_pow(2.0, 3)); -constexpr uint64_t POW2_2 = static_cast(_pow(2.0, 2)); - -struct PhrasematchSubq { - PhrasematchSubq(void* c, - char t, - double w, - std::string p, - bool pf, - unsigned short i, - unsigned short z, - uint32_t m, - langfield_type l) : cache(c), - type(t), - weight(w), - phrase(p), - prefix(pf), - idx(i), - zoom(z), - mask(m), - langfield(l) {} - void* cache; - char type; - double weight; - std::string phrase; - bool prefix; - unsigned short idx; - unsigned short zoom; - uint32_t mask; - langfield_type langfield; - PhrasematchSubq& operator=(PhrasematchSubq&& c) = default; - PhrasematchSubq(PhrasematchSubq&& c) = default; -}; - -struct Cover { - double relev; - uint32_t id; - uint32_t tmpid; - unsigned short x; - unsigned short y; - unsigned short score; - unsigned short idx; - uint32_t mask; - double distance; - double scoredist; - bool matches_language; -}; - -struct Context { - std::vector coverList; - uint32_t mask; - double relev; - - Context(Context const& c) = default; - Context(Cover&& cov, - uint32_t mask, - double relev) - : coverList(), - mask(mask), - relev(relev) { - coverList.emplace_back(std::move(cov)); - } - Context& operator=(Context&& c) { - coverList = std::move(c.coverList); - mask = std::move(c.mask); - relev = std::move(c.relev); - return *this; - } - Context(std::vector&& cl, - uint32_t mask, - double relev) - : coverList(std::move(cl)), - mask(mask), - relev(relev) {} - - Context(Context&& c) - : coverList(std::move(c.coverList)), - mask(std::move(c.mask)), - relev(std::move(c.relev)) {} -}; - -Cover numToCover(uint64_t num) { - Cover cover; - assert(((num >> 34) % POW2_14) <= static_cast(std::numeric_limits::max())); - assert(((num >> 34) % POW2_14) >= static_cast(std::numeric_limits::min())); - unsigned short y = static_cast((num >> 34) % POW2_14); - assert(((num >> 20) % POW2_14) <= static_cast(std::numeric_limits::max())); - assert(((num >> 20) % POW2_14) >= static_cast(std::numeric_limits::min())); - unsigned short x = static_cast((num >> 20) % POW2_14); - assert(((num >> 48) % POW2_3) <= static_cast(std::numeric_limits::max())); - assert(((num >> 48) % POW2_3) >= static_cast(std::numeric_limits::min())); - unsigned short score = static_cast((num >> 48) % POW2_3); - uint32_t id = static_cast(num % POW2_20); - bool matches_language = static_cast(num & LANGUAGE_MATCH_BOOST); - cover.x = x; - cover.y = y; - double relev = 0.4 + (0.2 * static_cast((num >> 51) % POW2_2)); - cover.relev = relev; - cover.score = score; - cover.id = id; - cover.matches_language = matches_language; - - // These are not derived from decoding the input num but by - // external values after initialization. - cover.idx = 0; - cover.mask = 0; - cover.tmpid = 0; - cover.distance = 0; - - return cover; -} - -struct ZXY { - unsigned z; - unsigned x; - unsigned y; -}; - -ZXY pxy2zxy(unsigned z, unsigned x, unsigned y, unsigned target_z) { - ZXY zxy; - zxy.z = target_z; - - // Interval between parent and target zoom level - signed zDist = static_cast(target_z) - static_cast(z); - signed zMult = zDist - 1; - if (zDist == 0) { - zxy.x = x; - zxy.y = y; - return zxy; - } - - // Midpoint length @ z for a tile at parent zoom level - double pMid_d = static_cast(std::pow(2, zDist) / 2); - assert(pMid_d <= static_cast(std::numeric_limits::max())); - assert(pMid_d >= static_cast(std::numeric_limits::min())); - signed pMid = static_cast(pMid_d); - zxy.x = static_cast((static_cast(x) * zMult) + pMid); - zxy.y = static_cast((static_cast(y) * zMult) + pMid); - return zxy; -} - -ZXY bxy2zxy(unsigned z, unsigned x, unsigned y, unsigned target_z, bool max = false) { - ZXY zxy; - zxy.z = target_z; - - // Interval between parent and target zoom level - signed zDist = static_cast(target_z) - static_cast(z); - if (zDist == 0) { - zxy.x = x; - zxy.y = y; - return zxy; - } - - // zoom conversion multiplier - float mult = static_cast(std::pow(2, zDist)); - - // zoom in min - if (zDist > 0 && !max) { - zxy.x = static_cast(static_cast(x) * mult); - zxy.y = static_cast(static_cast(y) * mult); - return zxy; - } - // zoom in max - else if (zDist > 0 && max) { - zxy.x = static_cast(static_cast(x) * mult + (mult - 1)); - zxy.y = static_cast(static_cast(y) * mult + (mult - 1)); - return zxy; - } - // zoom out - else { - unsigned mod = static_cast(std::pow(2, target_z)); - unsigned xDiff = x % mod; - unsigned yDiff = y % mod; - unsigned newX = x - xDiff; - unsigned newY = y - yDiff; - - zxy.x = static_cast(static_cast(newX) * mult); - zxy.y = static_cast(static_cast(newY) * mult); - return zxy; - } -} - -inline bool coverSortByRelev(Cover const& a, Cover const& b) noexcept { - if (b.relev > a.relev) - return false; - else if (b.relev < a.relev) - return true; - else if (b.scoredist > a.scoredist) - return false; - else if (b.scoredist < a.scoredist) - return true; - else if (b.idx < a.idx) - return false; - else if (b.idx > a.idx) - return true; - else if (b.id < a.id) - return false; - else if (b.id > a.id) - return true; - // sorting by x and y is arbitrary but provides a more deterministic output order - else if (b.x < a.x) - return false; - else if (b.x > a.x) - return true; - else - return (b.y > a.y); -} - -inline bool subqSortByZoom(PhrasematchSubq const& a, PhrasematchSubq const& b) noexcept { - if (a.zoom < b.zoom) return true; - if (a.zoom > b.zoom) return false; - return (a.idx < b.idx); -} - -inline bool contextSortByRelev(Context const& a, Context const& b) noexcept { - if (b.relev > a.relev) - return false; - else if (b.relev < a.relev) - return true; - else if (b.coverList[0].scoredist > a.coverList[0].scoredist) - return false; - else if (b.coverList[0].scoredist < a.coverList[0].scoredist) - return true; - else if (b.coverList[0].idx < a.coverList[0].idx) - return false; - else if (b.coverList[0].idx > a.coverList[0].idx) - return true; - return (b.coverList[0].id > a.coverList[0].id); -} - -inline double tileDist(unsigned px, unsigned py, unsigned tileX, unsigned tileY) { - const double dx = static_cast(px) - static_cast(tileX); - const double dy = static_cast(py) - static_cast(tileY); - const double distance = sqrt((dx * dx) + (dy * dy)); - - return distance; -} - -struct CoalesceBaton : carmen::noncopyable { - uv_work_t request; - // params - std::vector stack; - std::vector centerzxy; - std::vector bboxzxy; - double radius; - Nan::Persistent callback; - // return - std::vector features; - // error - std::string error; -}; - -// Equivalent of scoredist() function in carmen -// Combines score and distance into a single score that can be used for sorting. -// Unlike carmen the effect is not scaled by zoom level as regardless of index -// the score value at this stage is a 0-7 scalar (by comparison, in carmen, scores -// for indexes with features covering lower zooms often have exponentially higher -// scores - example: country@z9 vs poi@z14). -double scoredist(unsigned zoom, double distance, double score, double radius) { - if (zoom < 6) zoom = 6; - if (distance == 0.0) distance = 0.01; - double scoredist = 0; - - // Since distance is in tiles we calculate scoredist by converting the miles into - // a tile unit value at the appropriate zoom first. - // - // 32 tiles is about 40 miles at z14, use this as our mile <=> tile conversion. - scoredist = ((radius * (32.0 / 40.0)) / _pow(1.5, 14 - static_cast(zoom))) / distance; - - return score > scoredist ? score : scoredist; -} - -void coalesceFinalize(CoalesceBaton* baton, std::vector&& contexts) { - if (!contexts.empty()) { - // Coalesce stack, generate relevs. - double relevMax = contexts[0].relev; - std::size_t total = 0; - std::map sets; - std::map::iterator sit; - std::size_t max_contexts = 40; - baton->features.reserve(max_contexts); - for (auto&& context : contexts) { - // Maximum allowance of coalesced features: 40. - if (total >= max_contexts) break; - - // Since `coalesced` is sorted by relev desc at first - // threshold miss we can break the loop. - if (relevMax - context.relev >= 0.25) break; - - // Only collect each feature once. - uint32_t id = context.coverList[0].tmpid; - sit = sets.find(id); - if (sit != sets.end()) continue; - - sets.emplace(id, true); - baton->features.emplace_back(std::move(context)); - total++; - } - } -} -void coalesceSingle(uv_work_t* req) { - CoalesceBaton* baton = static_cast(req->data); - - try { - std::vector const& stack = baton->stack; - PhrasematchSubq const& subq = stack[0]; - - // proximity (optional) - bool proximity = !baton->centerzxy.empty(); - unsigned cz; - unsigned cx; - unsigned cy; - if (proximity) { - cz = static_cast(baton->centerzxy[0]); - cx = static_cast(baton->centerzxy[1]); - cy = static_cast(baton->centerzxy[2]); - } else { - cz = 0; - cx = 0; - cy = 0; - } - - // bbox (optional) - bool bbox = !baton->bboxzxy.empty(); - unsigned minx; - unsigned miny; - unsigned maxx; - unsigned maxy; - if (bbox) { - minx = static_cast(baton->bboxzxy[1]); - miny = static_cast(baton->bboxzxy[2]); - maxx = static_cast(baton->bboxzxy[3]); - maxy = static_cast(baton->bboxzxy[4]); - } else { - minx = 0; - miny = 0; - maxx = 0; - maxy = 0; - } - - // Load and concatenate grids for all ids in `phrases` - intarray grids; - grids = subq.type == TYPE_MEMORY ? __getmatching(reinterpret_cast(subq.cache), subq.phrase, subq.prefix, subq.langfield) : __getmatching(reinterpret_cast(subq.cache), subq.phrase, subq.prefix, subq.langfield); - - unsigned long m = grids.size(); - double relevMax = 0; - std::vector covers; - covers.reserve(m); - - uint32_t length = 0; - uint32_t lastId = 0; - double lastRelev = 0; - double lastScoredist = 0; - double lastDistance = 0; - double minScoredist = std::numeric_limits::max(); - for (unsigned long j = 0; j < m; j++) { - Cover cover = numToCover(grids[j]); - - cover.idx = subq.idx; - cover.tmpid = static_cast(cover.idx * POW2_25 + cover.id); - cover.relev = cover.relev * subq.weight; - if (!cover.matches_language) cover.relev *= .9; - cover.distance = proximity ? tileDist(cx, cy, cover.x, cover.y) : 0; - cover.scoredist = proximity ? scoredist(cz, cover.distance, cover.score, baton->radius) : cover.score; - - // only add cover id if it's got a higer scoredist - if (lastId == cover.id && cover.scoredist <= lastScoredist) continue; - - // short circuit based on relevMax thres - if (length > 40) { - if (cover.scoredist < minScoredist) continue; - if (cover.relev < lastRelev) break; - } - if (relevMax - cover.relev >= 0.25) break; - if (cover.relev > relevMax) relevMax = cover.relev; - - if (bbox) { - if (cover.x < minx || cover.y < miny || cover.x > maxx || cover.y > maxy) continue; - } - - covers.emplace_back(cover); - if (lastId != cover.id) length++; - if (!proximity && length > 40) break; - if (cover.scoredist < minScoredist) minScoredist = cover.scoredist; - lastId = cover.id; - lastRelev = cover.relev; - lastScoredist = cover.scoredist; - lastDistance = cover.distance; - } - - // sort grids by distance to proximity point - std::sort(covers.begin(), covers.end(), coverSortByRelev); - - uint32_t lastid = 0; - std::size_t added = 0; - std::vector contexts; - std::size_t max_contexts = 40; - contexts.reserve(max_contexts); - for (auto&& cover : covers) { - // Stop at 40 contexts - if (added == max_contexts) break; - - // Attempt not to add the same feature but by diff cover twice - if (lastid == cover.id) continue; - - lastid = cover.id; - added++; - - double relev = cover.relev; - uint32_t mask = 0; - contexts.emplace_back(std::move(cover), mask, relev); - } - - coalesceFinalize(baton, std::move(contexts)); - } catch (std::exception const& ex) { - baton->error = ex.what(); - } -} - -void coalesceMulti(uv_work_t* req) { - CoalesceBaton* baton = static_cast(req->data); - - try { - std::vector& stack = baton->stack; - std::sort(stack.begin(), stack.end(), subqSortByZoom); - std::size_t stackSize = stack.size(); - - // Cache zoom levels to iterate over as coalesce occurs. - std::vector zoomCache; - zoomCache.reserve(stackSize); - for (auto const& subq : stack) { - intarray zooms; - std::vector zoomUniq(22, false); - for (auto const& subqB : stack) { - if (subq.idx == subqB.idx) continue; - if (zoomUniq[subqB.zoom]) continue; - if (subq.zoom < subqB.zoom) continue; - zoomUniq[subqB.zoom] = true; - zooms.emplace_back(subqB.zoom); - } - zoomCache.push_back(std::move(zooms)); - } - - // Coalesce relevs into higher zooms, e.g. - // z5 inherits relev of overlapping tiles at z4. - // @TODO assumes sources are in zoom ascending order. - std::map> coalesced; - std::map>::iterator cit; - std::map>::iterator pit; - std::map done; - std::map::iterator dit; - - // proximity (optional) - bool proximity = baton->centerzxy.size() > 0; - unsigned cz; - unsigned cx; - unsigned cy; - if (proximity) { - cz = static_cast(baton->centerzxy[0]); - cx = static_cast(baton->centerzxy[1]); - cy = static_cast(baton->centerzxy[2]); - } else { - cz = 0; - cx = 0; - cy = 0; - } - - // bbox (optional) - bool bbox = !baton->bboxzxy.empty(); - unsigned bboxz; - unsigned minx; - unsigned miny; - unsigned maxx; - unsigned maxy; - if (bbox) { - bboxz = static_cast(baton->bboxzxy[0]); - minx = static_cast(baton->bboxzxy[1]); - miny = static_cast(baton->bboxzxy[2]); - maxx = static_cast(baton->bboxzxy[3]); - maxy = static_cast(baton->bboxzxy[4]); - } else { - bboxz = 0; - minx = 0; - miny = 0; - maxx = 0; - maxy = 0; - } - - std::vector contexts; - std::size_t i = 0; - for (auto const& subq : stack) { - // Load and concatenate grids for all ids in `phrases` - intarray grids; - grids = subq.type == TYPE_MEMORY ? __getmatching(reinterpret_cast(subq.cache), subq.phrase, subq.prefix, subq.langfield) : __getmatching(reinterpret_cast(subq.cache), subq.phrase, subq.prefix, subq.langfield); - - bool first = i == 0; - bool last = i == (stack.size() - 1); - unsigned short z = subq.zoom; - auto const& zCache = zoomCache[i]; - std::size_t zCacheSize = zCache.size(); - - unsigned long m = grids.size(); - - for (unsigned long j = 0; j < m; j++) { - Cover cover = numToCover(grids[j]); - cover.idx = subq.idx; - cover.mask = subq.mask; - cover.tmpid = static_cast(cover.idx * POW2_25 + cover.id); - cover.relev = cover.relev * subq.weight; - if (!cover.matches_language) cover.relev *= .9; - if (proximity) { - ZXY dxy = pxy2zxy(z, cover.x, cover.y, cz); - cover.distance = tileDist(cx, cy, dxy.x, dxy.y); - cover.scoredist = scoredist(cz, cover.distance, cover.score, baton->radius); - } else { - cover.distance = 0; - cover.scoredist = cover.score; - } - - if (bbox) { - ZXY min = bxy2zxy(bboxz, minx, miny, z, false); - ZXY max = bxy2zxy(bboxz, maxx, maxy, z, true); - if (cover.x < min.x || cover.y < min.y || cover.x > max.x || cover.y > max.y) continue; - } - - uint64_t zxy = (z * POW2_28) + (cover.x * POW2_14) + (cover.y); - - // Reserve stackSize for the coverList. The vector - // will grow no larger that the size of the input - // subqueries that are being coalesced. - std::vector covers; - covers.reserve(stackSize); - covers.push_back(cover); - uint32_t context_mask = cover.mask; - double context_relev = cover.relev; - - for (unsigned a = 0; a < zCacheSize; a++) { - uint64_t p = zCache[a]; - double s = static_cast(1 << (z - p)); - uint64_t pxy = static_cast(p * POW2_28) + - static_cast(std::floor(cover.x / s) * POW2_14) + - static_cast(std::floor(cover.y / s)); - pit = coalesced.find(pxy); - if (pit != coalesced.end()) { - uint32_t lastMask = 0; - double lastRelev = 0.0; - for (auto const& parents : pit->second) { - for (auto const& parent : parents.coverList) { - // this cover is functionally identical with previous and - // is more relevant, replace the previous. - if (parent.mask == lastMask && parent.relev > lastRelev) { - covers.pop_back(); - covers.emplace_back(parent); - context_relev -= lastRelev; - context_relev += parent.relev; - lastMask = parent.mask; - lastRelev = parent.relev; - // this cover doesn't overlap with used mask. - } else if (!(context_mask & parent.mask)) { - covers.emplace_back(parent); - context_relev += parent.relev; - context_mask = context_mask | parent.mask; - lastMask = parent.mask; - lastRelev = parent.relev; - } - } - } - } - } - - if (last) { - // Slightly penalize contexts that have no stacking - if (covers.size() == 1) { - context_relev -= 0.01; - // Slightly penalize contexts in ascending order - } else if (covers[0].mask > covers[1].mask) { - context_relev -= 0.01; - } - contexts.emplace_back(std::move(covers), context_mask, context_relev); - } else if (first || covers.size() > 1) { - cit = coalesced.find(zxy); - if (cit == coalesced.end()) { - std::vector local_contexts; - local_contexts.emplace_back(std::move(covers), context_mask, context_relev); - coalesced.emplace(zxy, std::move(local_contexts)); - } else { - cit->second.emplace_back(std::move(covers), context_mask, context_relev); - } - } - } - - i++; - } - - // append coalesced to contexts by moving memory - for (auto&& matched : coalesced) { - for (auto&& context : matched.second) { - contexts.emplace_back(std::move(context)); - } - } - - std::sort(contexts.begin(), contexts.end(), contextSortByRelev); - coalesceFinalize(baton, std::move(contexts)); - } catch (std::exception const& ex) { - baton->error = ex.what(); - } -} - -Local coverToObject(Cover const& cover) { - Local object = Nan::New(); - object->Set(Nan::New("x").ToLocalChecked(), Nan::New(cover.x)); - object->Set(Nan::New("y").ToLocalChecked(), Nan::New(cover.y)); - object->Set(Nan::New("relev").ToLocalChecked(), Nan::New(cover.relev)); - object->Set(Nan::New("score").ToLocalChecked(), Nan::New(cover.score)); - object->Set(Nan::New("id").ToLocalChecked(), Nan::New(cover.id)); - object->Set(Nan::New("idx").ToLocalChecked(), Nan::New(cover.idx)); - object->Set(Nan::New("tmpid").ToLocalChecked(), Nan::New(cover.tmpid)); - object->Set(Nan::New("distance").ToLocalChecked(), Nan::New(cover.distance)); - object->Set(Nan::New("scoredist").ToLocalChecked(), Nan::New(cover.scoredist)); - object->Set(Nan::New("matches_language").ToLocalChecked(), Nan::New(cover.matches_language)); - return object; -} -Local contextToArray(Context const& context) { - std::size_t size = context.coverList.size(); - Local array = Nan::New(static_cast(size)); - for (uint32_t i = 0; i < size; i++) { - array->Set(i, coverToObject(context.coverList[i])); - } - array->Set(Nan::New("relev").ToLocalChecked(), Nan::New(context.relev)); - return array; -} -void coalesceAfter(uv_work_t* req) { - Nan::HandleScope scope; - CoalesceBaton* baton = static_cast(req->data); - - // Reference count the cache objects - for (auto& subq : baton->stack) { - if (subq.type == TYPE_MEMORY) - reinterpret_cast(subq.cache)->_unref(); - else - reinterpret_cast(subq.cache)->_unref(); - } - - if (!baton->error.empty()) { - v8::Local argv[1] = {Nan::Error(baton->error.c_str())}; - Nan::MakeCallback(Nan::GetCurrentContext()->Global(), Nan::New(baton->callback), 1, argv); - } else { - std::vector const& features = baton->features; - - Local jsFeatures = Nan::New(static_cast(features.size())); - for (std::size_t i = 0; i < features.size(); i++) { - jsFeatures->Set(i, contextToArray(features[i])); - } - - Local argv[2] = {Nan::Null(), jsFeatures}; - Nan::MakeCallback(Nan::GetCurrentContext()->Global(), Nan::New(baton->callback), 2, argv); - } - - baton->callback.Reset(); - delete baton; -} - -NAN_METHOD(coalesce) { - // PhrasematchStack (js => cpp) - if (info.Length() < 3) { - return Nan::ThrowTypeError("Expects 3 arguments: a PhrasematchSubq array, an option object, and a callback"); - } - - if (!info[0]->IsArray()) { - return Nan::ThrowTypeError("Arg 1 must be a PhrasematchSubq array"); - } - - Local array = Local::Cast(info[0]); - auto array_length = array->Length(); - if (array_length < 1) { - return Nan::ThrowTypeError("Arg 1 must be an array with one or more PhrasematchSubq objects"); - } - - // Options object (js => cpp) - Local options_val = info[1]; - if (!options_val->IsObject()) { - return Nan::ThrowTypeError("Arg 2 must be an options object"); - } - Local options = options_val->ToObject(); - - // callback - Local callback = info[2]; - if (!callback->IsFunction()) { - return Nan::ThrowTypeError("Arg 3 must be a callback function"); - } - - // We use unique_ptr here to manage the heap allocated CoalesceBaton - // If an error is thrown the unique_ptr will go out of scope and delete - // its underlying baton. - // If no error is throw we release the underlying baton pointer before - // heading into the threadpool since we assume it will be deleted manually in coalesceAfter - std::unique_ptr baton_ptr = std::make_unique(); - CoalesceBaton* baton = baton_ptr.get(); - try { - for (uint32_t i = 0; i < array_length; i++) { - Local val = array->Get(i); - if (!val->IsObject()) { - return Nan::ThrowTypeError("All items in array must be valid PhrasematchSubq objects"); - } - Local jsStack = val->ToObject(); - if (jsStack->IsNull() || jsStack->IsUndefined()) { - return Nan::ThrowTypeError("All items in array must be valid PhrasematchSubq objects"); - } - - double weight; - std::string phrase; - bool prefix; - unsigned short idx; - unsigned short zoom; - uint32_t mask; - langfield_type langfield; - - // TODO: this is verbose: we could write some generic functions to do this robust conversion per type - if (!jsStack->Has(Nan::New("idx").ToLocalChecked())) { - return Nan::ThrowTypeError("missing idx property"); - } else { - Local prop_val = jsStack->Get(Nan::New("idx").ToLocalChecked()); - if (!prop_val->IsNumber()) { - return Nan::ThrowTypeError("idx value must be a number"); - } - int64_t _idx = prop_val->IntegerValue(); - if (_idx < 0 || _idx > std::numeric_limits::max()) { - return Nan::ThrowTypeError("encountered idx value too large to fit in unsigned short"); - } - idx = static_cast(_idx); - } - - if (!jsStack->Has(Nan::New("zoom").ToLocalChecked())) { - return Nan::ThrowTypeError("missing zoom property"); - } else { - Local prop_val = jsStack->Get(Nan::New("zoom").ToLocalChecked()); - if (!prop_val->IsNumber()) { - return Nan::ThrowTypeError("zoom value must be a number"); - } - int64_t _zoom = prop_val->IntegerValue(); - if (_zoom < 0 || _zoom > std::numeric_limits::max()) { - return Nan::ThrowTypeError("encountered zoom value too large to fit in unsigned short"); - } - zoom = static_cast(_zoom); - } - - if (!jsStack->Has(Nan::New("weight").ToLocalChecked())) { - return Nan::ThrowTypeError("missing weight property"); - } else { - Local prop_val = jsStack->Get(Nan::New("weight").ToLocalChecked()); - if (!prop_val->IsNumber()) { - return Nan::ThrowTypeError("weight value must be a number"); - } - double _weight = prop_val->NumberValue(); - if (_weight < 0 || _weight > std::numeric_limits::max()) { - return Nan::ThrowTypeError("encountered weight value too large to fit in double"); - } - weight = _weight; - } - - if (!jsStack->Has(Nan::New("phrase").ToLocalChecked())) { - return Nan::ThrowTypeError("missing phrase property"); - } else { - Local prop_val = jsStack->Get(Nan::New("phrase").ToLocalChecked()); - if (!prop_val->IsString()) { - return Nan::ThrowTypeError("phrase value must be a string"); - } - Nan::Utf8String _phrase(prop_val); - if (_phrase.length() < 1) { - return Nan::ThrowTypeError("encountered invalid phrase"); - } - phrase = *_phrase; - } - - if (!jsStack->Has(Nan::New("prefix").ToLocalChecked())) { - return Nan::ThrowTypeError("missing prefix property"); - } else { - Local prop_val = jsStack->Get(Nan::New("prefix").ToLocalChecked()); - if (!prop_val->IsBoolean()) { - return Nan::ThrowTypeError("prefix value must be a boolean"); - } - prefix = prop_val->BooleanValue(); - } - - if (!jsStack->Has(Nan::New("mask").ToLocalChecked())) { - return Nan::ThrowTypeError("missing mask property"); - } else { - Local prop_val = jsStack->Get(Nan::New("mask").ToLocalChecked()); - if (!prop_val->IsNumber()) { - return Nan::ThrowTypeError("mask value must be a number"); - } - int64_t _mask = prop_val->IntegerValue(); - if (_mask < 0 || _mask > std::numeric_limits::max()) { - return Nan::ThrowTypeError("encountered mask value too large to fit in uint32_t"); - } - mask = static_cast(_mask); - } - - langfield = ALL_LANGUAGES; - if (jsStack->Has(Nan::New("languages").ToLocalChecked())) { - Local c_array = jsStack->Get(Nan::New("languages").ToLocalChecked()); - if (!c_array->IsArray()) { - return Nan::ThrowTypeError("languages must be an array"); - } - Local carray = Local::Cast(c_array); - langfield = langarrayToLangfield(carray); - } - - if (!jsStack->Has(Nan::New("cache").ToLocalChecked())) { - return Nan::ThrowTypeError("missing cache property"); - } else { - Local prop_val = jsStack->Get(Nan::New("cache").ToLocalChecked()); - if (!prop_val->IsObject()) { - return Nan::ThrowTypeError("cache value must be a Cache object"); - } - Local _cache = prop_val->ToObject(); - if (_cache->IsNull() || _cache->IsUndefined()) { - return Nan::ThrowTypeError("cache value must be a Cache object"); - } - bool isMemoryCache = Nan::New(MemoryCache::constructor)->HasInstance(prop_val); - bool isRocksDBCache = Nan::New(RocksDBCache::constructor)->HasInstance(prop_val); - if (!(isMemoryCache || isRocksDBCache)) { - return Nan::ThrowTypeError("cache value must be a MemoryCache or RocksDBCache object"); - } - if (isMemoryCache) { - baton->stack.emplace_back( - (void*)node::ObjectWrap::Unwrap(_cache), - TYPE_MEMORY, - weight, - phrase, - prefix, - idx, - zoom, - mask, - langfield); - } else { - baton->stack.emplace_back( - (void*)node::ObjectWrap::Unwrap(_cache), - TYPE_ROCKSDB, - weight, - phrase, - prefix, - idx, - zoom, - mask, - langfield); - } - } - } - - if (options->Has(Nan::New("radius").ToLocalChecked())) { - Local prop_val = options->Get(Nan::New("radius").ToLocalChecked()); - if (!prop_val->IsNumber()) { - return Nan::ThrowTypeError("radius must be a number"); - } - int64_t _radius = prop_val->IntegerValue(); - if (_radius < 0 || _radius > std::numeric_limits::max()) { - return Nan::ThrowTypeError("encountered radius too large to fit in unsigned"); - } - baton->radius = static_cast(_radius); - } else { - baton->radius = 40.0; - } - - if (options->Has(Nan::New("centerzxy").ToLocalChecked())) { - Local c_array = options->Get(Nan::New("centerzxy").ToLocalChecked()); - if (!c_array->IsArray()) { - return Nan::ThrowTypeError("centerzxy must be an array"); - } - Local carray = Local::Cast(c_array); - if (carray->Length() != 3) { - return Nan::ThrowTypeError("centerzxy must be an array of 3 numbers"); - } - baton->centerzxy.reserve(carray->Length()); - for (uint32_t i = 0; i < carray->Length(); ++i) { - Local item = carray->Get(i); - if (!item->IsNumber()) { - return Nan::ThrowTypeError("centerzxy values must be number"); - } - int64_t a_val = item->IntegerValue(); - if (a_val < 0 || a_val > std::numeric_limits::max()) { - return Nan::ThrowTypeError("encountered centerzxy value too large to fit in uint32_t"); - } - baton->centerzxy.emplace_back(static_cast(a_val)); - } - } - - if (options->Has(Nan::New("bboxzxy").ToLocalChecked())) { - Local c_array = options->Get(Nan::New("bboxzxy").ToLocalChecked()); - if (!c_array->IsArray()) { - return Nan::ThrowTypeError("bboxzxy must be an array"); - } - Local carray = Local::Cast(c_array); - if (carray->Length() != 5) { - return Nan::ThrowTypeError("bboxzxy must be an array of 5 numbers"); - } - baton->bboxzxy.reserve(carray->Length()); - for (uint32_t i = 0; i < carray->Length(); ++i) { - Local item = carray->Get(i); - if (!item->IsNumber()) { - return Nan::ThrowTypeError("bboxzxy values must be number"); - } - int64_t a_val = item->IntegerValue(); - if (a_val < 0 || a_val > std::numeric_limits::max()) { - return Nan::ThrowTypeError("encountered bboxzxy value too large to fit in uint32_t"); - } - baton->bboxzxy.emplace_back(static_cast(a_val)); - } - } - - baton->callback.Reset(callback.As()); - - // queue work - baton->request.data = baton; - // Release the managed baton - baton_ptr.release(); - // Reference count the cache objects - for (auto& subq : baton->stack) { - if (subq.type == TYPE_MEMORY) - reinterpret_cast(subq.cache)->_ref(); - else - reinterpret_cast(subq.cache)->_ref(); - } - // optimization: for stacks of 1, use coalesceSingle - if (baton->stack.size() == 1) { - uv_queue_work(uv_default_loop(), &baton->request, coalesceSingle, (uv_after_work_cb)coalesceAfter); - } else { - uv_queue_work(uv_default_loop(), &baton->request, coalesceMulti, (uv_after_work_cb)coalesceAfter); - } - } catch (std::exception const& ex) { - return Nan::ThrowTypeError(ex.what()); - } - - info.GetReturnValue().Set(Nan::Undefined()); - return; -} - -template -inline NAN_METHOD(_genericgetmatching) { - if (info.Length() < 2) { - return Nan::ThrowTypeError("expected two or three info: id, match_prefixes, [languages]"); - } - if (!info[0]->IsString()) { - return Nan::ThrowTypeError("first arg must be a String"); - } - if (!info[1]->IsBoolean()) { - return Nan::ThrowTypeError("second arg must be a Bool"); - } - try { - Nan::Utf8String utf8_id(info[0]); - if (utf8_id.length() < 1) { - return Nan::ThrowTypeError("first arg must be a String"); - } - std::string id(*utf8_id); - - bool match_prefixes = info[1]->BooleanValue(); - - langfield_type langfield; - if (info.Length() > 2 && !(info[2]->IsNull() || info[2]->IsUndefined())) { - if (!info[2]->IsArray()) { - return Nan::ThrowTypeError("third arg, if supplied must be an Array"); - } - langfield = langarrayToLangfield(Local::Cast(info[2])); - } else { - langfield = ALL_LANGUAGES; - } - - T* c = node::ObjectWrap::Unwrap(info.This()); - intarray vector = __getmatching(c, id, match_prefixes, langfield); - if (!vector.empty()) { - std::size_t size = vector.size(); - Local array = Nan::New(static_cast(size)); - for (uint32_t i = 0; i < size; ++i) { - auto obj = coverToObject(numToCover(vector[i])); - - // these values don't make any sense outside the context of coalesce, so delete them - // it's a little clunky to set and then delete them, but this function as exposed - // to node is only used in debugging/testing, so, meh - obj->Delete(Nan::New("idx").ToLocalChecked()); - obj->Delete(Nan::New("tmpid").ToLocalChecked()); - obj->Delete(Nan::New("distance").ToLocalChecked()); - obj->Delete(Nan::New("scoredist").ToLocalChecked()); - array->Set(i, obj); - } - info.GetReturnValue().Set(array); - return; - } else { - info.GetReturnValue().Set(Nan::Undefined()); - return; - } - } catch (std::exception const& ex) { - return Nan::ThrowTypeError(ex.what()); - } -} - -NAN_METHOD(MemoryCache::_getmatching) { - return _genericgetmatching(info); -} - -NAN_METHOD(RocksDBCache::_getmatching) { - return _genericgetmatching(info); -} - -void NormalizationCache::Initialize(Handle target) { - Nan::HandleScope scope; - Local t = Nan::New(NormalizationCache::New); - t->InstanceTemplate()->SetInternalFieldCount(1); - t->SetClassName(Nan::New("NormalizationCache").ToLocalChecked()); - Nan::SetPrototypeMethod(t, "get", get); - Nan::SetPrototypeMethod(t, "getPrefixRange", getprefixrange); - Nan::SetPrototypeMethod(t, "getAll", getall); - Nan::SetPrototypeMethod(t, "writeBatch", writebatch); - - target->Set(Nan::New("NormalizationCache").ToLocalChecked(), t->GetFunction()); - constructor.Reset(t); -} - -NormalizationCache::NormalizationCache() - : ObjectWrap(), - db() {} - -NormalizationCache::~NormalizationCache() {} - -class UInt32Comparator : public rocksdb::Comparator { - public: - UInt32Comparator(const UInt32Comparator&) = delete; - UInt32Comparator& operator=(const UInt32Comparator&) = delete; - UInt32Comparator() = default; - - int Compare(const rocksdb::Slice& a, const rocksdb::Slice& b) const override { - uint32_t ia = 0, ib = 0; - if (a.size() >= sizeof(uint32_t)) memcpy(&ia, a.data(), sizeof(uint32_t)); - if (b.size() >= sizeof(uint32_t)) memcpy(&ib, b.data(), sizeof(uint32_t)); - - if (ia < ib) return -1; - if (ia > ib) return +1; - return 0; - } - - const char* Name() const override { return "UInt32Comparator"; } - void FindShortestSeparator(std::string* start, const rocksdb::Slice& limit) const override {} - void FindShortSuccessor(std::string* key) const override {} -}; -UInt32Comparator UInt32ComparatorInstance; - -NAN_METHOD(NormalizationCache::New) { - if (!info.IsConstructCall()) { - return Nan::ThrowTypeError("Cannot call constructor as function, you need to use 'new' keyword"); - } - try { - if (info.Length() < 2) { - return Nan::ThrowTypeError("expected arguments 'filename' and 'read-only'"); - } - if (!info[0]->IsString()) { - return Nan::ThrowTypeError("first argument 'filename' must be a String"); - } - if (!info[1]->IsBoolean()) { - return Nan::ThrowTypeError("second argument 'read-only' must be a Boolean"); - } - - Nan::Utf8String utf8_filename(info[0]); - if (utf8_filename.length() < 1) { - return Nan::ThrowTypeError("first arg must be a String"); - } - std::string filename(*utf8_filename); - bool read_only = info[1]->BooleanValue(); - - std::unique_ptr db; - rocksdb::Options options; - options.create_if_missing = true; - options.comparator = &UInt32ComparatorInstance; - - rocksdb::Status status; - if (read_only) { - status = OpenForReadOnlyDB(options, filename, db); - } else { - status = OpenDB(options, filename, db); - } - - if (!status.ok()) { - return Nan::ThrowTypeError("unable to open rocksdb file for normalization cache"); - } - NormalizationCache* im = new NormalizationCache(); - im->db = std::move(db); - im->Wrap(info.This()); - info.This()->Set(Nan::New("id").ToLocalChecked(), info[0]); - info.GetReturnValue().Set(info.This()); - return; - } catch (std::exception const& ex) { - return Nan::ThrowTypeError(ex.what()); - } - info.GetReturnValue().Set(Nan::Undefined()); - return; -} - -NAN_METHOD(NormalizationCache::get) { - if (info.Length() < 1) { - return Nan::ThrowTypeError("expected one info: id"); - } - if (!info[0]->IsNumber()) { - return Nan::ThrowTypeError("first arg must be a Number"); - } - - uint32_t id = static_cast(info[0]->IntegerValue()); - std::string sid(reinterpret_cast(&id), sizeof(uint32_t)); - - NormalizationCache* c = node::ObjectWrap::Unwrap(info.This()); - std::shared_ptr db = c->db; - - std::string message; - bool found; - rocksdb::Status s = db->Get(rocksdb::ReadOptions(), sid, &message); - found = s.ok(); - - size_t message_length = message.size(); - if (found && message_length >= sizeof(uint32_t)) { - Local out = Nan::New(); - uint32_t entry; - for (uint32_t i = 0; i * sizeof(uint32_t) < message_length; i++) { - memcpy(&entry, message.data() + (i * sizeof(uint32_t)), sizeof(uint32_t)); - out->Set(i, Nan::New(entry)); - } - info.GetReturnValue().Set(out); - return; - } else { - info.GetReturnValue().Set(Nan::Undefined()); - return; - } -} - -NAN_METHOD(NormalizationCache::getprefixrange) { - if (info.Length() < 1) { - return Nan::ThrowTypeError("expected at least two info: start_id, count, [scan_max], [return_max]"); - } - if (!info[0]->IsNumber()) { - return Nan::ThrowTypeError("first arg must be a Number"); - } - if (!info[1]->IsNumber()) { - return Nan::ThrowTypeError("second arg must be a Number"); - } - - uint32_t scan_max = 100; - uint32_t return_max = 10; - if (info.Length() > 2) { - if (!info[2]->IsNumber()) { - return Nan::ThrowTypeError("third arg, if supplied, must be a Number"); - } else { - scan_max = static_cast(info[2]->IntegerValue()); - } - } - if (info.Length() > 3) { - if (!info[3]->IsNumber()) { - return Nan::ThrowTypeError("third arg, if supplied, must be a Number"); - } else { - return_max = static_cast(info[3]->IntegerValue()); - } - } - - uint32_t start_id = static_cast(info[0]->IntegerValue()); - std::string sid(reinterpret_cast(&start_id), sizeof(uint32_t)); - uint32_t count = static_cast(info[1]->IntegerValue()); - uint32_t ceiling = start_id + count; - - uint32_t scan_count = 0, return_count = 0; - - Local out = Nan::New(); - unsigned out_idx = 0; - - NormalizationCache* c = node::ObjectWrap::Unwrap(info.This()); - std::shared_ptr db = c->db; - - std::unique_ptr rit(db->NewIterator(rocksdb::ReadOptions())); - for (rit->Seek(sid); rit->Valid(); rit->Next()) { - std::string skey = rit->key().ToString(); - uint32_t key; - memcpy(&key, skey.data(), sizeof(uint32_t)); - - if (key >= ceiling) break; - - uint32_t val; - std::string svalue = rit->value().ToString(); - for (uint32_t offset = 0; offset < svalue.length(); offset += sizeof(uint32_t)) { - memcpy(&val, svalue.data() + offset, sizeof(uint32_t)); - if (val < start_id || val >= ceiling) { - out->Set(out_idx++, Nan::New(val)); - - return_count++; - if (return_count >= return_max) break; - } - } - - scan_count++; - if (scan_count >= scan_max) break; - } - - info.GetReturnValue().Set(out); - return; -} - -NAN_METHOD(NormalizationCache::getall) { - Local out = Nan::New(); - unsigned out_idx = 0; - - NormalizationCache* c = node::ObjectWrap::Unwrap(info.This()); - std::shared_ptr db = c->db; - - std::unique_ptr rit(db->NewIterator(rocksdb::ReadOptions())); - for (rit->SeekToFirst(); rit->Valid(); rit->Next()) { - std::string skey = rit->key().ToString(); - uint32_t key = *reinterpret_cast(skey.data()); - - std::string svalue = rit->value().ToString(); - - Local row = Nan::New(); - row->Set(0, Nan::New(key)); - - Local vals = Nan::New(); - uint32_t entry; - for (uint32_t i = 0; i * sizeof(uint32_t) < svalue.length(); i++) { - memcpy(&entry, svalue.data() + (i * sizeof(uint32_t)), sizeof(uint32_t)); - vals->Set(i, Nan::New(entry)); - } - - row->Set(1, vals); - - out->Set(out_idx++, row); - } - - info.GetReturnValue().Set(out); - return; -} - -NAN_METHOD(NormalizationCache::writebatch) { - if (info.Length() < 1) { - return Nan::ThrowTypeError("expected one info: data"); - } - if (!info[0]->IsArray()) { - return Nan::ThrowTypeError("second arg must be an Array"); - } - Local data = Local::Cast(info[0]); - if (data->IsNull() || data->IsUndefined()) { - return Nan::ThrowTypeError("an array expected for second argument"); - } - - NormalizationCache* c = node::ObjectWrap::Unwrap(info.This()); - std::shared_ptr db = c->db; - - rocksdb::WriteBatch batch; - for (uint32_t i = 0; i < data->Length(); i++) { - if (!data->Get(i)->IsArray()) return Nan::ThrowTypeError("second argument must be an array of arrays"); - Local row = Local::Cast(data->Get(i)); - - if (row->Length() != 2) return Nan::ThrowTypeError("each element must have two values"); - - uint32_t key = static_cast(row->Get(0)->IntegerValue()); - std::string skey(reinterpret_cast(&key), sizeof(uint32_t)); - - std::string svalue(""); - - Local nvalue = row->Get(1); - uint32_t ivalue; - if (nvalue->IsNumber()) { - ivalue = static_cast(nvalue->IntegerValue()); - svalue.append(reinterpret_cast(&ivalue), sizeof(uint32_t)); - } else if (nvalue->IsArray()) { - Local nvalue_arr = Local::Cast(nvalue); - if (!nvalue_arr->IsNull() && !nvalue_arr->IsUndefined()) { - for (uint32_t j = 0; j < nvalue_arr->Length(); j++) { - ivalue = static_cast(nvalue_arr->Get(j)->IntegerValue()); - svalue.append(reinterpret_cast(&ivalue), sizeof(uint32_t)); - } - } else { - return Nan::ThrowTypeError("values should be either numbers or arrays of numbers"); - } - } else { - return Nan::ThrowTypeError("values should be either numbers or arrays of numbers"); - } - - batch.Put(skey, svalue); - } - db->Write(rocksdb::WriteOptions(), &batch); - - info.GetReturnValue().Set(Nan::Undefined()); - return; -} - extern "C" { static void start(Handle target) { MemoryCache::Initialize(target); @@ -2323,4 +15,8 @@ static void start(Handle target) { } // namespace carmen +// this macro expansion includes an old-style cast and is beyond our control +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wold-style-cast" NODE_MODULE(carmen, carmen::start) +#pragma clang diagnostic pop diff --git a/src/binding.hpp b/src/binding.hpp index b591d05..114e918 100644 --- a/src/binding.hpp +++ b/src/binding.hpp @@ -1,120 +1,22 @@ #ifndef __CARMEN_BINDING_HPP__ #define __CARMEN_BINDING_HPP__ +#include "coalesce.hpp" +#include "memorycache.hpp" +#include "node_util.hpp" +#include "normalizationcache.hpp" +#include "rocksdbcache.hpp" + +// this is an external library, so squash this warning #pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunknown-pragmas" -#pragma clang diagnostic ignored "-Wconversion" -#pragma clang diagnostic ignored "-Wshadow" -#pragma clang diagnostic ignored "-Wsign-compare" -#pragma clang diagnostic ignored "-Wunused-local-typedef" -#pragma clang diagnostic ignored "-Wunused-parameter" -#pragma clang diagnostic ignored "-Wpadded" -#pragma clang diagnostic ignored "-Wold-style-cast" #pragma clang diagnostic ignored "-Wsign-conversion" -#pragma clang diagnostic ignored "-Wshorten-64-to-32" #include "radix_max_heap.h" -#include "rocksdb/comparator.h" -#include "rocksdb/db.h" -#include "rocksdb/write_batch.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include #pragma clang diagnostic pop -#include -#include -#include -#include -#include namespace carmen { -class noncopyable { - protected: - constexpr noncopyable() = default; - ~noncopyable() = default; - noncopyable(noncopyable const&) = delete; - noncopyable& operator=(noncopyable const&) = delete; -}; - -typedef std::string key_type; -typedef uint64_t value_type; -typedef unsigned __int128 langfield_type; -// fully cached item -typedef std::vector intarray; -typedef std::vector keyarray; -typedef std::map arraycache; - -class MemoryCache : public node::ObjectWrap { - public: - ~MemoryCache(); - static Nan::Persistent constructor; - static void Initialize(v8::Handle target); - static NAN_METHOD(New); - static NAN_METHOD(pack); - static NAN_METHOD(list); - static NAN_METHOD(_get); - static NAN_METHOD(_getmatching); - static NAN_METHOD(_set); - static NAN_METHOD(coalesce); - explicit MemoryCache(); - void _ref() { Ref(); } - void _unref() { Unref(); } - arraycache cache_; -}; - -class RocksDBCache : public node::ObjectWrap { - public: - ~RocksDBCache(); - static Nan::Persistent constructor; - static void Initialize(v8::Handle target); - static NAN_METHOD(New); - static NAN_METHOD(pack); - static NAN_METHOD(merge); - static NAN_METHOD(list); - static NAN_METHOD(_get); - static NAN_METHOD(_getmatching); - static NAN_METHOD(_set); - static NAN_METHOD(coalesce); - explicit RocksDBCache(); - void _ref() { Ref(); } - void _unref() { Unref(); } - std::shared_ptr db; -}; - -class NormalizationCache : public node::ObjectWrap { - public: - ~NormalizationCache(); - static Nan::Persistent constructor; - static void Initialize(v8::Handle target); - static NAN_METHOD(New); - static NAN_METHOD(get); - static NAN_METHOD(getprefixrange); - static NAN_METHOD(getall); - static NAN_METHOD(writebatch); - explicit NormalizationCache(); - void _ref() { Ref(); } - void _unref() { Unref(); } - std::shared_ptr db; -}; - -#define CACHE_MESSAGE 1 -#define CACHE_ITEM 1 - -#define MEMO_PREFIX_LENGTH_T1 3 -#define MEMO_PREFIX_LENGTH_T2 6 -#define PREFIX_MAX_GRID_LENGTH 500000 - -constexpr langfield_type ALL_LANGUAGES = ~(langfield_type)(0); -#define LANGFIELD_SEPARATOR '|' +using namespace v8; -#define TYPE_MEMORY 1 -#define TYPE_ROCKSDB 2 } // namespace carmen #endif // __CARMEN_BINDING_HPP__ diff --git a/src/coalesce.cpp b/src/coalesce.cpp new file mode 100644 index 0000000..83c8cee --- /dev/null +++ b/src/coalesce.cpp @@ -0,0 +1,723 @@ + +#include "coalesce.hpp" +#include "binding.hpp" + +namespace carmen { + +using namespace v8; + +/** + * The PhrasematchSubqObject type describes the metadata known about possible matches to be assessed for stacking by + * coalesce as seen from Javascript. Note: it is of similar purpose to the PhrasematchSubq C++ struct type, but differs + * slightly in specific field names and types. + * + * @typedef PhrasematchSubqObject + * @name PhrasematchSubqObject + * @type {Object} + * @property {String} phrase - The matched string + * @property {Number} weight - A float between 0 and 1 representing how much of the query this string covers + * @property {Boolean} prefix - whether or not to do a prefix scan (as opposed to an exact match scan); used for autocomplete + * @property {Number} idx - an identifier of the index the match came from; opaque to carmen-cache but returned in results + * @property {Number} zoom - the configured tile zoom level for the index + * @property {Number} mask - a bitmask representing which tokens in the original query the subquery covers + * @property {Number[]} languages - a list of the language IDs to be considered matching + * @property {Object} cache - the carmen-cache from the index in which the match was found + */ + +/** + * @callback coalesceCallback + * @param err - error if any, or null if not + * @param {CoalesceResult[]} results - the results of the coalesce operation + */ + +/** + * A member of the result set from a coalesce operation. + * + * @typedef CoalesceResult + * @name CoalesceResult + * @type {Object} + * @property {Number} x - the X tile coordinate of the result + * @property {Number} y - the Y tile coordinate of the result + * @property {Number} relev - the computed relevance of the result + * @property {Number} score - the computed score of the result + * @property {Number} id - the feature ID of the result + * @property {Number} idx - the index ID (preserved from the inbound subquery) of the index the result came from + * @property {Number} tmpid - a composite ID used for uniquely identifying features across indexes that incorporates the ID and IDX + * @property {Number} distance - the distance metric computed using the feature and proximity, if supplied; 0 otherwise + * @property {Number} scoredist - the composite score incorporating the feature's score with the distance (or the score if distance is 0) + * @property {Boolean} matches_language - whether or not the match is valid for one of the languages in the inbound languages array + */ + +/** + * The coalesce function determines whether or not phrase matches in different + * carmen indexes align spatially, and computes information about successful matches + * such as combined relevance and score. The computation is done on the thread pool, + * and exposed asynchronously to JS via a callback argument. + * + * @name coalesce + * @param {PhrasematchSubqObject[]} phrasematches - an array of PhrasematchSubqObject objects, each of which describes a match candidate + * @param {Object} options - options for how to perform the coalesce operation that aren't specific to a particular subquery + * @param {Number} [options.radius] - the fall-off radius for determining how wide-reaching the effect of proximity bias is + * @param {Number[]} [options.centerzxy] - a 3-number array representing the ZXY of the tile on which the proximity point can be found + * @param {Number[]} [options.bboxzxy] - a 5-number array representing the zoom, minX, minY, maxX, and maxY values of the tile cover of the requested bbox, if any + * @param {coalesceCallback} callback - the callback function + */ +NAN_METHOD(coalesce) { + // PhrasematchStack (js => cpp) + if (info.Length() < 3) { + return Nan::ThrowTypeError("Expects 3 arguments: an array of PhrasematchSubqObjects, an option object, and a callback"); + } + + if (!info[0]->IsArray()) { + return Nan::ThrowTypeError("Arg 1 must be a PhrasematchSubqObject array"); + } + + Local array = Local::Cast(info[0]); + auto array_length = array->Length(); + if (array_length < 1) { + return Nan::ThrowTypeError("Arg 1 must be an array with one or more PhrasematchSubqObjects"); + } + + // Options object (js => cpp) + Local options_val = info[1]; + if (!options_val->IsObject()) { + return Nan::ThrowTypeError("Arg 2 must be an options object"); + } + Local options = options_val->ToObject(); + + // callback + Local callback = info[2]; + if (!callback->IsFunction()) { + return Nan::ThrowTypeError("Arg 3 must be a callback function"); + } + + // We use unique_ptr here to manage the heap allocated CoalesceBaton + // If an error is thrown the unique_ptr will go out of scope and delete + // its underlying baton. + // If no error is throw we release the underlying baton pointer before + // heading into the threadpool since we assume it will be deleted manually in coalesceAfter + std::unique_ptr baton_ptr = std::make_unique(); + CoalesceBaton* baton = baton_ptr.get(); + try { + for (uint32_t i = 0; i < array_length; i++) { + Local val = array->Get(i); + if (!val->IsObject()) { + return Nan::ThrowTypeError("All items in array must be valid PhrasematchSubqObjects"); + } + Local jsStack = val->ToObject(); + if (jsStack->IsNull() || jsStack->IsUndefined()) { + return Nan::ThrowTypeError("All items in array must be valid PhrasematchSubqObjects"); + } + + double weight; + std::string phrase; + bool prefix; + unsigned short idx; + unsigned short zoom; + uint32_t mask; + langfield_type langfield; + + // TODO: this is verbose: we could write some generic functions to do this robust conversion per type + if (!jsStack->Has(Nan::New("idx").ToLocalChecked())) { + return Nan::ThrowTypeError("missing idx property"); + } else { + Local prop_val = jsStack->Get(Nan::New("idx").ToLocalChecked()); + if (!prop_val->IsNumber()) { + return Nan::ThrowTypeError("idx value must be a number"); + } + int64_t _idx = prop_val->IntegerValue(); + if (_idx < 0 || _idx > std::numeric_limits::max()) { + return Nan::ThrowTypeError("encountered idx value too large to fit in unsigned short"); + } + idx = static_cast(_idx); + } + + if (!jsStack->Has(Nan::New("zoom").ToLocalChecked())) { + return Nan::ThrowTypeError("missing zoom property"); + } else { + Local prop_val = jsStack->Get(Nan::New("zoom").ToLocalChecked()); + if (!prop_val->IsNumber()) { + return Nan::ThrowTypeError("zoom value must be a number"); + } + int64_t _zoom = prop_val->IntegerValue(); + if (_zoom < 0 || _zoom > std::numeric_limits::max()) { + return Nan::ThrowTypeError("encountered zoom value too large to fit in unsigned short"); + } + zoom = static_cast(_zoom); + } + + if (!jsStack->Has(Nan::New("weight").ToLocalChecked())) { + return Nan::ThrowTypeError("missing weight property"); + } else { + Local prop_val = jsStack->Get(Nan::New("weight").ToLocalChecked()); + if (!prop_val->IsNumber()) { + return Nan::ThrowTypeError("weight value must be a number"); + } + double _weight = prop_val->NumberValue(); + if (_weight < 0 || _weight > std::numeric_limits::max()) { + return Nan::ThrowTypeError("encountered weight value too large to fit in double"); + } + weight = _weight; + } + + if (!jsStack->Has(Nan::New("phrase").ToLocalChecked())) { + return Nan::ThrowTypeError("missing phrase property"); + } else { + Local prop_val = jsStack->Get(Nan::New("phrase").ToLocalChecked()); + if (!prop_val->IsString()) { + return Nan::ThrowTypeError("phrase value must be a string"); + } + Nan::Utf8String _phrase(prop_val); + if (_phrase.length() < 1) { + return Nan::ThrowTypeError("encountered invalid phrase"); + } + phrase = *_phrase; + } + + if (!jsStack->Has(Nan::New("prefix").ToLocalChecked())) { + return Nan::ThrowTypeError("missing prefix property"); + } else { + Local prop_val = jsStack->Get(Nan::New("prefix").ToLocalChecked()); + if (!prop_val->IsBoolean()) { + return Nan::ThrowTypeError("prefix value must be a boolean"); + } + prefix = prop_val->BooleanValue(); + } + + if (!jsStack->Has(Nan::New("mask").ToLocalChecked())) { + return Nan::ThrowTypeError("missing mask property"); + } else { + Local prop_val = jsStack->Get(Nan::New("mask").ToLocalChecked()); + if (!prop_val->IsNumber()) { + return Nan::ThrowTypeError("mask value must be a number"); + } + int64_t _mask = prop_val->IntegerValue(); + if (_mask < 0 || _mask > std::numeric_limits::max()) { + return Nan::ThrowTypeError("encountered mask value too large to fit in uint32_t"); + } + mask = static_cast(_mask); + } + + langfield = ALL_LANGUAGES; + if (jsStack->Has(Nan::New("languages").ToLocalChecked())) { + Local c_array = jsStack->Get(Nan::New("languages").ToLocalChecked()); + if (!c_array->IsArray()) { + return Nan::ThrowTypeError("languages must be an array"); + } + Local carray = Local::Cast(c_array); + langfield = langarrayToLangfield(carray); + } + + if (!jsStack->Has(Nan::New("cache").ToLocalChecked())) { + return Nan::ThrowTypeError("missing cache property"); + } else { + Local prop_val = jsStack->Get(Nan::New("cache").ToLocalChecked()); + if (!prop_val->IsObject()) { + return Nan::ThrowTypeError("cache value must be a Cache object"); + } + Local _cache = prop_val->ToObject(); + if (_cache->IsNull() || _cache->IsUndefined()) { + return Nan::ThrowTypeError("cache value must be a Cache object"); + } + bool isMemoryCache = Nan::New(MemoryCache::constructor)->HasInstance(prop_val); + bool isRocksDBCache = Nan::New(RocksDBCache::constructor)->HasInstance(prop_val); + if (!(isMemoryCache || isRocksDBCache)) { + return Nan::ThrowTypeError("cache value must be a MemoryCache or RocksDBCache object"); + } + if (isMemoryCache) { + baton->stack.emplace_back( + static_cast(node::ObjectWrap::Unwrap(_cache)), + TYPE_MEMORY, + weight, + phrase, + prefix, + idx, + zoom, + mask, + langfield); + } else { + baton->stack.emplace_back( + static_cast(node::ObjectWrap::Unwrap(_cache)), + TYPE_ROCKSDB, + weight, + phrase, + prefix, + idx, + zoom, + mask, + langfield); + } + } + } + + if (options->Has(Nan::New("radius").ToLocalChecked())) { + Local prop_val = options->Get(Nan::New("radius").ToLocalChecked()); + if (!prop_val->IsNumber()) { + return Nan::ThrowTypeError("radius must be a number"); + } + int64_t _radius = prop_val->IntegerValue(); + if (_radius < 0 || _radius > std::numeric_limits::max()) { + return Nan::ThrowTypeError("encountered radius too large to fit in unsigned"); + } + baton->radius = static_cast(_radius); + } else { + baton->radius = 40.0; + } + + if (options->Has(Nan::New("centerzxy").ToLocalChecked())) { + Local c_array = options->Get(Nan::New("centerzxy").ToLocalChecked()); + if (!c_array->IsArray()) { + return Nan::ThrowTypeError("centerzxy must be an array"); + } + Local carray = Local::Cast(c_array); + if (carray->Length() != 3) { + return Nan::ThrowTypeError("centerzxy must be an array of 3 numbers"); + } + baton->centerzxy.reserve(carray->Length()); + for (uint32_t i = 0; i < carray->Length(); ++i) { + Local item = carray->Get(i); + if (!item->IsNumber()) { + return Nan::ThrowTypeError("centerzxy values must be number"); + } + int64_t a_val = item->IntegerValue(); + if (a_val < 0 || a_val > std::numeric_limits::max()) { + return Nan::ThrowTypeError("encountered centerzxy value too large to fit in uint32_t"); + } + baton->centerzxy.emplace_back(static_cast(a_val)); + } + } + + if (options->Has(Nan::New("bboxzxy").ToLocalChecked())) { + Local c_array = options->Get(Nan::New("bboxzxy").ToLocalChecked()); + if (!c_array->IsArray()) { + return Nan::ThrowTypeError("bboxzxy must be an array"); + } + Local carray = Local::Cast(c_array); + if (carray->Length() != 5) { + return Nan::ThrowTypeError("bboxzxy must be an array of 5 numbers"); + } + baton->bboxzxy.reserve(carray->Length()); + for (uint32_t i = 0; i < carray->Length(); ++i) { + Local item = carray->Get(i); + if (!item->IsNumber()) { + return Nan::ThrowTypeError("bboxzxy values must be number"); + } + int64_t a_val = item->IntegerValue(); + if (a_val < 0 || a_val > std::numeric_limits::max()) { + return Nan::ThrowTypeError("encountered bboxzxy value too large to fit in uint32_t"); + } + baton->bboxzxy.emplace_back(static_cast(a_val)); + } + } + + baton->callback.Reset(callback.As()); + + // queue work + baton->request.data = baton; + // Release the managed baton + baton_ptr.release(); + // Reference count the cache objects + for (auto& subq : baton->stack) { + if (subq.type == TYPE_MEMORY) + reinterpret_cast(subq.cache)->_ref(); + else + reinterpret_cast(subq.cache)->_ref(); + } + // optimization: for stacks of 1, use coalesceSingle + if (baton->stack.size() == 1) { + uv_queue_work(uv_default_loop(), &baton->request, coalesceSingle, static_cast(coalesceAfter)); + } else { + uv_queue_work(uv_default_loop(), &baton->request, coalesceMulti, static_cast(coalesceAfter)); + } + } catch (std::exception const& ex) { + return Nan::ThrowTypeError(ex.what()); + } + + info.GetReturnValue().Set(Nan::Undefined()); + return; +} + +// behind the scenes, coalesce has two different strategies, depending on whether +// it's actually trying to stack multiple matches or whether it's considering a +// single match that consumes the entire query; this function handles the latter case +// and takes as a parameter the libuv task that contains info about the job it's supposed to do +void coalesceSingle(uv_work_t* req) { + CoalesceBaton* baton = static_cast(req->data); + + try { + std::vector const& stack = baton->stack; + PhrasematchSubq const& subq = stack[0]; + + // proximity (optional) + bool proximity = !baton->centerzxy.empty(); + unsigned cz; + unsigned cx; + unsigned cy; + if (proximity) { + cz = static_cast(baton->centerzxy[0]); + cx = static_cast(baton->centerzxy[1]); + cy = static_cast(baton->centerzxy[2]); + } else { + cz = 0; + cx = 0; + cy = 0; + } + + // bbox (optional) + bool bbox = !baton->bboxzxy.empty(); + unsigned minx; + unsigned miny; + unsigned maxx; + unsigned maxy; + if (bbox) { + minx = static_cast(baton->bboxzxy[1]); + miny = static_cast(baton->bboxzxy[2]); + maxx = static_cast(baton->bboxzxy[3]); + maxy = static_cast(baton->bboxzxy[4]); + } else { + minx = 0; + miny = 0; + maxx = 0; + maxy = 0; + } + + // Load and concatenate grids for all ids in `phrases` + intarray grids; + grids = subq.type == TYPE_MEMORY ? __getmatching(reinterpret_cast(subq.cache), subq.phrase, subq.prefix, subq.langfield) : __getmatching(reinterpret_cast(subq.cache), subq.phrase, subq.prefix, subq.langfield); + + unsigned long m = grids.size(); + double relevMax = 0; + std::vector covers; + covers.reserve(m); + + uint32_t length = 0; + uint32_t lastId = 0; + double lastRelev = 0; + double lastScoredist = 0; + double lastDistance = 0; + double minScoredist = std::numeric_limits::max(); + for (unsigned long j = 0; j < m; j++) { + Cover cover = numToCover(grids[j]); + + cover.idx = subq.idx; + cover.tmpid = static_cast(cover.idx * POW2_25 + cover.id); + cover.relev = cover.relev * subq.weight; + if (!cover.matches_language) cover.relev *= .9; + cover.distance = proximity ? tileDist(cx, cy, cover.x, cover.y) : 0; + cover.scoredist = proximity ? scoredist(cz, cover.distance, cover.score, baton->radius) : cover.score; + + // only add cover id if it's got a higer scoredist + if (lastId == cover.id && cover.scoredist <= lastScoredist) continue; + + // short circuit based on relevMax thres + if (length > 40) { + if (cover.scoredist < minScoredist) continue; + if (cover.relev < lastRelev) break; + } + if (relevMax - cover.relev >= 0.25) break; + if (cover.relev > relevMax) relevMax = cover.relev; + + if (bbox) { + if (cover.x < minx || cover.y < miny || cover.x > maxx || cover.y > maxy) continue; + } + + covers.emplace_back(cover); + if (lastId != cover.id) length++; + if (!proximity && length > 40) break; + if (cover.scoredist < minScoredist) minScoredist = cover.scoredist; + lastId = cover.id; + lastRelev = cover.relev; + lastScoredist = cover.scoredist; + lastDistance = cover.distance; + } + + // sort grids by distance to proximity point + std::sort(covers.begin(), covers.end(), coverSortByRelev); + + uint32_t lastid = 0; + std::size_t added = 0; + std::vector contexts; + std::size_t max_contexts = 40; + contexts.reserve(max_contexts); + for (auto&& cover : covers) { + // Stop at 40 contexts + if (added == max_contexts) break; + + // Attempt not to add the same feature but by diff cover twice + if (lastid == cover.id) continue; + + lastid = cover.id; + added++; + + double relev = cover.relev; + uint32_t mask = 0; + contexts.emplace_back(std::move(cover), mask, relev); + } + + coalesceFinalize(baton, std::move(contexts)); + } catch (std::exception const& ex) { + baton->error = ex.what(); + } +} + +// this function handles the case where stacking is occurring between multiple subqueries +// again, it takes a libuv task as a parameter +void coalesceMulti(uv_work_t* req) { + CoalesceBaton* baton = static_cast(req->data); + + try { + std::vector& stack = baton->stack; + std::sort(stack.begin(), stack.end(), subqSortByZoom); + std::size_t stackSize = stack.size(); + + // Cache zoom levels to iterate over as coalesce occurs. + std::vector zoomCache; + zoomCache.reserve(stackSize); + for (auto const& subq : stack) { + intarray zooms; + std::vector zoomUniq(22, false); + for (auto const& subqB : stack) { + if (subq.idx == subqB.idx) continue; + if (zoomUniq[subqB.zoom]) continue; + if (subq.zoom < subqB.zoom) continue; + zoomUniq[subqB.zoom] = true; + zooms.emplace_back(subqB.zoom); + } + zoomCache.push_back(std::move(zooms)); + } + + // Coalesce relevs into higher zooms, e.g. + // z5 inherits relev of overlapping tiles at z4. + // @TODO assumes sources are in zoom ascending order. + std::map> coalesced; + std::map>::iterator cit; + std::map>::iterator pit; + std::map done; + std::map::iterator dit; + + // proximity (optional) + bool proximity = baton->centerzxy.size() > 0; + unsigned cz; + unsigned cx; + unsigned cy; + if (proximity) { + cz = static_cast(baton->centerzxy[0]); + cx = static_cast(baton->centerzxy[1]); + cy = static_cast(baton->centerzxy[2]); + } else { + cz = 0; + cx = 0; + cy = 0; + } + + // bbox (optional) + bool bbox = !baton->bboxzxy.empty(); + unsigned bboxz; + unsigned minx; + unsigned miny; + unsigned maxx; + unsigned maxy; + if (bbox) { + bboxz = static_cast(baton->bboxzxy[0]); + minx = static_cast(baton->bboxzxy[1]); + miny = static_cast(baton->bboxzxy[2]); + maxx = static_cast(baton->bboxzxy[3]); + maxy = static_cast(baton->bboxzxy[4]); + } else { + bboxz = 0; + minx = 0; + miny = 0; + maxx = 0; + maxy = 0; + } + + std::vector contexts; + std::size_t i = 0; + for (auto const& subq : stack) { + // Load and concatenate grids for all ids in `phrases` + intarray grids; + grids = subq.type == TYPE_MEMORY ? __getmatching(reinterpret_cast(subq.cache), subq.phrase, subq.prefix, subq.langfield) : __getmatching(reinterpret_cast(subq.cache), subq.phrase, subq.prefix, subq.langfield); + + bool first = i == 0; + bool last = i == (stack.size() - 1); + unsigned short z = subq.zoom; + auto const& zCache = zoomCache[i]; + std::size_t zCacheSize = zCache.size(); + + unsigned long m = grids.size(); + + for (unsigned long j = 0; j < m; j++) { + Cover cover = numToCover(grids[j]); + cover.idx = subq.idx; + cover.mask = subq.mask; + cover.tmpid = static_cast(cover.idx * POW2_25 + cover.id); + cover.relev = cover.relev * subq.weight; + if (!cover.matches_language) cover.relev *= .9; + if (proximity) { + ZXY dxy = pxy2zxy(z, cover.x, cover.y, cz); + cover.distance = tileDist(cx, cy, dxy.x, dxy.y); + cover.scoredist = scoredist(cz, cover.distance, cover.score, baton->radius); + } else { + cover.distance = 0; + cover.scoredist = cover.score; + } + + if (bbox) { + ZXY min = bxy2zxy(bboxz, minx, miny, z, false); + ZXY max = bxy2zxy(bboxz, maxx, maxy, z, true); + if (cover.x < min.x || cover.y < min.y || cover.x > max.x || cover.y > max.y) continue; + } + + uint64_t zxy = (z * POW2_28) + (cover.x * POW2_14) + (cover.y); + + // Reserve stackSize for the coverList. The vector + // will grow no larger that the size of the input + // subqueries that are being coalesced. + std::vector covers; + covers.reserve(stackSize); + covers.push_back(cover); + uint32_t context_mask = cover.mask; + double context_relev = cover.relev; + + for (unsigned a = 0; a < zCacheSize; a++) { + uint64_t p = zCache[a]; + double s = static_cast(1 << (z - p)); + uint64_t pxy = static_cast(p * POW2_28) + + static_cast(std::floor(cover.x / s) * POW2_14) + + static_cast(std::floor(cover.y / s)); + pit = coalesced.find(pxy); + if (pit != coalesced.end()) { + uint32_t lastMask = 0; + double lastRelev = 0.0; + for (auto const& parents : pit->second) { + for (auto const& parent : parents.coverList) { + // this cover is functionally identical with previous and + // is more relevant, replace the previous. + if (parent.mask == lastMask && parent.relev > lastRelev) { + covers.pop_back(); + covers.emplace_back(parent); + context_relev -= lastRelev; + context_relev += parent.relev; + lastMask = parent.mask; + lastRelev = parent.relev; + // this cover doesn't overlap with used mask. + } else if (!(context_mask & parent.mask)) { + covers.emplace_back(parent); + context_relev += parent.relev; + context_mask = context_mask | parent.mask; + lastMask = parent.mask; + lastRelev = parent.relev; + } + } + } + } + } + + if (last) { + // Slightly penalize contexts that have no stacking + if (covers.size() == 1) { + context_relev -= 0.01; + // Slightly penalize contexts in ascending order + } else if (covers[0].mask > covers[1].mask) { + context_relev -= 0.01; + } + contexts.emplace_back(std::move(covers), context_mask, context_relev); + } else if (first || covers.size() > 1) { + cit = coalesced.find(zxy); + if (cit == coalesced.end()) { + std::vector local_contexts; + local_contexts.emplace_back(std::move(covers), context_mask, context_relev); + coalesced.emplace(zxy, std::move(local_contexts)); + } else { + cit->second.emplace_back(std::move(covers), context_mask, context_relev); + } + } + } + + i++; + } + + // append coalesced to contexts by moving memory + for (auto&& matched : coalesced) { + for (auto&& context : matched.second) { + contexts.emplace_back(std::move(context)); + } + } + + std::sort(contexts.begin(), contexts.end(), contextSortByRelev); + coalesceFinalize(baton, std::move(contexts)); + } catch (std::exception const& ex) { + baton->error = ex.what(); + } +} + +// we don't use the 'status' parameter, but it's required as part of the uv_after_work_cb +// function signature, so suppress the warning about it +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" +// this function handles getting the results of either coalesceSingle or coalesceMulti +// ready to be passed back to JS land, and queues up a callback invokation on the main thread +void coalesceAfter(uv_work_t* req, int status) { + Nan::HandleScope scope; + CoalesceBaton* baton = static_cast(req->data); + + // Reference count the cache objects + for (auto& subq : baton->stack) { + if (subq.type == TYPE_MEMORY) + reinterpret_cast(subq.cache)->_unref(); + else + reinterpret_cast(subq.cache)->_unref(); + } + + if (!baton->error.empty()) { + v8::Local argv[1] = {Nan::Error(baton->error.c_str())}; + Nan::MakeCallback(Nan::GetCurrentContext()->Global(), Nan::New(baton->callback), 1, argv); + } else { + std::vector const& features = baton->features; + + Local jsFeatures = Nan::New(static_cast(features.size())); + for (uint32_t i = 0; i < features.size(); i++) { + jsFeatures->Set(i, contextToArray(features[i])); + } + + Local argv[2] = {Nan::Null(), jsFeatures}; + Nan::MakeCallback(Nan::GetCurrentContext()->Global(), Nan::New(baton->callback), 2, argv); + } + + baton->callback.Reset(); + delete baton; +} +#pragma clang diagnostic pop + +// this function tabulates the results of either coalesceSingle or coalesceMulti in +// a uniform way +void coalesceFinalize(CoalesceBaton* baton, std::vector&& contexts) { + if (!contexts.empty()) { + // Coalesce stack, generate relevs. + double relevMax = contexts[0].relev; + std::size_t total = 0; + std::map sets; + std::map::iterator sit; + std::size_t max_contexts = 40; + baton->features.reserve(max_contexts); + for (auto&& context : contexts) { + // Maximum allowance of coalesced features: 40. + if (total >= max_contexts) break; + + // Since `coalesced` is sorted by relev desc at first + // threshold miss we can break the loop. + if (relevMax - context.relev >= 0.25) break; + + // Only collect each feature once. + uint32_t id = context.coverList[0].tmpid; + sit = sets.find(id); + if (sit != sets.end()) continue; + + sets.emplace(id, true); + baton->features.emplace_back(std::move(context)); + total++; + } + } +} + +} // namespace carmen \ No newline at end of file diff --git a/src/coalesce.hpp b/src/coalesce.hpp new file mode 100644 index 0000000..a3a0cbf --- /dev/null +++ b/src/coalesce.hpp @@ -0,0 +1,47 @@ +#ifndef __CARMEN_COALESCE_HPP__ +#define __CARMEN_COALESCE_HPP__ + +#include "cpp_util.hpp" +#include "node_util.hpp" + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunknown-pragmas" +#pragma clang diagnostic ignored "-Wconversion" +#pragma clang diagnostic ignored "-Wshadow" +#pragma clang diagnostic ignored "-Wsign-compare" +#pragma clang diagnostic ignored "-Wunused-local-typedef" +#pragma clang diagnostic ignored "-Wunused-parameter" +#pragma clang diagnostic ignored "-Wpadded" +#pragma clang diagnostic ignored "-Wold-style-cast" +#pragma clang diagnostic ignored "-Wsign-conversion" +#pragma clang diagnostic ignored "-Wshorten-64-to-32" + +#include + +#pragma clang diagnostic pop + +namespace carmen { + +struct CoalesceBaton : carmen::noncopyable { + uv_work_t request; + // params + std::vector stack; + std::vector centerzxy; + std::vector bboxzxy; + double radius; + Nan::Persistent callback; + // return + std::vector features; + // error + std::string error; +}; + +NAN_METHOD(coalesce); +void coalesceSingle(uv_work_t* req); +void coalesceMulti(uv_work_t* req); +void coalesceAfter(uv_work_t* req, int status); +void coalesceFinalize(CoalesceBaton* baton, std::vector&& contexts); + +} // namespace carmen + +#endif // __CARMEN_COALESCE_HPP__ diff --git a/src/cpp_util.cpp b/src/cpp_util.cpp new file mode 100644 index 0000000..891d078 --- /dev/null +++ b/src/cpp_util.cpp @@ -0,0 +1,141 @@ + +#include "cpp_util.hpp" + +namespace carmen { + +// Converts from the packed integer into (relev, score, x, y, feature_id) +Cover numToCover(uint64_t num) { + Cover cover; + assert(((num >> 34) % POW2_14) <= static_cast(std::numeric_limits::max())); + assert(((num >> 34) % POW2_14) >= static_cast(std::numeric_limits::min())); + unsigned short y = static_cast((num >> 34) % POW2_14); + assert(((num >> 20) % POW2_14) <= static_cast(std::numeric_limits::max())); + assert(((num >> 20) % POW2_14) >= static_cast(std::numeric_limits::min())); + unsigned short x = static_cast((num >> 20) % POW2_14); + assert(((num >> 48) % POW2_3) <= static_cast(std::numeric_limits::max())); + assert(((num >> 48) % POW2_3) >= static_cast(std::numeric_limits::min())); + unsigned short score = static_cast((num >> 48) % POW2_3); + uint32_t id = static_cast(num % POW2_20); + bool matches_language = static_cast(num & LANGUAGE_MATCH_BOOST); + cover.x = x; + cover.y = y; + double relev = 0.4 + (0.2 * static_cast((num >> 51) % POW2_2)); + cover.relev = relev; + cover.score = score; + cover.id = id; + cover.matches_language = matches_language; + + // These are not derived from decoding the input num but by + // external values after initialization. + cover.idx = 0; + cover.mask = 0; + cover.tmpid = 0; + cover.distance = 0; + + return cover; +} + +// Converts the ZXY coordinates of the tile that contains a proximity point to a +// ZXY at the appropriate zoom level necessary for a given coalesce operation +ZXY pxy2zxy(unsigned z, unsigned x, unsigned y, unsigned target_z) { + ZXY zxy; + zxy.z = target_z; + + // Interval between parent and target zoom level + signed zDist = static_cast(target_z) - static_cast(z); + signed zMult = zDist - 1; + if (zDist == 0) { + zxy.x = x; + zxy.y = y; + return zxy; + } + + // Midpoint length @ z for a tile at parent zoom level + double pMid_d = static_cast(std::pow(2, zDist) / 2); + assert(pMid_d <= static_cast(std::numeric_limits::max())); + assert(pMid_d >= static_cast(std::numeric_limits::min())); + signed pMid = static_cast(pMid_d); + zxy.x = static_cast((static_cast(x) * zMult) + pMid); + zxy.y = static_cast((static_cast(y) * zMult) + pMid); + return zxy; +} + +// Calculates a ZXY for appropriate for a given coalesce operation out of the supplied bbox ZXY coordinates. +ZXY bxy2zxy(unsigned z, unsigned x, unsigned y, unsigned target_z, bool max) { + ZXY zxy; + zxy.z = target_z; + + // Interval between parent and target zoom level + signed zDist = static_cast(target_z) - static_cast(z); + if (zDist == 0) { + zxy.x = x; + zxy.y = y; + return zxy; + } + + // zoom conversion multiplier + float mult = static_cast(std::pow(2, zDist)); + + // zoom in min + if (zDist > 0 && !max) { + zxy.x = static_cast(static_cast(x) * mult); + zxy.y = static_cast(static_cast(y) * mult); + return zxy; + } + // zoom in max + else if (zDist > 0 && max) { + zxy.x = static_cast(static_cast(x) * mult + (mult - 1)); + zxy.y = static_cast(static_cast(y) * mult + (mult - 1)); + return zxy; + } + // zoom out + else { + unsigned mod = static_cast(std::pow(2, target_z)); + unsigned xDiff = x % mod; + unsigned yDiff = y % mod; + unsigned newX = x - xDiff; + unsigned newY = y - yDiff; + + zxy.x = static_cast(static_cast(newX) * mult); + zxy.y = static_cast(static_cast(newY) * mult); + return zxy; + } +} + +// Equivalent of scoredist() function in carmen +// Combines score and distance into a single score that can be used for sorting. +// Unlike carmen the effect is not scaled by zoom level as regardless of index +// the score value at this stage is a 0-7 scalar (by comparison, in carmen, scores +// for indexes with features covering lower zooms often have exponentially higher +// scores - example: country@z9 vs poi@z14). +double scoredist(unsigned zoom, double distance, double score, double radius) { + if (zoom < 6) zoom = 6; + if (distance == 0.0) distance = 0.01; + double scoredist = 0; + + // Since distance is in tiles we calculate scoredist by converting the miles into + // a tile unit value at the appropriate zoom first. + // + // 32 tiles is about 40 miles at z14, use this as our mile <=> tile conversion. + scoredist = ((radius * (32.0 / 40.0)) / _pow(1.5, 14 - static_cast(zoom))) / distance; + + return score > scoredist ? score : scoredist; +} + +// Open database for read-write availability +rocksdb::Status OpenDB(const rocksdb::Options& options, const std::string& name, std::unique_ptr& dbptr) { + rocksdb::DB* db; + rocksdb::Status status = rocksdb::DB::Open(options, name, &db); + dbptr = std::unique_ptr(db); + return status; +} + +// Open database for read-only availability +rocksdb::Status OpenForReadOnlyDB(const rocksdb::Options& options, const std::string& name, std::unique_ptr& dbptr) { + rocksdb::DB* db; + rocksdb::Status status = rocksdb::DB::OpenForReadOnly(options, name, &db); + dbptr = std::unique_ptr(db); + return status; +} + +} // namespace carmen diff --git a/src/cpp_util.hpp b/src/cpp_util.hpp new file mode 100644 index 0000000..c7f55b6 --- /dev/null +++ b/src/cpp_util.hpp @@ -0,0 +1,295 @@ +#ifndef __CARMEN_CPP_UTIL_HPP__ +#define __CARMEN_CPP_UTIL_HPP__ + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunknown-pragmas" +#pragma clang diagnostic ignored "-Wconversion" +#pragma clang diagnostic ignored "-Wshadow" +#pragma clang diagnostic ignored "-Wsign-compare" +#pragma clang diagnostic ignored "-Wunused-local-typedef" +#pragma clang diagnostic ignored "-Wunused-parameter" +#pragma clang diagnostic ignored "-Wpadded" +#pragma clang diagnostic ignored "-Wold-style-cast" +#pragma clang diagnostic ignored "-Wsign-conversion" +#pragma clang diagnostic ignored "-Wshorten-64-to-32" + +#include "rocksdb/db.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#pragma clang diagnostic pop + +namespace carmen { + +typedef std::string key_type; +typedef uint64_t value_type; +typedef unsigned __int128 langfield_type; +// fully cached item +typedef std::vector intarray; +typedef std::vector keyarray; +typedef std::map arraycache; + +class noncopyable { + protected: + constexpr noncopyable() = default; + ~noncopyable() = default; + noncopyable(noncopyable const&) = delete; + noncopyable& operator=(noncopyable const&) = delete; +}; + +typedef unsigned __int128 langfield_type; +constexpr uint64_t LANGUAGE_MATCH_BOOST = static_cast(1) << 63; + +//relev = 5 bits +//count = 3 bits +//reason = 12 bits +//* 1 bit gap +//id = 32 bits +constexpr double _pow(double x, int y) { + return y == 0 ? 1.0 : x * _pow(x, y - 1); +} + +constexpr uint64_t POW2_51 = static_cast(_pow(2.0, 51)); +constexpr uint64_t POW2_48 = static_cast(_pow(2.0, 48)); +constexpr uint64_t POW2_34 = static_cast(_pow(2.0, 34)); +constexpr uint64_t POW2_28 = static_cast(_pow(2.0, 28)); +constexpr uint64_t POW2_25 = static_cast(_pow(2.0, 25)); +constexpr uint64_t POW2_20 = static_cast(_pow(2.0, 20)); +constexpr uint64_t POW2_14 = static_cast(_pow(2.0, 14)); +constexpr uint64_t POW2_3 = static_cast(_pow(2.0, 3)); +constexpr uint64_t POW2_2 = static_cast(_pow(2.0, 2)); + +struct PhrasematchSubq { + PhrasematchSubq(void* c, + char t, + double w, + std::string p, + bool pf, + unsigned short i, + unsigned short z, + uint32_t m, + langfield_type l) : cache(c), + type(t), + weight(w), + phrase(p), + prefix(pf), + idx(i), + zoom(z), + mask(m), + langfield(l) {} + void* cache; + char type; + double weight; + std::string phrase; + bool prefix; + unsigned short idx; + unsigned short zoom; + uint32_t mask; + langfield_type langfield; + PhrasematchSubq& operator=(PhrasematchSubq&& c) = default; + PhrasematchSubq(PhrasematchSubq&& c) = default; +}; + +struct Cover { + double relev; + uint32_t id; + uint32_t tmpid; + unsigned short x; + unsigned short y; + unsigned short score; + unsigned short idx; + uint32_t mask; + double distance; + double scoredist; + bool matches_language; +}; + +struct Context { + std::vector coverList; + uint32_t mask; + double relev; + + Context() = delete; + Context(Context const& c) = delete; + Context& operator=(Context const& c) = delete; + Context(Cover&& cov, + uint32_t mask, + double relev) + : coverList(), + mask(mask), + relev(relev) { + coverList.emplace_back(std::move(cov)); + } + Context& operator=(Context&& c) { + coverList = std::move(c.coverList); + mask = std::move(c.mask); + relev = std::move(c.relev); + return *this; + } + Context(std::vector&& cl, + uint32_t mask, + double relev) + : coverList(std::move(cl)), + mask(mask), + relev(relev) {} + + Context(Context&& c) + : coverList(std::move(c.coverList)), + mask(std::move(c.mask)), + relev(std::move(c.relev)) {} +}; + +struct ZXY { + unsigned z; + unsigned x; + unsigned y; +}; + +Cover numToCover(uint64_t num); +ZXY pxy2zxy(unsigned z, unsigned x, unsigned y, unsigned target_z); +ZXY bxy2zxy(unsigned z, unsigned x, unsigned y, unsigned target_z, bool max = false); +double scoredist(unsigned zoom, double distance, double score, double radius); + +inline bool coverSortByRelev(Cover const& a, Cover const& b) noexcept { + if (b.relev > a.relev) + return false; + else if (b.relev < a.relev) + return true; + else if (b.scoredist > a.scoredist) + return false; + else if (b.scoredist < a.scoredist) + return true; + else if (b.idx < a.idx) + return false; + else if (b.idx > a.idx) + return true; + else if (b.id < a.id) + return false; + else if (b.id > a.id) + return true; + // sorting by x and y is arbitrary but provides a more deterministic output order + else if (b.x < a.x) + return false; + else if (b.x > a.x) + return true; + else + return (b.y > a.y); +} + +inline bool subqSortByZoom(PhrasematchSubq const& a, PhrasematchSubq const& b) noexcept { + if (a.zoom < b.zoom) return true; + if (a.zoom > b.zoom) return false; + return (a.idx < b.idx); +} + +inline bool contextSortByRelev(Context const& a, Context const& b) noexcept { + if (b.relev > a.relev) + return false; + else if (b.relev < a.relev) + return true; + else if (b.coverList[0].scoredist > a.coverList[0].scoredist) + return false; + else if (b.coverList[0].scoredist < a.coverList[0].scoredist) + return true; + else if (b.coverList[0].idx < a.coverList[0].idx) + return false; + else if (b.coverList[0].idx > a.coverList[0].idx) + return true; + return (b.coverList[0].id > a.coverList[0].id); +} + +inline double tileDist(unsigned px, unsigned py, unsigned tileX, unsigned tileY) { + const double dx = static_cast(px) - static_cast(tileX); + const double dy = static_cast(py) - static_cast(tileY); + const double distance = sqrt((dx * dx) + (dy * dy)); + + return distance; +} + +constexpr langfield_type ALL_LANGUAGES = ~static_cast(0); +#define LANGFIELD_SEPARATOR '|' + +inline void add_langfield(std::string& s, langfield_type langfield) { + if (langfield != ALL_LANGUAGES) { + char* lf_as_char = reinterpret_cast(&langfield); + + // we only want to copy over as many bytes as we're using + // so find the last byte that's not zero, and copy until there + // NOTE: this assumes little-endianness, where the bytes + // in use will be first rather than last + size_t highest(0); + for (size_t i = 0; i < sizeof(langfield_type); i++) { + if (lf_as_char[i] != 0) highest = i; + } + size_t field_length = highest + 1; + + s.reserve(sizeof(LANGFIELD_SEPARATOR) + field_length); + s.push_back(LANGFIELD_SEPARATOR); + s.append(lf_as_char, field_length); + } else { + s.push_back(LANGFIELD_SEPARATOR); + } +} + +// in general, the key format including language field is <8-byte langfield> +// but as a size optimization for language-less indexes, we omit the langfield +// if it would otherwise have been ALL_LANGUAGES (all 1's), so if the first occurence +// of the separator character is also the last character in the string, we just retur ALL_LANGUAGES +// otherwise we extract it from the string +// +// we centralize both the adding of the field and extracting of the field here to keep from having +// to handle that optimization everywhere +inline langfield_type extract_langfield(std::string const& s) { + size_t length = s.length(); + size_t langfield_start = s.find(LANGFIELD_SEPARATOR) + 1; + size_t distance_from_end = length - langfield_start; + + if (distance_from_end == 0) { + return ALL_LANGUAGES; + } else { + langfield_type result(0); + memcpy(&result, s.data() + langfield_start, distance_from_end); + return result; + } +} + +inline void packVec(intarray const& varr, std::unique_ptr const& db, std::string const& key) { + std::string message; + + protozero::pbf_writer item_writer(message); + + { + // Using new (in protozero 1.3.0) packed writing API + // https://github.com/mapbox/protozero/commit/4e7e32ac5350ea6d3dcf78ff5e74faeee513a6e1 + protozero::packed_field_uint64 field{item_writer, 1}; + uint64_t lastval = 0; + for (auto const& vitem : varr) { + if (lastval == 0) { + field.add_element(static_cast(vitem)); + } else { + field.add_element(static_cast(lastval - vitem)); + } + lastval = vitem; + } + } + + db->Put(rocksdb::WriteOptions(), key, message); +} + +// rocksdb is also used in memorycache, and normalizationcache +rocksdb::Status OpenDB(const rocksdb::Options& options, const std::string& name, std::unique_ptr& dbptr); +rocksdb::Status OpenForReadOnlyDB(const rocksdb::Options& options, const std::string& name, std::unique_ptr& dbptr); + +#define TYPE_MEMORY 1 +#define TYPE_ROCKSDB 2 + +} // namespace carmen + +#endif // __CARMEN_CPP_UTIL_HPP__ diff --git a/src/memorycache.cpp b/src/memorycache.cpp new file mode 100644 index 0000000..537e9bb --- /dev/null +++ b/src/memorycache.cpp @@ -0,0 +1,418 @@ + +#include "memorycache.hpp" +#include "cpp_util.hpp" + +namespace carmen { + +using namespace v8; + +/** +* @class MemoryCache +*/ + +Nan::Persistent MemoryCache::constructor; + +intarray __get(MemoryCache const* c, std::string phrase, langfield_type langfield) { + arraycache const& cache = c->cache_; + intarray array; + + add_langfield(phrase, langfield); + arraycache::const_iterator aitr = cache.find(phrase); + if (aitr != cache.end()) { + array = aitr->second; + } + std::sort(array.begin(), array.end(), std::greater()); + return array; +} + +intarray __getmatching(MemoryCache const* c, std::string phrase, bool match_prefixes, langfield_type langfield) { + intarray array; + + if (!match_prefixes) phrase.push_back(LANGFIELD_SEPARATOR); + size_t phrase_length = phrase.length(); + const char* phrase_data = phrase.data(); + + // Load values from memory cache + + for (auto const& item : c->cache_) { + const char* item_data = item.first.data(); + size_t item_length = item.first.length(); + + if (item_length < phrase_length) continue; + + if (memcmp(phrase_data, item_data, phrase_length) == 0) { + langfield_type message_langfield = extract_langfield(item.first); + + if (message_langfield & langfield) { + array.reserve(array.size() + item.second.size()); + for (auto const& grid : item.second) { + array.emplace_back(grid | LANGUAGE_MATCH_BOOST); + } + } else { + array.insert(array.end(), item.second.begin(), item.second.end()); + } + } + } + std::sort(array.begin(), array.end(), std::greater()); + return array; +} + +void MemoryCache::Initialize(Handle target) { + Nan::HandleScope scope; + Local t = Nan::New(MemoryCache::New); + t->InstanceTemplate()->SetInternalFieldCount(1); + t->SetClassName(Nan::New("MemoryCache").ToLocalChecked()); + Nan::SetPrototypeMethod(t, "pack", MemoryCache::pack); + Nan::SetPrototypeMethod(t, "list", MemoryCache::list); + Nan::SetPrototypeMethod(t, "_set", _set); + Nan::SetPrototypeMethod(t, "_get", _get); + Nan::SetPrototypeMethod(t, "_getMatching", _getmatching); + target->Set(Nan::New("MemoryCache").ToLocalChecked(), t->GetFunction()); + constructor.Reset(t); +} + +MemoryCache::MemoryCache() + : ObjectWrap(), + cache_() {} + +MemoryCache::~MemoryCache() {} + +/** + * creates database from filename + * optimize a memory cache and write to disc as rocksdbcache + * + * @name pack + * @memberof MemoryCache + * @param {String}, filename + * @returns undefined + * @example + * const cache = require('@mapbox/carmen-cache'); + * const MemoryCache = new cache.MemoryCache('a'); + * + * cache.pack('filename'); + * + */ + +NAN_METHOD(MemoryCache::pack) { + if (info.Length() < 1) { + return Nan::ThrowTypeError("expected one info: 'filename'"); + } + if (!info[0]->IsString()) { + return Nan::ThrowTypeError("first argument must be a String"); + } + try { + Nan::Utf8String utf8_filename(info[0]); + if (utf8_filename.length() < 1) { + return Nan::ThrowTypeError("first arg must be a String"); + } + std::string filename(*utf8_filename); + + MemoryCache* c = node::ObjectWrap::Unwrap(info.This()); + + std::unique_ptr db; + rocksdb::Options options; + options.create_if_missing = true; + rocksdb::Status status = OpenDB(options, filename, db); + + if (!status.ok()) { + return Nan::ThrowTypeError("unable to open rocksdb file for packing"); + } + + std::map> memoized_prefixes; + + for (auto const& item : c->cache_) { + std::size_t array_size = item.second.size(); + if (array_size > 0) { + // make copy of intarray so we can sort without + // modifying the original array + intarray varr = item.second; + + // delta-encode values, sorted in descending order. + std::sort(varr.begin(), varr.end(), std::greater()); + + packVec(varr, db, item.first); + + std::string prefix_t1 = ""; + std::string prefix_t2 = ""; + + // add this to the memoized prefix array too, maybe + auto phrase_length = item.first.find(LANGFIELD_SEPARATOR); + // use the full string for things shorter than the limit + // or the prefix otherwise + if (phrase_length < MEMO_PREFIX_LENGTH_T1) { + prefix_t1 = "=1" + item.first; + } else { + // get the prefix, then append the langfield back onto it again + langfield_type langfield = extract_langfield(item.first); + + prefix_t1 = "=1" + item.first.substr(0, MEMO_PREFIX_LENGTH_T1); + add_langfield(prefix_t1, langfield); + + if (phrase_length < MEMO_PREFIX_LENGTH_T2) { + prefix_t2 = "=2" + item.first; + } else { + prefix_t2 = "=2" + item.first.substr(0, MEMO_PREFIX_LENGTH_T2); + add_langfield(prefix_t2, langfield); + } + } + + if (prefix_t1 != "") { + std::map>::const_iterator mitr = memoized_prefixes.find(prefix_t1); + if (mitr == memoized_prefixes.end()) { + memoized_prefixes.emplace(prefix_t1, std::deque()); + } + std::deque& buf = memoized_prefixes[prefix_t1]; + + buf.insert(buf.end(), varr.begin(), varr.end()); + } + if (prefix_t2 != "") { + std::map>::const_iterator mitr = memoized_prefixes.find(prefix_t2); + if (mitr == memoized_prefixes.end()) { + memoized_prefixes.emplace(prefix_t2, std::deque()); + } + std::deque& buf = memoized_prefixes[prefix_t2]; + + buf.insert(buf.end(), varr.begin(), varr.end()); + } + } + } + + for (auto const& item : memoized_prefixes) { + // copy the deque into a vector so we can sort without + // modifying the original array + intarray varr(item.second.begin(), item.second.end()); + + // delta-encode values, sorted in descending order. + std::sort(varr.begin(), varr.end(), std::greater()); + + if (varr.size() > PREFIX_MAX_GRID_LENGTH) { + // for the prefix memos we're only going to ever use 500k max anyway + varr.resize(PREFIX_MAX_GRID_LENGTH); + } + + packVec(varr, db, item.first); + } + } catch (std::exception const& ex) { + return Nan::ThrowTypeError(ex.what()); + } + info.GetReturnValue().Set(Nan::Undefined()); + return; +} + +/** + * Lists the keys in the store of the MemoryCache Object + * + * @name list + * @memberof MemoryCache + * @param {String} id + * @returns {Array} + * @example + * const cache = require('@mapbox/carmen-cache'); + * const MemoryCache = new cache.MemoryCache('a'); + * + * cache.list('a', (err, result) => { + * if (err) throw err; + * console.log(result); + * }); + * + */ + +NAN_METHOD(MemoryCache::list) { + try { + Nan::Utf8String utf8_value(info[0]); + if (utf8_value.length() < 1) { + return Nan::ThrowTypeError("first arg must be a String"); + } + MemoryCache* c = node::ObjectWrap::Unwrap(info.This()); + Local ids = Nan::New(); + + unsigned idx = 0; + for (auto const& item : c->cache_) { + Local out = Nan::New(); + out->Set(0, Nan::New(item.first.substr(0, item.first.find(LANGFIELD_SEPARATOR))).ToLocalChecked()); + + langfield_type langfield = extract_langfield(item.first); + if (langfield == ALL_LANGUAGES) { + out->Set(1, Nan::Null()); + } else { + out->Set(1, langfieldToLangarray(langfield)); + } + + ids->Set(idx++, out); + } + + info.GetReturnValue().Set(ids); + return; + } catch (std::exception const& ex) { + return Nan::ThrowTypeError(ex.what()); + } +} + +/** + * Replaces or appends the data for a given key + * + * @name set + * @memberof MemoryCache + * @param {String} id + * @param {Array}, data; an array of numbers where each number represents a grid + * @param {Array} an array of relevant languages + * @param {Boolean} T: append to data, F: replace data + * @returns undefined + * @example + * const cache = require('@mapbox/carmen-cache'); + * const MemoryCache = new cache.MemoryCache('a'); + * + * cache.set('a', [1,2,3], (err, result) => { + * if (err) throw err; + * console.log(result) + *}); + * + */ + +NAN_METHOD(MemoryCache::_set) { + if (info.Length() < 2) { + return Nan::ThrowTypeError("expected at least two info: id, data, [languages], [append]"); + } + if (!info[0]->IsString()) { + return Nan::ThrowTypeError("first arg must be a String"); + } + if (!info[1]->IsArray()) { + return Nan::ThrowTypeError("second arg must be an Array"); + } + Local data = Local::Cast(info[1]); + if (data->IsNull() || data->IsUndefined()) { + return Nan::ThrowTypeError("an array expected for second argument"); + } + try { + + Nan::Utf8String utf8_id(info[0]); + if (utf8_id.length() < 1) { + return Nan::ThrowTypeError("first arg must be a String"); + } + std::string id(*utf8_id); + + langfield_type langfield; + if (info.Length() > 2 && !(info[2]->IsNull() || info[2]->IsUndefined())) { + if (!info[2]->IsArray()) { + return Nan::ThrowTypeError("third arg, if supplied must be an Array"); + } + langfield = langarrayToLangfield(Local::Cast(info[2])); + } else { + langfield = ALL_LANGUAGES; + } + + bool append = info.Length() > 3 && info[3]->IsBoolean() && info[3]->BooleanValue(); + + MemoryCache* c = node::ObjectWrap::Unwrap(info.This()); + arraycache& arrc = c->cache_; + key_type key_id = static_cast(id); + add_langfield(key_id, langfield); + + arraycache::iterator itr2 = arrc.find(key_id); + if (itr2 == arrc.end()) { + arrc.emplace(key_id, intarray()); + } + intarray& vv = arrc[key_id]; + + unsigned array_size = data->Length(); + if (append) { + vv.reserve(vv.size() + array_size); + } else { + if (itr2 != arrc.end()) vv.clear(); + vv.reserve(array_size); + } + + for (unsigned i = 0; i < array_size; ++i) { + vv.emplace_back(static_cast(data->Get(i)->NumberValue())); + } + } catch (std::exception const& ex) { + return Nan::ThrowTypeError(ex.what()); + } + info.GetReturnValue().Set(Nan::Undefined()); + return; +} + +/** + * Retrieves data exactly matching phrase and language settings by id + * + * @name get + * @memberof MemoryCache + * @param {String} id + * @param {Boolean} matches_prefixes: T if it matches exactly, F: if it does not + * @param {Array} optional; array of languages + * @returns {Array} integers referring to grids + * @example + * const cache = require('@mapbox/carmen-cache'); + * const MemoryCache = new cache.MemoryCache('a'); + * + * MemoryCache.get(id, languages); + * // => [grid, grid, grid, grid... ] + * + */ + +NAN_METHOD(MemoryCache::_get) { + return _genericget(info); +} + +/** + * Creates an in-memory key-value store mapping phrases and language IDs + * to lists of corresponding grids (grids ie are integer representations of occurrences of the phrase within an index) + * + * @name MemoryCache + * @memberof MemoryCache + * @param {String} id + * @returns {Array} grid of integers + * @example + * const cache = require('@mapbox/carmen-cache'); + * const MemoryCache = new cache.MemoryCache(id, languages); + * + */ + +NAN_METHOD(MemoryCache::New) { + if (!info.IsConstructCall()) { + return Nan::ThrowTypeError("Cannot call constructor as function, you need to use 'new' keyword"); + } + try { + if (info.Length() < 1) { + return Nan::ThrowTypeError("expected 'id' argument"); + } + if (!info[0]->IsString()) { + return Nan::ThrowTypeError("first argument 'id' must be a String"); + } + MemoryCache* im = new MemoryCache(); + + im->Wrap(info.This()); + info.This()->Set(Nan::New("id").ToLocalChecked(), info[0]); + info.GetReturnValue().Set(info.This()); + return; + } catch (std::exception const& ex) { + return Nan::ThrowTypeError(ex.what()); + } +} + +/** + * Retrieves data matching phrase and/or language settings by id. + * If match_prefixes is true, anything that starts with that phrase, + * and also for any entry regardless of language, + * and with a relevance penalty applied to languages that don't match those requested. + * + * @name getmatching + * @memberof MemoryCache + * @param {String} id + * @param {Boolean} matches_prefixes: T if it matches exactly, F: if it does not + * @param {Array} optional; array of languages + * @returns {Array} integers referring to grids + * @example + * const cache = require('@mapbox/carmen-cache'); + * const MemoryCache = new cache.MemoryCache('a'); + * + * MemoryCache.getmatching(id, languages); + * // => [grid, grid, grid, grid... ] + * + */ + +NAN_METHOD(MemoryCache::_getmatching) { + return _genericgetmatching(info); +} + +} // namespace carmen diff --git a/src/memorycache.hpp b/src/memorycache.hpp new file mode 100644 index 0000000..29d5c29 --- /dev/null +++ b/src/memorycache.hpp @@ -0,0 +1,48 @@ +#ifndef __CARMEN_MEMORYCACHE_HPP__ +#define __CARMEN_MEMORYCACHE_HPP__ + +#include "cpp_util.hpp" +#include "node_util.hpp" + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunknown-pragmas" +#pragma clang diagnostic ignored "-Wconversion" +#pragma clang diagnostic ignored "-Wshadow" +#pragma clang diagnostic ignored "-Wsign-compare" +#pragma clang diagnostic ignored "-Wunused-local-typedef" +#pragma clang diagnostic ignored "-Wunused-parameter" +#pragma clang diagnostic ignored "-Wpadded" +#pragma clang diagnostic ignored "-Wold-style-cast" +#pragma clang diagnostic ignored "-Wsign-conversion" +#pragma clang diagnostic ignored "-Wshorten-64-to-32" + +#include + +#pragma clang diagnostic pop + +namespace carmen { + +class MemoryCache : public node::ObjectWrap { + public: + ~MemoryCache(); + static Nan::Persistent constructor; + static void Initialize(v8::Handle target); + static NAN_METHOD(New); + static NAN_METHOD(pack); + static NAN_METHOD(list); + static NAN_METHOD(_get); + static NAN_METHOD(_getmatching); + static NAN_METHOD(_set); + static NAN_METHOD(coalesce); + explicit MemoryCache(); + void _ref() { Ref(); } + void _unref() { Unref(); } + arraycache cache_; +}; + +intarray __get(MemoryCache const* c, std::string phrase, langfield_type langfield); +intarray __getmatching(MemoryCache const* c, std::string phrase, bool match_prefixes, langfield_type langfield); + +} // namespace carmen + +#endif // __CARMEN_MEMORYCACHE_HPP__ diff --git a/src/node_util.cpp b/src/node_util.cpp new file mode 100644 index 0000000..6150d41 --- /dev/null +++ b/src/node_util.cpp @@ -0,0 +1,37 @@ + +#include "node_util.hpp" + +namespace carmen { + +using namespace v8; + +// convert from a C++ coalesce result object to an equivalent JS one; +// see the CoalesceResult JSDoc typedef in coalesce.cpp for an explanation +// of the what each property represents +Local coverToObject(Cover const& cover) { + Local object = Nan::New(); + object->Set(Nan::New("x").ToLocalChecked(), Nan::New(cover.x)); + object->Set(Nan::New("y").ToLocalChecked(), Nan::New(cover.y)); + object->Set(Nan::New("relev").ToLocalChecked(), Nan::New(cover.relev)); + object->Set(Nan::New("score").ToLocalChecked(), Nan::New(cover.score)); + object->Set(Nan::New("id").ToLocalChecked(), Nan::New(cover.id)); + object->Set(Nan::New("idx").ToLocalChecked(), Nan::New(cover.idx)); + object->Set(Nan::New("tmpid").ToLocalChecked(), Nan::New(cover.tmpid)); + object->Set(Nan::New("distance").ToLocalChecked(), Nan::New(cover.distance)); + object->Set(Nan::New("scoredist").ToLocalChecked(), Nan::New(cover.scoredist)); + object->Set(Nan::New("matches_language").ToLocalChecked(), Nan::New(cover.matches_language)); + return object; +} +// convert an array of Cover C++ objects to an array of CoalesceResult JS objects +// (basically just a wrapper around coverToObject, above) +Local contextToArray(Context const& context) { + std::size_t size = context.coverList.size(); + Local array = Nan::New(static_cast(size)); + for (uint32_t i = 0; i < size; i++) { + array->Set(i, coverToObject(context.coverList[i])); + } + array->Set(Nan::New("relev").ToLocalChecked(), Nan::New(context.relev)); + return array; +} + +} // namespace carmen diff --git a/src/node_util.hpp b/src/node_util.hpp new file mode 100644 index 0000000..92f4c24 --- /dev/null +++ b/src/node_util.hpp @@ -0,0 +1,169 @@ +#ifndef __CARMEN_NODE_UTIL_HPP__ +#define __CARMEN_NODE_UTIL_HPP__ + +#include "binding.hpp" +#include "cpp_util.hpp" +#include "memorycache.hpp" + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunknown-pragmas" +#pragma clang diagnostic ignored "-Wconversion" +#pragma clang diagnostic ignored "-Wshadow" +#pragma clang diagnostic ignored "-Wsign-compare" +#pragma clang diagnostic ignored "-Wunused-local-typedef" +#pragma clang diagnostic ignored "-Wunused-parameter" +#pragma clang diagnostic ignored "-Wpadded" +#pragma clang diagnostic ignored "-Wold-style-cast" +#pragma clang diagnostic ignored "-Wsign-conversion" +#pragma clang diagnostic ignored "-Wshorten-64-to-32" + +#include + +#pragma clang diagnostic pop + +namespace carmen { + +using namespace v8; + +Local coverToObject(Cover const& cover); +Local contextToArray(Context const& context); + +constexpr unsigned MAX_LANG = (sizeof(langfield_type) * 8) - 1; +// convert from a JS array of language IDs to a bitmask where the bits corresponding +// to those IDs are set to 1 +inline langfield_type langarrayToLangfield(Local const& array) { + size_t array_size = array->Length(); + langfield_type out = 0; + for (unsigned i = 0; i < array_size; i++) { + unsigned int val = static_cast(array->Get(i)->NumberValue()); + if (val > MAX_LANG) { + // this should probably throw something + continue; + } + out = out | (static_cast(1) << val); + } + return out; +} + +// convert from a bitmask where the bits corresponding to a set of language IDs +// are set to 1 to a JS array of those language IDs +inline Local langfieldToLangarray(langfield_type langfield) { + Local langs = Nan::New(); + + unsigned idx = 0; + for (unsigned i = 0; i <= MAX_LANG; i++) { + if (langfield & (static_cast(1) << i)) { + langs->Set(idx++, Nan::New(i)); + } + } + return langs; +} + +// this function is not exposed to JS directly, but rather, is exposed as +// either RocksDBCache::get or MemoryCache::get -- they share input validation +// and output formatting +template +inline NAN_METHOD(_genericget) { + if (info.Length() < 1) { + return Nan::ThrowTypeError("expected at least one info: id, [languages]"); + } + if (!info[0]->IsString()) { + return Nan::ThrowTypeError("first arg must be a String"); + } + try { + Nan::Utf8String utf8_id(info[0]); + if (utf8_id.length() < 1) { + return Nan::ThrowTypeError("first arg must be a String"); + } + std::string id(*utf8_id); + + langfield_type langfield; + if (info.Length() > 1 && !(info[1]->IsNull() || info[1]->IsUndefined())) { + if (!info[1]->IsArray()) { + return Nan::ThrowTypeError("second arg, if supplied must be an Array"); + } + langfield = langarrayToLangfield(Local::Cast(info[1])); + } else { + langfield = ALL_LANGUAGES; + } + + T* c = node::ObjectWrap::Unwrap(info.This()); + intarray vector = __get(c, id, langfield); + if (!vector.empty()) { + std::size_t size = vector.size(); + Local array = Nan::New(static_cast(size)); + for (uint32_t i = 0; i < size; ++i) { + array->Set(i, Nan::New(vector[i])); + } + info.GetReturnValue().Set(array); + return; + } else { + info.GetReturnValue().Set(Nan::Undefined()); + return; + } + } catch (std::exception const& ex) { + return Nan::ThrowTypeError(ex.what()); + } +} + +template +inline NAN_METHOD(_genericgetmatching) { + if (info.Length() < 2) { + return Nan::ThrowTypeError("expected two or three info: id, match_prefixes, [languages]"); + } + if (!info[0]->IsString()) { + return Nan::ThrowTypeError("first arg must be a String"); + } + if (!info[1]->IsBoolean()) { + return Nan::ThrowTypeError("second arg must be a Bool"); + } + try { + Nan::Utf8String utf8_id(info[0]); + if (utf8_id.length() < 1) { + return Nan::ThrowTypeError("first arg must be a String"); + } + std::string id(*utf8_id); + + bool match_prefixes = info[1]->BooleanValue(); + + langfield_type langfield; + if (info.Length() > 2 && !(info[2]->IsNull() || info[2]->IsUndefined())) { + if (!info[2]->IsArray()) { + return Nan::ThrowTypeError("third arg, if supplied must be an Array"); + } + langfield = langarrayToLangfield(Local::Cast(info[2])); + } else { + langfield = ALL_LANGUAGES; + } + + T* c = node::ObjectWrap::Unwrap(info.This()); + intarray vector = __getmatching(c, id, match_prefixes, langfield); + if (!vector.empty()) { + std::size_t size = vector.size(); + Local array = Nan::New(static_cast(size)); + for (uint32_t i = 0; i < size; ++i) { + auto obj = coverToObject(numToCover(vector[i])); + + // these values don't make any sense outside the context of coalesce, so delete them + // it's a little clunky to set and then delete them, but this function as exposed + // to node is only used in debugging/testing, so, meh + obj->Delete(Nan::New("idx").ToLocalChecked()); + obj->Delete(Nan::New("tmpid").ToLocalChecked()); + obj->Delete(Nan::New("distance").ToLocalChecked()); + obj->Delete(Nan::New("scoredist").ToLocalChecked()); + array->Set(i, obj); + } + info.GetReturnValue().Set(array); + return; + } else { + info.GetReturnValue().Set(Nan::Undefined()); + return; + } + } catch (std::exception const& ex) { + return Nan::ThrowTypeError(ex.what()); + } +} + +} // namespace carmen + +#endif // __CARMEN_NODE_UTIL_HPP__ diff --git a/src/normalizationcache.cpp b/src/normalizationcache.cpp new file mode 100644 index 0000000..9c2f46b --- /dev/null +++ b/src/normalizationcache.cpp @@ -0,0 +1,396 @@ + +#include "normalizationcache.hpp" + +namespace carmen { + +using namespace v8; + +Nan::Persistent NormalizationCache::constructor; + +/** + * NormalizationCache represents an one-to-many integer-to-integer + * mapping where each integer is assumed to be the position of a given term + * in a lexicographically-sorted vocabulary. The purpose of the cache is to capture + * equivalencies between different elements in the dictionary, such that further metadata + * (e.g., in a RocksDBCache) can be stored only for the canonical form of a given name. + * The structure is stored using a RocksDB database on disk. + * @class NormalizationCache + * + */ + +void NormalizationCache::Initialize(Handle target) { + Nan::HandleScope scope; + Local t = Nan::New(NormalizationCache::New); + t->InstanceTemplate()->SetInternalFieldCount(1); + t->SetClassName(Nan::New("NormalizationCache").ToLocalChecked()); + Nan::SetPrototypeMethod(t, "get", get); + Nan::SetPrototypeMethod(t, "getPrefixRange", getprefixrange); + Nan::SetPrototypeMethod(t, "getAll", getall); + Nan::SetPrototypeMethod(t, "writeBatch", writebatch); + + target->Set(Nan::New("NormalizationCache").ToLocalChecked(), t->GetFunction()); + constructor.Reset(t); +} + +NormalizationCache::NormalizationCache() + : ObjectWrap(), + db() {} + +NormalizationCache::~NormalizationCache() {} + +class UInt32Comparator : public rocksdb::Comparator { + public: + UInt32Comparator(const UInt32Comparator&) = delete; + UInt32Comparator& operator=(const UInt32Comparator&) = delete; + UInt32Comparator() = default; + + int Compare(const rocksdb::Slice& a, const rocksdb::Slice& b) const override { + uint32_t ia = 0, ib = 0; + if (a.size() >= sizeof(uint32_t)) memcpy(&ia, a.data(), sizeof(uint32_t)); + if (b.size() >= sizeof(uint32_t)) memcpy(&ib, b.data(), sizeof(uint32_t)); + + if (ia < ib) return -1; + if (ia > ib) return +1; + return 0; + } + + const char* Name() const override { return "UInt32Comparator"; } + +// these function signatures are mandated by rocksdb, so suppress warnings about not +// using all the parameters +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" + void FindShortestSeparator(std::string* start, const rocksdb::Slice& limit) const override {} + void FindShortSuccessor(std::string* key) const override {} +#pragma clang diagnostic pop +}; +UInt32Comparator UInt32ComparatorInstance; + +/** + * Constructor for NormalizationCache pointing to an on-disk RocksDB database + * to be used for reading or writing. + * + * @name NormalizationCache + * @memberof NormalizationCache + * @param {String} filename + * @param {String} read-only + * @returns {Object} + * @example + * const cache = require('@mapbox/carmen-cache'); + * const nc = new cache.NormalizationCache('file.norm.rocksdb', false); + * + */ + +NAN_METHOD(NormalizationCache::New) { + if (!info.IsConstructCall()) { + return Nan::ThrowTypeError("Cannot call constructor as function, you need to use 'new' keyword"); + } + try { + if (info.Length() < 2) { + return Nan::ThrowTypeError("expected arguments 'filename' and 'read-only'"); + } + if (!info[0]->IsString()) { + return Nan::ThrowTypeError("first argument 'filename' must be a String"); + } + if (!info[1]->IsBoolean()) { + return Nan::ThrowTypeError("second argument 'read-only' must be a Boolean"); + } + + Nan::Utf8String utf8_filename(info[0]); + if (utf8_filename.length() < 1) { + return Nan::ThrowTypeError("first arg must be a String"); + } + std::string filename(*utf8_filename); + bool read_only = info[1]->BooleanValue(); + + std::unique_ptr db; + rocksdb::Options options; + options.create_if_missing = true; + options.comparator = &UInt32ComparatorInstance; + + rocksdb::Status status; + if (read_only) { + status = OpenForReadOnlyDB(options, filename, db); + } else { + status = OpenDB(options, filename, db); + } + + if (!status.ok()) { + return Nan::ThrowTypeError("unable to open rocksdb file for normalization cache"); + } + NormalizationCache* im = new NormalizationCache(); + im->db = std::move(db); + im->Wrap(info.This()); + info.This()->Set(Nan::New("id").ToLocalChecked(), info[0]); + info.GetReturnValue().Set(info.This()); + return; + } catch (std::exception const& ex) { + return Nan::ThrowTypeError(ex.what()); + } +} + +/** + * retrieve the indices of the canonical labels for the index of a given non-canonical label + * + * @name get + * @memberof NormalizationCache + * @param {Number} id + * @returns {Array} + * @example + * const cache = require('@mapbox/carmen-cache'); + * const nc = new cache.NormalizationCache('file.norm.rocksdb', true); + * + * // for a normalization cache for the dictionary ['main st', 'main street'] + * // where 'main st' is canonical + * const canonical = nc.get(1); // returns [0] + */ + +NAN_METHOD(NormalizationCache::get) { + if (info.Length() < 1) { + return Nan::ThrowTypeError("expected one info: id"); + } + if (!info[0]->IsNumber()) { + return Nan::ThrowTypeError("first arg must be a Number"); + } + + uint32_t id = static_cast(info[0]->IntegerValue()); + std::string sid(reinterpret_cast(&id), sizeof(uint32_t)); + + NormalizationCache* c = node::ObjectWrap::Unwrap(info.This()); + std::shared_ptr db = c->db; + + std::string message; + bool found; + rocksdb::Status s = db->Get(rocksdb::ReadOptions(), sid, &message); + found = s.ok(); + + size_t message_length = message.size(); + if (found && message_length >= sizeof(uint32_t)) { + Local out = Nan::New(); + uint32_t entry; + for (uint32_t i = 0; i * sizeof(uint32_t) < message_length; i++) { + memcpy(&entry, message.data() + (i * sizeof(uint32_t)), sizeof(uint32_t)); + out->Set(i, Nan::New(entry)); + } + info.GetReturnValue().Set(out); + return; + } else { + info.GetReturnValue().Set(Nan::Undefined()); + return; + } +} + +/** + * given that in a lexicographically sorted list, all terms that share a prefix + * are grouped together, this function retrieves the indices of all canonical forms + * of all terms that share a given prefix as indicated by the index of the first term + * in the shared prefix list and the number of terms that share a prefix, for which the canonical + * form does not also share that same prefix + * + * @name getPrefixRange + * @memberof NormalizationCache + * @param {Number} start_id + * @param {Number} count + * @param {Number} [scan_max] - the maximum number of entries to scan + * @param {Number} [return_max] - the maximum number of indices to return + * @returns {Array} + * @example + * const cache = require('@mapbox/carmen-cache'); + * const nc = new cache.NormalizationCache('file.norm.rocksdb', true); + * + * // for a normalization cache for the dictionary + * // ['saint marks ave', 'saint peters ave', 'st marks ave', 'st peters ave'] + * // where the 'st ...' forms are canonical + * const canonical = nc.getPrefixRange(0, 2); // looks up all the canonical + * // forms for things that begin with + * // 'saint' + */ + +NAN_METHOD(NormalizationCache::getprefixrange) { + if (info.Length() < 1) { + return Nan::ThrowTypeError("expected at least two info: start_id, count, [scan_max], [return_max]"); + } + if (!info[0]->IsNumber()) { + return Nan::ThrowTypeError("first arg must be a Number"); + } + if (!info[1]->IsNumber()) { + return Nan::ThrowTypeError("second arg must be a Number"); + } + + uint32_t scan_max = 100; + uint32_t return_max = 10; + if (info.Length() > 2) { + if (!info[2]->IsNumber()) { + return Nan::ThrowTypeError("third arg, if supplied, must be a Number"); + } else { + scan_max = static_cast(info[2]->IntegerValue()); + } + } + if (info.Length() > 3) { + if (!info[3]->IsNumber()) { + return Nan::ThrowTypeError("third arg, if supplied, must be a Number"); + } else { + return_max = static_cast(info[3]->IntegerValue()); + } + } + + uint32_t start_id = static_cast(info[0]->IntegerValue()); + std::string sid(reinterpret_cast(&start_id), sizeof(uint32_t)); + uint32_t count = static_cast(info[1]->IntegerValue()); + uint32_t ceiling = start_id + count; + + uint32_t scan_count = 0, return_count = 0; + + Local out = Nan::New(); + unsigned out_idx = 0; + + NormalizationCache* c = node::ObjectWrap::Unwrap(info.This()); + std::shared_ptr db = c->db; + + std::unique_ptr rit(db->NewIterator(rocksdb::ReadOptions())); + for (rit->Seek(sid); rit->Valid(); rit->Next()) { + std::string skey = rit->key().ToString(); + uint32_t key; + memcpy(&key, skey.data(), sizeof(uint32_t)); + + if (key >= ceiling) break; + + uint32_t val; + std::string svalue = rit->value().ToString(); + for (uint32_t offset = 0; offset < svalue.length(); offset += sizeof(uint32_t)) { + memcpy(&val, svalue.data() + offset, sizeof(uint32_t)); + if (val < start_id || val >= ceiling) { + out->Set(out_idx++, Nan::New(val)); + + return_count++; + if (return_count >= return_max) break; + } + } + + scan_count++; + if (scan_count >= scan_max) break; + } + + info.GetReturnValue().Set(out); + return; +} + +/** + * retrieve the entire contents of a NormalizationCache, as an array of arrays + * + * @name getAll + * @memberof NormalizationCache + * @returns {Array} + * @example + * const cache = require('@mapbox/carmen-cache'); + * const nc = new cache.NormalizationCache('file.norm.rocksdb', true); + * + * // for a normalization cache for the dictionary + * // ['saint marks ave', 'saint peters ave', 'st marks ave', 'st peters ave'] + * // where the 'st ...' forms are canonical + * const canonical = nc.getAll() // returns [[0, [2]], [1, [3]]] + */ +NAN_METHOD(NormalizationCache::getall) { + Local out = Nan::New(); + unsigned out_idx = 0; + + NormalizationCache* c = node::ObjectWrap::Unwrap(info.This()); + std::shared_ptr db = c->db; + + std::unique_ptr rit(db->NewIterator(rocksdb::ReadOptions())); + for (rit->SeekToFirst(); rit->Valid(); rit->Next()) { + std::string skey = rit->key().ToString(); + uint32_t key = *reinterpret_cast(skey.data()); + + std::string svalue = rit->value().ToString(); + + Local row = Nan::New(); + row->Set(0, Nan::New(key)); + + Local vals = Nan::New(); + uint32_t entry; + for (uint32_t i = 0; i * sizeof(uint32_t) < svalue.length(); i++) { + memcpy(&entry, svalue.data() + (i * sizeof(uint32_t)), sizeof(uint32_t)); + vals->Set(i, Nan::New(entry)); + } + + row->Set(1, vals); + + out->Set(out_idx++, row); + } + + info.GetReturnValue().Set(out); + return; +} + +/** + * bulk-set the contents of a NormalizationCache to an array of arrays + * + * @name writeBatch + * @memberof NormalizationCache + * @param {Array} data - the values to be written to the cache, in the form [[from, [to, to, ...]], ...] + * @returns {Array} + * @example + * const cache = require('@mapbox/carmen-cache'); + * const nc = new cache.NormalizationCache('file.norm.rocksdb', true); + * + * // for a normalization cache for the dictionary + * // ['saint marks ave', 'saint peters ave', 'st marks ave', 'st peters ave'] + * // where the 'st ...' forms are canonical + * nc.writeBatch([[0, [2]], [1, [3]]]); + */ +NAN_METHOD(NormalizationCache::writebatch) { + if (info.Length() < 1) { + return Nan::ThrowTypeError("expected one info: data"); + } + if (!info[0]->IsArray()) { + return Nan::ThrowTypeError("first arg must be an Array"); + } + Local data = Local::Cast(info[0]); + if (data->IsNull() || data->IsUndefined()) { + return Nan::ThrowTypeError("an array expected for first argument"); + } + + NormalizationCache* c = node::ObjectWrap::Unwrap(info.This()); + std::shared_ptr db = c->db; + + rocksdb::WriteBatch batch; + for (uint32_t i = 0; i < data->Length(); i++) { + if (!data->Get(i)->IsArray()) return Nan::ThrowTypeError("second argument must be an array of arrays"); + Local row = Local::Cast(data->Get(i)); + + if (row->Length() != 2) return Nan::ThrowTypeError("each element must have two values"); + + uint32_t key = static_cast(row->Get(0)->IntegerValue()); + std::string skey(reinterpret_cast(&key), sizeof(uint32_t)); + + std::string svalue(""); + + Local nvalue = row->Get(1); + uint32_t ivalue; + if (nvalue->IsNumber()) { + ivalue = static_cast(nvalue->IntegerValue()); + svalue.append(reinterpret_cast(&ivalue), sizeof(uint32_t)); + } else if (nvalue->IsArray()) { + Local nvalue_arr = Local::Cast(nvalue); + if (!nvalue_arr->IsNull() && !nvalue_arr->IsUndefined()) { + for (uint32_t j = 0; j < nvalue_arr->Length(); j++) { + ivalue = static_cast(nvalue_arr->Get(j)->IntegerValue()); + svalue.append(reinterpret_cast(&ivalue), sizeof(uint32_t)); + } + } else { + return Nan::ThrowTypeError("values should be either numbers or arrays of numbers"); + } + } else { + return Nan::ThrowTypeError("values should be either numbers or arrays of numbers"); + } + + batch.Put(skey, svalue); + } + db->Write(rocksdb::WriteOptions(), &batch); + + info.GetReturnValue().Set(Nan::Undefined()); + return; +} + +} // namespace carmen diff --git a/src/normalizationcache.hpp b/src/normalizationcache.hpp new file mode 100644 index 0000000..c02be88 --- /dev/null +++ b/src/normalizationcache.hpp @@ -0,0 +1,45 @@ +#ifndef __CARMEN_NORMALIZATIONCACHE_HPP__ +#define __CARMEN_NORMALIZATIONCACHE_HPP__ + +#include "cpp_util.hpp" + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunknown-pragmas" +#pragma clang diagnostic ignored "-Wconversion" +#pragma clang diagnostic ignored "-Wshadow" +#pragma clang diagnostic ignored "-Wsign-compare" +#pragma clang diagnostic ignored "-Wunused-local-typedef" +#pragma clang diagnostic ignored "-Wunused-parameter" +#pragma clang diagnostic ignored "-Wpadded" +#pragma clang diagnostic ignored "-Wold-style-cast" +#pragma clang diagnostic ignored "-Wsign-conversion" +#pragma clang diagnostic ignored "-Wshorten-64-to-32" + +#include "rocksdb/comparator.h" +#include "rocksdb/db.h" +#include "rocksdb/write_batch.h" +#include + +#pragma clang diagnostic pop + +namespace carmen { + +class NormalizationCache : public node::ObjectWrap { + public: + ~NormalizationCache(); + static Nan::Persistent constructor; + static void Initialize(v8::Handle target); + static NAN_METHOD(New); + static NAN_METHOD(get); + static NAN_METHOD(getprefixrange); + static NAN_METHOD(getall); + static NAN_METHOD(writebatch); + explicit NormalizationCache(); + void _ref() { Ref(); } + void _unref() { Unref(); } + std::shared_ptr db; +}; + +} // namespace carmen + +#endif // __CARMEN_NORMALIZATIONCACHE_HPP__ diff --git a/src/rocksdbcache.cpp b/src/rocksdbcache.cpp new file mode 100644 index 0000000..ccd81f4 --- /dev/null +++ b/src/rocksdbcache.cpp @@ -0,0 +1,617 @@ + +#include "rocksdbcache.hpp" +#include "cpp_util.hpp" + +namespace carmen { + +using namespace v8; + +/** + * + * @class RocksDBCache + * + */ + +Nan::Persistent RocksDBCache::constructor; + +intarray __get(RocksDBCache const* c, std::string phrase, langfield_type langfield) { + std::shared_ptr db = c->db; + intarray array; + + add_langfield(phrase, langfield); + std::string message; + rocksdb::Status s = db->Get(rocksdb::ReadOptions(), phrase, &message); + if (s.ok()) { + decodeMessage(message, array); + } + + return array; +} + +intarray __getmatching(RocksDBCache const* c, std::string phrase, bool match_prefixes, langfield_type langfield) { + intarray array; + + if (!match_prefixes) phrase.push_back(LANGFIELD_SEPARATOR); + size_t phrase_length = phrase.length(); + + // Load values from message cache + std::vector> messages; + std::vector grids; + + if (match_prefixes) { + // if this is an autocomplete scan, use the prefix cache + if (phrase_length <= MEMO_PREFIX_LENGTH_T1) { + phrase = "=1" + phrase.substr(0, MEMO_PREFIX_LENGTH_T1); + } else if (phrase_length <= MEMO_PREFIX_LENGTH_T2) { + phrase = "=2" + phrase.substr(0, MEMO_PREFIX_LENGTH_T2); + } + } + + radix_max_heap::pair_radix_max_heap rh; + + std::shared_ptr db = c->db; + + std::unique_ptr rit(db->NewIterator(rocksdb::ReadOptions())); + for (rit->Seek(phrase); rit->Valid() && rit->key().ToString().compare(0, phrase.size(), phrase) == 0; rit->Next()) { + std::string key = rit->key().ToString(); + + // grab the langfield from the end of the key + langfield_type message_langfield = extract_langfield(key); + bool matches_language = static_cast(message_langfield & langfield); + + messages.emplace_back(std::make_tuple(rit->value().ToString(), matches_language)); + } + + // short-circuit the priority queue merging logic if we only found one message + // as will be the norm for exact matches in translationless indexes + if (messages.size() == 1) { + if (std::get<1>(messages[0])) { + decodeAndBoostMessage(std::get<0>(messages[0]), array); + } else { + decodeMessage(std::get<0>(messages[0]), array); + } + return array; + } + + for (std::tuple& message : messages) { + protozero::pbf_reader item(std::get<0>(message)); + bool matches_language = std::get<1>(message); + + item.next(CACHE_ITEM); + auto vals = item.get_packed_uint64(); + + if (vals.first != vals.second) { + value_type unadjusted_lastval = *(vals.first); + grids.emplace_back(sortableGrid{ + vals.first, + vals.second, + unadjusted_lastval, + matches_language}); + rh.push(matches_language ? unadjusted_lastval | LANGUAGE_MATCH_BOOST : unadjusted_lastval, grids.size() - 1); + } + } + + while (!rh.empty() && array.size() < PREFIX_MAX_GRID_LENGTH) { + size_t gridIdx = rh.top_value(); + uint64_t gridId = rh.top_key(); + rh.pop(); + + array.emplace_back(gridId); + sortableGrid* sg = &(grids[gridIdx]); + sg->it++; + if (sg->it != sg->end) { + sg->unadjusted_lastval -= *(grids[gridIdx].it); + rh.push( + sg->matches_language ? sg->unadjusted_lastval | LANGUAGE_MATCH_BOOST : sg->unadjusted_lastval, + gridIdx); + } + } + + return array; +} + +void RocksDBCache::Initialize(Handle target) { + Nan::HandleScope scope; + Local t = Nan::New(RocksDBCache::New); + t->InstanceTemplate()->SetInternalFieldCount(1); + t->SetClassName(Nan::New("RocksDBCache").ToLocalChecked()); + Nan::SetPrototypeMethod(t, "pack", RocksDBCache::pack); + Nan::SetPrototypeMethod(t, "list", RocksDBCache::list); + Nan::SetPrototypeMethod(t, "_get", _get); + Nan::SetPrototypeMethod(t, "_getMatching", _getmatching); + Nan::SetMethod(t, "merge", merge); + target->Set(Nan::New("RocksDBCache").ToLocalChecked(), t->GetFunction()); + constructor.Reset(t); +} + +RocksDBCache::RocksDBCache() + : ObjectWrap(), + db() {} + +RocksDBCache::~RocksDBCache() {} + +/** + * Writes an identical copy RocksDBCache from another RocksDBCache; not really used + * + * @name pack + * @memberof RocksDBCache + * @param {String}, filename + * @returns {Boolean} + * @example + * const cache = require('@mapbox/carmen-cache'); + * const RocksDBCache = new cache.RocksDBCache('a'); + * + * cache.pack('filename'); + * + */ + +NAN_METHOD(RocksDBCache::pack) { + if (info.Length() < 1) { + return Nan::ThrowTypeError("expected one info: 'filename'"); + } + if (!info[0]->IsString()) { + return Nan::ThrowTypeError("first argument must be a String"); + } + try { + Nan::Utf8String utf8_filename(info[0]); + if (utf8_filename.length() < 1) { + return Nan::ThrowTypeError("first arg must be a String"); + } + std::string filename(*utf8_filename); + + RocksDBCache* c = node::ObjectWrap::Unwrap(info.This()); + + if (c->db && c->db->GetName() == filename) { + return Nan::ThrowTypeError("rocksdb file is already loaded read-only; unload first"); + } else { + std::shared_ptr existing = c->db; + + std::unique_ptr db; + rocksdb::Options options; + options.create_if_missing = true; + rocksdb::Status status = OpenDB(options, filename, db); + + if (!status.ok()) { + return Nan::ThrowTypeError("unable to open rocksdb file for packing"); + } + + // if what we have now is already a rocksdb, and it's a different + // one from what we're being asked to pack into, copy from one to the other + std::unique_ptr existingIt(existing->NewIterator(rocksdb::ReadOptions())); + for (existingIt->SeekToFirst(); existingIt->Valid(); existingIt->Next()) { + db->Put(rocksdb::WriteOptions(), existingIt->key(), existingIt->value()); + } + } + info.GetReturnValue().Set(true); + return; + } catch (std::exception const& ex) { + return Nan::ThrowTypeError(ex.what()); + } +} + +// Used in merge() to queue files for merging; result is undefined +void mergeQueue(uv_work_t* req) { + MergeBaton* baton = static_cast(req->data); + std::string const& filename1 = baton->filename1; + std::string const& filename2 = baton->filename2; + std::string const& filename3 = baton->filename3; + std::string const& method = baton->method; + + // input 1 + std::unique_ptr db1; + rocksdb::Options options1; + options1.create_if_missing = true; + rocksdb::Status status1 = OpenForReadOnlyDB(options1, filename1, db1); + if (!status1.ok()) { + return Nan::ThrowTypeError("unable to open rocksdb input file #1"); + } + + // input 2 + std::unique_ptr db2; + rocksdb::Options options2; + options2.create_if_missing = true; + rocksdb::Status status2 = OpenForReadOnlyDB(options2, filename2, db2); + if (!status2.ok()) { + return Nan::ThrowTypeError("unable to open rocksdb input file #2"); + } + + // output + std::unique_ptr db3; + rocksdb::Options options3; + options3.create_if_missing = true; + rocksdb::Status status3 = OpenDB(options3, filename3, db3); + if (!status1.ok()) { + return Nan::ThrowTypeError("unable to open rocksdb output file"); + } + + // Ids that have been seen + std::map ids1; + std::map ids2; + + try { + // Store ids from 1 + std::unique_ptr it1(db1->NewIterator(rocksdb::ReadOptions())); + for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { + ids1.emplace(it1->key().ToString(), true); + } + + // Store ids from 2 + std::unique_ptr it2(db2->NewIterator(rocksdb::ReadOptions())); + for (it2->SeekToFirst(); it2->Valid(); it2->Next()) { + ids2.emplace(it2->key().ToString(), true); + } + + // No delta writes from message1 + it1 = std::unique_ptr(db1->NewIterator(rocksdb::ReadOptions())); + for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { + std::string key_id = it1->key().ToString(); + + // Skip this id if also in message 2 + if (ids2.find(key_id) != ids2.end()) continue; + + // get input proto + std::string in_message = it1->value().ToString(); + protozero::pbf_reader item(in_message); + item.next(CACHE_ITEM); + + std::string message; + message.clear(); + + protozero::pbf_writer item_writer(message); + { + protozero::packed_field_uint64 field{item_writer, 1}; + auto vals = item.get_packed_uint64(); + for (auto it = vals.first; it != vals.second; ++it) { + field.add_element(static_cast(*it)); + } + } + + rocksdb::Status putStatus = db3->Put(rocksdb::WriteOptions(), key_id, message); + assert(putStatus.ok()); + } + + // No delta writes from message2 + it2 = std::unique_ptr(db2->NewIterator(rocksdb::ReadOptions())); + for (it2->SeekToFirst(); it2->Valid(); it2->Next()) { + std::string key_id = it2->key().ToString(); + + // Skip this id if also in message 1 + if (ids1.find(key_id) != ids1.end()) continue; + + // get input proto + std::string in_message = it2->value().ToString(); + protozero::pbf_reader item(in_message); + item.next(CACHE_ITEM); + + std::string message; + message.clear(); + + protozero::pbf_writer item_writer(message); + { + protozero::packed_field_uint64 field{item_writer, 1}; + auto vals = item.get_packed_uint64(); + for (auto it = vals.first; it != vals.second; ++it) { + field.add_element(static_cast(*it)); + } + } + + rocksdb::Status putStatus = db3->Put(rocksdb::WriteOptions(), key_id, message); + assert(putStatus.ok()); + } + + // Delta writes for ids in both message1 and message2 + it1 = std::unique_ptr(db1->NewIterator(rocksdb::ReadOptions())); + for (it1->SeekToFirst(); it1->Valid(); it1->Next()) { + std::string key_id = it1->key().ToString(); + + // Skip ids that are only in one or the other lists + if (ids1.find(key_id) == ids1.end() || ids2.find(key_id) == ids2.end()) continue; + + // get input proto + std::string in_message1 = it1->value().ToString(); + protozero::pbf_reader item(in_message1); + item.next(CACHE_ITEM); + + uint64_t lastval = 0; + intarray varr; + + // Add values from filename1 + auto vals = item.get_packed_uint64(); + for (auto it = vals.first; it != vals.second; ++it) { + if (method == "freq") { + varr.emplace_back(*it); + break; + } else if (lastval == 0) { + lastval = *it; + varr.emplace_back(lastval); + } else { + lastval = lastval - *it; + varr.emplace_back(lastval); + } + } + + std::string in_message2; + std::string max_key = "__MAX__"; + auto max_key_length = max_key.length(); + rocksdb::Status s = db2->Get(rocksdb::ReadOptions(), key_id, &in_message2); + if (s.ok()) { + // get input proto 2 + protozero::pbf_reader item2(in_message2); + item2.next(CACHE_ITEM); + + auto vals2 = item2.get_packed_uint64(); + lastval = 0; + for (auto it = vals2.first; it != vals2.second; ++it) { + if (method == "freq") { + if (key_id.compare(0, max_key_length, max_key) == 0) { + varr[0] = varr[0] > *it ? varr[0] : *it; + } else { + varr[0] = varr[0] + *it; + } + break; + } else if (lastval == 0) { + lastval = *it; + varr.emplace_back(lastval); + } else { + lastval = lastval - *it; + varr.emplace_back(lastval); + } + } + } + + // Sort for proper delta encoding + std::sort(varr.begin(), varr.end(), std::greater()); + + // if this is the merging of a prefix cache entry + // (which would start with '=' and have been truncated) + // truncate the merged result + if (key_id.at(0) == '=' && varr.size() > PREFIX_MAX_GRID_LENGTH) { + varr.resize(PREFIX_MAX_GRID_LENGTH); + } + + // Write varr to merged protobuf + std::string message; + message.clear(); + + protozero::pbf_writer item_writer(message); + { + protozero::packed_field_uint64 field{item_writer, 1}; + lastval = 0; + for (auto const& vitem : varr) { + if (lastval == 0) { + field.add_element(static_cast(vitem)); + } else { + field.add_element(static_cast(lastval - vitem)); + } + lastval = vitem; + } + } + + rocksdb::Status putStatus = db3->Put(rocksdb::WriteOptions(), key_id, message); + assert(putStatus.ok()); + } + + } catch (std::exception const& ex) { + baton->error = ex.what(); + } +} + +// Used in merge() to queue files for merging + +// we don't use the 'status' parameter, but it's required as part of the uv_after_work_cb +// function signature, so suppress the warning about it +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" +void mergeAfter(uv_work_t* req, int status) { + Nan::HandleScope scope; + MergeBaton* baton = static_cast(req->data); + if (!baton->error.empty()) { + v8::Local argv[1] = {Nan::Error(baton->error.c_str())}; + Nan::MakeCallback(Nan::GetCurrentContext()->Global(), Nan::New(baton->callback), 1, argv); + } else { + Local argv[2] = {Nan::Null()}; + Nan::MakeCallback(Nan::GetCurrentContext()->Global(), Nan::New(baton->callback), 1, argv); + } + baton->callback.Reset(); + delete baton; +} +#pragma clang diagnostic pop + +/** + * Retrieves data exactly matching phrase and language settings by id + * + * @name get + * @memberof RocksDBCache + * @param {String} id + * @param {Boolean} matches_prefixes: T if it matches exactly, F: if it does not + * @param {Array} optional; array of languages + * @returns {Array} integers referring to grids + * @example + * const cache = require('@mapbox/carmen-cache'); + * const RocksDBCache = new cache.RocksDBCache('a'); + * + * RocksDBCache.get(id, languages); + * // => [grid, grid, grid, grid... ] + * + */ + +NAN_METHOD(RocksDBCache::_get) { + return _genericget(info); +} + +/** + * lists the keys in the RocksDBCache object + * + * @name list + * @memberof RocksDBCache + * @param {String} id + * @returns {Array} Set of keys/ids + * @example + * const cache = require('@mapbox/carmen-cache'); + * const RocksDBCache = new cache.RocksDBCache('a'); + * + * cache.list('a', (err, result) => { + * if (err) throw err; + * console.log(result); + * }); + * + */ + +NAN_METHOD(RocksDBCache::list) { + try { + RocksDBCache* c = node::ObjectWrap::Unwrap(info.This()); + Local ids = Nan::New(); + + std::shared_ptr db = c->db; + + std::unique_ptr it(db->NewIterator(rocksdb::ReadOptions())); + unsigned idx = 0; + for (it->SeekToFirst(); it->Valid(); it->Next()) { + std::string key_id = it->key().ToString(); + if (key_id.at(0) == '=') continue; + + Local out = Nan::New(); + out->Set(0, Nan::New(key_id.substr(0, key_id.find(LANGFIELD_SEPARATOR))).ToLocalChecked()); + + langfield_type langfield = extract_langfield(key_id); + if (langfield == ALL_LANGUAGES) { + out->Set(1, Nan::Null()); + } else { + out->Set(1, langfieldToLangarray(langfield)); + } + + ids->Set(idx++, out); + } + + info.GetReturnValue().Set(ids); + return; + } catch (std::exception const& ex) { + return Nan::ThrowTypeError(ex.what()); + } +} + +/** + * merges the contents from 2 RocksDBCaches + * + * @name merge + * @memberof RocksDBCache + * @param {String} RocksDBCache file 1 + * @param {String} RocksDBCache file 2 + * @param {String} result RocksDBCache file + * @param {String} method which is either concat or freq + * @param {Function} callback called from the mergeAfter method + * @returns {Set} Set of ids + * @example + * const cache = require('@mapbox/carmen-cache'); + * const RocksDBCache = new cache.RocksDBCache('a'); + * + * cache.merge('file1', 'file2', 'resultFile', 'method', (err, result) => { + * if (err) throw err; + * console.log(result); + * }); + * + */ + +NAN_METHOD(RocksDBCache::merge) { + if (!info[0]->IsString()) return Nan::ThrowTypeError("argument 1 must be a String (infile 1)"); + if (!info[1]->IsString()) return Nan::ThrowTypeError("argument 2 must be a String (infile 2)"); + if (!info[2]->IsString()) return Nan::ThrowTypeError("argument 3 must be a String (outfile)"); + if (!info[3]->IsString()) return Nan::ThrowTypeError("argument 4 must be a String (method)"); + if (!info[4]->IsFunction()) return Nan::ThrowTypeError("argument 5 must be a callback function"); + + std::string in1 = *String::Utf8Value(info[0]->ToString()); + std::string in2 = *String::Utf8Value(info[1]->ToString()); + std::string out = *String::Utf8Value(info[2]->ToString()); + Local callback = info[4]; + std::string method = *String::Utf8Value(info[3]->ToString()); + + MergeBaton* baton = new MergeBaton(); + baton->filename1 = in1; + baton->filename2 = in2; + baton->filename3 = out; + baton->method = method; + baton->callback.Reset(callback.As()); + baton->request.data = baton; + uv_queue_work(uv_default_loop(), &baton->request, mergeQueue, static_cast(mergeAfter)); + info.GetReturnValue().Set(Nan::Undefined()); + return; +} + +/** +* Creates an in-memory key-value store mapping phrases and language IDs +* to lists of corresponding grids (grids ie are integer representations of occurrences of the phrase within an index) +* + * @name RocksDBCache + * @memberof RocksDBCache + * @param {String} id + * @param {String} filename + * @returns {Object} + * @example + * const cache = require('@mapbox/carmen-cache'); + * const RocksDBCache = new cache.RocksDBCache('a', 'filename'); + * + */ + +NAN_METHOD(RocksDBCache::New) { + if (!info.IsConstructCall()) { + return Nan::ThrowTypeError("Cannot call constructor as function, you need to use 'new' keyword"); + } + try { + if (info.Length() < 2) { + return Nan::ThrowTypeError("expected arguments 'id' and 'filename'"); + } + if (!info[0]->IsString()) { + return Nan::ThrowTypeError("first argument 'id' must be a String"); + } + if (!info[1]->IsString()) { + return Nan::ThrowTypeError("second argument 'filename' must be a String"); + } + + Nan::Utf8String utf8_filename(info[1]); + if (utf8_filename.length() < 1) { + return Nan::ThrowTypeError("second arg must be a String"); + } + std::string filename(*utf8_filename); + + std::unique_ptr db; + rocksdb::Options options; + options.create_if_missing = true; + rocksdb::Status status = OpenForReadOnlyDB(options, filename, db); + + if (!status.ok()) { + return Nan::ThrowTypeError("unable to open rocksdb file for loading"); + } + RocksDBCache* im = new RocksDBCache(); + im->db = std::move(db); + im->Wrap(info.This()); + info.This()->Set(Nan::New("id").ToLocalChecked(), info[0]); + info.GetReturnValue().Set(info.This()); + return; + } catch (std::exception const& ex) { + return Nan::ThrowTypeError(ex.what()); + } +} + +/** + * Retrieves grid that at least partially matches phrase and/or language inputs + * + * @name get + * @memberof RocksDBCache + * @param {String} id + * @param {Boolean} matches_prefixes: T if it matches exactly, F: if it does not + * @param {Array} optional; array of languages + * @returns {Array} integers referring to grids + * @example + * const cache = require('@mapbox/carmen-cache'); + * const RocksDBCache = new cache.RocksDBCache('a'); + * + * RocksDBCache.get(id, languages); + * // => [grid, grid, grid, grid... ] + * + */ + +NAN_METHOD(RocksDBCache::_getmatching) { + return _genericgetmatching(info); +} + +} // namespace carmen diff --git a/src/rocksdbcache.hpp b/src/rocksdbcache.hpp new file mode 100644 index 0000000..2a2d924 --- /dev/null +++ b/src/rocksdbcache.hpp @@ -0,0 +1,110 @@ +#ifndef __CARMEN_ROCKSDBCACHE_HPP__ +#define __CARMEN_ROCKSDBCACHE_HPP__ + +#include "cpp_util.hpp" +#include "node_util.hpp" + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunknown-pragmas" +#pragma clang diagnostic ignored "-Wconversion" +#pragma clang diagnostic ignored "-Wshadow" +#pragma clang diagnostic ignored "-Wsign-compare" +#pragma clang diagnostic ignored "-Wunused-local-typedef" +#pragma clang diagnostic ignored "-Wunused-parameter" +#pragma clang diagnostic ignored "-Wpadded" +#pragma clang diagnostic ignored "-Wold-style-cast" +#pragma clang diagnostic ignored "-Wsign-conversion" +#pragma clang diagnostic ignored "-Wshorten-64-to-32" + +#include + +#pragma clang diagnostic pop + +namespace carmen { + +#define CACHE_MESSAGE 1 +#define CACHE_ITEM 1 + +#define MEMO_PREFIX_LENGTH_T1 3 +#define MEMO_PREFIX_LENGTH_T2 6 +#define PREFIX_MAX_GRID_LENGTH 500000 + +struct sortableGrid { + protozero::const_varint_iterator it; + protozero::const_varint_iterator end; + value_type unadjusted_lastval; + bool matches_language; +}; + +struct MergeBaton : carmen::noncopyable { + uv_work_t request; + std::string filename1; + std::string filename2; + std::string filename3; + std::string method; + std::string error; + Nan::Persistent callback; +}; + +inline void decodeMessage(std::string const& message, intarray& array) { + protozero::pbf_reader item(message); + item.next(CACHE_ITEM); + auto vals = item.get_packed_uint64(); + uint64_t lastval = 0; + // delta decode values. + for (auto it = vals.first; it != vals.second; ++it) { + if (lastval == 0) { + lastval = *it; + array.emplace_back(lastval); + } else { + lastval = lastval - *it; + array.emplace_back(lastval); + } + } +} + +inline void decodeAndBoostMessage(std::string const& message, intarray& array) { + protozero::pbf_reader item(message); + item.next(CACHE_ITEM); + auto vals = item.get_packed_uint64(); + uint64_t lastval = 0; + // delta decode values. + for (auto it = vals.first; it != vals.second; ++it) { + if (lastval == 0) { + lastval = *it; + array.emplace_back(lastval | LANGUAGE_MATCH_BOOST); + } else { + lastval = lastval - *it; + array.emplace_back(lastval | LANGUAGE_MATCH_BOOST); + } + } +} + +class RocksDBCache : public node::ObjectWrap { + public: + ~RocksDBCache(); + static Nan::Persistent constructor; + static void Initialize(v8::Handle target); + static NAN_METHOD(New); + static NAN_METHOD(pack); + static NAN_METHOD(merge); + static NAN_METHOD(list); + static NAN_METHOD(_get); + static NAN_METHOD(_getmatching); + static NAN_METHOD(_set); + static NAN_METHOD(coalesce); + explicit RocksDBCache(); + void _ref() { Ref(); } + void _unref() { Unref(); } + std::shared_ptr db; +}; + +void mergeQueue(uv_work_t* req); +void mergeAfter(uv_work_t* req, int status); + +intarray __get(RocksDBCache const* c, std::string phrase, langfield_type langfield); +intarray __getmatching(RocksDBCache const* c, std::string phrase, bool match_prefixes, langfield_type langfield); + +} // namespace carmen + +#endif // __CARMEN_ROCKSDBCACHE_HPP__ diff --git a/test/cache.test.js b/test/cache.test.js index 2bf7817..291610b 100644 --- a/test/cache.test.js +++ b/test/cache.test.js @@ -1,8 +1,12 @@ 'use strict'; const carmenCache = require('../index.js'); -const tape = require('tape'); +const test = require('tape'); const fs = require('fs'); +// These tests are testing to ensure both MemoryCache and RocksDBCache are identical. +// It loops over each to check that they are the same. +// Check API.md to see what each of these actually tests. + const tmpdir = '/tmp/temp.' + Math.random().toString(36).substr(2, 5); fs.mkdirSync(tmpdir); let tmpidx = 0; @@ -16,14 +20,14 @@ const sortedDescending = function(arr) { return [].concat(arr).sort((a, b) => { return b - a; }); }; -tape('list', (assert) => { +test('list', (t) => { const cache = new carmenCache.MemoryCache('a'); cache._set('5', [0,1,2]); - assert.deepEqual(cache.list().map((x) => { return x[0]; }), ['5']); - assert.end(); + t.deepEqual(cache.list().map((x) => { return x[0]; }), ['5'], 'listed keys match inserted keys'); + t.end(); }); -tape('get / set / list / pack / load (simple)', (assert) => { +test('get / set / list / pack / load (simple)', (t) => { const cache = new carmenCache.MemoryCache('a'); const ids = []; @@ -31,16 +35,16 @@ tape('get / set / list / pack / load (simple)', (assert) => { const id = Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 5); ids.push(id); - assert.deepEqual(cache._get(id), undefined, id + ' not set'); + t.deepEqual(cache._get(id), undefined, id + ' not set'); cache._set(id, [0,1,2]); - assert.deepEqual(cache._get(id), sortedDescending([0, 1, 2]), id + ' set to 0,1,2'); + t.deepEqual(cache._get(id), sortedDescending([0, 1, 2]), id + ' set to 0,1,2'); cache._set(id, [3,4,5]); - assert.deepEqual(cache._get(id), sortedDescending([3, 4, 5]), id + ' set to 3,4,5'); + t.deepEqual(cache._get(id), sortedDescending([3, 4, 5]), id + ' set to 3,4,5'); cache._set(id, [6,7,8], null, true); - assert.deepEqual(cache._get(id), sortedDescending([3, 4, 5, 6, 7, 8]), id + ' set to 3,4,5,6,7,8'); + t.deepEqual(cache._get(id), sortedDescending([3, 4, 5, 6, 7, 8]), id + ' set to 3,4,5,6,7,8'); } - assert.deepEqual(sorted(cache.list().map((x) => { return x[0]; })), sorted(ids), 'mem ids match'); + t.deepEqual(sorted(cache.list().map((x) => { return x[0]; })), sorted(ids), 'mem ids match'); const pack = tmpfile(); cache.pack(pack); @@ -49,14 +53,14 @@ tape('get / set / list / pack / load (simple)', (assert) => { for (let i = 0; i < 5; i++) { const id = ids[i]; - assert.deepEqual(cache._get(id), sortedDescending([3, 4, 5, 6, 7, 8]), id + ' set to 3,4,5,6,7,8'); + t.deepEqual(cache._get(id), sortedDescending([3, 4, 5, 6, 7, 8]), id + ' set to 3,4,5,6,7,8'); } - assert.deepEqual(sorted(loader.list().map((x) => { return x[0]; })), sorted(ids), 'rocks ids match'); - assert.end(); + t.deepEqual(sorted(loader.list().map((x) => { return x[0]; })), sorted(ids), 'rocks ids match'); + t.end(); }); -tape('get / set / list / pack / load (with lang codes)', (assert) => { +test('get / set / list / pack / load (with lang codes)', (t) => { const cache = new carmenCache.MemoryCache('a'); const ids = []; @@ -65,28 +69,28 @@ tape('get / set / list / pack / load (with lang codes)', (assert) => { const id = Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 5); ids.push(id); - assert.deepEqual(cache._get(id), undefined, id + ' not set'); + t.deepEqual(cache._get(id), undefined, id + ' not set'); cache._set(id, [0,1,2], [0]); cache._set(id, [7,8,9], [1]); cache._set(id, [12,13,14], [0,1]); - assert.deepEqual(cache._get(id, [0]), sortedDescending([0, 1, 2]), id + ' for lang [0] set to 0,1,2'); - assert.deepEqual(cache._get(id, [1]), sortedDescending([7, 8, 9]), id + ' for lang [1] set to 7,8,9'); - assert.deepEqual(cache._get(id, [0,1]), sortedDescending([12, 13, 14]), id + ' for lang [0,1] set to 12,13,14'); - assert.false(cache._get(id), id + ' without lang code returns nothing'); + t.deepEqual(cache._get(id, [0]), sortedDescending([0, 1, 2]), id + ' for lang [0] set to 0,1,2'); + t.deepEqual(cache._get(id, [1]), sortedDescending([7, 8, 9]), id + ' for lang [1] set to 7,8,9'); + t.deepEqual(cache._get(id, [0,1]), sortedDescending([12, 13, 14]), id + ' for lang [0,1] set to 12,13,14'); + t.false(cache._get(id), id + ' without lang code returns nothing'); cache._set(id, [3,4,5], [0]); - assert.deepEqual(cache._get(id, [0]), sortedDescending([3,4,5]), id + ' for lang [0] set to 3,4,5'); - assert.deepEqual(cache._get(id, [0,1]), sortedDescending([12, 13, 14]), id + ' for lang [0,1] still set to 12,13,14'); + t.deepEqual(cache._get(id, [0]), sortedDescending([3,4,5]), id + ' for lang [0] set to 3,4,5'); + t.deepEqual(cache._get(id, [0,1]), sortedDescending([12, 13, 14]), id + ' for lang [0,1] still set to 12,13,14'); cache._set(id, [6,7,8], [0], true); - assert.deepEqual(cache._get(id, [0]), sortedDescending([3, 4, 5, 6, 7, 8]), id + ' for lang [0] set to 3,4,5,6,7,8'); - assert.deepEqual(cache._get(id, [0,1]), sortedDescending([12, 13, 14]), id + ' for lang [0,1] still set to 12,13,14'); - assert.false(cache._get(id), id + ' without lang code still returns nothing'); + t.deepEqual(cache._get(id, [0]), sortedDescending([3, 4, 5, 6, 7, 8]), id + ' for lang [0] set to 3,4,5,6,7,8'); + t.deepEqual(cache._get(id, [0,1]), sortedDescending([12, 13, 14]), id + ' for lang [0,1] still set to 12,13,14'); + t.false(cache._get(id), id + ' without lang code still returns nothing'); expected.push([id, [0]]); expected.push([id, [1]]); expected.push([id, [0,1]]); } - assert.deepEqual(sorted(cache.list().map(JSON.stringify)), sorted(expected.map(JSON.stringify)), 'mem ids and langs match'); + t.deepEqual(sorted(cache.list().map(JSON.stringify)), sorted(expected.map(JSON.stringify)), 'mem ids and langs match'); const pack = tmpfile(); cache.pack(pack); @@ -94,23 +98,23 @@ tape('get / set / list / pack / load (with lang codes)', (assert) => { for (let i = 0; i < 5; i++) { const id = ids[i]; - assert.deepEqual(cache._get(id, [1]), sortedDescending([7, 8, 9]), id + ' for lang [1] set to 7,8,9'); - assert.deepEqual(cache._get(id, [0,1]), sortedDescending([12, 13, 14]), id + ' for lang [0,1] set to 12,13,14'); - assert.false(cache._get(id), id + ' without lang code returns nothing'); - assert.deepEqual(cache._get(id, [0]), sortedDescending([3, 4, 5, 6, 7, 8]), id + ' for lang [0] set to 3,4,5,6,7,8'); - assert.deepEqual(cache._get(id, [0,1]), sortedDescending([12, 13, 14]), id + ' for lang [0,1] still set to 12,13,14'); + t.deepEqual(cache._get(id, [1]), sortedDescending([7, 8, 9]), id + ' for lang [1] set to 7,8,9'); + t.deepEqual(cache._get(id, [0,1]), sortedDescending([12, 13, 14]), id + ' for lang [0,1] set to 12,13,14'); + t.false(cache._get(id), id + ' without lang code returns nothing'); + t.deepEqual(cache._get(id, [0]), sortedDescending([3, 4, 5, 6, 7, 8]), id + ' for lang [0] set to 3,4,5,6,7,8'); + t.deepEqual(cache._get(id, [0,1]), sortedDescending([12, 13, 14]), id + ' for lang [0,1] still set to 12,13,14'); } - assert.deepEqual(sorted(cache.list().map(JSON.stringify)), sorted(expected.map(JSON.stringify)), 'rocks ids and langs match'); - assert.end(); + t.deepEqual(sorted(cache.list().map(JSON.stringify)), sorted(expected.map(JSON.stringify)), 'rocks ids and langs match'); + t.end(); }); -tape('pack', (assert) => { +test('pack', (t) => { const cache = new carmenCache.MemoryCache('a'); cache._set('5', [0,1,2]); // set should replace data cache._set('5', [0,1,2,4]); - assert.throws(cache._set.bind(null, '5', []), 'can\'t set empty term'); + t.throws(cache._set.bind(null, '5', []), 'can\'t set empty term'); // fake data const array = []; @@ -122,26 +126,26 @@ tape('pack', (assert) => { packer._set('6', array); // invalid args - assert.throws(() => { new carmenCache.RocksDBCache('a'); }); - assert.throws(() => { const loader = new carmenCache.MemoryCache('a'); loader.pack(); }); - assert.throws(() => { const loader = new carmenCache.MemoryCache('a'); loader.pack(1); }); - assert.throws(() => { new carmenCache.RocksDBCache('a', 1); }); - assert.throws(() => { new carmenCache.RocksDBCache('a', null); }); - assert.throws(() => { new carmenCache.RocksDBCache('a', {}); }); - assert.throws(() => { new carmenCache.RocksDBCache('a', new Buffer('a')); }); + t.throws(() => { new carmenCache.RocksDBCache('a'); }, 'throws on invalid arguments'); + t.throws(() => { const loader = new carmenCache.MemoryCache('a'); loader.pack(); }, 'throws on invalid arguments'); + t.throws(() => { const loader = new carmenCache.MemoryCache('a'); loader.pack(1); }, 'throws on invalid arguments'); + t.throws(() => { new carmenCache.RocksDBCache('a', 1); }, 'throws on invalid arguments'); + t.throws(() => { new carmenCache.RocksDBCache('a', null); }, 'throws on invalid arguments'); + t.throws(() => { new carmenCache.RocksDBCache('a', {}); }, 'throws on invalid arguments'); + t.throws(() => { new carmenCache.RocksDBCache('a', new Buffer('a')); }, 'throws on invalid arguments'); // grab data right back out const directLoad = tmpfile(); packer.pack(directLoad); const loader = new carmenCache.RocksDBCache('a', directLoad); - assert.deepEqual(loader._get('5'), sortedDescending(array)); - assert.deepEqual(loader._get('6'), sortedDescending(array)); + t.deepEqual(loader._get('5'), sortedDescending(array), '_get output matches inserted values'); + t.deepEqual(loader._get('6'), sortedDescending(array), '_get output matches inserted values'); // test what happens when you pack a rocksdbcache - assert.throws(() => { loader.pack(); }, 'filename is required'); - assert.throws(() => { loader.pack(1); }, 'filename must be a string'); - assert.throws(() => { loader.pack(directLoad); }, 'can\'t pack into an already-loaded file'); - assert.ok(() => { loader.pack(tmpfile()); }, 'repacking works'); + t.throws(() => { loader.pack(); }, 'filename is required'); + t.throws(() => { loader.pack(1); }, 'filename must be a string'); + t.throws(() => { loader.pack(directLoad); }, 'can\'t pack into an already-loaded file'); + t.ok(() => { loader.pack(tmpfile()); }, 'repacking works'); - assert.end(); + t.end(); }); diff --git a/test/coalesce.proximity.test.js b/test/coalesce.proximity.test.js index 47f4a56..799d129 100644 --- a/test/coalesce.proximity.test.js +++ b/test/coalesce.proximity.test.js @@ -48,7 +48,7 @@ const test = require('tape'); // This centerpoint favors the N/S direction over E/W slightly to make // expected output order obvious - test('proximity ne', (assert) => { + test('proximity ne', (t) => { coalesce([{ cache: cache, mask: 1 << 0, @@ -61,14 +61,14 @@ const test = require('tape'); radius: 200, centerzxy: [14, 100 + 10, 100 + 15] }, (err, res) => { - assert.ifError(err); - assert.deepEqual(res.map((cover) => cover[0].id), [ne.id, nw.id, se.id, sw.id], 'right order'); - assert.deepEqual(res.map((cover) => Math.floor(cover[0].distance)), [123, 139, 146, 159], 'distances check out'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res.map((cover) => cover[0].id), [ne.id, nw.id, se.id, sw.id], 'right order'); + t.deepEqual(res.map((cover) => Math.floor(cover[0].distance)), [123, 139, 146, 159], 'distances check out'); + t.end(); }); }); - test('proximity se', (assert) => { + test('proximity se', (t) => { coalesce([{ cache: cache, mask: 1 << 0, @@ -81,14 +81,14 @@ const test = require('tape'); radius: 200, centerzxy: [14, 100 + 10, 100 - 15] }, (err, res) => { - assert.ifError(err); - assert.deepEqual(res.map((cover) => cover[0].id), [se.id, sw.id, ne.id, nw.id], 'right order'); - assert.deepEqual(res.map((cover) => Math.floor(cover[0].distance)), [123, 139, 146, 159], 'distances check out'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res.map((cover) => cover[0].id), [se.id, sw.id, ne.id, nw.id], 'right order'); + t.deepEqual(res.map((cover) => Math.floor(cover[0].distance)), [123, 139, 146, 159], 'distances check out'); + t.end(); }); }); - test('proximity sw', (assert) => { + test('proximity sw', (t) => { coalesce([{ cache: cache, mask: 1 << 0, @@ -101,14 +101,14 @@ const test = require('tape'); radius: 200, centerzxy: [14, 100 - 10, 100 - 15] }, (err, res) => { - assert.ifError(err); - assert.deepEqual(res.map((cover) => cover[0].id), [sw.id, se.id, nw.id, ne.id], 'right order'); - assert.deepEqual(res.map((cover) => Math.floor(cover[0].distance)), [123, 139, 146, 159], 'distances check out'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res.map((cover) => cover[0].id), [sw.id, se.id, nw.id, ne.id], 'right order'); + t.deepEqual(res.map((cover) => Math.floor(cover[0].distance)), [123, 139, 146, 159], 'distances check out'); + t.end(); }); }); - test('proximity nw', (assert) => { + test('proximity nw', (t) => { coalesce([{ cache: cache, mask: 1 << 0, @@ -121,10 +121,10 @@ const test = require('tape'); radius: 200, centerzxy: [14, 100 - 10, 100 + 15] }, (err, res) => { - assert.ifError(err); - assert.deepEqual(res.map((cover) => cover[0].id), [nw.id, ne.id, sw.id, se.id], 'right order'); - assert.deepEqual(res.map((cover) => Math.floor(cover[0].distance)), [123, 139, 146, 159], 'distances check out'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res.map((cover) => cover[0].id), [nw.id, ne.id, sw.id, se.id], 'right order'); + t.deepEqual(res.map((cover) => Math.floor(cover[0].distance)), [123, 139, 146, 159], 'distances check out'); + t.end(); }); }); })(); diff --git a/test/coalesce.test.js b/test/coalesce.test.js index f12180a..84365a3 100644 --- a/test/coalesce.test.js +++ b/test/coalesce.test.js @@ -17,42 +17,42 @@ const toRocksCache = function(memcache) { return new RocksDBCache(memcache.id + '.rocks', pack); }; -test('coalesce args', (assert) => { - assert.throws(() => { +test('coalesce args', (t) => { + t.throws(() => { coalesce(); }, /Expects 3 arguments/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([]); }, /Expects 3 arguments/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([], {} ); }, /Expects 3 arguments/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{}], {}, () => {} ); }, /missing idx property/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([-1], {}, () => {} ); - }, /All items in array must be valid PhrasematchSubq objects/, 'throws'); + }, /All items in array must be valid PhrasematchSubqObjects/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce(undefined, {}, () => {} ); - }, /Arg 1 must be a PhrasematchSubq array/, 'throws'); + }, /Arg 1 must be a PhrasematchSubqObject array/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([], {}, () => {} ); }, /Arg 1 must be an array with one or more/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([undefined], {}, () => {} ); - }, /All items in array must be valid PhrasematchSubq objects/, 'throws'); + }, /All items in array must be valid PhrasematchSubqObjects/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([null], {}, () => {} ); - }, /All items in array must be valid PhrasematchSubq objects/, 'throws'); + }, /All items in array must be valid PhrasematchSubqObjects/, 'throws'); const valid_subq = { cache: new MemoryCache('a'), @@ -64,64 +64,64 @@ test('coalesce args', (assert) => { prefix: false }; - assert.throws(() => { + t.throws(() => { coalesce([valid_subq],undefined,() => {}); }, /Arg 2 must be an options object/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq],undefined,() => {}); }, /Arg 2 must be an options object/, 'throws'); if (process.versions.node[0] !== '0') { - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ idx:-1 })],{},() => {}); }, /encountered idx value too large to fit/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ idx:null })],{},() => {}); }, /value must be a number/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ zoom:-1 })],{},() => {}); }, /encountered zoom value too large to fit/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ zoom:null })],{},() => {}); }, /value must be a number/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ zoom:-1 })],{},() => {}); }, /encountered zoom value too large to fit/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ mask:null })],{},() => {}); }, /value must be a number/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ mask:-1 })],{},() => {}); }, /encountered mask value too large to fit/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ weight:null })],{},() => {}); }, /weight value must be a number/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ weight:-1 })],{},() => {}); }, /encountered weight value too large to fit in double/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ phrase:null })],{},() => {}); }, /phrase value must be a string/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ phrase:'' })],{},() => {}); }, /encountered invalid phrase/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ cache:null })],{},() => {}); }, /cache value must be a Cache object/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([Object.assign({},valid_subq,{ cache:{} })],{},() => {}); }, /cache value must be/, 'throws'); @@ -146,113 +146,113 @@ test('coalesce args', (assert) => { } ]; - assert.throws(() => { + t.throws(() => { coalesce(valid_stack.concat([Object.assign({},valid_subq,{ cache:null })]),{},() => {}); }, /cache value must be a Cache object/, 'throws'); } - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { radius:null },() => {} ); }, /radius must be a number/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { radius:5e9 },() => {} ); }, /encountered radius too large to fit in unsigned/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { bboxzxy:null },() => {} ); }, /bboxzxy must be an array/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { bboxzxy:[0,0,0,0] },() => {} ); }, /bboxzxy must be an array of 5 numbers/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { bboxzxy:['',0,0,0,0] },() => {} ); }, /bboxzxy values must be number/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { bboxzxy:[-1,0,0,0,0] },() => {} ); }, /encountered bboxzxy value too large to fit in uint32_t/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { bboxzxy:[4294967296,0,0,0,0] },() => {} ); }, /encountered bboxzxy value too large to fit in uint32_t/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { centerzxy:null },() => {} ); }, /centerzxy must be an array/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { centerzxy:['',0,0] },() => {} ); }, /centerzxy values must be number/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { centerzxy:[0,0] },() => {} ); }, /centerzxy must be an array of 3 numbers/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { centerzxy:[-1,0,0] },() => {} ); }, /encountered centerzxy value too large to fit in uint32_t/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { centerzxy:[4294967296,0,0] },() => {} ); }, /encountered centerzxy value too large to fit in uint32_t/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([valid_subq], { centerzxy:[0,0,0] }, 5 ); }, /Arg 3 must be a callback/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{ mask: 1 << 0, idx: 1, zoom: 1, weight: .5, phrase: '1', prefix: false }],{},() => {}); }, /missing/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{ cache: new MemoryCache('b'), idx: 1, zoom: 1, weight: .5, phrase: '1', prefix: false }],{},() => {}); }, /missing/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{ cache: new MemoryCache('b'), mask: 1 << 0, zoom: 1, weight: .5, phrase: '1', prefix: false }],{},() => {}); }, /missing/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{ cache: new MemoryCache('b'), mask: 1 << 0, idx: 1, weight: .5, phrase: '1', prefix: false }],{},() => {}); }, /missing/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{ cache: new MemoryCache('b'), mask: 1 << 0, idx: 1, zoom: 1, phrase: '1', prefix: false }],{},() => {}); }, /missing/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{ cache: new MemoryCache('b'), mask: 1 << 0, idx: 1, zoom: 1, weight: .5, prefix: false }],{},() => {}); }, /missing/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{ cache: '', mask: 1 << 0, idx: 1, weight: .5, zoom: 1, phrase: '1', prefix: false }],{},() => {}); }, /cache value must be a Cache object/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{ cache: new MemoryCache('b'), mask: '', idx: 1, zoom: 1, weight: .5, phrase: '1', prefix: false }],{},() => {}); }, /mask value must be a number/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{ cache: new MemoryCache('b'), mask: 1 << 0, idx: '', weight: .5, zoom: 1, phrase: '1', prefix: false }],{},() => {}); }, /idx value must be a number/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{ cache: new MemoryCache('b'), mask: 1 << 0, idx: 1, weight: .5, zoom: '', phrase: '1', prefix: false }],{},() => {}); }, /zoom value must be a number/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{ cache: new MemoryCache('b'), mask: 1 << 0, idx: 1, weight: '', zoom: 1, phrase: '1', prefix: false }],{},() => {}); }, /weight value must be a number/, 'throws'); - assert.throws(() => { + t.throws(() => { coalesce([{ cache: new MemoryCache('b'), mask: 1 << 0, idx: 1, weight: .5, zoom: 1, phrase: '' }],{},() => {}); }, /encountered invalid phrase/, 'throws'); - assert.end(); + t.end(); }); (function() { @@ -283,7 +283,7 @@ test('coalesce args', (assert) => { const rockscache = toRocksCache(memcache); [memcache, rockscache].forEach((cache) => { - test('coalesceSingle: ' + cache.id, (assert) => { + test('coalesceSingle: ' + cache.id, (t) => { coalesce([{ cache: cache, mask: 1 << 0, @@ -293,17 +293,17 @@ test('coalesce args', (assert) => { phrase: '1', prefix: false }], {}, (err, res) => { - assert.ifError(err); - assert.deepEqual(res[0].relev, 1, '0.relev'); - assert.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 1.0, score: 7, scoredist: 7, tmpid: 1, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[1].relev, 1, '1.relev'); - assert.deepEqual(res[1][0], { matches_language: true, distance: 0, id: 3, idx: 0, relev: 1.0, score: 1, scoredist: 1, tmpid: 3, x: 3, y: 3 }, '1.0'); - assert.deepEqual(res[2].relev, 0.8, '2.relev'); - assert.deepEqual(res[2][0], { matches_language: true, distance: 0, id: 2, idx: 0, relev: 0.8, score: 3, scoredist: 3, tmpid: 2, x: 2, y: 2 }, '2.0'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res[0].relev, 1, '0.relev'); + t.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 1.0, score: 7, scoredist: 7, tmpid: 1, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[1].relev, 1, '1.relev'); + t.deepEqual(res[1][0], { matches_language: true, distance: 0, id: 3, idx: 0, relev: 1.0, score: 1, scoredist: 1, tmpid: 3, x: 3, y: 3 }, '1.0'); + t.deepEqual(res[2].relev, 0.8, '2.relev'); + t.deepEqual(res[2][0], { matches_language: true, distance: 0, id: 2, idx: 0, relev: 0.8, score: 3, scoredist: 3, tmpid: 2, x: 2, y: 2 }, '2.0'); + t.end(); }); }); - test('coalesceSingle proximity: ' + cache.id, (assert) => { + test('coalesceSingle proximity: ' + cache.id, (t) => { coalesce([{ cache: cache, mask: 1 << 0, @@ -315,17 +315,17 @@ test('coalesce args', (assert) => { }], { centerzxy: [3,3,3] }, (err, res) => { - assert.ifError(err); - assert.deepEqual(res[0].relev, 1, '0.relev'); - assert.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 3, idx: 0, relev: 1.0, score: 1, scoredist: 124.85901539399482, tmpid: 3, x: 3, y: 3 }, '0.0'); - assert.deepEqual(res[1].relev, 1, '1.relev'); - assert.deepEqual(res[1][0], { matches_language: true, distance: 2.8284271247461903, id: 1, idx: 0, relev: 1.0, score: 7, scoredist: 7, tmpid: 1, x: 1, y: 1 }, '1.0'); - assert.deepEqual(res[2].relev, 0.8, '2.relev'); - assert.deepEqual(res[2][0], { matches_language: true, distance: 1.4142135623730951, id: 2, idx: 0, relev: 0.8, score: 3, scoredist: 3, tmpid: 2, x: 2, y: 2 }, '2.0'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res[0].relev, 1, '0.relev'); + t.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 3, idx: 0, relev: 1.0, score: 1, scoredist: 124.85901539399482, tmpid: 3, x: 3, y: 3 }, '0.0'); + t.deepEqual(res[1].relev, 1, '1.relev'); + t.deepEqual(res[1][0], { matches_language: true, distance: 2.8284271247461903, id: 1, idx: 0, relev: 1.0, score: 7, scoredist: 7, tmpid: 1, x: 1, y: 1 }, '1.0'); + t.deepEqual(res[2].relev, 0.8, '2.relev'); + t.deepEqual(res[2][0], { matches_language: true, distance: 1.4142135623730951, id: 2, idx: 0, relev: 0.8, score: 3, scoredist: 3, tmpid: 2, x: 2, y: 2 }, '2.0'); + t.end(); }); }); - test('coalesceSingle bbox: ' + cache.id, (assert) => { + test('coalesceSingle bbox: ' + cache.id, (t) => { coalesce([{ cache: cache, mask: 1 << 0, @@ -337,17 +337,19 @@ test('coalesce args', (assert) => { }], { bboxzxy: [2, 1, 1, 1, 1] }, (err, res) => { - assert.ifError(err); - assert.deepEqual(res[0].relev, 1, '1.relev'); - assert.deepEqual(res.length, 1); - assert.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 1.0, score: 7, scoredist: 7, tmpid: 1, x: 1, y: 1 }, '1.0'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res[0].relev, 1, '1.relev'); + t.deepEqual(res.length, 1, 'got back 1 result'); + t.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 1.0, score: 7, scoredist: 7, tmpid: 1, x: 1, y: 1 }, '1.0'); + t.end(); }); }); }); })(); -// exercise the sort function +// exercise the sort function -- the coverage tool required testing all lines +// of the sort comparator function, so this test just ensures that the sort operator +// works for different kinds of sorting. It's pretty pointless. (function() { const caches = { 'x': new MemoryCache('x', 0), @@ -370,7 +372,7 @@ test('coalesce args', (assert) => { } caches[toIncrement]._set('1', entries.map(Grid.encode)); - test('coalesceSingle sort tests: ' + toIncrement, (assert) => { + test('coalesceSingle sort tests: ' + toIncrement, (t) => { coalesce([{ cache: caches[toIncrement], mask: 1 << 0, @@ -380,9 +382,9 @@ test('coalesce args', (assert) => { phrase: '1', prefix: false }], {}, (err, res) => { - assert.ifError(err); - assert.ok(res); - assert.end(); + t.ifError(err, 'no errors'); + t.ok(res, 'got back a result'); + t.end(); }); }); } @@ -410,7 +412,7 @@ test('coalesce args', (assert) => { memcache._set('1', grids); const rockscache = toRocksCache(memcache); [memcache, rockscache].forEach((cache) => { - test('coalesceSingle: ' + cache.id, (assert) => { + test('coalesceSingle: ' + cache.id, (t) => { coalesce([{ cache: cache, mask: 1 << 0, @@ -420,13 +422,13 @@ test('coalesce args', (assert) => { phrase: '1', prefix: false }], {}, (err, res) => { - assert.ifError(err); - assert.equal(res.length, 2); - assert.deepEqual(res[0].relev, 1, '0.relev'); - assert.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 1, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[1].relev, 1, '1.relev'); - assert.deepEqual(res[1][0], { matches_language: true, distance: 0, id: 2, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 2, x: 1, y: 1 }, '0.0'); - assert.end(); + t.ifError(err, 'no errors'); + t.equal(res.length, 2, 'got back 2 results'); + t.deepEqual(res[0].relev, 1, '0.relev'); + t.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 1, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[1].relev, 1, '1.relev'); + t.deepEqual(res[1][0], { matches_language: true, distance: 0, id: 2, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 2, x: 1, y: 1 }, '0.0'); + t.end(); }); }); }); @@ -465,7 +467,7 @@ test('coalesce args', (assert) => { })], [2]); const rockscache = toRocksCache(memcache); [memcache, rockscache].forEach((cache) => { - test('coalesceSingle, ALL_LANGUAGES: ' + cache.id, (assert) => { + test('coalesceSingle, ALL_LANGUAGES: ' + cache.id, (t) => { coalesce([{ cache: cache, mask: 1 << 0, @@ -475,20 +477,20 @@ test('coalesce args', (assert) => { phrase: '1', prefix: false }], {}, (err, res) => { - assert.ifError(err); - assert.equal(res.length, 4); - assert.deepEqual(res[0].relev, 1, '0.relev'); - assert.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 1, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[1].relev, 1, '1.relev'); - assert.deepEqual(res[1][0], { matches_language: true, distance: 0, id: 2, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 2, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[2].relev, 1, '2.relev'); - assert.deepEqual(res[2][0], { matches_language: true, distance: 0, id: 3, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 3, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[3].relev, 1, '3.relev'); - assert.deepEqual(res[3][0], { matches_language: true, distance: 0, id: 4, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 4, x: 1, y: 1 }, '0.0'); - assert.end(); + t.ifError(err, 'no errors'); + t.equal(res.length, 4, 'got back 4 results'); + t.deepEqual(res[0].relev, 1, '0.relev'); + t.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 1, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[1].relev, 1, '1.relev'); + t.deepEqual(res[1][0], { matches_language: true, distance: 0, id: 2, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 2, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[2].relev, 1, '2.relev'); + t.deepEqual(res[2][0], { matches_language: true, distance: 0, id: 3, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 3, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[3].relev, 1, '3.relev'); + t.deepEqual(res[3][0], { matches_language: true, distance: 0, id: 4, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 4, x: 1, y: 1 }, '0.0'); + t.end(); }); }); - test('coalesceSingle, [0]: ' + cache.id, (assert) => { + test('coalesceSingle, [0]: ' + cache.id, (t) => { coalesce([{ cache: cache, mask: 1 << 0, @@ -499,20 +501,20 @@ test('coalesce args', (assert) => { prefix: false, languages: [0] }], {}, (err, res) => { - assert.ifError(err); - assert.equal(res.length, 4); - assert.deepEqual(res[0].relev, 1, '0.relev'); - assert.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 1, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[1].relev, 1, '1.relev'); - assert.deepEqual(res[1][0], { matches_language: true, distance: 0, id: 3, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 3, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[2].relev, 0.9, '2.relev'); - assert.deepEqual(res[2][0], { matches_language: false, distance: 0, id: 2, idx: 0, relev: 0.9, score: 0, scoredist: 0, tmpid: 2, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[3].relev, 0.9, '3.relev'); - assert.deepEqual(res[3][0], { matches_language: false, distance: 0, id: 4, idx: 0, relev: 0.9, score: 0, scoredist: 0, tmpid: 4, x: 1, y: 1 }, '0.0'); - assert.end(); + t.ifError(err, 'no errors'); + t.equal(res.length, 4, 'got back 4 results'); + t.deepEqual(res[0].relev, 1, '0.relev'); + t.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 1, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[1].relev, 1, '1.relev'); + t.deepEqual(res[1][0], { matches_language: true, distance: 0, id: 3, idx: 0, relev: 1.0, score: 0, scoredist: 0, tmpid: 3, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[2].relev, 0.9, '2.relev'); + t.deepEqual(res[2][0], { matches_language: false, distance: 0, id: 2, idx: 0, relev: 0.9, score: 0, scoredist: 0, tmpid: 2, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[3].relev, 0.9, '3.relev'); + t.deepEqual(res[3][0], { matches_language: false, distance: 0, id: 4, idx: 0, relev: 0.9, score: 0, scoredist: 0, tmpid: 4, x: 1, y: 1 }, '0.0'); + t.end(); }); }); - test('coalesceSingle, [3]: ' + cache.id, (assert) => { + test('coalesceSingle, [3]: ' + cache.id, (t) => { coalesce([{ cache: cache, mask: 1 << 0, @@ -523,17 +525,17 @@ test('coalesce args', (assert) => { prefix: false, languages: [3] }], {}, (err, res) => { - assert.ifError(err); - assert.equal(res.length, 4); - assert.deepEqual(res[0].relev, 0.9, '0.relev'); - assert.deepEqual(res[0][0], { matches_language: false, distance: 0, id: 1, idx: 0, relev: 0.9, score: 0, scoredist: 0, tmpid: 1, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[1].relev, 0.9, '1.relev'); - assert.deepEqual(res[1][0], { matches_language: false, distance: 0, id: 2, idx: 0, relev: 0.9, score: 0, scoredist: 0, tmpid: 2, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[2].relev, 0.9, '2.relev'); - assert.deepEqual(res[2][0], { matches_language: false, distance: 0, id: 3, idx: 0, relev: 0.9, score: 0, scoredist: 0, tmpid: 3, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[3].relev, 0.9, '3.relev'); - assert.deepEqual(res[3][0], { matches_language: false, distance: 0, id: 4, idx: 0, relev: 0.9, score: 0, scoredist: 0, tmpid: 4, x: 1, y: 1 }, '0.0'); - assert.end(); + t.ifError(err, 'no errors'); + t.equal(res.length, 4, 'got back 4 results'); + t.deepEqual(res[0].relev, 0.9, '0.relev'); + t.deepEqual(res[0][0], { matches_language: false, distance: 0, id: 1, idx: 0, relev: 0.9, score: 0, scoredist: 0, tmpid: 1, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[1].relev, 0.9, '1.relev'); + t.deepEqual(res[1][0], { matches_language: false, distance: 0, id: 2, idx: 0, relev: 0.9, score: 0, scoredist: 0, tmpid: 2, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[2].relev, 0.9, '2.relev'); + t.deepEqual(res[2][0], { matches_language: false, distance: 0, id: 3, idx: 0, relev: 0.9, score: 0, scoredist: 0, tmpid: 3, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[3].relev, 0.9, '3.relev'); + t.deepEqual(res[3][0], { matches_language: false, distance: 0, id: 4, idx: 0, relev: 0.9, score: 0, scoredist: 0, tmpid: 4, x: 1, y: 1 }, '0.0'); + t.end(); }); }); }); @@ -588,7 +590,7 @@ test('coalesce args', (assert) => { const a = caches[0], b = caches[1]; - test('coalesceUV: ' + a.id + ', ' + b.id, (assert) => { + test('coalesceUV: ' + a.id + ', ' + b.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -606,18 +608,18 @@ test('coalesce args', (assert) => { phrase: '1', prefix: false }], {}, (err, res) => { - assert.ifError(err); + t.ifError(err, 'no errors'); // sorts by relev, score - assert.deepEqual(res[0].relev, 1, '0.relev'); - assert.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 2, idx: 1, relev: 0.5, score: 7, scoredist: 7, tmpid: 33554434, x: 2, y: 2 }, '0.0'); - assert.deepEqual(res[0][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '0.1'); - assert.deepEqual(res[1].relev, 1, '1.relev'); - assert.deepEqual(res[1][0], { matches_language: true, distance: 0, id: 3, idx: 1, relev: 0.5, score: 1, scoredist: 1, tmpid: 33554435, x: 3, y: 3 }, '1.0'); - assert.deepEqual(res[1][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '1.1'); - assert.end(); + t.deepEqual(res[0].relev, 1, '0.relev'); + t.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 2, idx: 1, relev: 0.5, score: 7, scoredist: 7, tmpid: 33554434, x: 2, y: 2 }, '0.0'); + t.deepEqual(res[0][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '0.1'); + t.deepEqual(res[1].relev, 1, '1.relev'); + t.deepEqual(res[1][0], { matches_language: true, distance: 0, id: 3, idx: 1, relev: 0.5, score: 1, scoredist: 1, tmpid: 33554435, x: 3, y: 3 }, '1.0'); + t.deepEqual(res[1][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '1.1'); + t.end(); }); }); - test('coalesceUV proximity: ' + a.id + ', ' + b.id, (assert) => { + test('coalesceUV proximity: ' + a.id + ', ' + b.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -637,15 +639,15 @@ test('coalesce args', (assert) => { }], { centerzxy: [2,3,3] }, (err, res) => { - assert.ifError(err); + t.ifError(err, 'no errors'); // sorts by relev, score - assert.deepEqual(res[0].relev, 1, '0.relev'); - assert.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 3, idx: 1, relev: 0.5, score: 1, scoredist: 124.85901539399482, tmpid: 33554435, x: 3, y: 3 }, '0.0'); - assert.deepEqual(res[0][1], { matches_language: true, distance: 2.8284271247461903, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '0.1'); - assert.deepEqual(res[1].relev, 1, '1.relev'); - assert.deepEqual(res[1][0], { matches_language: true, distance: 1.4142135623730951, id: 2, idx: 1, relev: 0.5, score: 7, scoredist: 7, tmpid: 33554434, x: 2, y: 2 }, '1.0'); - assert.deepEqual(res[1][1], { matches_language: true, distance: 2.8284271247461903, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '1.1'); - assert.end(); + t.deepEqual(res[0].relev, 1, '0.relev'); + t.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 3, idx: 1, relev: 0.5, score: 1, scoredist: 124.85901539399482, tmpid: 33554435, x: 3, y: 3 }, '0.0'); + t.deepEqual(res[0][1], { matches_language: true, distance: 2.8284271247461903, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '0.1'); + t.deepEqual(res[1].relev, 1, '1.relev'); + t.deepEqual(res[1][0], { matches_language: true, distance: 1.4142135623730951, id: 2, idx: 1, relev: 0.5, score: 7, scoredist: 7, tmpid: 33554434, x: 2, y: 2 }, '1.0'); + t.deepEqual(res[1][1], { matches_language: true, distance: 2.8284271247461903, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '1.1'); + t.end(); }); }); }); @@ -688,7 +690,7 @@ test('coalesce args', (assert) => { const a = caches[0], b = caches[1]; - test('coalesceMulti, ALL_LANGUAGES: ' + a.id + ', ' + b.id, (assert) => { + test('coalesceMulti, ALL_LANGUAGES: ' + a.id + ', ' + b.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -706,18 +708,18 @@ test('coalesce args', (assert) => { phrase: '1', prefix: false }], {}, (err, res) => { - assert.ifError(err); + t.ifError(err, 'no errors'); // sorts by relev, score - assert.deepEqual(res[0].relev, 1, '0.relev'); - assert.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 2, idx: 1, relev: 0.5, score: 1, scoredist: 1, tmpid: 33554434, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[0][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '0.1'); - assert.deepEqual(res[1].relev, 1, '1.relev'); - assert.deepEqual(res[1][0], { matches_language: true, distance: 0, id: 3, idx: 1, relev: 0.5, score: 1, scoredist: 1, tmpid: 33554435, x: 1, y: 1 }, '1.0'); - assert.deepEqual(res[1][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '1.1'); - assert.end(); + t.deepEqual(res[0].relev, 1, '0.relev'); + t.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 2, idx: 1, relev: 0.5, score: 1, scoredist: 1, tmpid: 33554434, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[0][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '0.1'); + t.deepEqual(res[1].relev, 1, '1.relev'); + t.deepEqual(res[1][0], { matches_language: true, distance: 0, id: 3, idx: 1, relev: 0.5, score: 1, scoredist: 1, tmpid: 33554435, x: 1, y: 1 }, '1.0'); + t.deepEqual(res[1][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '1.1'); + t.end(); }); }); - test('coalesceMulti, [0]: ' + a.id + ', ' + b.id, (assert) => { + test('coalesceMulti, [0]: ' + a.id + ', ' + b.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -736,19 +738,19 @@ test('coalesce args', (assert) => { prefix: false, languages: [0] }], {}, (err, res) => { - assert.ifError(err); + t.ifError(err, 'no errors'); // sorts by relev, score - assert.deepEqual(res[0].relev, 1, '0.relev'); - assert.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 2, idx: 1, relev: 0.5, score: 1, scoredist: 1, tmpid: 33554434, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[0][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '0.1'); + t.deepEqual(res[0].relev, 1, '0.relev'); + t.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 2, idx: 1, relev: 0.5, score: 1, scoredist: 1, tmpid: 33554434, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[0][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '0.1'); // one of our indexes has languages and the other does not, so relev will be 0.95 because it's (.5 + .9*.5) - assert.deepEqual(res[1].relev, 0.95, '1.relev'); - assert.deepEqual(res[1][0], { matches_language: false, distance: 0, id: 3, idx: 1, relev: 0.45, score: 1, scoredist: 1, tmpid: 33554435, x: 1, y: 1 }, '1.0'); - assert.deepEqual(res[1][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '1.1'); - assert.end(); + t.deepEqual(res[1].relev, 0.95, '1.relev'); + t.deepEqual(res[1][0], { matches_language: false, distance: 0, id: 3, idx: 1, relev: 0.45, score: 1, scoredist: 1, tmpid: 33554435, x: 1, y: 1 }, '1.0'); + t.deepEqual(res[1][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '1.1'); + t.end(); }); }); - test('coalesceMulti, [3]: ' + a.id + ', ' + b.id, (assert) => { + test('coalesceMulti, [3]: ' + a.id + ', ' + b.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -767,16 +769,16 @@ test('coalesce args', (assert) => { prefix: false, languages: [3] }], {}, (err, res) => { - assert.ifError(err); + t.ifError(err, 'no errors'); // sorts by relev, score - assert.deepEqual(res[0].relev, 0.95, '0.relev'); - assert.deepEqual(res[0][0], { matches_language: false, distance: 0, id: 2, idx: 1, relev: 0.45, score: 1, scoredist: 1, tmpid: 33554434, x: 1, y: 1 }, '0.0'); - assert.deepEqual(res[0][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '0.1'); + t.deepEqual(res[0].relev, 0.95, '0.relev'); + t.deepEqual(res[0][0], { matches_language: false, distance: 0, id: 2, idx: 1, relev: 0.45, score: 1, scoredist: 1, tmpid: 33554434, x: 1, y: 1 }, '0.0'); + t.deepEqual(res[0][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '0.1'); // one of our indexes has languages and the other does not, so relev will be 0.9 because it's (.5 + .8*.5) - assert.deepEqual(res[1].relev, 0.95, '1.relev'); - assert.deepEqual(res[1][0], { matches_language: false, distance: 0, id: 3, idx: 1, relev: 0.45, score: 1, scoredist: 1, tmpid: 33554435, x: 1, y: 1 }, '1.0'); - assert.deepEqual(res[1][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '1.1'); - assert.end(); + t.deepEqual(res[1].relev, 0.95, '1.relev'); + t.deepEqual(res[1][0], { matches_language: false, distance: 0, id: 3, idx: 1, relev: 0.45, score: 1, scoredist: 1, tmpid: 33554435, x: 1, y: 1 }, '1.0'); + t.deepEqual(res[1][1], { matches_language: true, distance: 0, id: 1, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 1, x: 1, y: 1 }, '1.1'); + t.end(); }); }); }); @@ -817,7 +819,7 @@ test('coalesce args', (assert) => { const a = caches[0], b = caches[1]; - test('coalesce scoredist (close proximity): ' + a.id + ', ' + b.id, (assert) => { + test('coalesce scoredist (close proximity): ' + a.id + ', ' + b.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -837,14 +839,14 @@ test('coalesce args', (assert) => { }], { centerzxy: [14,4601,6200] }, (err, res) => { - assert.ifError(err); - assert.deepEqual(res[0][0].id, 3, 'matches feat 3'); - assert.deepEqual(res[1][0].id, 2, 'matches feat 2'); - assert.deepEqual(res[0][0].distance < res[1][0].distance, true, 'feat 3 is closer than feat2'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res[0][0].id, 3, 'matches feat 3'); + t.deepEqual(res[1][0].id, 2, 'matches feat 2'); + t.deepEqual(res[0][0].distance < res[1][0].distance, true, 'feat 3 is closer than feat2'); + t.end(); }); }); - test('coalesce scoredist (far proximity): ' + a.id + ', ' + b.id, (assert) => { + test('coalesce scoredist (far proximity): ' + a.id + ', ' + b.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -864,11 +866,11 @@ test('coalesce args', (assert) => { }], { centerzxy: [14,4610,6200] }, (err, res) => { - assert.ifError(err); - assert.deepEqual(res[0][0].id, 2, 'matches feat 2 (higher score)'); - assert.deepEqual(res[1][0].id, 3, 'matches feat 3'); - assert.deepEqual(res[1][0].distance < res[0][0].distance, true, 'feat 3 is closer than feat2'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res[0][0].id, 2, 'matches feat 2 (higher score)'); + t.deepEqual(res[1][0].id, 3, 'matches feat 3'); + t.deepEqual(res[1][0].distance < res[0][0].distance, true, 'feat 3 is closer than feat2'); + t.end(); }); }); }); @@ -910,7 +912,7 @@ test('coalesce args', (assert) => { const a = caches[0], b = caches[1]; - test('coalesceMulti (higher relev wins): ' + a.id + ', ' + b.id, (assert) => { + test('coalesceMulti (higher relev wins): ' + a.id + ', ' + b.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -928,14 +930,14 @@ test('coalesce args', (assert) => { phrase: '1', prefix: false }], {}, (err, res) => { - assert.ifError(err); + t.ifError(err, 'no errors'); // sorts by relev, score - assert.deepEqual(res.length, 1, '1 result'); - assert.deepEqual(res[0].relev, 1, '0.relev'); - assert.deepEqual(res[0].length, 2, '0.length'); - assert.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 3, idx: 1, relev: 0.5, score: 1, scoredist: 1, tmpid: 33554435, x: 2, y: 2 }, '0.0'); - assert.deepEqual(res[0][1], { matches_language: true, distance: 0, id: 2, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 2, x: 1, y: 1 }, '0.1'); - assert.end(); + t.deepEqual(res.length, 1, '1 result'); + t.deepEqual(res[0].relev, 1, '0.relev'); + t.deepEqual(res[0].length, 2, '0.length'); + t.deepEqual(res[0][0], { matches_language: true, distance: 0, id: 3, idx: 1, relev: 0.5, score: 1, scoredist: 1, tmpid: 33554435, x: 2, y: 2 }, '0.0'); + t.deepEqual(res[0][1], { matches_language: true, distance: 0, id: 2, idx: 0, relev: 0.5, score: 1, scoredist: 1, tmpid: 2, x: 1, y: 1 }, '0.1'); + t.end(); }); }); }); @@ -1004,7 +1006,7 @@ test('coalesce args', (assert) => { b = caches[1], c = caches[2]; - test('coalesceMulti bbox: ' + a.id + ', ' + b.id + ', ' + c.id, (assert) => { + test('coalesceMulti bbox: ' + a.id + ', ' + b.id + ', ' + c.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -1024,12 +1026,12 @@ test('coalesce args', (assert) => { }], { bboxzxy: [1, 0, 0, 1, 0] }, (err, res) => { - assert.ifError(err); - assert.deepEqual(res.length, 2, '2 results: 1/0/0, 2/3/0'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res.length, 2, '2 results: 1/0/0, 2/3/0'); + t.end(); }); }); - test('coalesceMulti bbox: ' + a.id + ', ' + b.id + ', ' + c.id, (assert) => { + test('coalesceMulti bbox: ' + a.id + ', ' + b.id + ', ' + c.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -1049,12 +1051,12 @@ test('coalesce args', (assert) => { }], { bboxzxy: [2, 0, 0, 1, 3] }, (err, res) => { - assert.ifError(err); - assert.deepEqual(res.length, 2, '2 results: 1/0/0, 2/0/3'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res.length, 2, '2 results: 1/0/0, 2/0/3'); + t.end(); }); }); - test('coalesceMulti bbox: ' + a.id + ', ' + b.id + ', ' + c.id, (assert) => { + test('coalesceMulti bbox: ' + a.id + ', ' + b.id + ', ' + c.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -1074,12 +1076,12 @@ test('coalesce args', (assert) => { }], { bboxzxy: [6, 14, 30, 15, 64] }, (err, res) => { - assert.ifError(err); - assert.deepEqual(res.length, 2, '2 results: 1/0/0, 2/0/3'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res.length, 2, '2 results: 1/0/0, 2/0/3'); + t.end(); }); }); - test('coalesceMulti bbox: ' + a.id + ', ' + b.id + ', ' + c.id, (assert) => { + test('coalesceMulti bbox: ' + a.id + ', ' + b.id + ', ' + c.id, (t) => { coalesce([{ cache: b, mask: 1 << 1, @@ -1099,9 +1101,9 @@ test('coalesce args', (assert) => { }], { bboxzxy: [1, 0, 0, 1, 0] }, (err, res) => { - assert.ifError(err); - assert.deepEqual(res.length, 2, '2 results: 5/20/7, 2/3/0'); - assert.end(); + t.ifError(err, 'no errors'); + t.deepEqual(res.length, 2, '2 results: 5/20/7, 2/3/0'); + t.end(); }); }); }); @@ -1150,7 +1152,7 @@ test('coalesce args', (assert) => { const a = caches[0], b = caches[1]; - test('coalesceMulti sandwich: ' + a.id + ', ' + b.id, (assert) => { + test('coalesceMulti sandwich: ' + a.id + ', ' + b.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -1168,12 +1170,12 @@ test('coalesce args', (assert) => { phrase: '1', prefix: false }], {}, (err, res) => { - assert.ifError(err); - assert.equal(res.length, 2, 'res length = 2'); + t.ifError(err, 'no errors'); + t.equal(res.length, 2, 'res length = 2'); // sorts by relev, score - assert.deepEqual(res[0].map((f) => { return f.id; }), [1,4], '0.relev = 1'); - assert.deepEqual(res[1].map((f) => { return f.id; }), [2,4], '0.relev = 1'); - assert.end(); + t.deepEqual(res[0].map((f) => { return f.id; }), [1,4], '0.relev = 1'); + t.deepEqual(res[1].map((f) => { return f.id; }), [2,4], '0.relev = 1'); + t.end(); }); }); }); @@ -1216,7 +1218,7 @@ test('coalesce args', (assert) => { const a = caches[0], b = caches[1]; - test('coalesceMulti sandwich: ' + a.id + ', ' + b.id, (assert) => { + test('coalesceMulti sandwich: ' + a.id + ', ' + b.id, (t) => { coalesce([{ cache: a, mask: 1 << 1, @@ -1234,11 +1236,11 @@ test('coalesce args', (assert) => { phrase: '1', prefix: false }], {}, (err, res) => { - assert.ifError(err); - assert.equal(res.length, 2, 'res length = 2'); - assert.deepEqual(res[0].map((f) => { return f.id; }), [3,1], '0.relev = 1'); - assert.deepEqual(res[1].map((f) => { return f.id; }), [4,1], '0.relev = 1'); - assert.end(); + t.ifError(err, 'no errors'); + t.equal(res.length, 2, 'res length = 2'); + t.deepEqual(res[0].map((f) => { return f.id; }), [3,1], '0.relev = 1'); + t.deepEqual(res[1].map((f) => { return f.id; }), [4,1], '0.relev = 1'); + t.end(); }); }); }); @@ -1287,8 +1289,8 @@ test('coalesce args', (assert) => { b = caches[1], c = caches[2]; - test('coalesceMulti mask safe: ' + a.id + ', ' + b.id + ', ' + c.id, (assert) => { - assert.comment('start coalesce (mask: 2)'); + test('coalesceMulti mask safe: ' + a.id + ', ' + b.id + ', ' + c.id, (t) => { + t.comment('start coalesce (mask: 2)'); coalesce([{ cache: a, mask: 1 << 2, @@ -1314,15 +1316,15 @@ test('coalesce args', (assert) => { phrase: '1', prefix: false }], {}, (err, res) => { - assert.ifError(err); - assert.equal(res.length, 1, 'res length = 1'); - assert.deepEqual(res[0].map((f) => { return f.id; }), [1, 9999, 1], '0.relev = 0.99'); - assert.end(); + t.ifError(err, 'no errors'); + t.equal(res.length, 1, 'res length = 1'); + t.deepEqual(res[0].map((f) => { return f.id; }), [1, 9999, 1], '0.relev = 0.99'); + t.end(); }); }); - test('coalesceMulti mask overflow: ' + a.id + ', ' + b.id + ', ' + c.id, (assert) => { - assert.comment('start coalesce (mask: 18)'); + test('coalesceMulti mask overflow: ' + a.id + ', ' + b.id + ', ' + c.id, (t) => { + t.comment('start coalesce (mask: 18)'); coalesce([{ cache: a, mask: 1 << 18, @@ -1348,10 +1350,10 @@ test('coalesce args', (assert) => { phrase: '1', prefix: false }], {}, (err, res) => { - assert.ifError(err); - assert.equal(res.length, 1, 'res length = 1'); - assert.deepEqual(res[0].map((f) => { return f.id; }), [1, 9999, 1], '0.relev = 0.99'); - assert.end(); + t.ifError(err, 'no errors'); + t.equal(res.length, 1, 'res length = 1'); + t.deepEqual(res[0].map((f) => { return f.id; }), [1, 9999, 1], '0.relev = 0.99'); + t.end(); }); }); }); diff --git a/test/grid.js b/test/grid.js index d2e085d..75e7d8f 100644 --- a/test/grid.js +++ b/test/grid.js @@ -9,6 +9,9 @@ const mp14 = Math.pow(2,14); module.exports.encode = encode; module.exports.decode = decode; +// Utility functions used in coalesce. We need these functions in JS, whic is why they are in this folder. +// These functions are copied over from /carmen and /carmen does test them. + function encode(grid) { if (grid.id >= mp20) throw new Error('id must be < 2^20'); if (grid.x >= mp14) throw new Error('x must be < 2^14'); diff --git a/test/install_node.sh b/test/install_node.sh index 1ae9fe1..771d0a4 100755 --- a/test/install_node.sh +++ b/test/install_node.sh @@ -1,3 +1,4 @@ +# Travis uses this file to install Node in order to run tests in JS # here we set up the node version on the fly based on the matrix value. # This is done manually so that the build works the same on OS X rm -rf ~/.nvm/ && git clone --depth 1 https://github.com/creationix/nvm.git ~/.nvm diff --git a/test/langfield.fuzz.test.js b/test/langfield.fuzz.test.js index 3c9f70a..1d19488 100644 --- a/test/langfield.fuzz.test.js +++ b/test/langfield.fuzz.test.js @@ -1,14 +1,16 @@ 'use strict'; const carmenCache = require('../index.js'); -const tape = require('tape'); +const test = require('tape'); const fs = require('fs'); +// Checks the language field setting operation is working + const tmpdir = '/tmp/temp.' + Math.random().toString(36).substr(2, 5); fs.mkdirSync(tmpdir); let tmpidx = 0; const tmpfile = function() { return tmpdir + '/' + (tmpidx++) + '.dat'; }; -tape('language fuzzing', (assert) => { +test('language fuzzing', (t) => { const cache = new carmenCache.MemoryCache('a'); const records = new Map(); @@ -32,26 +34,26 @@ tape('language fuzzing', (assert) => { } let list = cache.list(); - assert.equal(list.length, records.size, 'got the same number of items out as went in'); + t.equal(list.length, records.size, 'got the same number of items out as went in'); let hasAll = true; for (const item of list) { const recordId = item[0] + '-' + (item[1] == null ? 'null' : item[1].join('-')); hasAll = hasAll && records.has(recordId); } - assert.ok(hasAll, 'all records and languages came out that went in'); + t.ok(hasAll, 'all records and languages came out that went in'); const pack = tmpfile(); cache.pack(pack); const loader = new carmenCache.RocksDBCache('b', pack); list = loader.list(); - assert.equal(list.length, records.size, 'got the same number of items out as went in'); + t.equal(list.length, records.size, 'got the same number of items out as went in'); hasAll = true; for (const item of list) { const recordId = item[0] + '-' + (item[1] == null ? 'null' : item[1].join('-')); hasAll = hasAll && records.has(recordId); } - assert.ok(hasAll, 'all records and languages came out that went in'); + t.ok(hasAll, 'all records and languages came out that went in'); - assert.end(); + t.end(); }); diff --git a/test/matching.test.js b/test/matching.test.js index 39055aa..53fdced 100644 --- a/test/matching.test.js +++ b/test/matching.test.js @@ -1,9 +1,12 @@ 'use strict'; const carmenCache = require('../index.js'); -const tape = require('tape'); +const test = require('tape'); const fs = require('fs'); const Grid = require('./grid.js'); +// Check API.md documentation for an explanationon getmatching operation. +// It is identical for both MemoryCache and RocksDBCache. + const tmpdir = '/tmp/temp.' + Math.random().toString(36).substr(2, 5); fs.mkdirSync(tmpdir); let tmpidx = 0; @@ -17,7 +20,7 @@ const getByLanguageMatch = function(grids, match) { return grids.filter((x) => { return x.matches_language === match; }); }; -tape('getMatching', (assert) => { +test('getMatching', (t) => { const cache = new carmenCache.MemoryCache('mem'); cache._set('test', [ @@ -70,38 +73,38 @@ tape('getMatching', (assert) => { [cache, loader].forEach((c) => { const test_all_langs_no_prefix = c._getMatching('test', false); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_no_prefix), [1, 2, 3, 11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43], "getMatching for 'test' with no prefix match and no language includes all IDs for 'test'" ); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_no_prefix), getIds(getByLanguageMatch(test_all_langs_no_prefix, true)), "getMatching for 'test' with no prefix match and no language includes only match_language: true" ); const test_all_langs_with_prefix = c._getMatching('test', true); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix), [1, 2, 3, 11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43, 51, 52, 53], "getMatching for 'test' with prefix match and no language includes all IDs for 'test' and 'testy'" ); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix), getIds(getByLanguageMatch(test_all_langs_with_prefix, true)), "getMatching for 'test' with prefix match and no language includes only match_language: true" ); - assert.false(c._getMatching('te', false), "getMatching for 'te' with no prefix match returns nothing"); + t.false(c._getMatching('te', false), "getMatching for 'te' with no prefix match returns nothing"); const te_all_langs_with_prefix = c._getMatching('te', true); - assert.deepEqual( + t.deepEqual( getIds(te_all_langs_with_prefix), [1, 2, 3, 11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43, 51, 52, 53, 61, 62, 63], "getMatching for 'te' with prefix match and no language includes all IDs for 'test' and 'testy' and 'tentacle'" ); - assert.deepEqual( + t.deepEqual( getIds(te_all_langs_with_prefix), getIds(getByLanguageMatch(te_all_langs_with_prefix, true)), "getMatching for 'te' with prefix match and no language includes only match_language: true" @@ -110,22 +113,22 @@ tape('getMatching', (assert) => { const test_all_langs_with_prefix_0 = c._getMatching('test', true, [0]); const test_all_langs_with_prefix_0_matched = getByLanguageMatch(test_all_langs_with_prefix_0, true); const test_all_langs_with_prefix_0_unmatched = getByLanguageMatch(test_all_langs_with_prefix_0, false); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix_0), [1, 2, 3, 11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43, 51, 52, 53], "getMatching for 'test' with prefix match and language [0] includes all IDs for 'test' and 'testy'" ); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix_0_matched), [1, 2, 3, 11, 12, 13, 51, 52, 53], "getMatching for 'test' with prefix match and language [0] includes language_match: true for IDs for 'test' and 'testy' with language 0" ); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix_0_unmatched), [21, 22, 23, 31, 32, 33, 41, 42, 43], "getMatching for 'test' with prefix match and language [0] includes language_match: false for IDs for 'test' and 'testy' without language 0" ); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix_0.slice(0, test_all_langs_with_prefix_0_matched.length)), getIds(test_all_langs_with_prefix_0_matched), 'all the language-matching results come first' @@ -134,22 +137,22 @@ tape('getMatching', (assert) => { const te_all_langs_with_prefix_0 = c._getMatching('te', true, [0]); const te_all_langs_with_prefix_0_matched = getByLanguageMatch(te_all_langs_with_prefix_0, true); const te_all_langs_with_prefix_0_unmatched = getByLanguageMatch(te_all_langs_with_prefix_0, false); - assert.deepEqual( + t.deepEqual( getIds(te_all_langs_with_prefix_0), [1, 2, 3, 11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43, 51, 52, 53, 61, 62, 63], "getMatching for 'te' with prefix match and language [0] includes all IDs for 'tentacle' and 'test' and 'testy'" ); - assert.deepEqual( + t.deepEqual( getIds(te_all_langs_with_prefix_0_matched), [1, 2, 3, 11, 12, 13, 51, 52, 53, 61, 62, 63], "getMatching for 'te' with prefix match and language [0] includes language_match: true for IDs for 'tentacle' and 'test' and 'testy' with language 0" ); - assert.deepEqual( + t.deepEqual( getIds(te_all_langs_with_prefix_0_unmatched), [21, 22, 23, 31, 32, 33, 41, 42, 43], "getMatching for 'te' with prefix match and language [0] includes language_match: false for IDs for 'tentacle' and 'test' and 'testy' without language 0" ); - assert.deepEqual( + t.deepEqual( getIds(te_all_langs_with_prefix_0.slice(0, te_all_langs_with_prefix_0_matched.length)), getIds(te_all_langs_with_prefix_0_matched), 'all the language-matching results come first' @@ -158,22 +161,22 @@ tape('getMatching', (assert) => { const test_all_langs_with_prefix_1 = c._getMatching('test', true, [1]); const test_all_langs_with_prefix_1_matched = getByLanguageMatch(test_all_langs_with_prefix_1, true); const test_all_langs_with_prefix_1_unmatched = getByLanguageMatch(test_all_langs_with_prefix_1, false); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix_1), [1, 2, 3, 11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43, 51, 52, 53], "getMatching for 'test' with prefix match and language [1] includes all IDs for 'test' and 'testy'" ); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix_1_matched), [1, 2, 3, 21, 22, 23, 31, 32, 33], "getMatching for 'test' with prefix match and language [1] includes language_match: true for IDs for 'test' and 'testy' for both language 1 and language 0,1" ); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix_1_unmatched), [11, 12, 13, 41, 42, 43, 51, 52, 53], "getMatching for 'test' with prefix match and language [1] includes language_match: false for IDs for 'test' and 'testy' without language 1" ); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix_1.slice(0, test_all_langs_with_prefix_1_matched.length)), getIds(test_all_langs_with_prefix_1_matched), 'all the language-matching results come first' @@ -182,28 +185,28 @@ tape('getMatching', (assert) => { const test_all_langs_with_prefix_7 = c._getMatching('test', true, [7]); const test_all_langs_with_prefix_7_matched = getByLanguageMatch(test_all_langs_with_prefix_7, true); const test_all_langs_with_prefix_7_unmatched = getByLanguageMatch(test_all_langs_with_prefix_7, false); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix_7), [1, 2, 3, 11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43, 51, 52, 53], "getMatching for 'test' with prefix match and language [7] includes all IDs for 'test' and 'testy'" ); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix_7_matched), [1, 2, 3], "getMatching for 'test' with prefix match and language [7] includes only language_match: true for IDs for 'test' and 'testy' with no language set" ); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix_7_unmatched), [11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43, 51, 52, 53], "getMatching for 'test' with prefix match and language [7] includes language_match: false for all IDs for 'test' and 'testy' with a language set" ); - assert.deepEqual( + t.deepEqual( getIds(test_all_langs_with_prefix_7.slice(0, test_all_langs_with_prefix_7_matched.length)), getIds(test_all_langs_with_prefix_7_matched), 'all the language-matching results come first' ); }); - // assert.deepEqual(loader.list('grid'), [ 'else.', 'something', 'test', 'test.' ], 'keys in shard'); - assert.end(); + // t.deepEqual(loader.list('grid'), [ 'else.', 'something', 'test', 'test.' ], 'keys in shard'); + t.end(); }); diff --git a/test/merge.test.js b/test/merge.test.js index b5de16e..f706b0e 100644 --- a/test/merge.test.js +++ b/test/merge.test.js @@ -2,7 +2,7 @@ const carmenCache = require('../index.js'); const Cache = carmenCache.MemoryCache; const RocksDBCache = carmenCache.RocksDBCache; -const tape = require('tape'); +const test = require('tape'); const fs = require('fs'); const tmpdir = '/tmp/temp.' + Math.random().toString(36).substr(2, 5); @@ -10,7 +10,10 @@ fs.mkdirSync(tmpdir); let tmpidx = 0; const tmpfile = function() { return tmpdir + '/' + (tmpidx++) + '.dat'; }; -tape('#merge concat', (assert) => { +// packs each MemoryCaches into a RocksDBCache, then merges all into a single RocksDBCache + +// merges grid caches +test('#merge concat', (t) => { const cacheA = new Cache('a'); cacheA._set('....1', [0,1,2,3]); cacheA._set('....2', [0,1,2,3]); @@ -26,16 +29,17 @@ tape('#merge concat', (assert) => { const merged = tmpfile(); RocksDBCache.merge(pbfA, pbfB, merged, 'concat', (err) => { - assert.ifError(err); + t.ifError(err, 'no errors'); const cacheC = new RocksDBCache('c', merged); - assert.deepEqual(cacheC._get('....2').sort(numSort), [0,1,2,3], 'a-only'); - assert.deepEqual(cacheC._get('....3').sort(numSort), [10,11,12,13], 'b-only'); - assert.deepEqual(cacheC._get('....1').sort(numSort), [0,1,2,3,10,11,12,13], 'a-b-merged'); - assert.end(); + t.deepEqual(cacheC._get('....2').sort(numSort), [0,1,2,3], 'a-only'); + t.deepEqual(cacheC._get('....3').sort(numSort), [10,11,12,13], 'b-only'); + t.deepEqual(cacheC._get('....1').sort(numSort), [0,1,2,3,10,11,12,13], 'a-b-merged'); + t.end(); }); }); -tape('#merge freq', (assert) => { +// merges frequency caches: used at index time to decide which words are important/not important +test('#merge freq', (t) => { const cacheA = new Cache('a'); // these would not ordinarily all be in the same shard, but force them to be // in the same shard to make this test actually work @@ -55,13 +59,13 @@ tape('#merge freq', (assert) => { const merged = tmpfile(); RocksDBCache.merge(pbfA, pbfB, merged, 'freq', (err) => { - assert.ifError(err); + t.ifError(err, 'no errors'); const cacheC = new RocksDBCache('c', merged); - assert.deepEqual(cacheC._get('__MAX__').sort(numSort), [2], 'a-b-max'); - assert.deepEqual(cacheC._get('__COUNT__').sort(numSort), [3], 'a-b-sum'); - assert.deepEqual(cacheC._get('3').sort(numSort), [1], 'a-only'); - assert.deepEqual(cacheC._get('4').sort(numSort), [2], 'b-only'); - assert.end(); + t.deepEqual(cacheC._get('__MAX__').sort(numSort), [2], 'a-b-max'); + t.deepEqual(cacheC._get('__COUNT__').sort(numSort), [3], 'a-b-sum'); + t.deepEqual(cacheC._get('3').sort(numSort), [1], 'a-only'); + t.deepEqual(cacheC._get('4').sort(numSort), [2], 'b-only'); + t.end(); }); }); diff --git a/test/normalize.test.js b/test/normalize.test.js index 52aeaa2..f4300fd 100644 --- a/test/normalize.test.js +++ b/test/normalize.test.js @@ -1,6 +1,6 @@ 'use strict'; const carmenCache = require('../index.js'); -const tape = require('tape'); +const test = require('tape'); const fs = require('fs'); const tmpdir = '/tmp/temp.' + Math.random().toString(36).substr(2, 5); @@ -31,7 +31,8 @@ const norm = { const file = tmpfile(); -tape('write/dump', (assert) => { +// test creating the cache and writing it to disk +test('write/dump', (t) => { const cache = new carmenCache.NormalizationCache(file, false); const map = []; @@ -43,7 +44,7 @@ tape('write/dump', (assert) => { // These tests just illustrate what the mapping actually is storing. // Note the mapping is based on index position of sorted text based // on how dawg-cache stores. - assert.deepEqual(words, [ + t.deepEqual(words, [ '1st st', 'apple lane', 'buerbarg', @@ -55,30 +56,31 @@ tape('write/dump', (assert) => { 'pear ave', 'pear avenue' ], 'confirm map sorted order simulating dawg text order'); - assert.deepEqual(map[0], [3, [2, 3]], 'burbarg => [buerbarg, burbarg]'); - assert.deepEqual(map[1], [4, [0]], 'first street => 1st st'); - assert.deepEqual(map[2], [6, [5]], 'frank boulevard => frank blvd'); - assert.deepEqual(map[3], [9, [8]], 'pear avenue => pear ave'); + t.deepEqual(map[0], [3, [2, 3]], 'burbarg => [buerbarg, burbarg]'); + t.deepEqual(map[1], [4, [0]], 'first street => 1st st'); + t.deepEqual(map[2], [6, [5]], 'frank boulevard => frank blvd'); + t.deepEqual(map[3], [9, [8]], 'pear avenue => pear ave'); cache.writeBatch(map); - assert.deepEqual(map, cache.getAll(), 'dumped contents match input'); + t.deepEqual(map, cache.getAll(), 'dumped contents match input'); // test some invalid input - assert.throws(() => { cache.writeBatch(); }); - assert.throws(() => { cache.writeBatch(7); }); - assert.throws(() => { cache.writeBatch([7]); }); - assert.throws(() => { cache.writeBatch([[7]]); }); - assert.throws(() => { cache.writeBatch([[7, 'asdf']]); }); + t.throws(() => { cache.writeBatch(); }, 'throws on invalid arguments'); + t.throws(() => { cache.writeBatch(7); }, 'throws on invalid arguments'); + t.throws(() => { cache.writeBatch([7]); }, 'throws on invalid arguments'); + t.throws(() => { cache.writeBatch([[7]]); }, 'throws on invalid arguments'); + t.throws(() => { cache.writeBatch([[7, 'asdf']]); }, 'throws on invalid arguments'); - return assert.end(); + return t.end(); }); -tape('read', (assert) => { +// tests reading cache from disk +test('read', (t) => { const cache = new carmenCache.NormalizationCache(file, true); - assert.deepEqual(cache.get(words.indexOf('first street')), [words.indexOf('1st st')]); - assert.equal(cache.get(8888), undefined); + t.deepEqual(cache.get(words.indexOf('first street')), [words.indexOf('1st st')], 'normalization value for "first street" is as expected'); + t.equal(cache.get(8888), undefined, 'returns no value for an index not in the cache'); const firstWithPrefix = function(p) { for (let i = 0; i < words.length; i++) if (words[i].startsWith(p)) return i; @@ -90,32 +92,32 @@ tape('read', (assert) => { return c; }; - assert.deepEqual(cache.getPrefixRange(firstWithPrefix('f'), countWithPrefix('f')), [words.indexOf('1st st')], 'found normalization for 1st st but not frank boulevard'); - assert.deepEqual(cache.getPrefixRange(firstWithPrefix('frank'), countWithPrefix('frank')), [], 'found nothing because all normalizations share the searched prefix'); - assert.deepEqual(cache.getPrefixRange(firstWithPrefix('frank bo'), countWithPrefix('frank bo')), [words.indexOf('frank blvd')], 'found frank boulevard because no prefixes are shared'); + t.deepEqual(cache.getPrefixRange(firstWithPrefix('f'), countWithPrefix('f')), [words.indexOf('1st st')], 'found normalization for 1st st but not frank boulevard'); + t.deepEqual(cache.getPrefixRange(firstWithPrefix('frank'), countWithPrefix('frank')), [], 'found nothing because all normalizations share the searched prefix'); + t.deepEqual(cache.getPrefixRange(firstWithPrefix('frank bo'), countWithPrefix('frank bo')), [words.indexOf('frank blvd')], 'found frank boulevard because no prefixes are shared'); - assert.deepEqual(cache.getPrefixRange(firstWithPrefix('bu'), countWithPrefix('bu')), [], 'found nothing because all normalizations share the searched prefix'); - assert.deepEqual(cache.getPrefixRange(firstWithPrefix('bue'), countWithPrefix('bue')), [], 'found nothing because bue... doesn\'t normalize to anything'); - assert.deepEqual(cache.getPrefixRange(firstWithPrefix('bur'), countWithPrefix('bur')), [words.indexOf('buerbarg')], 'found buerbarg but not burbarg because burbarg shares a prefix with itself'); - assert.deepEqual(cache.get(firstWithPrefix('bur')), [words.indexOf('buerbarg'), words.indexOf('burbarg')], 'found buerbarg and burbarg with regular get because nothing gets filtered'); + t.deepEqual(cache.getPrefixRange(firstWithPrefix('bu'), countWithPrefix('bu')), [], 'found nothing because all normalizations share the searched prefix'); + t.deepEqual(cache.getPrefixRange(firstWithPrefix('bue'), countWithPrefix('bue')), [], 'found nothing because bue... doesn\'t normalize to anything'); + t.deepEqual(cache.getPrefixRange(firstWithPrefix('bur'), countWithPrefix('bur')), [words.indexOf('buerbarg')], 'found buerbarg but not burbarg because burbarg shares a prefix with itself'); + t.deepEqual(cache.get(firstWithPrefix('bur')), [words.indexOf('buerbarg'), words.indexOf('burbarg')], 'found buerbarg and burbarg with regular get because nothing gets filtered'); // test some invalid input - assert.throws(() => { new carmenCache.NormalizationCache(); }); - assert.throws(() => { new carmenCache.NormalizationCache(7); }); - assert.throws(() => { new carmenCache.NormalizationCache('asdf', 7); }); + t.throws(() => { new carmenCache.NormalizationCache(); }, 'throws on invalid arguments'); + t.throws(() => { new carmenCache.NormalizationCache(7); }, 'throws on invalid arguments'); + t.throws(() => { new carmenCache.NormalizationCache('asdf', 7); }, 'throws on invalid arguments'); - assert.throws(() => { new carmenCache.NormalizationCache('/proc', true); }); + t.throws(() => { new carmenCache.NormalizationCache('/proc', true); }, 'throws on invalid arguments'); - assert.throws(() => { cache.get(); }); - assert.throws(() => { cache.getPrefixRange('asdf'); }); + t.throws(() => { cache.get(); }, 'throws on invalid arguments'); + t.throws(() => { cache.getPrefixRange('asdf'); }, 'throws on invalid arguments'); - assert.throws(() => { cache.getPrefixRange(); }); - assert.throws(() => { cache.getPrefixRange('asdf'); }); - assert.throws(() => { cache.getPrefixRange('asdf', 'asdf'); }); - assert.throws(() => { cache.getPrefixRange(1, 'asdf'); }); - assert.throws(() => { cache.getPrefixRange(1, 1, 'asdf'); }); - assert.throws(() => { cache.getPrefixRange(1, 1, 1, 'asdf'); }); + t.throws(() => { cache.getPrefixRange(); }, 'throws on invalid arguments'); + t.throws(() => { cache.getPrefixRange('asdf'); }, 'throws on invalid arguments'); + t.throws(() => { cache.getPrefixRange('asdf', 'asdf'); }, 'throws on invalid arguments'); + t.throws(() => { cache.getPrefixRange(1, 'asdf'); }, 'throws on invalid arguments'); + t.throws(() => { cache.getPrefixRange(1, 1, 'asdf'); }, 'throws on invalid arguments'); + t.throws(() => { cache.getPrefixRange(1, 1, 1, 'asdf'); }, 'throws on invalid arguments'); - assert.end(); + t.end(); }); diff --git a/yarn.lock b/yarn.lock index fc78598..18a4389 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6,6 +6,13 @@ version "1.0.1" resolved "https://registry.yarnpkg.com/@mapbox/eslint-config-geocoding/-/eslint-config-geocoding-1.0.1.tgz#3b6552881408b6dc10560de5e2701beaa44a69cd" +JSONStream@^1.0.3: + version "1.3.2" + resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.2.tgz#c102371b6ec3a7cf3b847ca00c20bb0fce4c6dea" + dependencies: + jsonparse "^1.2.0" + through ">=2.2.7 <3" + abbrev@1: version "1.1.1" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" @@ -20,7 +27,7 @@ acorn@^3.0.4: version "3.3.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-3.3.0.tgz#45e37fb39e8da3f25baee3ff5369e2bb5f22017a" -acorn@^5.4.0: +acorn@^5.2.1, acorn@^5.4.0: version "5.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.4.1.tgz#fdc58d9d17f4a4e98d102ded826a9b9759125102" @@ -48,6 +55,10 @@ ansi-escapes@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-3.0.0.tgz#ec3e8b4e9f8064fc02c3ac9b65f1c275bda8ef92" +ansi-html@^0.0.7: + version "0.0.7" + resolved "https://registry.yarnpkg.com/ansi-html/-/ansi-html-0.0.7.tgz#813584021962a9e9e6fd039f940d12f56ca7859e" + ansi-regex@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" @@ -56,16 +67,23 @@ ansi-regex@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998" -ansi-styles@^2.2.1: +ansi-styles@^2.0.1, ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" -ansi-styles@^3.1.0: +ansi-styles@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.0.tgz#c159b8d5be0f9e5a6f346dab94f16ce022161b88" dependencies: color-convert "^1.9.0" +anymatch@^1.3.0: + version "1.3.2" + resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-1.3.2.tgz#553dcb8f91e3c889845dfdba34c77721b90b9d7a" + dependencies: + micromatch "^2.1.5" + normalize-path "^2.0.0" + aproba@^1.0.3: version "1.2.0" resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" @@ -78,11 +96,37 @@ are-we-there-yet@~1.1.2: readable-stream "^2.0.6" argparse@^1.0.7: - version "1.0.9" - resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.9.tgz#73d83bc263f86e97f8cc4f6bae1b0e90a7d22c86" + version "1.0.10" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" dependencies: sprintf-js "~1.0.2" +arr-diff@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-2.0.0.tgz#8f3b827f955a8bd669697e4a4256ac3ceae356cf" + dependencies: + arr-flatten "^1.0.1" + +arr-diff@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-4.0.0.tgz#d6461074febfec71e7e15235761a329a5dc7c520" + +arr-flatten@^1.0.1, arr-flatten@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/arr-flatten/-/arr-flatten-1.1.0.tgz#36048bbff4e7b47e136644316c99669ea5ae91f1" + +arr-union@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/arr-union/-/arr-union-3.1.0.tgz#e39b09aea9def866a8f206e288af63919bae39c4" + +array-differ@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/array-differ/-/array-differ-1.0.0.tgz#eff52e3758249d33be402b8bb8e564bb2b5d4031" + +array-iterate@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/array-iterate/-/array-iterate-1.1.1.tgz#865bf7f8af39d6b0982c60902914ac76bc0108f6" + array-union@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/array-union/-/array-union-1.0.2.tgz#9a34410e4f4e3da23dea375be5be70f24778ec39" @@ -93,6 +137,14 @@ array-uniq@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/array-uniq/-/array-uniq-1.0.3.tgz#af6ac877a25cc7f74e058894753858dfdb24fdb6" +array-unique@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.2.1.tgz#a1d97ccafcbc2625cc70fadceb36a50c58b01a53" + +array-unique@^0.3.2: + version "0.3.2" + resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" + arrify@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d" @@ -109,13 +161,25 @@ assert-plus@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-0.2.0.tgz#d74e1b87e7affc0db8aadb7021f3fe48101ab234" +assign-symbols@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367" + +async-each@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d" + asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" +atob@^2.0.0: + version "2.0.3" + resolved "https://registry.yarnpkg.com/atob/-/atob-2.0.3.tgz#19c7a760473774468f20b2d2d03372ad7d4cbf5d" + aws-sdk@^2.55.0: - version "2.189.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.189.0.tgz#4df649b8cfa2008fee09f357fe3a9835465c4da4" + version "2.200.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.200.0.tgz#f460c96408725b0eb8c658fddea6e0bfe0ef5a44" dependencies: buffer "4.9.1" events "^1.1.1" @@ -135,7 +199,7 @@ aws4@^1.2.1: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" -babel-code-frame@^6.22.0: +babel-code-frame@^6.22.0, babel-code-frame@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b" dependencies: @@ -143,13 +207,726 @@ babel-code-frame@^6.22.0: esutils "^2.0.2" js-tokens "^3.0.2" +babel-core@^6.0.14, babel-core@^6.17.0, babel-core@^6.26.0: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-core/-/babel-core-6.26.0.tgz#af32f78b31a6fcef119c87b0fd8d9753f03a0bb8" + dependencies: + babel-code-frame "^6.26.0" + babel-generator "^6.26.0" + babel-helpers "^6.24.1" + babel-messages "^6.23.0" + babel-register "^6.26.0" + babel-runtime "^6.26.0" + babel-template "^6.26.0" + babel-traverse "^6.26.0" + babel-types "^6.26.0" + babylon "^6.18.0" + convert-source-map "^1.5.0" + debug "^2.6.8" + json5 "^0.5.1" + lodash "^4.17.4" + minimatch "^3.0.4" + path-is-absolute "^1.0.1" + private "^0.1.7" + slash "^1.0.0" + source-map "^0.5.6" + +babel-generator@6.25.0: + version "6.25.0" + resolved "https://registry.yarnpkg.com/babel-generator/-/babel-generator-6.25.0.tgz#33a1af70d5f2890aeb465a4a7793c1df6a9ea9fc" + dependencies: + babel-messages "^6.23.0" + babel-runtime "^6.22.0" + babel-types "^6.25.0" + detect-indent "^4.0.0" + jsesc "^1.3.0" + lodash "^4.2.0" + source-map "^0.5.0" + trim-right "^1.0.1" + +babel-generator@^6.26.0: + version "6.26.1" + resolved "https://registry.yarnpkg.com/babel-generator/-/babel-generator-6.26.1.tgz#1844408d3b8f0d35a404ea7ac180f087a601bd90" + dependencies: + babel-messages "^6.23.0" + babel-runtime "^6.26.0" + babel-types "^6.26.0" + detect-indent "^4.0.0" + jsesc "^1.3.0" + lodash "^4.17.4" + source-map "^0.5.7" + trim-right "^1.0.1" + +babel-helper-bindify-decorators@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-helper-bindify-decorators/-/babel-helper-bindify-decorators-6.24.1.tgz#14c19e5f142d7b47f19a52431e52b1ccbc40a330" + dependencies: + babel-runtime "^6.22.0" + babel-traverse "^6.24.1" + babel-types "^6.24.1" + +babel-helper-builder-binary-assignment-operator-visitor@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-helper-builder-binary-assignment-operator-visitor/-/babel-helper-builder-binary-assignment-operator-visitor-6.24.1.tgz#cce4517ada356f4220bcae8a02c2b346f9a56664" + dependencies: + babel-helper-explode-assignable-expression "^6.24.1" + babel-runtime "^6.22.0" + babel-types "^6.24.1" + +babel-helper-builder-react-jsx@^6.24.1: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-helper-builder-react-jsx/-/babel-helper-builder-react-jsx-6.26.0.tgz#39ff8313b75c8b65dceff1f31d383e0ff2a408a0" + dependencies: + babel-runtime "^6.26.0" + babel-types "^6.26.0" + esutils "^2.0.2" + +babel-helper-call-delegate@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-helper-call-delegate/-/babel-helper-call-delegate-6.24.1.tgz#ece6aacddc76e41c3461f88bfc575bd0daa2df8d" + dependencies: + babel-helper-hoist-variables "^6.24.1" + babel-runtime "^6.22.0" + babel-traverse "^6.24.1" + babel-types "^6.24.1" + +babel-helper-define-map@^6.24.1: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-helper-define-map/-/babel-helper-define-map-6.26.0.tgz#a5f56dab41a25f97ecb498c7ebaca9819f95be5f" + dependencies: + babel-helper-function-name "^6.24.1" + babel-runtime "^6.26.0" + babel-types "^6.26.0" + lodash "^4.17.4" + +babel-helper-explode-assignable-expression@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-helper-explode-assignable-expression/-/babel-helper-explode-assignable-expression-6.24.1.tgz#f25b82cf7dc10433c55f70592d5746400ac22caa" + dependencies: + babel-runtime "^6.22.0" + babel-traverse "^6.24.1" + babel-types "^6.24.1" + +babel-helper-explode-class@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-helper-explode-class/-/babel-helper-explode-class-6.24.1.tgz#7dc2a3910dee007056e1e31d640ced3d54eaa9eb" + dependencies: + babel-helper-bindify-decorators "^6.24.1" + babel-runtime "^6.22.0" + babel-traverse "^6.24.1" + babel-types "^6.24.1" + +babel-helper-function-name@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-helper-function-name/-/babel-helper-function-name-6.24.1.tgz#d3475b8c03ed98242a25b48351ab18399d3580a9" + dependencies: + babel-helper-get-function-arity "^6.24.1" + babel-runtime "^6.22.0" + babel-template "^6.24.1" + babel-traverse "^6.24.1" + babel-types "^6.24.1" + +babel-helper-get-function-arity@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-helper-get-function-arity/-/babel-helper-get-function-arity-6.24.1.tgz#8f7782aa93407c41d3aa50908f89b031b1b6853d" + dependencies: + babel-runtime "^6.22.0" + babel-types "^6.24.1" + +babel-helper-hoist-variables@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-helper-hoist-variables/-/babel-helper-hoist-variables-6.24.1.tgz#1ecb27689c9d25513eadbc9914a73f5408be7a76" + dependencies: + babel-runtime "^6.22.0" + babel-types "^6.24.1" + +babel-helper-optimise-call-expression@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-helper-optimise-call-expression/-/babel-helper-optimise-call-expression-6.24.1.tgz#f7a13427ba9f73f8f4fa993c54a97882d1244257" + dependencies: + babel-runtime "^6.22.0" + babel-types "^6.24.1" + +babel-helper-regex@^6.24.1: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-helper-regex/-/babel-helper-regex-6.26.0.tgz#325c59f902f82f24b74faceed0363954f6495e72" + dependencies: + babel-runtime "^6.26.0" + babel-types "^6.26.0" + lodash "^4.17.4" + +babel-helper-remap-async-to-generator@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-helper-remap-async-to-generator/-/babel-helper-remap-async-to-generator-6.24.1.tgz#5ec581827ad723fecdd381f1c928390676e4551b" + dependencies: + babel-helper-function-name "^6.24.1" + babel-runtime "^6.22.0" + babel-template "^6.24.1" + babel-traverse "^6.24.1" + babel-types "^6.24.1" + +babel-helper-replace-supers@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-helper-replace-supers/-/babel-helper-replace-supers-6.24.1.tgz#bf6dbfe43938d17369a213ca8a8bf74b6a90ab1a" + dependencies: + babel-helper-optimise-call-expression "^6.24.1" + babel-messages "^6.23.0" + babel-runtime "^6.22.0" + babel-template "^6.24.1" + babel-traverse "^6.24.1" + babel-types "^6.24.1" + +babel-helpers@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-helpers/-/babel-helpers-6.24.1.tgz#3471de9caec388e5c850e597e58a26ddf37602b2" + dependencies: + babel-runtime "^6.22.0" + babel-template "^6.24.1" + +babel-messages@^6.23.0: + version "6.23.0" + resolved "https://registry.yarnpkg.com/babel-messages/-/babel-messages-6.23.0.tgz#f3cdf4703858035b2a2951c6ec5edf6c62f2630e" + dependencies: + babel-runtime "^6.22.0" + +babel-plugin-check-es2015-constants@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-check-es2015-constants/-/babel-plugin-check-es2015-constants-6.22.0.tgz#35157b101426fd2ffd3da3f75c7d1e91835bbf8a" + dependencies: + babel-runtime "^6.22.0" + +babel-plugin-syntax-async-functions@^6.8.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-async-functions/-/babel-plugin-syntax-async-functions-6.13.0.tgz#cad9cad1191b5ad634bf30ae0872391e0647be95" + +babel-plugin-syntax-async-generators@^6.5.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-async-generators/-/babel-plugin-syntax-async-generators-6.13.0.tgz#6bc963ebb16eccbae6b92b596eb7f35c342a8b9a" + +babel-plugin-syntax-class-constructor-call@^6.18.0: + version "6.18.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-class-constructor-call/-/babel-plugin-syntax-class-constructor-call-6.18.0.tgz#9cb9d39fe43c8600bec8146456ddcbd4e1a76416" + +babel-plugin-syntax-class-properties@^6.8.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-class-properties/-/babel-plugin-syntax-class-properties-6.13.0.tgz#d7eb23b79a317f8543962c505b827c7d6cac27de" + +babel-plugin-syntax-decorators@^6.1.18, babel-plugin-syntax-decorators@^6.13.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-decorators/-/babel-plugin-syntax-decorators-6.13.0.tgz#312563b4dbde3cc806cee3e416cceeaddd11ac0b" + +babel-plugin-syntax-do-expressions@^6.8.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-do-expressions/-/babel-plugin-syntax-do-expressions-6.13.0.tgz#5747756139aa26d390d09410b03744ba07e4796d" + +babel-plugin-syntax-dynamic-import@^6.18.0: + version "6.18.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-dynamic-import/-/babel-plugin-syntax-dynamic-import-6.18.0.tgz#8d6a26229c83745a9982a441051572caa179b1da" + +babel-plugin-syntax-exponentiation-operator@^6.8.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-exponentiation-operator/-/babel-plugin-syntax-exponentiation-operator-6.13.0.tgz#9ee7e8337290da95288201a6a57f4170317830de" + +babel-plugin-syntax-export-extensions@^6.8.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-export-extensions/-/babel-plugin-syntax-export-extensions-6.13.0.tgz#70a1484f0f9089a4e84ad44bac353c95b9b12721" + +babel-plugin-syntax-flow@^6.18.0: + version "6.18.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-flow/-/babel-plugin-syntax-flow-6.18.0.tgz#4c3ab20a2af26aa20cd25995c398c4eb70310c8d" + +babel-plugin-syntax-function-bind@^6.8.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-function-bind/-/babel-plugin-syntax-function-bind-6.13.0.tgz#48c495f177bdf31a981e732f55adc0bdd2601f46" + +babel-plugin-syntax-jsx@^6.3.13, babel-plugin-syntax-jsx@^6.8.0: + version "6.18.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-jsx/-/babel-plugin-syntax-jsx-6.18.0.tgz#0af32a9a6e13ca7a3fd5069e62d7b0f58d0d8946" + +babel-plugin-syntax-object-rest-spread@^6.8.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-object-rest-spread/-/babel-plugin-syntax-object-rest-spread-6.13.0.tgz#fd6536f2bce13836ffa3a5458c4903a597bb3bf5" + +babel-plugin-syntax-trailing-function-commas@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-syntax-trailing-function-commas/-/babel-plugin-syntax-trailing-function-commas-6.22.0.tgz#ba0360937f8d06e40180a43fe0d5616fff532cf3" + +babel-plugin-system-import-transformer@3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/babel-plugin-system-import-transformer/-/babel-plugin-system-import-transformer-3.1.0.tgz#d37f0cae8e61ef39060208331d931b5e630d7c5f" + dependencies: + babel-plugin-syntax-dynamic-import "^6.18.0" + +babel-plugin-transform-async-generator-functions@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-async-generator-functions/-/babel-plugin-transform-async-generator-functions-6.24.1.tgz#f058900145fd3e9907a6ddf28da59f215258a5db" + dependencies: + babel-helper-remap-async-to-generator "^6.24.1" + babel-plugin-syntax-async-generators "^6.5.0" + babel-runtime "^6.22.0" + +babel-plugin-transform-async-to-generator@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-async-to-generator/-/babel-plugin-transform-async-to-generator-6.24.1.tgz#6536e378aff6cb1d5517ac0e40eb3e9fc8d08761" + dependencies: + babel-helper-remap-async-to-generator "^6.24.1" + babel-plugin-syntax-async-functions "^6.8.0" + babel-runtime "^6.22.0" + +babel-plugin-transform-class-constructor-call@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-class-constructor-call/-/babel-plugin-transform-class-constructor-call-6.24.1.tgz#80dc285505ac067dcb8d6c65e2f6f11ab7765ef9" + dependencies: + babel-plugin-syntax-class-constructor-call "^6.18.0" + babel-runtime "^6.22.0" + babel-template "^6.24.1" + +babel-plugin-transform-class-properties@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-class-properties/-/babel-plugin-transform-class-properties-6.24.1.tgz#6a79763ea61d33d36f37b611aa9def81a81b46ac" + dependencies: + babel-helper-function-name "^6.24.1" + babel-plugin-syntax-class-properties "^6.8.0" + babel-runtime "^6.22.0" + babel-template "^6.24.1" + +babel-plugin-transform-decorators-legacy@^1.3.4: + version "1.3.4" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-decorators-legacy/-/babel-plugin-transform-decorators-legacy-1.3.4.tgz#741b58f6c5bce9e6027e0882d9c994f04f366925" + dependencies: + babel-plugin-syntax-decorators "^6.1.18" + babel-runtime "^6.2.0" + babel-template "^6.3.0" + +babel-plugin-transform-decorators@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-decorators/-/babel-plugin-transform-decorators-6.24.1.tgz#788013d8f8c6b5222bdf7b344390dfd77569e24d" + dependencies: + babel-helper-explode-class "^6.24.1" + babel-plugin-syntax-decorators "^6.13.0" + babel-runtime "^6.22.0" + babel-template "^6.24.1" + babel-types "^6.24.1" + +babel-plugin-transform-do-expressions@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-do-expressions/-/babel-plugin-transform-do-expressions-6.22.0.tgz#28ccaf92812d949c2cd1281f690c8fdc468ae9bb" + dependencies: + babel-plugin-syntax-do-expressions "^6.8.0" + babel-runtime "^6.22.0" + +babel-plugin-transform-es2015-arrow-functions@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-arrow-functions/-/babel-plugin-transform-es2015-arrow-functions-6.22.0.tgz#452692cb711d5f79dc7f85e440ce41b9f244d221" + dependencies: + babel-runtime "^6.22.0" + +babel-plugin-transform-es2015-block-scoped-functions@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-block-scoped-functions/-/babel-plugin-transform-es2015-block-scoped-functions-6.22.0.tgz#bbc51b49f964d70cb8d8e0b94e820246ce3a6141" + dependencies: + babel-runtime "^6.22.0" + +babel-plugin-transform-es2015-block-scoping@^6.24.1: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-block-scoping/-/babel-plugin-transform-es2015-block-scoping-6.26.0.tgz#d70f5299c1308d05c12f463813b0a09e73b1895f" + dependencies: + babel-runtime "^6.26.0" + babel-template "^6.26.0" + babel-traverse "^6.26.0" + babel-types "^6.26.0" + lodash "^4.17.4" + +babel-plugin-transform-es2015-classes@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-classes/-/babel-plugin-transform-es2015-classes-6.24.1.tgz#5a4c58a50c9c9461e564b4b2a3bfabc97a2584db" + dependencies: + babel-helper-define-map "^6.24.1" + babel-helper-function-name "^6.24.1" + babel-helper-optimise-call-expression "^6.24.1" + babel-helper-replace-supers "^6.24.1" + babel-messages "^6.23.0" + babel-runtime "^6.22.0" + babel-template "^6.24.1" + babel-traverse "^6.24.1" + babel-types "^6.24.1" + +babel-plugin-transform-es2015-computed-properties@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-computed-properties/-/babel-plugin-transform-es2015-computed-properties-6.24.1.tgz#6fe2a8d16895d5634f4cd999b6d3480a308159b3" + dependencies: + babel-runtime "^6.22.0" + babel-template "^6.24.1" + +babel-plugin-transform-es2015-destructuring@^6.22.0: + version "6.23.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-destructuring/-/babel-plugin-transform-es2015-destructuring-6.23.0.tgz#997bb1f1ab967f682d2b0876fe358d60e765c56d" + dependencies: + babel-runtime "^6.22.0" + +babel-plugin-transform-es2015-duplicate-keys@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-duplicate-keys/-/babel-plugin-transform-es2015-duplicate-keys-6.24.1.tgz#73eb3d310ca969e3ef9ec91c53741a6f1576423e" + dependencies: + babel-runtime "^6.22.0" + babel-types "^6.24.1" + +babel-plugin-transform-es2015-for-of@^6.22.0: + version "6.23.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-for-of/-/babel-plugin-transform-es2015-for-of-6.23.0.tgz#f47c95b2b613df1d3ecc2fdb7573623c75248691" + dependencies: + babel-runtime "^6.22.0" + +babel-plugin-transform-es2015-function-name@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-function-name/-/babel-plugin-transform-es2015-function-name-6.24.1.tgz#834c89853bc36b1af0f3a4c5dbaa94fd8eacaa8b" + dependencies: + babel-helper-function-name "^6.24.1" + babel-runtime "^6.22.0" + babel-types "^6.24.1" + +babel-plugin-transform-es2015-literals@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-literals/-/babel-plugin-transform-es2015-literals-6.22.0.tgz#4f54a02d6cd66cf915280019a31d31925377ca2e" + dependencies: + babel-runtime "^6.22.0" + +babel-plugin-transform-es2015-modules-amd@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-modules-amd/-/babel-plugin-transform-es2015-modules-amd-6.24.1.tgz#3b3e54017239842d6d19c3011c4bd2f00a00d154" + dependencies: + babel-plugin-transform-es2015-modules-commonjs "^6.24.1" + babel-runtime "^6.22.0" + babel-template "^6.24.1" + +babel-plugin-transform-es2015-modules-commonjs@^6.24.1: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-modules-commonjs/-/babel-plugin-transform-es2015-modules-commonjs-6.26.0.tgz#0d8394029b7dc6abe1a97ef181e00758dd2e5d8a" + dependencies: + babel-plugin-transform-strict-mode "^6.24.1" + babel-runtime "^6.26.0" + babel-template "^6.26.0" + babel-types "^6.26.0" + +babel-plugin-transform-es2015-modules-systemjs@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-modules-systemjs/-/babel-plugin-transform-es2015-modules-systemjs-6.24.1.tgz#ff89a142b9119a906195f5f106ecf305d9407d23" + dependencies: + babel-helper-hoist-variables "^6.24.1" + babel-runtime "^6.22.0" + babel-template "^6.24.1" + +babel-plugin-transform-es2015-modules-umd@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-modules-umd/-/babel-plugin-transform-es2015-modules-umd-6.24.1.tgz#ac997e6285cd18ed6176adb607d602344ad38468" + dependencies: + babel-plugin-transform-es2015-modules-amd "^6.24.1" + babel-runtime "^6.22.0" + babel-template "^6.24.1" + +babel-plugin-transform-es2015-object-super@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-object-super/-/babel-plugin-transform-es2015-object-super-6.24.1.tgz#24cef69ae21cb83a7f8603dad021f572eb278f8d" + dependencies: + babel-helper-replace-supers "^6.24.1" + babel-runtime "^6.22.0" + +babel-plugin-transform-es2015-parameters@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-parameters/-/babel-plugin-transform-es2015-parameters-6.24.1.tgz#57ac351ab49caf14a97cd13b09f66fdf0a625f2b" + dependencies: + babel-helper-call-delegate "^6.24.1" + babel-helper-get-function-arity "^6.24.1" + babel-runtime "^6.22.0" + babel-template "^6.24.1" + babel-traverse "^6.24.1" + babel-types "^6.24.1" + +babel-plugin-transform-es2015-shorthand-properties@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-shorthand-properties/-/babel-plugin-transform-es2015-shorthand-properties-6.24.1.tgz#24f875d6721c87661bbd99a4622e51f14de38aa0" + dependencies: + babel-runtime "^6.22.0" + babel-types "^6.24.1" + +babel-plugin-transform-es2015-spread@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-spread/-/babel-plugin-transform-es2015-spread-6.22.0.tgz#d6d68a99f89aedc4536c81a542e8dd9f1746f8d1" + dependencies: + babel-runtime "^6.22.0" + +babel-plugin-transform-es2015-sticky-regex@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-sticky-regex/-/babel-plugin-transform-es2015-sticky-regex-6.24.1.tgz#00c1cdb1aca71112cdf0cf6126c2ed6b457ccdbc" + dependencies: + babel-helper-regex "^6.24.1" + babel-runtime "^6.22.0" + babel-types "^6.24.1" + +babel-plugin-transform-es2015-template-literals@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-template-literals/-/babel-plugin-transform-es2015-template-literals-6.22.0.tgz#a84b3450f7e9f8f1f6839d6d687da84bb1236d8d" + dependencies: + babel-runtime "^6.22.0" + +babel-plugin-transform-es2015-typeof-symbol@^6.22.0: + version "6.23.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-typeof-symbol/-/babel-plugin-transform-es2015-typeof-symbol-6.23.0.tgz#dec09f1cddff94b52ac73d505c84df59dcceb372" + dependencies: + babel-runtime "^6.22.0" + +babel-plugin-transform-es2015-unicode-regex@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-es2015-unicode-regex/-/babel-plugin-transform-es2015-unicode-regex-6.24.1.tgz#d38b12f42ea7323f729387f18a7c5ae1faeb35e9" + dependencies: + babel-helper-regex "^6.24.1" + babel-runtime "^6.22.0" + regexpu-core "^2.0.0" + +babel-plugin-transform-exponentiation-operator@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-exponentiation-operator/-/babel-plugin-transform-exponentiation-operator-6.24.1.tgz#2ab0c9c7f3098fa48907772bb813fe41e8de3a0e" + dependencies: + babel-helper-builder-binary-assignment-operator-visitor "^6.24.1" + babel-plugin-syntax-exponentiation-operator "^6.8.0" + babel-runtime "^6.22.0" + +babel-plugin-transform-export-extensions@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-export-extensions/-/babel-plugin-transform-export-extensions-6.22.0.tgz#53738b47e75e8218589eea946cbbd39109bbe653" + dependencies: + babel-plugin-syntax-export-extensions "^6.8.0" + babel-runtime "^6.22.0" + +babel-plugin-transform-flow-strip-types@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-flow-strip-types/-/babel-plugin-transform-flow-strip-types-6.22.0.tgz#84cb672935d43714fdc32bce84568d87441cf7cf" + dependencies: + babel-plugin-syntax-flow "^6.18.0" + babel-runtime "^6.22.0" + +babel-plugin-transform-function-bind@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-function-bind/-/babel-plugin-transform-function-bind-6.22.0.tgz#c6fb8e96ac296a310b8cf8ea401462407ddf6a97" + dependencies: + babel-plugin-syntax-function-bind "^6.8.0" + babel-runtime "^6.22.0" + +babel-plugin-transform-object-rest-spread@^6.22.0: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-object-rest-spread/-/babel-plugin-transform-object-rest-spread-6.26.0.tgz#0f36692d50fef6b7e2d4b3ac1478137a963b7b06" + dependencies: + babel-plugin-syntax-object-rest-spread "^6.8.0" + babel-runtime "^6.26.0" + +babel-plugin-transform-react-display-name@^6.23.0: + version "6.25.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-react-display-name/-/babel-plugin-transform-react-display-name-6.25.0.tgz#67e2bf1f1e9c93ab08db96792e05392bf2cc28d1" + dependencies: + babel-runtime "^6.22.0" + +babel-plugin-transform-react-jsx-self@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-react-jsx-self/-/babel-plugin-transform-react-jsx-self-6.22.0.tgz#df6d80a9da2612a121e6ddd7558bcbecf06e636e" + dependencies: + babel-plugin-syntax-jsx "^6.8.0" + babel-runtime "^6.22.0" + +babel-plugin-transform-react-jsx-source@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-react-jsx-source/-/babel-plugin-transform-react-jsx-source-6.22.0.tgz#66ac12153f5cd2d17b3c19268f4bf0197f44ecd6" + dependencies: + babel-plugin-syntax-jsx "^6.8.0" + babel-runtime "^6.22.0" + +babel-plugin-transform-react-jsx@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-react-jsx/-/babel-plugin-transform-react-jsx-6.24.1.tgz#840a028e7df460dfc3a2d29f0c0d91f6376e66a3" + dependencies: + babel-helper-builder-react-jsx "^6.24.1" + babel-plugin-syntax-jsx "^6.8.0" + babel-runtime "^6.22.0" + +babel-plugin-transform-regenerator@^6.24.1: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-regenerator/-/babel-plugin-transform-regenerator-6.26.0.tgz#e0703696fbde27f0a3efcacf8b4dca2f7b3a8f2f" + dependencies: + regenerator-transform "^0.10.0" + +babel-plugin-transform-strict-mode@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-plugin-transform-strict-mode/-/babel-plugin-transform-strict-mode-6.24.1.tgz#d5faf7aa578a65bbe591cf5edae04a0c67020758" + dependencies: + babel-runtime "^6.22.0" + babel-types "^6.24.1" + +babel-preset-es2015@^6.16.0: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-preset-es2015/-/babel-preset-es2015-6.24.1.tgz#d44050d6bc2c9feea702aaf38d727a0210538939" + dependencies: + babel-plugin-check-es2015-constants "^6.22.0" + babel-plugin-transform-es2015-arrow-functions "^6.22.0" + babel-plugin-transform-es2015-block-scoped-functions "^6.22.0" + babel-plugin-transform-es2015-block-scoping "^6.24.1" + babel-plugin-transform-es2015-classes "^6.24.1" + babel-plugin-transform-es2015-computed-properties "^6.24.1" + babel-plugin-transform-es2015-destructuring "^6.22.0" + babel-plugin-transform-es2015-duplicate-keys "^6.24.1" + babel-plugin-transform-es2015-for-of "^6.22.0" + babel-plugin-transform-es2015-function-name "^6.24.1" + babel-plugin-transform-es2015-literals "^6.22.0" + babel-plugin-transform-es2015-modules-amd "^6.24.1" + babel-plugin-transform-es2015-modules-commonjs "^6.24.1" + babel-plugin-transform-es2015-modules-systemjs "^6.24.1" + babel-plugin-transform-es2015-modules-umd "^6.24.1" + babel-plugin-transform-es2015-object-super "^6.24.1" + babel-plugin-transform-es2015-parameters "^6.24.1" + babel-plugin-transform-es2015-shorthand-properties "^6.24.1" + babel-plugin-transform-es2015-spread "^6.22.0" + babel-plugin-transform-es2015-sticky-regex "^6.24.1" + babel-plugin-transform-es2015-template-literals "^6.22.0" + babel-plugin-transform-es2015-typeof-symbol "^6.22.0" + babel-plugin-transform-es2015-unicode-regex "^6.24.1" + babel-plugin-transform-regenerator "^6.24.1" + +babel-preset-flow@^6.23.0: + version "6.23.0" + resolved "https://registry.yarnpkg.com/babel-preset-flow/-/babel-preset-flow-6.23.0.tgz#e71218887085ae9a24b5be4169affb599816c49d" + dependencies: + babel-plugin-transform-flow-strip-types "^6.22.0" + +babel-preset-react@^6.16.0: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-preset-react/-/babel-preset-react-6.24.1.tgz#ba69dfaea45fc3ec639b6a4ecea6e17702c91380" + dependencies: + babel-plugin-syntax-jsx "^6.3.13" + babel-plugin-transform-react-display-name "^6.23.0" + babel-plugin-transform-react-jsx "^6.24.1" + babel-plugin-transform-react-jsx-self "^6.22.0" + babel-plugin-transform-react-jsx-source "^6.22.0" + babel-preset-flow "^6.23.0" + +babel-preset-stage-0@^6.16.0: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-preset-stage-0/-/babel-preset-stage-0-6.24.1.tgz#5642d15042f91384d7e5af8bc88b1db95b039e6a" + dependencies: + babel-plugin-transform-do-expressions "^6.22.0" + babel-plugin-transform-function-bind "^6.22.0" + babel-preset-stage-1 "^6.24.1" + +babel-preset-stage-1@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-preset-stage-1/-/babel-preset-stage-1-6.24.1.tgz#7692cd7dcd6849907e6ae4a0a85589cfb9e2bfb0" + dependencies: + babel-plugin-transform-class-constructor-call "^6.24.1" + babel-plugin-transform-export-extensions "^6.22.0" + babel-preset-stage-2 "^6.24.1" + +babel-preset-stage-2@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-preset-stage-2/-/babel-preset-stage-2-6.24.1.tgz#d9e2960fb3d71187f0e64eec62bc07767219bdc1" + dependencies: + babel-plugin-syntax-dynamic-import "^6.18.0" + babel-plugin-transform-class-properties "^6.24.1" + babel-plugin-transform-decorators "^6.24.1" + babel-preset-stage-3 "^6.24.1" + +babel-preset-stage-3@^6.24.1: + version "6.24.1" + resolved "https://registry.yarnpkg.com/babel-preset-stage-3/-/babel-preset-stage-3-6.24.1.tgz#836ada0a9e7a7fa37cb138fb9326f87934a48395" + dependencies: + babel-plugin-syntax-trailing-function-commas "^6.22.0" + babel-plugin-transform-async-generator-functions "^6.24.1" + babel-plugin-transform-async-to-generator "^6.24.1" + babel-plugin-transform-exponentiation-operator "^6.24.1" + babel-plugin-transform-object-rest-spread "^6.22.0" + +babel-register@^6.26.0: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-register/-/babel-register-6.26.0.tgz#6ed021173e2fcb486d7acb45c6009a856f647071" + dependencies: + babel-core "^6.26.0" + babel-runtime "^6.26.0" + core-js "^2.5.0" + home-or-tmp "^2.0.0" + lodash "^4.17.4" + mkdirp "^0.5.1" + source-map-support "^0.4.15" + +babel-runtime@^6.18.0, babel-runtime@^6.2.0, babel-runtime@^6.22.0, babel-runtime@^6.26.0: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe" + dependencies: + core-js "^2.4.0" + regenerator-runtime "^0.11.0" + +babel-template@^6.24.1, babel-template@^6.26.0, babel-template@^6.3.0: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-template/-/babel-template-6.26.0.tgz#de03e2d16396b069f46dd9fff8521fb1a0e35e02" + dependencies: + babel-runtime "^6.26.0" + babel-traverse "^6.26.0" + babel-types "^6.26.0" + babylon "^6.18.0" + lodash "^4.17.4" + +babel-traverse@^6.16.0, babel-traverse@^6.24.1, babel-traverse@^6.26.0: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-traverse/-/babel-traverse-6.26.0.tgz#46a9cbd7edcc62c8e5c064e2d2d8d0f4035766ee" + dependencies: + babel-code-frame "^6.26.0" + babel-messages "^6.23.0" + babel-runtime "^6.26.0" + babel-types "^6.26.0" + babylon "^6.18.0" + debug "^2.6.8" + globals "^9.18.0" + invariant "^2.2.2" + lodash "^4.17.4" + +babel-types@^6.16.0, babel-types@^6.19.0, babel-types@^6.24.1, babel-types@^6.25.0, babel-types@^6.26.0: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-types/-/babel-types-6.26.0.tgz#a3b073f94ab49eb6fa55cd65227a334380632497" + dependencies: + babel-runtime "^6.26.0" + esutils "^2.0.2" + lodash "^4.17.4" + to-fast-properties "^1.0.3" + +babelify@^7.3.0: + version "7.3.0" + resolved "https://registry.yarnpkg.com/babelify/-/babelify-7.3.0.tgz#aa56aede7067fd7bd549666ee16dc285087e88e5" + dependencies: + babel-core "^6.0.14" + object-assign "^4.0.0" + +babylon@^6.17.2, babylon@^6.18.0: + version "6.18.0" + resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.18.0.tgz#af2f3b88fa6f5c1e4c634d1a0f8eac4f55b395e3" + +bail@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bail/-/bail-1.0.2.tgz#f7d6c1731630a9f9f0d4d35ed1f962e2074a1764" + balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" base64-js@^1.0.2: - version "1.2.1" - resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.1.tgz#a91947da1f4a516ea38e5b4ec0ec3773675e0886" + version "1.2.3" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.3.tgz#fb13668233d9614cf5fb4bce95a9ba4096cdf801" + +base@^0.11.1: + version "0.11.2" + resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f" + dependencies: + cache-base "^1.0.1" + class-utils "^0.3.5" + component-emitter "^1.2.1" + define-property "^1.0.0" + isobject "^3.0.1" + mixin-deep "^1.2.0" + pascalcase "^0.1.1" bcrypt-pbkdf@^1.0.0: version "1.0.1" @@ -157,12 +934,25 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" +binary-extensions@^1.0.0: + version "1.11.0" + resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.11.0.tgz#46aa1751fb6a2f93ee5e689bb1087d4b14c6c205" + block-stream@*: version "0.0.9" resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a" dependencies: inherits "~2.0.0" +body@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/body/-/body-5.1.0.tgz#e4ba0ce410a46936323367609ecb4e6553125069" + dependencies: + continuable-cache "^0.3.1" + error "^7.0.0" + raw-body "~1.1.0" + safe-json-parse "~1.0.1" + boom@2.x.x: version "2.10.1" resolved "https://registry.yarnpkg.com/boom/-/boom-2.10.1.tgz#39c8918ceff5799f83f9492a848f625add0c766f" @@ -170,12 +960,51 @@ boom@2.x.x: hoek "2.x.x" brace-expansion@^1.1.7: - version "1.1.8" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" dependencies: balanced-match "^1.0.0" concat-map "0.0.1" +braces@^1.8.2: + version "1.8.5" + resolved "https://registry.yarnpkg.com/braces/-/braces-1.8.5.tgz#ba77962e12dff969d6b76711e914b737857bf6a7" + dependencies: + expand-range "^1.8.1" + preserve "^0.2.0" + repeat-element "^1.1.2" + +braces@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.1.tgz#7086c913b4e5a08dbe37ac0ee6a2500c4ba691bb" + dependencies: + arr-flatten "^1.1.0" + array-unique "^0.3.2" + define-property "^1.0.0" + extend-shallow "^2.0.1" + fill-range "^4.0.0" + isobject "^3.0.1" + kind-of "^6.0.2" + repeat-element "^1.1.2" + snapdragon "^0.8.1" + snapdragon-node "^2.0.1" + split-string "^3.0.2" + to-regex "^3.0.1" + +browser-resolve@^1.7.0: + version "1.11.2" + resolved "https://registry.yarnpkg.com/browser-resolve/-/browser-resolve-1.11.2.tgz#8ff09b0a2c421718a1051c260b32e48f442938ce" + dependencies: + resolve "1.1.7" + +buf-compare@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/buf-compare/-/buf-compare-1.0.1.tgz#fef28da8b8113a0a0db4430b0b6467b69730b34a" + +buffer-shims@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" + buffer@4.9.1: version "4.9.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" @@ -184,6 +1013,28 @@ buffer@4.9.1: ieee754 "^1.1.4" isarray "^1.0.0" +builtin-modules@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" + +bytes@1: + version "1.0.0" + resolved "https://registry.yarnpkg.com/bytes/-/bytes-1.0.0.tgz#3569ede8ba34315fab99c3e92cb04c7220de1fa8" + +cache-base@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/cache-base/-/cache-base-1.0.1.tgz#0a7f46416831c8b662ee36fe4e7c59d76f666ab2" + dependencies: + collection-visit "^1.0.0" + component-emitter "^1.2.1" + get-value "^2.0.6" + has-value "^1.0.0" + isobject "^3.0.1" + set-value "^2.0.0" + to-object-path "^0.3.0" + union-value "^1.0.0" + unset-value "^1.0.0" + caller-path@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/caller-path/-/caller-path-0.1.0.tgz#94085ef63581ecd3daa92444a8fe94e82577751f" @@ -194,10 +1045,18 @@ callsites@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-0.2.0.tgz#afab96262910a7f33c19a5775825c69f34e350ca" +camelcase@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" + caseless@~0.12.0: version "0.12.0" resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" +ccount@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.0.2.tgz#53b6a2f815bb77b9c2871f7b9a72c3a25f1d8e89" + chalk@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" @@ -209,21 +1068,61 @@ chalk@^1.1.3: supports-color "^2.0.0" chalk@^2.0.0, chalk@^2.1.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.3.0.tgz#b5ea48efc9c1793dccc9b4767c93914d3f2d52ba" + version "2.3.1" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.3.1.tgz#523fe2678aec7b04e8041909292fe8b17059b796" dependencies: - ansi-styles "^3.1.0" + ansi-styles "^3.2.0" escape-string-regexp "^1.0.5" - supports-color "^4.0.0" + supports-color "^5.2.0" + +character-entities-html4@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/character-entities-html4/-/character-entities-html4-1.1.1.tgz#359a2a4a0f7e29d3dc2ac99bdbe21ee39438ea50" + +character-entities-legacy@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/character-entities-legacy/-/character-entities-legacy-1.1.1.tgz#f40779df1a101872bb510a3d295e1fccf147202f" + +character-entities@^1.0.0: + version "1.2.1" + resolved "https://registry.yarnpkg.com/character-entities/-/character-entities-1.2.1.tgz#f76871be5ef66ddb7f8f8e3478ecc374c27d6dca" + +character-reference-invalid@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/character-reference-invalid/-/character-reference-invalid-1.1.1.tgz#942835f750e4ec61a308e60c2ef8cc1011202efc" chardet@^0.4.0: version "0.4.2" resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.4.2.tgz#b5473b33dc97c424e5d98dc87d55d4d8a29c8bf2" +chokidar@^1.2.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-1.7.0.tgz#798e689778151c8076b4b360e5edd28cda2bb468" + dependencies: + anymatch "^1.3.0" + async-each "^1.0.0" + glob-parent "^2.0.0" + inherits "^2.0.1" + is-binary-path "^1.0.0" + is-glob "^2.0.0" + path-is-absolute "^1.0.0" + readdirp "^2.0.0" + optionalDependencies: + fsevents "^1.0.0" + circular-json@^0.3.1: version "0.3.3" resolved "https://registry.yarnpkg.com/circular-json/-/circular-json-0.3.3.tgz#815c99ea84f6809529d2f45791bdf82711352d66" +class-utils@^0.3.5: + version "0.3.6" + resolved "https://registry.yarnpkg.com/class-utils/-/class-utils-0.3.6.tgz#f93369ae8b9a7ce02fd41faad0ca83033190c463" + dependencies: + arr-union "^3.1.0" + define-property "^0.2.5" + isobject "^3.0.0" + static-extend "^0.1.1" + cli-cursor@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-2.1.0.tgz#b35dac376479facc3e94747d41d0d0f5238ffcb5" @@ -234,14 +1133,61 @@ cli-width@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.0.tgz#ff19ede8a9a5e579324147b0c11f0fbcbabed639" -co@^4.6.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" +cliui@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-3.2.0.tgz#120601537a916d29940f934da3b48d585a39213d" + dependencies: + string-width "^1.0.1" + strip-ansi "^3.0.1" + wrap-ansi "^2.0.0" + +clone-buffer@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/clone-buffer/-/clone-buffer-1.0.0.tgz#e3e25b207ac4e701af721e2cb5a16792cac3dc58" + +clone-stats@^0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/clone-stats/-/clone-stats-0.0.1.tgz#b88f94a82cf38b8791d58046ea4029ad88ca99d1" + +clone-stats@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/clone-stats/-/clone-stats-1.0.0.tgz#b3782dff8bb5474e18b9b6bf0fdfe782f8777680" + +clone@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.3.tgz#298d7e2231660f40c003c2ed3140decf3f53085f" + +clone@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.1.tgz#d217d1e961118e3ac9a4b8bba3285553bf647cdb" + +cloneable-readable@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/cloneable-readable/-/cloneable-readable-1.0.0.tgz#a6290d413f217a61232f95e458ff38418cfb0117" + dependencies: + inherits "^2.0.1" + process-nextick-args "^1.0.6" + through2 "^2.0.1" + +co@^4.6.0: + version "4.6.0" + resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" code-point-at@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" +collapse-white-space@^1.0.0, collapse-white-space@^1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/collapse-white-space/-/collapse-white-space-1.0.3.tgz#4b906f670e5a963a87b76b0e1689643341b6023c" + +collection-visit@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0" + dependencies: + map-visit "^1.0.0" + object-visit "^1.0.0" + color-convert@^1.9.0: version "1.9.1" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.1.tgz#c1261107aeb2f294ebffec9ed9ecad529a6097ed" @@ -253,16 +1199,26 @@ color-name@^1.1.1: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" combined-stream@^1.0.5, combined-stream@~1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.5.tgz#938370a57b4a51dea2c77c15d5c5fdf895164009" + version "1.0.6" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.6.tgz#723e7df6e801ac5613113a7e445a9b69cb632818" dependencies: delayed-stream "~1.0.0" +comma-separated-tokens@^1.0.1: + version "1.0.4" + resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-1.0.4.tgz#72083e58d4a462f01866f6617f4d98a3cd3b8a46" + dependencies: + trim "0.0.1" + +component-emitter@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.2.1.tgz#137918d6d78283f7df7a6b7c5a63e140e69425e6" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" -concat-stream@^1.6.0: +concat-stream@^1.5.0, concat-stream@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.0.tgz#0aac662fd52be78964d5532f694784e70110acf7" dependencies: @@ -270,10 +1226,40 @@ concat-stream@^1.6.0: readable-stream "^2.2.2" typedarray "^0.0.6" +concat-stream@~1.5.0: + version "1.5.2" + resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.5.2.tgz#708978624d856af41a5a741defdd261da752c266" + dependencies: + inherits "~2.0.1" + readable-stream "~2.0.0" + typedarray "~0.0.5" + console-control-strings@^1.0.0, console-control-strings@~1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" +continuable-cache@^0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/continuable-cache/-/continuable-cache-0.3.1.tgz#bd727a7faed77e71ff3985ac93351a912733ad0f" + +convert-source-map@^1.1.1, convert-source-map@^1.5.0: + version "1.5.1" + resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.5.1.tgz#b8278097b9bc229365de5c62cf5fcaed8b5599e5" + +copy-descriptor@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d" + +core-assert@^0.1.0: + version "0.1.3" + resolved "https://registry.yarnpkg.com/core-assert/-/core-assert-0.1.3.tgz#a339cadfe1898a2299cf7ee0df5c14993a32b3d1" + dependencies: + buf-compare "^1.0.0" + +core-js@^2.0.0, core-js@^2.4.0, core-js@^2.5.0: + version "2.5.3" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.5.3.tgz#8acc38345824f16d8365b7c9b4259168e8ed603e" + core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" @@ -298,7 +1284,7 @@ dashdash@^1.12.0: dependencies: assert-plus "^1.0.0" -debug@^2.2.0: +debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@~2.6.7: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" dependencies: @@ -310,6 +1296,14 @@ debug@^3.1.0: dependencies: ms "2.0.0" +decamelize@^1.1.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" + +decode-uri-component@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" + deep-equal@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5" @@ -322,6 +1316,12 @@ deep-is@~0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" +deep-strict-equal@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/deep-strict-equal/-/deep-strict-equal-0.1.0.tgz#5c3f90aa4c7dd9dee02693a662e574417e0b2e17" + dependencies: + core-assert "^0.1.0" + define-properties@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.2.tgz#83a73f2fea569898fb737193c8f873caf6d45c94" @@ -329,7 +1329,26 @@ define-properties@^1.1.2: foreach "^2.0.5" object-keys "^1.0.8" -defined@~1.0.0: +define-property@^0.2.5: + version "0.2.5" + resolved "https://registry.yarnpkg.com/define-property/-/define-property-0.2.5.tgz#c35b1ef918ec3c990f9a5bc57be04aacec5c8116" + dependencies: + is-descriptor "^0.1.0" + +define-property@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/define-property/-/define-property-1.0.0.tgz#769ebaaf3f4a63aad3af9e8d304c9bbe79bfb0e6" + dependencies: + is-descriptor "^1.0.0" + +define-property@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/define-property/-/define-property-2.0.2.tgz#d459689e8d654ba77e02a817f8710d702cb16e9d" + dependencies: + is-descriptor "^1.0.2" + isobject "^3.0.1" + +defined@^1.0.0, defined@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.0.tgz#c98d9bcef75674188e110969151199e39b1fa693" @@ -353,22 +1372,151 @@ delegates@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" +detab@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/detab/-/detab-2.0.1.tgz#531f5e326620e2fd4f03264a905fb3bcc8af4df4" + dependencies: + repeat-string "^1.5.4" + +detect-indent@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-4.0.0.tgz#f76d064352cdf43a1cb6ce619c4ee3a9475de208" + dependencies: + repeating "^2.0.0" + detect-libc@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" +detective@^4.0.0: + version "4.7.1" + resolved "https://registry.yarnpkg.com/detective/-/detective-4.7.1.tgz#0eca7314338442febb6d65da54c10bb1c82b246e" + dependencies: + acorn "^5.2.1" + defined "^1.0.0" + +diff@^1.3.2: + version "1.4.0" + resolved "https://registry.yarnpkg.com/diff/-/diff-1.4.0.tgz#7f28d2eb9ee7b15a97efd89ce63dcfdaa3ccbabf" + +disparity@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/disparity/-/disparity-2.0.0.tgz#57ddacb47324ae5f58d2cc0da886db4ce9eeb718" + dependencies: + ansi-styles "^2.0.1" + diff "^1.3.2" + +doctrine-temporary-fork@2.0.0-alpha-allowarrayindex: + version "2.0.0-alpha-allowarrayindex" + resolved "https://registry.yarnpkg.com/doctrine-temporary-fork/-/doctrine-temporary-fork-2.0.0-alpha-allowarrayindex.tgz#40015a867eb27e75b26c828b71524f137f89f9f0" + dependencies: + esutils "^2.0.2" + isarray "^1.0.0" + doctrine@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.1.0.tgz#5cd01fc101621b42c4cd7f5d1a66243716d3f39d" dependencies: esutils "^2.0.2" +documentation@^4.0.0-beta5: + version "4.0.0" + resolved "https://registry.yarnpkg.com/documentation/-/documentation-4.0.0.tgz#9a8a9a8e38969bfd49d34f35dfb27f1c75b647c0" + dependencies: + ansi-html "^0.0.7" + babel-core "^6.17.0" + babel-generator "6.25.0" + babel-plugin-system-import-transformer "3.1.0" + babel-plugin-transform-decorators-legacy "^1.3.4" + babel-preset-es2015 "^6.16.0" + babel-preset-react "^6.16.0" + babel-preset-stage-0 "^6.16.0" + babel-traverse "^6.16.0" + babel-types "^6.16.0" + babelify "^7.3.0" + babylon "^6.17.2" + chalk "^2.0.0" + chokidar "^1.2.0" + concat-stream "^1.5.0" + disparity "^2.0.0" + doctrine-temporary-fork "2.0.0-alpha-allowarrayindex" + get-comments "^1.0.1" + get-port "^3.1.0" + git-url-parse "^6.0.1" + github-slugger "1.1.3" + glob "^7.0.0" + globals-docs "^2.3.0" + highlight.js "^9.1.0" + js-yaml "^3.8.4" + lodash "^4.11.1" + mdast-util-inject "^1.1.0" + micromatch "^3.0.0" + mime "^1.3.4" + module-deps-sortable "4.0.6" + parse-filepath "^1.0.1" + pify "^3.0.0" + read-pkg-up "^2.0.0" + remark "^8.0.0" + remark-html "6.0.1" + remark-toc "^4.0.0" + remote-origin-url "0.4.0" + shelljs "^0.7.5" + stream-array "^1.1.0" + strip-json-comments "^2.0.0" + tiny-lr "^1.0.3" + unist-builder "^1.0.0" + unist-util-visit "^1.0.1" + vfile "^2.0.0" + vfile-reporter "^4.0.0" + vfile-sort "^2.0.0" + vinyl "^2.0.0" + vinyl-fs "^2.3.1" + yargs "^6.0.1" + +duplexer2@^0.1.2, duplexer2@~0.1.0: + version "0.1.4" + resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1" + dependencies: + readable-stream "^2.0.2" + +duplexify@^3.2.0: + version "3.5.3" + resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-3.5.3.tgz#8b5818800df92fd0125b27ab896491912858243e" + dependencies: + end-of-stream "^1.0.0" + inherits "^2.0.1" + readable-stream "^2.0.0" + stream-shift "^1.0.0" + ecc-jsbn@~0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz#0fc73a9ed5f0d53c38193398523ef7e543777505" dependencies: jsbn "~0.1.0" +"emoji-regex@>=6.0.0 <=6.1.1": + version "6.1.1" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-6.1.1.tgz#c6cd0ec1b0642e2a3c67a1137efc5e796da4f88e" + +end-of-stream@^1.0.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43" + dependencies: + once "^1.4.0" + +error-ex@^1.2.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.1.tgz#f855a86ce61adc4e8621c3cda21e7a7612c3a8dc" + dependencies: + is-arrayish "^0.2.1" + +error@^7.0.0: + version "7.0.2" + resolved "https://registry.yarnpkg.com/error/-/error-7.0.2.tgz#a5f75fff4d9926126ddac0ea5dc38e689153cb02" + dependencies: + string-template "~0.2.1" + xtend "~4.0.0" + es-abstract@^1.5.0: version "1.10.0" resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.10.0.tgz#1ecb36c197842a00d8ee4c2dfd8646bb97d60864" @@ -400,6 +1548,17 @@ eslint-plugin-node@^5.2.1: resolve "^1.3.3" semver "5.3.0" +eslint-plugin-tape@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-tape/-/eslint-plugin-tape-1.1.0.tgz#c19f730135e55c4b847e6ea62e566bd66dd9ce71" + dependencies: + deep-strict-equal "^0.1.0" + espree "^3.1.3" + espurify "^1.5.0" + multimatch "^2.1.0" + object-assign "^4.0.1" + pkg-up "^1.0.0" + eslint-scope@^3.7.1: version "3.7.1" resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-3.7.1.tgz#3d63c3edfda02e06e01a452ad88caacc7cdcb6e8" @@ -412,8 +1571,8 @@ eslint-visitor-keys@^1.0.0: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz#3f3180fb2e291017716acb4c9d6d5b5c34a6a81d" eslint@^4.17.0: - version "4.17.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-4.17.0.tgz#dc24bb51ede48df629be7031c71d9dc0ee4f3ddf" + version "4.18.1" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-4.18.1.tgz#b9138440cb1e98b2f44a0d578c6ecf8eae6150b0" dependencies: ajv "^5.3.0" babel-code-frame "^6.22.0" @@ -453,7 +1612,7 @@ eslint@^4.17.0: table "^4.0.1" text-table "~0.2.0" -espree@^3.5.2: +espree@^3.1.3, espree@^3.5.2: version "3.5.3" resolved "https://registry.yarnpkg.com/espree/-/espree-3.5.3.tgz#931e0af64e7fbbed26b050a29daad1fc64799fa6" dependencies: @@ -464,6 +1623,12 @@ esprima@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.0.tgz#4499eddcd1110e0b218bacf2fa7f7f59f55ca804" +espurify@^1.5.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/espurify/-/espurify-1.7.0.tgz#1c5cf6cbccc32e6f639380bd4f991fab9ba9d226" + dependencies: + core-js "^2.0.0" + esquery@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.0.0.tgz#cfba8b57d7fba93f17298a8a006a04cda13d80fa" @@ -489,7 +1654,44 @@ events@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" -extend@~3.0.0: +expand-brackets@^0.1.4: + version "0.1.5" + resolved "https://registry.yarnpkg.com/expand-brackets/-/expand-brackets-0.1.5.tgz#df07284e342a807cd733ac5af72411e581d1177b" + dependencies: + is-posix-bracket "^0.1.0" + +expand-brackets@^2.1.4: + version "2.1.4" + resolved "https://registry.yarnpkg.com/expand-brackets/-/expand-brackets-2.1.4.tgz#b77735e315ce30f6b6eff0f83b04151a22449622" + dependencies: + debug "^2.3.3" + define-property "^0.2.5" + extend-shallow "^2.0.1" + posix-character-classes "^0.1.0" + regex-not "^1.0.0" + snapdragon "^0.8.1" + to-regex "^3.0.1" + +expand-range@^1.8.1: + version "1.8.2" + resolved "https://registry.yarnpkg.com/expand-range/-/expand-range-1.8.2.tgz#a299effd335fe2721ebae8e257ec79644fc85337" + dependencies: + fill-range "^2.1.0" + +extend-shallow@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/extend-shallow/-/extend-shallow-2.0.1.tgz#51af7d614ad9a9f610ea1bafbb989d6b1c56890f" + dependencies: + is-extendable "^0.1.0" + +extend-shallow@^3.0.0, extend-shallow@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend-shallow/-/extend-shallow-3.0.2.tgz#26a71aaf073b39fb2127172746131c2704028db8" + dependencies: + assign-symbols "^1.0.0" + is-extendable "^1.0.1" + +extend@^3.0.0, extend@~3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" @@ -501,6 +1703,25 @@ external-editor@^2.0.4: iconv-lite "^0.4.17" tmp "^0.0.33" +extglob@^0.3.1: + version "0.3.2" + resolved "https://registry.yarnpkg.com/extglob/-/extglob-0.3.2.tgz#2e18ff3d2f49ab2765cec9023f011daa8d8349a1" + dependencies: + is-extglob "^1.0.0" + +extglob@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/extglob/-/extglob-2.0.4.tgz#ad00fe4dc612a9232e8718711dc5cb5ab0285543" + dependencies: + array-unique "^0.3.2" + define-property "^1.0.0" + expand-brackets "^2.1.4" + extend-shallow "^2.0.1" + fragment-cache "^0.2.1" + regex-not "^1.0.0" + snapdragon "^0.8.1" + to-regex "^3.0.1" + extsprintf@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" @@ -521,6 +1742,12 @@ fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" +faye-websocket@~0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/faye-websocket/-/faye-websocket-0.10.0.tgz#4e492f8d04dfb6f89003507f6edbf2d501e7c6f4" + dependencies: + websocket-driver ">=0.5.1" + figures@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/figures/-/figures-2.0.0.tgz#3ab1a2d2a62c8bfb431a0c94cb797a2fce27c962" @@ -534,6 +1761,46 @@ file-entry-cache@^2.0.0: flat-cache "^1.2.1" object-assign "^4.0.1" +filename-regex@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26" + +fill-range@^2.1.0: + version "2.2.3" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-2.2.3.tgz#50b77dfd7e469bc7492470963699fe7a8485a723" + dependencies: + is-number "^2.1.0" + isobject "^2.0.0" + randomatic "^1.1.3" + repeat-element "^1.1.2" + repeat-string "^1.5.2" + +fill-range@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-4.0.0.tgz#d544811d428f98eb06a63dc402d2403c328c38f7" + dependencies: + extend-shallow "^2.0.1" + is-number "^3.0.0" + repeat-string "^1.6.1" + to-regex-range "^2.1.0" + +find-up@^1.0.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f" + dependencies: + path-exists "^2.0.0" + pinkie-promise "^2.0.0" + +find-up@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-2.1.0.tgz#45d1b7e506c717ddd482775a2b77920a3c0c57a7" + dependencies: + locate-path "^2.0.0" + +first-chunk-stream@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/first-chunk-stream/-/first-chunk-stream-1.0.0.tgz#59bfb50cd905f60d7c394cd3d9acaab4e6ad934e" + flat-cache@^1.2.1: version "1.3.0" resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-1.3.0.tgz#d3030b32b38154f4e3b7e9c709f490f7ef97c481" @@ -549,6 +1816,16 @@ for-each@~0.3.2: dependencies: is-function "~1.0.0" +for-in@^1.0.1, for-in@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" + +for-own@^0.1.4: + version "0.1.5" + resolved "https://registry.yarnpkg.com/for-own/-/for-own-0.1.5.tgz#5265c681a4f294dabbf17c9509b6763aa84510ce" + dependencies: + for-in "^1.0.1" + foreach@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99" @@ -565,10 +1842,23 @@ form-data@~2.1.1: combined-stream "^1.0.5" mime-types "^2.1.12" +fragment-cache@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/fragment-cache/-/fragment-cache-0.2.1.tgz#4290fad27f13e89be7f33799c6bc5a0abfff0d19" + dependencies: + map-cache "^0.2.2" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" +fsevents@^1.0.0: + version "1.1.3" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-1.1.3.tgz#11f82318f5fe7bb2cd22965a108e9306208216d8" + dependencies: + nan "^2.3.0" + node-pre-gyp "^0.6.39" + fstream-ignore@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/fstream-ignore/-/fstream-ignore-1.0.5.tgz#9c31dae34767018fe1d249b24dada67d092da105" @@ -586,7 +1876,7 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2: mkdirp ">=0.5 0" rimraf "2" -function-bind@^1.0.2, function-bind@^1.1.1, function-bind@~1.1.0: +function-bind@^1.0.2, function-bind@^1.1.1, function-bind@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" @@ -607,13 +1897,97 @@ gauge@~2.7.3: strip-ansi "^3.0.1" wide-align "^1.1.0" +get-caller-file@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" + +get-comments@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/get-comments/-/get-comments-1.0.1.tgz#196759101bbbc4facf13060caaedd4870dee55be" + +get-port@^3.1.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/get-port/-/get-port-3.2.0.tgz#dd7ce7de187c06c8bf353796ac71e099f0980ebc" + +get-value@^2.0.3, get-value@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" + getpass@^0.1.1: version "0.1.7" resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" dependencies: assert-plus "^1.0.0" -glob@^7.0.3, glob@^7.0.5, glob@^7.1.2, glob@~7.1.2: +git-up@^2.0.0: + version "2.0.10" + resolved "https://registry.yarnpkg.com/git-up/-/git-up-2.0.10.tgz#20fe6bafbef4384cae253dc4f463c49a0c3bd2ec" + dependencies: + is-ssh "^1.3.0" + parse-url "^1.3.0" + +git-url-parse@^6.0.1: + version "6.2.2" + resolved "https://registry.yarnpkg.com/git-url-parse/-/git-url-parse-6.2.2.tgz#be49024e14b8487553436b4572b8b439532fa871" + dependencies: + git-up "^2.0.0" + +github-slugger@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/github-slugger/-/github-slugger-1.1.3.tgz#314a6e759a18c2b0cc5760d512ccbab549c549a7" + dependencies: + emoji-regex ">=6.0.0 <=6.1.1" + +github-slugger@^1.0.0, github-slugger@^1.1.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/github-slugger/-/github-slugger-1.2.0.tgz#8ada3286fd046d8951c3c952a8d7854cfd90fd9a" + dependencies: + emoji-regex ">=6.0.0 <=6.1.1" + +glob-base@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/glob-base/-/glob-base-0.3.0.tgz#dbb164f6221b1c0b1ccf82aea328b497df0ea3c4" + dependencies: + glob-parent "^2.0.0" + is-glob "^2.0.0" + +glob-parent@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-2.0.0.tgz#81383d72db054fcccf5336daa902f182f6edbb28" + dependencies: + is-glob "^2.0.0" + +glob-parent@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-3.1.0.tgz#9e6af6299d8d3bd2bd40430832bd113df906c5ae" + dependencies: + is-glob "^3.1.0" + path-dirname "^1.0.0" + +glob-stream@^5.3.2: + version "5.3.5" + resolved "https://registry.yarnpkg.com/glob-stream/-/glob-stream-5.3.5.tgz#a55665a9a8ccdc41915a87c701e32d4e016fad22" + dependencies: + extend "^3.0.0" + glob "^5.0.3" + glob-parent "^3.0.0" + micromatch "^2.3.7" + ordered-read-streams "^0.3.0" + through2 "^0.6.0" + to-absolute-glob "^0.1.1" + unique-stream "^2.0.2" + +glob@^5.0.3: + version "5.0.15" + resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1" + dependencies: + inflight "^1.0.4" + inherits "2" + minimatch "2 || 3" + once "^1.3.0" + path-is-absolute "^1.0.0" + +glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.1.2, glob@~7.1.2: version "7.1.2" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" dependencies: @@ -624,10 +1998,18 @@ glob@^7.0.3, glob@^7.0.5, glob@^7.1.2, glob@~7.1.2: once "^1.3.0" path-is-absolute "^1.0.0" +globals-docs@^2.3.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/globals-docs/-/globals-docs-2.4.0.tgz#f2c647544eb6161c7c38452808e16e693c2dafbb" + globals@^11.0.1: version "11.3.0" resolved "https://registry.yarnpkg.com/globals/-/globals-11.3.0.tgz#e04fdb7b9796d8adac9c8f64c14837b2313378b0" +globals@^9.18.0: + version "9.18.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-9.18.0.tgz#aa3896b3e69b487f17e31ed2143d69a8e30c2d8a" + globby@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/globby/-/globby-5.0.0.tgz#ebd84667ca0dbb330b99bcfc68eac2bc54370e0d" @@ -639,10 +2021,20 @@ globby@^5.0.0: pify "^2.0.0" pinkie-promise "^2.0.0" -graceful-fs@^4.1.2: +graceful-fs@^4.0.0, graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" +gulp-sourcemaps@1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/gulp-sourcemaps/-/gulp-sourcemaps-1.6.0.tgz#b86ff349d801ceb56e1d9e7dc7bbcb4b7dee600c" + dependencies: + convert-source-map "^1.1.1" + graceful-fs "^4.1.2" + strip-bom "^2.0.0" + through2 "^2.0.0" + vinyl "^1.0.0" + har-schema@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e" @@ -664,16 +2056,77 @@ has-flag@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51" +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + has-unicode@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" +has-value@^0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/has-value/-/has-value-0.3.1.tgz#7b1f58bada62ca827ec0a2078025654845995e1f" + dependencies: + get-value "^2.0.3" + has-values "^0.1.4" + isobject "^2.0.0" + +has-value@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/has-value/-/has-value-1.0.0.tgz#18b281da585b1c5c51def24c930ed29a0be6b177" + dependencies: + get-value "^2.0.6" + has-values "^1.0.0" + isobject "^3.0.0" + +has-values@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/has-values/-/has-values-0.1.4.tgz#6d61de95d91dfca9b9a02089ad384bff8f62b771" + +has-values@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/has-values/-/has-values-1.0.0.tgz#95b0b63fec2146619a6fe57fe75628d5a39efe4f" + dependencies: + is-number "^3.0.0" + kind-of "^4.0.0" + has@^1.0.1, has@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/has/-/has-1.0.1.tgz#8461733f538b0837c9361e39a9ab9e9704dc2f28" dependencies: function-bind "^1.0.2" +hast-util-is-element@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/hast-util-is-element/-/hast-util-is-element-1.0.0.tgz#3f7216978b2ae14d98749878782675f33be3ce00" + +hast-util-sanitize@^1.0.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/hast-util-sanitize/-/hast-util-sanitize-1.1.2.tgz#d10bd6757a21e59c13abc8ae3530dd3b6d7d679e" + dependencies: + xtend "^4.0.1" + +hast-util-to-html@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/hast-util-to-html/-/hast-util-to-html-3.1.0.tgz#882c99849e40130e991c042e456d453d95c36cff" + dependencies: + ccount "^1.0.0" + comma-separated-tokens "^1.0.1" + hast-util-is-element "^1.0.0" + hast-util-whitespace "^1.0.0" + html-void-elements "^1.0.0" + kebab-case "^1.0.0" + property-information "^3.1.0" + space-separated-tokens "^1.0.0" + stringify-entities "^1.0.1" + unist-util-is "^2.0.0" + xtend "^4.0.1" + +hast-util-whitespace@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/hast-util-whitespace/-/hast-util-whitespace-1.0.0.tgz#bd096919625d2936e1ff17bc4df7fd727f17ece9" + hawk@3.1.3, hawk@~3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4" @@ -683,10 +2136,33 @@ hawk@3.1.3, hawk@~3.1.3: hoek "2.x.x" sntp "1.x.x" +highlight.js@^9.1.0: + version "9.12.0" + resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-9.12.0.tgz#e6d9dbe57cbefe60751f02af336195870c90c01e" + hoek@2.x.x: version "2.16.3" resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" +home-or-tmp@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/home-or-tmp/-/home-or-tmp-2.0.0.tgz#e36c3f2d2cae7d746a857e38d18d5f32a7882db8" + dependencies: + os-homedir "^1.0.0" + os-tmpdir "^1.0.1" + +hosted-git-info@^2.1.4: + version "2.5.0" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" + +html-void-elements@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/html-void-elements/-/html-void-elements-1.0.2.tgz#9d22e0ca32acc95b3f45b8d5b4f6fbdc05affd55" + +http-parser-js@>=0.4.0: + version "0.4.10" + resolved "https://registry.yarnpkg.com/http-parser-js/-/http-parser-js-0.4.10.tgz#92c9c1374c35085f75db359ec56cc257cbb93fa4" + http-signature@~1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.1.1.tgz#df72e267066cd0ac67fb76adf8e134a8fbcf91bf" @@ -718,11 +2194,11 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@^2.0.3, inherits@~2.0.0, inherits@~2.0.3: +inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" -ini@~1.3.0: +ini@^1.3.3, ini@~1.3.0: version "1.3.5" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" @@ -745,58 +2221,264 @@ inquirer@^3.0.6: strip-ansi "^4.0.0" through "^2.3.6" -is-callable@^1.1.1, is-callable@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.1.3.tgz#86eb75392805ddc33af71c92a0eedf74ee7604b2" +interpret@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.1.0.tgz#7ed1b1410c6a0e0f78cf95d3b8440c63f78b8614" -is-date-object@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.1.tgz#9aa20eb6aeebbff77fbd33e74ca01b33581d3a16" +invariant@^2.2.2: + version "2.2.3" + resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.3.tgz#1a827dfde7dcbd7c323f0ca826be8fa7c5e9d688" + dependencies: + loose-envify "^1.0.0" -is-fullwidth-code-point@^1.0.0: +invert-kv@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" + resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" + +is-absolute@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-absolute/-/is-absolute-1.0.0.tgz#395e1ae84b11f26ad1795e73c17378e48a301576" dependencies: - number-is-nan "^1.0.0" + is-relative "^1.0.0" + is-windows "^1.0.1" -is-fullwidth-code-point@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f" +is-accessor-descriptor@^0.1.6: + version "0.1.6" + resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz#a9e12cb3ae8d876727eeef3843f8a0897b5c98d6" + dependencies: + kind-of "^3.0.2" -is-function@~1.0.0: +is-accessor-descriptor@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz#169c2f6d3df1f992618072365c9b0ea1f6878656" + dependencies: + kind-of "^6.0.0" + +is-alphabetical@^1.0.0: version "1.0.1" - resolved "https://registry.yarnpkg.com/is-function/-/is-function-1.0.1.tgz#12cfb98b65b57dd3d193a3121f5f6e2f437602b5" + resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-1.0.1.tgz#c77079cc91d4efac775be1034bf2d243f95e6f08" -is-path-cwd@^1.0.0: +is-alphanumeric@^1.0.0: version "1.0.0" - resolved "https://registry.yarnpkg.com/is-path-cwd/-/is-path-cwd-1.0.0.tgz#d225ec23132e89edd38fda767472e62e65f1106d" + resolved "https://registry.yarnpkg.com/is-alphanumeric/-/is-alphanumeric-1.0.0.tgz#4a9cef71daf4c001c1d81d63d140cf53fd6889f4" -is-path-in-cwd@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-path-in-cwd/-/is-path-in-cwd-1.0.0.tgz#6477582b8214d602346094567003be8a9eac04dc" +is-alphanumerical@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-alphanumerical/-/is-alphanumerical-1.0.1.tgz#dfb4aa4d1085e33bdb61c2dee9c80e9c6c19f53b" dependencies: - is-path-inside "^1.0.0" + is-alphabetical "^1.0.0" + is-decimal "^1.0.0" -is-path-inside@^1.0.0: +is-arrayish@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" + +is-binary-path@^1.0.0: version "1.0.1" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-1.0.1.tgz#8ef5b7de50437a3fdca6b4e865ef7aa55cb48036" + resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-1.0.1.tgz#75f16642b480f187a711c814161fd3a4a7655898" dependencies: - path-is-inside "^1.0.1" + binary-extensions "^1.0.0" -is-promise@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.1.0.tgz#79a2a9ece7f096e80f36d2b2f3bc16c1ff4bf3fa" +is-buffer@^1.1.4, is-buffer@^1.1.5: + version "1.1.6" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" -is-regex@^1.0.4: - version "1.0.4" +is-builtin-module@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-builtin-module/-/is-builtin-module-1.0.0.tgz#540572d34f7ac3119f8f76c30cbc1b1e037affbe" + dependencies: + builtin-modules "^1.0.0" + +is-callable@^1.1.1, is-callable@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.1.3.tgz#86eb75392805ddc33af71c92a0eedf74ee7604b2" + +is-data-descriptor@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56" + dependencies: + kind-of "^3.0.2" + +is-data-descriptor@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz#d84876321d0e7add03990406abbbbd36ba9268c7" + dependencies: + kind-of "^6.0.0" + +is-date-object@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.1.tgz#9aa20eb6aeebbff77fbd33e74ca01b33581d3a16" + +is-decimal@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-decimal/-/is-decimal-1.0.1.tgz#f5fb6a94996ad9e8e3761fbfbd091f1fca8c4e82" + +is-descriptor@^0.1.0: + version "0.1.6" + resolved "https://registry.yarnpkg.com/is-descriptor/-/is-descriptor-0.1.6.tgz#366d8240dde487ca51823b1ab9f07a10a78251ca" + dependencies: + is-accessor-descriptor "^0.1.6" + is-data-descriptor "^0.1.4" + kind-of "^5.0.0" + +is-descriptor@^1.0.0, is-descriptor@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-descriptor/-/is-descriptor-1.0.2.tgz#3b159746a66604b04f8c81524ba365c5f14d86ec" + dependencies: + is-accessor-descriptor "^1.0.0" + is-data-descriptor "^1.0.0" + kind-of "^6.0.2" + +is-dotfile@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/is-dotfile/-/is-dotfile-1.0.3.tgz#a6a2f32ffd2dfb04f5ca25ecd0f6b83cf798a1e1" + +is-equal-shallow@^0.1.3: + version "0.1.3" + resolved "https://registry.yarnpkg.com/is-equal-shallow/-/is-equal-shallow-0.1.3.tgz#2238098fc221de0bcfa5d9eac4c45d638aa1c534" + dependencies: + is-primitive "^2.0.0" + +is-extendable@^0.1.0, is-extendable@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89" + +is-extendable@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-1.0.1.tgz#a7470f9e426733d81bd81e1155264e3a3507cab4" + dependencies: + is-plain-object "^2.0.4" + +is-extglob@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-1.0.0.tgz#ac468177c4943405a092fc8f29760c6ffc6206c0" + +is-extglob@^2.1.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" + +is-finite@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-finite/-/is-finite-1.0.2.tgz#cc6677695602be550ef11e8b4aa6305342b6d0aa" + dependencies: + number-is-nan "^1.0.0" + +is-fullwidth-code-point@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" + dependencies: + number-is-nan "^1.0.0" + +is-fullwidth-code-point@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f" + +is-function@~1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-function/-/is-function-1.0.1.tgz#12cfb98b65b57dd3d193a3121f5f6e2f437602b5" + +is-glob@^2.0.0, is-glob@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-2.0.1.tgz#d096f926a3ded5600f3fdfd91198cb0888c2d863" + dependencies: + is-extglob "^1.0.0" + +is-glob@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-3.1.0.tgz#7ba5ae24217804ac70707b96922567486cc3e84a" + dependencies: + is-extglob "^2.1.0" + +is-hexadecimal@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-hexadecimal/-/is-hexadecimal-1.0.1.tgz#6e084bbc92061fbb0971ec58b6ce6d404e24da69" + +is-number@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-2.1.0.tgz#01fcbbb393463a548f2f466cce16dece49db908f" + dependencies: + kind-of "^3.0.2" + +is-number@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195" + dependencies: + kind-of "^3.0.2" + +is-number@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-4.0.0.tgz#0026e37f5454d73e356dfe6564699867c6a7f0ff" + +is-odd@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-odd/-/is-odd-2.0.0.tgz#7646624671fd7ea558ccd9a2795182f2958f1b24" + dependencies: + is-number "^4.0.0" + +is-path-cwd@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-path-cwd/-/is-path-cwd-1.0.0.tgz#d225ec23132e89edd38fda767472e62e65f1106d" + +is-path-in-cwd@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-path-in-cwd/-/is-path-in-cwd-1.0.0.tgz#6477582b8214d602346094567003be8a9eac04dc" + dependencies: + is-path-inside "^1.0.0" + +is-path-inside@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-1.0.1.tgz#8ef5b7de50437a3fdca6b4e865ef7aa55cb48036" + dependencies: + path-is-inside "^1.0.1" + +is-plain-obj@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e" + +is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677" + dependencies: + isobject "^3.0.1" + +is-posix-bracket@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz#3334dc79774368e92f016e6fbc0a88f5cd6e6bc4" + +is-primitive@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-primitive/-/is-primitive-2.0.0.tgz#207bab91638499c07b2adf240a41a87210034575" + +is-promise@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.1.0.tgz#79a2a9ece7f096e80f36d2b2f3bc16c1ff4bf3fa" + +is-regex@^1.0.4: + version "1.0.4" resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.0.4.tgz#5517489b547091b0930e095654ced25ee97e9491" dependencies: has "^1.0.1" +is-relative@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-relative/-/is-relative-1.0.0.tgz#a1bb6935ce8c5dba1e8b9754b9b2dcc020e2260d" + dependencies: + is-unc-path "^1.0.0" + is-resolvable@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-resolvable/-/is-resolvable-1.1.0.tgz#fb18f87ce1feb925169c9a407c19318a3206ed88" +is-ssh@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/is-ssh/-/is-ssh-1.3.0.tgz#ebea1169a2614da392a63740366c3ce049d8dff6" + dependencies: + protocols "^1.1.0" + +is-stream@^1.0.1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" + is-symbol@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.1.tgz#3cc59f00025194b6ab2e38dbae6689256b660572" @@ -805,7 +2487,37 @@ is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" -isarray@^1.0.0, isarray@~1.0.0: +is-unc-path@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-unc-path/-/is-unc-path-1.0.0.tgz#d731e8898ed090a12c352ad2eaed5095ad322c9d" + dependencies: + unc-path-regex "^0.1.2" + +is-utf8@^0.2.0: + version "0.2.1" + resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" + +is-valid-glob@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/is-valid-glob/-/is-valid-glob-0.3.0.tgz#d4b55c69f51886f9b65c70d6c2622d37e29f48fe" + +is-whitespace-character@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-whitespace-character/-/is-whitespace-character-1.0.1.tgz#9ae0176f3282b65457a1992cdb084f8a5f833e3b" + +is-windows@^1.0.1, is-windows@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" + +is-word-character@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-word-character/-/is-word-character-1.0.1.tgz#5a03fa1ea91ace8a6eb0c7cd770eb86d65c8befb" + +isarray@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" + +isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" @@ -813,6 +2525,16 @@ isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" +isobject@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89" + dependencies: + isarray "1.0.0" + +isobject@^3.0.0, isobject@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" + isstream@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" @@ -821,11 +2543,11 @@ jmespath@0.15.0: version "0.15.0" resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" -js-tokens@^3.0.2: +js-tokens@^3.0.0, js-tokens@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b" -js-yaml@^3.9.1: +js-yaml@^3.8.4, js-yaml@^3.9.1: version "3.10.0" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.10.0.tgz#2e78441646bd4682e963f22b6e92823c309c62dc" dependencies: @@ -836,6 +2558,14 @@ jsbn@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" +jsesc@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-1.3.0.tgz#46c3fec8c1892b12b0833db9bc7622176dbab34b" + +jsesc@~0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" + json-schema-traverse@^0.3.0: version "0.3.1" resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" @@ -848,7 +2578,7 @@ json-stable-stringify-without-jsonify@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" -json-stable-stringify@^1.0.1: +json-stable-stringify@^1.0.0, json-stable-stringify@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz#9a759d39c5f2ff503fd5300646ed445f88c4f9af" dependencies: @@ -858,10 +2588,18 @@ json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" +json5@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/json5/-/json5-0.5.1.tgz#1eade7acc012034ad84e2396767ead9fa5495821" + jsonify@~0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" +jsonparse@^1.2.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" + jsprim@^1.2.2: version "1.4.1" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" @@ -871,6 +2609,48 @@ jsprim@^1.2.2: json-schema "0.2.3" verror "1.10.0" +kebab-case@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/kebab-case/-/kebab-case-1.0.0.tgz#3f9e4990adcad0c686c0e701f7645868f75f91eb" + +kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: + version "3.2.2" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" + dependencies: + is-buffer "^1.1.5" + +kind-of@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-4.0.0.tgz#20813df3d712928b207378691a45066fae72dd57" + dependencies: + is-buffer "^1.1.5" + +kind-of@^5.0.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-5.1.0.tgz#729c91e2d857b7a419a1f9aa65685c4c33f5845d" + +kind-of@^6.0.0, kind-of@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.2.tgz#01146b36a6218e64e58f3a8d66de5d7fc6f6d051" + +lazy-cache@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-2.0.2.tgz#b9190a4f913354694840859f8a8f7084d8822264" + dependencies: + set-getter "^0.1.0" + +lazystream@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/lazystream/-/lazystream-1.0.0.tgz#f6995fe0f820392f61396be89462407bb77168e4" + dependencies: + readable-stream "^2.0.5" + +lcid@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/lcid/-/lcid-1.0.0.tgz#308accafa0bc483a3867b4b6f2b9506251d1b835" + dependencies: + invert-kv "^1.0.0" + levn@^0.3.0, levn@~0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" @@ -878,10 +2658,54 @@ levn@^0.3.0, levn@~0.3.0: prelude-ls "~1.1.2" type-check "~0.3.2" -lodash@^4.0.0, lodash@^4.17.4, lodash@^4.3.0: +livereload-js@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/livereload-js/-/livereload-js-2.3.0.tgz#c3ab22e8aaf5bf3505d80d098cbad67726548c9a" + +load-json-file@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0" + dependencies: + graceful-fs "^4.1.2" + parse-json "^2.2.0" + pify "^2.0.0" + pinkie-promise "^2.0.0" + strip-bom "^2.0.0" + +load-json-file@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-2.0.0.tgz#7947e42149af80d696cbf797bcaabcfe1fe29ca8" + dependencies: + graceful-fs "^4.1.2" + parse-json "^2.2.0" + pify "^2.0.0" + strip-bom "^3.0.0" + +locate-path@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e" + dependencies: + p-locate "^2.0.0" + path-exists "^3.0.0" + +lodash.isequal@^4.0.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" + +lodash@^4.0.0, lodash@^4.11.1, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.3.0: version "4.17.5" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.5.tgz#99a92d65c0272debe8c96b6057bc8fbfa3bed511" +longest-streak@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/longest-streak/-/longest-streak-2.0.2.tgz#2421b6ba939a443bb9ffebf596585a50b4c38e2e" + +loose-envify@^1.0.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.3.1.tgz#d1a8ad33fa9ce0e713d65fdd0ac8b748d478c848" + dependencies: + js-tokens "^3.0.0" + lru-cache@^4.0.1: version "4.1.1" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.1.tgz#622e32e82488b49279114a4f9ecf45e7cd6bba55" @@ -889,21 +2713,136 @@ lru-cache@^4.0.1: pseudomap "^1.0.2" yallist "^2.1.2" -mime-db@~1.30.0: - version "1.30.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" +map-cache@^0.2.0, map-cache@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/map-cache/-/map-cache-0.2.2.tgz#c32abd0bd6525d9b051645bb4f26ac5dc98a0dbf" + +map-visit@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/map-visit/-/map-visit-1.0.0.tgz#ecdca8f13144e660f1b5bd41f12f3479d98dfb8f" + dependencies: + object-visit "^1.0.0" + +markdown-escapes@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/markdown-escapes/-/markdown-escapes-1.0.1.tgz#1994df2d3af4811de59a6714934c2b2292734518" + +markdown-table@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/markdown-table/-/markdown-table-1.1.1.tgz#4b3dd3a133d1518b8ef0dbc709bf2a1b4824bc8c" + +mdast-util-compact@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/mdast-util-compact/-/mdast-util-compact-1.0.1.tgz#cdb5f84e2b6a2d3114df33bd05d9cb32e3c4083a" + dependencies: + unist-util-modify-children "^1.0.0" + unist-util-visit "^1.1.0" + +mdast-util-definitions@^1.2.0: + version "1.2.2" + resolved "https://registry.yarnpkg.com/mdast-util-definitions/-/mdast-util-definitions-1.2.2.tgz#673f4377c3e23d3de7af7a4fe2214c0e221c5ac7" + dependencies: + unist-util-visit "^1.0.0" + +mdast-util-inject@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/mdast-util-inject/-/mdast-util-inject-1.1.0.tgz#db06b8b585be959a2dcd2f87f472ba9b756f3675" + dependencies: + mdast-util-to-string "^1.0.0" + +mdast-util-to-hast@^2.1.1: + version "2.5.0" + resolved "https://registry.yarnpkg.com/mdast-util-to-hast/-/mdast-util-to-hast-2.5.0.tgz#f087844d255c7540f36906da30ba106c0ee5ee2f" + dependencies: + collapse-white-space "^1.0.0" + detab "^2.0.0" + mdast-util-definitions "^1.2.0" + mdurl "^1.0.1" + trim "0.0.1" + trim-lines "^1.0.0" + unist-builder "^1.0.1" + unist-util-generated "^1.1.0" + unist-util-position "^3.0.0" + unist-util-visit "^1.1.0" + xtend "^4.0.1" + +mdast-util-to-string@^1.0.0, mdast-util-to-string@^1.0.2: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-1.0.4.tgz#5c455c878c9355f0c1e7f3e8b719cf583691acfb" + +mdast-util-toc@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/mdast-util-toc/-/mdast-util-toc-2.0.1.tgz#b1d2cb23bfb01f812fa7b55bffe8b0a8bedf6f21" + dependencies: + github-slugger "^1.1.1" + mdast-util-to-string "^1.0.2" + unist-util-visit "^1.1.0" + +mdurl@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e" + +merge-stream@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-1.0.1.tgz#4041202d508a342ba00174008df0c251b8c135e1" + dependencies: + readable-stream "^2.0.1" + +micromatch@^2.1.5, micromatch@^2.3.7: + version "2.3.11" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-2.3.11.tgz#86677c97d1720b363431d04d0d15293bd38c1565" + dependencies: + arr-diff "^2.0.0" + array-unique "^0.2.1" + braces "^1.8.2" + expand-brackets "^0.1.4" + extglob "^0.3.1" + filename-regex "^2.0.0" + is-extglob "^1.0.0" + is-glob "^2.0.1" + kind-of "^3.0.2" + normalize-path "^2.0.1" + object.omit "^2.0.0" + parse-glob "^3.0.4" + regex-cache "^0.4.2" + +micromatch@^3.0.0: + version "3.1.8" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-3.1.8.tgz#5c8caa008de588eebb395e8c0ad12c128f25fff1" + dependencies: + arr-diff "^4.0.0" + array-unique "^0.3.2" + braces "^2.3.1" + define-property "^2.0.2" + extend-shallow "^3.0.2" + extglob "^2.0.4" + fragment-cache "^0.2.1" + kind-of "^6.0.2" + nanomatch "^1.2.9" + object.pick "^1.3.0" + regex-not "^1.0.0" + snapdragon "^0.8.1" + to-regex "^3.0.1" + +mime-db@~1.33.0: + version "1.33.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.33.0.tgz#a3492050a5cb9b63450541e39d9788d2272783db" mime-types@^2.1.12, mime-types@~2.1.7: - version "2.1.17" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" + version "2.1.18" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.18.tgz#6f323f60a83d11146f831ff11fd66e2fe5503bb8" dependencies: - mime-db "~1.30.0" + mime-db "~1.33.0" + +mime@^1.3.4: + version "1.6.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" mimic-fn@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022" -minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4: +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: @@ -913,33 +2852,85 @@ minimist@0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" -minimist@^1.2.0, minimist@~1.2.0: +minimist@^1.1.0, minimist@^1.2.0, minimist@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" -"mkdirp@>=0.5 0", mkdirp@^0.5.1: +mixin-deep@^1.2.0: + version "1.3.1" + resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.1.tgz#a49e7268dce1a0d9698e45326c5626df3543d0fe" + dependencies: + for-in "^1.0.2" + is-extendable "^1.0.1" + +"mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" dependencies: minimist "0.0.8" +module-deps-sortable@4.0.6: + version "4.0.6" + resolved "https://registry.yarnpkg.com/module-deps-sortable/-/module-deps-sortable-4.0.6.tgz#1251a4ba2c44a92df6989bd029da121a4f2109b0" + dependencies: + JSONStream "^1.0.3" + browser-resolve "^1.7.0" + concat-stream "~1.5.0" + defined "^1.0.0" + detective "^4.0.0" + duplexer2 "^0.1.2" + inherits "^2.0.1" + parents "^1.0.0" + readable-stream "^2.0.2" + resolve "^1.1.3" + stream-combiner2 "^1.1.1" + subarg "^1.0.0" + through2 "^2.0.0" + xtend "^4.0.0" + ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" +multimatch@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/multimatch/-/multimatch-2.1.0.tgz#9c7906a22fb4c02919e2f5f75161b4cdbd4b2a2b" + dependencies: + array-differ "^1.0.0" + array-union "^1.0.1" + arrify "^1.0.0" + minimatch "^3.0.0" + mute-stream@0.0.7: version "0.0.7" resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.7.tgz#3075ce93bc21b8fab43e1bc4da7e8115ed1e7bab" -nan@~2.5.1: +nan@^2.3.0, nan@~2.5.1: version "2.5.1" resolved "https://registry.yarnpkg.com/nan/-/nan-2.5.1.tgz#d5b01691253326a97a2bbee9e61c55d8d60351e2" +nanomatch@^1.2.9: + version "1.2.9" + resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.9.tgz#879f7150cb2dab7a471259066c104eee6e0fa7c2" + dependencies: + arr-diff "^4.0.0" + array-unique "^0.3.2" + define-property "^2.0.2" + extend-shallow "^3.0.2" + fragment-cache "^0.2.1" + is-odd "^2.0.0" + is-windows "^1.0.2" + kind-of "^6.0.2" + object.pick "^1.3.0" + regex-not "^1.0.0" + snapdragon "^0.8.1" + to-regex "^3.0.1" + natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" -node-pre-gyp@~0.6.32: +node-pre-gyp@^0.6.39, node-pre-gyp@~0.6.32: version "0.6.39" resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.39.tgz#c00e96860b23c0e1420ac7befc5044e1d78d8649" dependencies: @@ -962,6 +2953,21 @@ nopt@^4.0.1: abbrev "1" osenv "^0.1.4" +normalize-package-data@^2.3.2: + version "2.4.0" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.4.0.tgz#12f95a307d58352075a04907b84ac8be98ac012f" + dependencies: + hosted-git-info "^2.1.4" + is-builtin-module "^1.0.0" + semver "2 || 3 || 4 || 5" + validate-npm-package-license "^3.0.1" + +normalize-path@^2.0.0, normalize-path@^2.0.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-2.1.1.tgz#1ab28b556e198363a8c1a6f7e6fa20137fe6aed9" + dependencies: + remove-trailing-separator "^1.0.1" + npmlog@^4.0.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" @@ -979,19 +2985,46 @@ oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@^4.0.1, object-assign@^4.1.0: +object-assign@^4.0.0, object-assign@^4.0.1, object-assign@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" -object-inspect@~1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.3.0.tgz#5b1eb8e6742e2ee83342a637034d844928ba2f6d" +object-copy@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/object-copy/-/object-copy-0.1.0.tgz#7e7d858b781bd7c991a41ba975ed3812754e998c" + dependencies: + copy-descriptor "^0.1.0" + define-property "^0.2.5" + kind-of "^3.0.3" + +object-inspect@~1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.5.0.tgz#9d876c11e40f485c79215670281b767488f9bfe3" object-keys@^1.0.8: version "1.0.11" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d" -once@^1.3.0, once@^1.3.3: +object-visit@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/object-visit/-/object-visit-1.0.1.tgz#f79c4493af0c5377b59fe39d395e41042dd045bb" + dependencies: + isobject "^3.0.0" + +object.omit@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa" + dependencies: + for-own "^0.1.4" + is-extendable "^0.1.1" + +object.pick@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/object.pick/-/object.pick-1.3.0.tgz#87a10ac4c1694bd2e1cbf53591a66141fb5dd747" + dependencies: + isobject "^3.0.1" + +once@^1.3.0, once@^1.3.3, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" dependencies: @@ -1014,22 +3047,122 @@ optionator@^0.8.2: type-check "~0.3.2" wordwrap "~1.0.0" +ordered-read-streams@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/ordered-read-streams/-/ordered-read-streams-0.3.0.tgz#7137e69b3298bb342247a1bbee3881c80e2fd78b" + dependencies: + is-stream "^1.0.1" + readable-stream "^2.0.1" + os-homedir@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" -os-tmpdir@^1.0.0, os-tmpdir@~1.0.2: +os-locale@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/os-locale/-/os-locale-1.4.0.tgz#20f9f17ae29ed345e8bde583b13d2009803c14d9" + dependencies: + lcid "^1.0.0" + +os-tmpdir@^1.0.0, os-tmpdir@^1.0.1, os-tmpdir@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" osenv@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.4.tgz#42fe6d5953df06c8064be6f176c3d05aaaa34644" + version "0.1.5" + resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410" dependencies: os-homedir "^1.0.0" os-tmpdir "^1.0.0" -path-is-absolute@^1.0.0: +p-limit@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.2.0.tgz#0e92b6bedcb59f022c13d0f1949dc82d15909f1c" + dependencies: + p-try "^1.0.0" + +p-locate@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" + dependencies: + p-limit "^1.1.0" + +p-try@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" + +parents@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/parents/-/parents-1.0.1.tgz#fedd4d2bf193a77745fe71e371d73c3307d9c751" + dependencies: + path-platform "~0.11.15" + +parse-entities@^1.0.2: + version "1.1.1" + resolved "https://registry.yarnpkg.com/parse-entities/-/parse-entities-1.1.1.tgz#8112d88471319f27abae4d64964b122fe4e1b890" + dependencies: + character-entities "^1.0.0" + character-entities-legacy "^1.0.0" + character-reference-invalid "^1.0.0" + is-alphanumerical "^1.0.0" + is-decimal "^1.0.0" + is-hexadecimal "^1.0.0" + +parse-filepath@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/parse-filepath/-/parse-filepath-1.0.2.tgz#a632127f53aaf3d15876f5872f3ffac763d6c891" + dependencies: + is-absolute "^1.0.0" + map-cache "^0.2.0" + path-root "^0.1.1" + +parse-git-config@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/parse-git-config/-/parse-git-config-0.2.0.tgz#272833fdd15fea146fb75d336d236b963b6ff706" + dependencies: + ini "^1.3.3" + +parse-glob@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/parse-glob/-/parse-glob-3.0.4.tgz#b2c376cfb11f35513badd173ef0bb6e3a388391c" + dependencies: + glob-base "^0.3.0" + is-dotfile "^1.0.0" + is-extglob "^1.0.0" + is-glob "^2.0.0" + +parse-json@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-2.2.0.tgz#f480f40434ef80741f8469099f8dea18f55a4dc9" + dependencies: + error-ex "^1.2.0" + +parse-url@^1.3.0: + version "1.3.11" + resolved "https://registry.yarnpkg.com/parse-url/-/parse-url-1.3.11.tgz#57c15428ab8a892b1f43869645c711d0e144b554" + dependencies: + is-ssh "^1.3.0" + protocols "^1.4.0" + +pascalcase@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/pascalcase/-/pascalcase-0.1.1.tgz#b363e55e8006ca6fe21784d2db22bd15d7917f14" + +path-dirname@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/path-dirname/-/path-dirname-1.0.2.tgz#cc33d24d525e099a5388c0336c6e32b9160609e0" + +path-exists@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-2.1.0.tgz#0feb6c64f0fc518d9a754dd5efb62c7022761f4b" + dependencies: + pinkie-promise "^2.0.0" + +path-exists@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" + +path-is-absolute@^1.0.0, path-is-absolute@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" @@ -1041,6 +3174,34 @@ path-parse@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.5.tgz#3c1adf871ea9cd6c9431b6ea2bd74a0ff055c4c1" +path-platform@~0.11.15: + version "0.11.15" + resolved "https://registry.yarnpkg.com/path-platform/-/path-platform-0.11.15.tgz#e864217f74c36850f0852b78dc7bf7d4a5721bf2" + +path-root-regex@^0.1.0: + version "0.1.2" + resolved "https://registry.yarnpkg.com/path-root-regex/-/path-root-regex-0.1.2.tgz#bfccdc8df5b12dc52c8b43ec38d18d72c04ba96d" + +path-root@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/path-root/-/path-root-0.1.1.tgz#9a4a6814cac1c0cd73360a95f32083c8ea4745b7" + dependencies: + path-root-regex "^0.1.0" + +path-type@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" + dependencies: + graceful-fs "^4.1.2" + pify "^2.0.0" + pinkie-promise "^2.0.0" + +path-type@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-2.0.0.tgz#f012ccb8415b7096fc2daa1054c3d72389594c73" + dependencies: + pify "^2.0.0" + performance-now@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" @@ -1049,6 +3210,10 @@ pify@^2.0.0: version "2.3.0" resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" +pify@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176" + pinkie-promise@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/pinkie-promise/-/pinkie-promise-2.0.1.tgz#2135d6dfa7a358c069ac9b178776288228450ffa" @@ -1059,22 +3224,52 @@ pinkie@^2.0.0: version "2.0.4" resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870" +pkg-up@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/pkg-up/-/pkg-up-1.0.0.tgz#3e08fb461525c4421624a33b9f7e6d0af5b05a26" + dependencies: + find-up "^1.0.0" + pluralize@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-7.0.0.tgz#298b89df8b93b0221dbf421ad2b1b1ea23fc6777" +posix-character-classes@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab" + prelude-ls@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" -process-nextick-args@~1.0.6: +preserve@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/preserve/-/preserve-0.2.0.tgz#815ed1f6ebc65926f865b310c0713bcb3315ce4b" + +private@^0.1.6, private@^0.1.7: + version "0.1.8" + resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff" + +process-nextick-args@^1.0.6, process-nextick-args@~1.0.6: version "1.0.7" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3" +process-nextick-args@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" + progress@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.0.tgz#8a1be366bf8fc23db2bd23f10c6fe920b4389d1f" +property-information@^3.1.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/property-information/-/property-information-3.2.0.tgz#fd1483c8fbac61808f5fe359e7693a1f48a58331" + +protocols@^1.1.0, protocols@^1.4.0: + version "1.4.6" + resolved "https://registry.yarnpkg.com/protocols/-/protocols-1.4.6.tgz#f8bb263ea1b5fd7a7604d26b8be39bd77678bf8a" + protozero@~1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/protozero/-/protozero-1.5.1.tgz#5a27df6fb6e1ed743f510812ae76c082f5b16638" @@ -1087,38 +3282,283 @@ punycode@1.3.2: version "1.3.2" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" -punycode@^1.4.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" +punycode@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" + +qs@^6.4.0: + version "6.5.1" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" + +qs@~6.4.0: + version "6.4.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" + +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + +randomatic@^1.1.3: + version "1.1.7" + resolved "https://registry.yarnpkg.com/randomatic/-/randomatic-1.1.7.tgz#c7abe9cc8b87c0baa876b19fde83fd464797e38c" + dependencies: + is-number "^3.0.0" + kind-of "^4.0.0" + +raw-body@~1.1.0: + version "1.1.7" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-1.1.7.tgz#1d027c2bfa116acc6623bca8f00016572a87d425" + dependencies: + bytes "1" + string_decoder "0.10" + +rc@^1.1.7: + version "1.2.5" + resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.5.tgz#275cd687f6e3b36cc756baa26dfee80a790301fd" + dependencies: + deep-extend "~0.4.0" + ini "~1.3.0" + minimist "^1.2.0" + strip-json-comments "~2.0.1" + +read-pkg-up@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-1.0.1.tgz#9d63c13276c065918d57f002a57f40a1b643fb02" + dependencies: + find-up "^1.0.0" + read-pkg "^1.0.0" + +read-pkg-up@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-2.0.0.tgz#6b72a8048984e0c41e79510fd5e9fa99b3b549be" + dependencies: + find-up "^2.0.0" + read-pkg "^2.0.0" + +read-pkg@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-1.1.0.tgz#f5ffaa5ecd29cb31c0474bca7d756b6bb29e3f28" + dependencies: + load-json-file "^1.0.0" + normalize-package-data "^2.3.2" + path-type "^1.0.0" + +read-pkg@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-2.0.0.tgz#8ef1c0623c6a6db0dc6713c4bfac46332b2368f8" + dependencies: + load-json-file "^2.0.0" + normalize-package-data "^2.3.2" + path-type "^2.0.0" + +"readable-stream@>=1.0.33-1 <1.1.0-0": + version "1.0.34" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + +readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.4, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.1.4, readable-stream@^2.1.5, readable-stream@^2.2.2: + version "2.3.4" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.4.tgz#c946c3f47fa7d8eabc0b6150f4a12f69a4574071" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.0.3" + util-deprecate "~1.0.1" + +readable-stream@~2.0.0: + version "2.0.6" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.0.6.tgz#8f90341e68a53ccc928788dacfcd11b36eb9b78e" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "~1.0.0" + process-nextick-args "~1.0.6" + string_decoder "~0.10.x" + util-deprecate "~1.0.1" + +readable-stream@~2.1.0: + version "2.1.5" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.1.5.tgz#66fa8b720e1438b364681f2ad1a63c618448c9d0" + dependencies: + buffer-shims "^1.0.0" + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "~1.0.0" + process-nextick-args "~1.0.6" + string_decoder "~0.10.x" + util-deprecate "~1.0.1" + +readdirp@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-2.1.0.tgz#4ed0ad060df3073300c48440373f72d1cc642d78" + dependencies: + graceful-fs "^4.1.2" + minimatch "^3.0.2" + readable-stream "^2.0.2" + set-immediate-shim "^1.0.1" + +rechoir@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" + dependencies: + resolve "^1.1.6" + +regenerate@^1.2.1: + version "1.3.3" + resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.3.3.tgz#0c336d3980553d755c39b586ae3b20aa49c82b7f" + +regenerator-runtime@^0.11.0: + version "0.11.1" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz#be05ad7f9bf7d22e056f9726cee5017fbf19e2e9" + +regenerator-transform@^0.10.0: + version "0.10.1" + resolved "https://registry.yarnpkg.com/regenerator-transform/-/regenerator-transform-0.10.1.tgz#1e4996837231da8b7f3cf4114d71b5691a0680dd" + dependencies: + babel-runtime "^6.18.0" + babel-types "^6.19.0" + private "^0.1.6" + +regex-cache@^0.4.2: + version "0.4.4" + resolved "https://registry.yarnpkg.com/regex-cache/-/regex-cache-0.4.4.tgz#75bdc58a2a1496cec48a12835bc54c8d562336dd" + dependencies: + is-equal-shallow "^0.1.3" + +regex-not@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/regex-not/-/regex-not-1.0.2.tgz#1f4ece27e00b0b65e0247a6810e6a85d83a5752c" + dependencies: + extend-shallow "^3.0.2" + safe-regex "^1.1.0" + +regexpu-core@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-2.0.0.tgz#49d038837b8dcf8bfa5b9a42139938e6ea2ae240" + dependencies: + regenerate "^1.2.1" + regjsgen "^0.2.0" + regjsparser "^0.1.4" + +regjsgen@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/regjsgen/-/regjsgen-0.2.0.tgz#6c016adeac554f75823fe37ac05b92d5a4edb1f7" + +regjsparser@^0.1.4: + version "0.1.5" + resolved "https://registry.yarnpkg.com/regjsparser/-/regjsparser-0.1.5.tgz#7ee8f84dc6fa792d3fd0ae228d24bd949ead205c" + dependencies: + jsesc "~0.5.0" + +remark-html@6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/remark-html/-/remark-html-6.0.1.tgz#5094d2c71f7941fdb2ae865bac76627757ce09c1" + dependencies: + hast-util-sanitize "^1.0.0" + hast-util-to-html "^3.0.0" + mdast-util-to-hast "^2.1.1" + xtend "^4.0.1" + +remark-parse@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/remark-parse/-/remark-parse-4.0.0.tgz#99f1f049afac80382366e2e0d0bd55429dd45d8b" + dependencies: + collapse-white-space "^1.0.2" + is-alphabetical "^1.0.0" + is-decimal "^1.0.0" + is-whitespace-character "^1.0.0" + is-word-character "^1.0.0" + markdown-escapes "^1.0.0" + parse-entities "^1.0.2" + repeat-string "^1.5.4" + state-toggle "^1.0.0" + trim "0.0.1" + trim-trailing-lines "^1.0.0" + unherit "^1.0.4" + unist-util-remove-position "^1.0.0" + vfile-location "^2.0.0" + xtend "^4.0.1" + +remark-slug@^4.0.0: + version "4.2.3" + resolved "https://registry.yarnpkg.com/remark-slug/-/remark-slug-4.2.3.tgz#8d987d0e5e63d4a49ea37b90fe999a3dcfc81b72" + dependencies: + github-slugger "^1.0.0" + mdast-util-to-string "^1.0.0" + unist-util-visit "^1.0.0" + +remark-stringify@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/remark-stringify/-/remark-stringify-4.0.0.tgz#4431884c0418f112da44991b4e356cfe37facd87" + dependencies: + ccount "^1.0.0" + is-alphanumeric "^1.0.0" + is-decimal "^1.0.0" + is-whitespace-character "^1.0.0" + longest-streak "^2.0.1" + markdown-escapes "^1.0.0" + markdown-table "^1.1.0" + mdast-util-compact "^1.0.0" + parse-entities "^1.0.2" + repeat-string "^1.5.4" + state-toggle "^1.0.0" + stringify-entities "^1.0.1" + unherit "^1.0.4" + xtend "^4.0.1" + +remark-toc@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/remark-toc/-/remark-toc-4.0.1.tgz#ff36ff6de54ea07dd59e3f5334a4a3aac1e93185" + dependencies: + mdast-util-toc "^2.0.0" + remark-slug "^4.0.0" + +remark@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/remark/-/remark-8.0.0.tgz#287b6df2fe1190e263c1d15e486d3fa835594d6d" + dependencies: + remark-parse "^4.0.0" + remark-stringify "^4.0.0" + unified "^6.0.0" + +remote-origin-url@0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/remote-origin-url/-/remote-origin-url-0.4.0.tgz#4d3e2902f34e2d37d1c263d87710b77eb4086a30" + dependencies: + parse-git-config "^0.2.0" + +remove-trailing-separator@^1.0.1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef" -qs@~6.4.0: - version "6.4.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" +repeat-element@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/repeat-element/-/repeat-element-1.1.2.tgz#ef089a178d1483baae4d93eb98b4f9e4e11d990a" -querystring@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" +repeat-string@^1.5.0, repeat-string@^1.5.2, repeat-string@^1.5.4, repeat-string@^1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -rc@^1.1.7: - version "1.2.5" - resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.5.tgz#275cd687f6e3b36cc756baa26dfee80a790301fd" +repeating@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/repeating/-/repeating-2.0.1.tgz#5214c53a926d3552707527fbab415dbc08d06dda" dependencies: - deep-extend "~0.4.0" - ini "~1.3.0" - minimist "^1.2.0" - strip-json-comments "~2.0.1" + is-finite "^1.0.0" -readable-stream@^2.0.6, readable-stream@^2.1.4, readable-stream@^2.2.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c" - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.3" - isarray "~1.0.0" - process-nextick-args "~1.0.6" - safe-buffer "~5.1.1" - string_decoder "~1.0.3" - util-deprecate "~1.0.1" +replace-ext@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-0.0.1.tgz#29bbd92078a739f0bcce2b4ee41e837953522924" + +replace-ext@1.0.0, replace-ext@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-1.0.0.tgz#de63128373fcbf7c3ccfa4de5a480c45a67958eb" request@2.81.0: version "2.81.0" @@ -1147,6 +3587,14 @@ request@2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" +require-directory@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" + +require-main-filename@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1" + require-uncached@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/require-uncached/-/require-uncached-1.0.3.tgz#4e0d56d6c9662fd31e43011c4b95aa49955421d3" @@ -1158,18 +3606,20 @@ resolve-from@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-1.0.1.tgz#26cbfe935d1aeeeabb29bc3fe5aeb01e93d44226" -resolve@^1.3.3: +resolve-url@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a" + +resolve@1.1.7: + version "1.1.7" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b" + +resolve@^1.1.3, resolve@^1.1.6, resolve@^1.3.3, resolve@~1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.5.0.tgz#1f09acce796c9a762579f31b2c1cc4c3cddf9f36" dependencies: path-parse "^1.0.5" -resolve@~1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.4.0.tgz#a75be01c53da25d934a98ebd0e4c4a7312f92a86" - dependencies: - path-parse "^1.0.5" - restore-cursor@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-2.0.0.tgz#9f7ee287f82fd326d4fd162923d62129eee0dfaf" @@ -1183,6 +3633,10 @@ resumer@~0.0.0: dependencies: through "~2.3.4" +ret@~0.1.10: + version "0.1.15" + resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" + rimraf@2, rimraf@^2.2.8, rimraf@^2.5.1, rimraf@^2.6.1: version "2.6.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" @@ -1209,6 +3663,16 @@ safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" +safe-json-parse@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/safe-json-parse/-/safe-json-parse-1.0.1.tgz#3e76723e38dfdda13c9b1d29a1e07ffee4b30b57" + +safe-regex@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/safe-regex/-/safe-regex-1.1.0.tgz#40a3669f3b077d1e943d44629e157dd48023bf2e" + dependencies: + ret "~0.1.10" + sax@1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" @@ -1217,18 +3681,46 @@ sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" +"semver@2 || 3 || 4 || 5", semver@^5.3.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" + semver@5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" -semver@^5.3.0: - version "5.5.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" - -set-blocking@~2.0.0: +set-blocking@^2.0.0, set-blocking@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" +set-getter@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/set-getter/-/set-getter-0.1.0.tgz#d769c182c9d5a51f409145f2fba82e5e86e80376" + dependencies: + to-object-path "^0.3.0" + +set-immediate-shim@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz#4b2b1b27eb808a9f8dcc481a58e5e56f599f3f61" + +set-value@^0.4.3: + version "0.4.3" + resolved "https://registry.yarnpkg.com/set-value/-/set-value-0.4.3.tgz#7db08f9d3d22dc7f78e53af3c3bf4666ecdfccf1" + dependencies: + extend-shallow "^2.0.1" + is-extendable "^0.1.1" + is-plain-object "^2.0.1" + to-object-path "^0.3.0" + +set-value@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.0.tgz#71ae4a88f0feefbbf52d1ea604f3fb315ebb6274" + dependencies: + extend-shallow "^2.0.1" + is-extendable "^0.1.1" + is-plain-object "^2.0.3" + split-string "^3.0.1" + shebang-command@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" @@ -1239,22 +3731,111 @@ shebang-regex@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3" +shelljs@^0.7.5: + version "0.7.8" + resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.7.8.tgz#decbcf874b0d1e5fb72e14b164a9683048e9acb3" + dependencies: + glob "^7.0.0" + interpret "^1.0.0" + rechoir "^0.6.2" + signal-exit@^3.0.0, signal-exit@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" +slash@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55" + slice-ansi@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-1.0.0.tgz#044f1a49d8842ff307aad6b505ed178bd950134d" dependencies: is-fullwidth-code-point "^2.0.0" +snapdragon-node@^2.0.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b" + dependencies: + define-property "^1.0.0" + isobject "^3.0.0" + snapdragon-util "^3.0.1" + +snapdragon-util@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/snapdragon-util/-/snapdragon-util-3.0.1.tgz#f956479486f2acd79700693f6f7b805e45ab56e2" + dependencies: + kind-of "^3.2.0" + +snapdragon@^0.8.1: + version "0.8.1" + resolved "https://registry.yarnpkg.com/snapdragon/-/snapdragon-0.8.1.tgz#e12b5487faded3e3dea0ac91e9400bf75b401370" + dependencies: + base "^0.11.1" + debug "^2.2.0" + define-property "^0.2.5" + extend-shallow "^2.0.1" + map-cache "^0.2.2" + source-map "^0.5.6" + source-map-resolve "^0.5.0" + use "^2.0.0" + sntp@1.x.x: version "1.0.9" resolved "https://registry.yarnpkg.com/sntp/-/sntp-1.0.9.tgz#6541184cc90aeea6c6e7b35e2659082443c66198" dependencies: hoek "2.x.x" +source-map-resolve@^0.5.0: + version "0.5.1" + resolved "https://registry.yarnpkg.com/source-map-resolve/-/source-map-resolve-0.5.1.tgz#7ad0f593f2281598e854df80f19aae4b92d7a11a" + dependencies: + atob "^2.0.0" + decode-uri-component "^0.2.0" + resolve-url "^0.2.1" + source-map-url "^0.4.0" + urix "^0.1.0" + +source-map-support@^0.4.15: + version "0.4.18" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.4.18.tgz#0286a6de8be42641338594e97ccea75f0a2c585f" + dependencies: + source-map "^0.5.6" + +source-map-url@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3" + +source-map@^0.5.0, source-map@^0.5.6, source-map@^0.5.7: + version "0.5.7" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" + +space-separated-tokens@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/space-separated-tokens/-/space-separated-tokens-1.1.1.tgz#9695b9df9e65aec1811d4c3f9ce52520bc2f7e4d" + dependencies: + trim "0.0.1" + +spdx-correct@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-1.0.2.tgz#4b3073d933ff51f3912f03ac5519498a4150db40" + dependencies: + spdx-license-ids "^1.0.2" + +spdx-expression-parse@~1.0.0: + version "1.0.4" + resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-1.0.4.tgz#9bdf2f20e1f40ed447fbe273266191fced51626c" + +spdx-license-ids@^1.0.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-1.2.2.tgz#c9df7a3424594ade6bd11900d596696dc06bac57" + +split-string@^3.0.1, split-string@^3.0.2: + version "3.1.0" + resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2" + dependencies: + extend-shallow "^3.0.0" + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" @@ -1273,7 +3854,39 @@ sshpk@^1.7.0: jsbn "~0.1.0" tweetnacl "~0.14.0" -string-width@^1.0.1, string-width@^1.0.2: +state-toggle@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/state-toggle/-/state-toggle-1.0.0.tgz#d20f9a616bb4f0c3b98b91922d25b640aa2bc425" + +static-extend@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/static-extend/-/static-extend-0.1.2.tgz#60809c39cbff55337226fd5e0b520f341f1fb5c6" + dependencies: + define-property "^0.2.5" + object-copy "^0.1.0" + +stream-array@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/stream-array/-/stream-array-1.1.2.tgz#9e5f7345f2137c30ee3b498b9114e80b52bb7eb5" + dependencies: + readable-stream "~2.1.0" + +stream-combiner2@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/stream-combiner2/-/stream-combiner2-1.1.1.tgz#fb4d8a1420ea362764e21ad4780397bebcb41cbe" + dependencies: + duplexer2 "~0.1.0" + readable-stream "^2.0.2" + +stream-shift@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.0.tgz#d5c752825e5367e786f78e18e445ea223a155952" + +string-template@~0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/string-template/-/string-template-0.2.1.tgz#42932e598a352d01fc22ec3367d9d84eec6c9add" + +string-width@^1.0.0, string-width@^1.0.1, string-width@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" dependencies: @@ -1296,12 +3909,25 @@ string.prototype.trim@~1.1.2: es-abstract "^1.5.0" function-bind "^1.0.2" +string_decoder@0.10, string_decoder@~0.10.x: + version "0.10.31" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" + string_decoder@~1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab" dependencies: safe-buffer "~5.1.0" +stringify-entities@^1.0.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/stringify-entities/-/stringify-entities-1.3.1.tgz#b150ec2d72ac4c1b5f324b51fb6b28c9cdff058c" + dependencies: + character-entities-html4 "^1.0.0" + character-entities-legacy "^1.0.0" + is-alphanumerical "^1.0.0" + is-hexadecimal "^1.0.0" + stringstream@~0.0.4: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -1318,20 +3944,49 @@ strip-ansi@^4.0.0: dependencies: ansi-regex "^3.0.0" -strip-json-comments@~2.0.1: +strip-bom-stream@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/strip-bom-stream/-/strip-bom-stream-1.0.0.tgz#e7144398577d51a6bed0fa1994fa05f43fd988ee" + dependencies: + first-chunk-stream "^1.0.0" + strip-bom "^2.0.0" + +strip-bom@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e" + dependencies: + is-utf8 "^0.2.0" + +strip-bom@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" + +strip-json-comments@^2.0.0, strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" +subarg@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/subarg/-/subarg-1.0.0.tgz#f62cf17581e996b48fc965699f54c06ae268b8d2" + dependencies: + minimist "^1.1.0" + supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" -supports-color@^4.0.0: +supports-color@^4.1.0: version "4.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-4.5.0.tgz#be7a0de484dec5c5cddf8b3d59125044912f635b" dependencies: has-flag "^2.0.0" +supports-color@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.2.0.tgz#b0d5333b1184dd3666cbe5aa0b45c5ac7ac17a4a" + dependencies: + has-flag "^3.0.0" + table@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/table/-/table-4.0.2.tgz#a33447375391e766ad34d3486e6e2aedc84d2e36" @@ -1344,19 +3999,19 @@ table@^4.0.1: string-width "^2.1.1" tape@^4.6.3: - version "4.8.0" - resolved "https://registry.yarnpkg.com/tape/-/tape-4.8.0.tgz#f6a9fec41cc50a1de50fa33603ab580991f6068e" + version "4.9.0" + resolved "https://registry.yarnpkg.com/tape/-/tape-4.9.0.tgz#855c08360395133709d34d3fbf9ef341eb73ca6a" dependencies: deep-equal "~1.0.1" defined "~1.0.0" for-each "~0.3.2" - function-bind "~1.1.0" + function-bind "~1.1.1" glob "~7.1.2" has "~1.0.1" inherits "~2.0.3" minimist "~1.2.0" - object-inspect "~1.3.0" - resolve "~1.4.0" + object-inspect "~1.5.0" + resolve "~1.5.0" resumer "~0.0.0" string.prototype.trim "~1.1.2" through "~2.3.8" @@ -1386,22 +4041,105 @@ text-table@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" -through@^2.3.6, through@~2.3.4, through@~2.3.8: +through2-filter@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/through2-filter/-/through2-filter-2.0.0.tgz#60bc55a0dacb76085db1f9dae99ab43f83d622ec" + dependencies: + through2 "~2.0.0" + xtend "~4.0.0" + +through2@^0.6.0: + version "0.6.5" + resolved "https://registry.yarnpkg.com/through2/-/through2-0.6.5.tgz#41ab9c67b29d57209071410e1d7a7a968cd3ad48" + dependencies: + readable-stream ">=1.0.33-1 <1.1.0-0" + xtend ">=4.0.0 <4.1.0-0" + +through2@^2.0.0, through2@^2.0.1, through2@~2.0.0: + version "2.0.3" + resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.3.tgz#0004569b37c7c74ba39c43f3ced78d1ad94140be" + dependencies: + readable-stream "^2.1.5" + xtend "~4.0.1" + +"through@>=2.2.7 <3", through@^2.3.6, through@~2.3.4, through@~2.3.8: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" +tiny-lr@^1.0.3: + version "1.1.0" + resolved "https://registry.yarnpkg.com/tiny-lr/-/tiny-lr-1.1.0.tgz#a373bce2a4b58cef9a64433360ba593155f4cd45" + dependencies: + body "^5.1.0" + debug "~2.6.7" + faye-websocket "~0.10.0" + livereload-js "^2.3.0" + object-assign "^4.1.0" + qs "^6.4.0" + tmp@^0.0.33: version "0.0.33" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" dependencies: os-tmpdir "~1.0.2" +to-absolute-glob@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/to-absolute-glob/-/to-absolute-glob-0.1.1.tgz#1cdfa472a9ef50c239ee66999b662ca0eb39937f" + dependencies: + extend-shallow "^2.0.1" + +to-fast-properties@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-1.0.3.tgz#b83571fa4d8c25b82e231b06e3a3055de4ca1a47" + +to-object-path@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/to-object-path/-/to-object-path-0.3.0.tgz#297588b7b0e7e0ac08e04e672f85c1f4999e17af" + dependencies: + kind-of "^3.0.2" + +to-regex-range@^2.1.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-2.1.1.tgz#7c80c17b9dfebe599e27367e0d4dd5590141db38" + dependencies: + is-number "^3.0.0" + repeat-string "^1.6.1" + +to-regex@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/to-regex/-/to-regex-3.0.1.tgz#15358bee4a2c83bd76377ba1dc049d0f18837aae" + dependencies: + define-property "^0.2.5" + extend-shallow "^2.0.1" + regex-not "^1.0.0" + tough-cookie@~2.3.0: version "2.3.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561" dependencies: punycode "^1.4.1" +trim-lines@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/trim-lines/-/trim-lines-1.1.0.tgz#9926d03ede13ba18f7d42222631fb04c79ff26fe" + +trim-right@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/trim-right/-/trim-right-1.0.1.tgz#cb2e1203067e0c8de1f614094b9fe45704ea6003" + +trim-trailing-lines@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/trim-trailing-lines/-/trim-trailing-lines-1.1.0.tgz#7aefbb7808df9d669f6da2e438cac8c46ada7684" + +trim@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/trim/-/trim-0.0.1.tgz#5858547f6b290757ee95cccc666fb50084c460dd" + +trough@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/trough/-/trough-1.0.1.tgz#a9fd8b0394b0ae8fff82e0633a0a36ccad5b5f86" + tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" @@ -1418,7 +4156,7 @@ type-check@~0.3.2: dependencies: prelude-ls "~1.1.2" -typedarray@^0.0.6: +typedarray@^0.0.6, typedarray@~0.0.5: version "0.0.6" resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" @@ -1426,6 +4164,96 @@ uid-number@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81" +unc-path-regex@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/unc-path-regex/-/unc-path-regex-0.1.2.tgz#e73dd3d7b0d7c5ed86fbac6b0ae7d8c6a69d50fa" + +unherit@^1.0.4: + version "1.1.0" + resolved "https://registry.yarnpkg.com/unherit/-/unherit-1.1.0.tgz#6b9aaedfbf73df1756ad9e316dd981885840cd7d" + dependencies: + inherits "^2.0.1" + xtend "^4.0.1" + +unified@^6.0.0: + version "6.1.6" + resolved "https://registry.yarnpkg.com/unified/-/unified-6.1.6.tgz#5ea7f807a0898f1f8acdeefe5f25faa010cc42b1" + dependencies: + bail "^1.0.0" + extend "^3.0.0" + is-plain-obj "^1.1.0" + trough "^1.0.0" + vfile "^2.0.0" + x-is-function "^1.0.4" + x-is-string "^0.1.0" + +union-value@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/union-value/-/union-value-1.0.0.tgz#5c71c34cb5bad5dcebe3ea0cd08207ba5aa1aea4" + dependencies: + arr-union "^3.1.0" + get-value "^2.0.6" + is-extendable "^0.1.1" + set-value "^0.4.3" + +unique-stream@^2.0.2: + version "2.2.1" + resolved "https://registry.yarnpkg.com/unique-stream/-/unique-stream-2.2.1.tgz#5aa003cfbe94c5ff866c4e7d668bb1c4dbadb369" + dependencies: + json-stable-stringify "^1.0.0" + through2-filter "^2.0.0" + +unist-builder@^1.0.0, unist-builder@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/unist-builder/-/unist-builder-1.0.2.tgz#8c3b9903ef64bcfb117dd7cf6a5d98fc1b3b27b6" + dependencies: + object-assign "^4.1.0" + +unist-util-generated@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/unist-util-generated/-/unist-util-generated-1.1.1.tgz#99f16c78959ac854dee7c615c291924c8bf4de7f" + +unist-util-is@^2.0.0, unist-util-is@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-2.1.1.tgz#0c312629e3f960c66e931e812d3d80e77010947b" + +unist-util-modify-children@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/unist-util-modify-children/-/unist-util-modify-children-1.1.1.tgz#66d7e6a449e6f67220b976ab3cb8b5ebac39e51d" + dependencies: + array-iterate "^1.0.0" + +unist-util-position@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/unist-util-position/-/unist-util-position-3.0.0.tgz#e6e1e03eeeb81c5e1afe553e8d4adfbd7c0d8f82" + +unist-util-remove-position@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/unist-util-remove-position/-/unist-util-remove-position-1.1.1.tgz#5a85c1555fc1ba0c101b86707d15e50fa4c871bb" + dependencies: + unist-util-visit "^1.1.0" + +unist-util-stringify-position@^1.0.0, unist-util-stringify-position@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/unist-util-stringify-position/-/unist-util-stringify-position-1.1.1.tgz#3ccbdc53679eed6ecf3777dd7f5e3229c1b6aa3c" + +unist-util-visit@^1.0.0, unist-util-visit@^1.0.1, unist-util-visit@^1.1.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-1.3.0.tgz#41ca7c82981fd1ce6c762aac397fc24e35711444" + dependencies: + unist-util-is "^2.1.1" + +unset-value@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-1.0.0.tgz#8376873f7d2335179ffb1e6fc3a8ed0dfc8ab559" + dependencies: + has-value "^0.3.1" + isobject "^3.0.0" + +urix@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" + url@0.10.3: version "0.10.3" resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" @@ -1433,6 +4261,14 @@ url@0.10.3: punycode "1.3.2" querystring "0.2.0" +use@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/use/-/use-2.0.2.tgz#ae28a0d72f93bf22422a18a2e379993112dec8e8" + dependencies: + define-property "^0.2.5" + isobject "^3.0.0" + lazy-cache "^2.0.2" + util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" @@ -1445,6 +4281,17 @@ uuid@^3.0.0: version "3.2.1" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.2.1.tgz#12c528bb9d58d0b9265d9a2f6f0fe8be17ff1f14" +vali-date@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/vali-date/-/vali-date-1.0.0.tgz#1b904a59609fb328ef078138420934f6b86709a6" + +validate-npm-package-license@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz#2804babe712ad3379459acfbe24746ab2c303fbc" + dependencies: + spdx-correct "~1.0.0" + spdx-expression-parse "~1.0.0" + verror@1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" @@ -1453,6 +4300,99 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" +vfile-location@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/vfile-location/-/vfile-location-2.0.2.tgz#d3675c59c877498e492b4756ff65e4af1a752255" + +vfile-message@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/vfile-message/-/vfile-message-1.0.0.tgz#a6adb0474ea400fa25d929f1d673abea6a17e359" + dependencies: + unist-util-stringify-position "^1.1.1" + +vfile-reporter@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/vfile-reporter/-/vfile-reporter-4.0.0.tgz#ea6f0ae1342f4841573985e05f941736f27de9da" + dependencies: + repeat-string "^1.5.0" + string-width "^1.0.0" + supports-color "^4.1.0" + unist-util-stringify-position "^1.0.0" + vfile-statistics "^1.1.0" + +vfile-sort@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/vfile-sort/-/vfile-sort-2.1.0.tgz#49501c9e8bbe5adff2e9b3a7671ee1b1e20c5210" + +vfile-statistics@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/vfile-statistics/-/vfile-statistics-1.1.0.tgz#02104c60fdeed1d11b1f73ad65330b7634b3d895" + +vfile@^2.0.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/vfile/-/vfile-2.3.0.tgz#e62d8e72b20e83c324bc6c67278ee272488bf84a" + dependencies: + is-buffer "^1.1.4" + replace-ext "1.0.0" + unist-util-stringify-position "^1.0.0" + vfile-message "^1.0.0" + +vinyl-fs@^2.3.1: + version "2.4.4" + resolved "https://registry.yarnpkg.com/vinyl-fs/-/vinyl-fs-2.4.4.tgz#be6ff3270cb55dfd7d3063640de81f25d7532239" + dependencies: + duplexify "^3.2.0" + glob-stream "^5.3.2" + graceful-fs "^4.0.0" + gulp-sourcemaps "1.6.0" + is-valid-glob "^0.3.0" + lazystream "^1.0.0" + lodash.isequal "^4.0.0" + merge-stream "^1.0.0" + mkdirp "^0.5.0" + object-assign "^4.0.0" + readable-stream "^2.0.4" + strip-bom "^2.0.0" + strip-bom-stream "^1.0.0" + through2 "^2.0.0" + through2-filter "^2.0.0" + vali-date "^1.0.0" + vinyl "^1.0.0" + +vinyl@^1.0.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/vinyl/-/vinyl-1.2.0.tgz#5c88036cf565e5df05558bfc911f8656df218884" + dependencies: + clone "^1.0.0" + clone-stats "^0.0.1" + replace-ext "0.0.1" + +vinyl@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/vinyl/-/vinyl-2.1.0.tgz#021f9c2cf951d6b939943c89eb5ee5add4fd924c" + dependencies: + clone "^2.1.1" + clone-buffer "^1.0.0" + clone-stats "^1.0.0" + cloneable-readable "^1.0.0" + remove-trailing-separator "^1.0.1" + replace-ext "^1.0.0" + +websocket-driver@>=0.5.1: + version "0.7.0" + resolved "https://registry.yarnpkg.com/websocket-driver/-/websocket-driver-0.7.0.tgz#0caf9d2d755d93aee049d4bdd0d3fe2cca2a24eb" + dependencies: + http-parser-js ">=0.4.0" + websocket-extensions ">=0.1.1" + +websocket-extensions@>=0.1.1: + version "0.1.3" + resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.3.tgz#5d2ff22977003ec687a4b87073dfbbac146ccf29" + +which-module@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f" + which@^1.2.9: version "1.3.0" resolved "https://registry.yarnpkg.com/which/-/which-1.3.0.tgz#ff04bdfc010ee547d780bec38e1ac1c2777d253a" @@ -1469,6 +4409,13 @@ wordwrap@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" +wrap-ansi@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85" + dependencies: + string-width "^1.0.1" + strip-ansi "^3.0.1" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" @@ -1479,6 +4426,14 @@ write@^0.2.1: dependencies: mkdirp "^0.5.1" +x-is-function@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/x-is-function/-/x-is-function-1.0.4.tgz#5d294dc3d268cbdd062580e0c5df77a391d1fa1e" + +x-is-string@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/x-is-string/-/x-is-string-0.1.0.tgz#474b50865af3a49a9c4657f05acd145458f77d82" + xml2js@0.4.17: version "0.4.17" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.17.tgz#17be93eaae3f3b779359c795b419705a8817e868" @@ -1492,6 +4447,38 @@ xmlbuilder@4.2.1, xmlbuilder@^4.1.0: dependencies: lodash "^4.0.0" +"xtend@>=4.0.0 <4.1.0-0", xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.0, xtend@~4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af" + +y18n@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41" + yallist@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" + +yargs-parser@^4.2.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-4.2.1.tgz#29cceac0dc4f03c6c87b4a9f217dd18c9f74871c" + dependencies: + camelcase "^3.0.0" + +yargs@^6.0.1: + version "6.6.0" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-6.6.0.tgz#782ec21ef403345f830a808ca3d513af56065208" + dependencies: + camelcase "^3.0.0" + cliui "^3.2.0" + decamelize "^1.1.1" + get-caller-file "^1.0.1" + os-locale "^1.4.0" + read-pkg-up "^1.0.1" + require-directory "^2.1.1" + require-main-filename "^1.0.1" + set-blocking "^2.0.0" + string-width "^1.0.2" + which-module "^1.0.0" + y18n "^3.2.1" + yargs-parser "^4.2.0"