diff --git a/README.md b/README.md index d772630..0d2a87a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # abstract-level -**Abstract class for a lexicographically sorted key-value database.** The successor to [`abstract-leveldown`](https://github.com/Level/abstract-leveldown) with builtin encodings, sublevels, events, promises and support of Uint8Array. If you are upgrading please see [`UPGRADING.md`](UPGRADING.md). +**Abstract class for a lexicographically sorted key-value database.** The successor to [`abstract-leveldown`](https://github.com/Level/abstract-leveldown) with builtin encodings, sublevels, hooks, events, promises and support of Uint8Array. If you are upgrading please see [`UPGRADING.md`](UPGRADING.md). > :pushpin: Which module should I use? What happened to `levelup`? Head over to the [FAQ](https://github.com/Level/community#faq). @@ -39,7 +39,7 @@ - [`sublevel = db.sublevel(name[, options])`](#sublevel--dbsublevelname-options) - [`encoding = db.keyEncoding([encoding])`](#encoding--dbkeyencodingencoding) - [`encoding = db.valueEncoding([encoding])`](#encoding--dbvalueencodingencoding) - - [`key = db.prefixKey(key, keyFormat)`](#key--dbprefixkeykey-keyformat) + - [`key = db.prefixKey(key, keyFormat[, local])`](#key--dbprefixkeykey-keyformat-local) - [`db.defer(fn)`](#dbdeferfn) - [`chainedBatch`](#chainedbatch) - [`chainedBatch.put(key, value[, options])`](#chainedbatchputkey-value-options) @@ -63,9 +63,41 @@ - [`valueIterator`](#valueiterator) - [`sublevel`](#sublevel) - [`sublevel.prefix`](#sublevelprefix) + - [`sublevel.parent`](#sublevelparent) - [`sublevel.db`](#subleveldb) + - [Hooks](#hooks) + - [`hook = db.hooks.prewrite`](#hook--dbhooksprewrite) + - [Example](#example) + - [Arguments](#arguments) + - [`op` (object)](#op-object) + - [`batch` (object)](#batch-object) + - [`batch = batch.add(op)`](#batch--batchaddop) + - [`hook = db.hooks.postopen`](#hook--dbhookspostopen) + - [Example](#example-1) + - [Arguments](#arguments-1) + - [`options` (object)](#options-object) + - [`hook = db.hooks.newsub`](#hook--dbhooksnewsub) + - [Example](#example-2) + - [Arguments](#arguments-2) + - [`sublevel` (object)](#sublevel-object) + - [`options` (object)](#options-object-1) + - [`hook`](#hook) + - [`hook.add(fn)`](#hookaddfn) + - [`hook.delete(fn)`](#hookdeletefn) + - [Hook Error Handling](#hook-error-handling) + - [Hooks On Sublevels](#hooks-on-sublevels) - [Encodings](#encodings) - [Events](#events) + - [`opening`](#opening) + - [`open`](#open) + - [`ready` (deprecated)](#ready-deprecated) + - [`closing`](#closing) + - [`closed`](#closed) + - [`write`](#write) + - [`clear`](#clear) + - [`put` (deprecated)](#put-deprecated) + - [`del` (deprecated)](#del-deprecated) + - [`batch` (deprecated)](#batch-deprecated) - [Errors](#errors) - [`LEVEL_NOT_FOUND`](#level_not_found) - [`LEVEL_DATABASE_NOT_OPEN`](#level_database_not_open) @@ -84,12 +116,13 @@ - [`LEVEL_NOT_SUPPORTED`](#level_not_supported) - [`LEVEL_LEGACY`](#level_legacy) - [`LEVEL_LOCKED`](#level_locked) + - [`LEVEL_HOOK_ERROR`](#level_hook_error) - [`LEVEL_READONLY`](#level_readonly) - [`LEVEL_CONNECTION_LOST`](#level_connection_lost) - [`LEVEL_REMOTE_ERROR`](#level_remote_error) - [Shared Access](#shared-access) - [Private API For Implementors](#private-api-for-implementors) - - [Example](#example) + - [Example](#example-3) - [`db = AbstractLevel(manifest[, options])`](#db--abstractlevelmanifest-options) - [`db._open(options, callback)`](#db_openoptions-callback) - [`db._close(callback)`](#db_closecallback) @@ -112,7 +145,8 @@ - [`iterator._close(callback)`](#iterator_closecallback) - [`keyIterator = AbstractKeyIterator(db, options)`](#keyiterator--abstractkeyiteratordb-options) - [`valueIterator = AbstractValueIterator(db, options)`](#valueiterator--abstractvalueiteratordb-options) - - [`chainedBatch = AbstractChainedBatch(db)`](#chainedbatch--abstractchainedbatchdb) + - [`chainedBatch = AbstractChainedBatch(db, options)`](#chainedbatch--abstractchainedbatchdb-options) + - [`chainedBatch._add(op)`](#chainedbatch_addop) - [`chainedBatch._put(key, value, options)`](#chainedbatch_putkey-value-options) - [`chainedBatch._del(key, options)`](#chainedbatch_delkey-options) - [`chainedBatch._clear()`](#chainedbatch_clear) @@ -300,7 +334,7 @@ Perform multiple _put_ and/or _del_ operations in bulk. The `operations` argumen Each operation must be an object with at least a `type` property set to either `'put'` or `'del'`. If the `type` is `'put'`, the operation must have `key` and `value` properties. It may optionally have `keyEncoding` and / or `valueEncoding` properties to encode keys or values with a custom encoding for just that operation. If the `type` is `'del'`, the operation must have a `key` property and may optionally have a `keyEncoding` property. -An operation of either type may also have a `sublevel` property, to prefix the key of the operation with the prefix of that sublevel. This allows atomically committing data to multiple sublevels. Keys and values will be encoded by the sublevel, to the same effect as a `sublevel.batch(..)` call. In the following example, the first `value` will be encoded with `'json'` rather than the default encoding of `db`: +An operation of either type may also have a `sublevel` property, to prefix the key of the operation with the prefix of that sublevel. This allows atomically committing data to multiple sublevels. The given `sublevel` must be a descendant of `db`. Keys and values will be encoded by the sublevel, to the same effect as a `sublevel.batch(..)` call. In the following example, the first `value` will be encoded with `'json'` rather than the default encoding of `db`: ```js const people = db.sublevel('people', { valueEncoding: 'json' }) @@ -411,7 +445,7 @@ The `gte` and `lte` range options take precedence over `gt` and `lt` respectivel ### `sublevel = db.sublevel(name[, options])` -Create a [sublevel](#sublevel) that has the same interface as `db` (except for additional, implementation-specific methods) and prefixes the keys of operations before passing them on to `db`. The `name` argument is required and must be a string. +Create a [sublevel](#sublevel) that has the same interface as `db` (except for additional, implementation-specific methods) and prefixes the keys of operations before passing them on to `db`. The `name` argument is required and must be a string, or an array of strings (explained further below). ```js const example = db.sublevel('example') @@ -425,7 +459,7 @@ for await (const [key, value] of example.iterator()) { } ``` -Sublevels effectively separate a database into sections. Think SQL tables, but evented, ranged and realtime! Each sublevel is an `AbstractLevel` instance with its own keyspace, [events](https://github.com/Level/abstract-level#events) and [encodings](https://github.com/Level/abstract-level#encodings). For example, it's possible to have one sublevel with `'buffer'` keys and another with `'utf8'` keys. The same goes for values. Like so: +Sublevels effectively separate a database into sections. Think SQL tables, but evented, ranged and realtime! Each sublevel is an `AbstractLevel` instance with its own keyspace, [encodings](https://github.com/Level/abstract-level#encodings), [hooks](https://github.com/Level/abstract-level#hooks) and [events](https://github.com/Level/abstract-level#events). For example, it's possible to have one sublevel with `'buffer'` keys and another with `'utf8'` keys. The same goes for values. Like so: ```js db.sublevel('one', { valueEncoding: 'json' }) @@ -457,6 +491,21 @@ The `keyEncoding` and `valueEncoding` options are forwarded to the `AbstractLeve Like regular databases, sublevels open themselves but they do not affect the state of the parent database. This means a sublevel can be individually closed and (re)opened. If the sublevel is created while the parent database is opening, it will wait for that to finish. If the parent database is closed, then opening the sublevel will fail and subsequent operations on the sublevel will yield errors with code [`LEVEL_DATABASE_NOT_OPEN`](#errors). +Lastly, the `name` argument can be an array as a shortcut to create nested sublevels. Those are normally created like so: + +```js +const indexes = db.sublevel('idx') +const colorIndex = indexes.sublevel('colors') +``` + +Here, the parent database of `colorIndex` is `indexes`. Operations made on `colorIndex` are thus forwarded from that sublevel to `indexes` and from there to `db`. At each step, hooks and events are available to transform and react to data from a different perspective. Which comes at a (typically small) performance cost that increases with further nested sublevels. If the `indexes` sublevel is only used to organize keys and not directly interfaced with, operations on `colorIndex` can be made faster by skipping `indexes`: + +```js +const colorIndex = db.sublevel(['idx', 'colors']) +``` + +In this case, the parent database of `colorIndex` is `db`. Note that it's still possible to separately create the `indexes` sublevel, but it will be disconnected from `colorIndex`, meaning that `indexes` will not see (live) operations made on `colorIndex`. + ### `encoding = db.keyEncoding([encoding])` Returns the given `encoding` argument as a normalized encoding object that follows the [`level-transcoder`](https://github.com/Level/transcoder) encoding interface. See [Encodings](#encodings) for an introduction. The `encoding` argument may be: @@ -476,7 +525,7 @@ Assume that e.g. `db.keyEncoding().encode(key)` is safe to call at any time incl Same as `db.keyEncoding([encoding])` except that it returns the default `valueEncoding` of the database (if the `encoding` argument is omitted, `null` or `undefined`). -### `key = db.prefixKey(key, keyFormat)` +### `key = db.prefixKey(key, keyFormat[, local])` Add sublevel prefix to the given `key`, which must be already-encoded. If this database is not a sublevel, the given `key` is returned as-is. The `keyFormat` must be one of `'utf8'`, `'buffer'`, `'view'`. If `'utf8'` then `key` must be a string and the return value will be a string. If `'buffer'` then Buffer, if `'view'` then Uint8Array. @@ -487,6 +536,16 @@ console.log(db.prefixKey('a', 'utf8')) // 'a' console.log(sublevel.prefixKey('a', 'utf8')) // '!example!a' ``` +By default, the given `key` will be prefixed to form a fully-qualified key in the context of the _root_ (i.e. top-most) database, as the following example will demonstrate. If `local` is true, the given `key` will instead be prefixed to form a fully-qualified key in the context of the _parent_ database. + +```js +const sublevel = db.sublevel('example') +const nested = sublevel.sublevel('nested') + +console.log(nested.prefixKey('a', 'utf8')) // '!example!!nested!a' +console.log(nested.prefixKey('a', 'utf8', true)) // '!nested!a' +``` + ### `db.defer(fn)` Call the function `fn` at a later time when [`db.status`](#dbstatus) changes to `'open'` or `'closed'`. Used by `abstract-level` itself to implement "deferred open" which is a feature that makes it possible to call operations like `db.put()` before the database has finished opening. The `defer()` method is exposed for implementations and plugins to achieve the same on their custom operations: @@ -507,26 +566,26 @@ When deferring a custom operation, do it early: after normalizing optional argum #### `chainedBatch.put(key, value[, options])` -Queue a `put` operation on this batch, not committed until `write()` is called. This will throw a [`LEVEL_INVALID_KEY`](#errors) or [`LEVEL_INVALID_VALUE`](#errors) error if `key` or `value` is invalid. The optional `options` object may contain: +Add a `put` operation to this chained batch, not committed until `write()` is called. This will throw a [`LEVEL_INVALID_KEY`](#errors) or [`LEVEL_INVALID_VALUE`](#errors) error if `key` or `value` is invalid. The optional `options` object may contain: - `keyEncoding`: custom key encoding for this operation, used to encode the `key`. - `valueEncoding`: custom value encoding for this operation, used to encode the `value`. -- `sublevel` (sublevel instance): act as though the `put` operation is performed on the given sublevel, to similar effect as `sublevel.batch().put(key, value)`. This allows atomically committing data to multiple sublevels. The `key` will be prefixed with the `prefix` of the sublevel, and the `key` and `value` will be encoded by the sublevel (using the default encodings of the sublevel unless `keyEncoding` and / or `valueEncoding` are provided). +- `sublevel` (sublevel instance): act as though the `put` operation is performed on the given sublevel, to similar effect as `sublevel.batch().put(key, value)`. This allows atomically committing data to multiple sublevels. The given `sublevel` must be a descendant of `db`. The `key` will be prefixed with the prefix of the sublevel, and the `key` and `value` will be encoded by the sublevel (using the default encodings of the sublevel unless `keyEncoding` and / or `valueEncoding` are provided). #### `chainedBatch.del(key[, options])` -Queue a `del` operation on this batch, not committed until `write()` is called. This will throw a [`LEVEL_INVALID_KEY`](#errors) error if `key` is invalid. The optional `options` object may contain: +Add a `del` operation to this chained batch, not committed until `write()` is called. This will throw a [`LEVEL_INVALID_KEY`](#errors) error if `key` is invalid. The optional `options` object may contain: - `keyEncoding`: custom key encoding for this operation, used to encode the `key`. -- `sublevel` (sublevel instance): act as though the `del` operation is performed on the given sublevel, to similar effect as `sublevel.batch().del(key)`. This allows atomically committing data to multiple sublevels. The `key` will be prefixed with the `prefix` of the sublevel, and the `key` will be encoded by the sublevel (using the default key encoding of the sublevel unless `keyEncoding` is provided). +- `sublevel` (sublevel instance): act as though the `del` operation is performed on the given sublevel, to similar effect as `sublevel.batch().del(key)`. This allows atomically committing data to multiple sublevels. The given `sublevel` must be a descendant of `db`. The `key` will be prefixed with the prefix of the sublevel, and the `key` will be encoded by the sublevel (using the default key encoding of the sublevel unless `keyEncoding` is provided). #### `chainedBatch.clear()` -Clear all queued operations on this batch. +Remove all operations from this chained batch, so that they will not be committed. #### `chainedBatch.write([options][, callback])` -Commit the queued operations for this batch. All operations will be written atomically, that is, they will either all succeed or fail with no partial commits. +Commit the operations. All operations will be written atomically, that is, they will either all succeed or fail with no partial commits. There are no `options` by default but implementations may add theirs. Note that `write()` does not take encoding options. Those can only be set on `put()` and `del()` because implementations may synchronously forward such calls to an underlying store and thus need keys and values to be encoded at that point. @@ -536,11 +595,11 @@ After `write()` or `close()` has been called, no further operations are allowed. #### `chainedBatch.close([callback])` -Free up underlying resources. This should be done even if the chained batch has zero queued operations. Automatically called by `write()` so normally not necessary to call, unless the intent is to discard a chained batch without committing it. The `callback` function will be called with no arguments. If no callback is provided, a promise is returned. Closing the batch is an idempotent operation, such that calling `close()` more than once is allowed and makes no difference. +Free up underlying resources. This should be done even if the chained batch has zero operations. Automatically called by `write()` so normally not necessary to call, unless the intent is to discard a chained batch without committing it. The `callback` function will be called with no arguments. If no callback is provided, a promise is returned. Closing the batch is an idempotent operation, such that calling `close()` more than once is allowed and makes no difference. #### `chainedBatch.length` -The number of queued operations on the current batch. +The number of operations in this chained batch, including operations that were added by [`prewrite`](#hook--dbhooksprewrite) hook functions if any. #### `chainedBatch.db` @@ -693,7 +752,7 @@ console.log(example.prefix) // '!example!' console.log(nested.prefix) // '!example!!nested!' ``` -#### `sublevel.db` +#### `sublevel.parent` Parent database. A read-only property. @@ -701,10 +760,156 @@ Parent database. A read-only property. const example = db.sublevel('example') const nested = example.sublevel('nested') +console.log(example.parent === db) // true +console.log(nested.parent === example) // true +``` + +#### `sublevel.db` + +Root database. A read-only property. + +```js +const example = db.sublevel('example') +const nested = example.sublevel('nested') + console.log(example.db === db) // true console.log(nested.db === db) // true ``` +### Hooks + +Hooks allow userland _hook functions_ to customize behavior of the database. Each hook is a different extension point, accessible via `db.hooks`. Some are shared between database methods to encapsulate common behavior. A hook is either synchronous or asynchronous, and functions added to a hook must respect that trait. + +#### `hook = db.hooks.prewrite` + +A synchronous hook for modifying or adding operations to [`db.batch([])`](#dbbatchoperations-options-callback), [`db.batch().put()`](#chainedbatchputkey-value-options), [`db.batch().del()`](#chainedbatchdelkey-options), [`db.put()`](#dbputkey-value-options-callback) and [`db.del()`](#dbdelkey-options-callback) calls. It does not include [`db.clear()`](#dbclearoptions-callback) because the entries deleted by such a call are not communicated back to `db`. + +Functions added to this hook will receive two arguments: `op` and `batch`. + +##### Example + +```js +const charwise = require('charwise-compact') +const books = db.sublevel('books', { valueEncoding: 'json' }) +const index = db.sublevel('authors', { keyEncoding: charwise }) + +books.hooks.prewrite.add(function (op, batch) { + if (op.type === 'put') { + batch.add({ + type: 'put', + key: [op.value.author, op.key], + value: '', + sublevel: index + }) + } +}) + +// Will atomically commit it to the author index as well +await books.put('12', { title: 'Siddhartha', author: 'Hesse' }) +``` + +##### Arguments + +###### `op` (object) + +The `op` argument reflects the input operation and has the following properties: `type`, `key`, `keyEncoding`, an optional `sublevel`, and if `type` is `'put'` then also `value` and `valueEncoding`. It can also include userland options, that were provided either in the input operation object (if it originated from [`db.batch([])`](#db_batchoperations-options-callback)) or in the `options` argument of the originating call, for example the `options` in `db.del(key, options)`. + +The `key` and `value` have not yet been encoded at this point. The `keyEncoding` and `valueEncoding` properties are always encoding objects (rather than encoding names like `'json'`) which means hook functions can call (for example) `op.keyEncoding.encode(123)`. + +Hook functions can modify the `key`, `value`, `keyEncoding` and `valueEncoding` properties, but not `type` or `sublevel`. If a hook function modifies `keyEncoding` or `valueEncoding` it can use either encoding names or encoding objects, which will subsequently be normalized to encoding objects. Hook functions can also add custom properties to `op` which will be visible to other hook functions, the private API of the database and in the [`write`](#write) event. + +###### `batch` (object) + +The `batch` argument of the hook function is an interface to add operations, to be committed in the same batch as the input operation(s). This also works if the originating call was a singular operation like `db.put()` because the presence of one or more hook functions will change `db.put()` and `db.del()` to internally use a batch. For originating calls like [`db.batch([])`](#dbbatchoperations-options-callback) that provide multiple input operations, operations will be added after the last input operation, rather than interleaving. The hook function will not be called for operations that were added by either itself or other hook functions. + +###### `batch = batch.add(op)` + +Add a batch operation, using the same format as the operations that [`db.batch([])`](#dbbatchoperations-options-callback) takes. However, it is assumed that `op` can be freely mutated by `abstract-level`. Unlike input operations it will not be cloned before doing so. The `add` method returns `batch` which allows for chaining, similar to the [chained batch](#chainedbatch) API. + +For hook functions to be generic, it is recommended to explicitly define `keyEncoding` and `valueEncoding` properties on `op` (instead of relying on database defaults) or to use an isolated sublevel with known defaults. + +#### `hook = db.hooks.postopen` + +An asynchronous hook that runs after the database has succesfully opened, but before deferred operations are executed and before events are emitted. It thus allows for additional initialization, including reading and writing data that deferred operations might need. + +Functions added to this hook must return a promise and will receive one argument: `options`. If one of the hook functions yields an error (or itself closes the database) then the database will be closed. The postopen hook always runs before the prewrite hook. + +##### Example + +```js +db.hooks.postopen.add(async function (options) { + // Can read and write like usual + return db.put('example', 123, { + valueEncoding: 'json' + }) +}) +``` + +##### Arguments + +###### `options` (object) + +The `options` that were provided in the originating [`db.open(options)`](#dbopenoptions-callback) call, merged with constructor options and defaults. Equivalent to what the private API received in [`db._open(options)`](#db_openoptions-callback). + +#### `hook = db.hooks.newsub` + +A synchronous hook that runs when a `AbstractSublevel` instance has been created by [`db.sublevel(options)`](#sublevel--dbsublevelname-options). Functions added to this hook will receive two arguments: `sublevel` and `options`. + +##### Example + +This hook can be useful to hook into a database and any sublevels created on that database. Userland modules that act like plugins might like the following pattern: + +```js +module.exports = function logger (db, options) { + // Recurse so that db.sublevel('foo', opts) will call logger(sublevel, opts) + db.hooks.newsub.add(logger) + + db.hooks.prewrite.add(function (op, batch) { + console.log('writing', { db, op }) + }) +} +``` + +##### Arguments + +###### `sublevel` (object) + +The `AbstractSublevel` instance that was created. + +###### `options` (object) + +The `options` that were provided in the originating `db.sublevel(options)` call, merged with defaults. Equivalent to what the private API received in [`db._sublevel(options)`](#sublevel--db_sublevelname-options). + +#### `hook` + +##### `hook.add(fn)` + +Add the given `fn` function to this hook, if it wasn't already added. + +##### `hook.delete(fn)` + +Remove the given `fn` function from this hook. + +#### Hook Error Handling + +If a hook function throws an error, it will be wrapped in an error with code [`LEVEL_HOOK_ERROR`](#errors) and abort the originating call: + +```js +try { + await db.put('abc', 123) +} catch (err) { + if (err.code === 'LEVEL_HOOK_ERROR') { + console.log(err.cause) + } +} +``` + +As a result, other hook functions will not be called. + +#### Hooks On Sublevels + +On sublevels and their parent database(s), hooks are triggered in bottom-up order. For example, `db.sublevel('a').sublevel('b').batch(..)` will trigger the `prewrite` hook of sublevel `a`, then the `prewrite` hook of sublevel `b` and then of `db`. Only direct operations on a database will trigger hooks, not when a sublevel is provided as an option. This means `db.batch([{ sublevel, ... }])` will trigger the `prewrite` hook of `db` but not of `sublevel`. These behaviors are symmetrical to [events](#events): `db.batch([{ sublevel, ... }])` will only emit a `write` event from `db` while `db.sublevel(..).batch([{ ... }])` will emit a `write` event from the sublevel and then another from `db` (this time with fully-qualified keys). + ### Encodings Any method that takes a `key` argument, `value` argument or range options like `gte`, hereby jointly referred to as `data`, runs that `data` through an _encoding_. This means to encode input `data` and decode output `data`. @@ -764,29 +969,166 @@ Lastly, one way or another, every implementation _must_ support `data` of type S ### Events -An `abstract-level` database is an [`EventEmitter`](https://nodejs.org/api/events.html) and emits the following events. +An `abstract-level` database is an [`EventEmitter`](https://nodejs.org/api/events.html) and emits the events listed below. + +The `put`, `del` and `batch` events are deprecated in favor of the `write` event and will be removed in a future version of `abstract-level`. If one or more `write` event listeners exist or if the [`prewrite`](#hook--dbhooksprewrite) hook is in use, either of which implies opting-in to the `write` event, then the deprecated events will not be emitted. + +#### `opening` + +Emitted when database is opening. Receives 0 arguments: + +```js +db.once('opening', function () { + console.log('Opening...') +}) +``` -| Event | Description | Arguments | -| :-------- | :------------------- | :------------------- | -| `put` | Entry was updated | `key, value` (any) | -| `del` | Entry was deleted | `key` (any) | -| `batch` | Batch has executed | `operations` (array) | -| `clear` | Entries were deleted | `options` (object) | -| `opening` | Database is opening | - | -| `open` | Database has opened | - | -| `ready` | Alias for `open` | - | -| `closing` | Database is closing | - | -| `closed` | Database has closed. | - | +#### `open` -For example you can do: +Emitted when database has successfully opened. Receives 0 arguments: + +```js +db.once('open', function () { + console.log('Opened!') +}) +``` + +#### `ready` (deprecated) + +Alias for the `open` event. Deprecated in favor of the `open` event. + +#### `closing` + +Emitted when database is closing. Receives 0 arguments. + +#### `closed` + +Emitted when database has successfully closed. Receives 0 arguments. + +#### `write` + +Emitted when data was successfully written to the database as the result of `db.batch()`, `db.put()` or `db.del()`. Receives a single `operations` argument, which is an array containing normalized operation objects. The array will contain at least one operation object and reflects modifications made (and operations added) by the [`prewrite`](#hook--dbhooksprewrite) hook. Normalized means that every operation object has `keyEncoding` and (if `type` is `'put'`) `valueEncoding` properties and these are always encoding objects, rather than their string names like `'utf8'` or whatever was given in the input. + +Operation objects also include userland options that were provided in the `options` argument of the originating call, for example the `options` in a `db.put(key, value, options)` call: + +```js +db.on('write', function (operations) { + for (const op of operations) { + if (op.type === 'put') { + console.log(op.key, op.value, op.foo) + } + } +}) + +// Put with a userland 'foo' option +await db.put('abc', 'xyz', { foo: true }) +``` + +The `key` and `value` of the operation object match the original input, before having encoded it. To provide access to encoded data, the operation object additionally has `encodedKey` and (if `type` is `'put'`) `encodedValue` properties. Event listeners can inspect [`keyEncoding.format`](https://github.com/Level/transcoder#encodingformat) and `valueEncoding.format` to determine the data type of `encodedKey` and `encodedValue`. + +As an example, given a sublevel created with `users = db.sublevel('users', { valueEncoding: 'json' })`, a call like `users.put('isa', { score: 10 })` will emit a `write` event from the sublevel with an `operations` argument that looks like the following. Note that specifics (in data types and encodings) may differ per database at it depends on which encodings an implementation supports and uses internally. This example assumes that the database uses `'utf8'`. + +```js +[{ + type: 'put', + key: 'isa', + value: { score: 10 }, + keyEncoding: users.keyEncoding('utf8'), + valueEncoding: users.valueEncoding('json'), + encodedKey: 'isa', // No change (was already utf8) + encodedValue: '{"score":10}', // JSON-encoded +}] +``` + +Because sublevels encode and then forward operations to their parent database, a separate `write` event will be emitted from `db` with: + +```js +[{ + type: 'put', + key: '!users!isa', // Prefixed + value: '{"score":10}', // No change + keyEncoding: db.keyEncoding('utf8'), + valueEncoding: db.valueEncoding('utf8'), + encodedKey: '!users!isa', + encodedValue: '{"score":10}' +}] +``` + +Similarly, if a `sublevel` option was provided: + +```js +await db.batch() + .del('isa', { sublevel: users }) + .write() +``` + +We'll get: + +```js +[{ + type: 'del', + key: '!users!isa', // Prefixed + keyEncoding: db.keyEncoding('utf8'), + encodedKey: '!users!isa' +}] +``` + +Lastly, newly added `write` event listeners are only called for subsequently created batches (including chained batches): + +```js +const promise = db.batch([{ type: 'del', key: 'abc' }]) +db.on('write', listener) // Too late +await promise +``` + +For the event listener to be called it must be added earlier: + +```js +db.on('write', listener) +await db.batch([{ type: 'del', key: 'abc' }]) +``` + +The same is true for `db.put()` and `db.del()`. + +#### `clear` + +Emitted when a `db.clear()` call completed and entries were thus successfully deleted from the database. Receives a single `options` argument, which is the verbatim `options` argument that was passed to `db.clear(options)` (or an empty object if none) before having encoded range options. + +#### `put` (deprecated) + +Emitted when a `db.put()` call completed and an entry was thus successfully written to the database. Receives `key` and `value` arguments, which are the verbatim `key` and `value` that were passed to `db.put(key, value)` before having encoded them. ```js db.on('put', function (key, value) { - console.log('Updated', { key, value }) + console.log('Wrote', key, value) }) ``` -Any keys, values and range options in these events are the original arguments passed to the relevant operation that triggered the event, before having encoded them. +#### `del` (deprecated) + +Emitted when a `db.del()` call completed and an entry was thus successfully deleted from the database. Receives a single `key` argument, which is the verbatim `key` that was passed to `db.del(key)` before having encoded it. + +```js +db.on('del', function (key) { + console.log('Deleted', key) +}) +``` + +#### `batch` (deprecated) + +Emitted when a `db.batch([])` or chained `db.batch().write()` call completed and the data was thus successfully written to the database. Receives a single `operations` argument, which is the verbatim `operations` array that was passed to `db.batch(operations)` before having encoded it, or the equivalent for a chained `db.batch().write()`. + +```js +db.on('batch', function (operations) { + for (const op of operations) { + if (op.type === 'put') { + console.log('Wrote', op.key, op.value) + } else { + console.log('Deleted', op.key) + } + } +}) +``` ### Errors @@ -901,6 +1243,10 @@ When a method, option or other property was used that has been removed from the When an attempt was made to open a database that is already open in another process or instance. Used by `classic-level` and other implementations of `abstract-level` that use exclusive locks. +#### `LEVEL_HOOK_ERROR` + +An error occurred while running a hook function. The error will have a `cause` property set to the original error thrown from the hook function. + #### `LEVEL_READONLY` When an attempt was made to write data to a read-only database. Used by `many-level`. @@ -1100,7 +1446,7 @@ The default `_del()` invokes `callback` on a next tick. It must be overridden. Perform multiple _put_ and/or _del_ operations in bulk. The `operations` argument is always an array containing a list of operations to be executed sequentially, although as a whole they should be performed as an atomic operation. The `_batch()` method will not be called if the `operations` array is empty. Each operation is guaranteed to have at least `type`, `key` and `keyEncoding` properties. If the type is `put`, the operation will also have `value` and `valueEncoding` properties. There are no default options but `options` will always be an object. If the batch failed, call the `callback` function with an error. Otherwise call `callback` without any arguments. -The public `batch()` method supports encoding options both in the `options` argument and per operation. The private `_batch()` method only receives encoding options per operation. +The public `batch()` method supports encoding options both in the `options` argument and per operation. The private `_batch()` method should only support encoding options per operation, which are guaranteed to be set and to be normalized (the `options` argument in the private API might also contain encoding options but only because it's cheaper to not remove them). The default `_batch()` invokes `callback` on a next tick. It must be overridden. @@ -1224,11 +1570,11 @@ class ExampleSublevel extends AbstractSublevel { const keyEncoding = this.keyEncoding(options.keyEncoding) const keyFormat = keyEncoding.format - key = this.prefixKey(keyEncoding.encode(key), keyFormat) + key = this.prefixKey(keyEncoding.encode(key), keyFormat, true) // The parent database can be accessed like so. Make sure // to forward encoding options and use the full key. - this.db.del(key, { keyEncoding: keyFormat }, ...) + this.parent.del(key, { keyEncoding: keyFormat }, ...) } } ``` @@ -1291,25 +1637,31 @@ The `options` argument must be the original `options` object that was passed to A value iterator has the same interface and constructor arguments as `AbstractIterator` except that it must yields values instead of entries. For further details, see `keyIterator` above. -### `chainedBatch = AbstractChainedBatch(db)` +### `chainedBatch = AbstractChainedBatch(db, options)` The first argument to this constructor must be an instance of the relevant `AbstractLevel` implementation. The constructor will set `chainedBatch.db` which is used (among other things) to access encodings and ensures that `db` will not be garbage collected in case there are no other references to it. +There are two ways to implement a chained batch. If `options.add` is true, only `_add()` will be called. If `options.add` is false or not provided, only `_put()` and `_del()` will be called. + +#### `chainedBatch._add(op)` + +Add a `put` or `del` operation. The `op` object will always have the following properties: `type`, `key`, `keyEncoding` and (if `type` is `'put'`) `value` and `valueEncoding`. + #### `chainedBatch._put(key, value, options)` -Queue a `put` operation on this batch. The `options` object will always have the following properties: `keyEncoding` and `valueEncoding`. +Add a `put` operation. The `options` object will always have the following properties: `keyEncoding` and `valueEncoding`. #### `chainedBatch._del(key, options)` -Queue a `del` operation on this batch. The `options` object will always have the following properties: `keyEncoding`. +Add a `del` operation. The `options` object will always have the following properties: `keyEncoding`. #### `chainedBatch._clear()` -Clear all queued operations on this batch. +Remove all operations from this batch. #### `chainedBatch._write(options, callback)` -The default `_write` method uses `db._batch`. If the `_write` method is overridden it must atomically commit the queued operations. There are no default options but `options` will always be an object. If committing fails, call the `callback` function with an error. Otherwise call `callback` without any arguments. The `_write()` method will not be called if the chained batch has zero queued operations. +The default `_write` method uses `db._batch`. If the `_write` method is overridden it must atomically commit the operations. There are no default options but `options` will always be an object. If committing fails, call the `callback` function with an error. Otherwise call `callback` without any arguments. The `_write()` method will not be called if the chained batch contains zero operations. #### `chainedBatch._close(callback)` diff --git a/UPGRADING.md b/UPGRADING.md index 5b82feb..91f9a97 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -6,6 +6,7 @@ This document describes breaking changes and how to upgrade. For a complete list
Click to expand +- [Upcoming](#upcoming) - [1.0.0](#100) - [1. API parity with `levelup`](#1-api-parity-with-levelup) - [1.1. New: promises](#11-new-promises) @@ -31,6 +32,39 @@ This document describes breaking changes and how to upgrade. For a complete list
+## Upcoming + +The next major release adds [hooks](./README.md#hooks). To achieve hooks, two low-impact breaking changes have been made to nested sublevels. Nested sublevels, no matter their depth, were previously all connected to the same parent database rather than forming a tree. In the following example, the `colorIndex` sublevel would previously forward its operations directly to `db`: + +```js +const indexes = db.sublevel('idx') +const colorIndex = indexes.sublevel('colors') +``` + +It will now forward its operations to `indexes`, which in turn forwards them to `db`. At each step, hooks and events are available to transform and react to data from a different perspective. Which comes at a (typically small) performance cost that increases with further nested sublevels. This decreased performance is the **first breaking change** and mainly affects sublevels nested at a depth of more than 2. + +To optionally negate it, a new feature has been added to `db.sublevel(name)`: it now also accepts a `name` that is an array. If the `indexes` sublevel is only used to organize keys and not directly interfaced with, operations on `colorIndex` can be made faster by skipping `indexes`: + +```js +const colorIndex = db.sublevel(['idx', 'colors']) +``` + +The **second breaking change** is that if a `sublevel` is provided as an option to `db.batch()`, that sublevel must now be a descendant of `db`: + +```js +const colorIndex = indexes.sublevel('colors') +const flavorIndex = indexes.sublevel('flavors') + +// No longer works because colorIndex isn't a descendant of flavorIndex +flavorIndex.batch([{ type: 'del', key: 'blue', sublevel: colorIndex }]) + +// OK +indexes.batch([{ type: 'del', key: 'blue', sublevel: colorIndex }]) + +// OK +db.batch([{ type: 'del', key: 'blue', sublevel: colorIndex }]) +``` + ## 1.0.0 **Introducing `abstract-level`: a fork of [`abstract-leveldown`](https://github.com/Level/abstract-leveldown) that removes the need for [`levelup`](https://github.com/Level/levelup), [`encoding-down`](https://github.com/Level/encoding-down) and more. An `abstract-level` database is a complete solution that doesn't need to be wrapped. It has the same API as `level(up)` including encodings, promises and events. In addition, implementations can now choose to use Uint8Array instead of Buffer. Consumers of an implementation can use both. Sublevels are builtin.** diff --git a/abstract-chained-batch.js b/abstract-chained-batch.js index 3f77b3c..ae3df2b 100644 --- a/abstract-chained-batch.js +++ b/abstract-chained-batch.js @@ -2,25 +2,62 @@ const { fromCallback } = require('catering') const ModuleError = require('module-error') -const { getCallback, getOptions } = require('./lib/common') +const { getCallback, getOptions, emptyOptions } = require('./lib/common') +const { prefixDescendantKey } = require('./lib/prefixes') +const { PrewriteBatch } = require('./lib/prewrite-batch') const kPromise = Symbol('promise') const kStatus = Symbol('status') -const kOperations = Symbol('operations') +const kPublicOperations = Symbol('publicOperations') +const kLegacyOperations = Symbol('legacyOperations') +const kPrivateOperations = Symbol('privateOperations') const kFinishClose = Symbol('finishClose') const kCloseCallbacks = Symbol('closeCallbacks') +const kLength = Symbol('length') +const kPrewriteRun = Symbol('prewriteRun') +const kPrewriteBatch = Symbol('prewriteBatch') +const kPrewriteData = Symbol('prewriteData') +const kAddMode = Symbol('addMode') class AbstractChainedBatch { - constructor (db) { + constructor (db, options) { if (typeof db !== 'object' || db === null) { const hint = db === null ? 'null' : typeof db throw new TypeError(`The first argument must be an abstract-level database, received ${hint}`) } - this[kOperations] = [] + const enableWriteEvent = db.listenerCount('write') > 0 + const enablePrewriteHook = !db.hooks.prewrite.noop + + // Operations for write event. We can skip populating this array (and cloning of + // operations, which is the expensive part) if there are 0 write event listeners. + this[kPublicOperations] = enableWriteEvent ? [] : null + + // Operations for legacy batch event. If user opted-in to write event or prewrite + // hook, skip legacy batch event. We can't skip the batch event based on listener + // count, because a listener may be added between put() or del() and write(). + this[kLegacyOperations] = enableWriteEvent || enablePrewriteHook ? null : [] + + this[kLength] = 0 this[kCloseCallbacks] = [] this[kStatus] = 'open' this[kFinishClose] = this[kFinishClose].bind(this) + this[kAddMode] = getOptions(options, emptyOptions).add === true + + if (enablePrewriteHook) { + // Use separate arrays to collect operations added by hook functions, because + // we wait to apply those until write(). Store these arrays in PrewriteData which + // exists to separate internal data from the public PrewriteBatch interface. + const data = new PrewriteData([], enableWriteEvent ? [] : null) + + this[kPrewriteData] = data + this[kPrewriteBatch] = new PrewriteBatch(db, data[kPrivateOperations], data[kPublicOperations]) + this[kPrewriteRun] = db.hooks.prewrite.run // TODO: document why + } else { + this[kPrewriteData] = null + this[kPrewriteBatch] = null + this[kPrewriteRun] = null + } this.db = db this.db.attachResource(this) @@ -28,85 +65,203 @@ class AbstractChainedBatch { } get length () { - return this[kOperations].length + if (this[kPrewriteData] !== null) { + return this[kLength] + this[kPrewriteData].length + } else { + return this[kLength] + } } put (key, value, options) { - if (this[kStatus] !== 'open') { - throw new ModuleError('Batch is not open: cannot call put() after write() or close()', { - code: 'LEVEL_BATCH_NOT_OPEN' - }) - } - - const err = this.db._checkKey(key) || this.db._checkValue(value) - if (err) throw err + assertStatus(this) + options = getOptions(options, emptyOptions) - const db = options && options.sublevel != null ? options.sublevel : this.db + const delegated = options.sublevel != null + const db = delegated ? options.sublevel : this.db const original = options - const keyEncoding = db.keyEncoding(options && options.keyEncoding) - const valueEncoding = db.valueEncoding(options && options.valueEncoding) - const keyFormat = keyEncoding.format + const keyError = db._checkKey(key) + const valueError = db._checkValue(value) + + if (keyError != null) throw keyError + if (valueError != null) throw valueError + + // Avoid spread operator because of https://bugs.chromium.org/p/chromium/issues/detail?id=1204540 + const op = Object.assign({}, options, { + type: 'put', + key, + value, + keyEncoding: db.keyEncoding(options.keyEncoding), + valueEncoding: db.valueEncoding(options.valueEncoding) + }) + + if (this[kPrewriteRun] !== null) { + try { + // Note: we could have chosen to recurse here so that prewriteBatch.put() would + // call this.put(). But then operations added by hook functions would be inserted + // before rather than after user operations. Instead we process those operations + // lazily in write(). This does hurt the only performance benefit benefit of a + // chained batch though, which is that it avoids blocking the event loop with + // more than one operation at a time. On the other hand, if operations added by + // hook functions are adjacent (i.e. sorted) committing them should be faster. + this[kPrewriteRun](op, this[kPrewriteBatch]) + + // Normalize encodings again in case they were modified + op.keyEncoding = db.keyEncoding(op.keyEncoding) + op.valueEncoding = db.valueEncoding(op.valueEncoding) + } catch (err) { + throw new ModuleError('The prewrite hook failed on batch.put()', { + code: 'LEVEL_HOOK_ERROR', + cause: err + }) + } + } - // Forward encoding options - options = { ...options, keyEncoding: keyFormat, valueEncoding: valueEncoding.format } + // Encode data for private API + const keyEncoding = op.keyEncoding + const preencodedKey = keyEncoding.encode(op.key) + const keyFormat = keyEncoding.format + const encodedKey = delegated ? prefixDescendantKey(preencodedKey, keyFormat, db, this.db) : preencodedKey + const valueEncoding = op.valueEncoding + const encodedValue = valueEncoding.encode(op.value) + const valueFormat = valueEncoding.format // Prevent double prefixing - if (db !== this.db) { - options.sublevel = null + if (delegated) op.sublevel = null + + if (this[kPublicOperations] !== null) { + // Clone op before we mutate it for the private API + const publicOperation = Object.assign({}, op) + publicOperation.encodedKey = encodedKey + publicOperation.encodedValue = encodedValue + + if (delegated) { + // Ensure emitted data makes sense in the context of this db + publicOperation.key = encodedKey + publicOperation.value = encodedValue + publicOperation.keyEncoding = this.db.keyEncoding(keyFormat) + publicOperation.valueEncoding = this.db.valueEncoding(valueFormat) + } + + this[kPublicOperations].push(publicOperation) + } else if (this[kLegacyOperations] !== null) { + const legacyOperation = Object.assign({}, original) + + legacyOperation.type = 'put' + legacyOperation.key = key + legacyOperation.value = value + + this[kLegacyOperations].push(legacyOperation) } - const mappedKey = db.prefixKey(keyEncoding.encode(key), keyFormat) - const mappedValue = valueEncoding.encode(value) + op.key = this.db.prefixKey(encodedKey, keyFormat, true) + op.value = encodedValue + op.keyEncoding = keyFormat + op.valueEncoding = valueFormat - this._put(mappedKey, mappedValue, options) - this[kOperations].push({ ...original, type: 'put', key, value }) + if (this[kAddMode]) { + this._add(op) + } else { + // This "operation as options" trick avoids further cloning + this._put(op.key, encodedValue, op) + } + // Increment only on success + this[kLength]++ return this } _put (key, value, options) {} del (key, options) { - if (this[kStatus] !== 'open') { - throw new ModuleError('Batch is not open: cannot call del() after write() or close()', { - code: 'LEVEL_BATCH_NOT_OPEN' - }) - } - - const err = this.db._checkKey(key) - if (err) throw err + assertStatus(this) + options = getOptions(options, emptyOptions) - const db = options && options.sublevel != null ? options.sublevel : this.db + const delegated = options.sublevel != null + const db = delegated ? options.sublevel : this.db const original = options - const keyEncoding = db.keyEncoding(options && options.keyEncoding) - const keyFormat = keyEncoding.format + const keyError = db._checkKey(key) + + if (keyError != null) throw keyError + + // Avoid spread operator because of https://bugs.chromium.org/p/chromium/issues/detail?id=1204540 + const op = Object.assign({}, options, { + type: 'del', + key, + keyEncoding: db.keyEncoding(options.keyEncoding) + }) + + if (this[kPrewriteRun] !== null) { + try { + this[kPrewriteRun](op, this[kPrewriteBatch]) + + // Normalize encoding again in case it was modified + op.keyEncoding = db.keyEncoding(op.keyEncoding) + } catch (err) { + throw new ModuleError('The prewrite hook failed on batch.del()', { + code: 'LEVEL_HOOK_ERROR', + cause: err + }) + } + } - // Forward encoding options - options = { ...options, keyEncoding: keyFormat } + // Encode data for private API + const keyEncoding = op.keyEncoding + const preencodedKey = keyEncoding.encode(op.key) + const keyFormat = keyEncoding.format + const encodedKey = delegated ? prefixDescendantKey(preencodedKey, keyFormat, db, this.db) : preencodedKey // Prevent double prefixing - if (db !== this.db) { - options.sublevel = null + if (delegated) op.sublevel = null + + if (this[kPublicOperations] !== null) { + // Clone op before we mutate it for the private API + const publicOperation = Object.assign({}, op) + publicOperation.encodedKey = encodedKey + + if (delegated) { + // Ensure emitted data makes sense in the context of this db + publicOperation.key = encodedKey + publicOperation.keyEncoding = this.db.keyEncoding(keyFormat) + } + + this[kPublicOperations].push(publicOperation) + } else if (this[kLegacyOperations] !== null) { + const legacyOperation = Object.assign({}, original) + + legacyOperation.type = 'del' + legacyOperation.key = key + + this[kLegacyOperations].push(legacyOperation) } - this._del(db.prefixKey(keyEncoding.encode(key), keyFormat), options) - this[kOperations].push({ ...original, type: 'del', key }) + op.key = this.db.prefixKey(encodedKey, keyFormat, true) + op.keyEncoding = keyFormat + if (this[kAddMode]) { + this._add(op) + } else { + // This "operation as options" trick avoids further cloning + this._del(op.key, op) + } + + // Increment only on success + this[kLength]++ return this } _del (key, options) {} - clear () { - if (this[kStatus] !== 'open') { - throw new ModuleError('Batch is not open: cannot call clear() after write() or close()', { - code: 'LEVEL_BATCH_NOT_OPEN' - }) - } + _add (op) {} + clear () { + assertStatus(this) this._clear() - this[kOperations] = [] + if (this[kPublicOperations] !== null) this[kPublicOperations] = [] + if (this[kLegacyOperations] !== null) this[kLegacyOperations] = [] + if (this[kPrewriteData] !== null) this[kPrewriteData].clear() + + this[kLength] = 0 return this } @@ -121,17 +276,51 @@ class AbstractChainedBatch { this.nextTick(callback, new ModuleError('Batch is not open: cannot call write() after write() or close()', { code: 'LEVEL_BATCH_NOT_OPEN' })) - } else if (this.length === 0) { + } else if (this[kLength] === 0) { this.close(callback) } else { this[kStatus] = 'writing' + + // Process operations added by prewrite hook functions + if (this[kPrewriteData] !== null) { + const publicOperations = this[kPrewriteData][kPublicOperations] + const privateOperations = this[kPrewriteData][kPrivateOperations] + const length = this[kPrewriteData].length + + for (let i = 0; i < length; i++) { + const op = privateOperations[i] + + // We can _add(), _put() or _del() even though status is now 'writing' because + // status isn't exposed to the private API, so there's no difference in state + // from that perspective, unless an implementation overrides the public write() + // method at its own risk. + if (this[kAddMode]) { + this._add(op) + } else if (op.type === 'put') { + this._put(op.key, op.value, op) + } else { + this._del(op.key, op) + } + } + + if (publicOperations !== null && length !== 0) { + this[kPublicOperations] = this[kPublicOperations].concat(publicOperations) + } + } + this._write(options, (err) => { this[kStatus] = 'closing' this[kCloseCallbacks].push(() => callback(err)) // Emit after setting 'closing' status, because event may trigger a // db close which in turn triggers (idempotently) closing this batch. - if (!err) this.db.emit('batch', this[kOperations]) + if (!err) { + if (this[kPublicOperations] !== null) { + this.db.emit('write', this[kPublicOperations]) + } else if (this[kLegacyOperations] !== null) { + this.db.emit('batch', this[kLegacyOperations]) + } + } this._close(this[kFinishClose]) }) @@ -178,4 +367,42 @@ class AbstractChainedBatch { } } +class PrewriteData { + constructor (privateOperations, publicOperations) { + this[kPrivateOperations] = privateOperations + this[kPublicOperations] = publicOperations + } + + get length () { + return this[kPrivateOperations].length + } + + clear () { + // Clear operation arrays if present. + for (const k of [kPublicOperations, kPrivateOperations]) { + const ops = this[k] + + if (ops !== null) { + // Keep array alive because PrewriteBatch has a reference to it + ops.splice(0, ops.length) + } + } + } +} + +function assertStatus (batch) { + if (batch[kStatus] !== 'open') { + throw new ModuleError('Batch is not open: cannot change operations after write() or close()', { + code: 'LEVEL_BATCH_NOT_OPEN' + }) + } + + // TODO (next major): enforce this regardless of hooks + if (batch[kPrewriteBatch] !== null && batch.db.status !== 'open') { + throw new ModuleError('Chained batch is not available until database is open', { + code: 'LEVEL_DATABASE_NOT_OPEN' + }) + } +} + exports.AbstractChainedBatch = AbstractChainedBatch diff --git a/abstract-iterator.js b/abstract-iterator.js index 48330ec..954b2ab 100644 --- a/abstract-iterator.js +++ b/abstract-iterator.js @@ -2,7 +2,7 @@ const { fromCallback } = require('catering') const ModuleError = require('module-error') -const { getOptions, getCallback } = require('./lib/common') +const { getOptions, getCallback, emptyOptions, noop, deprecate } = require('./lib/common') const kPromise = Symbol('promise') const kCallback = Symbol('callback') @@ -25,10 +25,6 @@ const kValues = Symbol('values') const kLimit = Symbol('limit') const kCount = Symbol('count') -const emptyOptions = Object.freeze({}) -const noop = () => {} -let warnedEnd = false - // This class is an internal utility for common functionality between AbstractIterator, // AbstractKeyIterator and AbstractValueIterator. It's not exported. class CommonIterator { @@ -264,7 +260,7 @@ class CommonIterator { options = { ...options, keyEncoding: keyFormat } } - const mapped = this.db.prefixKey(keyEncoding.encode(target), keyFormat) + const mapped = this.db.prefixKey(keyEncoding.encode(target), keyFormat, false) this._seek(mapped, options) } } @@ -377,14 +373,7 @@ class AbstractIterator extends CommonIterator { } end (callback) { - if (!warnedEnd && typeof console !== 'undefined') { - warnedEnd = true - console.warn(new ModuleError( - 'The iterator.end() method was renamed to close() and end() is an alias that will be removed in a future version', - { code: 'LEVEL_LEGACY' } - )) - } - + deprecate('The iterator.end() method was renamed to close() and end() is an alias that will be removed in a future version') return this.close(callback) } } diff --git a/abstract-level.js b/abstract-level.js index 82878ea..2e59407 100644 --- a/abstract-level.js +++ b/abstract-level.js @@ -3,13 +3,18 @@ const { supports } = require('level-supports') const { Transcoder } = require('level-transcoder') const { EventEmitter } = require('events') -const { fromCallback } = require('catering') +const { fromCallback, fromPromise } = require('catering') const ModuleError = require('module-error') +const combineErrors = require('maybe-combine-errors') const { AbstractIterator } = require('./abstract-iterator') const { DefaultKeyIterator, DefaultValueIterator } = require('./lib/default-kv-iterator') const { DeferredIterator, DeferredKeyIterator, DeferredValueIterator } = require('./lib/deferred-iterator') const { DefaultChainedBatch } = require('./lib/default-chained-batch') -const { getCallback, getOptions } = require('./lib/common') +const { DatabaseHooks } = require('./lib/hooks') +const { PrewriteBatch } = require('./lib/prewrite-batch') +const { EventMonitor } = require('./lib/event-monitor') +const { getCallback, getOptions, noop, emptyOptions } = require('./lib/common') +const { prefixDescendantKey } = require('./lib/prefixes') const rangeOptions = require('./lib/range-options') const kPromise = Symbol('promise') @@ -25,7 +30,7 @@ const kDefaultOptions = Symbol('defaultOptions') const kTranscoder = Symbol('transcoder') const kKeyEncoding = Symbol('keyEncoding') const kValueEncoding = Symbol('valueEncoding') -const noop = () => {} +const kEventMonitor = Symbol('eventMonitor') class AbstractLevel extends EventEmitter { constructor (manifest, options) { @@ -44,6 +49,8 @@ class AbstractLevel extends EventEmitter { this[kOptions] = forward this[kStatus] = 'opening' + this.hooks = new DatabaseHooks() + this.supports = supports(manifest, { status: true, promises: true, @@ -61,12 +68,18 @@ class AbstractLevel extends EventEmitter { iteratorNextv: true, iteratorAll: true, + // TODO: add to level-supports + // We don't have to make this an object (e.g. db.supports.hooks.prewrite) because + // that information is already available in e.g. db.hooks.prewrite != null. + hooks: true, + encodings: manifest.encodings || {}, events: Object.assign({}, manifest.events, { opening: true, open: true, closing: true, closed: true, + write: true, put: true, del: true, batch: true, @@ -74,6 +87,15 @@ class AbstractLevel extends EventEmitter { }) }) + // Monitor event listeners + this[kEventMonitor] = new EventMonitor(this, [ + { name: 'write' }, + { name: 'put', deprecated: true, alt: 'write' }, + { name: 'del', deprecated: true, alt: 'write' }, + { name: 'batch', deprecated: true, alt: 'write' }, + { name: 'ready', deprecated: true, alt: 'open' } + ]) + this[kTranscoder] = new Transcoder(formats(this)) this[kKeyEncoding] = this[kTranscoder].encoding(keyEncoding || 'utf8') this[kValueEncoding] = this[kTranscoder].encoding(valueEncoding || 'utf8') @@ -86,17 +108,25 @@ class AbstractLevel extends EventEmitter { } this[kDefaultOptions] = { - empty: Object.freeze({}), + empty: emptyOptions, entry: Object.freeze({ keyEncoding: this[kKeyEncoding].commonName, valueEncoding: this[kValueEncoding].commonName }), + entryFormat: Object.freeze({ + keyEncoding: this[kKeyEncoding].format, + valueEncoding: this[kValueEncoding].format + }), key: Object.freeze({ keyEncoding: this[kKeyEncoding].commonName + }), + keyFormat: Object.freeze({ + keyEncoding: this[kKeyEncoding].format }) } - // Let subclass finish its constructor + // Before we start opening, let subclass finish its constructor + // and allow events and postopen hook functions to be added. this.nextTick(() => { if (this[kDeferOpen]) { this.open({ passive: false }, noop) @@ -108,6 +138,10 @@ class AbstractLevel extends EventEmitter { return this[kStatus] } + get parent () { + return null + } + keyEncoding (encoding) { return this[kTranscoder].encoding(encoding != null ? encoding : this[kKeyEncoding]) } @@ -165,6 +199,37 @@ class AbstractLevel extends EventEmitter { } this[kStatus] = 'open' + + // Skip postopen hook if it has 0 hook functions + // TODO (not urgent): freeze postopen.run before we start opening + if (this.hooks.postopen.noop) { + return finishOpen() + } + + // Run postopen hook and convert promise to callback + fromPromise(this.hooks.postopen.run(options), (hookErr) => { + // Cancel opening if a hook function threw or closed the database + if (hookErr || this[kStatus] !== 'open') { + return this.close((closeErr) => { + if (hookErr) { + callback(new ModuleError('The postopen hook failed on open()', { + code: 'LEVEL_HOOK_ERROR', + cause: combineErrors([hookErr, closeErr]) + })) + } else { + // Means the hook function is responsible for handling closeErr + callback(new ModuleError('The postopen hook has closed the database', { + code: 'LEVEL_HOOK_ERROR' + })) + } + }) + } + + finishOpen() + }) + }) + + const finishOpen = () => { this[kUndefer]() this.emit(kLanded) @@ -175,7 +240,7 @@ class AbstractLevel extends EventEmitter { if (this[kStatus] === 'open') this.emit('ready') maybeOpened() - }) + } } else if (this[kStatus] === 'open') { this.nextTick(maybeOpened) } else { @@ -301,7 +366,7 @@ class AbstractLevel extends EventEmitter { options = Object.assign({}, options, { keyEncoding: keyFormat, valueEncoding: valueFormat }) } - this._get(this.prefixKey(keyEncoding.encode(key), keyFormat), options, (err, value) => { + this._get(this.prefixKey(keyEncoding.encode(key), keyFormat, true), options, (err, value) => { if (err) { // Normalize not found error for backwards compatibility with abstract-leveldown and level(up) if (err.code === 'LEVEL_NOT_FOUND' || err.notFound || /NotFound/i.test(err)) { @@ -377,7 +442,7 @@ class AbstractLevel extends EventEmitter { return callback[kPromise] } - mappedKeys[i] = this.prefixKey(keyEncoding.encode(key), keyFormat) + mappedKeys[i] = this.prefixKey(keyEncoding.encode(key), keyFormat, true) } this._getMany(mappedKeys, options, (err, values) => { @@ -407,6 +472,13 @@ class AbstractLevel extends EventEmitter { } put (key, value, options, callback) { + if (!this.hooks.prewrite.noop) { + // Forward to batch() which will run the hook + // Note: technically means that put() supports the sublevel option in this case, + // but it generally doesn't per documentation (which makes sense). Same for del(). + return this.batch([{ type: 'put', key, value }], options, callback) + } + callback = getCallback(options, callback) callback = fromCallback(callback, kPromise) options = getOptions(options, this[kDefaultOptions].entry) @@ -427,22 +499,46 @@ class AbstractLevel extends EventEmitter { return callback[kPromise] } + // Encode data for private API const keyEncoding = this.keyEncoding(options.keyEncoding) const valueEncoding = this.valueEncoding(options.valueEncoding) const keyFormat = keyEncoding.format const valueFormat = valueEncoding.format + const enableWriteEvent = this[kEventMonitor].write + const original = options - // Forward encoding options - if (options.keyEncoding !== keyFormat || options.valueEncoding !== valueFormat) { + // Avoid Object.assign() for default options + // TODO: also apply this tweak to get() and getMany() + if (options === this[kDefaultOptions].entry) { + options = this[kDefaultOptions].entryFormat + } else if (options.keyEncoding !== keyFormat || options.valueEncoding !== valueFormat) { options = Object.assign({}, options, { keyEncoding: keyFormat, valueEncoding: valueFormat }) } - const mappedKey = this.prefixKey(keyEncoding.encode(key), keyFormat) - const mappedValue = valueEncoding.encode(value) + const encodedKey = keyEncoding.encode(key) + const prefixedKey = this.prefixKey(encodedKey, keyFormat, true) + const encodedValue = valueEncoding.encode(value) - this._put(mappedKey, mappedValue, options, (err) => { + this._put(prefixedKey, encodedValue, options, (err) => { if (err) return callback(err) - this.emit('put', key, value) + + if (enableWriteEvent) { + const op = Object.assign({}, original, { + type: 'put', + key, + value, + keyEncoding, + valueEncoding, + encodedKey, + encodedValue + }) + + this.emit('write', [op]) + } else { + // TODO (semver-major): remove + this.emit('put', key, value) + } + callback() }) @@ -454,6 +550,11 @@ class AbstractLevel extends EventEmitter { } del (key, options, callback) { + if (!this.hooks.prewrite.noop) { + // Forward to batch() which will run the hook + return this.batch([{ type: 'del', key }], options, callback) + } + callback = getCallback(options, callback) callback = fromCallback(callback, kPromise) options = getOptions(options, this[kDefaultOptions].key) @@ -474,17 +575,39 @@ class AbstractLevel extends EventEmitter { return callback[kPromise] } + // Encode data for private API const keyEncoding = this.keyEncoding(options.keyEncoding) const keyFormat = keyEncoding.format + const enableWriteEvent = this[kEventMonitor].write + const original = options - // Forward encoding options - if (options.keyEncoding !== keyFormat) { + // Avoid Object.assign() for default options + if (options === this[kDefaultOptions].key) { + options = this[kDefaultOptions].keyFormat + } else if (options.keyEncoding !== keyFormat) { options = Object.assign({}, options, { keyEncoding: keyFormat }) } - this._del(this.prefixKey(keyEncoding.encode(key), keyFormat), options, (err) => { + const encodedKey = keyEncoding.encode(key) + const prefixedKey = this.prefixKey(encodedKey, keyFormat, true) + + this._del(prefixedKey, options, (err) => { if (err) return callback(err) - this.emit('del', key) + + if (enableWriteEvent) { + const op = Object.assign({}, original, { + type: 'del', + key, + keyEncoding, + encodedKey + }) + + this.emit('write', [op]) + } else { + // TODO (semver-major): remove + this.emit('del', key) + } + callback() }) @@ -495,6 +618,9 @@ class AbstractLevel extends EventEmitter { this.nextTick(callback) } + // TODO (future): add way for implementations to declare which options are for the + // whole batch rather than defaults for individual operations. E.g. the sync option + // of classic-level, that should not be copied to individual operations. batch (operations, options, callback) { if (!arguments.length) { if (this[kStatus] === 'opening') return new DefaultChainedBatch(this) @@ -512,6 +638,7 @@ class AbstractLevel extends EventEmitter { callback = fromCallback(callback, kPromise) options = getOptions(options, this[kDefaultOptions].empty) + // TODO (not urgent): freeze prewrite hook and write event if (this[kStatus] === 'opening') { this.defer(() => this.batch(operations, options, callback)) return callback[kPromise] @@ -531,61 +658,131 @@ class AbstractLevel extends EventEmitter { return callback[kPromise] } - const mapped = new Array(operations.length) - const { keyEncoding: ke, valueEncoding: ve, ...forward } = options - - for (let i = 0; i < operations.length; i++) { - if (typeof operations[i] !== 'object' || operations[i] === null) { - this.nextTick(callback, new TypeError('A batch operation must be an object')) + const length = operations.length + const enablePrewriteHook = !this.hooks.prewrite.noop + const enableWriteEvent = this[kEventMonitor].write + const publicOperations = enableWriteEvent ? new Array(length) : null + const privateOperations = new Array(length) + const prewriteBatch = enablePrewriteHook + ? new PrewriteBatch(this, privateOperations, publicOperations) + : null + + for (let i = 0; i < length; i++) { + // Clone the op so that we can freely mutate it. We can't use a class because the + // op can have userland properties that we'd have to copy, negating the performance + // benefits of a class. So use a plain object. + const op = Object.assign({}, options, operations[i]) + + // Hook functions can modify op but not its type or sublevel, so cache those + const isPut = op.type === 'put' + const delegated = op.sublevel != null + const db = delegated ? op.sublevel : this + const keyError = db._checkKey(op.key) + + if (keyError != null) { + this.nextTick(callback, keyError) return callback[kPromise] } - const op = Object.assign({}, operations[i]) + op.keyEncoding = db.keyEncoding(op.keyEncoding) + + if (isPut) { + const valueError = db._checkValue(op.value) + + if (valueError != null) { + this.nextTick(callback, valueError) + return callback[kPromise] + } - if (op.type !== 'put' && op.type !== 'del') { + op.valueEncoding = db.valueEncoding(op.valueEncoding) + } else if (op.type !== 'del') { this.nextTick(callback, new TypeError("A batch operation must have a type property that is 'put' or 'del'")) return callback[kPromise] } - const err = this._checkKey(op.key) + if (enablePrewriteHook) { + try { + this.hooks.prewrite.run(op, prewriteBatch) - if (err) { - this.nextTick(callback, err) - return callback[kPromise] + // Normalize encodings again in case they were modified + op.keyEncoding = db.keyEncoding(op.keyEncoding) + if (isPut) op.valueEncoding = db.valueEncoding(op.valueEncoding) + } catch (err) { + this.nextTick(callback, new ModuleError('The prewrite hook failed on batch()', { + code: 'LEVEL_HOOK_ERROR', + cause: err + })) + + return callback[kPromise] + } } - const db = op.sublevel != null ? op.sublevel : this - const keyEncoding = db.keyEncoding(op.keyEncoding || ke) + // Encode data for private API + // TODO: benchmark a try/catch around this + const keyEncoding = op.keyEncoding + const preencodedKey = keyEncoding.encode(op.key) const keyFormat = keyEncoding.format + const encodedKey = delegated ? prefixDescendantKey(preencodedKey, keyFormat, db, this) : preencodedKey - op.key = db.prefixKey(keyEncoding.encode(op.key), keyFormat) - op.keyEncoding = keyFormat + // Prevent double prefixing + if (delegated) op.sublevel = null - if (op.type === 'put') { - const valueErr = this._checkValue(op.value) + let publicOperation = null - if (valueErr) { - this.nextTick(callback, valueErr) - return callback[kPromise] - } + if (enableWriteEvent) { + // Clone op before we mutate it for the private API + // TODO (future semver-major): consider sending this shape to private API too + publicOperation = Object.assign({}, op) + publicOperation.encodedKey = encodedKey - const valueEncoding = db.valueEncoding(op.valueEncoding || ve) + if (delegated) { + // Ensure emitted data makes sense in the context of this db + publicOperation.key = encodedKey + publicOperation.keyEncoding = this.keyEncoding(keyFormat) + } - op.value = valueEncoding.encode(op.value) - op.valueEncoding = valueEncoding.format + publicOperations[i] = publicOperation } - // Prevent double prefixing - if (db !== this) { - op.sublevel = null + op.key = this.prefixKey(encodedKey, keyFormat, true) + op.keyEncoding = keyFormat + + if (isPut) { + const valueEncoding = op.valueEncoding + const encodedValue = valueEncoding.encode(op.value) + const valueFormat = valueEncoding.format + + op.value = encodedValue + op.valueEncoding = valueFormat + + if (enableWriteEvent) { + publicOperation.encodedValue = encodedValue + + if (delegated) { + publicOperation.value = encodedValue + publicOperation.valueEncoding = this.valueEncoding(valueFormat) + } + } } - mapped[i] = op + privateOperations[i] = op } - this._batch(mapped, forward, (err) => { + // TODO (future): maybe add separate hook to run on private data. Currently can't work + // because prefixing happens too soon; we need to move that logic to the private + // API of AbstractSublevel (or reimplement with hooks). TBD how it'd work in chained + // batch. Hook would look something like hooks.midwrite.run(privateOperations, ...). + + this._batch(privateOperations, options, (err) => { if (err) return callback(err) - this.emit('batch', operations) + + if (enableWriteEvent) { + this.emit('write', publicOperations) + } else if (!enablePrewriteHook) { + // TODO (semver-major): remove + this.emit('batch', operations) + } + callback() }) @@ -597,14 +794,28 @@ class AbstractLevel extends EventEmitter { } sublevel (name, options) { - return this._sublevel(name, AbstractSublevel.defaults(options)) + const xopts = AbstractSublevel.defaults(options) + const sublevel = this._sublevel(name, xopts) + + if (!this.hooks.newsub.noop) { + try { + this.hooks.newsub.run(sublevel, xopts) + } catch (err) { + throw new ModuleError('The newsub hook failed on sublevel()', { + code: 'LEVEL_HOOK_ERROR', + cause: err + }) + } + } + + return sublevel } _sublevel (name, options) { return new AbstractSublevel(this, name, options) } - prefixKey (key, keyFormat) { + prefixKey (key, keyFormat, local) { return key } diff --git a/index.d.ts b/index.d.ts index 3cdcc8c..f102c34 100644 --- a/index.d.ts +++ b/index.d.ts @@ -10,7 +10,9 @@ export { AbstractBatchOperation, AbstractBatchPutOperation, AbstractBatchDelOperation, - AbstractClearOptions + AbstractClearOptions, + AbstractDatabaseHooks, + AbstractHook } from './types/abstract-level' export { diff --git a/lib/abstract-sublevel.js b/lib/abstract-sublevel.js index aaca0fc..1b4d9e2 100644 --- a/lib/abstract-sublevel.js +++ b/lib/abstract-sublevel.js @@ -8,9 +8,11 @@ const { AbstractSublevelValueIterator } = require('./abstract-sublevel-iterator') -const kPrefix = Symbol('prefix') -const kUpperBound = Symbol('upperBound') +const kGlobalPrefix = Symbol('prefix') +const kLocalPrefix = Symbol('localPrefix') +const kGlobalUpperBound = Symbol('upperBound') const kPrefixRange = Symbol('prefixRange') +const kRoot = Symbol('root') const kParent = Symbol('parent') const kUnfix = Symbol('unfix') @@ -45,41 +47,49 @@ module.exports = function ({ AbstractLevel }) { constructor (db, name, options) { // Don't forward AbstractSublevel options to AbstractLevel const { separator, manifest, ...forward } = AbstractSublevel.defaults(options) - name = trim(name, separator) + const names = [].concat(name).map(name => trim(name, separator)) // Reserve one character between separator and name to give us an upper bound const reserved = separator.charCodeAt(0) + 1 - const parent = db[kParent] || db + const root = db[kRoot] || db // Keys should sort like ['!a!', '!a!!a!', '!a"', '!aa!', '!b!']. // Use ASCII for consistent length between string, Buffer and Uint8Array - if (!textEncoder.encode(name).every(x => x > reserved && x < 127)) { - throw new ModuleError(`Prefix must use bytes > ${reserved} < ${127}`, { + if (!names.every(name => textEncoder.encode(name).every(x => x > reserved && x < 127))) { + throw new ModuleError(`Sublevel name must use bytes > ${reserved} < ${127}`, { code: 'LEVEL_INVALID_PREFIX' }) } - super(mergeManifests(parent, manifest), forward) + super(mergeManifests(db, manifest), forward) - const prefix = (db.prefix || '') + separator + name + separator - const upperBound = prefix.slice(0, -1) + String.fromCharCode(reserved) + const localPrefix = names.map(name => separator + name + separator).join('') + const globalPrefix = (db.prefix || '') + localPrefix + const globalUpperBound = globalPrefix.slice(0, -1) + String.fromCharCode(reserved) - this[kParent] = parent - this[kPrefix] = new MultiFormat(prefix) - this[kUpperBound] = new MultiFormat(upperBound) + // Most operations are forwarded to the parent database, but clear() and iterators + // still forward to the root database - which is older logic and does not yet need + // to change, until we add some form of preread or postread hooks. + this[kRoot] = root + this[kParent] = db + this[kGlobalPrefix] = new MultiFormat(globalPrefix) + this[kGlobalUpperBound] = new MultiFormat(globalUpperBound) + this[kLocalPrefix] = new MultiFormat(localPrefix) this[kUnfix] = new Unfixer() - this.nextTick = parent.nextTick + this.nextTick = db.nextTick } - prefixKey (key, keyFormat) { + prefixKey (key, keyFormat, local) { + const prefix = local ? this[kLocalPrefix] : this[kGlobalPrefix] + if (keyFormat === 'utf8') { - return this[kPrefix].utf8 + key + return prefix.utf8 + key } else if (key.byteLength === 0) { // Fast path for empty key (no copy) - return this[kPrefix][keyFormat] + return prefix[keyFormat] } else if (keyFormat === 'view') { - const view = this[kPrefix].view + const view = prefix.view const result = new Uint8Array(view.byteLength + key.byteLength) result.set(view, 0) @@ -87,7 +97,7 @@ module.exports = function ({ AbstractLevel }) { return result } else { - const buffer = this[kPrefix].buffer + const buffer = prefix.buffer return Buffer.concat([buffer, key], buffer.byteLength + key.byteLength) } } @@ -95,27 +105,31 @@ module.exports = function ({ AbstractLevel }) { // Not exposed for now. [kPrefixRange] (range, keyFormat) { if (range.gte !== undefined) { - range.gte = this.prefixKey(range.gte, keyFormat) + range.gte = this.prefixKey(range.gte, keyFormat, false) } else if (range.gt !== undefined) { - range.gt = this.prefixKey(range.gt, keyFormat) + range.gt = this.prefixKey(range.gt, keyFormat, false) } else { - range.gte = this[kPrefix][keyFormat] + range.gte = this[kGlobalPrefix][keyFormat] } if (range.lte !== undefined) { - range.lte = this.prefixKey(range.lte, keyFormat) + range.lte = this.prefixKey(range.lte, keyFormat, false) } else if (range.lt !== undefined) { - range.lt = this.prefixKey(range.lt, keyFormat) + range.lt = this.prefixKey(range.lt, keyFormat, false) } else { - range.lte = this[kUpperBound][keyFormat] + range.lte = this[kGlobalUpperBound][keyFormat] } } get prefix () { - return this[kPrefix].utf8 + return this[kGlobalPrefix].utf8 } get db () { + return this[kRoot] + } + + get parent () { return this[kParent] } @@ -145,30 +159,32 @@ module.exports = function ({ AbstractLevel }) { this[kParent].batch(operations, options, callback) } + // TODO: call parent instead of root _clear (options, callback) { // TODO (refactor): move to AbstractLevel this[kPrefixRange](options, options.keyEncoding) - this[kParent].clear(options, callback) + this[kRoot].clear(options, callback) } + // TODO: call parent instead of root _iterator (options) { // TODO (refactor): move to AbstractLevel this[kPrefixRange](options, options.keyEncoding) - const iterator = this[kParent].iterator(options) - const unfix = this[kUnfix].get(this[kPrefix].utf8.length, options.keyEncoding) + const iterator = this[kRoot].iterator(options) + const unfix = this[kUnfix].get(this[kGlobalPrefix].utf8.length, options.keyEncoding) return new AbstractSublevelIterator(this, options, iterator, unfix) } _keys (options) { this[kPrefixRange](options, options.keyEncoding) - const iterator = this[kParent].keys(options) - const unfix = this[kUnfix].get(this[kPrefix].utf8.length, options.keyEncoding) + const iterator = this[kRoot].keys(options) + const unfix = this[kUnfix].get(this[kGlobalPrefix].utf8.length, options.keyEncoding) return new AbstractSublevelKeyIterator(this, options, iterator, unfix) } _values (options) { this[kPrefixRange](options, options.keyEncoding) - const iterator = this[kParent].values(options) + const iterator = this[kRoot].values(options) return new AbstractSublevelValueIterator(this, options, iterator) } } diff --git a/lib/common.js b/lib/common.js index 2786f5d..492048f 100644 --- a/lib/common.js +++ b/lib/common.js @@ -1,5 +1,8 @@ 'use strict' +const ModuleError = require('module-error') +const deprecations = new Set() + exports.getCallback = function (options, callback) { return typeof options === 'function' ? options : callback } @@ -15,3 +18,19 @@ exports.getOptions = function (options, def) { return {} } + +exports.emptyOptions = Object.freeze({}) +exports.noop = function () {} + +exports.deprecate = function (message) { + if (!deprecations.has(message)) { + deprecations.add(message) + + // Avoid polyfills + const c = globalThis.console + + if (typeof c !== 'undefined' && typeof c.warn === 'function') { + c.warn(new ModuleError(message, { code: 'LEVEL_LEGACY' })) + } + } +} diff --git a/lib/default-chained-batch.js b/lib/default-chained-batch.js index cebb31c..6c2ef59 100644 --- a/lib/default-chained-batch.js +++ b/lib/default-chained-batch.js @@ -7,27 +7,25 @@ const kEncoded = Symbol('encoded') // Functional default for chained batch, with support of deferred open class DefaultChainedBatch extends AbstractChainedBatch { constructor (db) { - super(db) + // Opt-in to _add() instead of _put() and _del() + super(db, { add: true }) this[kEncoded] = [] } - _put (key, value, options) { - this[kEncoded].push({ ...options, type: 'put', key, value }) - } - - _del (key, options) { - this[kEncoded].push({ ...options, type: 'del', key }) + _add (op) { + this[kEncoded].push(op) } _clear () { this[kEncoded] = [] } - // Assumes this[kEncoded] cannot change after write() _write (options, callback) { if (this.db.status === 'opening') { this.db.defer(() => this._write(options, callback)) } else if (this.db.status === 'open') { + // Need to call the private rather than public method, to prevent + // recursion, double prefixing, double encoding and double hooks. if (this[kEncoded].length === 0) this.nextTick(callback) else this.db._batch(this[kEncoded], options, callback) } else { diff --git a/lib/event-monitor.js b/lib/event-monitor.js new file mode 100644 index 0000000..311f6ba --- /dev/null +++ b/lib/event-monitor.js @@ -0,0 +1,41 @@ +'use strict' + +const { deprecate } = require('./common') + +exports.EventMonitor = class EventMonitor { + constructor (emitter, events) { + for (const event of events) { + // Track whether listeners are present + this[event.name] = false + + // Prepare deprecation message + if (event.deprecated) { + event.message = `The '${event.name}' event is deprecated in favor of '${event.alt}' and will be removed in a future version of abstract-level` + } + } + + const map = new Map(events.map(e => [e.name, e])) + const monitor = this + + emitter.on('newListener', beforeAdded) + emitter.on('removeListener', afterRemoved) + + function beforeAdded (name) { + const event = map.get(name) + + if (event !== undefined) { + monitor[name] = true + + if (event.deprecated) { + deprecate(event.message) + } + } + } + + function afterRemoved (name) { + if (map.has(name)) { + monitor[name] = this.listenerCount(name) > 0 + } + } + } +} diff --git a/lib/hooks.js b/lib/hooks.js new file mode 100644 index 0000000..3468c94 --- /dev/null +++ b/lib/hooks.js @@ -0,0 +1,79 @@ +'use strict' + +const { noop } = require('./common') + +const kFunctions = Symbol('functions') +const kAsync = Symbol('async') + +class DatabaseHooks { + constructor () { + this.postopen = new Hook({ async: true }) + this.prewrite = new Hook({ async: false }) + this.newsub = new Hook({ async: false }) + } +} + +class Hook { + constructor (options) { + this[kAsync] = options.async + this[kFunctions] = new Set() + + // Offer a fast way to check if hook functions are present. We could also expose a + // size getter, which would be slower, or check it by hook.run !== noop, which would + // not allow userland to do the same check. + this.noop = true + this.run = runner(this) + } + + add (fn) { + // Validate now rather than in asynchronous code paths + assertFunction(fn) + this[kFunctions].add(fn) + this.noop = false + this.run = runner(this) + } + + delete (fn) { + assertFunction(fn) + this[kFunctions].delete(fn) + this.noop = this[kFunctions].size === 0 + this.run = runner(this) + } +} + +const assertFunction = function (fn) { + if (typeof fn !== 'function') { + const hint = fn === null ? 'null' : typeof fn + throw new TypeError(`The first argument must be a function, received ${hint}`) + } +} + +const runner = function (hook) { + if (hook.noop) { + return noop + } else if (hook[kFunctions].size === 1) { + const [fn] = hook[kFunctions] + return fn + } else if (hook[kAsync]) { + // The run function should not reference hook, so that consumers like chained batch + // and db.open() can save a reference to hook.run and safely assume it won't change + // during their lifetime or async work. + const run = async function (functions, ...args) { + for (const fn of functions) { + await fn(...args) + } + } + + return run.bind(null, Array.from(hook[kFunctions])) + } else { + const run = function (functions, ...args) { + for (const fn of functions) { + fn(...args) + } + } + + return run.bind(null, Array.from(hook[kFunctions])) + } +} + +exports.DatabaseHooks = DatabaseHooks diff --git a/lib/prefixes.js b/lib/prefixes.js new file mode 100644 index 0000000..8fa43f7 --- /dev/null +++ b/lib/prefixes.js @@ -0,0 +1,12 @@ +'use strict' + +exports.prefixDescendantKey = function (key, keyFormat, descendant, ancestor) { + // TODO: optimize + // TODO: throw when ancestor is not descendant's ancestor? + while (descendant !== null && descendant !== ancestor) { + key = descendant.prefixKey(key, keyFormat, true) + descendant = descendant.parent + } + + return key +} diff --git a/lib/prewrite-batch.js b/lib/prewrite-batch.js new file mode 100644 index 0000000..452e27f --- /dev/null +++ b/lib/prewrite-batch.js @@ -0,0 +1,90 @@ +'use strict' + +const { prefixDescendantKey } = require('./prefixes') + +const kDb = Symbol('db') +const kPrivateOperations = Symbol('privateOperations') +const kPublicOperations = Symbol('publicOperations') + +// An interface for prewrite hook functions to add operations +class PrewriteBatch { + constructor (db, privateOperations, publicOperations) { + this[kDb] = db + + // Note: if for db.batch([]), these arrays include input operations (or empty slots + // for them) but if for chained batch then it does not. Small implementation detail. + this[kPrivateOperations] = privateOperations + this[kPublicOperations] = publicOperations + } + + add (op) { + const isPut = op.type === 'put' + const delegated = op.sublevel != null + const db = delegated ? op.sublevel : this[kDb] + + const keyError = db._checkKey(op.key) + if (keyError != null) throw keyError + + op.keyEncoding = db.keyEncoding(op.keyEncoding) + + if (isPut) { + const valueError = db._checkValue(op.value) + if (valueError != null) throw valueError + + op.valueEncoding = db.valueEncoding(op.valueEncoding) + } else if (op.type !== 'del') { + throw new TypeError("A batch operation must have a type property that is 'put' or 'del'") + } + + // Encode data for private API + const keyEncoding = op.keyEncoding + const preencodedKey = keyEncoding.encode(op.key) + const keyFormat = keyEncoding.format + const encodedKey = delegated ? prefixDescendantKey(preencodedKey, keyFormat, db, this[kDb]) : preencodedKey + + // Prevent double prefixing + if (delegated) op.sublevel = null + + let publicOperation = null + + if (this[kPublicOperations] !== null) { + // Clone op before we mutate it for the private API + publicOperation = Object.assign({}, op) + publicOperation.encodedKey = encodedKey + + if (delegated) { + // Ensure emitted data makes sense in the context of this[kDb] + publicOperation.key = encodedKey + publicOperation.keyEncoding = this[kDb].keyEncoding(keyFormat) + } + + this[kPublicOperations].push(publicOperation) + } + + op.key = this[kDb].prefixKey(encodedKey, keyFormat, true) + op.keyEncoding = keyFormat + + if (isPut) { + const valueEncoding = op.valueEncoding + const encodedValue = valueEncoding.encode(op.value) + const valueFormat = valueEncoding.format + + op.value = encodedValue + op.valueEncoding = valueFormat + + if (publicOperation !== null) { + publicOperation.encodedValue = encodedValue + + if (delegated) { + publicOperation.value = encodedValue + publicOperation.valueEncoding = this[kDb].valueEncoding(valueFormat) + } + } + } + + this[kPrivateOperations].push(op) + return this + } +} + +exports.PrewriteBatch = PrewriteBatch diff --git a/package.json b/package.json index 53ca2d5..2a36e8f 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "is-buffer": "^2.0.5", "level-supports": "^4.0.0", "level-transcoder": "^1.0.1", + "maybe-combine-errors": "^1.0.0", "module-error": "^1.0.1", "queue-microtask": "^1.2.3" }, diff --git a/test/batch-test.js b/test/batch-test.js index 927316f..8926429 100644 --- a/test/batch-test.js +++ b/test/batch-test.js @@ -133,16 +133,18 @@ exports.args = function (test, testCommon) { const type = operation === null ? 'null' : typeof operation test('test batch() with ' + type + ' operation', assertAsync.ctx(function (t) { - t.plan(5) + t.plan(3) db.batch([operation], assertAsync(function (err) { - t.is(err && err.name, 'TypeError') - t.is(err && err.message, 'A batch operation must be an object', 'correct error message (callback)') + // We can either explicitly check the type of the op and throw a TypeError, + // or skip that for performance reasons in which case the next thing checked + // will be op.key or op.type. Doesn't matter, because we've documented that + // TypeErrors and such are not part of the semver contract. + t.ok(err && (err.name === 'TypeError' || err.code === 'LEVEL_INVALID_KEY')) })) db.batch([operation]).catch(function (err) { - t.is(err.name, 'TypeError') - t.is(err.message, 'A batch operation must be an object', 'correct error message (promise)') + t.ok(err.name === 'TypeError' || err.code === 'LEVEL_INVALID_KEY') }) })) }) diff --git a/test/events/write.js b/test/events/write.js new file mode 100644 index 0000000..2f2e17f --- /dev/null +++ b/test/events/write.js @@ -0,0 +1,181 @@ +'use strict' + +module.exports = function (test, testCommon) { + for (const deferred of [false, true]) { + for (const method of ['batch', 'chained batch', 'singular']) { + for (const withSublevel of (method === 'singular' ? [false] : [false, true])) { + test(`db emits write event for ${method} put operation (deferred: ${deferred}, sublevel: ${withSublevel})`, async function (t) { + t.plan(1) + + const db = testCommon.factory() + const sublevel = withSublevel ? db.sublevel('abc') : null + + if (!deferred) { + await db.open() + if (withSublevel) await sublevel.open() + } + + // Note: may return a transcoder encoding, which unfortunately makes the below + // assertions a little less precise (i.e. we can't compare output data). But + // in places where we expect encoded data, we can use strings (rather than + // numbers) as the input to encode(), which'll tell us that encoding did happen. + const dbEncoding = db.keyEncoding('utf8') + const subEncoding = withSublevel ? sublevel.keyEncoding('utf8') : null + + db.on('write', function (ops) { + t.same(ops, [ + { + type: 'put', + key: withSublevel ? sublevel.prefixKey(subEncoding.encode('456'), subEncoding.format, true) : 456, + value: withSublevel ? subEncoding.encode('99') : 99, + keyEncoding: db.keyEncoding(withSublevel ? subEncoding.format : 'utf8'), + valueEncoding: db.valueEncoding(withSublevel ? subEncoding.format : 'utf8'), + encodedKey: withSublevel ? sublevel.prefixKey(subEncoding.encode('456'), subEncoding.format, true) : dbEncoding.encode('456'), + encodedValue: (withSublevel ? subEncoding : dbEncoding).encode('99'), + custom: 123, + sublevel: null // Should be unset + } + ], 'got write event') + }) + + switch (method) { + case 'batch': + await db.batch([{ type: 'put', key: 456, value: 99, custom: 123, sublevel }]) + break + case 'chained batch': + await db.batch().put(456, 99, { custom: 123, sublevel }).write() + break + case 'singular': + // Does not support sublevel option + await db.put(456, 99, { custom: 123, sublevel }) + break + } + + return db.close() + }) + + test(`db emits write event for ${method} del operation (deferred: ${deferred}, sublevel: ${withSublevel})`, async function (t) { + t.plan(1) + + const db = testCommon.factory() + const sublevel = withSublevel ? db.sublevel('abc') : null + + if (!deferred) { + await db.open() + if (withSublevel) await sublevel.open() + } + + // See notes above, in the put test + const dbEncoding = db.keyEncoding('utf8') + const subEncoding = withSublevel ? sublevel.keyEncoding('utf8') : null + + db.on('write', function (ops) { + t.same(ops, [ + { + type: 'del', + key: withSublevel ? sublevel.prefixKey(subEncoding.encode('456'), subEncoding.format, true) : 456, + keyEncoding: db.keyEncoding(withSublevel ? subEncoding.format : 'utf8'), + encodedKey: withSublevel ? sublevel.prefixKey(subEncoding.encode('456'), subEncoding.format, true) : dbEncoding.encode('456'), + custom: 123, + sublevel: null // Should be unset + } + ], 'got write event') + }) + + switch (method) { + case 'batch': + await db.batch([{ type: 'del', key: 456, custom: 123, sublevel }]) + break + case 'chained batch': + await db.batch().del(456, { custom: 123, sublevel }).write() + break + case 'singular': + // Does not support sublevel option + await db.del(456, { custom: 123, sublevel }) + break + } + + return db.close() + }) + } + } + + for (const method of ['batch', 'chained batch']) { + test(`db emits write event for multiple ${method} operations (deferred: ${deferred})`, async function (t) { + t.plan(1) + + const db = testCommon.factory() + if (!deferred) await db.open() + + db.on('write', function (ops) { + t.same(ops.map(op => op.key), ['a', 'b'], 'got multiple operations in one event') + }) + + switch (method) { + case 'batch': + await db.batch([{ type: 'put', key: 'a', value: 'foo' }, { type: 'del', key: 'b' }]) + break + case 'chained batch': + await db.batch().put('a', 'foo').del('b').write() + break + } + + return db.close() + }) + } + + for (const method of ['batch', 'chained batch', 'singular']) { + test(`db emits write event for ${method} operation in favor of deprecated events (deferred: ${deferred})`, async function (t) { + t.plan(5) + + const keys = [] + const db = testCommon.factory() + if (!deferred) await db.open() + + db.on('write', function (ops) { + keys.push(...ops.map(op => op.key)) + }) + + db.on('batch', function () { + t.fail('should not get batch event') + }) + + db.on('put', function () { + t.fail('should not get put event') + }) + + db.on('del', function () { + t.fail('should not get del event') + }) + + // Once we remove the deprecated events, this test would still pass, but we should then remove it. + t.ok(db.supports.events.batch, 'supports batch event') + t.ok(db.supports.events.put, 'supports put event') + t.ok(db.supports.events.del, 'supports del event') + + switch (method) { + case 'batch': + await db.batch([{ type: 'put', key: 'a', value: 'a' }]) + t.is(keys.pop(), 'a', 'got write event for batch put') + await db.batch([{ type: 'del', key: 'b' }]) + t.is(keys.pop(), 'b', 'got write event for batch del') + break + case 'chained batch': + await db.batch().put('c', 'c').write() + t.is(keys.pop(), 'c', 'got write event for chained batch put') + await db.batch().del('d').write() + t.is(keys.pop(), 'd', 'got write event for chained batch del') + break + case 'singular': + await db.put('e', 'e') + t.is(keys.pop(), 'e', 'got write event for put') + await db.del('f') + t.is(keys.pop(), 'f', 'got write event for del') + break + } + + return db.close() + }) + } + } +} diff --git a/test/hooks/newsub.js b/test/hooks/newsub.js new file mode 100644 index 0000000..33ef7db --- /dev/null +++ b/test/hooks/newsub.js @@ -0,0 +1,57 @@ +'use strict' + +const shared = require('./shared') + +module.exports = function (test, testCommon) { + shared(test, testCommon, 'newsub') + + test('newsub hook function receives sublevel and default options', async function (t) { + t.plan(3) + + const db = testCommon.factory() + + let instance + db.hooks.newsub.add(function (sublevel, options) { + instance = sublevel + + // Recursing is the main purpose of this hook + t.ok(sublevel.hooks, 'can access sublevel hooks') + t.same(options, { separator: '!' }) + }) + + t.ok(db.sublevel('sub') === instance) + return db.close() + }) + + test('newsub hook function receives userland options', async function (t) { + t.plan(1) + + const db = testCommon.factory() + + db.hooks.newsub.add(function (sublevel, options) { + t.same(options, { separator: '!', userland: 123 }) + }) + + db.sublevel('sub', { userland: 123 }) + return db.close() + }) + + test('db wraps error from newsub hook function', async function (t) { + t.plan(2) + + const db = testCommon.factory() + + db.hooks.newsub.add(function (sublevel, options) { + throw new Error('test') + }) + + try { + db.sublevel('sub') + } catch (err) { + t.is(err.code, 'LEVEL_HOOK_ERROR') + t.is(err.cause.message, 'test') + } + + return db.close() + }) +} diff --git a/test/hooks/postopen.js b/test/hooks/postopen.js new file mode 100644 index 0000000..4b77892 --- /dev/null +++ b/test/hooks/postopen.js @@ -0,0 +1,169 @@ +'use strict' + +const shared = require('./shared') + +module.exports = function (test, testCommon) { + shared(test, testCommon, 'postopen') + + test('postopen hook function is called before deferred operations and open event', async function (t) { + t.plan(5) + + const db = testCommon.factory() + const order = [] + + db.hooks.postopen.add(async function (options) { + t.is(db.status, 'open') + order.push('postopen') + }) + + db.on('opening', function () { + t.is(db.status, 'opening') + order.push('opening') + }) + + db.defer(function () { + t.is(db.status, 'open') + order.push('undefer') + }) + + db.on('open', function () { + t.is(db.status, 'open') + order.push('open') + }) + + await db.open() + t.same(order, ['opening', 'postopen', 'undefer', 'open']) + + return db.close() + }) + + test('postopen hook functions are called sequentially', async function (t) { + t.plan(1) + + const db = testCommon.factory() + + let waited = false + db.hooks.postopen.add(async function (options) { + return new Promise(function (resolve) { + setTimeout(function () { + waited = true + resolve() + }, 100) + }) + }) + + db.hooks.postopen.add(async function (options) { + t.ok(waited) + }) + + await db.open() + return db.close() + }) + + test('postopen hook function receives options from constructor', async function (t) { + t.plan(1) + + const db = testCommon.factory({ userland: 123 }) + + db.hooks.postopen.add(async function (options) { + t.same(options, { + createIfMissing: true, + errorIfExists: false, + userland: 123 + }) + }) + + await db.open() + return db.close() + }) + + test('postopen hook function receives options from open()', async function (t) { + t.plan(1) + + const db = testCommon.factory() + + db.hooks.postopen.add(async function (options) { + t.same(options, { + createIfMissing: true, + errorIfExists: false, + userland: 456 + }) + }) + + await db.open({ userland: 456 }) + return db.close() + }) + + test('error from postopen hook function closes the db', async function (t) { + t.plan(4) + + const db = testCommon.factory() + + db.hooks.postopen.add(async function (options) { + t.is(db.status, 'open') + throw new Error('test') + }) + + try { + await db.open() + } catch (err) { + t.is(db.status, 'closed') + t.is(err.code, 'LEVEL_HOOK_ERROR') + t.is(err.cause.message, 'test') + } + }) + + test('postopen hook function that fully closes the db results in error', async function (t) { + t.plan(5) + + const db = testCommon.factory() + + db.hooks.postopen.add(async function (options) { + t.is(db.status, 'open') + return db.close() + }) + + db.on('open', function () { + t.fail('should not open') + }) + + db.on('closed', function () { + t.pass('closed') + }) + + try { + await db.open() + } catch (err) { + t.is(db.status, 'closed') + t.is(err.code, 'LEVEL_HOOK_ERROR') + t.is(err.message, 'The postopen hook has closed the database') + } + }) + + test('postopen hook function that partially closes the db results in error', async function (t) { + t.plan(5) + + const db = testCommon.factory() + + db.hooks.postopen.add(async function (options) { + t.is(db.status, 'open') + db.close() // Don't await + }) + + db.on('open', function () { + t.fail('should not open') + }) + + db.on('closed', function () { + t.pass('closed') + }) + + try { + await db.open() + } catch (err) { + t.is(db.status, 'closed') + t.is(err.code, 'LEVEL_HOOK_ERROR') + t.is(err.message, 'The postopen hook has closed the database') + } + }) +} diff --git a/test/hooks/prewrite.js b/test/hooks/prewrite.js new file mode 100644 index 0000000..5b30e29 --- /dev/null +++ b/test/hooks/prewrite.js @@ -0,0 +1,764 @@ +'use strict' + +const shared = require('./shared') + +module.exports = function (test, testCommon) { + shared(test, testCommon, 'prewrite') + + for (const deferred of [false, true]) { + for (const type of ['put', 'del']) { + for (const method of ['batch', 'chained batch', 'singular']) { + test(`prewrite hook function is called after open (deferred: ${deferred})`, async function (t) { + t.plan(1) + + const db = testCommon.factory() + if (!deferred) await db.open() + + db.hooks.prewrite.add(function (op, batch) { + t.is(db.status, 'open') + }) + + if (type === 'put') { + switch (method) { + case 'batch': + await db.batch([{ type: 'put', key: 'beep', value: 'boop' }]) + break + case 'chained batch': + // Does not support deferred open + await db.open() + await db.batch().put('beep', 'boop').write() + break + case 'singular': + await db.put('beep', 'boop') + break + } + } else if (type === 'del') { + switch (method) { + case 'batch': + await db.batch([{ type: 'del', key: 'beep' }]) + break + case 'chained batch': + // Does not support deferred open + await db.open() + await db.batch().del('beep').write() + break + case 'singular': + await db.del('beep') + break + } + } + + return db.close() + }) + } + } + } + + test('prewrite hook function receives put op', async function (t) { + t.plan(3) + + const db = testCommon.factory() + + db.hooks.prewrite.add(function (op, batch) { + t.same(op, { + type: 'put', + key: 'beep', + value: 'boop', + keyEncoding: db.keyEncoding('utf8'), + valueEncoding: db.valueEncoding('utf8') + }) + }) + + await db.put('beep', 'boop') + await db.batch([{ type: 'put', key: 'beep', value: 'boop' }]) + await db.batch().put('beep', 'boop').write() + + return db.close() + }) + + test('prewrite hook function receives del op', async function (t) { + t.plan(3) + + const db = testCommon.factory() + + db.hooks.prewrite.add(function (op, batch) { + t.same(op, { + type: 'del', + key: 'beep', + keyEncoding: db.keyEncoding('utf8') + }) + }) + + await db.del('beep') + await db.batch([{ type: 'del', key: 'beep' }]) + await db.batch().del('beep').write() + + return db.close() + }) + + test('prewrite hook function receives put op with custom encodings and userland option', async function (t) { + t.plan(3) + + const db = testCommon.factory() + + db.hooks.prewrite.add(function (op, batch) { + t.same(op, { + type: 'put', + key: 123, // Should not be JSON-encoded + value: 'boop', + keyEncoding: db.keyEncoding('json'), + valueEncoding: db.valueEncoding('json'), + userland: 456 + }) + }) + + await db.put(123, 'boop', { keyEncoding: 'json', valueEncoding: 'json', userland: 456 }) + await db.batch([{ type: 'put', key: 123, value: 'boop', keyEncoding: 'json', valueEncoding: 'json', userland: 456 }]) + await db.batch().put(123, 'boop', { keyEncoding: 'json', valueEncoding: 'json', userland: 456 }).write() + + return db.close() + }) + + test('prewrite hook function receives del op with custom encodings and userland option', async function (t) { + t.plan(3) + + const db = testCommon.factory() + + db.hooks.prewrite.add(function (op, batch) { + t.same(op, { + type: 'del', + key: 123, // Should not be JSON-encoded + keyEncoding: db.keyEncoding('json'), + userland: 456 + }) + }) + + await db.del(123, { keyEncoding: 'json', userland: 456 }) + await db.batch([{ type: 'del', key: 123, keyEncoding: 'json', userland: 456 }]) + await db.batch().del(123, { keyEncoding: 'json', userland: 456 }).write() + + return db.close() + }) + + test('prewrite hook function can modify put operation', async function (t) { + t.plan(10 * 3) + + const db = testCommon.factory({ keyEncoding: 'json', valueEncoding: 'utf8' }) + + db.hooks.prewrite.add(function (op, batch) { + t.is(op.keyEncoding, db.keyEncoding('json')) + t.is(op.valueEncoding, db.valueEncoding('utf8')) + + op.key = '456' + op.value = { x: 1 } + + // Flip the encodings + op.keyEncoding = 'utf8' + op.valueEncoding = 'json' + + // Test adding a userland option + op.userland = 456 + }) + + db.on('write', function (ops) { + t.is(ops.length, 1) + t.is(ops[0].key, '456') + t.same(ops[0].value, { x: 1 }) + t.is(ops[0].keyEncoding, db.keyEncoding('utf8')) + t.is(ops[0].valueEncoding, db.valueEncoding('json')) + t.same(ops[0].encodedKey, db.keyEncoding('utf8').encode('456')) + t.same(ops[0].encodedValue, db.valueEncoding('json').encode({ x: 1 })) + t.is(ops[0].userland, 456) + }) + + await db.put(123, 'boop') + await db.batch([{ type: 'put', key: 123, value: 'boop' }]) + await db.batch().put(123, 'boop').write() + + return db.close() + }) + + test('prewrite hook function can modify del operation', async function (t) { + t.plan(6 * 3) + + const db = testCommon.factory({ keyEncoding: 'json' }) + + db.hooks.prewrite.add(function (op, batch) { + t.is(op.keyEncoding, db.keyEncoding('json')) + + op.key = '456' + op.keyEncoding = 'utf8' + + // Test adding a userland option + op.userland = 456 + }) + + db.on('write', function (ops) { + t.is(ops.length, 1) + t.is(ops[0].key, '456') + t.is(ops[0].keyEncoding, db.keyEncoding('utf8')) + t.same(ops[0].encodedKey, db.keyEncoding('utf8').encode('456')) + t.is(ops[0].userland, 456) + }) + + await db.del(123) + await db.batch([{ type: 'del', key: 123 }]) + await db.batch().del(123).write() + + return db.close() + }) + + test('second prewrite hook function sees modified operation of first', async function (t) { + t.plan(6 * 2) + + const db = testCommon.factory() + + db.hooks.prewrite.add(function (op, batch) { + t.is(op.key, '1') + op.key = '2' + }) + + db.hooks.prewrite.add(function (op, batch) { + t.is(op.key, '2') + }) + + await db.put('1', 'boop') + await db.batch([{ type: 'put', key: '1', value: 'boop' }]) + await db.batch().put('1', 'boop').write() + + await db.del('1') + await db.batch([{ type: 'del', key: '1' }]) + await db.batch().del('1').write() + + return db.close() + }) + + test('prewrite hook function triggered by put can add put operation', async function (t) { + t.plan(3) + + const db = testCommon.factory() + + // Note: may return a transcoder encoding + const utf8 = db.keyEncoding('utf8') + const json = db.valueEncoding('json') + + db.hooks.prewrite.add(function (op, batch) { + batch.add({ + type: 'put', + key: 'from-hook', + value: { abc: 123 }, + valueEncoding: 'json' + }) + }) + + db.on('write', function (ops) { + t.same(ops, [ + { + type: 'put', + key: 'beep', + value: 'boop', + keyEncoding: db.keyEncoding('utf8'), + valueEncoding: db.valueEncoding('utf8'), + encodedKey: utf8.encode('beep'), + encodedValue: utf8.encode('boop') + }, + { + type: 'put', + key: 'from-hook', + value: { abc: 123 }, + keyEncoding: db.keyEncoding('utf8'), + valueEncoding: db.valueEncoding('json'), + encodedKey: utf8.encode('from-hook'), + encodedValue: json.encode({ abc: 123 }) + } + ]) + }) + + await db.put('beep', 'boop') + await db.batch([{ type: 'put', key: 'beep', value: 'boop' }]) + await db.batch().put('beep', 'boop').write() + + return db.close() + }) + + test('prewrite hook function triggered by del can add del operation', async function (t) { + t.plan(3) + + const db = testCommon.factory() + + // Note: may return a transcoder encoding + const utf8 = db.keyEncoding('utf8') + + db.hooks.prewrite.add(function (op, batch) { + batch.add({ type: 'del', key: 'from-hook' }) + }) + + db.on('write', function (ops) { + t.same(ops, [ + { + type: 'del', + key: 'beep', + keyEncoding: db.keyEncoding('utf8'), + encodedKey: utf8.encode('beep') + }, + { + type: 'del', + key: 'from-hook', + keyEncoding: db.keyEncoding('utf8'), + encodedKey: utf8.encode('from-hook') + } + ]) + }) + + await db.del('beep') + await db.batch([{ type: 'del', key: 'beep' }]) + await db.batch().del('beep').write() + + return db.close() + }) + + test('prewrite hook function can add operations with sublevel option', async function (t) { + t.plan(2 * 6) + + const db = testCommon.factory() + const sublevel = db.sublevel('sub', { keyEncoding: 'json', valueEncoding: 'json' }) + + // Note: may return a transcoder encoding + const utf8 = db.keyEncoding('utf8') + + db.hooks.prewrite.add(function (op, batch) { + batch.add({ type: 'put', key: 'from-hook-1', value: { x: 22 }, sublevel }) + batch.add({ type: 'del', key: 'from-hook-2', sublevel }) + }) + + db.on('write', function (ops) { + t.is(ops[0].key, 'from-input') + t.same(ops.slice(1), [ + { + type: 'put', + key: utf8.encode('!sub!"from-hook-1"'), + value: utf8.encode('{"x":22}'), + keyEncoding: db.keyEncoding(sublevel.keyEncoding().format), + valueEncoding: db.valueEncoding(sublevel.valueEncoding().format), + encodedKey: utf8.encode('!sub!"from-hook-1"'), + encodedValue: utf8.encode('{"x":22}'), + sublevel: null // Should be unset + }, + { + type: 'del', + key: utf8.encode('!sub!"from-hook-2"'), + keyEncoding: db.keyEncoding(sublevel.keyEncoding().format), + encodedKey: utf8.encode('!sub!"from-hook-2"'), + sublevel: null // Should be unset + } + ]) + }) + + await db.put('from-input', 'abc') + await db.batch([{ type: 'put', key: 'from-input', value: 'abc' }]) + await db.batch().put('from-input', 'abc').write() + + await db.del('from-input') + await db.batch([{ type: 'del', key: 'from-input' }]) + await db.batch().del('from-input').write() + + return db.close() + }) + + test('prewrite hook function can add operations with descendant sublevel option', async function (t) { + t.plan(20) + + const db = testCommon.factory() + await db.open() + + const a = db.sublevel('a') + const b = a.sublevel('b') + const c = b.sublevel('c') + + // Note: may return a transcoder encoding + const utf8 = db.keyEncoding('utf8') + + const put = async (db, key, opts) => { + const fn = function (op, batch) { + batch.add({ type: 'put', key, value: 'x', ...opts }) + } + + db.hooks.prewrite.add(fn) + + try { + await db.put('0', '0') + } finally { + db.hooks.prewrite.delete(fn) + } + } + + const del = async (db, key, opts) => { + const fn = function (op, batch) { + batch.add({ type: 'del', key, ...opts }) + } + + db.hooks.prewrite.add(fn) + + try { + await db.del('0') + } finally { + db.hooks.prewrite.delete(fn) + } + } + + // Note: not entirely a noop. Use of sublevel option triggers data to be encoded early + db.on('write', (ops) => t.same(ops[1].key, utf8.encode('1'), 'got put 1')) + await put(db, '1', { sublevel: db }) + + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[1].key, utf8.encode('!a!2'), 'got put 2')) + await put(db, '2', { sublevel: a }) + await put(a, '2', { sublevel: a }) // Same + + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[1].key, utf8.encode('!a!!b!3'), 'got put 3')) + await put(db, '3', { sublevel: b }) + await put(a, '3', { sublevel: b }) // Same + await put(b, '3', { sublevel: b }) // Same + + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[1].key, utf8.encode('!a!!b!!c!4'), 'got put 4')) + await put(db, '4', { sublevel: c }) + await put(a, '4', { sublevel: c }) // Same + await put(b, '4', { sublevel: c }) // Same + await put(c, '4', { sublevel: c }) // Same + + // Test deletes + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[1].key, utf8.encode('1'), 'got del 1')) + await del(db, '1', { sublevel: db }) + + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[1].key, utf8.encode('!a!2'), 'got del 2')) + await del(db, '2', { sublevel: a }) + await del(a, '2', { sublevel: a }) // Same + + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[1].key, utf8.encode('!a!!b!3'), 'got del 3')) + await del(db, '3', { sublevel: b }) + await del(a, '3', { sublevel: b }) // Same + await del(b, '3', { sublevel: b }) // Same + + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[1].key, utf8.encode('!a!!b!!c!4'), 'got del 4')) + await del(db, '4', { sublevel: c }) + await del(a, '4', { sublevel: c }) // Same + await del(b, '4', { sublevel: c }) // Same + await del(c, '4', { sublevel: c }) // Same + + return db.close() + }) + + test('prewrite hook is triggered bottom-up for nested sublevels', async function (t) { + const db = testCommon.factory() + const a = db.sublevel('a') + const b = a.sublevel('b') + const order = [] + const triggers = [ + [['b', 'a', 'root'], () => b.put('a', 'a')], + [['b', 'a', 'root'], () => b.batch([{ type: 'put', key: 'a', value: 'a' }])], + [['b', 'a', 'root'], () => b.batch().put('a', 'a').write()], + [['b', 'a', 'root'], () => b.del('a')], + [['b', 'a', 'root'], () => b.batch([{ type: 'del', key: 'a' }])], + [['b', 'a', 'root'], () => b.batch().del('a').write()], + + [['a', 'root'], () => a.put('a', 'a')], + [['a', 'root'], () => a.batch([{ type: 'put', key: 'a', value: 'a' }])], + [['a', 'root'], () => a.batch().put('a', 'a').write()], + [['a', 'root'], () => a.del('a')], + [['a', 'root'], () => a.batch([{ type: 'del', key: 'a' }])], + [['a', 'root'], () => a.batch().del('a').write()], + + [['root'], () => db.put('a', 'a')], + [['root'], () => db.batch([{ type: 'put', key: 'a', value: 'a' }])], + [['root'], () => db.batch().put('a', 'a').write()], + [['root'], () => db.del('a')], + [['root'], () => db.batch([{ type: 'del', key: 'a' }])], + [['root'], () => db.batch().del('a').write()], + + // The sublevel option should not trigger the prewrite hook + [['root'], () => db.put('a', 'a', { sublevel: a })], + [['root'], () => db.batch([{ type: 'put', key: 'a', value: 'a', sublevel: a }])], + [['root'], () => db.batch().put('a', 'a', { sublevel: a }).write()], + [['root'], () => db.del('a', { sublevel: a })], + [['root'], () => db.batch([{ type: 'del', key: 'a', sublevel: a }])], + [['root'], () => db.batch().del('a', { sublevel: a }).write()] + ] + + t.plan(triggers.length) + + db.hooks.prewrite.add((op, batch) => { order.push('root') }) + a.hooks.prewrite.add((op, batch) => { order.push('a') }) + b.hooks.prewrite.add((op, batch) => { order.push('b') }) + + for (const [expectedOrder, trigger] of triggers) { + await trigger() + t.same(order.splice(0, order.length), expectedOrder) + } + + return db.close() + }) + + test('db catches invalid operations added by prewrite hook function', async function (t) { + const db = testCommon.factory() + const errEncoding = { + name: 'test', + format: 'utf8', + encode () { + throw new Error() + }, + decode () { + throw new Error() + } + } + + const hookFunctions = [ + (op, batch) => batch.add(), + (op, batch) => batch.add({}), + (op, batch) => batch.add({ type: 'del' }), + (op, batch) => batch.add({ type: 'del', key: null }), + (op, batch) => batch.add({ type: 'del', key: undefined }), + (op, batch) => batch.add({ type: 'put', key: 'a' }), + (op, batch) => batch.add({ type: 'put', key: 'a', value: null }), + (op, batch) => batch.add({ type: 'put', key: 'a', value: undefined }), + (op, batch) => batch.add({ type: 'nope', key: 'a', value: 'b' }), + (op, batch) => batch.add({ type: 'del', key: 'a', keyEncoding: errEncoding }), + (op, batch) => batch.add({ type: 'put', key: 'a', value: 'b', keyEncoding: errEncoding }), + (op, batch) => batch.add({ type: 'put', key: 'a', value: 'b', valueEncoding: errEncoding }) + ] + + const triggers = [ + () => db.put('beep', 'boop'), + () => db.batch([{ type: 'put', key: 'beep', value: 'boop' }]), + () => db.batch().put('beep', 'boop').write(), + () => db.del('beep'), + () => db.batch([{ type: 'del', key: 'beep' }]), + () => db.batch().del('beep').write() + ] + + t.plan(hookFunctions.length * triggers.length * 2) + + db.on('write', function (ops) { + t.fail('should not write') + }) + + for (const trigger of triggers) { + for (const fn of hookFunctions) { + db.hooks.prewrite.add(fn) + + try { + await trigger() + } catch (err) { + t.is(err.code, 'LEVEL_HOOK_ERROR') + } + + db.hooks.prewrite.delete(fn) + t.is(db.hooks.prewrite.noop, true) + } + } + + return db.close() + }) + + test('prewrite hook function is called once for every input operation', async function (t) { + t.plan(2) + + const calls = [] + const db = testCommon.factory() + + db.hooks.prewrite.add(function (op, batch) { + calls.push(op.key) + }) + + await db.batch([{ type: 'del', key: '1' }, { type: 'put', key: '2', value: '123' }]) + t.same(calls.splice(0, calls.length), ['1', '2']) + + await db.batch().del('1').put('2', '123').write() + t.same(calls.splice(0, calls.length), ['1', '2']) + + return db.close() + }) + + test('prewrite hook adds operations after input operations', async function (t) { + t.plan(2) + + const db = testCommon.factory() + + db.hooks.prewrite.add(function (op, batch) { + if (op.key === 'input1') { + batch + .add({ type: 'del', key: 'hook1' }) + .add({ type: 'del', key: 'hook2' }) + .add({ type: 'put', key: 'hook3', value: 'foo' }) + } + }) + + db.on('write', function (ops) { + t.same(ops.map(op => op.key), [ + 'input1', 'input2', 'hook1', 'hook2', 'hook3' + ], 'order is correct') + }) + + await db.batch([{ type: 'del', key: 'input1' }, { type: 'put', key: 'input2', value: '123' }]) + await db.batch().del('input1').put('input2', '123').write() + + return db.close() + }) + + test('prewrite hook does not copy input options to added operations', async function (t) { + t.plan(6) + + const db = testCommon.factory() + + db.hooks.prewrite.add(function (op, batch) { + batch.add({ type: 'put', key: 'from-hook-a', value: 'xyz' }) + batch.add({ type: 'del', key: 'from-hook-b' }) + }) + + db.on('write', function (ops) { + const relevant = ops.map(op => { + return { + key: op.key, + hasOption: 'userland' in op, + keyEncoding: op.keyEncoding.commonName + } + }) + + t.same(relevant, [ + { + key: 'input-a', + keyEncoding: 'json', + hasOption: true + }, + { + key: 'from-hook-a', + keyEncoding: 'utf8', // Should be the database default (2x) + hasOption: false + }, + { + key: 'from-hook-b', + keyEncoding: 'utf8', + hasOption: false + } + ]) + }) + + await db.put('input-a', 'boop', { keyEncoding: 'json', userland: 123 }) + await db.batch([{ type: 'put', key: 'input-a', value: 'boop', keyEncoding: 'json', userland: 123 }]) + await db.batch().put('input-a', 'boop', { keyEncoding: 'json', userland: 123 }).write() + + await db.del('input-a', { keyEncoding: 'json', userland: 123 }) + await db.batch([{ type: 'del', key: 'input-a', keyEncoding: 'json', userland: 123 }]) + await db.batch().del('input-a', { keyEncoding: 'json', userland: 123 }).write() + + return db.close() + }) + + test('error thrown from prewrite hook function is catched', async function (t) { + t.plan(6 * 2) + + const db = testCommon.factory() + + db.hooks.prewrite.add(function (op, batch) { + throw new Error('test') + }) + + const verify = (err) => { + t.is(err.code, 'LEVEL_HOOK_ERROR') + t.is(err.cause.message, 'test') + } + + await db.batch([{ type: 'del', key: '1' }]).catch(verify) + await db.batch([{ type: 'put', key: '1', value: '2' }]).catch(verify) + + const batch1 = db.batch() + const batch2 = db.batch() + + try { batch1.del('1') } catch (err) { verify(err) } + try { batch2.put('1', '2') } catch (err) { verify(err) } + + await batch1.close() + await batch2.close() + + await db.del('1').catch(verify) + await db.put('1', '2').catch(verify) + + return db.close() + }) + + test('operations added by prewrite hook function count towards chained batch length', async function (t) { + t.plan(2) + + const db = testCommon.factory() + await db.open() + + db.hooks.prewrite.add(function (op, batch) { + batch.add({ type: 'del', key: 'hook1' }) + }) + + const batch = db.batch() + + batch.del('input1') + t.is(batch.length, 2) + + batch.put('input2', 'foo') + t.is(batch.length, 4) + + await batch.close() + return db.close() + }) + + test('operations added by prewrite hook function can be cleared from chained batch', async function (t) { + t.plan(3) + + const db = testCommon.factory() + await db.open() + + db.hooks.prewrite.add(function (op, batch) { + batch.add({ type: 'put', key: 'x', value: 'y' }) + }) + + const batch = db.batch() + + batch.del('a') + t.is(batch.length, 2) + + batch.clear() + t.is(batch.length, 0) + + db.on('write', t.fail.bind(t)) + await batch.write() + + t.same(await db.keys().all(), [], 'did not write to db') + return db.close() + }) + + test('prewrite hook function is not called for earlier chained batch', async function (t) { + t.plan(2) + + const db = testCommon.factory() + await db.open() + + const calls = [] + const batchBefore = db.batch() + + db.hooks.prewrite.add(function (op, batch) { + calls.push(op.key) + }) + + batchBefore.del('before') + t.same(calls, []) + + const batchAfter = db.batch() + batchAfter.del('after') + t.same(calls, ['after']) + + await Promise.all([batchBefore.close(), batchAfter.close()]) + return db.close() + }) +} diff --git a/test/hooks/shared.js b/test/hooks/shared.js new file mode 100644 index 0000000..e0a7300 --- /dev/null +++ b/test/hooks/shared.js @@ -0,0 +1,38 @@ +'use strict' + +module.exports = function (test, testCommon, hook) { + test(`can add and remove functions to/from ${hook} hook`, async function (t) { + const db = testCommon.factory() + const fn1 = function () {} + const fn2 = function () {} + + t.is(db.hooks[hook].noop, true, 'is initially a noop') + t.is(typeof db.hooks[hook].run, 'function') + + db.hooks[hook].add(fn1) + t.is(db.hooks[hook].noop, false, 'not a noop') + t.is(typeof db.hooks[hook].run, 'function') + + db.hooks[hook].add(fn2) + t.is(db.hooks[hook].noop, false, 'not a noop') + t.is(typeof db.hooks[hook].run, 'function') + + db.hooks[hook].delete(fn1) + t.is(db.hooks[hook].noop, false, 'not a noop') + t.is(typeof db.hooks[hook].run, 'function') + + db.hooks[hook].delete(fn2) + t.is(db.hooks[hook].noop, true, 'is a noop again') + t.is(typeof db.hooks[hook].run, 'function') + + for (const invalid of [null, undefined, 123]) { + t.throws(() => db.hooks[hook].add(invalid), (err) => err.name === 'TypeError') + t.throws(() => db.hooks[hook].delete(invalid), (err) => err.name === 'TypeError') + } + + t.is(db.hooks[hook].noop, true, 'still a noop') + t.is(typeof db.hooks[hook].run, 'function') + + return db.close() + }) +} diff --git a/test/index.js b/test/index.js index d56b4a8..c9fb4e4 100644 --- a/test/index.js +++ b/test/index.js @@ -54,6 +54,11 @@ function suite (options) { require('./clear-range-test').all(test, testCommon) require('./sublevel-test').all(test, testCommon) + require('./events/write')(test, testCommon) + require('./hooks/postopen')(test, testCommon) + require('./hooks/newsub')(test, testCommon) + require('./hooks/prewrite')(test, testCommon) + // Run the same suite on a sublevel if (!testCommon.internals[kSublevels]) { const factory = testCommon.factory diff --git a/test/self.js b/test/self.js index d4e9761..312ef8d 100644 --- a/test/self.js +++ b/test/self.js @@ -480,7 +480,7 @@ test('test batch([]) (array-form) extensibility', function (t) { t.equal(spy.callCount, 2, 'got _batch() call') t.equal(spy.getCall(1).thisValue, test, '`this` on _batch() was correct') t.equal(spy.getCall(1).args.length, 3, 'got three arguments') - t.deepEqual(spy.getCall(1).args[0], expectedArray, 'got expected array argument') + t.deepEqual(spy.getCall(1).args[0], expectedArray.map(o => ({ ...expectedOptions, ...o })), 'got expected array argument') t.deepEqual(spy.getCall(1).args[1], expectedOptions, 'got expected options argument') t.equal(typeof spy.getCall(1).args[2], 'function', 'got callback argument') @@ -688,7 +688,7 @@ test('test AbstractChainedBatch#write() extensibility with options', function (t }) test('test AbstractChainedBatch#put() extensibility', function (t) { - t.plan(7) + t.plan(8) const spy = sinon.spy() const expectedKey = 'key' @@ -705,7 +705,11 @@ test('test AbstractChainedBatch#put() extensibility', function (t) { t.equal(spy.getCall(0).args.length, 3, 'got 3 arguments') t.equal(spy.getCall(0).args[0], expectedKey, 'got expected key argument') t.equal(spy.getCall(0).args[1], expectedValue, 'got expected value argument') - t.same(spy.getCall(0).args[2], { keyEncoding: 'utf8', valueEncoding: 'utf8' }, 'got expected options argument') + + // May contain more options, just because it's cheaper to not remove them + t.is(spy.getCall(0).args[2].keyEncoding, 'utf8', 'got expected keyEncoding option') + t.is(spy.getCall(0).args[2].valueEncoding, 'utf8', 'got expected valueEncoding option') + t.equal(returnValue, test, 'get expected return value') }) }) @@ -726,7 +730,10 @@ test('test AbstractChainedBatch#del() extensibility', function (t) { t.equal(spy.getCall(0).thisValue, test, '`this` on _del() was correct') t.equal(spy.getCall(0).args.length, 2, 'got 2 arguments') t.equal(spy.getCall(0).args[0], expectedKey, 'got expected key argument') - t.same(spy.getCall(0).args[1], { keyEncoding: 'utf8' }, 'got expected options argument') + + // May contain more options, just because it's cheaper to not remove them + t.is(spy.getCall(0).args[1].keyEncoding, 'utf8', 'got expected keyEncoding option') + t.equal(returnValue, test, 'get expected return value') }) }) diff --git a/test/self/encoding-test.js b/test/self/encoding-test.js index dd381f6..2af621b 100644 --- a/test/self/encoding-test.js +++ b/test/self/encoding-test.js @@ -222,7 +222,7 @@ for (const deferred of [false, true]) { // NOTE: adapted from encoding-down test(`chainedBatch.put() and del() encode utf8 key and value (deferred: ${deferred})`, async function (t) { - t.plan(deferred ? 2 : 4) + t.plan(deferred ? 2 : 5) let db @@ -243,11 +243,14 @@ for (const deferred of [false, true]) { return mockChainedBatch(this, { _put: function (key, value, options) { t.same({ key, value }, { key: '1', value: '2' }) - t.same(options, { keyEncoding: 'utf8', valueEncoding: 'utf8' }) + + // May contain additional options just because it's cheaper to not remove them + t.is(options.keyEncoding, 'utf8') + t.is(options.valueEncoding, 'utf8') }, _del: function (key, options) { t.is(key, '3') - t.same(options, { keyEncoding: 'utf8' }) + t.is(options.keyEncoding, 'utf8') } }) } @@ -260,7 +263,7 @@ for (const deferred of [false, true]) { // NOTE: adapted from encoding-down test(`chainedBatch.put() and del() take encoding options (deferred: ${deferred})`, async function (t) { - t.plan(deferred ? 2 : 4) + t.plan(deferred ? 2 : 5) let db @@ -284,11 +287,14 @@ for (const deferred of [false, true]) { return mockChainedBatch(this, { _put: function (key, value, options) { t.same({ key, value }, { key: '"1"', value: '{"x":[2]}' }) - t.same(options, { keyEncoding: 'utf8', valueEncoding: 'utf8' }) + + // May contain additional options just because it's cheaper to not remove them + t.is(options.keyEncoding, 'utf8') + t.is(options.valueEncoding, 'utf8') }, _del: function (key, options) { t.is(key, '"3"') - t.same(options, { keyEncoding: 'utf8' }) + t.is(options.keyEncoding, 'utf8') } }) } diff --git a/test/self/sublevel-test.js b/test/self/sublevel-test.js index b6fcbb2..aff589c 100644 --- a/test/self/sublevel-test.js +++ b/test/self/sublevel-test.js @@ -58,6 +58,7 @@ test('sublevel is extensible', function (t) { open: true, closing: true, closed: true, + write: true, put: true, del: true, batch: true, @@ -70,6 +71,8 @@ test('sublevel is extensible', function (t) { // NOTE: adapted from subleveldown test('sublevel prefix and options', function (t) { + // TODO: rename "prefix" to "name" where appropriate. E.g. this test should be + // called 'empty name' rather than 'empty prefix'. t.test('empty prefix', function (t) { const sub = new NoopLevel().sublevel('') t.is(sub.prefix, '!!') @@ -88,6 +91,38 @@ test('sublevel prefix and options', function (t) { t.end() }) + t.test('array name', function (t) { + const sub = new NoopLevel().sublevel(['a', 'b']) + t.is(sub.prefix, '!a!!b!') + const alt = new NoopLevel().sublevel('a').sublevel('b') + t.is(alt.prefix, sub.prefix) + t.end() + }) + + t.test('empty array name', function (t) { + const sub = new NoopLevel().sublevel(['', '']) + t.is(sub.prefix, '!!!!') + const alt = new NoopLevel().sublevel('').sublevel('') + t.is(alt.prefix, sub.prefix) + t.end() + }) + + t.test('array name with single element', function (t) { + const sub = new NoopLevel().sublevel(['a']) + t.is(sub.prefix, '!a!') + const alt = new NoopLevel().sublevel('a') + t.is(alt.prefix, sub.prefix) + t.end() + }) + + t.test('array name and separator option', function (t) { + const sub = new NoopLevel().sublevel(['a', 'b'], { separator: '%' }) + t.is(sub.prefix, '%a%%b%') + const alt = new NoopLevel().sublevel('a', { separator: '%' }).sublevel('b', { separator: '%' }) + t.is(alt.prefix, sub.prefix) + t.end() + }) + t.test('separator is trimmed from prefix', function (t) { const sub1 = new NoopLevel().sublevel('!prefix') t.is(sub1.prefix, '!prefix!') @@ -101,19 +136,29 @@ test('sublevel prefix and options', function (t) { const sub4 = new NoopLevel().sublevel('@prefix@', { separator: '@' }) t.is(sub4.prefix, '@prefix@') + const sub5 = new NoopLevel().sublevel(['!!!a', 'b!!!']) + t.is(sub5.prefix, '!a!!b!') + + const sub6 = new NoopLevel().sublevel(['a@@@', '@@@b'], { separator: '@' }) + t.is(sub6.prefix, '@a@@b@') + t.end() }) t.test('repeated separator can not result in empty prefix', function (t) { - const sub = new NoopLevel().sublevel('!!!!') - t.is(sub.prefix, '!!') + const sub1 = new NoopLevel().sublevel('!!!!') + t.is(sub1.prefix, '!!') + const sub2 = new NoopLevel().sublevel(['!!!!', '!!!!']) + t.is(sub2.prefix, '!!!!') t.end() }) t.test('invalid sublevel prefix', function (t) { t.throws(() => new NoopLevel().sublevel('foo\x05'), (err) => err.code === 'LEVEL_INVALID_PREFIX') t.throws(() => new NoopLevel().sublevel('foo\xff'), (err) => err.code === 'LEVEL_INVALID_PREFIX') + t.throws(() => new NoopLevel().sublevel(['ok', 'foo\xff']), (err) => err.code === 'LEVEL_INVALID_PREFIX') t.throws(() => new NoopLevel().sublevel('foo!', { separator: '@' }), (err) => err.code === 'LEVEL_INVALID_PREFIX') + t.throws(() => new NoopLevel().sublevel(['ok', 'foo!'], { separator: '@' }), (err) => err.code === 'LEVEL_INVALID_PREFIX') t.end() }) @@ -161,6 +206,10 @@ test('sublevel.prefixKey()', function (t) { t.same(sub.prefixKey('', 'utf8'), '!test!') t.same(sub.prefixKey('a', 'utf8'), '!test!a') + t.same(sub.prefixKey('', 'utf8', false), '!test!', 'explicitly global') + t.same(sub.prefixKey('a', 'utf8', false), '!test!a', 'explicitly global') + t.same(sub.prefixKey('', 'utf8', true), '!test!', 'local') + t.same(sub.prefixKey('a', 'utf8', true), '!test!a', 'local') t.same(sub.prefixKey(Buffer.from(''), 'buffer'), Buffer.from('!test!')) t.same(sub.prefixKey(Buffer.from('a'), 'buffer'), Buffer.from('!test!a')) @@ -168,6 +217,14 @@ test('sublevel.prefixKey()', function (t) { t.same(sub.prefixKey(textEncoder.encode(''), 'view'), textEncoder.encode('!test!')) t.same(sub.prefixKey(textEncoder.encode('a'), 'view'), textEncoder.encode('!test!a')) + const nested = sub.sublevel('nested') + t.same(nested.prefixKey('', 'utf8'), '!test!!nested!') + t.same(nested.prefixKey('a', 'utf8'), '!test!!nested!a') + t.same(nested.prefixKey('', 'utf8', false), '!test!!nested!', 'explicitly global') + t.same(nested.prefixKey('a', 'utf8', false), '!test!!nested!a', 'explicitly global') + t.same(nested.prefixKey('', 'utf8', true), '!nested!', 'local') + t.same(nested.prefixKey('a', 'utf8', true), '!nested!a', 'local') + t.end() }) diff --git a/test/sublevel-test.js b/test/sublevel-test.js index 38c8dc5..187c02d 100644 --- a/test/sublevel-test.js +++ b/test/sublevel-test.js @@ -45,6 +45,83 @@ exports.all = function (test, testCommon) { }) } + for (const method of ['batch', 'chained batch']) { + test(`${method} with descendant sublevel option`, async function (t) { + t.plan(25) + + const db = testCommon.factory() + await db.open() + + const a = db.sublevel('a') + const b = a.sublevel('b') + const c = b.sublevel('c') + + // Note: may return a transcoder encoding + const utf8 = db.keyEncoding('utf8') + + const put = method === 'batch' + ? (db, key, opts) => db.batch([{ type: 'put', key, value: 'x', ...opts }]) + : (db, key, opts) => db.batch().put(key, key, opts).write() + + const del = method === 'batch' + ? (db, key, opts) => db.batch([{ type: 'del', key, ...opts }]) + : (db, key, opts) => db.batch().del(key, opts).write() + + // Note: not entirely a noop. Use of sublevel option triggers data to be encoded early + db.on('write', (ops) => t.same(ops[0].key, utf8.encode('1'), 'got put 1')) + await put(db, '1', { sublevel: db }) + + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[0].key, utf8.encode('!a!2'), 'got put 2')) + await put(db, '2', { sublevel: a }) + await put(a, '2', { sublevel: a }) // Same + + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[0].key, utf8.encode('!a!!b!3'), 'got put 3')) + await put(db, '3', { sublevel: b }) + await put(a, '3', { sublevel: b }) // Same + await put(b, '3', { sublevel: b }) // Same + + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[0].key, utf8.encode('!a!!b!!c!4'), 'got put 4')) + await put(db, '4', { sublevel: c }) + await put(a, '4', { sublevel: c }) // Same + await put(b, '4', { sublevel: c }) // Same + await put(c, '4', { sublevel: c }) // Same + + t.same(await db.keys().all(), ['!a!!b!!c!4', '!a!!b!3', '!a!2', '1'], 'db has entries') + t.same(await a.keys().all(), ['!b!!c!4', '!b!3', '2'], 'sublevel a has entries') + t.same(await b.keys().all(), ['!c!4', '3'], 'sublevel b has entries') + t.same(await c.keys().all(), ['4'], 'sublevel c has entries') + + // Test deletes + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[0].key, utf8.encode('1'), 'got del 1')) + await del(db, '1', { sublevel: db }) + + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[0].key, utf8.encode('!a!2'), 'got del 2')) + await del(db, '2', { sublevel: a }) + await del(a, '2', { sublevel: a }) // Same + + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[0].key, utf8.encode('!a!!b!3'), 'got del 3')) + await del(db, '3', { sublevel: b }) + await del(a, '3', { sublevel: b }) // Same + await del(b, '3', { sublevel: b }) // Same + + db.removeAllListeners('write') + db.on('write', (ops) => t.same(ops[0].key, utf8.encode('!a!!b!!c!4'), 'got del 4')) + await del(db, '4', { sublevel: c }) + await del(a, '4', { sublevel: c }) // Same + await del(b, '4', { sublevel: c }) // Same + await del(c, '4', { sublevel: c }) // Same + + t.same(await db.keys().all(), [], 'db has no entries') + return db.close() + }) + } + for (const deferred of [false, true]) { for (const keyEncoding of ['buffer', 'view']) { if (!testCommon.supports.encodings[keyEncoding]) return diff --git a/types/abstract-level.d.ts b/types/abstract-level.d.ts index 6a09b05..4f77f6c 100644 --- a/types/abstract-level.d.ts +++ b/types/abstract-level.d.ts @@ -42,6 +42,11 @@ declare class AbstractLevel */ supports: IManifest + /** + * Allows userland _hook functions_ to customize behavior of the database. + */ + hooks: AbstractDatabaseHooks + /** * Read-only getter that returns a string reflecting the current state of the database: * @@ -215,9 +220,9 @@ declare class AbstractLevel * Create a sublevel. * @param name Name of the sublevel, used to prefix keys. */ - sublevel (name: string): AbstractSublevel + sublevel (name: string | string[]): AbstractSublevel sublevel ( - name: string, + name: string | string[], options: AbstractSublevelOptions ): AbstractSublevel @@ -229,10 +234,11 @@ declare class AbstractLevel * @param keyFormat Format of {@link key}. One of `'utf8'`, `'buffer'`, `'view'`. * If `'utf8'` then {@link key} must be a string and the return value will be a string. * If `'buffer'` then Buffer, if `'view'` then Uint8Array. + * @param local If true, add prefix for parent database, else for root database (default). */ - prefixKey (key: string, keyFormat: 'utf8'): string - prefixKey (key: Buffer, keyFormat: 'buffer'): Buffer - prefixKey (key: Uint8Array, keyFormat: 'view'): Uint8Array + prefixKey (key: string, keyFormat: 'utf8', local?: boolean | undefined): string + prefixKey (key: Buffer, keyFormat: 'buffer', local?: boolean | undefined): Buffer + prefixKey (key: Uint8Array, keyFormat: 'view', local?: boolean | undefined): Uint8Array /** * Returns the given {@link encoding} argument as a normalized encoding object @@ -483,3 +489,75 @@ export interface AbstractClearOptions extends RangeOptions { */ keyEncoding?: string | Transcoder.PartialEncoding | undefined } + +/** + * Allows userland _hook functions_ to customize behavior of the database. + * + * @template TDatabase Type of database. + */ +export interface AbstractDatabaseHooks< + TDatabase, + TOpenOptions = AbstractOpenOptions, + TBatchOperation = AbstractBatchOperation> { + /** + * An asynchronous hook that runs after the database has succesfully opened, but before + * deferred operations are executed and before events are emitted. Example: + * + * ```js + * db.hooks.postopen.add(async function () { + * // Initialize data + * }) + * ``` + */ + postopen: AbstractHook<(options: TOpenOptions) => Promise> + + /** + * A synchronous hook for modifying or adding operations. Example: + * + * ```js + * db.hooks.prewrite.add(function (op, batch) { + * op.key = op.key.toUpperCase() + * }) + * ``` + * + * @todo Define type of `op`. + */ + prewrite: AbstractHook<(op: any, batch: AbstractPrewriteBatch) => void> + + /** + * A synchronous hook that runs when an {@link AbstractSublevel} instance has been + * created by {@link AbstractLevel.sublevel()}. + */ + newsub: AbstractHook<( + sublevel: AbstractSublevel, + options: AbstractSublevelOptions + ) => void> +} + +/** + * An interface for prewrite hook functions to add operations, to be committed in the + * same batch as the input operation(s). + */ +export interface AbstractPrewriteBatch { + /** + * Add a batch operation. + */ + add: (op: TBatchOperation) => this +} + +/** + * @template TFn The hook-specific function signature. + */ +export interface AbstractHook { + /** + * Add the given {@link fn} function to this hook, if it wasn't already added. + * @param fn Hook function. + */ + add: (fn: TFn) => void + + /** + * Remove the given {@link fn} function from this hook. + * @param fn Hook function. + */ + delete: (fn: TFn) => void +} diff --git a/types/abstract-sublevel.d.ts b/types/abstract-sublevel.d.ts index 8b26364..345344a 100644 --- a/types/abstract-sublevel.d.ts +++ b/types/abstract-sublevel.d.ts @@ -29,7 +29,12 @@ declare class AbstractSublevel /** * Parent database. A read-only property. */ - get db (): TDatabase + get parent (): TDatabase + + /** + * Root database. A read-only property. + */ + get db (): AbstractLevel } /**