Skip to content

Commit 55d1463

Browse files
committed
zlib: add dictionary support to zstdCompress and zstdDecompress
Adds optional dictionary support to zlib’s zstdCompress and zstdDecompress APIs. This enables better compression ratios when the dictionary matches expected input structure or content patterns. The implementation allows passing a `dictionary` buffer through the options object. Support was added to both streaming and convenience methods. Tests and documentation were also updated to reflect this new capability. Fixes: #59105
1 parent 3b57443 commit 55d1463

File tree

4 files changed

+100
-13
lines changed

4 files changed

+100
-13
lines changed

doc/api/zlib.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1069,6 +1069,9 @@ Each Zstd-based class takes an `options` object. All options are optional.
10691069
* `maxOutputLength` {integer} Limits output size when using
10701070
[convenience methods][]. **Default:** [`buffer.kMaxLength`][]
10711071
* `info` {boolean} If `true`, returns an object with `buffer` and `engine`. **Default:** `false`
1072+
* `dictionary` {Buffer} Optional dictionary used to
1073+
improve compression efficiency when compressing or decompressing data that
1074+
shares common patterns with the dictionary.
10721075

10731076
For example:
10741077

lib/zlib.js

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -893,12 +893,26 @@ class Zstd extends ZlibBase {
893893
const pledgedSrcSize = opts?.pledgedSrcSize ?? undefined;
894894

895895
const writeState = new Uint32Array(2);
896-
handle.init(
897-
initParamsArray,
898-
pledgedSrcSize,
899-
writeState,
900-
processCallback,
901-
);
896+
897+
if (opts?.dictionary && Buffer.isBuffer(opts.dictionary)) {
898+
handle.init(
899+
initParamsArray,
900+
pledgedSrcSize,
901+
writeState,
902+
processCallback,
903+
opts.dictionary.buffer,
904+
opts.dictionary.byteOffset,
905+
opts.dictionary.length,
906+
);
907+
} else {
908+
handle.init(
909+
initParamsArray,
910+
pledgedSrcSize,
911+
writeState,
912+
processCallback,
913+
);
914+
}
915+
902916
super(opts, mode, handle, zstdDefaultOpts);
903917
this._writeState = writeState;
904918
}

src/node_zlib.cc

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,9 @@ class ZstdCompressContext final : public ZstdContext {
324324
CompressionError ResetStream();
325325

326326
// Zstd specific:
327-
CompressionError Init(uint64_t pledged_src_size);
327+
CompressionError Init(uint64_t pledged_src_size,
328+
const char* dict_data = nullptr,
329+
size_t dict_size = 0);
328330
CompressionError SetParameter(int key, int value);
329331

330332
// Wrap ZSTD_freeCCtx to remove the return type.
@@ -349,7 +351,9 @@ class ZstdDecompressContext final : public ZstdContext {
349351
CompressionError ResetStream();
350352

351353
// Zstd specific:
352-
CompressionError Init(uint64_t pledged_src_size);
354+
CompressionError Init(uint64_t pledged_src_size,
355+
const char* dict_data = nullptr,
356+
size_t dict_size = 0);
353357
CompressionError SetParameter(int key, int value);
354358

355359
// Wrap ZSTD_freeDCtx to remove the return type.
@@ -875,8 +879,9 @@ class ZstdStream final : public CompressionStream<CompressionContext> {
875879
Environment* env = Environment::GetCurrent(args);
876880
Local<Context> context = env->context();
877881

878-
CHECK(args.Length() == 4 &&
879-
"init(params, pledgedSrcSize, writeResult, writeCallback)");
882+
CHECK((args.Length() == 4 || args.Length() == 7) &&
883+
"init(params, pledgedSrcSize, writeResult, writeCallback[, "
884+
"dictBuf, dictOffset, dictLen])");
880885
ZstdStream* wrap;
881886
ASSIGN_OR_RETURN_UNWRAP(&wrap, args.This());
882887

@@ -904,7 +909,23 @@ class ZstdStream final : public CompressionStream<CompressionContext> {
904909
}
905910

906911
AllocScope alloc_scope(wrap);
907-
CompressionError err = wrap->context()->Init(pledged_src_size);
912+
const char* dict_data = nullptr;
913+
size_t dict_size = 0;
914+
915+
if (args.Length() == 7) {
916+
Local<Context> context = env->context();
917+
v8::Local<v8::ArrayBuffer> buffer = args[4].As<v8::ArrayBuffer>();
918+
919+
dict_data = static_cast<const char*>(
920+
buffer->GetBackingStore()->Data()) +
921+
args[5]->Uint32Value(context).ToChecked();
922+
923+
924+
dict_size = args[6]->Uint32Value(context).ToChecked();
925+
}
926+
927+
CompressionError err =
928+
wrap->context()->Init(pledged_src_size, dict_data, dict_size);
908929
if (err.IsError()) {
909930
wrap->EmitError(err);
910931
THROW_ERR_ZLIB_INITIALIZATION_FAILED(wrap->env(), err.message);
@@ -1509,14 +1530,26 @@ CompressionError ZstdCompressContext::SetParameter(int key, int value) {
15091530
return {};
15101531
}
15111532

1512-
CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size) {
1533+
CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size,
1534+
const char* dict_data,
1535+
size_t dict_size) {
15131536
pledged_src_size_ = pledged_src_size;
15141537
cctx_.reset(ZSTD_createCCtx());
15151538
if (!cctx_) {
15161539
return CompressionError("Could not initialize zstd instance",
15171540
"ERR_ZLIB_INITIALIZATION_FAILED",
15181541
-1);
15191542
}
1543+
1544+
if (dict_data != nullptr && dict_size > 0) {
1545+
size_t ret = ZSTD_CCtx_loadDictionary(cctx_.get(), dict_data, dict_size);
1546+
if (ZSTD_isError(ret)) {
1547+
return CompressionError("Failed to load zstd dictionary",
1548+
"ERR_ZLIB_DICTIONARY_LOAD_FAILED",
1549+
-1);
1550+
}
1551+
}
1552+
15201553
size_t result = ZSTD_CCtx_setPledgedSrcSize(cctx_.get(), pledged_src_size);
15211554
if (ZSTD_isError(result)) {
15221555
return CompressionError(
@@ -1549,13 +1582,24 @@ CompressionError ZstdDecompressContext::SetParameter(int key, int value) {
15491582
return {};
15501583
}
15511584

1552-
CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size) {
1585+
CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size,
1586+
const char* dict_data,
1587+
size_t dict_size) {
15531588
dctx_.reset(ZSTD_createDCtx());
15541589
if (!dctx_) {
15551590
return CompressionError("Could not initialize zstd instance",
15561591
"ERR_ZLIB_INITIALIZATION_FAILED",
15571592
-1);
15581593
}
1594+
1595+
if (dict_data != nullptr && dict_size > 0) {
1596+
size_t ret = ZSTD_DCtx_loadDictionary(dctx_.get(), dict_data, dict_size);
1597+
if (ZSTD_isError(ret)) {
1598+
return CompressionError("Failed to load zstd dictionary",
1599+
"ERR_ZLIB_DICTIONARY_LOAD_FAILED",
1600+
-1);
1601+
}
1602+
}
15591603
return {};
15601604
}
15611605

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
'use strict';
2+
3+
const common = require('../common');
4+
const assert = require('assert');
5+
const zlib = require('zlib');
6+
7+
const dictionary = Buffer.from(
8+
`Lorem ipsum dolor sit amet, consectetur adipiscing elit.
9+
Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
10+
Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.`
11+
);
12+
13+
const input = Buffer.from(
14+
`Lorem ipsum dolor sit amet, consectetur adipiscing elit.
15+
Lorem ipsum dolor sit amet, consectetur adipiscing elit.
16+
Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
17+
Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
18+
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.`
19+
);
20+
21+
zlib.zstdCompress(input, { dictionary }, common.mustSucceed((compressed) => {
22+
assert(compressed.length < input.length);
23+
zlib.zstdDecompress(compressed, { dictionary }, common.mustSucceed((decompressed) => {
24+
assert.strictEqual(decompressed.toString(), input.toString());
25+
}));
26+
}));

0 commit comments

Comments
 (0)