diff --git a/.github/actions/build-test-wasm/action.yaml b/.github/actions/build-test-wasm/action.yaml index 0effdc1..595c124 100644 --- a/.github/actions/build-test-wasm/action.yaml +++ b/.github/actions/build-test-wasm/action.yaml @@ -22,10 +22,10 @@ runs: wasm-pack build --target nodejs ./crates/${{ inputs.crate }} --out-dir ../../prebuilds/${{ inputs.crate }} shell: bash - name: Test WASM - run: node test_wasm.js + run: node test_wasm.js ${{ inputs.crate }} shell: bash - uses: actions/upload-artifact@v4 with: - name: prebuilds-wasm + name: prebuilds-wasm-${{ inputs.crate }} if-no-files-found: ignore path: ./prebuilds/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e534dc5..0533478 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,6 +13,7 @@ jobs: matrix: crate: - library_config + - datadog-js-zstd steps: - uses: actions/checkout@v4 - name: 'Use composite action' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d9b1a7a..eb820ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,6 +12,7 @@ jobs: matrix: crate: - library_config + - datadog-js-zstd steps: - uses: actions/checkout@v4 - name: 'Use composite action' diff --git a/Cargo.lock b/Cargo.lock index 5e2ff17..00faea7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -267,16 +267,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "console_error_panic_hook" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" -dependencies = [ - "cfg-if", - "wasm-bindgen", -] - [[package]] name = "convert_case" version = "0.6.0" @@ -393,6 +383,16 @@ dependencies = [ "prost", ] +[[package]] +name = "datadog-js-zstd" +version = "0.1.0" +dependencies = [ + "js-sys", + "wasm-bindgen", + "wasm-bindgen-test", + "zstd", +] + [[package]] name = "datadog-library-config" version = "0.0.1" @@ -964,10 +964,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -1286,6 +1287,12 @@ dependencies = [ "neon", ] +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "portable-atomic" version = "1.9.0" @@ -1592,6 +1599,12 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + [[package]] name = "ryu" version = "1.0.18" @@ -1640,12 +1653,6 @@ dependencies = [ "syn 2.0.87", ] -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "sct" version = "0.7.1" @@ -2043,24 +2050,24 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn 2.0.87", @@ -2069,21 +2076,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.45" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2091,9 +2099,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", @@ -2104,20 +2112,21 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-bindgen-test" -version = "0.3.45" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d381749acb0943d357dcbd8f0b100640679883fcdeeef04def49daf8d33a5426" +checksum = "66c8d5e33ca3b6d9fa3b4676d774c5778031d27a578c2b007f905acf816152c3" dependencies = [ - "console_error_panic_hook", "js-sys", "minicov", - "scoped-tls", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test-macro", @@ -2125,9 +2134,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.45" +version = "0.3.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c97b2ef2c8d627381e51c071c2ab328eac606d3f69dd82bcbca20a9e389d95f0" +checksum = "17d5042cc5fa009658f9a7333ef24291b1291a25b6382dd68862a7f3b969f69b" dependencies = [ "proc-macro2", "quote", @@ -2136,9 +2145,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.72" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -2380,3 +2389,31 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/crates/datadog-js-zstd/Cargo.toml b/crates/datadog-js-zstd/Cargo.toml new file mode 100644 index 0000000..308740f --- /dev/null +++ b/crates/datadog-js-zstd/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "datadog-js-zstd" +version = "0.1.0" +edition = "2018" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +wasm-bindgen = "0.2.100" +zstd = "0.13.3" +js-sys = "0.3.77" + +[dev-dependencies] +wasm-bindgen-test = "0.3.50" diff --git a/crates/datadog-js-zstd/src/lib.rs b/crates/datadog-js-zstd/src/lib.rs new file mode 100644 index 0000000..4896301 --- /dev/null +++ b/crates/datadog-js-zstd/src/lib.rs @@ -0,0 +1,12 @@ +use wasm_bindgen::prelude::*; +use js_sys::Uint8Array; + +#[wasm_bindgen] +pub fn zstd_compress( + data: Uint8Array, + level: i32, +) -> Uint8Array { + let vecdata = data.to_vec(); + let compressed_data = zstd::encode_all(&vecdata[..], level).expect("Failed to compress data"); + Uint8Array::from(compressed_data.as_slice()) +} diff --git a/load.js b/load.js index 21970c3..334e609 100644 --- a/load.js +++ b/load.js @@ -39,7 +39,7 @@ function findWASM (name) { const prebuilds = path.join(root, 'prebuilds') const folders = readdirSync(prebuilds) if (folders.find(f => f === name)) { - return path.join(prebuilds, name, `${name}.js`) + return path.join(prebuilds, name, `${name.replaceAll('-', '_')}.js`) } } diff --git a/package.json b/package.json index 897c058..13201f7 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "build-debug": "mkdir -p target && yarn -s cargo-build > ./target/out.ndjson && yarn -s copy-artifacts", "build-release": "mkdir -p target && yarn -s cargo-build-release > ./target/out.ndjson && yarn -s copy-artifacts", "build-all": "mkdir -p target && yarn -s cargo-build -- --workspace > ./target/out.ndjson && yarn -s copy-artifacts && yarn -s build-wasm", - "build-wasm": "yarn -s install-wasm-pack && wasm-pack build --target nodejs ./crates/library_config --out-dir ../../prebuilds/library_config", + "build-wasm": "yarn -s install-wasm-pack && node scripts/build-wasm.js library_config && node scripts/build-wasm.js datadog-js-zstd", "cargo-build-release": "yarn -s cargo-build -- --release", "cargo-build": "cargo build --message-format=json-render-diagnostics", "copy-artifacts": "node ./scripts/copy-artifacts", diff --git a/scripts/build-wasm.js b/scripts/build-wasm.js new file mode 100644 index 0000000..ae5bcdd --- /dev/null +++ b/scripts/build-wasm.js @@ -0,0 +1,46 @@ +// This script builds a WebAssembly module using wasm-pack. It is essentially invoking +// wasm-pack build. All the special handling is for macOS, because Apple's Clang version suffers +// from some issues that prevent it from compiling at least the zstd crate. +// See https://github.com/gyscos/zstd-rs/issues/302 +// This is solved by requiring the homebrew version of LLVM to be installed and available in the +// PATH. Unfortunately, this version then suffers from a different issue that requires wasm-opt to +// be disabled. +// See https://github.com/WebAssembly/wasi-sdk/issues/254 +// See https://github.com/llvm/llvm-project/issues/64909 +// Our releases are built on Linux, and fortunately no special handling is required there. This +// script only allows development to happen on macOS. + +const os = require('os'); +const childProcess = require('child_process'); + +const isMacOS = os.platform() === 'darwin'; +const noWasmOpt = isMacOS ? '--no-opt' : ''; +const library = process.argv[2]; + +const env = { + ...process.env, +}; + +if (isMacOS) { + const homebrewDir = env.HOMEBREW_DIR ?? '/opt/homebrew'; + const llvmDir = `${homebrewDir}/opt/llvm/`; + const llvmBinDir = `${llvmDir}/bin`; + + try { + childProcess.execSync(`${llvmBinDir}/llvm-config --version`); + } catch (error) { + console.error(`‼️ LLVM not found in ${llvmDir}.\n‼️ Please install LLVM using Homebrew:\n📝 brew install llvm`); + process.exit(1); + } + + if (!env.PATH.includes(llvmBinDir)) { + // Add LLVM to PATH if not already included + env.PATH = `${llvmBinDir}:${env.PATH}`; + } +} + +childProcess.execSync( + `wasm-pack build ${noWasmOpt} --target nodejs ./crates/${library} --out-dir ../../prebuilds/${library}`, { + env + } +); diff --git a/test/wasm/datadog-js-zstd/index.js b/test/wasm/datadog-js-zstd/index.js new file mode 100644 index 0000000..fd7a4e8 --- /dev/null +++ b/test/wasm/datadog-js-zstd/index.js @@ -0,0 +1,50 @@ +const loader = require('../../../load.js'); +const assert = require('assert'); + +const zstd = loader.load('datadog-js-zstd'); +assert(zstd !== undefined); + +// Create some compressible data +const SAMPLE_SIZE = 512; +const SAMPLE_COUNT = 1024; +const DATA_SIZE = SAMPLE_COUNT * 4 * SAMPLE_SIZE; + +const samples = [] +for (let i = 0; i < SAMPLE_COUNT; i++) { + const sample = new Array(SAMPLE_SIZE); + for (let j = 0; j < SAMPLE_SIZE; j++) { + sample[j] = (Math.random() * 256) | 0; + } + samples.push(sample); +} +const data = new Array(DATA_SIZE); +for (let i = 0; i < DATA_SIZE; i+= SAMPLE_SIZE) { + data.push(...samples[Math.random() * SAMPLE_COUNT | 0]); +} +// Introduce some irregularities +for (let i = 0; i < SAMPLE_COUNT; i++) { + data[Math.random() * DATA_SIZE | 0] = 0; +} +const dataArr = new Uint8Array(data); +const compressed3 = zstd.zstd_compress(dataArr, 3) +ensureCompressed(compressed3); + +// Test that 0 means default compression level +const compressed0 = zstd.zstd_compress(dataArr, 0) +ensureCompressed(compressed3); +assert(compressed0.length == compressed3.length); + +// Test that compression levels are correctly passed on. +// Level 18 should produce a smaller output than level 3. +// We can go all the way up to 22, but it is significantly slower. +const compressed18 = zstd.zstd_compress(dataArr, 18) +ensureCompressed(compressed18); +assert(compressed18.length < compressed3.length); + +function ensureCompressed(compressed) { + assert(compressed.length > 4); + assert.equal(compressed[0], 0x28); + assert.equal(compressed[1], 0xb5); + assert.equal(compressed[2], 0x2f); + assert.equal(compressed[3], 0xfd); +} diff --git a/test_wasm.js b/test_wasm.js index 762dd03..66cc869 100644 --- a/test_wasm.js +++ b/test_wasm.js @@ -1,10 +1,10 @@ 'use strict' const fs = require('fs') -const { execSync } = require('child_process') -fs.readdirSync('test/wasm') +const crateTestsDir = `./test/wasm/${process.argv[2]}` +fs.readdirSync(crateTestsDir) .filter(file => file.endsWith('.js') || !file.includes('.')) .forEach(file => { - require('./test/wasm/' + file) + require(`${crateTestsDir}/${file}`) })