Add a crate for wasm zstd implementation

.github/actions/build-test-wasm/action.yaml

@@ -22,10 +22,10 @@ runs:
           wasm-pack build --target nodejs ./crates/${{ inputs.crate }} --out-dir ../../prebuilds/${{ inputs.crate }}
         shell: bash
       - name: Test WASM
-        run: node test_wasm.js
+        run: node test_wasm.js ${{ inputs.crate }}
         shell: bash
       - uses: actions/upload-artifact@v4
         with:
-          name: prebuilds-wasm
+          name: prebuilds-wasm-${{ inputs.crate }}
           if-no-files-found: ignore
           path: ./prebuilds/*

.github/workflows/build.yml

@@ -13,6 +13,7 @@ jobs:
       matrix:
         crate:
           - library_config
+          - datadog-js-zstd
     steps:
       - uses: actions/checkout@v4
       - name: 'Use composite action'

.github/workflows/release.yml

@@ -12,6 +12,7 @@ jobs:
       matrix:
         crate:
           - library_config
+          - datadog-js-zstd
     steps:
       - uses: actions/checkout@v4
       - name: 'Use composite action'

crates/datadog-js-zstd/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "datadog-js-zstd"
+version = "0.1.0"
+edition = "2018"
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+wasm-bindgen = "0.2.100"
+zstd = "0.13.3"
+js-sys = "0.3.77"
+
+[dev-dependencies]
+wasm-bindgen-test = "0.3.50"

crates/datadog-js-zstd/src/lib.rs

@@ -0,0 +1,12 @@
+use wasm_bindgen::prelude::*;
+use js_sys::Uint8Array;
+
+#[wasm_bindgen]
+pub fn zstd_compress(
+    data: Uint8Array,
+    level: i32,
+) -> Uint8Array {
+    let vecdata = data.to_vec();
+    let compressed_data = zstd::encode_all(&vecdata[..], level).expect("Failed to compress data");
+    Uint8Array::from(compressed_data.as_slice())
+}

load.js

@@ -39,7 +39,7 @@ function findWASM (name) {
   const prebuilds = path.join(root, 'prebuilds')
   const folders = readdirSync(prebuilds)
   if (folders.find(f => f === name)) {
-    return path.join(prebuilds, name, `${name}.js`)
+    return path.join(prebuilds, name, `${name.replaceAll('-', '_')}.js`)
   }
 }
 

package.json

@@ -9,7 +9,7 @@
     "build-debug": "mkdir -p target && yarn -s cargo-build > ./target/out.ndjson && yarn -s copy-artifacts",
     "build-release": "mkdir -p target && yarn -s cargo-build-release > ./target/out.ndjson && yarn -s copy-artifacts",
     "build-all": "mkdir -p target && yarn -s cargo-build -- --workspace > ./target/out.ndjson && yarn -s copy-artifacts && yarn -s build-wasm",
-    "build-wasm": "yarn -s install-wasm-pack && wasm-pack build --target nodejs ./crates/library_config --out-dir ../../prebuilds/library_config",
+    "build-wasm": "yarn -s install-wasm-pack && node scripts/build-wasm.js library_config && node scripts/build-wasm.js datadog-js-zstd",
     "cargo-build-release": "yarn -s cargo-build -- --release",
     "cargo-build": "cargo build --message-format=json-render-diagnostics",
     "copy-artifacts": "node ./scripts/copy-artifacts",

scripts/build-wasm.js

@@ -0,0 +1,46 @@
+// This script builds a WebAssembly module using wasm-pack. It is essentially invoking
+// wasm-pack build. All the special handling is for macOS, because Apple's Clang version suffers
+// from some issues that prevent it from compiling at least the zstd crate.
+// See https://github.com/gyscos/zstd-rs/issues/302
+// This is solved by requiring the homebrew version of LLVM to be installed and available in the
+// PATH. Unfortunately, this version then suffers from a different issue that requires wasm-opt to
+// be disabled.
+// See https://github.com/WebAssembly/wasi-sdk/issues/254
+// See https://github.com/llvm/llvm-project/issues/64909
+// Our releases are built on Linux, and fortunately no special handling is required there. This
+// script only allows development to happen on macOS.
+
+const os = require('os');
+const childProcess = require('child_process');
+
+const isMacOS = os.platform() === 'darwin';
+const noWasmOpt = isMacOS ? '--no-opt' : '';
+const library = process.argv[2];
+
+const env = {
+  ...process.env,
+};
+
+if (isMacOS) {
+  const homebrewDir = env.HOMEBREW_DIR ?? '/opt/homebrew';
+  const llvmDir = `${homebrewDir}/opt/llvm/`;
+  const llvmBinDir = `${llvmDir}/bin`;
+
+  try {
+    childProcess.execSync(`${llvmBinDir}/llvm-config --version`);
+  } catch (error) {
+    console.error(`‼️ LLVM not found in ${llvmDir}.\n‼️ Please install LLVM using Homebrew:\n📝   brew install llvm`);
+    process.exit(1);
+  }
+
+  if (!env.PATH.includes(llvmBinDir)) {
+    // Add LLVM to PATH if not already included
+    env.PATH = `${llvmBinDir}:${env.PATH}`;
+  }
+}
+
+childProcess.execSync(
+  `wasm-pack build ${noWasmOpt} --target nodejs ./crates/${library} --out-dir ../../prebuilds/${library}`, {
+    env
+  }
+);