diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 576670a..f68793c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,6 @@ version: 2 updates: - # Version updates for npm dependencies - - package-ecosystem: "npm" + - package-ecosystem: "cargo" directory: "/" schedule: interval: "weekly" @@ -12,19 +11,55 @@ updates: labels: - "chore" commit-message: - prefix: "chore(deps)" - prefix-development: "chore(deps-dev)" - rebase-strategy: "auto" - versioning-strategy: "auto" + prefix: "chore(deps-rs)" groups: - npm-dependencies: + rust-dependencies: + patterns: + - "*" + update-types: + - "minor" + - "patch" + + - package-ecosystem: "pip" + directory: "/packages/nvisy-ai" + schedule: + interval: "weekly" + timezone: "Europe/Berlin" + day: "monday" + time: "04:00" + open-pull-requests-limit: 3 + labels: + - "chore" + commit-message: + prefix: "chore(deps-py)" + groups: + rust-dependencies: + patterns: + - "*" + update-types: + - "minor" + - "patch" + + - package-ecosystem: "pip" + directory: "/packages/nvisy-exif" + schedule: + interval: "weekly" + timezone: "Europe/Berlin" + day: "monday" + time: "04:00" + open-pull-requests-limit: 3 + labels: + - "chore" + commit-message: + prefix: "chore(deps-py)" + groups: + rust-dependencies: patterns: - "*" update-types: - "minor" - "patch" - # Version updates for GitHub Actions - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 769e7e8..c55ade4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,72 +4,87 @@ on: push: branches: [main] paths: - - "packages/**" - - "package.json" - - "package-lock.json" - - "tsconfig.json" - - "vitest.config.ts" - - "biome.json" + - "crates/**" + - "Cargo.toml" + - "Cargo.lock" - ".github/workflows/build.yml" pull_request: branches: [main] paths: - - "packages/**" - - "package.json" - - "package-lock.json" - - "tsconfig.json" - - "vitest.config.ts" - - "biome.json" + - "crates/**" + - "Cargo.toml" + - "Cargo.lock" - ".github/workflows/build.yml" workflow_dispatch: env: - NODE_VERSION: 22 + CARGO_TERM_COLOR: always + PYO3_USE_ABI3_FORWARD_COMPATIBILITY: 1 concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: - lint: - name: Lint + check: + name: Check runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 - - - name: Install Node.js - uses: actions/setup-node@v6 - with: - node-version: ${{ env.NODE_VERSION }} - cache: npm + uses: actions/checkout@v4 - - name: Install dependencies - run: npm ci + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable - - name: Check formatting & linting - run: npx biome check . + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" - check: - name: Typecheck + - name: Cache cargo registry and build + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-check-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo-check- + + - name: Cargo check + run: cargo check --workspace + + clippy: + name: Clippy runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: clippy - - name: Install Node.js - uses: actions/setup-node@v6 + - name: Install Python + uses: actions/setup-python@v5 with: - node-version: ${{ env.NODE_VERSION }} - cache: npm + python-version: "3.11" - - name: Install dependencies - run: npm ci + - name: Cache cargo registry and build + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo-clippy- - - name: Typecheck - run: npx tsc -b packages/*/tsconfig.json + - name: Clippy + run: cargo clippy --workspace -- -D warnings test: name: Test @@ -78,19 +93,28 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v4 - - name: Install Node.js - uses: actions/setup-node@v6 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install Python + uses: actions/setup-python@v5 with: - node-version: ${{ env.NODE_VERSION }} - cache: npm + python-version: "3.11" - - name: Install dependencies - run: npm ci + - name: Cache cargo registry and build + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo-test- - name: Run tests - run: npx vitest run --coverage + run: cargo test --workspace build: name: Build @@ -99,16 +123,25 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v4 - - name: Install Node.js - uses: actions/setup-node@v6 - with: - node-version: ${{ env.NODE_VERSION }} - cache: npm + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable - - name: Install dependencies - run: npm ci + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" - - name: Build - run: npm run build --workspaces --if-present + - name: Cache cargo registry and build + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo-build- + + - name: Build release + run: cargo build --release diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 4069d7c..660fc76 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -4,16 +4,16 @@ on: push: branches: [main] paths: - - "packages/**" - - "package.json" - - "package-lock.json" + - "crates/**" + - "Cargo.toml" + - "Cargo.lock" - ".github/workflows/security.yml" pull_request: branches: [main] paths: - - "packages/**" - - "package.json" - - "package-lock.json" + - "crates/**" + - "Cargo.toml" + - "Cargo.lock" - ".github/workflows/security.yml" schedule: # Monday at 06:00 UTC @@ -27,9 +27,6 @@ concurrency: permissions: contents: read -env: - NODE_VERSION: 22 - jobs: audit: name: Audit @@ -37,16 +34,13 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v4 - - name: Install Node.js - uses: actions/setup-node@v6 - with: - node-version: ${{ env.NODE_VERSION }} - cache: npm + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable - - name: Install dependencies - run: npm ci + - name: Install cargo-audit + run: cargo install cargo-audit - - name: Audit dependencies - run: npm audit --omit=dev + - name: Run cargo audit + run: cargo audit diff --git a/.gitignore b/.gitignore index 3705388..9c1daf2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,50 +2,57 @@ Thumbs.db .DS_Store -# Editors +# IDE and Editors .vs/ .vscode/ .idea/ .zed/ -# Node & JavaScript -node_modules/ -.vite/ -.astro/ -.output/ -.nuxt/ -.nitro/ -.cache/ - -# Testing -playwright-report/ -test-results/ +# Rust +debug/ +target/ +**/*.rs.bk +*.pdb + +# Python +__pycache__/ +*.py[cod] +.venv/ +*.egg-info/ +.ruff_cache/ +.pytest_cache/ + +# Generated files +*.pem +encryption.key +*.backup coverage/ -.lighthouse/ +*.lcov + +# Intermediate output +.diesel_lock +crates/nvisy-postgres/src/migrations/ +crates/nvisy-postgres/src/schema.rs.bak -# Build Output +# Build output dist/ build/ -output/ -# Environment +# Environment files .env* !.env.example # Logs +logs/ *.log *.log* -# Generated -*.pem +# Backup and temporary files +*.bak *.backup -*.tsbuildinfo - -# Generated config output files -*.config.d.ts -*.config.d.ts.map -*.config.js -*.config.js.map +*.tmp +tmp/ +temp/ # Other .ignore*/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..c8c1ce5 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,4301 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ab_glyph" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c0457472c38ea5bd1c3b5ada5e368271cb550be7a4ca4a0b4634e9913f6cc2" +dependencies = [ + "ab_glyph_rasterizer", + "owned_ttf_parser", +] + +[[package]] +name = "ab_glyph_rasterizer" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618" + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "adobe-cmap-parser" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8abfa9a4688de8fc9f42b3f013b6fffec18ed8a554f5f113577e0b9b3212a3" +dependencies = [ + "pom", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "aligned" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" +dependencies = [ + "as-slice", +] + +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "as-slice" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "atoi_simd" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ad17c7c205c2c28b527b9845eeb91cf1b4d008b438f98ce0e628227a822758e" +dependencies = [ + "debug_unsafe", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "av-scenechange" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394" +dependencies = [ + "aligned", + "anyhow", + "arg_enum_proc_macro", + "arrayvec", + "log", + "num-rational", + "num-traits", + "pastey", + "rayon", + "thiserror", + "v_frame", + "y4m", +] + +[[package]] +name = "av1-grain" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom 8.0.0", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47c8fbc0f831f4519fe8b810b6a7a91410ec83031b8233f730a0480029f6a23f" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bit_field" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "bitstream-io" +version = "4.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60d4bd9d1db2c6bdf285e223a7fa369d5ce98ec767dec949c6ca62863ce61757" +dependencies = [ + "core2", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "built" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] + +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "calamine" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96ae094b353c7810cd5efd2e69413ebb9354816138a387c09f7b90d4e826a49f" +dependencies = [ + "atoi_simd", + "byteorder", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml 0.38.4", + "serde", + "zip 7.4.0", +] + +[[package]] +name = "cc" +version = "1.2.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfb" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f" +dependencies = [ + "byteorder", + "fnv", + "uuid", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cssparser" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c66d1cd8ed61bf80b38432613a7a2f09401ab8d0501110655f8b341484a3e3" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.114", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + +[[package]] +name = "deflate64" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "derive_more" +version = "0.99.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", + "unicode-xid", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ego-tree" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_filter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "euclid" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb7ef65b3777a325d1eeefefab5b6d4959da54747e33bd6258e789640f307ad" +dependencies = [ + "num-traits", +] + +[[package]] +name = "exr" +version = "1.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4300e043a56aa2cb633c01af81ca8f699a321879a7854d3896a0ba89056363be" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "fast-float2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fax" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05de7d48f37cd6730705cbca900770cab77a89f413d23e100ad7fad7795a0ab" +dependencies = [ + "fax_derive", +] + +[[package]] +name = "fax_derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "gif" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5df2ba84018d80c213569363bdcd0c64e6933c67fe4c1d60ecf822971a3c35e" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hipstr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97971ffc85d4c98de12e2608e992a43f5294ebb625fdb045b27c731b64c4c6d6" +dependencies = [ + "serde", + "serde_bytes", + "sptr", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "html5ever" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c" +dependencies = [ + "log", + "mac", + "markup5ever", + "match_token", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "image" +version = "0.25.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6506c6c10786659413faa717ceebcb8f70731c0a60cbae39795fdf114519c1a" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core 0.5.1", + "zune-jpeg 0.5.12", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imageproc" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2393fb7808960751a52e8a154f67e7dd3f8a2ef9bd80d1553078a7b4e8ed3f0d" +dependencies = [ + "ab_glyph", + "approx", + "getrandom 0.2.17", + "image", + "itertools 0.12.1", + "nalgebra", + "num", + "rand 0.8.5", + "rand_distr", + "rayon", +] + +[[package]] +name = "imgref" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c5cedc30da3a610cac6b4ba17597bdf7152cf974e8aab3afb3d54455e371c8" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "infer" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7" +dependencies = [ + "cfb", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "jiff" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89a5b5e10d5a9ad6e5d1f4bd58225f655d6fe9767575a5e8ac5a6fe64e04495" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-sys 0.61.2", +] + +[[package]] +name = "jiff-static" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff7a39c8862fc1369215ccf0a8f12dd4598c7f6484704359f0351bd617034dbf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + +[[package]] +name = "libc" +version = "0.2.181" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459427e2af2b9c839b132acb702a1c654d95e10f8c326bfc2ad11310e458b1c5" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5037190e1f70cbeef565bd267599242926f724d3b8a9f510fd7e0b540cfa4404" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + +[[package]] +name = "lopdf" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5c8ecfc6c72051981c0459f75ccc585e7ff67c70829560cda8e647882a9abff" +dependencies = [ + "chrono", + "encoding_rs", + "flate2", + "indexmap", + "itoa", + "log", + "md-5", + "nom 7.1.3", + "rangemap", + "rayon", + "time", + "weezl", +] + +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18" +dependencies = [ + "log", + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "match_token" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "minio" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3824101357fa899d01c729e4a245776e20a03f2f6645979e86b9d3d5d9c42741" +dependencies = [ + "async-recursion", + "async-trait", + "base64", + "byteorder", + "bytes", + "chrono", + "crc", + "dashmap", + "derivative", + "env_logger", + "futures", + "futures-util", + "hex", + "hmac", + "http", + "hyper", + "lazy_static", + "log", + "md5", + "multimap", + "percent-encoding", + "rand 0.8.5", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "tokio", + "tokio-stream", + "tokio-util", + "urlencoding", + "xmltree", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moxcms" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac9557c559cd6fc9867e122e20d2cbefc9ca29d80d027a8e39310920ed2f0a97" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +dependencies = [ + "serde", +] + +[[package]] +name = "nalgebra" +version = "0.32.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5c17de023a86f59ed79891b2e5d5a94c705dbe904a5b5c9c952ea6221b03e4" +dependencies = [ + "approx", + "matrixmultiply", + "num-complex", + "num-rational", + "num-traits", + "simba", + "typenum", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "nvisy-core" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "derive_more 1.0.0", + "hex", + "hipstr", + "infer", + "jiff", + "schemars", + "serde", + "serde_json", + "sha2", + "strum", + "tempfile", + "thiserror", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "nvisy-engine" +version = "0.1.0" +dependencies = [ + "anyhow", + "jiff", + "nvisy-core", + "nvisy-ontology", + "petgraph", + "rand 0.9.2", + "schemars", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "uuid", +] + +[[package]] +name = "nvisy-ingest" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "calamine", + "csv", + "futures", + "image", + "infer", + "lopdf", + "nvisy-core", + "nvisy-ontology", + "pdf-extract", + "quick-xml 0.37.5", + "scraper", + "serde", + "serde_json", + "tokio", + "tracing", + "uuid", + "zip 2.4.2", +] + +[[package]] +name = "nvisy-object" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "futures", + "minio", + "nvisy-core", + "nvisy-pipeline", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "nvisy-ontology" +version = "0.1.0" +dependencies = [ + "derive_more 1.0.0", + "jiff", + "nvisy-core", + "schemars", + "semver", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "nvisy-pattern" +version = "0.1.0" +dependencies = [ + "nvisy-ontology", + "serde", + "serde_json", +] + +[[package]] +name = "nvisy-pipeline" +version = "0.1.0" +dependencies = [ + "aho-corasick", + "async-trait", + "bytes", + "image", + "imageproc", + "jiff", + "nvisy-core", + "nvisy-ingest", + "nvisy-ontology", + "nvisy-pattern", + "regex", + "serde", + "serde_json", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "nvisy-python" +version = "0.1.0" +dependencies = [ + "async-trait", + "nvisy-core", + "nvisy-ingest", + "nvisy-ontology", + "nvisy-pipeline", + "pyo3", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "owned_ttf_parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36820e9051aca1014ddc75770aab4d68bc1e9e632f0f5627c4086bc216fb583b" +dependencies = [ + "ttf-parser", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pdf-extract" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb3a5387b94b9053c1e69d8abfd4dd6dae7afda65a5c5279bc1f42ab39df575" +dependencies = [ + "adobe-cmap-parser", + "encoding_rs", + "euclid", + "lopdf", + "postscript", + "type1-encoding-parser", + "unicode-normalization", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", + "serde", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "png" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97baced388464909d42d89643fe4361939af9b7ce7a31ee32a168f832a70f2a0" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "pom" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postscript" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306" + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "profiling" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb8486b569e12e2c32ad3e204dbaba5e4b5b216e9367044f25f1dba42341773" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52717f9a02b6965224f95ca2a81e2e0c5c43baacd28ca057577988930b6c3d5b" +dependencies = [ + "quote", + "syn 2.0.114", +] + +[[package]] +name = "pxfm" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7186d3822593aa4393561d186d1393b3923e9d6163d3fbfd6e825e3e6cf3e6a8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "pyo3" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "serde", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", +] + +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "encoding_rs", + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "rav1e" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b" +dependencies = [ + "aligned-vec", + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av-scenechange", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools 0.14.0", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "paste", + "profiling", + "rand 0.9.2", + "rand_chacha 0.9.0", + "simd_helpers", + "thiserror", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef69c1990ceef18a116855938e74793a5f7496ee907562bd0857b6ac734ab285" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "rgb" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "bytes", + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "schemars_derive" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.114", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scraper" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc3d051b884f40e309de6c149734eab57aa8cc1347992710dc80bcc1c2194c15" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "precomputed-hash", + "selectors", + "tendril", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "selectors" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" +dependencies = [ + "bitflags", + "cssparser", + "derive_more 0.99.20", + "fxhash", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simba" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061507c94fc6ab4ba1c9a0305018408e312e17c041eb63bef8aa726fa33aceae" +dependencies = [ + "approx", + "num-complex", + "num-traits", + "paste", + "wide", +] + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "sptr" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "string_cache" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.114", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "tempfile" +version = "3.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "tiff" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9605de7fee8d9551863fd692cce7637f548dbd9db9180fcc07ccc6d26c336f" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg 0.4.21", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + +[[package]] +name = "type1-encoding-parser" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3d6cc09e1a99c7e01f2afe4953789311a1c50baebbdac5b477ecf78e2e92a5b" +dependencies = [ + "pom", +] + +[[package]] +name = "typed-path" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3015e6ce46d5ad8751e4a772543a30c7511468070e98e64e20165f8f81155b64" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537dd038a89878be9b64dd4bd1b260315c1bb94f4d784956b81e27a088d9a09e" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "v_frame" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.114", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "xml-rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" + +[[package]] +name = "xmltree" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b619f8c85654798007fb10afa5125590b43b088c225a25fc2fec100a9fad0fc6" +dependencies = [ + "xml-rs", +] + +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + +[[package]] +name = "y4m" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "getrandom 0.3.4", + "hmac", + "indexmap", + "lzma-rs", + "memchr", + "pbkdf2", + "sha1", + "thiserror", + "time", + "xz2", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zip" +version = "7.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc12baa6db2b15a140161ce53d72209dacea594230798c24774139b54ecaa980" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c" + +[[package]] +name = "zmij" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de98dfa5d5b7fef4ee834d0073d560c9ca7b6c46a71d058c48db7960f8cfaf7" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713" +dependencies = [ + "zune-core 0.4.12", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "410e9ecef634c709e3831c2cfdb8d9c32164fae1c67496d5b68fff728eec37fe" +dependencies = [ + "zune-core 0.5.1", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..f62acce --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,108 @@ +# https://doc.rust-lang.org/cargo/reference/manifest.html + +[workspace] +resolver = "2" +members = [ + "./crates/nvisy-core", + "./crates/nvisy-engine", + "./crates/nvisy-ingest", + "./crates/nvisy-object", + "./crates/nvisy-pattern", + "./crates/nvisy-pipeline", + "./crates/nvisy-ontology", + "./crates/nvisy-python", +] + +[workspace.package] +version = "0.1.0" +rust-version = "1.85" +edition = "2024" +license = "MIT" +publish = false + +authors = ["nvisy "] +repository = "https://github.com/nvisycom/runtime" +homepage = "https://github.com/nvisycom/runtime" +documentation = "https://docs.rs/nvisy-runtime" + +[workspace.dependencies] +# Default features are disabled for certain dependencies to allow +# downstream workspaces/crates to selectively enable them as needed. +# +# See for more details: https://github.com/rust-lang/cargo/issues/11329 + +# Internal crates +nvisy-core = { path = "./crates/nvisy-core", version = "0.1.0" } +nvisy-engine = { path = "./crates/nvisy-engine", version = "0.1.0" } +nvisy-ingest = { path = "./crates/nvisy-ingest", version = "0.1.0" } +nvisy-object = { path = "./crates/nvisy-object", version = "0.1.0" } +nvisy-pattern = { path = "./crates/nvisy-pattern", version = "0.1.0" } +nvisy-pipeline = { path = "./crates/nvisy-pipeline", version = "0.1.0" } +nvisy-ontology = { path = "./crates/nvisy-ontology", version = "0.1.0" } +nvisy-python = { path = "./crates/nvisy-python", version = "0.1.0" } + +# Async runtime +tokio = { version = "1", features = [] } +tokio-util = { version = "0.7", features = [] } +futures = { version = "0.3", features = [] } +async-trait = { version = "0.1", features = [] } + +# Observability +tracing = { version = "0.1", features = [] } + +# (De)serialization +serde = { version = "1.0", features = ["derive"] } +serde_json = { version = "1.0", features = [] } +schemars = { version = "1", features = ["uuid1", "bytes1"] } + +# Derive macros and error handling +thiserror = { version = "2.0", features = [] } +anyhow = { version = "1.0", features = [] } +derive_more = { version = "1", features = ["display", "from"] } +strum = { version = "0.26", features = ["derive"] } + +# Primitive datatypes +uuid = { version = "1", features = ["serde", "v4", "v7"] } +bytes = { version = "1", features = ["serde"] } +jiff = { version = "0.2", features = ["serde"] } +sha2 = { version = "0.10", features = [] } +hex = { version = "0.4", features = [] } + +# Text processing +hipstr = { version = "0.6", features = [] } +regex = { version = "1.0", features = [] } +aho-corasick = { version = "1", features = [] } +csv = { version = "1", features = [] } + +# Graph data structures +petgraph = { version = "0.8", features = [] } + +# File type detection +infer = { version = "0.19", features = [] } + +# Python interop +pyo3 = { version = "0.23", features = [] } + +# S3-compatible object storage +minio = { version = "0.3", features = [] } + +# Image processing +image = { version = "0.25", default-features = false, features = ["png", "jpeg", "tiff"] } +imageproc = { version = "0.25", features = [] } + +# Document parsing +pdf-extract = { version = "0.7", features = [] } +lopdf = { version = "0.34", features = [] } +scraper = { version = "0.22", features = [] } +calamine = { version = "0.33", features = [] } +zip = { version = "2", features = [] } +quick-xml = { version = "0.37", features = [] } + +# Semantic versioning +semver = { version = "1", features = ["serde"] } + +# Testing +tempfile = { version = "3", features = [] } + +# Randomness +rand = { version = "0.9", features = [] } diff --git a/Makefile b/Makefile index f740844..9ab97d1 100644 --- a/Makefile +++ b/Makefile @@ -5,48 +5,50 @@ ifneq (,$(wildcard ./.env)) export endif -# Shell-level logger (expands to a printf that runs in the shell). +export PYO3_USE_ABI3_FORWARD_COMPATIBILITY := 1 + define log printf "[%s] [MAKE] [$(MAKECMDGOALS)] $(1)\n" "$$(date '+%Y-%m-%d %H:%M:%S')" endef -WATCH_PATHS := $(foreach p,$(wildcard packages/*/dist),--watch-path=$(p)) - .PHONY: dev -dev: ## Starts build watchers and dev server concurrently. - @for pkg in packages/*/; do \ - npm run build:watch --workspace=$$pkg & \ - done; \ - node $(WATCH_PATHS) packages/nvisy-server/dist/main.js & \ - wait - -.PHONY: dev\:prod -dev\:prod: ## Starts dev server with production log level (info). - @for pkg in packages/*/; do \ - npm run build:watch --workspace=$$pkg & \ - done; \ - NODE_ENV=production node $(WATCH_PATHS) packages/nvisy-server/dist/main.js & \ - wait +dev: ## Starts cargo-watch for the server binary. + @cargo watch -x 'run -p nvisy-server' + +.PHONY: build +build: ## Builds all crates in release mode. + @$(call log,Building workspace...) + @cargo build --workspace --release + @$(call log,Build complete.) + +.PHONY: check +check: ## Runs cargo check on all crates. + @cargo check --workspace + +.PHONY: test +test: ## Runs all tests. + @cargo test --workspace + +.PHONY: lint +lint: ## Runs clippy and format check. + @$(call log,Running format check...) + @cargo fmt --all -- --check + @$(call log,Running clippy...) + @cargo clippy --workspace -- -D warnings + @$(call log,Lint passed.) + +.PHONY: fmt +fmt: ## Formats all Rust code. + @cargo fmt --all .PHONY: ci -ci: ## Runs all CI checks locally (lint, typecheck, test, build). - @$(call log,Running lint...) - @npx biome check . - @$(call log,Running typecheck...) - @npx tsc -b packages/*/tsconfig.json - @$(call log,Running tests...) - @npx vitest run --coverage - @$(call log,Running build...) - @npm run build --workspaces --if-present +ci: lint check test build ## Runs all CI checks locally. @$(call log,All CI checks passed!) .PHONY: clean -clean: ## Removes all build artifacts and node_modules. +clean: ## Removes build artifacts. @$(call log,Cleaning build artifacts...) - @npx tsc -b --clean packages/*/tsconfig.json - @rm -rf packages/*/dist - @$(call log,Removing node_modules...) - @rm -rf node_modules packages/*/node_modules package-lock.json + @cargo clean @$(call log,Clean complete.) .PHONY: docker @@ -54,3 +56,8 @@ docker: ## Builds the Docker image. @$(call log,Building Docker image...) @docker build -f docker/Dockerfile -t nvisy-runtime . @$(call log,Docker image built.) + +.PHONY: help +help: ## Shows this help message. + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-12s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 1783c89..7201a3e 100644 --- a/README.md +++ b/README.md @@ -2,41 +2,45 @@ [![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/build.yml) -An open-source ETL platform purpose-built for LLM and AI data pipelines. +A data protection runtime for AI pipelines — detect, redact, and audit sensitive data across documents, images, and streams. -Nvisy Runtime treats AI data as a first-class citizen: embeddings, completions, -structured outputs, tool-call traces, images, audio, and fine-tuning datasets -all flow through typed, validated primitives with full lineage tracking. +Built in Rust with Python extensions for AI-powered detection. -## Packages +## Workspace -| Package | Description | -|---------|-------------| -| [`nvisy-core`](packages/nvisy-core/) | Core data types, errors, and utilities | -| [`nvisy-runtime`](packages/nvisy-runtime/) | Graph definition, DAG compiler, execution engine | -| [`nvisy-plugin-ai`](packages/nvisy-plugin-ai/) | AI provider integrations (OpenAI, Anthropic, Google) | -| [`nvisy-plugin-object`](packages/nvisy-plugin-object/) | Object store integrations (S3, GCS, Parquet, JSONL, CSV) | -| [`nvisy-plugin-sql`](packages/nvisy-plugin-sql/) | SQL provider integrations (Postgres, MySQL, MSSQL) | -| [`nvisy-plugin-vector`](packages/nvisy-plugin-vector/) | Vector database integrations (Qdrant, Milvus, Weaviate, Pinecone) | -| [`nvisy-server`](packages/nvisy-server/) | HTTP execution worker | - -See [packages/](packages/README.md) for detailed descriptions. +``` +crates/ + nvisy-core/ Types, traits, plugin registry, error handling + nvisy-detect/ Regex patterns, policy evaluation, redaction actions + nvisy-engine/ DAG graph compiler and execution engine + nvisy-object/ Object storage connectors (S3) + nvisy-python/ Python interop for AI-powered NER via PyO3 + nvisy-server/ Axum HTTP server with REST API + +packages/ + nvisy-ai/ Python: LLM-based entity detection + nvisy-exif/ Python: EXIF metadata reading and stripping +``` ## Quick Start ```bash -npm install -npm run build +cargo build --workspace +cargo test --workspace +cargo run -p nvisy-server ``` -## Documentation +## Development -See [`docs/`](docs/) for architecture, intelligence capabilities, provider -design, and security documentation. +```bash +make dev # cargo-watch dev server +make ci # lint + check + test + build +make help # list all targets +``` -## Changelog +## Documentation -See [CHANGELOG.md](CHANGELOG.md) for release notes and version history. +See [`docs/`](docs/) for architecture and development documentation. ## License diff --git a/biome.json b/biome.json deleted file mode 100644 index b8fb431..0000000 --- a/biome.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "$schema": "https://biomejs.dev/schemas/2.3.14/schema.json", - "vcs": { - "enabled": true, - "clientKind": "git", - "useIgnoreFile": true - }, - "files": { - "ignoreUnknown": false, - "includes": ["**", "!node_modules", "!dist", "!coverage"] - }, - "formatter": { - "enabled": true, - "indentStyle": "tab" - }, - "linter": { - "enabled": true, - "rules": { - "recommended": true, - "style": { - "recommended": true, - "noNonNullAssertion": "off" - }, - "correctness": { - "noUnusedImports": "warn", - "noUnusedPrivateClassMembers": "warn", - "useYield": "warn" - }, - "complexity": { - "noUselessConstructor": "warn" - }, - "suspicious": { - "noEmptyInterface": "off" - } - } - }, - "javascript": { - "formatter": { - "quoteStyle": "double" - } - } -} diff --git a/crates/nvisy-core/Cargo.toml b/crates/nvisy-core/Cargo.toml new file mode 100644 index 0000000..8172dd7 --- /dev/null +++ b/crates/nvisy-core/Cargo.toml @@ -0,0 +1,60 @@ +# https://doc.rust-lang.org/cargo/reference/manifest.html + +[package] +name = "nvisy-core" +description = "Domain types, traits, and errors for the Nvisy platform" +keywords = ["nvisy", "core", "domain", "types"] +categories = ["data-structures"] + +version = { workspace = true } +rust-version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } + +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } + +[dependencies] +# JSON Schema generation +schemars = { workspace = true, features = [] } + +# (De)serialization +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = [] } + +# Async runtime +tokio = { workspace = true, features = ["sync", "fs", "io-util", "rt"] } + +# Primitive datatypes +uuid = { workspace = true, features = ["serde", "v4", "v7"] } +bytes = { workspace = true, features = ["serde"] } + +# File type detection +infer = { workspace = true, features = [] } + +# Error handling +thiserror = { workspace = true, features = [] } +anyhow = { workspace = true, features = [] } +derive_more = { workspace = true, features = ["display", "deref", "as_ref"] } + +# Time +jiff = { workspace = true, features = [] } + +# Interned strings +hipstr = { workspace = true, features = [] } + +# Hashing +sha2 = { workspace = true, features = [] } +hex = { workspace = true, features = [] } + +# Enum derives +strum = { workspace = true, features = [] } + +# Observability +tracing = { workspace = true, features = [] } + +[dev-dependencies] +tempfile = { workspace = true, features = [] } diff --git a/crates/nvisy-core/README.md b/crates/nvisy-core/README.md new file mode 100644 index 0000000..63e5fd2 --- /dev/null +++ b/crates/nvisy-core/README.md @@ -0,0 +1,3 @@ +# nvisy-core + +Foundational crate for the Nvisy runtime. Defines domain types, error types, the plugin trait system, and the action/provider registry that all other crates build on. diff --git a/crates/nvisy-core/src/error.rs b/crates/nvisy-core/src/error.rs new file mode 100644 index 0000000..3d669b7 --- /dev/null +++ b/crates/nvisy-core/src/error.rs @@ -0,0 +1,158 @@ +//! Unified error types for the nvisy platform. +//! +//! All crates in the nvisy workspace use [`Error`] as their primary error +//! type and [`ErrorKind`] to classify failures. + +use derive_more::Display; + +/// Classification of error kinds. +/// +/// Used to tag every [`Error`] so callers can programmatically decide +/// how to handle a failure (e.g. retry on `Timeout`, surface to user +/// on `Validation`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Display)] +pub enum ErrorKind { + /// Input or configuration failed validation checks. + Validation, + /// Could not connect to an external service. + Connection, + /// An operation exceeded its time limit. + Timeout, + /// The operation was explicitly cancelled. + Cancellation, + /// A policy rule was violated. + Policy, + /// An internal runtime error occurred. + Runtime, + /// An error originating from the embedded Python bridge. + Python, + /// An internal infrastructure error (filesystem, I/O). + InternalError, + /// The input was invalid or out of bounds. + InvalidInput, + /// A serialization or encoding error. + Serialization, + /// An error that does not fit any other category. + Other, +} + +/// Unified error type for the nvisy platform. +/// +/// Carries a [`kind`](ErrorKind), a human-readable message, an optional +/// source component name, a retryable flag, and an optional wrapped cause. +#[derive(Debug, thiserror::Error)] +#[error("{kind}: {message}")] +pub struct Error { + /// Classification of the error. + pub kind: ErrorKind, + /// Human-readable description of what went wrong. + pub message: String, + /// Name of the component that produced this error (e.g. `"s3-read"`, `"detect-regex"`). + pub source_component: Option, + /// Whether the operation that failed can be safely retried. + pub retryable: bool, + /// The underlying cause, if any. + #[source] + pub source: Option>, +} + +impl Error { + /// Create a new error with the given kind and message. + pub fn new(kind: ErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + source_component: None, + retryable: false, + source: None, + } + } + + /// Attach an underlying cause to this error. + pub fn with_source(mut self, source: impl std::error::Error + Send + Sync + 'static) -> Self { + self.source = Some(Box::new(source)); + self + } + + /// Tag this error with the name of the component that produced it. + pub fn with_component(mut self, component: impl Into) -> Self { + self.source_component = Some(component.into()); + self + } + + /// Mark whether this error is safe to retry. + pub fn with_retryable(mut self, retryable: bool) -> Self { + self.retryable = retryable; + self + } + + /// Shorthand for a validation error with a source component. + pub fn validation(message: impl Into, source: impl Into) -> Self { + Self::new(ErrorKind::Validation, message).with_component(source) + } + + /// Shorthand for a connection error with a source component and retryable flag. + pub fn connection( + message: impl Into, + source: impl Into, + retryable: bool, + ) -> Self { + Self::new(ErrorKind::Connection, message) + .with_component(source) + .with_retryable(retryable) + } + + /// Shorthand for a timeout error (always retryable). + pub fn timeout(message: impl Into) -> Self { + Self::new(ErrorKind::Timeout, message).with_retryable(true) + } + + /// Shorthand for a cancellation error. + pub fn cancellation(message: impl Into) -> Self { + Self::new(ErrorKind::Cancellation, message) + } + + /// Shorthand for a policy violation error. + pub fn policy(message: impl Into) -> Self { + Self::new(ErrorKind::Policy, message) + } + + /// Shorthand for a runtime error with a source component and retryable flag. + pub fn runtime( + message: impl Into, + source: impl Into, + retryable: bool, + ) -> Self { + Self::new(ErrorKind::Runtime, message) + .with_component(source) + .with_retryable(retryable) + } + + /// Shorthand for a Python bridge error. + pub fn python(message: impl Into) -> Self { + Self::new(ErrorKind::Python, message) + } + + /// Whether this error is retryable. + pub fn is_retryable(&self) -> bool { + self.retryable + } +} + +impl From for Error { + fn from(err: std::io::Error) -> Self { + Self::new(ErrorKind::InternalError, err.to_string()) + .with_source(err) + } +} + +impl From for Error { + fn from(err: anyhow::Error) -> Self { + // anyhow::Error doesn't implement std::error::Error, so we capture the + // full chain as text instead of storing it as a boxed source. + Self::new(ErrorKind::Other, format!("{err:#}")) + } +} + +/// Convenience type alias for results using the Nvisy error type. +pub type Result = std::result::Result; diff --git a/crates/nvisy-core/src/fs/content_file.rs b/crates/nvisy-core/src/fs/content_file.rs new file mode 100644 index 0000000..0d46e06 --- /dev/null +++ b/crates/nvisy-core/src/fs/content_file.rs @@ -0,0 +1,637 @@ +//! Content file handling for filesystem operations +//! +//! This module provides the [`ContentFile`] struct for working with files +//! on the filesystem while maintaining content source tracking and metadata. + +use std::io; +use std::path::{Path, PathBuf}; + +use bytes::Bytes; +use tokio::fs::{File, OpenOptions}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, SeekFrom}; + +use crate::error::{Error, ErrorKind, Result}; +use crate::fs::ContentMetadata; +use crate::io::{AsyncContentRead, AsyncContentWrite, ContentData}; +use crate::path::ContentSource; + +/// A file wrapper that combines filesystem operations with content tracking +/// +/// This struct provides a high-level interface for working with files while +/// maintaining content source identification and metadata throughout the +/// processing pipeline. +#[derive(Debug)] +pub struct ContentFile { + /// Unique identifier for this content source + content_source: ContentSource, + /// The underlying tokio file handle + file: File, + /// Path to the file + path: PathBuf, +} + +impl ContentFile { + /// Create a new `ContentFile` by opening an existing file + /// + /// # Errors + /// + /// Returns an error if the file cannot be opened or doesn't exist. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::fs::ContentFile; + /// use std::path::Path; + /// + /// async fn open_file() -> Result<(), Box> { + /// let content_file = ContentFile::open("example.txt").await?; + /// println!("Opened file with source: {}", content_file.content_source()); + /// Ok(()) + /// } + /// ``` + pub async fn open(path: impl AsRef) -> io::Result { + let path_buf = path.as_ref().to_path_buf(); + let file = File::open(&path_buf).await?; + let content_source = ContentSource::new(); + + Ok(Self { + content_source, + file, + path: path_buf, + }) + } + + /// Create a new `ContentFile` with a specific content source + /// + /// # Errors + /// + /// Returns an error if the file cannot be opened or read. + pub async fn open_with_source( + path: impl AsRef, + content_source: ContentSource, + ) -> io::Result { + let path_buf = path.as_ref().to_path_buf(); + let file = File::open(&path_buf).await?; + + Ok(Self { + content_source, + file, + path: path_buf, + }) + } + + /// Create a new file and return a `ContentFile` + /// + /// # Errors + /// + /// Returns an error if the file cannot be created. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::fs::ContentFile; + /// + /// async fn create_file() -> Result<(), Box> { + /// let content_file = ContentFile::create("new_file.txt").await?; + /// println!("Created file with source: {}", content_file.content_source()); + /// Ok(()) + /// } + /// ``` + pub async fn create(path: impl AsRef) -> io::Result { + let path_buf = path.as_ref().to_path_buf(); + let file = File::create(&path_buf).await?; + let content_source = ContentSource::new(); + + Ok(Self { + content_source, + file, + path: path_buf, + }) + } + + /// Create a new file with a specific content source + /// + /// # Errors + /// + /// Returns an error if the file cannot be created or written to. + pub async fn create_with_source( + path: impl AsRef, + content_source: ContentSource, + ) -> io::Result { + let path_buf = path.as_ref().to_path_buf(); + let file = File::create(&path_buf).await?; + + Ok(Self { + content_source, + file, + path: path_buf, + }) + } + + /// Open a file with custom options + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::fs::ContentFile; + /// use tokio::fs::OpenOptions; + /// + /// async fn open_with_options() -> Result<(), Box> { + /// let mut options = OpenOptions::new(); + /// options.read(true) + /// .write(true) + /// .create(true); + /// + /// let content_file = ContentFile::open_with_options("data.txt", &options).await?; + /// Ok(()) + /// } + /// ``` + /// + /// # Errors + /// + /// Returns an error if the file cannot be opened with the specified options. + pub async fn open_with_options( + path: impl AsRef, + options: &OpenOptions, + ) -> io::Result { + let path_buf = path.as_ref().to_path_buf(); + let file = options.open(&path_buf).await?; + let content_source = ContentSource::new(); + + Ok(Self { + content_source, + file, + path: path_buf, + }) + } + + /// Read all content from the file into a `ContentData` structure + /// + /// # Errors + /// + /// Returns an error if the file cannot be read or if an I/O error occurs. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::fs::ContentFile; + /// + /// async fn read_content() -> Result<(), Box> { + /// let mut content_file = ContentFile::open("example.txt").await?; + /// let content_data = content_file.read_to_content_data().await?; + /// + /// println!("Read {} bytes", content_data.size()); + /// Ok(()) + /// } + /// ``` + pub async fn read_to_content_data(&mut self) -> Result { + let mut buffer = Vec::new(); + self.file.read_to_end(&mut buffer).await?; + + let content_data = ContentData::new(self.content_source, Bytes::from(buffer)); + + Ok(content_data) + } + + /// Read content with size limit to prevent memory issues + /// + /// # Errors + /// + /// Returns an error if the file cannot be read, if an I/O error occurs, + /// or if the file size exceeds the specified maximum size. + pub async fn read_to_content_data_limited(&mut self, max_size: usize) -> Result { + let mut buffer = Vec::new(); + let mut temp_buffer = vec![0u8; 8192]; + let mut total_read = 0; + + loop { + let bytes_read = self.file.read(&mut temp_buffer).await?; + if bytes_read == 0 { + break; // EOF + } + + if total_read + bytes_read > max_size { + return Err(Error::new(ErrorKind::InvalidInput, format!( + "File size exceeds maximum limit of {max_size} bytes" + ))); + } + + buffer.extend_from_slice(&temp_buffer[..bytes_read]); + total_read += bytes_read; + } + + let content_data = ContentData::new(self.content_source, Bytes::from(buffer)); + + Ok(content_data) + } + + /// Write `ContentData` to the file + /// + /// # Errors + /// + /// Returns an error if the data cannot be written or if an I/O error occurs. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::fs::ContentFile; + /// use nvisy_core::io::ContentData; + /// + /// async fn write_content() -> Result<(), Box> { + /// let mut content_file = ContentFile::create("output.txt").await?; + /// let content_data = ContentData::from("Hello, world!"); + /// + /// let metadata = content_file.write_from_content_data(content_data).await?; + /// println!("Written to: {:?}", metadata.source_path); + /// Ok(()) + /// } + /// ``` + pub async fn write_from_content_data( + &mut self, + content_data: ContentData, + ) -> Result { + self.file.write_all(content_data.as_bytes()).await?; + self.file.flush().await?; + + let metadata = ContentMetadata::with_path(content_data.content_source, self.path.clone()); + Ok(metadata) + } + + /// Append `ContentData` to the file + /// + /// # Errors + /// + /// Returns an error if the data cannot be appended or if an I/O error occurs. + pub async fn append_from_content_data( + &mut self, + content_data: ContentData, + ) -> Result { + self.file.seek(SeekFrom::End(0)).await?; + self.file.write_all(content_data.as_bytes()).await?; + self.file.flush().await?; + + let metadata = ContentMetadata::with_path(content_data.content_source, self.path.clone()); + Ok(metadata) + } + + /// Write `ContentData` in chunks for better memory efficiency + /// + /// # Errors + /// + /// Returns an error if the data cannot be written or if an I/O error occurs. + pub async fn write_from_content_data_chunked( + &mut self, + content_data: ContentData, + chunk_size: usize, + ) -> Result { + let data = content_data.as_bytes(); + + for chunk in data.chunks(chunk_size) { + self.file.write_all(chunk).await?; + } + + self.file.flush().await?; + + let metadata = ContentMetadata::with_path(content_data.content_source, self.path.clone()); + Ok(metadata) + } + + /// Get content metadata for this file + pub fn content_metadata(&self) -> ContentMetadata { + ContentMetadata::with_path(self.content_source, self.path.clone()) + } + + /// Get the file path + pub fn path(&self) -> &Path { + &self.path + } + + /// Get the content source + pub fn content_source(&self) -> ContentSource { + self.content_source + } + + /// Get the source identifier for this content + pub fn source(&self) -> ContentSource { + self.content_source + } + + /// Get a reference to the underlying file + pub fn as_file(&self) -> &File { + &self.file + } + + /// Get a mutable reference to the underlying file + pub fn as_file_mut(&mut self) -> &mut File { + &mut self.file + } + + /// Convert into the underlying file, consuming the `ContentFile` + pub fn into_file(self) -> File { + self.file + } + + /// Get file size in bytes + /// + /// # Errors + /// + /// Returns an error if the file metadata cannot be retrieved. + pub async fn size(&mut self) -> Result { + let metadata = self.file.metadata().await?; + Ok(metadata.len()) + } + + /// Check if the file exists + pub fn exists(&self) -> bool { + self.path.exists() + } + + /// Get the filename + pub fn filename(&self) -> Option<&str> { + self.path.file_name().and_then(|name| name.to_str()) + } + + /// Get the file extension + pub fn extension(&self) -> Option<&str> { + self.path.extension().and_then(|ext| ext.to_str()) + } + + /// Sync all data to disk + /// + /// # Errors + /// + /// Returns an error if the sync operation fails. + pub async fn sync_all(&mut self) -> Result<()> { + self.file.sync_all().await?; + Ok(()) + } + + /// Sync data (but not metadata) to disk + /// + /// # Errors + /// + /// Returns an error if the sync operation fails. + pub async fn sync_data(&mut self) -> Result<()> { + self.file.sync_data().await?; + Ok(()) + } + + /// Seek to a specific position in the file + /// + /// # Errors + /// + /// Returns an error if the seek operation fails. + pub async fn seek(&mut self, pos: SeekFrom) -> Result { + let position = self.file.seek(pos).await?; + Ok(position) + } + + /// Get current position in the file + /// + /// # Errors + /// + /// Returns an error if the current position cannot be determined. + pub async fn stream_position(&mut self) -> Result { + let position = self.file.stream_position().await?; + Ok(position) + } +} + +// Implement AsyncRead for ContentFile by delegating to the underlying file +impl AsyncRead for ContentFile { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.file).poll_read(cx, buf) + } +} + +// Implement AsyncWrite for ContentFile by delegating to the underlying file +impl AsyncWrite for ContentFile { + fn poll_write( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.file).poll_write(cx, buf) + } + + fn poll_flush( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.file).poll_flush(cx) + } + + fn poll_shutdown( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.file).poll_shutdown(cx) + } +} + +// Implement AsyncContentRead for ContentFile by delegating to the underlying file +impl AsyncContentRead for ContentFile { + // Default implementations from the trait will work since File implements AsyncRead +} + +// Implement AsyncContentWrite for ContentFile by delegating to the underlying file +impl AsyncContentWrite for ContentFile { + // Default implementations from the trait will work since File implements AsyncWrite +} + +#[cfg(test)] +mod tests { + use tempfile::NamedTempFile; + + use super::*; + + #[tokio::test] + async fn test_create_and_open() { + let temp_file = NamedTempFile::new().unwrap(); + let path = temp_file.path(); + + // Create file + let content_file = ContentFile::create(path).await.unwrap(); + assert_eq!(content_file.path(), path); + assert!(!content_file.content_source.as_uuid().is_nil()); + + // Clean up + drop(content_file); + + // Open existing file + let content_file = ContentFile::open(path).await.unwrap(); + assert_eq!(content_file.path(), path); + } + + #[tokio::test] + async fn test_write_and_read_content_data() { + let temp_file = NamedTempFile::new().unwrap(); + let path = temp_file.path(); + + // Write content + let mut content_file = ContentFile::create(path).await.unwrap(); + let content_data = ContentData::from("Hello, world!"); + let metadata = content_file + .write_from_content_data(content_data) + .await + .unwrap(); + + assert_eq!(metadata.source_path, Some(path.to_path_buf())); + + // Read content back + drop(content_file); + let mut content_file = ContentFile::open(path).await.unwrap(); + let read_content = content_file.read_to_content_data().await.unwrap(); + + assert_eq!(read_content.as_string().unwrap(), "Hello, world!"); + } + + #[tokio::test] + async fn test_file_extension() { + let temp_file = NamedTempFile::new().unwrap(); + let mut path = temp_file.path().to_path_buf(); + path.set_extension("txt"); + + let content_file = ContentFile::create(&path).await.unwrap(); + assert_eq!(content_file.extension(), Some("txt")); + assert_eq!( + content_file.filename(), + path.file_name().and_then(|n| n.to_str()) + ); + } + + #[tokio::test] + async fn test_write_chunked() { + let temp_file = NamedTempFile::new().unwrap(); + let path = temp_file.path(); + + let mut content_file = ContentFile::create(path).await.unwrap(); + let large_data = vec![b'A'; 1000]; + let content_data = ContentData::from(large_data.clone()); + + let metadata = content_file + .write_from_content_data_chunked(content_data, 100) + .await + .unwrap(); + assert_eq!(metadata.source_path, Some(path.to_path_buf())); + + // Verify content + drop(content_file); + let mut content_file = ContentFile::open(path).await.unwrap(); + let read_content = content_file.read_to_content_data().await.unwrap(); + + assert_eq!(read_content.as_bytes(), large_data.as_slice()); + } + + #[tokio::test] + async fn test_append_content() { + let temp_file = NamedTempFile::new().unwrap(); + let path = temp_file.path(); + + // Write initial content + let mut content_file = ContentFile::create(path).await.unwrap(); + let initial_content = ContentData::from("Hello, "); + content_file + .write_from_content_data(initial_content) + .await + .unwrap(); + + // Append more content + let append_content = ContentData::from("world!"); + content_file + .append_from_content_data(append_content) + .await + .unwrap(); + + // Verify combined content + drop(content_file); + let mut content_file = ContentFile::open(path).await.unwrap(); + let read_content = content_file.read_to_content_data().await.unwrap(); + + assert_eq!(read_content.as_string().unwrap(), "Hello, world!"); + } + + #[tokio::test] + async fn test_read_with_limit() { + let temp_file = NamedTempFile::new().unwrap(); + let path = temp_file.path(); + + // Write content larger than limit + let mut content_file = ContentFile::create(path).await.unwrap(); + let large_content = ContentData::from(vec![b'X'; 1000]); + content_file + .write_from_content_data(large_content) + .await + .unwrap(); + + drop(content_file); + + // Try to read with small limit + let mut content_file = ContentFile::open(path).await.unwrap(); + let result = content_file.read_to_content_data_limited(100).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_file_operations() { + let temp_file = NamedTempFile::new().unwrap(); + let path = temp_file.path(); + + let mut content_file = ContentFile::create(path).await.unwrap(); + + // Test size (should be 0 for new file) + let size = content_file.size().await.unwrap(); + assert_eq!(size, 0); + + // Test existence + assert!(content_file.exists()); + + // Write some content + let content = ContentData::from("Test content"); + content_file.write_from_content_data(content).await.unwrap(); + + // Test size after writing + let size = content_file.size().await.unwrap(); + assert!(size > 0); + + // Test sync operations + content_file.sync_all().await.unwrap(); + content_file.sync_data().await.unwrap(); + } + + #[tokio::test] + async fn test_seeking() { + let temp_file = NamedTempFile::new().unwrap(); + let path = temp_file.path(); + + let mut content_file = ContentFile::create(path).await.unwrap(); + let content = ContentData::from("0123456789"); + content_file.write_from_content_data(content).await.unwrap(); + + // Test seeking + let pos = content_file.seek(SeekFrom::Start(5)).await.unwrap(); + assert_eq!(pos, 5); + + let current_pos = content_file.stream_position().await.unwrap(); + assert_eq!(current_pos, 5); + } + + #[tokio::test] + async fn test_with_specific_source() { + let temp_file = NamedTempFile::new().unwrap(); + let path = temp_file.path(); + + let source = ContentSource::new(); + let content_file = ContentFile::create_with_source(path, source).await.unwrap(); + + assert_eq!(content_file.content_source, source); + + let metadata = content_file.content_metadata(); + assert_eq!(metadata.content_source, source); + assert_eq!(metadata.source_path, Some(path.to_path_buf())); + } +} diff --git a/crates/nvisy-core/src/fs/content_handler.rs b/crates/nvisy-core/src/fs/content_handler.rs new file mode 100644 index 0000000..8e54505 --- /dev/null +++ b/crates/nvisy-core/src/fs/content_handler.rs @@ -0,0 +1,129 @@ +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use tokio::runtime::Handle; + +use crate::path::ContentSource; + +/// Inner state cleaned up when the last `ContentHandler` reference is dropped. +struct ContentHandlerInner { + content_source: ContentSource, + dir: PathBuf, + runtime_handle: Handle, +} + +impl fmt::Debug for ContentHandlerInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ContentHandlerInner") + .field("content_source", &self.content_source) + .field("dir", &self.dir) + .finish() + } +} + +impl Drop for ContentHandlerInner { + fn drop(&mut self) { + let dir = self.dir.clone(); + let source = self.content_source; + + self.runtime_handle.spawn(async move { + if let Err(err) = tokio::fs::remove_dir_all(&dir).await { + tracing::warn!( + target: "nvisy_core::fs", + content_source = %source, + path = %dir.display(), + error = %err, + "Failed to clean up temporary content directory" + ); + } else { + tracing::trace!( + target: "nvisy_core::fs", + content_source = %source, + path = %dir.display(), + "Cleaned up temporary content directory" + ); + } + }); + } +} + +/// Handle to content stored in a managed temporary directory. +/// +/// Cloning is cheap — clones share the same underlying directory via `Arc`. +/// When the last clone is dropped, the temporary directory is deleted. +#[derive(Debug, Clone)] +pub struct ContentHandler { + inner: Arc, +} + +impl ContentHandler { + /// Creates a new content handler. + pub(crate) fn new(source: ContentSource, dir: PathBuf, handle: Handle) -> Self { + Self { + inner: Arc::new(ContentHandlerInner { + content_source: source, + dir, + runtime_handle: handle, + }), + } + } + + /// Returns the content source identifier. + pub fn content_source(&self) -> ContentSource { + self.inner.content_source + } + + /// Returns the path to the temporary directory. + pub fn dir(&self) -> &Path { + &self.inner.dir + } +} + +#[cfg(test)] +mod tests { + use crate::fs::ContentRegistry; + use crate::io::{Content, ContentData}; + + #[tokio::test] + async fn test_handler_has_valid_source() { + let temp = tempfile::TempDir::new().unwrap(); + let registry = ContentRegistry::new(temp.path().join("content")); + let content = Content::new(ContentData::from("test data")); + let handler = registry.register(content).await.unwrap(); + + assert!(!handler.content_source().as_uuid().is_nil()); + assert!(handler.dir().exists()); + } + + #[tokio::test] + async fn test_clone_shares_same_directory() { + let temp = tempfile::TempDir::new().unwrap(); + let registry = ContentRegistry::new(temp.path().join("content")); + let content = Content::new(ContentData::from("shared")); + let handler1 = registry.register(content).await.unwrap(); + let handler2 = handler1.clone(); + + assert_eq!(handler1.dir(), handler2.dir()); + } + + #[tokio::test] + async fn test_directory_cleaned_on_last_drop() { + let temp = tempfile::TempDir::new().unwrap(); + let registry = ContentRegistry::new(temp.path().join("content")); + let content = Content::new(ContentData::from("cleanup test")); + let handler = registry.register(content).await.unwrap(); + let dir = handler.dir().to_path_buf(); + let handler2 = handler.clone(); + + assert!(dir.exists()); + + drop(handler); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!(dir.exists()); + + drop(handler2); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!(!dir.exists()); + } +} diff --git a/crates/nvisy-core/src/fs/content_kind.rs b/crates/nvisy-core/src/fs/content_kind.rs new file mode 100644 index 0000000..288f488 --- /dev/null +++ b/crates/nvisy-core/src/fs/content_kind.rs @@ -0,0 +1,132 @@ +//! Content type classification for different categories of data +//! +//! This module provides the [`ContentKind`] enum for classifying content +//! into broad categories. Extension-to-kind mapping is handled by the +//! engine's format registry. + +use serde::{Deserialize, Serialize}; +use strum::{AsRefStr, Display, EnumIter, EnumString}; + +/// Content type classification for different categories of data +/// +/// This enum represents high-level content categories without knowledge +/// of specific file extensions or MIME types. The engine's format registry +/// handles the mapping from extensions/MIME types to content kinds. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(AsRefStr, Display, EnumString, EnumIter)] +#[derive(Serialize, Deserialize)] +#[strum(serialize_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum ContentKind { + /// Plain text content + Text, + /// Document files (PDF, Word, etc.) + Document, + /// Spreadsheet files (Excel, CSV, etc.) + Spreadsheet, + /// Image files + Image, + /// Archive files (ZIP, TAR, etc.) + Archive, + /// Unknown or unsupported content type + #[default] + Unknown, +} + +impl ContentKind { + /// Check if this content kind represents text-based content + #[must_use] + pub fn is_text_based(&self) -> bool { + matches!(self, Self::Text) + } + + /// Check if this content kind represents a document + #[must_use] + pub fn is_document(&self) -> bool { + matches!(self, Self::Document) + } + + /// Check if this content kind represents a spreadsheet + #[must_use] + pub fn is_spreadsheet(&self) -> bool { + matches!(self, Self::Spreadsheet) + } + + /// Check if this content kind represents an image + #[must_use] + pub fn is_image(&self) -> bool { + matches!(self, Self::Image) + } + + /// Check if this content kind represents an archive + #[must_use] + pub fn is_archive(&self) -> bool { + matches!(self, Self::Archive) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_content_kind_predicates() { + assert!(ContentKind::Text.is_text_based()); + assert!(!ContentKind::Document.is_text_based()); + + assert!(ContentKind::Document.is_document()); + assert!(!ContentKind::Text.is_document()); + + assert!(ContentKind::Spreadsheet.is_spreadsheet()); + assert!(!ContentKind::Document.is_spreadsheet()); + + assert!(ContentKind::Image.is_image()); + assert!(!ContentKind::Text.is_image()); + + assert!(ContentKind::Archive.is_archive()); + assert!(!ContentKind::Document.is_archive()); + } + + #[test] + fn test_content_kind_display() { + assert_eq!(ContentKind::Text.to_string(), "text"); + assert_eq!(ContentKind::Document.to_string(), "document"); + assert_eq!(ContentKind::Spreadsheet.to_string(), "spreadsheet"); + assert_eq!(ContentKind::Image.to_string(), "image"); + assert_eq!(ContentKind::Archive.to_string(), "archive"); + assert_eq!(ContentKind::Unknown.to_string(), "unknown"); + } + + #[test] + fn test_content_kind_as_ref() { + assert_eq!(ContentKind::Text.as_ref(), "text"); + assert_eq!(ContentKind::Document.as_ref(), "document"); + } + + #[test] + fn test_content_kind_from_str() { + use std::str::FromStr; + + assert_eq!(ContentKind::from_str("text").unwrap(), ContentKind::Text); + assert_eq!( + ContentKind::from_str("document").unwrap(), + ContentKind::Document + ); + assert!(ContentKind::from_str("invalid").is_err()); + } + + #[test] + fn test_default() { + assert_eq!(ContentKind::default(), ContentKind::Unknown); + } + + #[test] + fn test_serialization() { + let kind = ContentKind::Spreadsheet; + let serialized = serde_json::to_string(&kind).unwrap(); + assert_eq!(serialized, "\"spreadsheet\""); + + let deserialized: ContentKind = serde_json::from_str(&serialized).unwrap(); + assert_eq!(deserialized, kind); + } +} diff --git a/crates/nvisy-core/src/fs/content_metadata.rs b/crates/nvisy-core/src/fs/content_metadata.rs new file mode 100644 index 0000000..11cb976 --- /dev/null +++ b/crates/nvisy-core/src/fs/content_metadata.rs @@ -0,0 +1,189 @@ +//! Content metadata for filesystem operations +//! +//! This module provides the [`ContentMetadata`] struct for handling metadata +//! about content files, including paths and source tracking. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::path::ContentSource; + +/// Metadata associated with content files +/// +/// This struct stores metadata about content including its source identifier +/// and file path. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ContentMetadata { + /// Unique identifier for the content source + pub content_source: ContentSource, + /// Optional path to the source file + pub source_path: Option, + /// Arbitrary key-value metadata associated with this content. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option>, +} + +impl ContentMetadata { + /// Create new content metadata with just a source + /// + /// # Example + /// + /// ``` + /// use nvisy_core::{fs::ContentMetadata, path::ContentSource}; + /// + /// let source = ContentSource::new(); + /// let metadata = ContentMetadata::new(source); + /// ``` + #[must_use] + pub fn new(content_source: ContentSource) -> Self { + Self { + content_source, + source_path: None, + metadata: None, + } + } + + /// Create content metadata with a file path + /// + /// # Example + /// + /// ``` + /// use nvisy_core::{fs::ContentMetadata, path::ContentSource}; + /// use std::path::PathBuf; + /// + /// let source = ContentSource::new(); + /// let metadata = ContentMetadata::with_path(source, PathBuf::from("document.pdf")); + /// assert_eq!(metadata.file_extension(), Some("pdf")); + /// ``` + pub fn with_path(content_source: ContentSource, path: impl Into) -> Self { + Self { + content_source, + source_path: Some(path.into()), + metadata: None, + } + } + + /// Get the file extension if available + #[must_use] + pub fn file_extension(&self) -> Option<&str> { + self.source_path + .as_ref() + .and_then(|path| path.extension()) + .and_then(|ext| ext.to_str()) + } + + /// Get the filename if available + #[must_use] + pub fn filename(&self) -> Option<&str> { + self.source_path + .as_ref() + .and_then(|path| path.file_name()) + .and_then(|name| name.to_str()) + } + + /// Get the parent directory if available + #[must_use] + pub fn parent_directory(&self) -> Option<&Path> { + self.source_path.as_ref().and_then(|path| path.parent()) + } + + /// Get the full path if available + #[must_use] + pub fn path(&self) -> Option<&Path> { + self.source_path.as_deref() + } + + /// Set the source path + pub fn set_path(&mut self, path: impl Into) { + self.source_path = Some(path.into()); + } + + /// Remove the source path + pub fn clear_path(&mut self) { + self.source_path = None; + } + + /// Check if this metadata has a path + #[must_use] + pub fn has_path(&self) -> bool { + self.source_path.is_some() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_content_metadata_creation() { + let source = ContentSource::new(); + let metadata = ContentMetadata::new(source); + + assert_eq!(metadata.content_source, source); + assert!(metadata.source_path.is_none()); + assert!(!metadata.has_path()); + } + + #[test] + fn test_content_metadata_with_path() { + let source = ContentSource::new(); + let path = PathBuf::from("/path/to/document.pdf"); + let metadata = ContentMetadata::with_path(source, path.clone()); + + assert_eq!(metadata.content_source, source); + assert_eq!(metadata.source_path, Some(path)); + assert!(metadata.has_path()); + } + + #[test] + fn test_file_extension_detection() { + let source = ContentSource::new(); + let metadata = ContentMetadata::with_path(source, PathBuf::from("document.pdf")); + + assert_eq!(metadata.file_extension(), Some("pdf")); + } + + #[test] + fn test_metadata_filename() { + let source = ContentSource::new(); + let metadata = ContentMetadata::with_path(source, PathBuf::from("/path/to/file.txt")); + + assert_eq!(metadata.filename(), Some("file.txt")); + } + + #[test] + fn test_metadata_parent_directory() { + let source = ContentSource::new(); + let metadata = ContentMetadata::with_path(source, PathBuf::from("/path/to/file.txt")); + + assert_eq!(metadata.parent_directory(), Some(Path::new("/path/to"))); + } + + #[test] + fn test_path_operations() { + let source = ContentSource::new(); + let mut metadata = ContentMetadata::new(source); + + assert!(!metadata.has_path()); + + metadata.set_path("test.txt"); + assert!(metadata.has_path()); + assert_eq!(metadata.filename(), Some("test.txt")); + + metadata.clear_path(); + assert!(!metadata.has_path()); + assert_eq!(metadata.filename(), None); + } + + #[test] + fn test_serde_serialization() { + let source = ContentSource::new(); + let metadata = ContentMetadata::with_path(source, PathBuf::from("test.json")); + + let serialized = serde_json::to_string(&metadata).unwrap(); + let deserialized: ContentMetadata = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(metadata, deserialized); + } +} diff --git a/crates/nvisy-core/src/fs/content_registry.rs b/crates/nvisy-core/src/fs/content_registry.rs new file mode 100644 index 0000000..6a190a5 --- /dev/null +++ b/crates/nvisy-core/src/fs/content_registry.rs @@ -0,0 +1,108 @@ +use std::path::{Path, PathBuf}; + +use crate::error::{Error, ErrorKind, Result}; +use crate::fs::ContentHandler; +use crate::io::Content; + +/// Registry that accepts content, creates temporary directories, and returns +/// handlers that manage the directory lifecycle. +/// +/// Each call to [`register`](ContentRegistry::register) creates a subdirectory +/// under the base path, named by the content's [`ContentSource`](crate::path::ContentSource) +/// UUID. The directory is automatically cleaned up when the last +/// [`ContentHandler`] referencing it is dropped. +#[derive(Debug, Clone)] +pub struct ContentRegistry { + base_dir: PathBuf, +} + +impl ContentRegistry { + /// Creates a new content registry with the specified base directory. + /// + /// The directory does not need to exist yet — it is created lazily + /// when content is first registered. + pub fn new(base_dir: impl Into) -> Self { + Self { + base_dir: base_dir.into(), + } + } + + /// Registers content and creates a managed temporary directory for it. + /// + /// Creates a subdirectory named by the content's `ContentSource` UUID, + /// writes the content data as `content.bin`, and returns a handler that + /// deletes the directory when the last reference is dropped. + pub async fn register(&self, content: Content) -> Result { + let content_source = content.content_source(); + let dir = self.base_dir.join(content_source.to_string()); + + tokio::fs::create_dir_all(&dir).await.map_err(|err| { + Error::new(ErrorKind::InternalError, format!( + "Failed to create temporary content directory (path: {})", dir.display() + )).with_source(err) + })?; + + let data_path = dir.join("content.bin"); + tokio::fs::write(&data_path, content.as_bytes()) + .await + .map_err(|err| { + Error::new(ErrorKind::InternalError, format!( + "Failed to write content data (path: {})", data_path.display() + )).with_source(err) + })?; + + let runtime_handle = tokio::runtime::Handle::current(); + + Ok(ContentHandler::new(content_source, dir, runtime_handle)) + } + + /// Returns the base directory path. + pub fn base_dir(&self) -> &Path { + &self.base_dir + } +} + +#[cfg(test)] +mod tests { + use crate::io::{Content, ContentData}; + + use super::*; + + #[tokio::test] + async fn test_register_creates_directory() { + let temp = tempfile::TempDir::new().unwrap(); + let registry = ContentRegistry::new(temp.path().join("content")); + let content = Content::new(ContentData::from("Hello, world!")); + let handler = registry.register(content).await.unwrap(); + + assert!(handler.dir().exists()); + assert!(handler.dir().join("content.bin").exists()); + } + + #[tokio::test] + async fn test_base_dir() { + let temp = tempfile::TempDir::new().unwrap(); + let base = temp.path().join("content"); + let registry = ContentRegistry::new(&base); + assert_eq!(registry.base_dir(), base); + } + + #[tokio::test] + async fn test_register_multiple() { + let temp = tempfile::TempDir::new().unwrap(); + let registry = ContentRegistry::new(temp.path().join("content")); + + let h1 = registry + .register(Content::new(ContentData::from("first"))) + .await + .unwrap(); + let h2 = registry + .register(Content::new(ContentData::from("second"))) + .await + .unwrap(); + + assert_ne!(h1.dir(), h2.dir()); + assert!(h1.dir().exists()); + assert!(h2.dir().exists()); + } +} diff --git a/crates/nvisy-core/src/fs/mod.rs b/crates/nvisy-core/src/fs/mod.rs new file mode 100644 index 0000000..920670e --- /dev/null +++ b/crates/nvisy-core/src/fs/mod.rs @@ -0,0 +1,42 @@ +//! Filesystem module for content file operations +//! +//! This module provides filesystem-specific functionality for working with +//! content files, including file metadata handling and archive operations. +//! +//! # Core Types +//! +//! - [`ContentFile`]: A file wrapper that combines filesystem operations with content tracking +//! - [`ContentMetadata`]: Metadata information for content files +//! - [`ContentKind`]: Classification of content types by file extension +//! +//! # Example +//! +//! ```no_run +//! use nvisy_core::fs::ContentFile; +//! use nvisy_core::io::ContentData; +//! +//! async fn example() -> Result<(), Box> { +//! // Create a new file +//! let mut content_file = ContentFile::create("example.txt").await?; +//! +//! // Write some content +//! let content_data = ContentData::from("Hello, world!"); +//! let metadata = content_file.write_from_content_data(content_data).await?; +//! +//! println!("Written to: {:?}", metadata.source_path); +//! Ok(()) +//! } +//! ``` + +mod content_file; +mod content_handler; +mod content_kind; +mod content_metadata; +mod content_registry; + +// Re-export main types +pub use content_file::ContentFile; +pub use content_handler::ContentHandler; +pub use content_kind::ContentKind; +pub use content_metadata::ContentMetadata; +pub use content_registry::ContentRegistry; diff --git a/crates/nvisy-core/src/io/content.rs b/crates/nvisy-core/src/io/content.rs new file mode 100644 index 0000000..ed6f3cb --- /dev/null +++ b/crates/nvisy-core/src/io/content.rs @@ -0,0 +1,234 @@ +//! Content representation combining data with metadata +//! +//! This module provides the [`Content`] struct that combines [`ContentData`] +//! with optional [`ContentMetadata`] for complete content representation. + +use derive_more::{AsRef, Deref}; +use serde::{Deserialize, Serialize}; + +use super::ContentData; +use crate::error::Result; +use crate::fs::ContentMetadata; +use crate::path::ContentSource; + +/// Complete content representation with data and metadata +/// +/// This struct combines [`ContentData`] (the actual content bytes) with +/// optional [`ContentMetadata`] (path, extension info, etc.) to provide +/// a complete content representation. +/// +/// # Examples +/// +/// ```rust +/// use nvisy_core::io::{Content, ContentData}; +/// use nvisy_core::fs::ContentMetadata; +/// use nvisy_core::path::ContentSource; +/// +/// // Create content from data +/// let data = ContentData::from("Hello, world!"); +/// let content = Content::new(data); +/// +/// assert_eq!(content.size(), 13); +/// assert!(content.is_likely_text()); +/// +/// // Create content with metadata +/// let source = ContentSource::new(); +/// let data = ContentData::from_text(source, "Sample text"); +/// let metadata = ContentMetadata::with_path(source, "document.txt"); +/// let content = Content::with_metadata(data, metadata); +/// +/// assert_eq!(content.metadata().and_then(|m| m.filename()), Some("document.txt")); +/// ``` +#[derive(Debug, Clone, PartialEq)] +#[derive(AsRef, Deref, Serialize, Deserialize)] +pub struct Content { + /// The actual content data + #[deref] + #[as_ref] + data: ContentData, + /// Optional metadata about the content + metadata: Option, +} + +impl From for Content { + fn from(data: ContentData) -> Self { + Self::new(data) + } +} + +impl Content { + /// Create new content from data without metadata + pub fn new(data: ContentData) -> Self { + Self { + data, + metadata: None, + } + } + + /// Create new content with metadata + pub fn with_metadata(data: ContentData, metadata: ContentMetadata) -> Self { + Self { + data, + metadata: Some(metadata), + } + } + + /// Get the content data + pub fn data(&self) -> &ContentData { + &self.data + } + + /// Get the content metadata if available + pub fn metadata(&self) -> Option<&ContentMetadata> { + self.metadata.as_ref() + } + + /// Get the content source + pub fn content_source(&self) -> ContentSource { + self.data.content_source + } + + /// Get the content as bytes + pub fn as_bytes(&self) -> &[u8] { + self.data.as_bytes() + } + + /// Returns `true` if the content appears to be text. + pub fn is_likely_text(&self) -> bool { + self.data.is_likely_text() + } + + /// Try to get the content as a string slice. + /// + /// # Errors + /// + /// Returns an error if the content is not valid UTF-8. + pub fn as_str(&self) -> Result<&str> { + self.data.as_str() + } + + /// Get the file extension from metadata if available + pub fn file_extension(&self) -> Option<&str> { + self.metadata.as_ref().and_then(|m| m.file_extension()) + } + + /// Get the filename from metadata if available + pub fn filename(&self) -> Option<&str> { + self.metadata.as_ref().and_then(|m| m.filename()) + } + + /// Set the metadata + pub fn set_metadata(&mut self, metadata: ContentMetadata) { + self.metadata = Some(metadata); + } + + /// Remove the metadata + pub fn clear_metadata(&mut self) { + self.metadata = None; + } + + /// Consume and return the inner [`ContentData`]. + pub fn into_data(self) -> ContentData { + self.data + } + + /// Consume and return both data and metadata + pub fn into_parts(self) -> (ContentData, Option) { + (self.data, self.metadata) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_content_creation() { + let data = ContentData::from("Hello, world!"); + let content = Content::new(data.clone()); + + assert_eq!(content.size(), 13); + assert!(content.is_likely_text()); + assert!(content.metadata().is_none()); + } + + #[test] + fn test_content_with_metadata() { + let source = ContentSource::new(); + let data = ContentData::from_text(source, "Test content"); + let metadata = ContentMetadata::with_path(source, "test.txt"); + let content = Content::with_metadata(data, metadata); + + assert!(content.metadata().is_some()); + assert_eq!(content.file_extension(), Some("txt")); + assert_eq!(content.filename(), Some("test.txt")); + } + + #[test] + fn test_content_deref() { + let data = ContentData::from("Hello"); + let content = Content::new(data); + + // Test that Deref works - we can call ContentData methods directly + assert_eq!(content.size(), 5); + assert_eq!(content.as_str().unwrap(), "Hello"); + } + + #[test] + fn test_content_from() { + let data = ContentData::from("Test"); + let content: Content = data.into(); + + assert_eq!(content.size(), 4); + } + + #[test] + fn test_metadata_operations() { + let data = ContentData::from("Test"); + let mut content = Content::new(data); + + assert!(content.metadata().is_none()); + + let source = content.content_source(); + let metadata = ContentMetadata::with_path(source, "file.pdf"); + content.set_metadata(metadata); + + assert!(content.metadata().is_some()); + assert_eq!(content.file_extension(), Some("pdf")); + + content.clear_metadata(); + assert!(content.metadata().is_none()); + } + + #[test] + fn test_into_parts() { + let source = ContentSource::new(); + let data = ContentData::from_text(source, "Test"); + let metadata = ContentMetadata::with_path(source, "test.txt"); + let content = Content::with_metadata(data.clone(), metadata.clone()); + + let (recovered_data, recovered_metadata) = content.into_parts(); + assert_eq!(recovered_data, data); + assert_eq!(recovered_metadata, Some(metadata)); + } + + #[test] + fn test_serialization() { + let data = ContentData::from("Test content"); + let content = Content::new(data); + + let json = serde_json::to_string(&content).unwrap(); + let deserialized: Content = serde_json::from_str(&json).unwrap(); + + assert_eq!(content, deserialized); + } + + #[test] + fn test_content_source() { + let source = ContentSource::new(); + let data = ContentData::from_text(source, "Test"); + let content = Content::new(data); + + assert_eq!(content.content_source(), source); + } +} diff --git a/crates/nvisy-core/src/io/content_data.rs b/crates/nvisy-core/src/io/content_data.rs new file mode 100644 index 0000000..e1adb0c --- /dev/null +++ b/crates/nvisy-core/src/io/content_data.rs @@ -0,0 +1,686 @@ +//! Content data structure for storing and managing content with metadata +//! +//! This module provides the [`ContentData`] struct for storing content data +//! along with its metadata and source information. + +use std::fmt; +use std::ops::Deref; +use std::sync::OnceLock; + +use bytes::Bytes; +use hipstr::HipStr; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::error::{Error, ErrorKind, Result}; +use crate::path::ContentSource; + +/// A wrapper around `Bytes` for content storage. +/// +/// This struct wraps `bytes::Bytes` and provides additional methods +/// for text conversion. It's cheap to clone as `Bytes` uses reference +/// counting internally. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[derive(Serialize, Deserialize)] +#[serde(transparent)] +pub struct ContentBytes(Bytes); + +impl ContentBytes { + /// Creates a new `ContentBytes` from raw bytes. + #[must_use] + pub fn new(bytes: Bytes) -> Self { + Self(bytes) + } + + /// Returns the size of the content in bytes. + #[must_use] + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns `true` if the content is empty. + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Returns the content as a byte slice. + #[must_use] + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } + + /// Tries to return the content as a string slice. + /// + /// Returns `None` if the content is not valid UTF-8. + #[must_use] + pub fn as_str(&self) -> Option<&str> { + std::str::from_utf8(&self.0).ok() + } + + /// Converts to a `HipStr` if the content is valid UTF-8. + /// + /// # Errors + /// + /// Returns an error if the content is not valid UTF-8. + pub fn as_hipstr(&self) -> Result> { + let s = std::str::from_utf8(&self.0).map_err(|e| { + Error::new(ErrorKind::Serialization, format!("Invalid UTF-8: {e}")) + })?; + Ok(HipStr::from(s)) + } + + /// Returns the underlying `Bytes`. + #[must_use] + pub fn to_bytes(&self) -> Bytes { + self.0.clone() + } + + /// Consumes and returns the underlying `Bytes`. + #[must_use] + pub fn into_bytes(self) -> Bytes { + self.0 + } + + /// Returns `true` if the content appears to be text. + /// + /// Uses a simple heuristic: checks if all bytes are ASCII printable + /// or whitespace characters. + #[must_use] + pub fn is_likely_text(&self) -> bool { + self.0 + .iter() + .all(|&b| b.is_ascii_graphic() || b.is_ascii_whitespace()) + } +} + +impl Deref for ContentBytes { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl AsRef<[u8]> for ContentBytes { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl From<&str> for ContentBytes { + fn from(s: &str) -> Self { + Self(Bytes::copy_from_slice(s.as_bytes())) + } +} + +impl From for ContentBytes { + fn from(s: String) -> Self { + Self(Bytes::from(s)) + } +} + +impl From> for ContentBytes { + fn from(s: HipStr<'static>) -> Self { + Self(Bytes::copy_from_slice(s.as_bytes())) + } +} + +impl From<&[u8]> for ContentBytes { + fn from(bytes: &[u8]) -> Self { + Self(Bytes::copy_from_slice(bytes)) + } +} + +impl From> for ContentBytes { + fn from(vec: Vec) -> Self { + Self(Bytes::from(vec)) + } +} + +impl From for ContentBytes { + fn from(bytes: Bytes) -> Self { + Self(bytes) + } +} + +/// Content data with metadata and computed hashes. +/// +/// This struct wraps [`ContentBytes`] and stores content data along with +/// metadata about its source and optional computed SHA256 hash. +/// It's designed to be cheap to clone using reference-counted types. +/// The SHA256 hash is lazily computed using `OnceLock` for lock-free +/// access after initialization. +#[derive(Debug, Serialize, Deserialize)] +pub struct ContentData { + /// Unique identifier for the content source. + pub content_source: ContentSource, + /// The actual content data. + data: ContentBytes, + /// Lazily computed SHA256 hash of the content. + #[serde(skip)] + sha256_cache: OnceLock, + /// Caller-supplied MIME type (e.g. from HTTP Content-Type header). + #[serde(skip_serializing_if = "Option::is_none")] + pub mime: Option, + /// MIME type detected from magic bytes. + #[serde(skip_serializing_if = "Option::is_none")] + pub detected_mime: Option, +} + +impl ContentData { + /// Creates new content data from bytes. + /// + /// # Example + /// + /// ``` + /// use nvisy_core::{io::ContentData, path::ContentSource}; + /// use bytes::Bytes; + /// + /// let source = ContentSource::new(); + /// let data = Bytes::from("Hello, world!"); + /// let content = ContentData::new(source, data); + /// + /// assert_eq!(content.size(), 13); + /// ``` + pub fn new(content_source: ContentSource, data: Bytes) -> Self { + Self { + content_source, + data: ContentBytes::new(data), + sha256_cache: OnceLock::new(), + mime: None, + detected_mime: None, + } + } + + /// Creates new content data from text. + /// + /// # Example + /// + /// ``` + /// use nvisy_core::{io::ContentData, path::ContentSource}; + /// + /// let source = ContentSource::new(); + /// let content = ContentData::from_text(source, "Hello, world!"); + /// + /// assert_eq!(content.as_str().unwrap(), "Hello, world!"); + /// ``` + pub fn from_text(content_source: ContentSource, text: impl Into) -> Self { + Self { + content_source, + data: ContentBytes::from(text.into()), + sha256_cache: OnceLock::new(), + mime: None, + detected_mime: None, + } + } + + /// Creates content data with explicit `ContentBytes`. + pub fn with_content_bytes(content_source: ContentSource, data: ContentBytes) -> Self { + Self { + content_source, + data, + sha256_cache: OnceLock::new(), + mime: None, + detected_mime: None, + } + } + + /// Set the caller-provided MIME type (builder pattern). + #[must_use] + pub fn with_content_type(mut self, mime: impl Into) -> Self { + self.mime = Some(mime.into()); + self + } + + /// Get the best-available MIME type (provided takes precedence over detected). + #[must_use] + pub fn content_type(&self) -> Option<&str> { + self.mime.as_deref().or(self.detected_mime.as_deref()) + } + + /// Detect the MIME type from magic bytes and store it. + pub fn detect_mime(&mut self) { + self.detected_mime = infer::get(self.data.as_bytes()).map(|t| t.mime_type().to_string()); + } + + /// Returns the size of the content in bytes. + #[must_use] + pub fn size(&self) -> usize { + self.data.len() + } + + /// Returns a pretty formatted size string. + #[allow(clippy::cast_precision_loss)] + #[must_use] + pub fn get_pretty_size(&self) -> String { + let bytes = self.size(); + match bytes { + 0..=1023 => format!("{bytes} B"), + 1024..=1_048_575 => format!("{:.1} KB", bytes as f64 / 1024.0), + 1_048_576..=1_073_741_823 => format!("{:.1} MB", bytes as f64 / 1_048_576.0), + _ => format!("{:.1} GB", bytes as f64 / 1_073_741_824.0), + } + } + + /// Returns the content data as a byte slice. + #[must_use] + pub fn as_bytes(&self) -> &[u8] { + self.data.as_bytes() + } + + /// Returns a reference to the underlying `ContentBytes`. + #[must_use] + pub fn content_bytes(&self) -> &ContentBytes { + &self.data + } + + /// Converts the content data to `Bytes`. + #[must_use] + pub fn to_bytes(&self) -> Bytes { + self.data.to_bytes() + } + + /// Consumes and converts into `Bytes`. + #[must_use] + pub fn into_bytes(self) -> Bytes { + self.data.into_bytes() + } + + /// Returns `true` if the content appears to be text. + /// + /// Uses a simple heuristic: checks if all bytes are ASCII printable + /// or whitespace characters. + #[must_use] + pub fn is_likely_text(&self) -> bool { + self.data.is_likely_text() + } + + /// Tries to convert the content data to a UTF-8 string. + /// + /// # Errors + /// + /// Returns an error if the content data contains invalid UTF-8 sequences. + pub fn as_string(&self) -> Result { + self.data.as_hipstr().map(|s| s.to_string()) + } + + /// Tries to convert the content data to a UTF-8 string slice. + /// + /// # Errors + /// + /// Returns an error if the content data contains invalid UTF-8 sequences. + pub fn as_str(&self) -> Result<&str> { + std::str::from_utf8(self.data.as_bytes()).map_err(|e| { + Error::new(ErrorKind::Serialization, format!("Invalid UTF-8: {e}")) + }) + } + + /// Converts to a `HipStr` if the content is valid UTF-8. + /// + /// # Errors + /// + /// Returns an error if the content is not valid UTF-8. + pub fn as_hipstr(&self) -> Result> { + self.data.as_hipstr() + } + + /// Computes SHA256 hash of the content. + fn compute_sha256_internal(&self) -> Bytes { + let mut hasher = Sha256::new(); + hasher.update(self.data.as_bytes()); + Bytes::from(hasher.finalize().to_vec()) + } + + /// Returns the SHA256 hash, computing it if not already done. + #[must_use] + pub fn sha256(&self) -> &Bytes { + self.sha256_cache + .get_or_init(|| self.compute_sha256_internal()) + } + + /// Returns the SHA256 hash as a hex string. + #[must_use] + pub fn sha256_hex(&self) -> String { + hex::encode(self.sha256()) + } + + /// Verifies the content against a provided SHA256 hash. + /// + /// # Errors + /// + /// Returns an error if the computed hash does not match the expected hash. + pub fn verify_sha256(&self, expected_hash: impl AsRef<[u8]>) -> Result<()> { + let actual_hash = self.sha256(); + let expected = expected_hash.as_ref(); + + if actual_hash.as_ref() == expected { + Ok(()) + } else { + Err(Error::new(ErrorKind::InvalidInput, format!( + "Hash mismatch: expected {}, got {}", + hex::encode(expected), + hex::encode(actual_hash) + ))) + } + } + + /// Returns a slice of the content data. + /// + /// # Errors + /// + /// Returns an error if the end index is beyond the content length + /// or if start is greater than end. + pub fn slice(&self, start: usize, end: usize) -> Result { + let bytes = self.data.as_bytes(); + if end > bytes.len() { + return Err(Error::new(ErrorKind::InvalidInput, format!( + "Slice end {} exceeds content length {}", + end, + bytes.len() + ))); + } + if start > end { + return Err(Error::new(ErrorKind::InvalidInput, + format!("Slice start {start} is greater than end {end}"))); + } + Ok(Bytes::copy_from_slice(&bytes[start..end])) + } + + /// Returns `true` if the content is empty. + #[must_use] + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } +} + +impl Clone for ContentData { + fn clone(&self) -> Self { + let new_lock = OnceLock::new(); + // Copy the computed hash if available + if let Some(hash) = self.sha256_cache.get() { + let _ = new_lock.set(hash.clone()); + } + + Self { + content_source: self.content_source, + data: self.data.clone(), + sha256_cache: new_lock, + mime: self.mime.clone(), + detected_mime: self.detected_mime.clone(), + } + } +} + +impl PartialEq for ContentData { + fn eq(&self, other: &Self) -> bool { + self.content_source == other.content_source + && self.data == other.data + && self.mime == other.mime + && self.detected_mime == other.detected_mime + } +} + +impl Eq for ContentData {} + +impl From<&str> for ContentData { + fn from(s: &str) -> Self { + let source = ContentSource::new(); + Self::from_text(source, s) + } +} + +impl From for ContentData { + fn from(s: String) -> Self { + let source = ContentSource::new(); + Self::from_text(source, s) + } +} + +impl From<&[u8]> for ContentData { + fn from(bytes: &[u8]) -> Self { + let source = ContentSource::new(); + Self::new(source, Bytes::copy_from_slice(bytes)) + } +} + +impl From> for ContentData { + fn from(vec: Vec) -> Self { + let source = ContentSource::new(); + Self::new(source, Bytes::from(vec)) + } +} + +impl From for ContentData { + fn from(bytes: Bytes) -> Self { + let source = ContentSource::new(); + Self::new(source, bytes) + } +} + +impl From> for ContentData { + fn from(text: HipStr<'static>) -> Self { + let source = ContentSource::new(); + Self::from_text(source, text.to_string()) + } +} + +impl fmt::Display for ContentData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Ok(text) = self.as_str() { + write!(f, "{text}") + } else { + write!(f, "[Binary data: {} bytes]", self.size()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_content_data_creation() { + let source = ContentSource::new(); + let data = Bytes::from("Hello, world!"); + let content = ContentData::new(source, data); + + assert_eq!(content.content_source, source); + assert_eq!(content.size(), 13); + assert!(content.sha256_cache.get().is_none()); + } + + #[test] + fn test_content_data_from_text() { + let source = ContentSource::new(); + let content = ContentData::from_text(source, "Hello, world!"); + + assert_eq!(content.as_str().unwrap(), "Hello, world!"); + } + + #[test] + fn test_content_bytes_wrapper() { + let bytes = ContentBytes::from("Hello"); + assert_eq!(bytes.as_str(), Some("Hello")); + assert_eq!(bytes.len(), 5); + assert!(!bytes.is_empty()); + } + + #[test] + fn test_content_bytes_as_hipstr() { + let bytes = ContentBytes::from("Hello, HipStr!"); + let hipstr = bytes.as_hipstr().unwrap(); + assert_eq!(hipstr.as_str(), "Hello, HipStr!"); + + // Test with invalid UTF-8 + let invalid = ContentBytes::from(vec![0xFF, 0xFE]); + assert!(invalid.as_hipstr().is_err()); + } + + #[test] + fn test_content_bytes_binary() { + let binary = ContentBytes::from(vec![0xFF, 0xFE]); + assert_eq!(binary.len(), 2); + assert!(binary.as_str().is_none()); + assert!(!binary.is_likely_text()); + } + + #[test] + fn test_size_methods() { + let content = ContentData::from("Hello"); + assert_eq!(content.size(), 5); + + let pretty_size = content.get_pretty_size(); + assert!(!pretty_size.is_empty()); + } + + #[test] + fn test_sha256_computation() { + let content = ContentData::from("Hello, world!"); + let hash = content.sha256(); + + assert!(content.sha256_cache.get().is_some()); + assert_eq!(hash.len(), 32); + + let hash2 = content.sha256(); + assert_eq!(hash, hash2); + } + + #[test] + fn test_sha256_verification() { + let content = ContentData::from("Hello, world!"); + let hash = content.sha256().clone(); + + assert!(content.verify_sha256(&hash).is_ok()); + + let wrong_hash = vec![0u8; 32]; + assert!(content.verify_sha256(&wrong_hash).is_err()); + } + + #[test] + fn test_string_conversion() { + let content = ContentData::from("Hello, world!"); + assert_eq!(content.as_string().unwrap(), "Hello, world!"); + assert_eq!(content.as_str().unwrap(), "Hello, world!"); + + let binary_content = ContentData::from(vec![0xFF, 0xFE, 0xFD]); + assert!(binary_content.as_string().is_err()); + assert!(binary_content.as_str().is_err()); + } + + #[test] + fn test_as_hipstr() { + let content = ContentData::from("Hello, HipStr!"); + let hipstr = content.as_hipstr().unwrap(); + assert_eq!(hipstr.as_str(), "Hello, HipStr!"); + + let binary_content = ContentData::from(vec![0xFF, 0xFE]); + assert!(binary_content.as_hipstr().is_err()); + } + + #[test] + fn test_is_likely_text() { + let text_content = ContentData::from("Hello, world!"); + assert!(text_content.is_likely_text()); + + let binary_content = ContentData::from(vec![0xFF, 0xFE, 0xFD]); + assert!(!binary_content.is_likely_text()); + } + + #[test] + fn test_slice() { + let content = ContentData::from("Hello, world!"); + + let slice = content.slice(0, 5).unwrap(); + assert_eq!(slice, Bytes::from("Hello")); + + let slice = content.slice(7, 12).unwrap(); + assert_eq!(slice, Bytes::from("world")); + + assert!(content.slice(0, 100).is_err()); + assert!(content.slice(10, 5).is_err()); + } + + #[test] + fn test_from_conversions() { + let from_str = ContentData::from("test"); + let from_string = ContentData::from("test".to_string()); + let from_bytes = ContentData::from(b"test".as_slice()); + let from_vec = ContentData::from(b"test".to_vec()); + let from_bytes_type = ContentData::from(Bytes::from("test")); + + assert_eq!(from_str.as_str().unwrap(), "test"); + assert_eq!(from_string.as_str().unwrap(), "test"); + assert_eq!(from_bytes.as_str().unwrap(), "test"); + assert_eq!(from_vec.as_str().unwrap(), "test"); + assert_eq!(from_bytes_type.as_str().unwrap(), "test"); + } + + #[test] + fn test_display() { + let text_content = ContentData::from("Hello"); + assert_eq!(format!("{text_content}"), "Hello"); + + let binary_content = ContentData::from(vec![0xFF, 0xFE]); + assert!(format!("{binary_content}").contains("Binary data")); + } + + #[test] + fn test_cloning_preserves_hash() { + let original = ContentData::from("Hello, world!"); + let _ = original.sha256(); + + let cloned = original.clone(); + + assert!(original.sha256_cache.get().is_some()); + assert!(cloned.sha256_cache.get().is_some()); + assert_eq!(original.sha256(), cloned.sha256()); + } + + #[test] + fn test_cloning_is_cheap() { + let original = ContentData::from("Hello, world!"); + let cloned = original.clone(); + + assert_eq!(original, cloned); + } + + #[test] + fn test_into_bytes() { + let content = ContentData::from("Hello, world!"); + let bytes = content.into_bytes(); + assert_eq!(bytes, Bytes::from("Hello, world!")); + } + + #[test] + fn test_empty_content() { + let content = ContentData::from(""); + assert!(content.is_empty()); + assert_eq!(content.size(), 0); + } + + #[test] + fn test_to_bytes() { + let text_content = ContentData::from_text(ContentSource::new(), "Hello"); + let bytes = text_content.to_bytes(); + assert_eq!(bytes.as_ref(), b"Hello"); + + let binary_content = ContentData::new(ContentSource::new(), Bytes::from("World")); + let bytes = binary_content.to_bytes(); + assert_eq!(bytes.as_ref(), b"World"); + } + + #[test] + fn test_from_hipstr() { + let hipstr = HipStr::from("Hello from HipStr"); + let content = ContentData::from(hipstr); + assert_eq!(content.as_str().unwrap(), "Hello from HipStr"); + } + + #[test] + fn test_content_bytes_deref() { + let bytes = ContentBytes::from("Hello"); + assert_eq!(&*bytes, b"Hello"); + assert_eq!(bytes.as_ref(), b"Hello"); + } +} diff --git a/crates/nvisy-core/src/io/content_read.rs b/crates/nvisy-core/src/io/content_read.rs new file mode 100644 index 0000000..f889aea --- /dev/null +++ b/crates/nvisy-core/src/io/content_read.rs @@ -0,0 +1,372 @@ +//! Content reading trait for async I/O operations +//! +//! This module provides the [`AsyncContentRead`] trait for reading content data +//! from various async sources into [`ContentData`] structures. + +use std::future::Future; +use std::io; + +use bytes::Bytes; +use tokio::io::{AsyncRead, AsyncReadExt}; + +use super::ContentData; +use crate::path::ContentSource; + +/// Trait for reading content from async sources +/// +/// This trait provides methods for reading content data from async sources +/// and converting them into [`ContentData`] structures with various options +/// for size limits, and verification. +pub trait AsyncContentRead: AsyncRead + Unpin + Send { + /// Read all content from the source into a `ContentData` structure + /// + /// # Errors + /// + /// Returns an error if the read operation fails or if there are I/O issues. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::io::{AsyncContentRead, ContentData}; + /// use tokio::fs::File; + /// use std::io; + /// + /// async fn read_file() -> io::Result { + /// let mut file = File::open("example.txt").await?; + /// file.read_content().await + /// } + /// ``` + fn read_content(&mut self) -> impl Future> + Send + where + Self: Sized, + { + async move { + let mut buffer = Vec::new(); + self.read_to_end(&mut buffer).await?; + + let content_data = ContentData::new(ContentSource::new(), Bytes::from(buffer)); + Ok(content_data) + } + } + + /// Read content with a specified content source + /// + /// # Errors + /// + /// Returns an error if the read operation fails or if there are I/O issues. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::{io::{AsyncContentRead, ContentData}, path::ContentSource}; + /// use tokio::fs::File; + /// use std::io; + /// + /// async fn read_with_source() -> io::Result { + /// let mut file = File::open("example.txt").await?; + /// let source = ContentSource::new(); + /// file.read_content_with_source(source).await + /// } + /// ``` + fn read_content_with_source( + &mut self, + source: ContentSource, + ) -> impl Future> + Send + where + Self: Sized, + { + async move { + let mut buffer = Vec::new(); + self.read_to_end(&mut buffer).await?; + + let content_data = ContentData::new(source, Bytes::from(buffer)); + Ok(content_data) + } + } + + /// Read content up to a maximum size limit + /// + /// This method prevents reading extremely large files that could cause + /// memory issues. + /// + /// # Errors + /// + /// Returns an error if the read operation fails, if there are I/O issues, + /// or if the content exceeds the maximum size limit. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::io::{AsyncContentRead, ContentData}; + /// use tokio::fs::File; + /// use std::io; + /// + /// async fn read_limited_content() -> io::Result { + /// let mut file = File::open("example.txt").await?; + /// // Limit to 1MB + /// file.read_content_limited(1024 * 1024).await + /// } + /// ``` + fn read_content_limited( + &mut self, + max_size: usize, + ) -> impl Future> + Send + where + Self: Sized, + { + async move { + let mut buffer = Vec::with_capacity(std::cmp::min(max_size, 8192)); + let mut total_read = 0; + + loop { + let mut temp_buf = vec![0u8; 8192]; + let bytes_read = self.read(&mut temp_buf).await?; + + if bytes_read == 0 { + break; // EOF reached + } + + if total_read + bytes_read > max_size { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Content size exceeds maximum limit of {max_size} bytes"), + )); + } + + buffer.extend_from_slice(&temp_buf[..bytes_read]); + total_read += bytes_read; + } + + let content_data = ContentData::new(ContentSource::new(), Bytes::from(buffer)); + Ok(content_data) + } + } + + /// Read content in chunks, calling a callback for each chunk + /// + /// This is useful for processing large files without loading them + /// entirely into memory. + /// + /// # Errors + /// + /// Returns an error if the read operation fails or if the callback + /// returns an error. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::io::AsyncContentRead; + /// use tokio::fs::File; + /// use bytes::Bytes; + /// use std::io; + /// + /// async fn process_chunks() -> io::Result<()> { + /// let mut file = File::open("large_file.txt").await?; + /// + /// file.read_content_chunked(8192, |chunk| { + /// println!("Processing chunk of {} bytes", chunk.len()); + /// Ok(()) + /// }).await + /// } + /// ``` + fn read_content_chunked( + &mut self, + chunk_size: usize, + mut callback: impl FnMut(Bytes) -> std::result::Result<(), E> + Send, + ) -> impl Future> + Send + where + Self: Sized, + E: From + Send, + { + async move { + let mut buffer = vec![0u8; chunk_size]; + + loop { + let bytes_read = self.read(&mut buffer).await?; + if bytes_read == 0 { + break; // EOF reached + } + + let chunk = Bytes::copy_from_slice(&buffer[..bytes_read]); + callback(chunk)?; + } + + Ok(()) + } + } + + /// Read content with verification + /// + /// This method reads the content and optionally verifies it meets + /// certain criteria. + /// + /// # Errors + /// + /// Returns an error if the read operation fails, if there are I/O issues, + /// or if verification fails. + fn read_content_verified( + &mut self, + verify_fn: F, + ) -> impl Future> + Send + where + Self: Sized, + F: FnOnce(&[u8]) -> bool + Send, + { + async move { + let mut buffer = Vec::new(); + self.read_to_end(&mut buffer).await?; + + // Verify with a reference to the buffer data + if !verify_fn(&buffer) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Content verification failed", + )); + } + + // Convert to ContentData after verification + let content_data = ContentData::new(ContentSource::new(), Bytes::from(buffer)); + Ok(content_data) + } + } +} + +// Implementations for common types +impl AsyncContentRead for tokio::fs::File {} +impl AsyncContentRead for Box {} + +// Test-specific implementations +#[cfg(test)] +impl + Unpin + Send> AsyncContentRead for std::io::Cursor {} + +#[cfg(test)] +mod tests { + use std::io::{Cursor, Result}; + + use super::*; + + #[tokio::test] + async fn test_read_content() -> Result<()> { + let data = b"Hello, world!"; + let mut cursor = Cursor::new(data); + + let content = cursor.read_content().await.unwrap(); + assert_eq!(content.as_bytes(), data); + assert_eq!(content.size(), data.len()); + + Ok(()) + } + + #[tokio::test] + async fn test_read_content_with_source() -> Result<()> { + let data = b"Hello, world!"; + let mut cursor = Cursor::new(data); + let source = ContentSource::new(); + + let content = cursor.read_content_with_source(source).await.unwrap(); + assert_eq!(content.content_source, source); + assert_eq!(content.as_bytes(), data); + + Ok(()) + } + + #[tokio::test] + async fn test_read_content_limited() -> Result<()> { + let data = b"Hello, world!"; + let mut cursor = Cursor::new(data); + + // Should succeed within limit + let content = cursor.read_content_limited(20).await?; + assert_eq!(content.as_bytes(), data); + + Ok(()) + } + + #[tokio::test] + async fn test_read_content_limited_exceeds() -> Result<()> { + let data = b"Hello, world!"; + let mut cursor = Cursor::new(data); + + // Should fail when exceeding limit + let result = cursor.read_content_limited(5).await; + assert!(result.is_err()); + + Ok(()) + } + + #[tokio::test] + async fn test_read_content_chunked() -> Result<()> { + let data = b"Hello, world!"; + let mut cursor = Cursor::new(data); + + let mut chunks = Vec::new(); + let result = cursor + .read_content_chunked(5, |chunk| { + chunks.push(chunk); + Ok::<(), io::Error>(()) + }) + .await; + + assert!(result.is_ok()); + assert!(!chunks.is_empty()); + + // Concatenate chunks and verify they match original data + let concatenated: Vec = chunks + .into_iter() + .flat_map(|chunk| chunk.to_vec()) + .collect(); + assert_eq!(concatenated, data); + + Ok(()) + } + + #[tokio::test] + async fn test_read_content_verified() -> Result<()> { + let data = b"Hello, world!"; + let mut cursor = Cursor::new(data); + + // Should succeed with passing verification + let content = cursor + .read_content_verified(|data| !data.is_empty()) + .await?; + assert_eq!(content.as_bytes(), data); + + Ok(()) + } + + #[tokio::test] + async fn test_read_content_verified_fails() -> Result<()> { + let data = b"Hello, world!"; + let mut cursor = Cursor::new(data); + + // Should fail with failing verification + let result = cursor.read_content_verified(<[u8]>::is_empty).await; + assert!(result.is_err()); + + Ok(()) + } + + #[tokio::test] + async fn test_read_empty_content() -> Result<()> { + let data = b""; + let mut cursor = Cursor::new(data); + + let content = cursor.read_content().await?; + assert_eq!(content.size(), 0); + assert!(content.is_empty()); + + Ok(()) + } + + #[tokio::test] + async fn test_read_large_content() -> Result<()> { + let data = vec![42u8; 10000]; + let mut cursor = Cursor::new(data.clone()); + + let content = cursor.read_content().await?; + assert_eq!(content.as_bytes(), data.as_slice()); + assert_eq!(content.size(), 10000); + + Ok(()) + } +} diff --git a/crates/nvisy-core/src/io/content_write.rs b/crates/nvisy-core/src/io/content_write.rs new file mode 100644 index 0000000..99e749e --- /dev/null +++ b/crates/nvisy-core/src/io/content_write.rs @@ -0,0 +1,372 @@ +//! Content writing trait for async I/O operations +//! +//! This module provides the [`AsyncContentWrite`] trait for writing content data +//! to various async destinations from [`ContentData`] structures. + +use std::future::Future; +use std::io; + +use tokio::io::{AsyncWrite, AsyncWriteExt}; + +use super::ContentData; +use crate::fs::ContentMetadata; + +/// Trait for writing content to async destinations +/// +/// This trait provides methods for writing content data to async destinations +/// with various options for chunking, and verification. +pub trait AsyncContentWrite: AsyncWrite + Unpin + Send { + /// Write content data to the destination + /// + /// # Errors + /// + /// Returns an error if the write operation fails or if there are I/O issues. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::io::{AsyncContentWrite, ContentData}; + /// use nvisy_core::fs::ContentMetadata; + /// use tokio::fs::File; + /// use std::io; + /// + /// async fn write_file() -> io::Result { + /// let mut file = File::create("output.txt").await?; + /// let content = ContentData::from("Hello, world!"); + /// file.write_content(content).await + /// } + /// ``` + fn write_content( + &mut self, + content_data: ContentData, + ) -> impl Future> + Send + where + Self: Sized, + { + async move { + self.write_all(content_data.as_bytes()).await?; + self.flush().await?; + + let metadata = ContentMetadata::new(content_data.content_source); + Ok(metadata) + } + } + + /// Write content data and return metadata with specified source path + /// + /// # Errors + /// + /// Returns an error if the write operation fails or if there are I/O issues. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::io::{AsyncContentWrite, ContentData}; + /// use nvisy_core::fs::ContentMetadata; + /// use tokio::fs::File; + /// use std::path::PathBuf; + /// use std::io; + /// + /// async fn write_with_path() -> io::Result { + /// let mut file = File::create("output.txt").await?; + /// let content = ContentData::from("Hello, world!"); + /// let path = PathBuf::from("output.txt"); + /// file.write_content_with_path(content, path).await + /// } + /// ``` + fn write_content_with_path( + &mut self, + content_data: ContentData, + path: impl Into + Send, + ) -> impl Future> + Send + where + Self: Sized, + { + async move { + self.write_all(content_data.as_bytes()).await?; + self.flush().await?; + + let metadata = ContentMetadata::with_path(content_data.content_source, path); + Ok(metadata) + } + } + + /// Write content data in chunks for better memory efficiency + /// + /// This method is useful for writing large content without keeping it + /// all in memory at once. + /// + /// # Errors + /// + /// Returns an error if the write operation fails or if there are I/O issues. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::io::{AsyncContentWrite, ContentData}; + /// use nvisy_core::fs::ContentMetadata; + /// use tokio::fs::File; + /// use std::io; + /// + /// async fn write_chunked() -> io::Result { + /// let mut file = File::create("output.txt").await?; + /// let content = ContentData::from(vec![0u8; 1_000_000]); // 1MB + /// file.write_content_chunked(content, 8192).await + /// } + /// ``` + fn write_content_chunked( + &mut self, + content_data: ContentData, + chunk_size: usize, + ) -> impl Future> + Send + where + Self: Sized, + { + async move { + let data = content_data.as_bytes(); + + for chunk in data.chunks(chunk_size) { + self.write_all(chunk).await?; + } + + self.flush().await?; + + let metadata = ContentMetadata::new(content_data.content_source); + Ok(metadata) + } + } + + /// Write multiple content data items sequentially + /// + /// # Errors + /// + /// Returns an error if any write operation fails or if there are I/O issues. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::io::{AsyncContentWrite, ContentData}; + /// use nvisy_core::fs::ContentMetadata; + /// use tokio::fs::File; + /// use std::io; + /// + /// async fn write_multiple() -> io::Result> { + /// let mut file = File::create("output.txt").await?; + /// let contents = vec![ + /// ContentData::from("Hello, "), + /// ContentData::from("world!"), + /// ]; + /// file.write_multiple_content(contents).await + /// } + /// ``` + fn write_multiple_content( + &mut self, + content_data_list: Vec, + ) -> impl Future>> + Send + where + Self: Sized, + { + async move { + let mut metadata_list = Vec::with_capacity(content_data_list.len()); + + for content_data in content_data_list { + self.write_all(content_data.as_bytes()).await?; + let metadata = ContentMetadata::new(content_data.content_source); + metadata_list.push(metadata); + } + + self.flush().await?; + Ok(metadata_list) + } + } + + /// Append content data to the destination without truncating + /// + /// This method assumes the destination supports append operations. + /// + /// # Errors + /// + /// Returns an error if the write operation fails or if there are I/O issues. + /// + /// # Example + /// + /// ```no_run + /// use nvisy_core::io::{AsyncContentWrite, ContentData}; + /// use nvisy_core::fs::ContentMetadata; + /// use tokio::fs::OpenOptions; + /// use std::io; + /// + /// async fn append_content() -> io::Result { + /// let mut file = OpenOptions::new() + /// .create(true) + /// .append(true) + /// .open("log.txt") + /// .await?; + /// + /// let content = ContentData::from("New log entry\n"); + /// file.append_content(content).await + /// } + /// ``` + fn append_content( + &mut self, + content_data: ContentData, + ) -> impl Future> + Send + where + Self: Sized, + { + async move { + self.write_all(content_data.as_bytes()).await?; + self.flush().await?; + + let metadata = ContentMetadata::new(content_data.content_source); + Ok(metadata) + } + } + + /// Write content data with verification + /// + /// This method writes the content and then optionally verifies it was + /// written correctly by checking the expected size. + /// + /// # Errors + /// + /// Returns an error if the write operation fails, if there are I/O issues, + /// or if verification fails. + fn write_content_verified( + &mut self, + content_data: ContentData, + verify_size: bool, + ) -> impl Future> + Send + where + Self: Sized, + { + async move { + let expected_size = content_data.size(); + let data = content_data.as_bytes(); + + let bytes_written = self.write(data).await?; + self.flush().await?; + + if verify_size && bytes_written != expected_size { + return Err(io::Error::new( + io::ErrorKind::WriteZero, + format!( + "Expected to write {expected_size} bytes, but only wrote {bytes_written} bytes" + ), + )); + } + + let metadata = ContentMetadata::new(content_data.content_source); + Ok(metadata) + } + } +} + +// Implementations for common types +impl AsyncContentWrite for tokio::fs::File {} +impl AsyncContentWrite for Vec {} +impl AsyncContentWrite for Box {} + +#[cfg(test)] +mod tests { + use std::io::Result; + + use super::*; + + #[tokio::test] + async fn test_write_content() -> Result<()> { + let mut writer = Vec::::new(); + let content = ContentData::from("Hello, world!"); + + let metadata = writer.write_content(content).await?; + assert!(!metadata.content_source.as_uuid().is_nil()); + + Ok(()) + } + + #[tokio::test] + async fn test_write_content_with_path() -> Result<()> { + let mut writer = Vec::::new(); + let content = ContentData::from("Hello, world!"); + + let metadata = writer.write_content_with_path(content, "test.txt").await?; + assert!(metadata.has_path()); + assert_eq!(metadata.filename(), Some("test.txt")); + + Ok(()) + } + + #[tokio::test] + async fn test_write_content_chunked() -> Result<()> { + let mut writer = Vec::::new(); + let data = vec![42u8; 1000]; + let content = ContentData::from(data.clone()); + + let metadata = writer.write_content_chunked(content, 100).await?; + assert!(!metadata.content_source.as_uuid().is_nil()); + assert_eq!(writer.as_slice(), data.as_slice()); + + Ok(()) + } + + #[tokio::test] + async fn test_write_multiple_content() -> Result<()> { + let mut writer = Vec::::new(); + let contents = vec![ContentData::from("Hello, "), ContentData::from("world!")]; + + let metadata_list = writer.write_multiple_content(contents).await?; + assert_eq!(metadata_list.len(), 2); + assert_eq!(writer.as_slice(), b"Hello, world!"); + + Ok(()) + } + + #[tokio::test] + async fn test_append_content() -> Result<()> { + let mut writer = Vec::::new(); + let content = ContentData::from("Hello, world!"); + + let metadata = writer.append_content(content).await?; + assert!(!metadata.content_source.as_uuid().is_nil()); + assert_eq!(writer.as_slice(), b"Hello, world!"); + + Ok(()) + } + + #[tokio::test] + async fn test_write_content_verified() -> Result<()> { + let mut writer = Vec::::new(); + let content = ContentData::from("Hello, world!"); + + let metadata = writer.write_content_verified(content, true).await?; + assert!(!metadata.content_source.as_uuid().is_nil()); + assert_eq!(writer.as_slice(), b"Hello, world!"); + + Ok(()) + } + + #[tokio::test] + async fn test_write_empty_content() -> Result<()> { + let mut writer = Vec::::new(); + let content = ContentData::from(""); + + let metadata = writer.write_content(content).await?; + assert!(!metadata.content_source.as_uuid().is_nil()); + assert_eq!(writer.as_slice(), b""); + + Ok(()) + } + + #[tokio::test] + async fn test_write_large_content() -> Result<()> { + let mut writer = Vec::::new(); + let data = vec![123u8; 10000]; + let content = ContentData::from(data.clone()); + + let metadata = writer.write_content(content).await?; + assert!(!metadata.content_source.as_uuid().is_nil()); + assert_eq!(writer.as_slice(), data.as_slice()); + + Ok(()) + } +} diff --git a/crates/nvisy-core/src/io/data_reference.rs b/crates/nvisy-core/src/io/data_reference.rs new file mode 100644 index 0000000..7397498 --- /dev/null +++ b/crates/nvisy-core/src/io/data_reference.rs @@ -0,0 +1,141 @@ +//! Data reference definitions +//! +//! This module provides the `DataReference` struct for referencing and +//! tracking content within the Nvisy system. + +use serde::{Deserialize, Serialize}; + +use crate::io::Content; +use crate::path::ContentSource; + +/// Reference to data with source tracking and content information +/// +/// A `DataReference` provides a lightweight way to reference data content +/// while maintaining information about its source location and optional +/// mapping within that source. +/// +/// # Examples +/// +/// ```rust +/// use nvisy_core::io::{DataReference, Content, ContentData}; +/// +/// let content = Content::new(ContentData::from("Hello, world!")); +/// let data_ref = DataReference::new(content) +/// .with_mapping_id("line-42"); +/// +/// assert!(data_ref.mapping_id().is_some()); +/// assert_eq!(data_ref.mapping_id().unwrap(), "line-42"); +/// ``` +#[derive(Debug, Clone)] +#[derive(Serialize, Deserialize)] +pub struct DataReference { + /// Unique identifier for the source containing this data + /// Using `UUIDv7` for time-ordered, globally unique identification + source: ContentSource, + + /// Optional identifier that defines the position/location of the data within the source + /// Examples: line numbers, byte offsets, element IDs, `XPath` expressions + mapping_id: Option, + + /// The actual content data + content: Content, +} + +impl DataReference { + /// Create a new data reference with auto-generated source ID (`UUIDv7`) + pub fn new(content: Content) -> Self { + Self { + source: ContentSource::new(), + mapping_id: None, + content, + } + } + + /// Create a new data reference with specific source + pub fn with_source(source: ContentSource, content: Content) -> Self { + Self { + source, + mapping_id: None, + content, + } + } + + /// Set the mapping ID for this data reference + #[must_use] + pub fn with_mapping_id>(mut self, mapping_id: S) -> Self { + self.mapping_id = Some(mapping_id.into()); + self + } + + /// Get the content source + pub fn source(&self) -> ContentSource { + self.source + } + + /// Get the mapping ID, if any + pub fn mapping_id(&self) -> Option<&str> { + self.mapping_id.as_deref() + } + + /// Get a reference to the content + pub fn content(&self) -> &Content { + &self.content + } + + /// Check if the content is text-based + pub fn is_likely_text(&self) -> bool { + self.content.is_likely_text() + } + + /// Get the size of the content in bytes + pub fn size(&self) -> usize { + self.content.size() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::io::ContentData; + + #[test] + fn test_data_reference_creation() { + let content = Content::new(ContentData::from("Hello, world!")); + let data_ref = DataReference::new(content); + + assert!(data_ref.is_likely_text()); + assert!(data_ref.mapping_id().is_none()); + assert_eq!(data_ref.size(), 13); + // Verify UUIDv7 is used + assert_eq!(data_ref.source().as_uuid().get_version_num(), 7); + } + + #[test] + fn test_data_reference_with_mapping() { + let content = Content::new(ContentData::from("Test content")); + let data_ref = DataReference::new(content).with_mapping_id("line-42"); + + assert_eq!(data_ref.mapping_id(), Some("line-42")); + } + + #[test] + fn test_data_reference_with_source() { + let source = ContentSource::new(); + let content = Content::new(ContentData::from("Test content")); + let data_ref = DataReference::with_source(source, content); + + assert_eq!(data_ref.source(), source); + } + + #[test] + fn test_serialization() { + let content = Content::new(ContentData::from("Test content")); + let data_ref = DataReference::new(content).with_mapping_id("test-mapping"); + + let json = serde_json::to_string(&data_ref).unwrap(); + let deserialized: DataReference = serde_json::from_str(&json).unwrap(); + + assert_eq!(data_ref.source(), deserialized.source()); + assert_eq!(data_ref.mapping_id(), deserialized.mapping_id()); + } +} diff --git a/crates/nvisy-core/src/io/mod.rs b/crates/nvisy-core/src/io/mod.rs new file mode 100644 index 0000000..aa33482 --- /dev/null +++ b/crates/nvisy-core/src/io/mod.rs @@ -0,0 +1,26 @@ +//! I/O module for content handling and processing +//! +//! This module provides the core I/O abstractions for handling content data, +//! including content data structures and async read/write traits. +//! +//! # Core Types +//! +//! - [`ContentData`]: Container for content data with metadata, hashing, and size utilities +//! +//! # Traits +//! +//! - [`AsyncContentRead`]: Async trait for reading content from various sources +//! - [`AsyncContentWrite`]: Async trait for writing content to various destinations + +mod content; +mod content_data; +mod content_read; +mod content_write; +mod data_reference; + +// Re-export core types and traits +pub use content::Content; +pub use content_data::{ContentBytes, ContentData}; +pub use content_read::AsyncContentRead; +pub use content_write::AsyncContentWrite; +pub use data_reference::DataReference; diff --git a/crates/nvisy-core/src/lib.rs b/crates/nvisy-core/src/lib.rs new file mode 100644 index 0000000..ad85b54 --- /dev/null +++ b/crates/nvisy-core/src/lib.rs @@ -0,0 +1,11 @@ +#![forbid(unsafe_code)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![doc = include_str!("../README.md")] + +pub mod error; +pub mod fs; +pub mod io; +pub mod path; + +#[doc(hidden)] +pub mod prelude; diff --git a/crates/nvisy-core/src/path/mod.rs b/crates/nvisy-core/src/path/mod.rs new file mode 100644 index 0000000..08cb0c4 --- /dev/null +++ b/crates/nvisy-core/src/path/mod.rs @@ -0,0 +1,9 @@ +//! Path module for content source identification +//! +//! This module provides functionality for uniquely identifying content sources +//! throughout the nvisy system using UUIDv7-based identifiers. + +mod source; + +// Re-export core types +pub use source::ContentSource; diff --git a/crates/nvisy-core/src/path/source.rs b/crates/nvisy-core/src/path/source.rs new file mode 100644 index 0000000..ff8a1cd --- /dev/null +++ b/crates/nvisy-core/src/path/source.rs @@ -0,0 +1,317 @@ +//! Content source identification module +//! +//! This module provides the [`ContentSource`] struct for uniquely identifying +//! data sources throughout the nvisy system using `UUIDv7`. + +use std::fmt; + +use jiff::Zoned; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Unique identifier for content sources in the system +/// +/// Uses `UUIDv7` for time-ordered, globally unique identification of data sources. +/// +/// This allows for efficient tracking and correlation of content throughout +/// the processing pipeline. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Serialize, Deserialize, schemars::JsonSchema)] +pub struct ContentSource { + /// `UUIDv7` identifier + id: Uuid, + /// Optional parent source for lineage tracking + #[serde(skip_serializing_if = "Option::is_none")] + parent_id: Option, +} + +impl ContentSource { + /// Create a new content source with a fresh `UUIDv7` + /// + /// # Example + /// + /// ``` + /// use nvisy_core::path::ContentSource; + /// + /// let source = ContentSource::new(); + /// assert!(!source.as_uuid().is_nil()); + /// ``` + #[must_use] + pub fn new() -> Self { + let now = Zoned::now(); + let timestamp = uuid::Timestamp::from_unix( + uuid::NoContext, + now.timestamp().as_second().unsigned_abs(), + now.timestamp().subsec_nanosecond().unsigned_abs(), + ); + + Self { + id: Uuid::new_v7(timestamp), + parent_id: None, + } + } + + /// Create a content source from an existing UUID + /// + /// # Example + /// + /// ``` + /// use nvisy_core::path::ContentSource; + /// use uuid::Uuid; + /// + /// let source = ContentSource::new(); + /// let uuid = source.as_uuid(); + /// let source2 = ContentSource::from_uuid(uuid); + /// assert_eq!(source2.as_uuid(), uuid); + /// ``` + #[must_use] + pub fn from_uuid(id: Uuid) -> Self { + Self { id, parent_id: None } + } + + /// Get the underlying UUID + /// + /// # Example + /// + /// ``` + /// use nvisy_core::path::ContentSource; + /// + /// let source = ContentSource::new(); + /// let uuid = source.as_uuid(); + /// assert_eq!(uuid.get_version_num(), 7); + /// ``` + #[must_use] + pub fn as_uuid(&self) -> Uuid { + self.id + } + + /// Get the UUID as a string + /// + /// # Example + /// + /// ``` + /// use nvisy_core::path::ContentSource; + /// + /// let source = ContentSource::new(); + /// let id_str = source.to_string(); + /// assert_eq!(id_str.len(), 36); // Standard UUID string length + /// ``` + /// + /// Parse a content source from a string + /// + /// # Errors + /// + /// Returns an error if the string is not a valid UUID format. + /// + /// # Example + /// + /// ``` + /// use nvisy_core::path::ContentSource; + /// + /// let source = ContentSource::new(); + /// let id_str = source.to_string(); + /// let parsed = ContentSource::parse(&id_str).unwrap(); + /// assert_eq!(source, parsed); + /// ``` + pub fn parse(s: &str) -> Result { + let id = Uuid::parse_str(s)?; + Ok(Self { id, parent_id: None }) + } + + /// Get the parent source identifier, if any. + #[must_use] + pub fn parent_id(&self) -> Option { + self.parent_id + } + + /// Set the parent source identifier. + pub fn set_parent_id(&mut self, parent_id: Option) { + self.parent_id = parent_id; + } + + /// Create a copy of this source with the given parent (builder pattern). + #[must_use] + pub fn with_parent(mut self, parent: &ContentSource) -> Self { + self.parent_id = Some(parent.id); + self + } + + /// Create a new content source derived from this one (new ID, self as parent). + #[must_use] + pub fn derive(&self) -> Self { + let mut child = Self::new(); + child.parent_id = Some(self.id); + child + } + + /// Get the timestamp component from the `UUIDv7` + /// + /// Returns the Unix timestamp in milliseconds when this UUID was generated, + /// or None if this is not a `UUIDv7`. + /// + /// # Example + /// + /// ``` + /// use nvisy_core::path::ContentSource; + /// use std::time::{SystemTime, UNIX_EPOCH}; + /// + /// let source = ContentSource::new(); + /// let timestamp = source.timestamp().expect("UUIDv7 should have timestamp"); + /// let now = SystemTime::now() + /// .duration_since(UNIX_EPOCH) + /// .unwrap() + /// .as_millis() as u64; + /// + /// // Should be very close to current time (within a few seconds) + /// assert!((timestamp as i64 - now as i64).abs() < 5000); + /// ``` + #[must_use] + pub fn timestamp(&self) -> Option { + self.id.get_timestamp().map(|timestamp| { + let (seconds, nanos) = timestamp.to_unix(); + seconds * 1000 + u64::from(nanos) / 1_000_000 + }) + } + + /// Check if this content source was created before another + /// + /// Returns false if either UUID is not a `UUIDv7` and thus has no timestamp. + /// + /// # Example + /// + /// ``` + /// use nvisy_core::path::ContentSource; + /// use std::thread; + /// use std::time::Duration; + /// + /// let source1 = ContentSource::new(); + /// thread::sleep(Duration::from_millis(1)); + /// let source2 = ContentSource::new(); + /// + /// assert!(source1.created_before(&source2)); + /// assert!(!source2.created_before(&source1)); + /// ``` + #[must_use] + pub fn created_before(&self, other: &ContentSource) -> bool { + match (self.timestamp(), other.timestamp()) { + (Some(self_ts), Some(other_ts)) => self_ts < other_ts, + _ => false, + } + } + + /// Check if this content source was created after another + /// + /// Returns false if either UUID is not a `UUIDv7` and thus has no timestamp. + #[must_use] + pub fn created_after(&self, other: &ContentSource) -> bool { + match (self.timestamp(), other.timestamp()) { + (Some(self_ts), Some(other_ts)) => self_ts > other_ts, + _ => false, + } + } +} + +impl Default for ContentSource { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Display for ContentSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.id) + } +} + +impl From for ContentSource { + fn from(id: Uuid) -> Self { + Self::from_uuid(id) + } +} + +impl From for Uuid { + fn from(source: ContentSource) -> Self { + source.id + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::thread; + use std::time::Duration; + + use super::*; + + #[test] + fn test_new_content_source() { + let source = ContentSource::new(); + assert_eq!(source.as_uuid().get_version_num(), 7); + assert!(!source.as_uuid().is_nil()); + } + + #[test] + fn test_uniqueness() { + let mut sources = HashSet::new(); + + // Generate 1000 sources and ensure they're all unique + for _ in 0..1000 { + let source = ContentSource::new(); + assert!(sources.insert(source), "Duplicate content source found"); + } + } + + #[test] + fn test_string_conversion() { + let source = ContentSource::new(); + let string_repr = source.to_string(); + let parsed = ContentSource::parse(&string_repr).unwrap(); + assert_eq!(source, parsed); + } + + #[test] + fn test_invalid_string_parsing() { + let result = ContentSource::parse("invalid-uuid"); + assert!(result.is_err()); + } + + #[test] + fn test_ordering() { + let source1 = ContentSource::new(); + thread::sleep(Duration::from_millis(2)); + let source2 = ContentSource::new(); + + assert!(source1.created_before(&source2)); + assert!(source2.created_after(&source1)); + assert!(source1 < source2); // Test PartialOrd + } + + #[test] + fn test_display() { + let source = ContentSource::new(); + let display_str = format!("{source}"); + let uuid_str = source.as_uuid().to_string(); + assert_eq!(display_str, uuid_str); + } + + #[test] + fn test_serde_serialization() { + let source = ContentSource::new(); + let serialized = serde_json::to_string(&source).unwrap(); + let deserialized: ContentSource = serde_json::from_str(&serialized).unwrap(); + assert_eq!(source, deserialized); + } + + #[test] + fn test_hash_consistency() { + let source = ContentSource::new(); + let mut set = HashSet::new(); + + set.insert(source); + assert!(set.contains(&source)); + + // Same source should hash the same way + let cloned_source = source; + assert!(set.contains(&cloned_source)); + } +} diff --git a/crates/nvisy-core/src/prelude.rs b/crates/nvisy-core/src/prelude.rs new file mode 100644 index 0000000..5690858 --- /dev/null +++ b/crates/nvisy-core/src/prelude.rs @@ -0,0 +1,9 @@ +//! Convenience re-exports for common nvisy-core types. +//! +//! Import everything from this module to get the most commonly used +//! types without individual `use` statements. + +pub use crate::error::{Error, ErrorKind, Result}; +pub use crate::fs::{ContentFile, ContentHandler, ContentKind, ContentMetadata, ContentRegistry}; +pub use crate::io::{AsyncContentRead, AsyncContentWrite, Content, ContentBytes, ContentData, DataReference}; +pub use crate::path::ContentSource; diff --git a/crates/nvisy-engine/Cargo.toml b/crates/nvisy-engine/Cargo.toml new file mode 100644 index 0000000..095deba --- /dev/null +++ b/crates/nvisy-engine/Cargo.toml @@ -0,0 +1,51 @@ +# https://doc.rust-lang.org/cargo/reference/manifest.html + +[package] +name = "nvisy-engine" +description = "DAG compiler and executor for Nvisy pipeline graphs" +keywords = ["nvisy", "engine", "dag", "pipeline"] +categories = ["concurrency"] + +version = { workspace = true } +rust-version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } + +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } + +[dependencies] +# Internal crates +nvisy-core = { workspace = true, features = [] } +nvisy-ontology = { workspace = true, features = [] } + +# JSON Schema generation +schemars = { workspace = true, features = [] } + +# (De)serialization +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = [] } + +# Async runtime +tokio = { workspace = true, features = ["rt", "sync", "time", "macros"] } +tokio-util = { workspace = true, features = [] } + +# Primitive datatypes +uuid = { workspace = true, features = ["v4"] } +jiff = { workspace = true, features = [] } + +# Graph data structures +petgraph = { workspace = true, features = [] } + +# Error handling +thiserror = { workspace = true, features = [] } +anyhow = { workspace = true, features = [] } + +# Randomness +rand = { workspace = true, features = [] } + +# Observability +tracing = { workspace = true, features = [] } diff --git a/crates/nvisy-engine/README.md b/crates/nvisy-engine/README.md new file mode 100644 index 0000000..0ae582f --- /dev/null +++ b/crates/nvisy-engine/README.md @@ -0,0 +1,3 @@ +# nvisy-engine + +DAG compiler and executor for the Nvisy runtime. Compiles graph definitions into executable pipelines, manages run lifecycles, and coordinates policy resolution and connection routing between nodes. diff --git a/crates/nvisy-engine/src/compiler/graph.rs b/crates/nvisy-engine/src/compiler/graph.rs new file mode 100644 index 0000000..ab5056c --- /dev/null +++ b/crates/nvisy-engine/src/compiler/graph.rs @@ -0,0 +1,130 @@ +//! Graph data model for pipeline definitions. +//! +//! A pipeline is represented as a set of [`GraphNode`]s connected by +//! [`GraphEdge`]s, collected into a [`Graph`]. + +use serde::{Deserialize, Serialize}; + +use crate::policies::retry::RetryPolicy; + +/// A node in the pipeline graph, tagged by its role. +/// +/// Nodes are serialized with a `"type"` discriminator so JSON definitions +/// can specify `"source"`, `"action"`, or `"target"`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(schemars::JsonSchema)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum GraphNode { + /// A data source that reads from an external provider via a named stream. + Source { + /// Unique identifier for this node within the graph. + id: String, + /// Provider name used to resolve the connection (e.g. `"s3"`). + provider: String, + /// Stream name on the provider (e.g. `"read"`). + stream: String, + /// Arbitrary provider-specific parameters. + #[serde(default)] + params: serde_json::Value, + /// Optional retry policy applied to this node's execution. + #[serde(skip_serializing_if = "Option::is_none")] + retry: Option, + /// Optional per-node timeout in milliseconds. + #[serde(skip_serializing_if = "Option::is_none")] + timeout_ms: Option, + }, + /// A transformation or detection step applied to data flowing through the pipeline. + Action { + /// Unique identifier for this node within the graph. + id: String, + /// Registered action name (e.g. `"detect_regex"`, `"classify"`). + action: String, + /// Arbitrary action-specific parameters. + #[serde(default)] + params: serde_json::Value, + /// Optional retry policy applied to this node's execution. + #[serde(skip_serializing_if = "Option::is_none")] + retry: Option, + /// Optional per-node timeout in milliseconds. + #[serde(skip_serializing_if = "Option::is_none")] + timeout_ms: Option, + }, + /// A data sink that writes to an external provider via a named stream. + Target { + /// Unique identifier for this node within the graph. + id: String, + /// Provider name used to resolve the connection (e.g. `"s3"`). + provider: String, + /// Stream name on the provider (e.g. `"write"`). + stream: String, + /// Arbitrary provider-specific parameters. + #[serde(default)] + params: serde_json::Value, + /// Optional retry policy applied to this node's execution. + #[serde(skip_serializing_if = "Option::is_none")] + retry: Option, + /// Optional per-node timeout in milliseconds. + #[serde(skip_serializing_if = "Option::is_none")] + timeout_ms: Option, + }, +} + +impl GraphNode { + /// Returns the unique identifier shared by all node variants. + pub fn id(&self) -> &str { + match self { + GraphNode::Source { id, .. } => id, + GraphNode::Action { id, .. } => id, + GraphNode::Target { id, .. } => id, + } + } + + /// Returns the parameters value for this node. + pub fn params(&self) -> &serde_json::Value { + match self { + GraphNode::Source { params, .. } => params, + GraphNode::Action { params, .. } => params, + GraphNode::Target { params, .. } => params, + } + } + + /// Returns the retry policy, if one is configured. + pub fn retry(&self) -> Option<&RetryPolicy> { + match self { + GraphNode::Source { retry, .. } => retry.as_ref(), + GraphNode::Action { retry, .. } => retry.as_ref(), + GraphNode::Target { retry, .. } => retry.as_ref(), + } + } + + /// Returns the per-node timeout in milliseconds, if one is configured. + pub fn timeout_ms(&self) -> Option { + match self { + GraphNode::Source { timeout_ms, .. } => *timeout_ms, + GraphNode::Action { timeout_ms, .. } => *timeout_ms, + GraphNode::Target { timeout_ms, .. } => *timeout_ms, + } + } +} + +/// A directed edge connecting two nodes by their IDs. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(schemars::JsonSchema)] +pub struct GraphEdge { + /// ID of the upstream (source) node. + pub from: String, + /// ID of the downstream (destination) node. + pub to: String, +} + +/// A complete pipeline graph definition containing nodes and edges. +/// +/// The graph must be a valid DAG (directed acyclic graph) with unique node IDs. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(schemars::JsonSchema)] +pub struct Graph { + /// All nodes in the pipeline. + pub nodes: Vec, + /// Directed edges describing data flow between nodes. + pub edges: Vec, +} diff --git a/crates/nvisy-engine/src/compiler/mod.rs b/crates/nvisy-engine/src/compiler/mod.rs new file mode 100644 index 0000000..9b21fc8 --- /dev/null +++ b/crates/nvisy-engine/src/compiler/mod.rs @@ -0,0 +1,11 @@ +//! Pipeline compilation: parsing, graph construction, and execution planning. +//! +//! The compiler takes a JSON pipeline definition, validates it, builds a +//! directed graph, and produces a topologically-sorted execution plan. + +pub mod graph; +pub mod parse; +pub mod plan; + +pub use parse::parse_graph; +pub use plan::{build_plan, ExecutionPlan, ResolvedNode}; diff --git a/crates/nvisy-engine/src/compiler/parse.rs b/crates/nvisy-engine/src/compiler/parse.rs new file mode 100644 index 0000000..68b53c0 --- /dev/null +++ b/crates/nvisy-engine/src/compiler/parse.rs @@ -0,0 +1,54 @@ +//! JSON parsing and validation for pipeline graph definitions. +//! +//! Deserializes a [`serde_json::Value`] into a [`Graph`] and validates +//! structural invariants (non-empty, unique IDs, valid edge references). + +use crate::compiler::graph::Graph; +use nvisy_core::error::Error; + +/// Parses and validates a [`Graph`] from a JSON value. +/// +/// Performs the following validations: +/// - The graph must contain at least one node. +/// - All node IDs must be unique. +/// - All edge endpoints must reference existing node IDs. +pub fn parse_graph(value: &serde_json::Value) -> Result { + let graph: Graph = serde_json::from_value(value.clone()).map_err(|e| { + Error::validation(format!("Invalid graph definition: {}", e), "compiler") + })?; + + // Validate: must have at least one node + if graph.nodes.is_empty() { + return Err(Error::validation("Graph must have at least one node", "compiler")); + } + + // Validate: no duplicate node IDs + let mut seen = std::collections::HashSet::new(); + for node in &graph.nodes { + if !seen.insert(node.id()) { + return Err(Error::validation( + format!("Duplicate node ID: {}", node.id()), + "compiler", + )); + } + } + + // Validate: all edge endpoints reference existing nodes + let node_ids: std::collections::HashSet<&str> = graph.nodes.iter().map(|n| n.id()).collect(); + for edge in &graph.edges { + if !node_ids.contains(edge.from.as_str()) { + return Err(Error::validation( + format!("Edge references unknown source node: {}", edge.from), + "compiler", + )); + } + if !node_ids.contains(edge.to.as_str()) { + return Err(Error::validation( + format!("Edge references unknown target node: {}", edge.to), + "compiler", + )); + } + } + + Ok(graph) +} diff --git a/crates/nvisy-engine/src/compiler/plan.rs b/crates/nvisy-engine/src/compiler/plan.rs new file mode 100644 index 0000000..44be2d0 --- /dev/null +++ b/crates/nvisy-engine/src/compiler/plan.rs @@ -0,0 +1,104 @@ +//! Execution planning via topological sort. +//! +//! Converts a validated [`Graph`] into an [`ExecutionPlan`] by performing +//! cycle detection and topological sorting using `petgraph`. + +use std::collections::HashMap; +use petgraph::algo::{is_cyclic_directed, toposort}; +use petgraph::graph::{DiGraph, NodeIndex}; +use crate::compiler::graph::{Graph, GraphNode}; +use nvisy_core::error::Error; + +/// A graph node enriched with topological ordering and adjacency information. +#[derive(Debug, Clone)] +pub struct ResolvedNode { + /// The original graph node definition. + pub node: GraphNode, + /// Zero-based position in the topological ordering. + pub topo_order: usize, + /// IDs of nodes that feed data into this node. + pub upstream_ids: Vec, + /// IDs of nodes that receive data from this node. + pub downstream_ids: Vec, +} + +/// A compiled execution plan ready for the executor. +/// +/// Contains all nodes in topological order along with their adjacency +/// information so the executor can wire channels and schedule tasks. +pub struct ExecutionPlan { + /// Resolved nodes sorted in topological order. + pub nodes: Vec, + /// Node IDs in topological order. + pub topo_order: Vec, +} + +/// Builds an execution plan from a parsed [`Graph`]. +/// +/// Validates that the graph is acyclic, performs a topological sort, and +/// computes upstream/downstream adjacency lists for each node. +/// +/// Returns an error if the graph contains a cycle or references unknown nodes. +pub fn build_plan(graph: &Graph) -> Result { + // Build petgraph + let mut pg: DiGraph<&str, ()> = DiGraph::new(); + let mut index_map: HashMap<&str, NodeIndex> = HashMap::new(); + + for node in &graph.nodes { + let idx = pg.add_node(node.id()); + index_map.insert(node.id(), idx); + } + + for edge in &graph.edges { + let from = index_map.get(edge.from.as_str()).ok_or_else(|| { + Error::validation(format!("Unknown edge source: {}", edge.from), "compiler") + })?; + let to = index_map.get(edge.to.as_str()).ok_or_else(|| { + Error::validation(format!("Unknown edge target: {}", edge.to), "compiler") + })?; + pg.add_edge(*from, *to, ()); + } + + // Cycle detection + if is_cyclic_directed(&pg) { + return Err(Error::validation("Graph contains a cycle", "compiler")); + } + + // Topological sort + let topo = toposort(&pg, None).map_err(|_| { + Error::validation("Graph contains a cycle", "compiler") + })?; + + let topo_order: Vec = topo.iter().map(|idx| pg[*idx].to_string()).collect(); + + // Build resolved nodes with adjacency info + let node_map: HashMap<&str, &GraphNode> = graph.nodes.iter().map(|n| (n.id(), n)).collect(); + let mut resolved = Vec::new(); + + for (order, node_id) in topo_order.iter().enumerate() { + let node = node_map[node_id.as_str()]; + let idx = index_map[node_id.as_str()]; + + let upstream_ids: Vec = pg + .neighbors_directed(idx, petgraph::Direction::Incoming) + .map(|n| pg[n].to_string()) + .collect(); + + let downstream_ids: Vec = pg + .neighbors_directed(idx, petgraph::Direction::Outgoing) + .map(|n| pg[n].to_string()) + .collect(); + + resolved.push(ResolvedNode { + node: node.clone(), + topo_order: order, + upstream_ids, + downstream_ids, + }); + } + + Ok(ExecutionPlan { + nodes: resolved, + topo_order, + }) +} diff --git a/crates/nvisy-engine/src/connections/mod.rs b/crates/nvisy-engine/src/connections/mod.rs new file mode 100644 index 0000000..cab9543 --- /dev/null +++ b/crates/nvisy-engine/src/connections/mod.rs @@ -0,0 +1,25 @@ +//! External service connection definitions. +//! +//! A [`Connection`] holds the provider type, credentials, and optional context +//! needed to interact with an external service (e.g. S3, a database). +//! [`Connections`] is a type alias mapping connection IDs to their definitions. + +use std::collections::HashMap; +use serde::{Deserialize, Serialize}; + +/// A validated connection to an external service such as S3 or a database. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(schemars::JsonSchema)] +pub struct Connection { + /// Provider type identifier (e.g. `"s3"`, `"postgres"`). + #[serde(rename = "type")] + pub provider_type: String, + /// Opaque credentials payload specific to the provider. + pub credentials: serde_json::Value, + /// Optional provider-specific context (e.g. region, endpoint overrides). + #[serde(default)] + pub context: serde_json::Value, +} + +/// Map of connection IDs to their [`Connection`] definitions. +pub type Connections = HashMap; diff --git a/crates/nvisy-engine/src/engine.rs b/crates/nvisy-engine/src/engine.rs new file mode 100644 index 0000000..b271095 --- /dev/null +++ b/crates/nvisy-engine/src/engine.rs @@ -0,0 +1,70 @@ +//! Top-level engine contract and I/O types. +//! +//! The [`Engine`] trait defines the high-level redaction pipeline contract: +//! given a content handler, policies, and an execution graph, produce redacted +//! output together with a full audit trail and per-phase breakdown. + +use std::future::Future; + +use uuid::Uuid; + +use nvisy_core::error::Error; +use nvisy_core::fs::ContentHandler; +use nvisy_ontology::audit::Audit; +use nvisy_ontology::detection::{ClassificationResult, DetectionResult}; +use nvisy_ontology::policy::{Policies, PolicyEvaluation}; +use nvisy_ontology::redaction::RedactionSummary; + +use crate::compiler::graph::Graph; +use crate::connections::Connections; +use crate::executor::runner::RunResult; + +/// Everything the caller must provide to run a redaction pipeline. +pub struct EngineInput { + /// Handle to the managed directory containing the files to process. + pub source: ContentHandler, + /// Policies to apply (at least one). + pub policies: Policies, + /// Execution graph defining the pipeline DAG. + pub graph: Graph, + /// External service connections for source/target nodes. + pub connections: Connections, + /// Human or service account identity. + pub actor: Option, +} + +/// Full result of a pipeline run. +/// +/// Contains a content handler for the redacted output, per-phase breakdown +/// (detection, classification, policy evaluation), per-source summaries, +/// audit records, and the raw DAG execution result. +pub struct EngineOutput { + /// Unique run identifier. + pub run_id: Uuid, + /// Handle to the managed directory containing redacted output files. + pub output: ContentHandler, + /// Full detection result (entities, sensitivity, risk). + pub detection: DetectionResult, + /// Sensitivity classification. + pub classification: ClassificationResult, + /// Policy evaluation breakdown (redactions, reviews, suppressions, blocks, alerts). + pub evaluation: PolicyEvaluation, + /// Per-source redaction summaries. + pub summaries: Vec, + /// Immutable audit trail. + pub audits: Vec, + /// Per-node execution results from the DAG runner. + pub run_result: RunResult, +} + +/// The top-level redaction engine contract. +/// +/// Takes a content handler, policies, and an execution graph; returns redacted +/// output, audit records, and a full breakdown of every pipeline phase. +pub trait Engine: Send + Sync { + /// Execute a full redaction pipeline. + fn run( + &self, + input: EngineInput, + ) -> impl Future> + Send; +} diff --git a/crates/nvisy-engine/src/executor/context.rs b/crates/nvisy-engine/src/executor/context.rs new file mode 100644 index 0000000..58c5c7d --- /dev/null +++ b/crates/nvisy-engine/src/executor/context.rs @@ -0,0 +1,58 @@ +//! Channel primitives used to wire data flow between pipeline nodes. +//! +//! [`EdgeChannel`] carries [`ContentData`] items along a graph edge, while +//! [`NodeSignal`] broadcasts node completion. + +use tokio::sync::{mpsc, watch}; +use nvisy_core::io::ContentData; + +/// Default buffer size for bounded inter-node MPSC channels. +pub const CHANNEL_BUFFER_SIZE: usize = 256; + +/// A bounded MPSC channel pair used to transfer [`ContentData`] items along a +/// single graph edge from an upstream node to a downstream node. +pub struct EdgeChannel { + /// Sending half, held by the upstream node. + pub sender: mpsc::Sender, + /// Receiving half, held by the downstream node. + pub receiver: mpsc::Receiver, +} + +impl Default for EdgeChannel { + fn default() -> Self { + Self::new() + } +} + +impl EdgeChannel { + /// Creates a new edge channel with [`CHANNEL_BUFFER_SIZE`] capacity. + pub fn new() -> Self { + let (sender, receiver) = mpsc::channel(CHANNEL_BUFFER_SIZE); + Self { sender, receiver } + } +} + +/// A watch channel pair used to signal that a node has completed execution. +/// +/// The sender broadcasts `true` when the node finishes, and downstream nodes +/// wait on the receiver before starting. +pub struct NodeSignal { + /// Sending half; set to `true` when the node completes. + pub sender: watch::Sender, + /// Receiving half; downstream tasks call `wait_for(|&done| done)`. + pub receiver: watch::Receiver, +} + +impl Default for NodeSignal { + fn default() -> Self { + Self::new() + } +} + +impl NodeSignal { + /// Creates a new node signal initialized to `false` (not completed). + pub fn new() -> Self { + let (sender, receiver) = watch::channel(false); + Self { sender, receiver } + } +} diff --git a/crates/nvisy-engine/src/executor/mod.rs b/crates/nvisy-engine/src/executor/mod.rs new file mode 100644 index 0000000..7669b01 --- /dev/null +++ b/crates/nvisy-engine/src/executor/mod.rs @@ -0,0 +1,9 @@ +//! Pipeline execution runtime. +//! +//! Spawns concurrent Tokio tasks for each node in topological order, +//! wires inter-node channels, and collects per-node results. + +pub mod context; +pub mod runner; + +pub use runner::run_graph; diff --git a/crates/nvisy-engine/src/executor/runner.rs b/crates/nvisy-engine/src/executor/runner.rs new file mode 100644 index 0000000..4a949ac --- /dev/null +++ b/crates/nvisy-engine/src/executor/runner.rs @@ -0,0 +1,165 @@ +//! Graph runner that executes a compiled [`ExecutionPlan`]. +//! +//! Each node is spawned as a concurrent Tokio task. Data flows between nodes +//! via bounded MPSC channels, and upstream completion is signalled via watch +//! channels so downstream tasks wait before starting. + +use std::collections::HashMap; +use tokio::sync::{mpsc, watch}; +use tokio::task::JoinSet; +use uuid::Uuid; +use nvisy_core::io::ContentData; +use nvisy_core::error::Error; +use crate::compiler::plan::ExecutionPlan; +use crate::connections::Connections; +use crate::executor::context::CHANNEL_BUFFER_SIZE; +use crate::compiler::graph::GraphNode; + +/// Outcome of executing a single node in the pipeline. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(schemars::JsonSchema)] +pub struct NodeResult { + /// ID of the node that produced this result. + pub node_id: String, + /// Number of data items processed by this node. + pub items_processed: u64, + /// Error message if the node failed, or `None` on success. + pub error: Option, +} + +/// Aggregate outcome of executing an entire pipeline graph. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(schemars::JsonSchema)] +pub struct RunResult { + /// Unique identifier for this execution run. + pub run_id: Uuid, + /// Per-node results in completion order. + pub node_results: Vec, + /// `true` if all nodes completed without error. + pub success: bool, +} + +/// Executes a compiled [`ExecutionPlan`] by spawning concurrent tasks for each node. +/// +/// Returns a [`RunResult`] containing per-node outcomes and an overall success flag. +pub async fn run_graph( + plan: &ExecutionPlan, + _connections: &Connections, +) -> Result { + let run_id = Uuid::new_v4(); + + // Create channels for each edge + let mut senders: HashMap>> = HashMap::new(); + let mut receivers: HashMap>> = HashMap::new(); + + for node in &plan.nodes { + let node_id = node.node.id(); + for downstream_id in &node.downstream_ids { + let (tx, rx) = mpsc::channel(CHANNEL_BUFFER_SIZE); + senders.entry(node_id.to_string()).or_default().push(tx); + receivers.entry(downstream_id.clone()).or_default().push(rx); + } + } + + // Create completion signals per node + let mut signal_senders: HashMap> = HashMap::new(); + let mut signal_receivers: HashMap> = HashMap::new(); + + for node in &plan.nodes { + let (tx, rx) = watch::channel(false); + signal_senders.insert(node.node.id().to_string(), tx); + signal_receivers.insert(node.node.id().to_string(), rx); + } + + // Spawn tasks + let mut join_set: JoinSet = JoinSet::new(); + + for resolved in &plan.nodes { + let node = resolved.node.clone(); + let node_id = node.id().to_string(); + let upstream_ids = resolved.upstream_ids.clone(); + + // Collect upstream watch receivers + let upstream_watches: Vec> = upstream_ids + .iter() + .filter_map(|id| signal_receivers.get(id).cloned()) + .collect(); + + let completion_tx = signal_senders.remove(&node_id); + let node_senders = senders.remove(&node_id).unwrap_or_default(); + let node_receivers = receivers.remove(&node_id).unwrap_or_default(); + + join_set.spawn(async move { + // Wait for upstream nodes to complete + for mut rx in upstream_watches { + let _ = rx.wait_for(|&done| done).await; + } + + let result = execute_node(&node, node_senders, node_receivers).await; + + // Signal completion + if let Some(tx) = completion_tx { + let _ = tx.send(true); + } + + match result { + Ok(count) => NodeResult { + node_id, + items_processed: count, + error: None, + }, + Err(e) => NodeResult { + node_id, + items_processed: 0, + error: Some(e.to_string()), + }, + } + }); + } + + // Collect results + let mut node_results = Vec::new(); + while let Some(result) = join_set.join_next().await { + match result { + Ok(nr) => node_results.push(nr), + Err(e) => node_results.push(NodeResult { + node_id: "unknown".to_string(), + items_processed: 0, + error: Some(format!("Task panicked: {}", e)), + }), + } + } + + let success = node_results.iter().all(|r| r.error.is_none()); + + Ok(RunResult { + run_id, + node_results, + success, + }) +} + +/// Execute a single node with its channels (simplified -- does not use registry directly). +async fn execute_node( + _node: &GraphNode, + senders: Vec>, + mut receivers: Vec>, +) -> Result { + // For now, forward items from receivers to senders (passthrough behavior). + // The actual registry-based dispatch happens via the Engine wrapper. + let mut count = 0u64; + + for rx in &mut receivers { + while let Some(item) = rx.recv().await { + count += 1; + for tx in &senders { + let _ = tx.send(item.clone()).await; + } + } + } + + // Drop senders to signal downstream completion + drop(senders); + + Ok(count) +} diff --git a/crates/nvisy-engine/src/lib.rs b/crates/nvisy-engine/src/lib.rs new file mode 100644 index 0000000..b3e6edc --- /dev/null +++ b/crates/nvisy-engine/src/lib.rs @@ -0,0 +1,19 @@ +//! DAG execution engine for nvisy pipelines. +//! +//! This crate compiles pipeline definitions into directed acyclic graphs (DAGs), +//! plans topologically-ordered execution, and runs nodes concurrently with +//! retry and timeout policies. + +#![forbid(unsafe_code)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![doc = include_str!("../README.md")] + +pub mod compiler; +pub mod connections; +pub mod engine; +pub mod executor; +pub mod policies; +pub mod runs; + +#[doc(hidden)] +pub mod prelude; diff --git a/crates/nvisy-engine/src/policies/mod.rs b/crates/nvisy-engine/src/policies/mod.rs new file mode 100644 index 0000000..eed231b --- /dev/null +++ b/crates/nvisy-engine/src/policies/mod.rs @@ -0,0 +1,76 @@ +//! Retry and timeout policies for pipeline execution. +//! +//! Provides [`compute_delay`] for backoff calculation, [`with_retry`] for +//! automatic retry of fallible futures, and [`with_timeout`] for deadline +//! enforcement. + +use std::time::Duration; +use tokio::time; +use nvisy_core::error::Error; +pub mod retry; + +use crate::policies::retry::{BackoffStrategy, RetryPolicy}; + +/// Computes the sleep duration before a retry attempt based on the policy's +/// [`BackoffStrategy`] and the zero-based attempt number. +pub fn compute_delay(policy: &RetryPolicy, attempt: u32) -> Duration { + let base = Duration::from_millis(policy.delay_ms); + match policy.backoff { + BackoffStrategy::Fixed => base, + BackoffStrategy::Exponential => base * 2u32.saturating_pow(attempt), + BackoffStrategy::Jitter => { + let exp = base * 2u32.saturating_pow(attempt); + let jitter_range = exp.as_millis() as u64 + 1; + let jitter = Duration::from_millis(rand::random_range(0..jitter_range)); + exp + jitter + } + } +} + +/// Executes a fallible async closure with automatic retry according to the +/// given [`RetryPolicy`]. +/// +/// The closure is invoked up to `max_retries + 1` times. Non-retryable errors +/// (as determined by [`Error::is_retryable`]) are returned immediately. +pub async fn with_retry( + policy: &RetryPolicy, + mut f: F, +) -> Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut last_err = None; + for attempt in 0..=policy.max_retries { + match f().await { + Ok(v) => return Ok(v), + Err(e) => { + if !e.is_retryable() || attempt == policy.max_retries { + return Err(e); + } + last_err = Some(e); + let delay = compute_delay(policy, attempt); + time::sleep(delay).await; + } + } + } + Err(last_err.unwrap_or_else(|| Error::runtime("Retry exhausted", "policies", false))) +} + +/// Wraps a future with a deadline, returning an [`Error::timeout`] if it +/// does not complete within `timeout_ms` milliseconds. +pub async fn with_timeout( + timeout_ms: u64, + f: F, +) -> Result +where + F: std::future::Future>, +{ + match time::timeout(Duration::from_millis(timeout_ms), f).await { + Ok(result) => result, + Err(_) => Err(Error::timeout(format!( + "Operation timed out after {}ms", + timeout_ms + ))), + } +} diff --git a/crates/nvisy-engine/src/policies/retry.rs b/crates/nvisy-engine/src/policies/retry.rs new file mode 100644 index 0000000..859d6fe --- /dev/null +++ b/crates/nvisy-engine/src/policies/retry.rs @@ -0,0 +1,52 @@ +//! Retry policy types and backoff strategies. +//! +//! [`RetryPolicy`] configures how many times a failed node should be retried, +//! the base delay between attempts, and the [`BackoffStrategy`] to use. + +use serde::{Deserialize, Serialize}; + +/// Retry policy attached to a pipeline node. +/// +/// Defaults to 3 retries with a 1 000 ms fixed delay. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(schemars::JsonSchema)] +pub struct RetryPolicy { + /// Maximum number of retry attempts after the initial failure. + #[serde(default = "default_max_retries")] + pub max_retries: u32, + /// Base delay in milliseconds between retry attempts. + #[serde(default = "default_delay_ms")] + pub delay_ms: u64, + /// Strategy used to compute the delay between successive retries. + #[serde(default)] + pub backoff: BackoffStrategy, +} + +/// Returns the default maximum retry count (3). +fn default_max_retries() -> u32 { 3 } +/// Returns the default base delay in milliseconds (1 000). +fn default_delay_ms() -> u64 { 1000 } + +impl Default for RetryPolicy { + fn default() -> Self { + Self { + max_retries: 3, + delay_ms: 1000, + backoff: BackoffStrategy::default(), + } + } +} + +/// Strategy for computing the delay between retry attempts. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[derive(schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum BackoffStrategy { + /// Constant delay equal to `delay_ms` on every attempt. + #[default] + Fixed, + /// Delay doubles with each attempt: `delay_ms * 2^attempt`. + Exponential, + /// Exponential backoff with an added random jitter to prevent thundering herd. + Jitter, +} diff --git a/crates/nvisy-engine/src/prelude.rs b/crates/nvisy-engine/src/prelude.rs new file mode 100644 index 0000000..eddd8f5 --- /dev/null +++ b/crates/nvisy-engine/src/prelude.rs @@ -0,0 +1,6 @@ +//! Convenience re-exports. +pub use crate::compiler::graph::{Graph, GraphEdge, GraphNode}; +pub use crate::compiler::plan::{build_plan, ExecutionPlan, ResolvedNode}; +pub use crate::engine::{Engine, EngineInput, EngineOutput}; +pub use crate::executor::runner::{run_graph, RunResult}; +pub use crate::runs::{RunManager, RunState, RunStatus, RunSummary}; diff --git a/crates/nvisy-engine/src/runs/mod.rs b/crates/nvisy-engine/src/runs/mod.rs new file mode 100644 index 0000000..da290e5 --- /dev/null +++ b/crates/nvisy-engine/src/runs/mod.rs @@ -0,0 +1,208 @@ +//! Pipeline run lifecycle management. +//! +//! Tracks the status of every pipeline execution from creation through +//! completion or cancellation. Provides [`RunManager`] for concurrent +//! read/write access to run state. + +use std::collections::HashMap; +use std::sync::Arc; +use jiff::Timestamp; +use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; +use crate::executor::runner::RunResult; + +/// Lifecycle status of a pipeline run. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunStatus { + /// The run has been created but not yet started. + Pending, + /// The run is actively executing nodes. + Running, + /// All nodes completed without error. + Success, + /// Some nodes succeeded while others failed. + PartialFailure, + /// All nodes failed. + Failure, + /// The run was cancelled by the caller. + Cancelled, +} + +/// Execution progress of a single node within a run. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(schemars::JsonSchema)] +pub struct NodeProgress { + /// ID of the node this progress belongs to. + pub node_id: String, + /// Current status of this node. + pub status: RunStatus, + /// Number of data items processed so far. + pub items_processed: u64, + /// Error message if the node failed. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +/// Complete mutable state of a pipeline run. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(schemars::JsonSchema)] +pub struct RunState { + /// Unique run identifier. + pub id: Uuid, + /// Current overall status. + pub status: RunStatus, + /// Timestamp when the run was created. + #[schemars(with = "String")] + pub created_at: Timestamp, + /// Timestamp when the run finished, if applicable. + #[serde(skip_serializing_if = "Option::is_none")] + #[schemars(with = "Option")] + pub completed_at: Option, + /// Per-node progress keyed by node ID. + pub node_progress: HashMap, + /// Final result after the run completes. + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option, +} + +/// Lightweight summary of a run for listing endpoints. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(schemars::JsonSchema)] +pub struct RunSummary { + /// Unique run identifier. + pub id: Uuid, + /// Current overall status. + pub status: RunStatus, + /// Timestamp when the run was created. + #[schemars(with = "String")] + pub created_at: Timestamp, + /// Timestamp when the run finished, if applicable. + #[serde(skip_serializing_if = "Option::is_none")] + #[schemars(with = "Option")] + pub completed_at: Option, +} + +/// Thread-safe manager that tracks all pipeline runs. +/// +/// Internally uses [`RwLock`]-protected maps so multiple readers can inspect +/// run state concurrently while writes are serialized. +pub struct RunManager { + /// All known runs keyed by their UUID. + runs: Arc>>, + /// Cancellation tokens for runs that are still in progress. + cancel_tokens: Arc>>, +} + +impl RunManager { + /// Creates a new, empty run manager. + pub fn new() -> Self { + Self { + runs: Arc::new(RwLock::new(HashMap::new())), + cancel_tokens: Arc::new(RwLock::new(HashMap::new())), + } + } + + /// Create a new pending run and return its ID and cancellation token. + pub async fn create_run(&self) -> (Uuid, CancellationToken) { + let id = Uuid::new_v4(); + let token = CancellationToken::new(); + + let state = RunState { + id, + status: RunStatus::Pending, + created_at: Timestamp::now(), + completed_at: None, + node_progress: HashMap::new(), + result: None, + }; + + self.runs.write().await.insert(id, state); + self.cancel_tokens.write().await.insert(id, token.clone()); + + (id, token) + } + + /// Update a run to running status. + pub async fn set_running(&self, id: Uuid) { + if let Some(state) = self.runs.write().await.get_mut(&id) { + state.status = RunStatus::Running; + } + } + + /// Complete a run with a result. + pub async fn complete_run(&self, id: Uuid, result: RunResult) { + if let Some(state) = self.runs.write().await.get_mut(&id) { + state.status = if result.success { + RunStatus::Success + } else if result.node_results.iter().any(|r| r.error.is_none()) { + RunStatus::PartialFailure + } else { + RunStatus::Failure + }; + state.completed_at = Some(Timestamp::now()); + + for nr in &result.node_results { + state.node_progress.insert( + nr.node_id.clone(), + NodeProgress { + node_id: nr.node_id.clone(), + status: if nr.error.is_none() { + RunStatus::Success + } else { + RunStatus::Failure + }, + items_processed: nr.items_processed, + error: nr.error.clone(), + }, + ); + } + + state.result = Some(result); + } + self.cancel_tokens.write().await.remove(&id); + } + + /// Get the current state of a run. + pub async fn get(&self, id: Uuid) -> Option { + self.runs.read().await.get(&id).cloned() + } + + /// List all runs, optionally filtered by status. + pub async fn list(&self, status: Option) -> Vec { + self.runs + .read() + .await + .values() + .filter(|s| status.is_none_or(|st| s.status == st)) + .map(|s| RunSummary { + id: s.id, + status: s.status, + created_at: s.created_at, + completed_at: s.completed_at, + }) + .collect() + } + + /// Cancel a running or pending run. Returns false if not found or already finished. + pub async fn cancel(&self, id: Uuid) -> bool { + if let Some(token) = self.cancel_tokens.read().await.get(&id) { + token.cancel(); + if let Some(state) = self.runs.write().await.get_mut(&id) { + state.status = RunStatus::Cancelled; + state.completed_at = Some(Timestamp::now()); + } + true + } else { + false + } + } +} + +impl Default for RunManager { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/nvisy-ingest/Cargo.toml b/crates/nvisy-ingest/Cargo.toml new file mode 100644 index 0000000..d367962 --- /dev/null +++ b/crates/nvisy-ingest/Cargo.toml @@ -0,0 +1,80 @@ +# https://doc.rust-lang.org/cargo/reference/manifest.html + +[package] +name = "nvisy-ingest" +description = "File-format loaders and unified Document type for the Nvisy multimodal redaction platform" +keywords = ["nvisy", "ingest", "loader", "pdf", "docx"] +categories = ["parser-implementations"] + +version = { workspace = true } +rust-version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } + +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[features] +default = ["pdf", "docx", "html", "xlsx", "image", "wav", "mp3"] +# PDF parsing and text extraction via pdf-extract + lopdf; enables png for extracted images +pdf = ["dep:pdf-extract", "dep:lopdf", "png"] +# Microsoft Word (.docx) parsing via zip + quick-xml; enables image formats for extracted images +docx = ["dep:zip", "dep:quick-xml", "jpeg", "png"] +# HTML parsing and text extraction via scraper +html = ["dep:scraper"] +# Excel (.xlsx) spreadsheet parsing via calamine +xlsx = ["dep:calamine"] +# Convenience alias: all image formats +image = ["jpeg", "png"] +# Individual image format handlers (each requires dep:image) +jpeg = ["dep:image"] +png = ["dep:image"] +# Audio format handlers (no additional dependencies) +wav = [] +mp3 = [] + +[dependencies] +# Internal crates +nvisy-core = { workspace = true, features = [] } +nvisy-ontology = { workspace = true, features = [] } + +# (De)serialization +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = [] } + +# Text parsing +csv = { workspace = true, features = [] } + +# Async runtime +tokio = { workspace = true, features = ["sync"] } +async-trait = { workspace = true, features = [] } +futures = { workspace = true, features = [] } + +# Primitive datatypes +uuid = { workspace = true, features = ["v4"] } +bytes = { workspace = true, features = [] } + +# Observability +tracing = { workspace = true, features = [] } + +# File type detection +infer = { workspace = true, features = [] } + +# Document parsing (feature-gated) +pdf-extract = { workspace = true, optional = true, features = [] } +lopdf = { workspace = true, optional = true, features = [] } +zip = { workspace = true, optional = true, features = [] } +quick-xml = { workspace = true, optional = true, features = [] } +scraper = { workspace = true, optional = true, features = [] } +calamine = { workspace = true, optional = true, features = [] } +image = { workspace = true, optional = true, features = [] } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/crates/nvisy-ingest/README.md b/crates/nvisy-ingest/README.md new file mode 100644 index 0000000..378ce7b --- /dev/null +++ b/crates/nvisy-ingest/README.md @@ -0,0 +1,9 @@ +# nvisy-ingest + +File-format loaders for the Nvisy multimodal redaction platform. + +This crate provides loaders for PDF, DOCX, HTML, Image, XLSX, Audio, +CSV, JSON, and plain-text files. Each loader implements the +[`Loader`](crate::loaders::Loader) trait and converts raw +blob bytes into structured `Document`, `ImageData`, or `TabularData` +artifacts. diff --git a/crates/nvisy-ingest/src/document/edit_stream.rs b/crates/nvisy-ingest/src/document/edit_stream.rs new file mode 100644 index 0000000..c520da1 --- /dev/null +++ b/crates/nvisy-ingest/src/document/edit_stream.rs @@ -0,0 +1,43 @@ +//! Async span edit stream for [`Handler::edit_spans`]. +//! +//! [`Handler::edit_spans`]: crate::handler::Handler::edit_spans + +use std::pin::Pin; +use std::task::{Context, Poll}; + +use futures::Stream; + +use crate::handler::span::SpanEdit; + +/// Async stream of edits consumed by [`Handler::edit_spans`]. +/// +/// Wraps a `Pin>` so that callers can pass any +/// iterator/stream of edits without exposing a concrete type. +/// +/// [`Handler::edit_spans`]: crate::handler::Handler::edit_spans +pub struct SpanEditStream<'a, Id, Data> { + inner: Pin> + Send + 'a>>, +} + +impl<'a, Id, Data> SpanEditStream<'a, Id, Data> { + /// Wrap any `Send` stream of span edits. + pub fn new(stream: impl Stream> + Send + 'a) -> Self { + Self { + inner: Box::pin(stream), + } + } +} + +impl Unpin for SpanEditStream<'_, Id, Data> {} + +impl Stream for SpanEditStream<'_, Id, Data> { + type Item = SpanEdit; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.inner.as_mut().poll_next(cx) + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} diff --git a/crates/nvisy-ingest/src/document/mod.rs b/crates/nvisy-ingest/src/document/mod.rs new file mode 100644 index 0000000..c74fb3e --- /dev/null +++ b/crates/nvisy-ingest/src/document/mod.rs @@ -0,0 +1,63 @@ +//! Unified document representation. + +pub mod view_stream; +pub mod edit_stream; + +use nvisy_core::io::ContentData; +use nvisy_core::path::ContentSource; +use nvisy_ontology::entity::DocumentType; + +use crate::handler::Handler; + +/// A unified representation of any content that can be handled by the pipeline. +/// +/// `Document` is generic over `H`, a [`Handler`] that holds the loaded data +/// and provides methods to read and manipulate it. +#[derive(Debug)] +pub struct Document { + /// Content source identity and lineage. + pub source: ContentSource, + + /// Format handler (holds the loaded data). + handler: H, +} + +impl Clone for Document { + fn clone(&self) -> Self { + Self { + source: self.source, + handler: self.handler.clone(), + } + } +} + +impl Document { + /// Create a new document with the given handler. + pub fn new(handler: H) -> Self { + Self { + source: ContentSource::new(), + handler, + } + } + + /// Get a reference to the format handler. + pub fn handler(&self) -> &H { + &self.handler + } + + /// Get a mutable reference to the format handler. + pub fn handler_mut(&mut self) -> &mut H { + &mut self.handler + } + + /// The document type of the loaded content. + pub fn document_type(&self) -> DocumentType { + self.handler.document_type() + } + + /// Set this document's parent to the given content source. + pub fn with_parent(mut self, content: &ContentData) -> Self { + self.source.set_parent_id(Some(content.content_source.as_uuid())); + self + } +} diff --git a/crates/nvisy-ingest/src/document/view_stream.rs b/crates/nvisy-ingest/src/document/view_stream.rs new file mode 100644 index 0000000..c17e52d --- /dev/null +++ b/crates/nvisy-ingest/src/document/view_stream.rs @@ -0,0 +1,43 @@ +//! Async span stream for [`Handler::view_spans`]. +//! +//! [`Handler::view_spans`]: crate::handler::Handler::view_spans + +use std::pin::Pin; +use std::task::{Context, Poll}; + +use futures::Stream; + +use crate::handler::span::Span; + +/// Async stream of spans returned by [`Handler::view_spans`]. +/// +/// Wraps a `Pin>` so that handler implementations +/// can return any iterator/stream without exposing a concrete type. +/// +/// [`Handler::view_spans`]: crate::handler::Handler::view_spans +pub struct SpanStream<'a, Id, Data> { + inner: Pin> + Send + 'a>>, +} + +impl<'a, Id, Data> SpanStream<'a, Id, Data> { + /// Wrap any `Send` stream of spans. + pub fn new(stream: impl Stream> + Send + 'a) -> Self { + Self { + inner: Box::pin(stream), + } + } +} + +impl Unpin for SpanStream<'_, Id, Data> {} + +impl Stream for SpanStream<'_, Id, Data> { + type Item = Span; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.inner.as_mut().poll_next(cx) + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} diff --git a/crates/nvisy-ingest/src/handler/audio/mod.rs b/crates/nvisy-ingest/src/handler/audio/mod.rs new file mode 100644 index 0000000..5a1fb22 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/audio/mod.rs @@ -0,0 +1,6 @@ +//! Audio format handlers. + +#[cfg(feature = "wav")] +pub mod wav; +#[cfg(feature = "mp3")] +pub mod mp3; diff --git a/crates/nvisy-ingest/src/handler/audio/mp3.rs b/crates/nvisy-ingest/src/handler/audio/mp3.rs new file mode 100644 index 0000000..5e0a962 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/audio/mp3.rs @@ -0,0 +1,46 @@ +//! MP3 handler (stub — awaiting migration to Loader/Handler pattern). + +use bytes::Bytes; + +use nvisy_core::error::Error; +use nvisy_ontology::entity::DocumentType; + +use crate::document::edit_stream::SpanEditStream; +use crate::document::view_stream::SpanStream; +use crate::handler::Handler; + +#[derive(Debug, Clone)] +pub struct Mp3Handler { + pub(crate) bytes: Bytes, +} + +impl Mp3Handler { + pub fn new(bytes: Bytes) -> Self { + Self { bytes } + } + + pub fn bytes(&self) -> &Bytes { + &self.bytes + } +} + +#[async_trait::async_trait] +impl Handler for Mp3Handler { + fn document_type(&self) -> DocumentType { + DocumentType::Mp3 + } + + type SpanId = (); + type SpanData = (); + + async fn view_spans(&self) -> SpanStream<'_, (), ()> { + SpanStream::new(futures::stream::empty()) + } + + async fn edit_spans( + &mut self, + _edits: SpanEditStream<'_, (), ()>, + ) -> Result<(), Error> { + Ok(()) + } +} diff --git a/crates/nvisy-ingest/src/handler/audio/wav.rs b/crates/nvisy-ingest/src/handler/audio/wav.rs new file mode 100644 index 0000000..c8cb4e4 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/audio/wav.rs @@ -0,0 +1,46 @@ +//! WAV handler (stub — awaiting migration to Loader/Handler pattern). + +use bytes::Bytes; + +use nvisy_core::error::Error; +use nvisy_ontology::entity::DocumentType; + +use crate::document::edit_stream::SpanEditStream; +use crate::document::view_stream::SpanStream; +use crate::handler::Handler; + +#[derive(Debug, Clone)] +pub struct WavHandler { + pub(crate) bytes: Bytes, +} + +impl WavHandler { + pub fn new(bytes: Bytes) -> Self { + Self { bytes } + } + + pub fn bytes(&self) -> &Bytes { + &self.bytes + } +} + +#[async_trait::async_trait] +impl Handler for WavHandler { + fn document_type(&self) -> DocumentType { + DocumentType::Wav + } + + type SpanId = (); + type SpanData = (); + + async fn view_spans(&self) -> SpanStream<'_, (), ()> { + SpanStream::new(futures::stream::empty()) + } + + async fn edit_spans( + &mut self, + _edits: SpanEditStream<'_, (), ()>, + ) -> Result<(), Error> { + Ok(()) + } +} diff --git a/crates/nvisy-ingest/src/handler/document/docx.rs b/crates/nvisy-ingest/src/handler/document/docx.rs new file mode 100644 index 0000000..90b7d9e --- /dev/null +++ b/crates/nvisy-ingest/src/handler/document/docx.rs @@ -0,0 +1,32 @@ +//! DOCX handler (stub — awaiting migration to Loader/Handler pattern). + +use nvisy_core::error::Error; +use nvisy_ontology::entity::DocumentType; + +use crate::document::edit_stream::SpanEditStream; +use crate::document::view_stream::SpanStream; +use crate::handler::Handler; + +#[derive(Debug)] +pub struct DocxHandler; + +#[async_trait::async_trait] +impl Handler for DocxHandler { + fn document_type(&self) -> DocumentType { + DocumentType::Docx + } + + type SpanId = (); + type SpanData = (); + + async fn view_spans(&self) -> SpanStream<'_, (), ()> { + SpanStream::new(futures::stream::empty()) + } + + async fn edit_spans( + &mut self, + _edits: SpanEditStream<'_, (), ()>, + ) -> Result<(), Error> { + Ok(()) + } +} diff --git a/crates/nvisy-ingest/src/handler/document/mod.rs b/crates/nvisy-ingest/src/handler/document/mod.rs new file mode 100644 index 0000000..9c62d32 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/document/mod.rs @@ -0,0 +1,6 @@ +//! Rich document format handlers. + +#[cfg(feature = "pdf")] +pub mod pdf; +#[cfg(feature = "docx")] +pub mod docx; diff --git a/crates/nvisy-ingest/src/handler/document/pdf.rs b/crates/nvisy-ingest/src/handler/document/pdf.rs new file mode 100644 index 0000000..4c8ac68 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/document/pdf.rs @@ -0,0 +1,32 @@ +//! PDF handler (stub — awaiting migration to Loader/Handler pattern). + +use nvisy_core::error::Error; +use nvisy_ontology::entity::DocumentType; + +use crate::document::edit_stream::SpanEditStream; +use crate::document::view_stream::SpanStream; +use crate::handler::Handler; + +#[derive(Debug)] +pub struct PdfHandler; + +#[async_trait::async_trait] +impl Handler for PdfHandler { + fn document_type(&self) -> DocumentType { + DocumentType::Pdf + } + + type SpanId = (); + type SpanData = (); + + async fn view_spans(&self) -> SpanStream<'_, (), ()> { + SpanStream::new(futures::stream::empty()) + } + + async fn edit_spans( + &mut self, + _edits: SpanEditStream<'_, (), ()>, + ) -> Result<(), Error> { + Ok(()) + } +} diff --git a/crates/nvisy-ingest/src/handler/encoding.rs b/crates/nvisy-ingest/src/handler/encoding.rs new file mode 100644 index 0000000..c90d92e --- /dev/null +++ b/crates/nvisy-ingest/src/handler/encoding.rs @@ -0,0 +1,25 @@ +//! Character encoding for text-based loaders. + +use nvisy_core::error::Error; + +/// Character encoding used to decode raw bytes before parsing. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TextEncoding { + /// UTF-8 (the default and by far the most common encoding). + #[default] + Utf8, +} + +impl TextEncoding { + /// Decode raw bytes to a UTF-8 string. + /// + /// `origin` identifies the caller for error messages + /// (e.g. `"json-loader"`). + pub fn decode_bytes(self, bytes: &[u8], origin: &str) -> Result { + match self { + Self::Utf8 => String::from_utf8(bytes.to_vec()).map_err(|e| { + Error::validation(format!("Invalid UTF-8: {e}"), origin) + }), + } + } +} diff --git a/crates/nvisy-ingest/src/handler/image/jpeg.rs b/crates/nvisy-ingest/src/handler/image/jpeg.rs new file mode 100644 index 0000000..9b56f21 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/image/jpeg.rs @@ -0,0 +1,32 @@ +//! JPEG handler (stub — awaiting migration to Loader/Handler pattern). + +use nvisy_core::error::Error; +use nvisy_ontology::entity::DocumentType; + +use crate::document::edit_stream::SpanEditStream; +use crate::document::view_stream::SpanStream; +use crate::handler::Handler; + +#[derive(Debug)] +pub struct JpegHandler; + +#[async_trait::async_trait] +impl Handler for JpegHandler { + fn document_type(&self) -> DocumentType { + DocumentType::Jpeg + } + + type SpanId = (); + type SpanData = (); + + async fn view_spans(&self) -> SpanStream<'_, (), ()> { + SpanStream::new(futures::stream::empty()) + } + + async fn edit_spans( + &mut self, + _edits: SpanEditStream<'_, (), ()>, + ) -> Result<(), Error> { + Ok(()) + } +} diff --git a/crates/nvisy-ingest/src/handler/image/mod.rs b/crates/nvisy-ingest/src/handler/image/mod.rs new file mode 100644 index 0000000..46bcfdd --- /dev/null +++ b/crates/nvisy-ingest/src/handler/image/mod.rs @@ -0,0 +1,6 @@ +//! Image format handlers and shared decode helper. + +#[cfg(feature = "jpeg")] +pub mod jpeg; +#[cfg(feature = "png")] +pub mod png; diff --git a/crates/nvisy-ingest/src/handler/image/png.rs b/crates/nvisy-ingest/src/handler/image/png.rs new file mode 100644 index 0000000..a761862 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/image/png.rs @@ -0,0 +1,46 @@ +//! PNG handler (stub — awaiting migration to Loader/Handler pattern). + +use bytes::Bytes; + +use nvisy_core::error::Error; +use nvisy_ontology::entity::DocumentType; + +use crate::document::edit_stream::SpanEditStream; +use crate::document::view_stream::SpanStream; +use crate::handler::Handler; + +#[derive(Debug, Clone)] +pub struct PngHandler { + pub(crate) bytes: Bytes, +} + +impl PngHandler { + pub fn new(bytes: Bytes) -> Self { + Self { bytes } + } + + pub fn bytes(&self) -> &Bytes { + &self.bytes + } +} + +#[async_trait::async_trait] +impl Handler for PngHandler { + fn document_type(&self) -> DocumentType { + DocumentType::Png + } + + type SpanId = (); + type SpanData = (); + + async fn view_spans(&self) -> SpanStream<'_, (), ()> { + SpanStream::new(futures::stream::empty()) + } + + async fn edit_spans( + &mut self, + _edits: SpanEditStream<'_, (), ()>, + ) -> Result<(), Error> { + Ok(()) + } +} diff --git a/crates/nvisy-ingest/src/handler/mod.rs b/crates/nvisy-ingest/src/handler/mod.rs new file mode 100644 index 0000000..13fea3b --- /dev/null +++ b/crates/nvisy-ingest/src/handler/mod.rs @@ -0,0 +1,95 @@ +//! Loader and handler traits. +//! +//! A [`Loader`] validates and parses raw content, producing a +//! [`Document`] containing the corresponding [`Handler`]. The handler +//! holds the loaded data and provides methods to read and manipulate it. +//! +//! Each handler defines its own span types and exposes them as async +//! streams via [`Handler::view_spans`] and [`Handler::edit_spans`]. + +use nvisy_core::error::Error; +use nvisy_core::io::ContentData; +use nvisy_ontology::entity::DocumentType; + +use crate::document::edit_stream::SpanEditStream; +use crate::document::view_stream::SpanStream; +use crate::document::Document; + +pub mod encoding; +pub mod span; + +pub mod text; +pub mod document; +pub mod image; +pub mod tabular; +pub mod audio; + +pub use encoding::TextEncoding; +pub use span::{Span, SpanEdit}; + +pub use text::txt_handler::{TxtData, TxtHandler, TxtSpan}; +pub use text::txt_loader::{TxtLoader, TxtParams}; +pub use text::csv_handler::{CsvData, CsvHandler, CsvSpan}; +pub use text::csv_loader::{CsvLoader, CsvParams}; +pub use text::json_handler::{ + JsonData, JsonHandler, JsonIndent, JsonPath, +}; +pub use text::json_loader::{JsonParams, JsonLoader}; + +#[cfg(feature = "png")] +pub use image::png::PngHandler; + +#[cfg(feature = "wav")] +pub use audio::wav::WavHandler; +#[cfg(feature = "mp3")] +pub use audio::mp3::Mp3Handler; + +/// Trait implemented by all format handlers. +/// +/// A handler holds loaded, validated content and provides methods to +/// read and manipulate it. Handlers are produced by their corresponding +/// [`Loader`]. +/// +/// Each handler defines its own span addressing scheme ([`SpanId`](Self::SpanId)) +/// and data type ([`SpanData`](Self::SpanData)). Pipeline actions +/// constrain `SpanData` to express what they need (e.g. `AsRef` +/// for text scanning). +#[async_trait::async_trait] +pub trait Handler: Send + Sync + 'static { + /// The document type this handler represents. + fn document_type(&self) -> DocumentType; + + /// Strongly-typed identifier for a span within this handler. + type SpanId: Send + Sync + Clone + 'static; + /// The data type carried by each span. + type SpanData: Send + 'static; + + /// Return the loaded content as an async stream of spans. + async fn view_spans(&self) -> SpanStream<'_, Self::SpanId, Self::SpanData>; + + /// Apply edits from an async stream back to the source structure. + async fn edit_spans( + &mut self, + edits: SpanEditStream<'_, Self::SpanId, Self::SpanData>, + ) -> Result<(), Error>; +} + +/// Trait implemented by format loaders. +/// +/// A loader validates and parses raw content, producing a +/// [`Document`] with the corresponding handler. +#[async_trait::async_trait] +pub trait Loader: Send + Sync + 'static { + /// The handler type this loader produces. + type Handler: Handler; + /// Strongly-typed parameters for loading. + type Params: Send; + + /// Validate and parse the content, returning a document with + /// the loaded handler. + async fn load( + &self, + content: &ContentData, + params: &Self::Params, + ) -> Result>, Error>; +} diff --git a/crates/nvisy-ingest/src/handler/span.rs b/crates/nvisy-ingest/src/handler/span.rs new file mode 100644 index 0000000..4bbd276 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/span.rs @@ -0,0 +1,19 @@ +//! Span types for content traversal and editing. + +/// A span of content tagged with its origin in the source structure. +#[derive(Debug, Clone)] +pub struct Span { + /// Identifier locating this span within the handler's data model. + pub id: Id, + /// The content of this span. + pub data: Data, +} + +/// An edit to apply to a specific span. +#[derive(Debug, Clone)] +pub struct SpanEdit { + /// Which span to edit (must match a `Span::id`). + pub id: Id, + /// Replacement data for this span. + pub data: Data, +} diff --git a/crates/nvisy-ingest/src/handler/tabular/mod.rs b/crates/nvisy-ingest/src/handler/tabular/mod.rs new file mode 100644 index 0000000..bb7cea7 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/tabular/mod.rs @@ -0,0 +1,4 @@ +//! Tabular/spreadsheet format handlers. + +#[cfg(feature = "xlsx")] +pub mod xlsx; diff --git a/crates/nvisy-ingest/src/handler/tabular/xlsx.rs b/crates/nvisy-ingest/src/handler/tabular/xlsx.rs new file mode 100644 index 0000000..c4c3bad --- /dev/null +++ b/crates/nvisy-ingest/src/handler/tabular/xlsx.rs @@ -0,0 +1,32 @@ +//! XLSX handler (stub — awaiting migration to Loader/Handler pattern). + +use nvisy_core::error::Error; +use nvisy_ontology::entity::DocumentType; + +use crate::document::edit_stream::SpanEditStream; +use crate::document::view_stream::SpanStream; +use crate::handler::Handler; + +#[derive(Debug)] +pub struct XlsxHandler; + +#[async_trait::async_trait] +impl Handler for XlsxHandler { + fn document_type(&self) -> DocumentType { + DocumentType::Xlsx + } + + type SpanId = (); + type SpanData = (); + + async fn view_spans(&self) -> SpanStream<'_, (), ()> { + SpanStream::new(futures::stream::empty()) + } + + async fn edit_spans( + &mut self, + _edits: SpanEditStream<'_, (), ()>, + ) -> Result<(), Error> { + Ok(()) + } +} diff --git a/crates/nvisy-ingest/src/handler/text/csv_handler.rs b/crates/nvisy-ingest/src/handler/text/csv_handler.rs new file mode 100644 index 0000000..f05416f --- /dev/null +++ b/crates/nvisy-ingest/src/handler/text/csv_handler.rs @@ -0,0 +1,408 @@ +//! CSV handler — holds parsed CSV content and provides span-based +//! access via [`Handler`]. +//! +//! The handler stores the parsed rows (and optional headers) together +//! with the detected delimiter so the file can be reconstructed after +//! edits. +//! +//! # Span model +//! +//! [`Handler::view_spans`] yields one [`Span`] per cell. If headers +//! are present, header cells are emitted first (with +//! [`CsvSpan::header`] set to `true`), followed by data cells in +//! row-major order. +//! +//! [`Handler::edit_spans`] replaces cell content at the given +//! (row, col) position. Header cells can also be edited. + +use futures::StreamExt; + +use nvisy_core::error::Error; +use nvisy_ontology::entity::DocumentType; + +use crate::document::edit_stream::SpanEditStream; +use crate::document::view_stream::SpanStream; +use crate::handler::{Handler, Span}; + +/// Cell address within a CSV document. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CsvSpan { + /// 0-based row index (within data rows, not counting the header). + pub row: usize, + /// 0-based column index. + pub col: usize, + /// `true` when this span addresses a header cell rather than a + /// data cell. + pub header: bool, + /// Column name (from the header row) or column index as a string + /// when no headers are present. + pub key: String, +} + +impl CsvSpan { + /// Address a data cell with a column key. + pub fn cell(row: usize, col: usize, key: impl Into) -> Self { + Self { + row, + col, + header: false, + key: key.into(), + } + } + + /// Address a header cell. + pub fn header_cell(col: usize, key: impl Into) -> Self { + Self { + row: 0, + col, + header: true, + key: key.into(), + } + } +} + +/// Parsed CSV content. +#[derive(Debug, Clone)] +pub struct CsvData { + /// Column headers, if present. + pub headers: Option>, + /// Data rows (excluding the header row). + pub rows: Vec>, + /// Field delimiter byte (e.g. `b','`, `b'\t'`, `b';'`). + pub delimiter: u8, + /// Whether the original source had a trailing newline. + pub trailing_newline: bool, +} + +/// Handler for loaded CSV content. +#[derive(Debug, Clone)] +pub struct CsvHandler { + pub(crate) data: CsvData, +} + +#[async_trait::async_trait] +impl Handler for CsvHandler { + fn document_type(&self) -> DocumentType { + DocumentType::Csv + } + + type SpanId = CsvSpan; + type SpanData = String; + + async fn view_spans(&self) -> SpanStream<'_, CsvSpan, String> { + SpanStream::new(futures::stream::iter(CsvSpanIter::new(&self.data))) + } + + async fn edit_spans( + &mut self, + edits: SpanEditStream<'_, CsvSpan, String>, + ) -> Result<(), Error> { + let edits: Vec<_> = edits.collect().await; + for edit in edits { + if edit.id.header { + let headers = self.data.headers.as_mut().ok_or_else(|| { + Error::validation("no headers to edit", "csv-handler") + })?; + let cell = headers.get_mut(edit.id.col).ok_or_else(|| { + Error::validation( + format!("header column {} out of bounds", edit.id.col), + "csv-handler", + ) + })?; + *cell = edit.data; + } else { + let row = self.data.rows.get_mut(edit.id.row).ok_or_else(|| { + Error::validation( + format!("row {} out of bounds", edit.id.row), + "csv-handler", + ) + })?; + let cell = row.get_mut(edit.id.col).ok_or_else(|| { + Error::validation( + format!( + "column {} out of bounds in row {}", + edit.id.col, edit.id.row, + ), + "csv-handler", + ) + })?; + *cell = edit.data; + } + } + Ok(()) + } +} + +impl CsvHandler { + /// Column headers, if present. + pub fn headers(&self) -> Option<&[String]> { + self.data.headers.as_deref() + } + + /// All data rows. + pub fn rows(&self) -> &[Vec] { + &self.data.rows + } + + /// Mutable access to all data rows. + pub fn rows_mut(&mut self) -> &mut Vec> { + &mut self.data.rows + } + + /// A specific cell by (row, col). + pub fn cell(&self, row: usize, col: usize) -> Option<&str> { + self.data + .rows + .get(row) + .and_then(|r| r.get(col)) + .map(|s| s.as_str()) + } + + /// Number of data rows (excluding the header). + pub fn row_count(&self) -> usize { + self.data.rows.len() + } + + /// Detected field delimiter. + pub fn delimiter(&self) -> u8 { + self.data.delimiter + } + + /// Whether the original source had a trailing newline. + pub fn trailing_newline(&self) -> bool { + self.data.trailing_newline + } + + /// Consume the handler and return the inner [`CsvData`]. + pub fn into_data(self) -> CsvData { + self.data + } +} + +/// Iterator over cells of a CSV document. +/// +/// Yields header cells first (if present), then data cells in +/// row-major order. +struct CsvSpanIter<'a> { + headers: Option<&'a [String]>, + rows: &'a [Vec], + /// Current position: `None` = in headers, `Some(row)` = in data. + phase: CsvIterPhase, + col: usize, +} + +enum CsvIterPhase { + Headers, + Data(usize), +} + +impl<'a> CsvSpanIter<'a> { + fn new(data: &'a CsvData) -> Self { + let phase = if data.headers.is_some() { + CsvIterPhase::Headers + } else { + CsvIterPhase::Data(0) + }; + Self { + headers: data.headers.as_deref(), + rows: &data.rows, + phase, + col: 0, + } + } +} + +impl<'a> Iterator for CsvSpanIter<'a> { + type Item = Span; + + fn next(&mut self) -> Option { + loop { + match &self.phase { + CsvIterPhase::Headers => { + let headers = self.headers?; + if let Some(value) = headers.get(self.col) { + let col = self.col; + self.col += 1; + return Some(Span { + id: CsvSpan::header_cell(col, value.clone()), + data: value.clone(), + }); + } + self.phase = CsvIterPhase::Data(0); + self.col = 0; + } + CsvIterPhase::Data(row) => { + let row_idx = *row; + let row_data = self.rows.get(row_idx)?; + if let Some(value) = row_data.get(self.col) { + let col = self.col; + self.col += 1; + let key = self + .headers + .and_then(|h| h.get(col)) + .cloned() + .unwrap_or_else(|| col.to_string()); + return Some(Span { + id: CsvSpan::cell(row_idx, col, key), + data: value.clone(), + }); + } + self.phase = CsvIterPhase::Data(row_idx + 1); + self.col = 0; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::handler::SpanEdit; + use futures::StreamExt; + + fn handler_with_headers( + headers: Vec<&str>, + rows: Vec>, + ) -> CsvHandler { + CsvHandler { + data: CsvData { + headers: Some(headers.into_iter().map(String::from).collect()), + rows: rows + .into_iter() + .map(|r| r.into_iter().map(String::from).collect()) + .collect(), + delimiter: b',', + trailing_newline: true, + }, + } + } + + fn handler_no_headers(rows: Vec>) -> CsvHandler { + CsvHandler { + data: CsvData { + headers: None, + rows: rows + .into_iter() + .map(|r| r.into_iter().map(String::from).collect()) + .collect(), + delimiter: b',', + trailing_newline: true, + }, + } + } + + #[tokio::test] + async fn view_spans_with_headers() { + let h = handler_with_headers( + vec!["name", "age"], + vec![vec!["Alice", "30"], vec!["Bob", "25"]], + ); + let spans: Vec<_> = h.view_spans().await.collect().await; + + // 2 header cells + 4 data cells + assert_eq!(spans.len(), 6); + + // Headers + assert_eq!(spans[0].id, CsvSpan::header_cell(0, "name")); + assert_eq!(spans[0].data, "name"); + assert_eq!(spans[1].id, CsvSpan::header_cell(1, "age")); + assert_eq!(spans[1].data, "age"); + + // Row 0 + assert_eq!(spans[2].id, CsvSpan::cell(0, 0, "name")); + assert_eq!(spans[2].id.key, "name"); + assert_eq!(spans[2].data, "Alice"); + assert_eq!(spans[3].id, CsvSpan::cell(0, 1, "age")); + assert_eq!(spans[3].id.key, "age"); + assert_eq!(spans[3].data, "30"); + + // Row 1 + assert_eq!(spans[4].id, CsvSpan::cell(1, 0, "name")); + assert_eq!(spans[4].data, "Bob"); + assert_eq!(spans[5].id, CsvSpan::cell(1, 1, "age")); + assert_eq!(spans[5].data, "25"); + } + + #[tokio::test] + async fn view_spans_no_headers() { + let h = handler_no_headers(vec![vec!["x", "y"], vec!["1", "2"]]); + let spans: Vec<_> = h.view_spans().await.collect().await; + + assert_eq!(spans.len(), 4); + assert_eq!(spans[0].id, CsvSpan::cell(0, 0, "0")); + assert_eq!(spans[0].id.key, "0"); + assert_eq!(spans[0].data, "x"); + } + + #[tokio::test] + async fn view_spans_empty() { + let h = handler_no_headers(vec![]); + let spans: Vec<_> = h.view_spans().await.collect().await; + assert!(spans.is_empty()); + } + + #[tokio::test] + async fn edit_spans_data_cell() { + let mut h = handler_with_headers( + vec!["ssn"], + vec![vec!["123-45-6789"]], + ); + h.edit_spans(SpanEditStream::new(futures::stream::iter(vec![ + SpanEdit { + id: CsvSpan::cell(0, 0, "ssn"), + data: "[REDACTED]".into(), + }, + ]))) + .await + .unwrap(); + assert_eq!(h.cell(0, 0), Some("[REDACTED]")); + } + + #[tokio::test] + async fn edit_spans_header_cell() { + let mut h = handler_with_headers( + vec!["secret_field"], + vec![vec!["value"]], + ); + h.edit_spans(SpanEditStream::new(futures::stream::iter(vec![ + SpanEdit { + id: CsvSpan::header_cell(0, "secret_field"), + data: "redacted".into(), + }, + ]))) + .await + .unwrap(); + assert_eq!(h.headers(), Some(["redacted".to_string()].as_slice())); + } + + #[tokio::test] + async fn edit_spans_row_out_of_bounds() { + let mut h = handler_no_headers(vec![vec!["a"]]); + let err = h + .edit_spans(SpanEditStream::new(futures::stream::iter(vec![ + SpanEdit { + id: CsvSpan::cell(5, 0, "0"), + data: "x".into(), + }, + ]))) + .await + .unwrap_err(); + assert!(err.to_string().contains("out of bounds")); + } + + #[tokio::test] + async fn edit_spans_col_out_of_bounds() { + let mut h = handler_no_headers(vec![vec!["a"]]); + let err = h + .edit_spans(SpanEditStream::new(futures::stream::iter(vec![ + SpanEdit { + id: CsvSpan::cell(0, 5, "5"), + data: "x".into(), + }, + ]))) + .await + .unwrap_err(); + assert!(err.to_string().contains("out of bounds")); + } +} diff --git a/crates/nvisy-ingest/src/handler/text/csv_loader.rs b/crates/nvisy-ingest/src/handler/text/csv_loader.rs new file mode 100644 index 0000000..40c7d45 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/text/csv_loader.rs @@ -0,0 +1,234 @@ +//! CSV loader — validates and parses raw CSV content into a +//! [`Document`]. +//! +//! The loader auto-detects the field delimiter (comma, tab, semicolon, +//! pipe) by inspecting the first line. + +use nvisy_core::error::Error; +use nvisy_core::io::ContentData; + +use crate::document::Document; +use crate::handler::{CsvData, CsvHandler, Loader, TextEncoding}; + +/// Parameters for [`CsvLoader`]. +#[derive(Debug)] +pub struct CsvParams { + /// Character encoding of the input bytes. + pub encoding: TextEncoding, + /// Whether the first row contains column headers. + /// Defaults to `true`. + pub has_headers: bool, + /// Override the field delimiter. If `None`, the loader will + /// auto-detect from the first line. + pub delimiter: Option, +} + +impl Default for CsvParams { + fn default() -> Self { + Self { + encoding: TextEncoding::Utf8, + has_headers: true, + delimiter: None, + } + } +} + +/// Loader that validates and parses CSV files. +/// +/// Produces a single [`Document`] per input. +#[derive(Debug)] +pub struct CsvLoader; + +#[async_trait::async_trait] +impl Loader for CsvLoader { + type Handler = CsvHandler; + type Params = CsvParams; + + async fn load( + &self, + content: &ContentData, + params: &Self::Params, + ) -> Result>, Error> { + let raw = content.to_bytes(); + let text = params.encoding.decode_bytes(&raw, "csv-loader")?; + let trailing_newline = text.ends_with('\n'); + let delimiter = params + .delimiter + .unwrap_or_else(|| detect_delimiter(&text)); + + let mut reader = csv::ReaderBuilder::new() + .has_headers(params.has_headers) + .delimiter(delimiter) + .flexible(true) + .from_reader(text.as_bytes()); + + let headers = if params.has_headers { + let hdr = reader.headers().map_err(|e| { + Error::validation(format!("CSV header error: {e}"), "csv-loader") + })?; + Some(hdr.iter().map(String::from).collect()) + } else { + None + }; + + let mut rows = Vec::new(); + for result in reader.records() { + let record = result.map_err(|e| { + Error::validation(format!("CSV parse error: {e}"), "csv-loader") + })?; + rows.push(record.iter().map(String::from).collect()); + } + + let handler = CsvHandler { + data: CsvData { + headers, + rows, + delimiter, + trailing_newline, + }, + }; + let doc = Document::new(handler).with_parent(content); + Ok(vec![doc]) + } +} + +/// Auto-detect the CSV delimiter by counting candidate characters +/// in the first line. +fn detect_delimiter(text: &str) -> u8 { + let first_line = text.lines().next().unwrap_or(""); + let candidates: &[(u8, char)] = &[ + (b',', ','), + (b'\t', '\t'), + (b';', ';'), + (b'|', '|'), + ]; + candidates + .iter() + .max_by_key(|(_, ch)| first_line.matches(*ch).count()) + .map(|(b, _)| *b) + .unwrap_or(b',') +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::handler::Handler; + use bytes::Bytes; + use futures::StreamExt; + use nvisy_core::path::ContentSource; + use nvisy_ontology::entity::DocumentType; + + fn content_from_str(s: &str) -> ContentData { + ContentData::new(ContentSource::new(), Bytes::from(s.to_owned())) + } + + #[tokio::test] + async fn load_with_headers() { + let content = content_from_str("name,age\nAlice,30\nBob,25\n"); + let docs = CsvLoader + .load(&content, &CsvParams::default()) + .await + .unwrap(); + + assert_eq!(docs.len(), 1); + assert_eq!(docs[0].document_type(), DocumentType::Csv); + + let h = docs[0].handler(); + assert_eq!(h.headers(), Some(["name", "age"].map(String::from).as_slice())); + assert_eq!(h.row_count(), 2); + assert_eq!(h.cell(0, 0), Some("Alice")); + assert_eq!(h.cell(1, 1), Some("25")); + assert!(h.trailing_newline()); + } + + #[tokio::test] + async fn load_without_headers() { + let params = CsvParams { + has_headers: false, + ..CsvParams::default() + }; + let content = content_from_str("x,y\n1,2\n"); + let docs = CsvLoader.load(&content, ¶ms).await.unwrap(); + + let h = docs[0].handler(); + assert!(h.headers().is_none()); + assert_eq!(h.row_count(), 2); + assert_eq!(h.cell(0, 0), Some("x")); + } + + #[tokio::test] + async fn load_tab_delimited() { + let content = content_from_str("a\tb\n1\t2\n"); + let docs = CsvLoader + .load(&content, &CsvParams::default()) + .await + .unwrap(); + let h = docs[0].handler(); + assert_eq!(h.delimiter(), b'\t'); + assert_eq!(h.headers(), Some(["a", "b"].map(String::from).as_slice())); + } + + #[tokio::test] + async fn load_semicolon_delimited() { + let content = content_from_str("a;b\n1;2\n"); + let docs = CsvLoader + .load(&content, &CsvParams::default()) + .await + .unwrap(); + assert_eq!(docs[0].handler().delimiter(), b';'); + } + + #[tokio::test] + async fn load_quoted_fields() { + let content = content_from_str("name,bio\n\"Alice\",\"Has a, comma\"\n"); + let docs = CsvLoader + .load(&content, &CsvParams::default()) + .await + .unwrap(); + let h = docs[0].handler(); + assert_eq!(h.cell(0, 1), Some("Has a, comma")); + } + + #[tokio::test] + async fn load_empty() { + let content = content_from_str(""); + let docs = CsvLoader + .load(&content, &CsvParams::default()) + .await + .unwrap(); + let h = docs[0].handler(); + assert_eq!(h.row_count(), 0); + } + + #[tokio::test] + async fn load_spans_round_trip() { + let content = content_from_str("name,age\nAlice,30\n"); + let docs = CsvLoader + .load(&content, &CsvParams::default()) + .await + .unwrap(); + let spans: Vec<_> = docs[0].handler().view_spans().await.collect().await; + + // 2 header + 2 data + assert_eq!(spans.len(), 4); + assert_eq!(spans[0].data, "name"); + assert_eq!(spans[1].data, "age"); + assert_eq!(spans[2].data, "Alice"); + assert_eq!(spans[2].id.key, "name"); + assert_eq!(spans[3].data, "30"); + assert_eq!(spans[3].id.key, "age"); + } + + #[tokio::test] + async fn load_invalid_utf8() { + let content = ContentData::new( + ContentSource::new(), + Bytes::from_static(&[0xFF, 0xFE, 0x00]), + ); + let err = CsvLoader + .load(&content, &CsvParams::default()) + .await + .unwrap_err(); + assert!(err.to_string().contains("UTF-8")); + } +} diff --git a/crates/nvisy-ingest/src/handler/text/html.rs b/crates/nvisy-ingest/src/handler/text/html.rs new file mode 100644 index 0000000..318bd46 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/text/html.rs @@ -0,0 +1,32 @@ +//! HTML handler (stub — awaiting migration to Loader/Handler pattern). + +use nvisy_core::error::Error; +use nvisy_ontology::entity::DocumentType; + +use crate::document::edit_stream::SpanEditStream; +use crate::document::view_stream::SpanStream; +use crate::handler::Handler; + +#[derive(Debug)] +pub struct HtmlHandler; + +#[async_trait::async_trait] +impl Handler for HtmlHandler { + fn document_type(&self) -> DocumentType { + DocumentType::Html + } + + type SpanId = (); + type SpanData = (); + + async fn view_spans(&self) -> SpanStream<'_, (), ()> { + SpanStream::new(futures::stream::empty()) + } + + async fn edit_spans( + &mut self, + _edits: SpanEditStream<'_, (), ()>, + ) -> Result<(), Error> { + Ok(()) + } +} diff --git a/crates/nvisy-ingest/src/handler/text/json_handler.rs b/crates/nvisy-ingest/src/handler/text/json_handler.rs new file mode 100644 index 0000000..74598cb --- /dev/null +++ b/crates/nvisy-ingest/src/handler/text/json_handler.rs @@ -0,0 +1,578 @@ +//! JSON handler — holds parsed JSON content and provides span-based +//! access via [`Handler`]. +//! +//! The handler stores the parsed [`serde_json::Value`] tree together +//! with formatting metadata captured during loading, so the original +//! file can be reconstructed with identical whitespace after edits. +//! +//! # Span model +//! +//! [`Handler::view_spans`] yields one [`Span`] per node in the JSON +//! tree. **Every** value is emitted — leaf scalars, and object keys +//! (as string-valued spans). Each span is addressed by a [`JsonPath`]: +//! an [RFC 6901] JSON Pointer such as `/address/city` plus a flag +//! indicating whether the span targets the key name or the value. +//! +//! [`Handler::edit_spans`] accepts [`SpanEdit`]s. For value spans the +//! value at the pointer is replaced; for key spans the object key is +//! renamed. +//! +//! [RFC 6901]: https://www.rfc-editor.org/rfc/rfc6901 + +use std::num::NonZeroU32; + +use futures::StreamExt; +use serde::{Deserialize, Serialize}; + +use nvisy_core::error::Error; +use nvisy_ontology::entity::DocumentType; + +use crate::document::edit_stream::SpanEditStream; +use crate::document::view_stream::SpanStream; +use crate::handler::{Handler, Span}; + +const DEFAULT_INDENT: NonZeroU32 = NonZeroU32::new(2).unwrap(); + +/// [RFC 6901] JSON Pointer identifying a span within a JSON document. +/// +/// `pointer` follows JSON Pointer syntax: `""` for the root, +/// `"/foo/0/bar"` for nested paths. Object keys containing `~` or `/` +/// are escaped as `~0` and `~1` respectively. +/// +/// When `key_of` is `true` the span addresses the **key name** of the +/// object entry at `pointer`, rather than its value. Editing a key +/// span renames the key; editing a value span replaces the value. +/// +/// [RFC 6901]: https://www.rfc-editor.org/rfc/rfc6901 +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct JsonPath { + pub pointer: String, + pub key_of: bool, +} + +impl JsonPath { + /// Create a value-addressing path. + pub fn value(pointer: impl Into) -> Self { + Self { + pointer: pointer.into(), + key_of: false, + } + } + + /// Create a key-addressing path. + pub fn key(pointer: impl Into) -> Self { + Self { + pointer: pointer.into(), + key_of: true, + } + } +} + +/// Indentation style detected in the original JSON source. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum JsonIndent { + /// No whitespace between tokens (`{"a":1}`). + Compact, + /// N-space indentation. + Spaces(NonZeroU32), + /// Tab indentation. + Tab, +} + +impl JsonIndent { + /// Two-space indentation. + pub fn two_spaces() -> Self { + Self::Spaces(NonZeroU32::new(2).unwrap()) + } + + /// Four-space indentation. + pub fn four_spaces() -> Self { + Self::Spaces(NonZeroU32::new(4).unwrap()) + } +} + +/// Parsed JSON content together with its original formatting. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JsonData { + pub value: serde_json::Value, + pub indent: JsonIndent, + pub trailing_newline: bool, +} + +impl Default for JsonData { + fn default() -> Self { + Self { + value: serde_json::Value::Null, + indent: JsonIndent::Spaces(DEFAULT_INDENT), + trailing_newline: true, + } + } +} + +/// Handler for loaded JSON content. +/// +/// Provides direct access to the parsed [`serde_json::Value`] tree +/// for reading and mutation, plus [`Handler`] implementation for +/// pipeline-driven span-based editing. +#[derive(Debug)] +pub struct JsonHandler { + pub(crate) data: JsonData, +} + +#[async_trait::async_trait] +impl Handler for JsonHandler { + fn document_type(&self) -> DocumentType { + DocumentType::Json + } + + type SpanId = JsonPath; + type SpanData = serde_json::Value; + + async fn view_spans(&self) -> SpanStream<'_, JsonPath, serde_json::Value> { + SpanStream::new(futures::stream::iter(JsonSpanIter::new(&self.data.value))) + } + + async fn edit_spans( + &mut self, + edits: SpanEditStream<'_, JsonPath, serde_json::Value>, + ) -> Result<(), Error> { + let edits: Vec<_> = edits.collect().await; + // Apply value edits first so that pointers remain valid when + // key renames change the path structure. + for edit in edits.iter().filter(|e| !e.id.key_of) { + let target = + self.data.value.pointer_mut(&edit.id.pointer).ok_or_else(|| { + Error::validation( + format!("JSON pointer not found: {}", edit.id.pointer), + "json-handler", + ) + })?; + *target = edit.data.clone(); + } + for edit in edits.iter().filter(|e| e.id.key_of) { + rename_key(&mut self.data.value, &edit.id.pointer, &edit.data)?; + } + Ok(()) + } +} + +impl JsonHandler { + /// Reference to the root JSON value. + pub fn value(&self) -> &serde_json::Value { + &self.data.value + } + + /// Mutable reference to the root JSON value. + pub fn value_mut(&mut self) -> &mut serde_json::Value { + &mut self.data.value + } + + /// Look up a value by [RFC 6901] JSON Pointer (e.g. `"/a/0/b"`). + /// + /// [RFC 6901]: https://www.rfc-editor.org/rfc/rfc6901 + pub fn pointer(&self, pointer: &str) -> Option<&serde_json::Value> { + self.data.value.pointer(pointer) + } + + /// Mutably look up a value by [RFC 6901] JSON Pointer. + /// + /// [RFC 6901]: https://www.rfc-editor.org/rfc/rfc6901 + pub fn pointer_mut(&mut self, pointer: &str) -> Option<&mut serde_json::Value> { + self.data.value.pointer_mut(pointer) + } + + /// Replace the entire root value. + pub fn set_value(&mut self, value: serde_json::Value) { + self.data.value = value; + } + + /// Indentation style detected in the original source. + pub fn indent(&self) -> JsonIndent { + self.data.indent + } + + /// Whether the original source had a trailing newline. + pub fn trailing_newline(&self) -> bool { + self.data.trailing_newline + } + + /// Consume the handler and return the inner [`JsonData`]. + pub fn into_data(self) -> JsonData { + self.data + } +} + +/// Rename an object key addressed by `pointer`. +/// +/// `new_name` must be a `Value::String`; the pointer must resolve to +/// an entry inside an object. +fn rename_key( + root: &mut serde_json::Value, + pointer: &str, + new_name: &serde_json::Value, +) -> Result<(), Error> { + let new_key = new_name.as_str().ok_or_else(|| { + Error::validation("key rename requires a string value", "json-handler") + })?; + + let (parent_ptr, old_key) = split_pointer(pointer)?; + + let parent = if parent_ptr.is_empty() { + root as &mut serde_json::Value + } else { + root.pointer_mut(parent_ptr).ok_or_else(|| { + Error::validation( + format!("JSON pointer not found: {parent_ptr}"), + "json-handler", + ) + })? + }; + + let obj = parent.as_object_mut().ok_or_else(|| { + Error::validation( + format!("parent at {parent_ptr} is not an object"), + "json-handler", + ) + })?; + + let value = obj.remove(&old_key).ok_or_else(|| { + Error::validation( + format!("key {old_key:?} not found in object at {parent_ptr}"), + "json-handler", + ) + })?; + + obj.insert(new_key.to_owned(), value); + Ok(()) +} + +/// Split a JSON Pointer into parent pointer and last segment (unescaped). +fn split_pointer(pointer: &str) -> Result<(&str, String), Error> { + let last_slash = pointer.rfind('/').ok_or_else(|| { + Error::validation( + format!("invalid JSON pointer for key rename: {pointer}"), + "json-handler", + ) + })?; + let parent = &pointer[..last_slash]; + let segment = unescape_json_pointer(&pointer[last_slash + 1..]); + Ok((parent, segment)) +} + +/// Unescape a JSON Pointer segment ([RFC 6901]): `~1` → `/`, `~0` → `~`. +/// +/// [RFC 6901]: https://www.rfc-editor.org/rfc/rfc6901 +fn unescape_json_pointer(segment: &str) -> String { + if segment.contains('~') { + segment.replace("~1", "/").replace("~0", "~") + } else { + segment.to_owned() + } +} + +/// Stack frame for iterative JSON tree traversal. +enum IterFrame<'a> { + /// A leaf or unexpanded node to process. + Pending { + value: &'a serde_json::Value, + pointer: String, + }, + /// A key span to yield before descending into its value. + KeySpan { + value: &'a serde_json::Value, + pointer: String, + key: String, + }, + /// An object whose entries are being yielded. + Object(String, serde_json::map::Iter<'a>), + /// An array whose elements are being yielded. + Array(String, std::iter::Enumerate>), +} + +/// Stack-based depth-first iterator over a JSON tree. +/// +/// Yields one [`Span`] per leaf value **and** one per object key. +/// Key spans have [`JsonPath::key_of`] set to `true` and carry the +/// key name as `Value::String`. Objects and arrays are expanded in +/// place without recursion, so arbitrarily deep documents are safe +/// to iterate. +struct JsonSpanIter<'a> { + stack: Vec>, +} + +impl<'a> JsonSpanIter<'a> { + fn new(root: &'a serde_json::Value) -> Self { + Self { + stack: vec![IterFrame::Pending { + value: root, + pointer: String::new(), + }], + } + } +} + +impl<'a> Iterator for JsonSpanIter<'a> { + type Item = Span; + + fn next(&mut self) -> Option { + loop { + let frame = self.stack.last_mut()?; + + match frame { + IterFrame::Pending { .. } => { + let IterFrame::Pending { value, pointer } = + self.stack.pop().unwrap() + else { + unreachable!() + }; + match value { + serde_json::Value::Object(map) => { + self.stack.push(IterFrame::Object(pointer, map.iter())); + } + serde_json::Value::Array(arr) => { + self.stack + .push(IterFrame::Array(pointer, arr.iter().enumerate())); + } + leaf => { + return Some(Span { + id: JsonPath::value(pointer), + data: leaf.clone(), + }); + } + } + } + IterFrame::KeySpan { .. } => { + let IterFrame::KeySpan { value, pointer, key } = + self.stack.pop().unwrap() + else { + unreachable!() + }; + // Push the value traversal so it runs after we yield the key. + self.stack.push(IterFrame::Pending { + value, + pointer: pointer.clone(), + }); + return Some(Span { + id: JsonPath::key(&pointer), + data: serde_json::Value::String(key), + }); + } + IterFrame::Object(pointer, iter) => match iter.next() { + Some((key, child)) => { + let child_pointer = + format!("{}/{}", pointer, escape_json_pointer(key)); + self.stack.push(IterFrame::KeySpan { + value: child, + pointer: child_pointer, + key: key.clone(), + }); + } + None => { + self.stack.pop(); + } + }, + IterFrame::Array(pointer, iter) => match iter.next() { + Some((i, child)) => { + let child_pointer = format!("{}/{i}", pointer); + self.stack.push(IterFrame::Pending { + value: child, + pointer: child_pointer, + }); + } + None => { + self.stack.pop(); + } + }, + } + } + } +} + +/// Escape a JSON object key for use in a JSON Pointer ([RFC 6901]). +/// +/// [RFC 6901]: https://www.rfc-editor.org/rfc/rfc6901 +fn escape_json_pointer(key: &str) -> String { + if key.contains('~') || key.contains('/') { + key.replace('~', "~0").replace('/', "~1") + } else { + key.to_owned() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::handler::SpanEdit; + use futures::StreamExt; + use serde_json::json; + + fn handler(value: serde_json::Value) -> JsonHandler { + JsonHandler { + data: JsonData { + value, + ..JsonData::default() + }, + } + } + + #[tokio::test] + async fn view_spans_flat_object() { + let h = handler(json!({"name": "Alice", "age": 30})); + let spans: Vec<_> = h.view_spans().await.collect().await; + + // BTreeMap (alphabetical): age before name. + // Each key emits a key span followed by a value span. + assert_eq!(spans.len(), 4); + assert_eq!(spans[0].id, JsonPath::key("/age")); + assert_eq!(spans[0].data, json!("age")); + assert_eq!(spans[1].id, JsonPath::value("/age")); + assert_eq!(spans[1].data, json!(30)); + assert_eq!(spans[2].id, JsonPath::key("/name")); + assert_eq!(spans[2].data, json!("name")); + assert_eq!(spans[3].id, JsonPath::value("/name")); + assert_eq!(spans[3].data, json!("Alice")); + } + + #[tokio::test] + async fn view_spans_nested() { + let h = handler(json!({"a": {"b": [1, "two", null]}})); + let spans: Vec<_> = h.view_spans().await.collect().await; + + // key "a", key "b", values 0/1/2 + assert_eq!(spans.len(), 5); + assert_eq!(spans[0].id, JsonPath::key("/a")); + assert_eq!(spans[1].id, JsonPath::key("/a/b")); + assert_eq!(spans[2].id, JsonPath::value("/a/b/0")); + assert_eq!(spans[2].data, json!(1)); + assert_eq!(spans[3].id, JsonPath::value("/a/b/1")); + assert_eq!(spans[3].data, json!("two")); + assert_eq!(spans[4].id, JsonPath::value("/a/b/2")); + assert_eq!(spans[4].data, json!(null)); + } + + #[tokio::test] + async fn view_spans_key_escaping() { + let h = handler(json!({"a/b": "x", "c~d": "y"})); + let spans: Vec<_> = h.view_spans().await.collect().await; + + // key span, value span, key span, value span + assert_eq!(spans.len(), 4); + assert_eq!(spans[0].id, JsonPath::key("/a~1b")); + assert_eq!(spans[0].data, json!("a/b")); + assert_eq!(spans[1].id, JsonPath::value("/a~1b")); + assert_eq!(spans[1].data, json!("x")); + assert_eq!(spans[2].id, JsonPath::key("/c~0d")); + assert_eq!(spans[2].data, json!("c~d")); + assert_eq!(spans[3].id, JsonPath::value("/c~0d")); + assert_eq!(spans[3].data, json!("y")); + } + + #[tokio::test] + async fn view_spans_empty_object() { + let h = handler(json!({})); + let spans: Vec<_> = h.view_spans().await.collect().await; + assert!(spans.is_empty()); + } + + #[tokio::test] + async fn view_spans_scalar_root() { + let h = handler(json!("hello")); + let spans: Vec<_> = h.view_spans().await.collect().await; + assert_eq!(spans.len(), 1); + assert_eq!(spans[0].id, JsonPath::value("")); + assert_eq!(spans[0].data, json!("hello")); + } + + #[tokio::test] + async fn edit_spans_replace_value() { + let mut h = handler(json!({"ssn": "123-45-6789"})); + h.edit_spans(SpanEditStream::new(futures::stream::iter(vec![ + SpanEdit { + id: JsonPath::value("/ssn"), + data: json!(null), + }, + ]))) + .await + .unwrap(); + assert_eq!(h.value(), &json!({"ssn": null})); + } + + #[tokio::test] + async fn edit_spans_rename_key() { + let mut h = handler(json!({"John Smith": {"age": 30}})); + h.edit_spans(SpanEditStream::new(futures::stream::iter(vec![ + SpanEdit { + id: JsonPath::key("/John Smith"), + data: json!("[REDACTED]"), + }, + ]))) + .await + .unwrap(); + assert_eq!(h.value(), &json!({"[REDACTED]": {"age": 30}})); + } + + #[tokio::test] + async fn edit_spans_rename_nested_key() { + let mut h = handler(json!({"a": {"secret_field": 42}})); + h.edit_spans(SpanEditStream::new(futures::stream::iter(vec![ + SpanEdit { + id: JsonPath::key("/a/secret_field"), + data: json!("redacted"), + }, + ]))) + .await + .unwrap(); + assert_eq!(h.value(), &json!({"a": {"redacted": 42}})); + } + + #[tokio::test] + async fn edit_spans_rename_key_requires_string() { + let mut h = handler(json!({"a": 1})); + let err = h + .edit_spans(SpanEditStream::new(futures::stream::iter(vec![ + SpanEdit { + id: JsonPath::key("/a"), + data: json!(42), + }, + ]))) + .await + .unwrap_err(); + assert!(err.to_string().contains("string")); + } + + #[tokio::test] + async fn edit_spans_bad_pointer() { + let mut h = handler(json!({"a": 1})); + let err = h + .edit_spans(SpanEditStream::new(futures::stream::iter(vec![ + SpanEdit { + id: JsonPath::value("/nonexistent"), + data: json!(null), + }, + ]))) + .await + .unwrap_err(); + assert!(err.to_string().contains("not found")); + } + + #[tokio::test] + async fn edit_spans_value_before_key_rename() { + let mut h = handler(json!({"name": "Alice"})); + // Key rename listed first, but value edit must apply first + // (while /name still exists) before the key is renamed. + h.edit_spans(SpanEditStream::new(futures::stream::iter(vec![ + SpanEdit { + id: JsonPath::key("/name"), + data: json!("[REDACTED]"), + }, + SpanEdit { + id: JsonPath::value("/name"), + data: json!("***"), + }, + ]))) + .await + .unwrap(); + assert_eq!(h.value(), &json!({"[REDACTED]": "***"})); + } +} diff --git a/crates/nvisy-ingest/src/handler/text/json_loader.rs b/crates/nvisy-ingest/src/handler/text/json_loader.rs new file mode 100644 index 0000000..5baf657 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/text/json_loader.rs @@ -0,0 +1,186 @@ +//! JSON loader — validates and parses raw JSON content into a +//! [`Document`]. +//! +//! The loader detects the indentation style and trailing-newline +//! convention of the source file so that [`JsonData`] preserves +//! whitespace for round-trip fidelity. + +use std::num::NonZeroU32; + +use nvisy_core::error::Error; +use nvisy_core::io::ContentData; + +use crate::document::Document; +use crate::handler::{ + JsonData, JsonHandler, JsonIndent, Loader, TextEncoding, +}; + +/// Parameters for [`JsonLoader`]. +#[derive(Debug, Default)] +pub struct JsonParams { + /// Character encoding of the input bytes. + pub encoding: TextEncoding, +} + +/// Loader that validates and parses JSON files. +/// +/// Produces a single [`Document`] per input. The +/// loaded handler stores the parsed [`serde_json::Value`] tree +/// together with formatting metadata for round-trip fidelity. +#[derive(Debug)] +pub struct JsonLoader; + +#[async_trait::async_trait] +impl Loader for JsonLoader { + type Handler = JsonHandler; + type Params = JsonParams; + + async fn load( + &self, + content: &ContentData, + params: &Self::Params, + ) -> Result>, Error> { + let raw = content.to_bytes(); + let text = params.encoding.decode_bytes(&raw, "json-loader")?; + let (indent, trailing_newline) = detect_formatting(&text); + + let value: serde_json::Value = serde_json::from_str(&text).map_err(|e| { + Error::validation(format!("Invalid JSON: {e}"), "json-loader") + })?; + + let handler = JsonHandler { + data: JsonData { + value, + indent, + trailing_newline, + }, + }; + let doc = Document::new(handler).with_parent(content); + Ok(vec![doc]) + } +} + +/// Detect indentation style and trailing newline from raw JSON source. +/// +/// Inspects the first indented line to determine the whitespace +/// convention. Falls back to [`JsonIndent::Compact`] when no +/// indentation is present (single-line JSON). +fn detect_formatting(source: &str) -> (JsonIndent, bool) { + let trailing_newline = source.ends_with('\n'); + + let indent = source + .lines() + .find_map(|line| { + let stripped = line.trim_start(); + if stripped.len() == line.len() { + return None; + } + let ws = &line[..line.len() - stripped.len()]; + if ws.starts_with('\t') { + Some(JsonIndent::Tab) + } else { + let n = u32::try_from(ws.len()).unwrap_or(u32::MAX); + Some(JsonIndent::Spaces( + NonZeroU32::new(n).unwrap_or(NonZeroU32::new(2).unwrap()), + )) + } + }) + .unwrap_or(JsonIndent::Compact); + + (indent, trailing_newline) +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use nvisy_core::path::ContentSource; + use nvisy_ontology::entity::DocumentType; + use serde_json::json; + + fn content_from_str(s: &str) -> ContentData { + ContentData::new(ContentSource::new(), Bytes::from(s.to_owned())) + } + + #[tokio::test] + async fn load_simple_object() { + let content = content_from_str(r#"{"name": "Alice", "age": 30}"#); + let docs = JsonLoader + .load(&content, &JsonParams::default()) + .await + .unwrap(); + + assert_eq!(docs.len(), 1); + assert_eq!(docs[0].document_type(), DocumentType::Json); + + let handler = docs[0].handler(); + assert_eq!(handler.value(), &json!({"name": "Alice", "age": 30})); + } + + #[tokio::test] + async fn load_detects_compact_formatting() { + let content = content_from_str(r#"{"a":1}"#); + let docs = JsonLoader + .load(&content, &JsonParams::default()) + .await + .unwrap(); + let h = docs[0].handler(); + assert_eq!(h.indent(), JsonIndent::Compact); + assert!(!h.trailing_newline()); + } + + #[tokio::test] + async fn load_detects_two_space_indent() { + let content = content_from_str("{\n \"a\": 1\n}\n"); + let docs = JsonLoader + .load(&content, &JsonParams::default()) + .await + .unwrap(); + let h = docs[0].handler(); + assert_eq!(h.indent(), JsonIndent::two_spaces()); + assert!(h.trailing_newline()); + } + + #[tokio::test] + async fn load_detects_four_space_indent() { + let content = content_from_str("{\n \"a\": 1\n}\n"); + let docs = JsonLoader + .load(&content, &JsonParams::default()) + .await + .unwrap(); + assert_eq!(docs[0].handler().indent(), JsonIndent::four_spaces()); + } + + #[tokio::test] + async fn load_detects_tab_indent() { + let content = content_from_str("{\n\t\"a\": 1\n}\n"); + let docs = JsonLoader + .load(&content, &JsonParams::default()) + .await + .unwrap(); + assert_eq!(docs[0].handler().indent(), JsonIndent::Tab); + } + + #[tokio::test] + async fn load_invalid_utf8() { + let content = ContentData::new( + ContentSource::new(), + Bytes::from_static(&[0xFF, 0xFE, 0x00]), + ); + let err = JsonLoader + .load(&content, &JsonParams::default()) + .await + .unwrap_err(); + assert!(err.to_string().contains("UTF-8")); + } + + #[tokio::test] + async fn load_invalid_json() { + let content = content_from_str("{not json}"); + let err = JsonLoader + .load(&content, &JsonParams::default()) + .await + .unwrap_err(); + assert!(err.to_string().contains("JSON")); + } +} diff --git a/crates/nvisy-ingest/src/handler/text/mod.rs b/crates/nvisy-ingest/src/handler/text/mod.rs new file mode 100644 index 0000000..22b6542 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/text/mod.rs @@ -0,0 +1,10 @@ +//! Text-based format handlers. + +pub mod txt_handler; +pub mod txt_loader; +pub mod csv_handler; +pub mod csv_loader; +pub mod json_handler; +pub mod json_loader; +#[cfg(feature = "html")] +pub mod html; diff --git a/crates/nvisy-ingest/src/handler/text/txt_handler.rs b/crates/nvisy-ingest/src/handler/text/txt_handler.rs new file mode 100644 index 0000000..d6c5932 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/text/txt_handler.rs @@ -0,0 +1,219 @@ +//! Plain-text handler — holds loaded text content and provides +//! span-based access via [`Handler`]. +//! +//! The handler stores the text as a vector of lines together with a +//! trailing-newline flag so the original file can be reconstructed +//! byte-for-byte after edits. +//! +//! # Span model +//! +//! [`Handler::view_spans`] yields one [`Span`] per line. Each span +//! is addressed by a [`TxtSpan`] (0-based line index) and carries the +//! line content as a `String`. +//! +//! [`Handler::edit_spans`] replaces the content of lines at the given +//! indices. + +use futures::StreamExt; + +use nvisy_core::error::Error; +use nvisy_ontology::entity::DocumentType; + +use crate::document::edit_stream::SpanEditStream; +use crate::document::view_stream::SpanStream; +use crate::handler::{Handler, Span}; + +/// 0-based line index identifying a span within a plain-text document. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct TxtSpan(pub usize); + +/// Parsed plain-text content. +#[derive(Debug, Clone)] +pub struct TxtData { + pub lines: Vec, + pub trailing_newline: bool, +} + +/// Handler for loaded plain-text content. +/// +/// Each line is independently addressable via [`TxtSpan`]. +#[derive(Debug, Clone)] +pub struct TxtHandler { + pub(crate) data: TxtData, +} + +#[async_trait::async_trait] +impl Handler for TxtHandler { + fn document_type(&self) -> DocumentType { + DocumentType::Txt + } + + type SpanId = TxtSpan; + type SpanData = String; + + async fn view_spans(&self) -> SpanStream<'_, TxtSpan, String> { + SpanStream::new(futures::stream::iter(TxtSpanIter { + lines: &self.data.lines, + index: 0, + })) + } + + async fn edit_spans( + &mut self, + edits: SpanEditStream<'_, TxtSpan, String>, + ) -> Result<(), Error> { + let edits: Vec<_> = edits.collect().await; + for edit in edits { + let line = self.data.lines.get_mut(edit.id.0).ok_or_else(|| { + Error::validation( + format!("line index out of bounds: {}", edit.id.0), + "txt-handler", + ) + })?; + *line = edit.data; + } + Ok(()) + } +} + +impl TxtHandler { + /// Create a new handler from parsed text data. + pub fn new(data: TxtData) -> Self { + Self { data } + } + + /// All lines in the document. + pub fn lines(&self) -> &[String] { + &self.data.lines + } + + /// A specific line by 0-based index. + pub fn line(&self, index: usize) -> Option<&str> { + self.data.lines.get(index).map(|s| s.as_str()) + } + + /// Whether the original source had a trailing newline. + pub fn trailing_newline(&self) -> bool { + self.data.trailing_newline + } + + /// Total number of lines. + pub fn line_count(&self) -> usize { + self.data.lines.len() + } + + /// Consume the handler and return the inner [`TxtData`]. + pub fn into_data(self) -> TxtData { + self.data + } +} + +/// Iterator over lines of a plain-text document. +struct TxtSpanIter<'a> { + lines: &'a [String], + index: usize, +} + +impl<'a> Iterator for TxtSpanIter<'a> { + type Item = Span; + + fn next(&mut self) -> Option { + let line = self.lines.get(self.index)?; + let span = Span { + id: TxtSpan(self.index), + data: line.clone(), + }; + self.index += 1; + Some(span) + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = self.lines.len() - self.index; + (remaining, Some(remaining)) + } +} + +impl<'a> ExactSizeIterator for TxtSpanIter<'a> {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::handler::SpanEdit; + use futures::{Stream, StreamExt}; + + fn handler(text: &str) -> TxtHandler { + let trailing_newline = text.ends_with('\n'); + let lines = text.lines().map(String::from).collect(); + TxtHandler { + data: TxtData { + lines, + trailing_newline, + }, + } + } + + #[tokio::test] + async fn view_spans_multiline() { + let h = handler("hello\nworld\n"); + let spans: Vec<_> = h.view_spans().await.collect().await; + + assert_eq!(spans.len(), 2); + assert_eq!(spans[0].id, TxtSpan(0)); + assert_eq!(spans[0].data, "hello"); + assert_eq!(spans[1].id, TxtSpan(1)); + assert_eq!(spans[1].data, "world"); + } + + #[tokio::test] + async fn view_spans_single_line_no_newline() { + let h = handler("no newline"); + let spans: Vec<_> = h.view_spans().await.collect().await; + + assert_eq!(spans.len(), 1); + assert_eq!(spans[0].data, "no newline"); + assert!(!h.trailing_newline()); + } + + #[tokio::test] + async fn view_spans_empty() { + let h = handler(""); + let spans: Vec<_> = h.view_spans().await.collect().await; + assert!(spans.is_empty()); + } + + #[tokio::test] + async fn edit_spans_replace_line() { + let mut h = handler("hello\nworld\n"); + h.edit_spans(SpanEditStream::new(futures::stream::iter(vec![ + SpanEdit { + id: TxtSpan(1), + data: "[REDACTED]".into(), + }, + ]))) + .await + .unwrap(); + assert_eq!(h.lines(), &["hello", "[REDACTED]"]); + } + + #[tokio::test] + async fn edit_spans_out_of_bounds() { + let mut h = handler("one line"); + let err = h + .edit_spans(SpanEditStream::new(futures::stream::iter(vec![ + SpanEdit { + id: TxtSpan(5), + data: "nope".into(), + }, + ]))) + .await + .unwrap_err(); + assert!(err.to_string().contains("out of bounds")); + } + + #[tokio::test] + async fn view_spans_size_hint() { + let h = handler("a\nb\nc\n"); + let stream = h.view_spans().await; + assert_eq!(stream.size_hint(), (3, Some(3))); + } +} diff --git a/crates/nvisy-ingest/src/handler/text/txt_loader.rs b/crates/nvisy-ingest/src/handler/text/txt_loader.rs new file mode 100644 index 0000000..5347415 --- /dev/null +++ b/crates/nvisy-ingest/src/handler/text/txt_loader.rs @@ -0,0 +1,136 @@ +//! Plain-text loader — validates and parses raw text content into a +//! [`Document`]. +//! +//! The loader splits the input into lines and records whether the +//! source ended with a trailing newline so the file can be +//! reconstructed after edits. + +use nvisy_core::error::Error; +use nvisy_core::io::ContentData; + +use crate::document::Document; +use crate::handler::{Loader, TxtData, TxtHandler}; + +/// Parameters for [`TxtLoader`]. +#[derive(Debug, Default)] +pub struct TxtParams { + /// Character encoding of the input bytes. + pub encoding: crate::handler::TextEncoding, +} + +/// Loader that validates and parses plain-text files. +/// +/// Produces a single [`Document`] per input. +#[derive(Debug)] +pub struct TxtLoader; + +#[async_trait::async_trait] +impl Loader for TxtLoader { + type Handler = TxtHandler; + type Params = TxtParams; + + async fn load( + &self, + content: &ContentData, + params: &Self::Params, + ) -> Result>, Error> { + let raw = content.to_bytes(); + let text = params.encoding.decode_bytes(&raw, "txt-loader")?; + let trailing_newline = text.ends_with('\n'); + let lines = text.lines().map(String::from).collect(); + + let handler = TxtHandler { + data: TxtData { + lines, + trailing_newline, + }, + }; + let doc = Document::new(handler).with_parent(content); + Ok(vec![doc]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::handler::Handler; + use bytes::Bytes; + use futures::StreamExt; + use nvisy_core::path::ContentSource; + use nvisy_ontology::entity::DocumentType; + + fn content_from_str(s: &str) -> ContentData { + ContentData::new(ContentSource::new(), Bytes::from(s.to_owned())) + } + + #[tokio::test] + async fn load_multiline() { + let content = content_from_str("hello\nworld\n"); + let docs = TxtLoader + .load(&content, &TxtParams::default()) + .await + .unwrap(); + + assert_eq!(docs.len(), 1); + assert_eq!(docs[0].document_type(), DocumentType::Txt); + + let h = docs[0].handler(); + assert_eq!(h.lines(), &["hello", "world"]); + assert!(h.trailing_newline()); + } + + #[tokio::test] + async fn load_no_trailing_newline() { + let content = content_from_str("single line"); + let docs = TxtLoader + .load(&content, &TxtParams::default()) + .await + .unwrap(); + + let h = docs[0].handler(); + assert_eq!(h.line_count(), 1); + assert_eq!(h.line(0), Some("single line")); + assert!(!h.trailing_newline()); + } + + #[tokio::test] + async fn load_empty() { + let content = content_from_str(""); + let docs = TxtLoader + .load(&content, &TxtParams::default()) + .await + .unwrap(); + + let h = docs[0].handler(); + assert_eq!(h.line_count(), 0); + assert!(!h.trailing_newline()); + } + + #[tokio::test] + async fn load_preserves_spans_through_round_trip() { + let content = content_from_str("Alice\nBob\nCharlie\n"); + let docs = TxtLoader + .load(&content, &TxtParams::default()) + .await + .unwrap(); + + let spans: Vec<_> = docs[0].handler().view_spans().await.collect().await; + assert_eq!(spans.len(), 3); + assert_eq!(spans[0].data, "Alice"); + assert_eq!(spans[1].data, "Bob"); + assert_eq!(spans[2].data, "Charlie"); + } + + #[tokio::test] + async fn load_invalid_utf8() { + let content = ContentData::new( + ContentSource::new(), + Bytes::from_static(&[0xFF, 0xFE, 0x00]), + ); + let err = TxtLoader + .load(&content, &TxtParams::default()) + .await + .unwrap_err(); + assert!(err.to_string().contains("UTF-8")); + } +} diff --git a/crates/nvisy-ingest/src/lib.rs b/crates/nvisy-ingest/src/lib.rs new file mode 100644 index 0000000..ed421f6 --- /dev/null +++ b/crates/nvisy-ingest/src/lib.rs @@ -0,0 +1,9 @@ +#![forbid(unsafe_code)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![doc = include_str!("../README.md")] + +pub mod handler; +pub mod document; + +#[doc(hidden)] +pub mod prelude; diff --git a/crates/nvisy-ingest/src/prelude.rs b/crates/nvisy-ingest/src/prelude.rs new file mode 100644 index 0000000..245b2be --- /dev/null +++ b/crates/nvisy-ingest/src/prelude.rs @@ -0,0 +1,15 @@ +//! Convenience re-exports. + +pub use crate::handler::{ + Handler, Loader, TextEncoding, + Span, SpanEdit, + TxtData, TxtHandler, TxtSpan, + TxtLoader, TxtParams, + CsvData, CsvHandler, CsvSpan, + CsvLoader, CsvParams, + JsonData, JsonHandler, JsonIndent, + JsonParams, JsonLoader, JsonPath, +}; +pub use crate::document::view_stream::SpanStream; +pub use crate::document::edit_stream::SpanEditStream; +pub use crate::document::Document; diff --git a/crates/nvisy-object/Cargo.toml b/crates/nvisy-object/Cargo.toml new file mode 100644 index 0000000..ab392ea --- /dev/null +++ b/crates/nvisy-object/Cargo.toml @@ -0,0 +1,51 @@ +# https://doc.rust-lang.org/cargo/reference/manifest.html + +[package] +name = "nvisy-object" +description = "Object store providers and streams (S3, Azure, GCS) for Nvisy" +keywords = ["nvisy", "object-store", "s3", "storage"] +categories = ["filesystem"] + +version = { workspace = true } +rust-version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } + +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +# Internal crates +nvisy-core = { workspace = true, features = [] } +nvisy-pipeline = { workspace = true, features = [] } + +# (De)serialization +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = [] } + +# Async runtime +tokio = { workspace = true, features = ["sync"] } +async-trait = { workspace = true, features = [] } + +# Primitive datatypes +uuid = { workspace = true, features = ["v4"] } +bytes = { workspace = true, features = [] } + +# S3-compatible object storage +minio = { workspace = true, features = [] } + +# Async streams +futures = { workspace = true, features = [] } + +# Error handling +thiserror = { workspace = true, features = [] } + +# Observability +tracing = { workspace = true, features = [] } diff --git a/crates/nvisy-object/README.md b/crates/nvisy-object/README.md new file mode 100644 index 0000000..cb82b98 --- /dev/null +++ b/crates/nvisy-object/README.md @@ -0,0 +1,3 @@ +# nvisy-object + +Object store plugin for the Nvisy runtime. Provides cloud storage providers (S3) and streaming read/write interfaces for ingesting and outputting data through the processing pipeline. diff --git a/crates/nvisy-object/src/client/mod.rs b/crates/nvisy-object/src/client/mod.rs new file mode 100644 index 0000000..df925d2 --- /dev/null +++ b/crates/nvisy-object/src/client/mod.rs @@ -0,0 +1,53 @@ +//! Abstract object-store client trait and helper types. +//! +//! The [`ObjectStoreClient`] trait defines the CRUD surface that every backend +//! (S3, GCS, local filesystem, etc.) must implement. [`ObjectStoreBox`] wraps +//! a concrete client so it can be passed through the engine as `Box`. + +use bytes::Bytes; + +/// Result returned by [`ObjectStoreClient::list`]. +pub struct ListResult { + /// Object keys matching the requested prefix. + pub keys: Vec, + /// Opaque pagination cursor; `None` when there are no more pages. + pub next_cursor: Option, +} + +/// Abstract client for object storage operations. +/// +/// Implementations provide list, get, put, and delete over a single bucket +/// or container. +#[async_trait::async_trait] +pub trait ObjectStoreClient: Send + Sync + 'static { + /// List object keys under `prefix`, optionally continuing from `cursor`. + async fn list(&self, prefix: &str, cursor: Option<&str>) -> Result>; + /// Retrieve the object stored at `key`. + async fn get(&self, key: &str) -> Result>; + /// Upload `data` to `key`, optionally setting the content-type header. + async fn put(&self, key: &str, data: Bytes, content_type: Option<&str>) -> Result<(), Box>; + /// Delete the object at `key`. + async fn delete(&self, key: &str) -> Result<(), Box>; +} + +/// Result returned by [`ObjectStoreClient::get`]. +pub struct GetResult { + /// Raw bytes of the retrieved object. + pub data: Bytes, + /// MIME content-type, if the backend provides one. + pub content_type: Option, +} + +/// Type-erased wrapper around a boxed [`ObjectStoreClient`]. +/// +/// This allows the client to be stored as `Box` inside the +/// engine's `ConnectedInstance` while still being downcasted back to a usable +/// object-store client. +pub struct ObjectStoreBox(pub Box); + +impl ObjectStoreBox { + /// Wrap a concrete [`ObjectStoreClient`] implementation. + pub fn new(client: impl ObjectStoreClient) -> Self { + Self(Box::new(client)) + } +} diff --git a/crates/nvisy-object/src/lib.rs b/crates/nvisy-object/src/lib.rs new file mode 100644 index 0000000..806462b --- /dev/null +++ b/crates/nvisy-object/src/lib.rs @@ -0,0 +1,15 @@ +//! Object storage providers and streams for the nvisy pipeline. +//! +//! This crate provides an abstraction layer over cloud object stores (currently S3) +//! and exposes streaming read/write interfaces that plug into the nvisy engine. + +#![forbid(unsafe_code)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![doc = include_str!("../README.md")] + +pub mod client; +pub mod providers; +pub mod streams; + +#[doc(hidden)] +pub mod prelude; diff --git a/crates/nvisy-object/src/prelude.rs b/crates/nvisy-object/src/prelude.rs new file mode 100644 index 0000000..0217bb4 --- /dev/null +++ b/crates/nvisy-object/src/prelude.rs @@ -0,0 +1,5 @@ +//! Convenience re-exports. +pub use crate::providers::s3::S3Provider; +pub use crate::streams::read::ObjectReadStream; +pub use crate::streams::write::ObjectWriteStream; +pub use crate::streams::{StreamSource, StreamTarget}; diff --git a/crates/nvisy-object/src/providers/mod.rs b/crates/nvisy-object/src/providers/mod.rs new file mode 100644 index 0000000..17b9082 --- /dev/null +++ b/crates/nvisy-object/src/providers/mod.rs @@ -0,0 +1,3 @@ +//! Object storage provider factories. + +pub mod s3; diff --git a/crates/nvisy-object/src/providers/s3.rs b/crates/nvisy-object/src/providers/s3.rs new file mode 100644 index 0000000..c1cc8e7 --- /dev/null +++ b/crates/nvisy-object/src/providers/s3.rs @@ -0,0 +1,183 @@ +//! S3-compatible provider implementation using the MinIO Rust SDK. +//! +//! Provides [`S3ObjectStoreClient`] which implements [`ObjectStoreClient`] and +//! [`S3Provider`] which plugs into the engine's provider system. +//! +//! Works with MinIO, AWS S3, and any S3-compatible service. + +use bytes::Bytes; +use serde::Deserialize; + +use minio::s3::creds::StaticProvider; +use minio::s3::http::BaseUrl; +use minio::s3::types::{S3Api, ToStream}; +use minio::s3::{Client as MinioClient, ClientBuilder as MinioClientBuilder}; + +use nvisy_core::error::Error; +use nvisy_pipeline::provider::{ConnectedInstance, Provider}; +use crate::client::{GetResult, ListResult, ObjectStoreBox, ObjectStoreClient}; + +/// S3-compatible object store client. +/// +/// Wraps the MinIO [`MinioClient`] and scopes all operations to a single bucket. +pub struct S3ObjectStoreClient { + /// Underlying MinIO client. + client: MinioClient, + /// Target S3 bucket name. + bucket: String, +} + +impl S3ObjectStoreClient { + /// Create a new client bound to the given `bucket`. + pub fn new(client: MinioClient, bucket: String) -> Self { + Self { client, bucket } + } +} + +#[async_trait::async_trait] +impl ObjectStoreClient for S3ObjectStoreClient { + async fn list(&self, prefix: &str, cursor: Option<&str>) -> Result> { + use futures::StreamExt; + + let mut builder = self.client + .list_objects(&self.bucket) + .recursive(true) + .use_api_v1(false); + + if !prefix.is_empty() { + builder = builder.prefix(Some(prefix.to_string())); + } + + if let Some(token) = cursor { + builder = builder.continuation_token(Some(token.to_string())); + } + + let mut stream = builder.to_stream().await; + + // Fetch one page + if let Some(result) = stream.next().await { + let resp = result?; + let keys: Vec = resp.contents + .iter() + .filter(|entry| !entry.is_prefix) + .map(|entry| entry.name.clone()) + .collect(); + + let next_cursor = resp.next_continuation_token.clone(); + + Ok(ListResult { keys, next_cursor }) + } else { + Ok(ListResult { keys: vec![], next_cursor: None }) + } + } + + async fn get(&self, key: &str) -> Result> { + let resp = self.client + .get_object(&self.bucket, key) + .send() + .await?; + + let data = resp.content.to_segmented_bytes().await?.to_bytes(); + + Ok(GetResult { data, content_type: None }) + } + + async fn put(&self, key: &str, data: Bytes, content_type: Option<&str>) -> Result<(), Box> { + let content = minio::s3::builders::ObjectContent::from(data); + let mut builder = self.client + .put_object_content(&self.bucket, key, content); + + if let Some(ct) = content_type { + builder = builder.content_type(ct.to_string()); + } + + builder.send().await?; + Ok(()) + } + + async fn delete(&self, key: &str) -> Result<(), Box> { + self.client + .delete_object(&self.bucket, key) + .send() + .await?; + Ok(()) + } +} + +/// Typed credentials for S3-compatible provider. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct S3Credentials { + /// S3 bucket name. + pub bucket: String, + /// AWS region (defaults to `us-east-1`). + #[serde(default = "default_region")] + pub region: String, + /// Endpoint URL (e.g. `http://localhost:9000` for MinIO). + /// Required for non-AWS S3-compatible services. + #[serde(default)] + pub endpoint: Option, + /// Access key ID for static credentials. + #[serde(default)] + pub access_key_id: Option, + /// Secret access key for static credentials. + #[serde(default)] + pub secret_access_key: Option, + /// Session token for temporary credentials. + #[serde(default)] + pub session_token: Option, +} + +fn default_region() -> String { "us-east-1".to_string() } + +/// Factory that creates [`S3ObjectStoreClient`] instances from typed credentials. +pub struct S3Provider; + +#[async_trait::async_trait] +impl Provider for S3Provider { + type Credentials = S3Credentials; + type Client = ObjectStoreBox; + + fn id(&self) -> &str { "s3" } + + fn validate_credentials(&self, _creds: &Self::Credentials) -> Result<(), Error> { + Ok(()) + } + + async fn verify(&self, creds: &Self::Credentials) -> Result<(), Error> { + self.validate_credentials(creds)?; + Ok(()) + } + + async fn connect(&self, creds: &Self::Credentials) -> Result, Error> { + let endpoint = creds.endpoint.as_deref().unwrap_or("https://s3.amazonaws.com"); + + let mut base_url: BaseUrl = endpoint.parse().map_err(|e| { + Error::runtime(format!("invalid endpoint URL: {e}"), "s3/connect", true) + })?; + base_url.region = creds.region.clone(); + + let mut builder = MinioClientBuilder::new(base_url); + + // If access_key and secret_key provided, use static credentials + if let (Some(access_key), Some(secret_key)) = (&creds.access_key_id, &creds.secret_access_key) { + let provider = StaticProvider::new( + access_key, + secret_key, + creds.session_token.as_deref(), + ); + builder = builder.provider(Some(Box::new(provider))); + } + + let client = builder.build().map_err(|e| { + Error::runtime(format!("failed to build MinIO client: {e}"), "s3/connect", true) + })?; + + let store_client = S3ObjectStoreClient::new(client, creds.bucket.clone()); + + Ok(ConnectedInstance { + client: ObjectStoreBox::new(store_client), + disconnect: None, + }) + } +} diff --git a/crates/nvisy-object/src/streams/mod.rs b/crates/nvisy-object/src/streams/mod.rs new file mode 100644 index 0000000..3befadf --- /dev/null +++ b/crates/nvisy-object/src/streams/mod.rs @@ -0,0 +1,60 @@ +//! Streaming read and write adapters for object stores. + +use serde::de::DeserializeOwned; +use tokio::sync::mpsc; + +use nvisy_core::io::ContentData; +use nvisy_core::error::Error; + +/// A source stream that reads content from an external system into the pipeline. +/// +/// Implementations connect to a storage backend (e.g. S3, local filesystem) +/// and emit content data into the pipeline's input channel. +#[async_trait::async_trait] +pub trait StreamSource: Send + Sync + 'static { + /// Strongly-typed parameters for this stream source. + type Params: DeserializeOwned + Send; + /// The client type this stream requires. + type Client: Send + 'static; + + /// Unique identifier for this stream source (e.g. `"s3-read"`). + fn id(&self) -> &str; + + /// Read content from the external system and send it to `output`. + /// + /// Returns the number of items read. + async fn read( + &self, + output: mpsc::Sender, + params: Self::Params, + client: Self::Client, + ) -> Result; +} + +/// A target stream that writes content from the pipeline to an external system. +/// +/// Implementations receive processed content data from the pipeline and persist +/// it to a storage backend. +#[async_trait::async_trait] +pub trait StreamTarget: Send + Sync + 'static { + /// Strongly-typed parameters for this stream target. + type Params: DeserializeOwned + Send; + /// The client type this stream requires. + type Client: Send + 'static; + + /// Unique identifier for this stream target (e.g. `"s3-write"`). + fn id(&self) -> &str; + + /// Receive content from `input` and write it to the external system. + /// + /// Returns the number of items written. + async fn write( + &self, + input: mpsc::Receiver, + params: Self::Params, + client: Self::Client, + ) -> Result; +} + +pub mod read; +pub mod write; diff --git a/crates/nvisy-object/src/streams/read.rs b/crates/nvisy-object/src/streams/read.rs new file mode 100644 index 0000000..90fb8a4 --- /dev/null +++ b/crates/nvisy-object/src/streams/read.rs @@ -0,0 +1,84 @@ +//! Streaming reader that pulls objects from an S3-compatible store. + +use serde::Deserialize; +use tokio::sync::mpsc; + +use nvisy_core::io::ContentData; +use nvisy_core::path::ContentSource; +use nvisy_core::error::Error; +use super::StreamSource; +use crate::client::ObjectStoreBox; + +/// Typed parameters for [`ObjectReadStream`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ObjectReadParams { + /// Object key prefix to filter by. + #[serde(default)] + pub prefix: String, + /// Number of keys to fetch per page. + #[serde(default = "default_batch_size")] + pub batch_size: usize, +} + +fn default_batch_size() -> usize { 100 } + +/// A [`StreamSource`] that lists and fetches objects from an S3-compatible store, +/// emitting each object as a [`ContentData`] onto the output channel. +pub struct ObjectReadStream; + +#[async_trait::async_trait] +impl StreamSource for ObjectReadStream { + type Params = ObjectReadParams; + type Client = ObjectStoreBox; + + fn id(&self) -> &str { "read" } + + async fn read( + &self, + output: mpsc::Sender, + params: Self::Params, + client: Self::Client, + ) -> Result { + let store_client = &client.0; + + let prefix = ¶ms.prefix; + let batch_size = params.batch_size; + + let mut cursor: Option = None; + let mut total = 0u64; + + loop { + let result = store_client + .list(prefix, cursor.as_deref()) + .await + .map_err(|e| Error::runtime(format!("List failed: {}", e), "object/read", true))?; + + let keys_count = result.keys.len(); + + for key in &result.keys { + let get_result = store_client + .get(key) + .await + .map_err(|e| Error::runtime(format!("Get failed for {}: {}", key, e), "object/read", true))?; + + let mut content = ContentData::new(ContentSource::new(), get_result.data); + if let Some(ct) = get_result.content_type { + content = content.with_content_type(ct); + } + + total += 1; + if output.send(content).await.is_err() { + return Ok(total); + } + } + + if keys_count < batch_size || result.next_cursor.is_none() { + break; + } + cursor = result.next_cursor; + } + + Ok(total) + } +} diff --git a/crates/nvisy-object/src/streams/write.rs b/crates/nvisy-object/src/streams/write.rs new file mode 100644 index 0000000..75902a9 --- /dev/null +++ b/crates/nvisy-object/src/streams/write.rs @@ -0,0 +1,60 @@ +//! Streaming writer that uploads content to an S3-compatible store. + +use serde::Deserialize; +use tokio::sync::mpsc; + +use nvisy_core::io::ContentData; +use nvisy_core::error::Error; +use super::StreamTarget; +use crate::client::ObjectStoreBox; + +/// Typed parameters for [`ObjectWriteStream`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ObjectWriteParams { + /// Key prefix prepended to each content source UUID. + #[serde(default)] + pub prefix: String, +} + +/// A [`StreamTarget`] that receives [`ContentData`] from the input channel and +/// uploads each one to an S3-compatible object store. +pub struct ObjectWriteStream; + +#[async_trait::async_trait] +impl StreamTarget for ObjectWriteStream { + type Params = ObjectWriteParams; + type Client = ObjectStoreBox; + + fn id(&self) -> &str { "write" } + + async fn write( + &self, + mut input: mpsc::Receiver, + params: Self::Params, + client: Self::Client, + ) -> Result { + let store_client = &client.0; + + let prefix = ¶ms.prefix; + let mut total = 0u64; + + while let Some(content) = input.recv().await { + let source_id = content.content_source.to_string(); + let key = if prefix.is_empty() { + source_id + } else { + format!("{}{}", prefix, source_id) + }; + + store_client + .put(&key, content.to_bytes(), content.content_type()) + .await + .map_err(|e| Error::runtime(format!("Put failed for {}: {}", key, e), "object/write", true))?; + + total += 1; + } + + Ok(total) + } +} diff --git a/crates/nvisy-ontology/Cargo.toml b/crates/nvisy-ontology/Cargo.toml new file mode 100644 index 0000000..a5bb2e8 --- /dev/null +++ b/crates/nvisy-ontology/Cargo.toml @@ -0,0 +1,43 @@ +# https://doc.rust-lang.org/cargo/reference/manifest.html + +[package] +name = "nvisy-ontology" +description = "Detection ontology and redaction policy types for the Nvisy platform" +keywords = ["nvisy", "ontology", "redaction", "policy"] +categories = ["data-structures"] + +version = { workspace = true } +rust-version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } + +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } + +[features] +default = [] +jsonschema = ["dep:schemars"] + +[dependencies] +# Internal crates +nvisy-core = { workspace = true, features = [] } + +# JSON Schema generation (optional) +schemars = { workspace = true, optional = true, features = [] } + +# (De)serialization +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = [] } + +# Primitive datatypes +uuid = { workspace = true, features = ["serde", "v4"] } +jiff = { workspace = true, features = [] } + +# Semantic versioning +semver = { workspace = true, features = [] } + +# Error handling +derive_more = { workspace = true, features = ["display", "from"] } diff --git a/crates/nvisy-ontology/README.md b/crates/nvisy-ontology/README.md new file mode 100644 index 0000000..00ccb86 --- /dev/null +++ b/crates/nvisy-ontology/README.md @@ -0,0 +1,3 @@ +# nvisy-ontology + +Detection ontology and redaction policy types for the Nvisy platform. Defines entities, redaction methods, audit records, and policy rules that all detection and redaction crates depend on. diff --git a/crates/nvisy-ontology/src/audit/explanation.rs b/crates/nvisy-ontology/src/audit/explanation.rs new file mode 100644 index 0000000..c4b70fd --- /dev/null +++ b/crates/nvisy-ontology/src/audit/explanation.rs @@ -0,0 +1,48 @@ +//! Explainability metadata for data protection decisions. +//! +//! An [`Explanation`] records why an action was taken — which model, rule, +//! and confidence level were involved. Types that carry this metadata +//! implement the [`Explainable`] trait. + +use semver::Version; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::entity::{DetectionMethod, ModelInfo}; + +/// Types that carry explainability metadata. +pub trait Explainable { + /// Why this action was taken. + fn explanation(&self) -> Option<&Explanation>; +} + +/// Structured explainability metadata for a data protection decision. +/// +/// Records why an action was taken, which model and rule were involved, +/// and who reviewed it. Complements the freeform `details` field on [`Audit`](super::Audit). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct Explanation { + /// Detection model that produced the decision. + #[serde(skip_serializing_if = "Option::is_none")] + pub model: Option, + /// Identifier of the policy rule that triggered the action. + #[serde(skip_serializing_if = "Option::is_none")] + pub rule_id: Option, + /// Detection confidence score. + #[serde(skip_serializing_if = "Option::is_none")] + pub confidence: Option, + /// Detection method used. + #[serde(skip_serializing_if = "Option::is_none")] + pub detection_method: Option, + /// Human-readable reason for the action. + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// Version of the policy that was evaluated. + #[serde(skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "jsonschema", schemars(with = "Option"))] + pub policy_version: Option, + /// Identifier of the reviewer who approved/rejected. + #[serde(skip_serializing_if = "Option::is_none")] + pub reviewer_id: Option, +} diff --git a/crates/nvisy-ontology/src/audit/mod.rs b/crates/nvisy-ontology/src/audit/mod.rs new file mode 100644 index 0000000..fad2171 --- /dev/null +++ b/crates/nvisy-ontology/src/audit/mod.rs @@ -0,0 +1,73 @@ +//! Audit trail records for data protection events. +//! +//! An [`Audit`] entry records an immutable event in the data protection +//! pipeline, carrying structured [`Explanation`] metadata for compliance. + +mod explanation; +mod retention; + +pub use explanation::{Explainable, Explanation}; +pub use retention::{RetentionPolicy, RetentionScope}; + +use jiff::Timestamp; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use nvisy_core::path::ContentSource; + +/// Types that emit audit records. +pub trait Auditable { + /// Produce an audit record for this event. + fn to_audit(&self) -> Audit; +} + +/// Kind of auditable action recorded in an [`Audit`] entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum AuditAction { + /// A sensitive entity was detected. + Detection, + /// A redaction was applied to an entity. + Redaction, + /// A human review was performed on a redaction. + Review, +} + +/// An immutable audit record tracking a data protection event. +/// +/// Audit entries are emitted by pipeline actions and form a tamper-evident +/// log of all detection, redaction, and policy decisions. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct Audit { + /// Content source identity and lineage. + #[serde(flatten)] + pub source: ContentSource, + /// The kind of event this audit entry records. + pub action: AuditAction, + /// UTC timestamp when the event occurred. + #[cfg_attr(feature = "jsonschema", schemars(with = "String"))] + pub timestamp: Timestamp, + /// Identifier of the related entity, if applicable. + #[serde(skip_serializing_if = "Option::is_none")] + pub entity_id: Option, + /// Identifier of the related redaction, if applicable. + #[serde(skip_serializing_if = "Option::is_none")] + pub redaction_id: Option, + /// Identifier of the policy that was evaluated, if applicable. + #[serde(skip_serializing_if = "Option::is_none")] + pub policy_id: Option, + /// Identifier of the source blob or document. + #[serde(skip_serializing_if = "Option::is_none")] + pub source_id: Option, + /// Identifier of the pipeline run that produced this entry. + #[serde(skip_serializing_if = "Option::is_none")] + pub run_id: Option, + /// Human or service account that triggered the event. + #[serde(skip_serializing_if = "Option::is_none")] + pub actor: Option, + /// Structured explainability metadata. + #[serde(skip_serializing_if = "Option::is_none")] + pub explanation: Option, +} diff --git a/crates/nvisy-ontology/src/audit/retention.rs b/crates/nvisy-ontology/src/audit/retention.rs new file mode 100644 index 0000000..612b636 --- /dev/null +++ b/crates/nvisy-ontology/src/audit/retention.rs @@ -0,0 +1,47 @@ +//! Data retention policy types. + +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +/// What class of data a retention policy applies to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum RetentionScope { + /// Original ingested content before redaction. + OriginalContent, + /// Redacted output artifacts. + RedactedOutput, + /// Audit log entries. + AuditLogs, +} + +/// A retention policy governing how long data is kept. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct RetentionPolicy { + /// What class of data this policy applies to. + pub scope: RetentionScope, + /// Maximum number of days to retain data. `None` means indefinite. + #[serde(skip_serializing_if = "Option::is_none")] + pub max_duration_days: Option, + /// If true, delete data immediately after processing (zero-retention mode). + pub zero_retention: bool, + /// Description of the retention policy. + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, +} + +impl RetentionPolicy { + /// Returns the retention duration, or `None` for indefinite retention. + /// + /// Returns [`Duration::ZERO`] when `zero_retention` is `true`. + pub fn duration(&self) -> Option { + if self.zero_retention { + return Some(Duration::ZERO); + } + self.max_duration_days + .map(|days| Duration::from_secs(days * 24 * 60 * 60)) + } +} diff --git a/crates/nvisy-ontology/src/detection/annotation.rs b/crates/nvisy-ontology/src/detection/annotation.rs new file mode 100644 index 0000000..eb7b8ae --- /dev/null +++ b/crates/nvisy-ontology/src/detection/annotation.rs @@ -0,0 +1,78 @@ +//! Annotation types for pre-identified regions and classification labels. +//! +//! Annotations allow users and upstream systems to mark regions of content +//! before detection runs. They replace the previous `ManualAnnotation` type +//! with a unified model supporting three kinds: inclusions (pre-identified +//! sensitive regions), exclusions (known-safe regions to skip), and +//! classification labels. + +use serde::{Deserialize, Serialize}; + +use crate::entity::{ + AudioLocation, EntityCategory, ImageLocation, TabularLocation, TextLocation, VideoLocation, +}; + +/// The kind of annotation applied to a content region. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum AnnotationKind { + /// Pre-identified sensitive region that should be treated as a detection. + Inclusion, + /// Known-safe region that detection should skip. + Exclusion, + /// Classification label attached to a document or region. + Label, +} + +/// A classification label attached to a document or region. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct AnnotationLabel { + /// Label name (e.g. `"contains-phi"`, `"gdpr-request"`). + pub name: String, + /// Scope of the label: `"document"` or a region identifier. + #[serde(skip_serializing_if = "Option::is_none")] + pub scope: Option, + /// Confidence of the label assignment. + #[serde(skip_serializing_if = "Option::is_none")] + pub confidence: Option, +} + +/// A user-provided or upstream annotation on a content region. +/// +/// Replaces the previous `ManualAnnotation` with a unified type that +/// supports inclusions, exclusions, and classification labels. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct Annotation { + /// What kind of annotation this is. + pub kind: AnnotationKind, + /// Entity category, if applicable. + #[serde(skip_serializing_if = "Option::is_none")] + pub category: Option, + /// Entity type label, if applicable. + #[serde(skip_serializing_if = "Option::is_none")] + pub entity_type: Option, + /// The annotated text or value. + #[serde(skip_serializing_if = "Option::is_none")] + pub value: Option, + /// Text location of the annotated region. + #[serde(skip_serializing_if = "Option::is_none")] + pub text_location: Option, + /// Image location of the annotated region. + #[serde(skip_serializing_if = "Option::is_none")] + pub image_location: Option, + /// Tabular location of the annotated region. + #[serde(skip_serializing_if = "Option::is_none")] + pub tabular_location: Option, + /// Audio location of the annotated region. + #[serde(skip_serializing_if = "Option::is_none")] + pub audio_location: Option, + /// Video location of the annotated region. + #[serde(skip_serializing_if = "Option::is_none")] + pub video_location: Option, + /// Classification labels attached to this annotation. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub labels: Vec, +} diff --git a/crates/nvisy-ontology/src/detection/classification.rs b/crates/nvisy-ontology/src/detection/classification.rs new file mode 100644 index 0000000..f107961 --- /dev/null +++ b/crates/nvisy-ontology/src/detection/classification.rs @@ -0,0 +1,15 @@ +//! Sensitivity classification result. + +use serde::{Deserialize, Serialize}; + +use super::Sensitivity; + +/// Result of sensitivity classification over a set of detected entities. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct ClassificationResult { + /// Sensitivity assessment (level + risk score). + pub sensitivity: Sensitivity, + /// Total number of entities considered. + pub total_entities: usize, +} diff --git a/crates/nvisy-ontology/src/detection/mod.rs b/crates/nvisy-ontology/src/detection/mod.rs new file mode 100644 index 0000000..9af8a56 --- /dev/null +++ b/crates/nvisy-ontology/src/detection/mod.rs @@ -0,0 +1,40 @@ +//! Detection result types. +//! +//! A [`DetectionResult`] groups the output of a detection pass as a +//! first-class type, carrying the detected entities alongside pipeline +//! and policy metadata. + +mod annotation; +mod classification; +mod sensitivity; + +pub use annotation::{Annotation, AnnotationKind, AnnotationLabel}; +pub use classification::ClassificationResult; +pub use sensitivity::{Sensitivity, SensitivityLevel}; + +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use nvisy_core::path::ContentSource; + +use crate::entity::Entity; + +/// The output of a detection pass over a single content source. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct DetectionResult { + /// Content source identity and lineage. + #[serde(flatten)] + pub source: ContentSource, + /// Entities detected in the content. + pub entities: Vec, + /// Identifier of the policy that governed detection. + #[serde(skip_serializing_if = "Option::is_none")] + pub policy_id: Option, + /// Processing time in milliseconds. + #[serde(skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + /// Overall sensitivity assessment derived from the detected entities. + #[serde(skip_serializing_if = "Option::is_none")] + pub sensitivity: Option, +} diff --git a/crates/nvisy-ontology/src/detection/sensitivity.rs b/crates/nvisy-ontology/src/detection/sensitivity.rs new file mode 100644 index 0000000..783f2a7 --- /dev/null +++ b/crates/nvisy-ontology/src/detection/sensitivity.rs @@ -0,0 +1,38 @@ +//! Sensitivity level and assessment types. + +use serde::{Deserialize, Serialize}; + +/// Sensitivity classification assigned to a document or content region. +/// +/// Drives downstream policy: rules can be scoped to specific sensitivity +/// levels via [`RuleCondition`](crate::policy::RuleCondition). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum SensitivityLevel { + /// No sensitive data detected or all data is publicly available. + Public, + /// Internal use only — not intended for external distribution. + Internal, + /// Contains sensitive data requiring access controls. + Confidential, + /// Highly sensitive — regulated data requiring strict controls. + Restricted, +} + +/// Combined sensitivity assessment for a content source. +/// +/// Pairs a discrete [`SensitivityLevel`] with an optional continuous +/// re-identification risk score in `[0.0, 1.0]`. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct Sensitivity { + /// Discrete sensitivity classification. + pub level: SensitivityLevel, + /// Re-identification risk score in the range `[0.0, 1.0]`. + /// + /// Estimates the likelihood that a data subject could be re-identified + /// from the entities remaining after redaction. + #[serde(skip_serializing_if = "Option::is_none")] + pub risk_score: Option, +} diff --git a/crates/nvisy-ontology/src/entity/document.rs b/crates/nvisy-ontology/src/entity/document.rs new file mode 100644 index 0000000..01d6d68 --- /dev/null +++ b/crates/nvisy-ontology/src/entity/document.rs @@ -0,0 +1,32 @@ +//! Document format classification. + +use serde::{Deserialize, Serialize}; + +/// Document format that content can be classified as. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum DocumentType { + /// Plain text (`.txt`, `.log`, etc.). + Txt, + /// Comma-separated values. + Csv, + /// JSON data. + Json, + /// HTML pages. + Html, + /// PDF documents. + Pdf, + /// Microsoft Word (`.docx`). + Docx, + /// Microsoft Excel (`.xlsx`). + Xlsx, + /// PNG image. + Png, + /// JPEG image. + Jpeg, + /// WAV audio. + Wav, + /// MP3 audio. + Mp3, +} diff --git a/crates/nvisy-ontology/src/entity/location.rs b/crates/nvisy-ontology/src/entity/location.rs new file mode 100644 index 0000000..d8a4ba9 --- /dev/null +++ b/crates/nvisy-ontology/src/entity/location.rs @@ -0,0 +1,115 @@ +//! Spatial and temporal location types for entity positions. + +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// A time interval within an audio or video stream. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct TimeSpan { + /// Start time in seconds from the beginning of the stream. + pub start_secs: f64, + /// End time in seconds from the beginning of the stream. + pub end_secs: f64, +} + +/// Axis-aligned bounding box for image-based entity locations. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct BoundingBox { + /// Horizontal offset of the top-left corner (pixels or normalized). + pub x: f64, + /// Vertical offset of the top-left corner (pixels or normalized). + pub y: f64, + /// Width of the bounding box. + pub width: f64, + /// Height of the bounding box. + pub height: f64, +} + +/// Location of an entity within text content. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct TextLocation { + /// Byte or character offset where the entity starts. + pub start_offset: usize, + /// Byte or character offset where the entity ends. + pub end_offset: usize, + /// Start offset of the surrounding context window for redaction. + /// + /// When set, the redaction may expand to cover surrounding text + /// (e.g. +/- N characters around an SSN) to prevent contextual + /// re-identification. + #[serde(skip_serializing_if = "Option::is_none")] + pub context_start_offset: Option, + /// End offset of the surrounding context window for redaction. + #[serde(skip_serializing_if = "Option::is_none")] + pub context_end_offset: Option, + /// Identifier of the document element containing this entity. + #[serde(skip_serializing_if = "Option::is_none")] + pub element_id: Option, + /// 1-based page number where the entity was found. + #[serde(skip_serializing_if = "Option::is_none")] + pub page_number: Option, +} + +/// Location of an entity within an image. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct ImageLocation { + /// Bounding box of the entity in the image. + pub bounding_box: BoundingBox, + /// Links this entity to a specific image document. + #[serde(skip_serializing_if = "Option::is_none")] + pub image_id: Option, + /// 1-based page number (for multi-page documents like PDFs). + #[serde(skip_serializing_if = "Option::is_none")] + pub page_number: Option, +} + +/// Location of an entity within tabular data. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct TabularLocation { + /// Row index (0-based). + pub row_index: usize, + /// Column index (0-based). + pub column_index: usize, + /// Byte offset within the cell where the entity starts, if applicable. + #[serde(skip_serializing_if = "Option::is_none")] + pub start_offset: Option, + /// Byte offset within the cell where the entity ends, if applicable. + #[serde(skip_serializing_if = "Option::is_none")] + pub end_offset: Option, +} + +/// Location of an entity within an audio stream. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct AudioLocation { + /// Time interval of the entity. + pub time_span: TimeSpan, + /// Speaker identifier from diarization. + #[serde(skip_serializing_if = "Option::is_none")] + pub speaker_id: Option, +} + +/// Location of an entity within a video stream. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct VideoLocation { + /// Bounding box of the entity in the frame. + pub bounding_box: BoundingBox, + /// 0-based frame number where the entity was detected. + pub frame_number: u64, + /// Time interval of the entity in the video. + #[serde(skip_serializing_if = "Option::is_none")] + pub time_span: Option, + /// Tracking identifier for an entity across multiple frames. + #[serde(skip_serializing_if = "Option::is_none")] + pub track_id: Option, + /// Speaker identifier from diarization (for audio track). + #[serde(skip_serializing_if = "Option::is_none")] + pub speaker_id: Option, +} + diff --git a/crates/nvisy-ontology/src/entity/mod.rs b/crates/nvisy-ontology/src/entity/mod.rs new file mode 100644 index 0000000..88fe2d3 --- /dev/null +++ b/crates/nvisy-ontology/src/entity/mod.rs @@ -0,0 +1,183 @@ +//! Sensitive-data entity types and detection metadata. +//! +//! An [`Entity`] represents a single occurrence of sensitive data detected +//! within a document. Entities are produced by detection actions and consumed +//! by redaction and audit stages of the pipeline. + +mod document; +mod location; +mod model; +mod selector; + +pub use document::DocumentType; +pub use location::{ + AudioLocation, BoundingBox, ImageLocation, TabularLocation, + TextLocation, TimeSpan, VideoLocation, +}; +pub use model::{ModelInfo, ModelKind}; +pub use selector::EntitySelector; + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use nvisy_core::path::ContentSource; + +/// Category of sensitive data an entity belongs to. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum EntityCategory { + /// Personally Identifiable Information (names, SSNs, addresses, etc.). + Pii, + /// Protected Health Information (HIPAA-regulated data). + Phi, + /// Financial data (credit card numbers, bank accounts, etc.). + Financial, + /// Secrets and credentials (API keys, passwords, tokens). + Credentials, + /// Legal documents and privileged communications. + Legal, + /// Biometric data (fingerprints, iris scans, voiceprints). + Biometric, + /// User-defined or plugin-specific category. + Custom(String), +} + +/// Method used to detect a sensitive entity. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum DetectionMethod { + /// Regular expression pattern matching. + Regex, + /// Checksum or Luhn-algorithm validation. + Checksum, + /// Lookup in a known-value dictionary. + Dictionary, + /// Named-entity recognition via AI model. + Ner, + /// Contextual NLP analysis (discourse-level understanding). + ContextualNlp, + /// OCR text extraction with bounding boxes. + Ocr, + /// Face detection in images or video frames. + FaceDetection, + /// Object detection in images or video frames. + ObjectDetection, + /// Entity detection from speech transcription. + SpeechTranscript, + /// Multiple methods combined to produce a single detection. + Composite, + /// User-provided annotations. + Manual, +} + +/// A detected sensitive data occurrence within a document. +/// +/// Entities are produced by detection actions (regex, NER, checksum, etc.) +/// and later consumed by redaction and audit actions. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct Entity { + /// Content source identity and lineage. + #[serde(flatten)] + pub source: ContentSource, + /// Broad classification of the sensitive data. + pub category: EntityCategory, + /// Specific type label (e.g. `"ssn"`, `"email"`, `"credit_card"`). + pub entity_type: String, + /// The matched text or value. + pub value: String, + /// How this entity was detected. + pub detection_method: DetectionMethod, + /// Detection confidence score in the range `[0.0, 1.0]`. + pub confidence: f64, + /// Text location, if this entity was found in text content. + #[serde(skip_serializing_if = "Option::is_none")] + pub text_location: Option, + /// Image location, if this entity was found in an image. + #[serde(skip_serializing_if = "Option::is_none")] + pub image_location: Option, + /// Tabular location, if this entity was found in tabular data. + #[serde(skip_serializing_if = "Option::is_none")] + pub tabular_location: Option, + /// Audio location, if this entity was found in audio. + #[serde(skip_serializing_if = "Option::is_none")] + pub audio_location: Option, + /// Video location, if this entity was found in video. + #[serde(skip_serializing_if = "Option::is_none")] + pub video_location: Option, + /// BCP-47 language tag of the detected content. + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + /// Detection model that produced this entity. + #[serde(skip_serializing_if = "Option::is_none")] + pub model: Option, + /// Additional unstructured metadata. + #[serde(skip_serializing_if = "Option::is_none")] + pub metadata: Option>, +} + +impl Entity { + /// Create a new entity with the given detection details. + pub fn new( + category: EntityCategory, + entity_type: impl Into, + value: impl Into, + detection_method: DetectionMethod, + confidence: f64, + ) -> Self { + Self { + source: ContentSource::new(), + category, + entity_type: entity_type.into(), + value: value.into(), + detection_method, + confidence, + text_location: None, + image_location: None, + tabular_location: None, + audio_location: None, + video_location: None, + language: None, + model: None, + metadata: None, + } + } + + /// Set a text location on this entity. + pub fn with_text_location(mut self, location: TextLocation) -> Self { + self.text_location = Some(location); + self + } + + /// Set an image location on this entity. + pub fn with_image_location(mut self, location: ImageLocation) -> Self { + self.image_location = Some(location); + self + } + + /// Set a tabular location on this entity. + pub fn with_tabular_location(mut self, location: TabularLocation) -> Self { + self.tabular_location = Some(location); + self + } + + /// Set an audio location on this entity. + pub fn with_audio_location(mut self, location: AudioLocation) -> Self { + self.audio_location = Some(location); + self + } + + /// Set a video location on this entity. + pub fn with_video_location(mut self, location: VideoLocation) -> Self { + self.video_location = Some(location); + self + } + + /// Set the parent source for lineage tracking. + pub fn with_parent(mut self, parent: &ContentSource) -> Self { + self.source = self.source.with_parent(parent); + self + } +} diff --git a/crates/nvisy-ontology/src/entity/model.rs b/crates/nvisy-ontology/src/entity/model.rs new file mode 100644 index 0000000..a372704 --- /dev/null +++ b/crates/nvisy-ontology/src/entity/model.rs @@ -0,0 +1,30 @@ +//! Detection model identity and provenance. + +use serde::{Deserialize, Serialize}; + +/// Provenance or licensing classification of a detection model. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum ModelKind { + /// Open-source model (e.g. spaCy, Hugging Face community models). + OpenSource, + /// Proprietary model (e.g. vendor-specific NER). + Proprietary, + /// Model accessed through a third-party API gateway. + Gateway, + /// Self-hosted model served behind an internal endpoint. + SelfHosted, +} + +/// Identity and version of the model used for detection. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct ModelInfo { + /// Model name (e.g. `"spacy-en-core-web-lg"`, `"gpt-4"`). + pub name: String, + /// Provenance / licensing classification. + pub kind: ModelKind, + /// Model version string. + pub version: String, +} diff --git a/crates/nvisy-ontology/src/entity/selector.rs b/crates/nvisy-ontology/src/entity/selector.rs new file mode 100644 index 0000000..19bfb51 --- /dev/null +++ b/crates/nvisy-ontology/src/entity/selector.rs @@ -0,0 +1,65 @@ +//! Entity selection criteria for policy rules. +//! +//! An [`EntitySelector`] describes which entities a policy rule or redaction +//! applies to, based on category, type, and confidence constraints. + +use serde::{Deserialize, Serialize}; + +use super::EntityCategory; + +/// Criteria for selecting which entities a policy rule applies to. +/// +/// All fields use "empty means all" semantics: an empty `categories` list +/// matches every category, an empty `entity_types` list matches every type, +/// and so on. When multiple fields are set, they are combined with AND logic. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct EntitySelector { + /// Entity categories this selector matches. Empty means all categories. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub categories: Vec, + /// Specific entity type names this selector matches. Empty means all types. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub entity_types: Vec, + /// Minimum detection confidence required. Entities below this threshold + /// are not matched. + #[serde(default = "default_confidence_threshold")] + pub confidence_threshold: f64, +} + +fn default_confidence_threshold() -> f64 { + 0.0 +} + +impl Default for EntitySelector { + fn default() -> Self { + Self { + categories: Vec::new(), + entity_types: Vec::new(), + confidence_threshold: default_confidence_threshold(), + } + } +} + +impl EntitySelector { + /// Create a selector that matches all entities. + pub fn all() -> Self { + Self::default() + } + + /// Returns `true` if the given entity properties match this selector. + pub fn matches(&self, category: &EntityCategory, entity_type: &str, confidence: f64) -> bool { + if confidence < self.confidence_threshold { + return false; + } + if !self.categories.is_empty() && !self.categories.contains(category) { + return false; + } + if !self.entity_types.is_empty() + && !self.entity_types.iter().any(|t| t == entity_type) + { + return false; + } + true + } +} diff --git a/crates/nvisy-ontology/src/lib.rs b/crates/nvisy-ontology/src/lib.rs new file mode 100644 index 0000000..48686f2 --- /dev/null +++ b/crates/nvisy-ontology/src/lib.rs @@ -0,0 +1,12 @@ +#![forbid(unsafe_code)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![doc = include_str!("../README.md")] + +pub mod audit; +pub mod detection; +pub mod entity; +pub mod policy; +pub mod redaction; + +#[doc(hidden)] +pub mod prelude; diff --git a/crates/nvisy-ontology/src/policy/evaluation.rs b/crates/nvisy-ontology/src/policy/evaluation.rs new file mode 100644 index 0000000..d0f540b --- /dev/null +++ b/crates/nvisy-ontology/src/policy/evaluation.rs @@ -0,0 +1,27 @@ +//! Policy evaluation outcome. + +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::redaction::Redaction; + +/// Full outcome of evaluating a [`Policy`](crate::policy::Policy) against a set of entities. +/// +/// Captures every rule kind's effect: redactions to apply, entities pending +/// human review, entities suppressed from output, blocked entities, and alerts. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct PolicyEvaluation { + /// Identifier of the policy that was evaluated. + pub policy_id: Uuid, + /// Redactions produced by `Redact` rules. + pub redactions: Vec, + /// Entity IDs routed to human review by `Review` rules. + pub pending_review: Vec, + /// Entity IDs suppressed from output by `Suppress` rules. + pub suppressed: Vec, + /// Entity IDs blocked from processing by `Block` rules. + pub blocked: Vec, + /// Entity IDs that triggered alert notifications via `Alert` rules. + pub alerted: Vec, +} diff --git a/crates/nvisy-ontology/src/policy/mod.rs b/crates/nvisy-ontology/src/policy/mod.rs new file mode 100644 index 0000000..35de77e --- /dev/null +++ b/crates/nvisy-ontology/src/policy/mod.rs @@ -0,0 +1,58 @@ +//! Redaction policies and rules. +//! +//! A [`Policy`] is a named, versioned set of [`PolicyRule`]s that govern +//! how detected entities are redacted. Policies may be associated with a +//! [`RegulationKind`] and support inheritance via the `extends` field. + +mod evaluation; +mod regulation; +mod rule; + +pub use evaluation::PolicyEvaluation; +pub use regulation::RegulationKind; +pub use rule::{PolicyRule, RuleCondition, RuleKind}; + +use semver::Version; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::redaction::RedactionSpec; + +/// A named redaction policy containing an ordered set of rules. +/// +/// Policies are pure configuration — they describe *what* to detect and +/// *how* to handle it, independent of any specific content source. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct Policy { + /// Unique identifier for this policy. + pub id: Uuid, + /// Human-readable policy name. + pub name: String, + /// Policy version. + #[cfg_attr(feature = "jsonschema", schemars(with = "String"))] + pub version: Version, + /// Description of the policy's purpose. + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Parent policy identifier for inheritance. + #[serde(skip_serializing_if = "Option::is_none")] + pub extends: Option, + /// Compliance regulation this policy targets. + #[serde(skip_serializing_if = "Option::is_none")] + pub regulation: Option, + /// Ordered list of rules. + pub rules: Vec, + /// Fallback redaction specification when no rule matches. + pub default_spec: RedactionSpec, + /// Fallback confidence threshold when no rule matches. + pub default_confidence_threshold: f64, +} + +/// A collection of policies to apply during a pipeline run. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct Policies { + /// The policies to evaluate, in order. + pub policies: Vec, +} diff --git a/crates/nvisy-ontology/src/policy/regulation.rs b/crates/nvisy-ontology/src/policy/regulation.rs new file mode 100644 index 0000000..8007b1f --- /dev/null +++ b/crates/nvisy-ontology/src/policy/regulation.rs @@ -0,0 +1,26 @@ +//! Regulatory framework identifiers. + +use serde::{Deserialize, Serialize}; + +/// A compliance regulation or framework that a policy targets. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum RegulationKind { + /// Health Insurance Portability and Accountability Act. + Hipaa, + /// General Data Protection Regulation (EU). + Gdpr, + /// California Consumer Privacy Act. + Ccpa, + /// Payment Card Industry Data Security Standard. + PciDss, + /// Criminal Justice Information Services Security Policy. + Cjis, + /// Family Educational Rights and Privacy Act. + Ferpa, + /// Sarbanes-Oxley Act. + Sox, + /// User-defined regulation or framework. + Custom(String), +} diff --git a/crates/nvisy-ontology/src/policy/rule.rs b/crates/nvisy-ontology/src/policy/rule.rs new file mode 100644 index 0000000..c7e1af5 --- /dev/null +++ b/crates/nvisy-ontology/src/policy/rule.rs @@ -0,0 +1,71 @@ +//! Policy rule types. +//! +//! A [`PolicyRule`] defines when and how a specific redaction is applied, +//! based on entity categories, types, and confidence thresholds. + +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::detection::SensitivityLevel; +use crate::entity::{DocumentType, EntitySelector}; +use crate::redaction::RedactionSpec; + +/// Conditions that must be met for a [`PolicyRule`] to apply. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct RuleCondition { + /// Document formats this rule applies to. Empty means all formats. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub document_types: Vec, + /// User roles this rule applies to. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub roles: Vec, + /// Labels that must be present on the document. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub required_labels: Vec, + /// Sensitivity levels this rule applies to. Empty means all levels. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub sensitivity_levels: Vec, +} + +/// Classifies what a policy rule does when it matches. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum RuleKind { + /// Apply a redaction to the matched entity. + Redaction, + /// Require human review before any action is taken. + Review, + /// Flag the entity without redacting (for reporting / alerting). + Alert, + /// Block processing of the entire document. + Block, + /// Suppress a detection (treat as false positive). + Suppress, +} + +/// A single rule within a redaction [`Policy`](super::Policy). +/// +/// Rules specify which entity categories and types they match, the minimum +/// confidence threshold, and the action to take. Rules are evaluated in +/// ascending priority order. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct PolicyRule { + /// Unique identifier for this rule. + pub id: Uuid, + /// What this rule does when it matches. + pub kind: RuleKind, + /// Which entities this rule applies to. + pub selector: EntitySelector, + /// Redaction specification to apply when this rule matches (relevant when `kind` is `Redaction`). + pub spec: RedactionSpec, + /// Template string for the replacement value (e.g. `"[REDACTED]"`). + pub replacement_template: String, + /// Evaluation priority (lower numbers are evaluated first). + pub priority: i32, + /// Additional conditions for this rule to apply. + #[serde(skip_serializing_if = "Option::is_none")] + pub conditions: Option, +} diff --git a/crates/nvisy-ontology/src/prelude.rs b/crates/nvisy-ontology/src/prelude.rs new file mode 100644 index 0000000..c2dc957 --- /dev/null +++ b/crates/nvisy-ontology/src/prelude.rs @@ -0,0 +1,23 @@ +//! Convenience re-exports for common nvisy-ontology types. + +pub use crate::audit::{ + Audit, AuditAction, Auditable, Explainable, Explanation, RetentionPolicy, RetentionScope, +}; +pub use crate::detection::{ + Annotation, AnnotationKind, AnnotationLabel, ClassificationResult, DetectionResult, + Sensitivity, SensitivityLevel, +}; +pub use crate::entity::{ + AudioLocation, BoundingBox, DetectionMethod, DocumentType, Entity, EntityCategory, + EntitySelector, ImageLocation, ModelInfo, ModelKind, TabularLocation, + TextLocation, TimeSpan, VideoLocation, +}; +pub use crate::policy::{ + Policies, Policy, PolicyEvaluation, PolicyRule, RegulationKind, RuleCondition, RuleKind, +}; +pub use crate::redaction::{ + AudioRedactionMethod, AudioRedactionOutput, AudioRedactionSpec, ImageRedactionMethod, + ImageRedactionOutput, ImageRedactionSpec, Redactable, Redaction, RedactionMethod, + RedactionOutput, RedactionSpec, RedactionSummary, ReviewDecision, ReviewStatus, + TextRedactionMethod, TextRedactionOutput, TextRedactionSpec, +}; diff --git a/crates/nvisy-ontology/src/redaction/method.rs b/crates/nvisy-ontology/src/redaction/method.rs new file mode 100644 index 0000000..13290a2 --- /dev/null +++ b/crates/nvisy-ontology/src/redaction/method.rs @@ -0,0 +1,109 @@ +//! Plain-tag redaction method enums. +//! +//! These are lightweight identifiers that name a redaction algorithm without +//! carrying any configuration data. For a data-carrying request see +//! [`RedactionSpec`](super::RedactionSpec); for a data-carrying result see +//! [`RedactionOutput`](super::RedactionOutput). + +use derive_more::From; +use serde::{Deserialize, Serialize}; + +/// Redaction strategies for text and tabular content. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum TextRedactionMethod { + /// Replace characters with a mask character (e.g. `***-**-1234`). + Mask, + /// Substitute with a fixed placeholder string. + Replace, + /// Replace with a one-way hash of the original value. + Hash, + /// Encrypt the value so it can be recovered later with a key. + Encrypt, + /// Remove the value entirely from the output. + Remove, + /// Replace with a synthetically generated realistic value. + Synthesize, + /// Replace with a consistent pseudonym across the document. + Pseudonymize, + /// Replace with a vault-backed reversible token (e.g. `USER_001`). + Tokenize, + /// Aggregate value into a range or bucket (e.g. age 34 → 30-39). + Aggregate, + /// Generalize to a less precise value (e.g. street → city → country). + Generalize, + /// Shift dates by a random but consistent offset, preserving intervals. + DateShift, +} + +/// Redaction strategies for image and video regions. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum ImageRedactionMethod { + /// Apply a gaussian blur to the region. + Blur, + /// Overlay an opaque block over the region. + Block, + /// Apply pixelation to the region (mosaic effect). + Pixelate, + /// Replace with a synthetically generated region. + Synthesize, +} + +/// Redaction strategies for audio segments. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum AudioRedactionMethod { + /// Replace the audio segment with silence. + Silence, + /// Remove the audio segment entirely. + Remove, + /// Replace with synthetically generated audio. + Synthesize, +} + +/// Unified redaction strategy tag that wraps modality-specific methods. +/// +/// This is a lightweight identifier — it names the algorithm but carries no +/// configuration data. For a data-carrying request use [`RedactionSpec`](super::RedactionSpec); +/// for a data-carrying result use [`RedactionOutput`](super::RedactionOutput). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, From, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum RedactionMethod { + /// Text/tabular redaction strategy. + Text(TextRedactionMethod), + /// Image/video redaction strategy. + Image(ImageRedactionMethod), + /// Audio redaction strategy. + Audio(AudioRedactionMethod), +} + +impl RedactionMethod { + /// Returns the text redaction method if this is a text variant. + pub fn as_text(&self) -> Option { + match self { + Self::Text(m) => Some(*m), + _ => None, + } + } + + /// Returns the image redaction method if this is an image variant. + pub fn as_image(&self) -> Option { + match self { + Self::Image(m) => Some(*m), + _ => None, + } + } + + /// Returns the audio redaction method if this is an audio variant. + pub fn as_audio(&self) -> Option { + match self { + Self::Audio(m) => Some(*m), + _ => None, + } + } +} diff --git a/crates/nvisy-ontology/src/redaction/mod.rs b/crates/nvisy-ontology/src/redaction/mod.rs new file mode 100644 index 0000000..568a419 --- /dev/null +++ b/crates/nvisy-ontology/src/redaction/mod.rs @@ -0,0 +1,111 @@ +//! Redaction methods, specifications, outputs, and records. +//! +//! This module contains three layers of redaction types: +//! +//! 1. **Method** ([`RedactionMethod`]) — a plain tag enum naming a redaction +//! strategy. Used as a lightweight identifier (e.g. in logs, serialized +//! references, or when the caller only needs to know *which* algorithm). +//! +//! 2. **Spec** ([`RedactionSpec`]) — a data-carrying enum that describes a +//! redaction request submitted to the engine: which method to apply and +//! the configuration parameters it needs (mask char, blur sigma, key id, +//! etc.). Used on [`PolicyRule`](crate::policy::PolicyRule) and +//! [`Policy`](crate::policy::Policy). +//! +//! 3. **Output** ([`RedactionOutput`]) — a data-carrying enum that records +//! what was actually done and the result data (replacement string, +//! ciphertext, shifted date, etc.). Stored on [`Redaction`]. +//! +//! All three are organized by modality: +//! - Text / tabular: [`TextRedactionMethod`], [`TextRedactionSpec`], [`TextRedactionOutput`] +//! - Image / video: [`ImageRedactionMethod`], [`ImageRedactionSpec`], [`ImageRedactionOutput`] +//! - Audio: [`AudioRedactionMethod`], [`AudioRedactionSpec`], [`AudioRedactionOutput`] + +mod method; +mod output; +mod review; +mod spec; +mod summary; + +pub use method::{ + AudioRedactionMethod, ImageRedactionMethod, RedactionMethod, TextRedactionMethod, +}; +pub use output::{ + AudioRedactionOutput, ImageRedactionOutput, RedactionOutput, TextRedactionOutput, +}; +pub use review::{ReviewDecision, ReviewStatus}; +pub use spec::{AudioRedactionSpec, ImageRedactionSpec, RedactionSpec, TextRedactionSpec}; +pub use summary::RedactionSummary; + +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use nvisy_core::path::ContentSource; + +use crate::entity::Entity; +use crate::policy::Policy; + +/// Types that produce redaction decisions. +pub trait Redactable { + /// The entities detected in this content. + fn entities(&self) -> &[Entity]; + /// The policy governing redaction. + fn policy(&self) -> Option<&Policy>; +} + +/// A redaction decision recording how a specific entity was (or will be) redacted. +/// +/// Each `Redaction` is linked to exactly one [`Entity`](crate::entity::Entity) +/// via `entity_id`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct Redaction { + /// Content source identity and lineage. + #[serde(flatten)] + pub source: ContentSource, + /// Identifier of the entity being redacted. + pub entity_id: Uuid, + /// Redaction output recording the method used and its result data. + pub output: RedactionOutput, + /// The original sensitive value, retained for audit purposes. + #[serde(skip_serializing_if = "Option::is_none")] + pub original_value: Option, + /// Identifier of the policy rule that triggered this redaction. + #[serde(skip_serializing_if = "Option::is_none")] + pub policy_rule_id: Option, + /// Whether the redaction has been applied to the output content. + pub applied: bool, + /// Version of this redaction record (starts at 1, incremented on modification). + pub version: u32, + /// Human review decision, if any. + #[serde(skip_serializing_if = "Option::is_none")] + pub review: Option, +} + +impl Redaction { + /// Create a new pending redaction for the given entity. + pub fn new(entity_id: Uuid, output: impl Into) -> Self { + Self { + source: ContentSource::new(), + entity_id, + output: output.into(), + original_value: None, + policy_rule_id: None, + applied: false, + version: 1, + review: None, + } + } + + /// Record the original sensitive value for audit trail purposes. + pub fn with_original_value(mut self, value: impl Into) -> Self { + self.original_value = Some(value.into()); + self + } + + /// Associate this redaction with the policy rule that triggered it. + pub fn with_policy_rule_id(mut self, id: Uuid) -> Self { + self.policy_rule_id = Some(id); + self + } +} diff --git a/crates/nvisy-ontology/src/redaction/output.rs b/crates/nvisy-ontology/src/redaction/output.rs new file mode 100644 index 0000000..5392b9c --- /dev/null +++ b/crates/nvisy-ontology/src/redaction/output.rs @@ -0,0 +1,152 @@ +//! Data-carrying redaction output enums recording what was done. +//! +//! A [`RedactionOutput`] records the method that was applied and its result +//! data (replacement string, ciphertext, blur sigma, etc.). Stored on +//! [`Redaction`](super::Redaction). + +use derive_more::From; +use serde::{Deserialize, Serialize}; + +use super::method::{ + AudioRedactionMethod, ImageRedactionMethod, RedactionMethod, TextRedactionMethod, +}; + +/// Text redaction output — records the method used and its replacement data. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(tag = "method", rename_all = "snake_case")] +pub enum TextRedactionOutput { + /// Characters replaced with a mask character. + Mask { + replacement: String, + mask_char: char, + }, + /// Substituted with a fixed placeholder string. + Replace { replacement: String }, + /// Replaced with a one-way hash. + Hash { hash_value: String }, + /// Encrypted; recoverable with the referenced key. + Encrypt { ciphertext: String, key_id: String }, + /// Removed entirely from the output. + Remove, + /// Replaced with a synthetically generated value. + Synthesize { replacement: String }, + /// Replaced with a consistent pseudonym. + Pseudonymize { pseudonym: String }, + /// Replaced with a vault-backed reversible token. + Tokenize { + token: String, + vault_id: Option, + }, + /// Aggregated into a range or bucket. + Aggregate { replacement: String }, + /// Generalized to a less precise value. + Generalize { + replacement: String, + level: Option, + }, + /// Date shifted by a consistent offset. + DateShift { + replacement: String, + offset_days: i64, + }, +} + +/// Image redaction output — records the method used and its parameters. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(tag = "method", rename_all = "snake_case")] +pub enum ImageRedactionOutput { + /// Gaussian blur applied to the region. + Blur { sigma: f32 }, + /// Opaque block overlay on the region. + Block { color: [u8; 4] }, + /// Pixelation (mosaic) applied to the region. + Pixelate { block_size: u32 }, + /// Region replaced with a synthetic image. + Synthesize, +} + +/// Audio redaction output — records the method used. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(tag = "method", rename_all = "snake_case")] +pub enum AudioRedactionOutput { + /// Segment replaced with silence. + Silence, + /// Segment removed entirely. + Remove, + /// Segment replaced with synthetic audio. + Synthesize, +} + +/// Unified redaction output that wraps modality-specific output variants. +/// +/// Carries method-specific result data (replacement strings, ciphertext, +/// blur sigma, etc.). Stored on [`Redaction`](super::Redaction). +#[derive(Debug, Clone, PartialEq, From, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum RedactionOutput { + /// Text/tabular redaction output. + Text(TextRedactionOutput), + /// Image/video redaction output. + Image(ImageRedactionOutput), + /// Audio redaction output. + Audio(AudioRedactionOutput), +} + +impl RedactionOutput { + /// Returns the text replacement string, regardless of specific method. + /// + /// Used by apply actions that just need to know "what string goes here". + /// Returns `None` for image and audio outputs, or text `Remove`. + pub fn replacement_value(&self) -> Option<&str> { + match self { + Self::Text(t) => match t { + TextRedactionOutput::Mask { replacement, .. } => Some(replacement), + TextRedactionOutput::Replace { replacement } => Some(replacement), + TextRedactionOutput::Hash { hash_value } => Some(hash_value), + TextRedactionOutput::Encrypt { ciphertext, .. } => Some(ciphertext), + TextRedactionOutput::Remove => None, + TextRedactionOutput::Synthesize { replacement } => Some(replacement), + TextRedactionOutput::Pseudonymize { pseudonym } => Some(pseudonym), + TextRedactionOutput::Tokenize { token, .. } => Some(token), + TextRedactionOutput::Aggregate { replacement } => Some(replacement), + TextRedactionOutput::Generalize { replacement, .. } => Some(replacement), + TextRedactionOutput::DateShift { replacement, .. } => Some(replacement), + }, + Self::Image(_) | Self::Audio(_) => None, + } + } + + /// Returns the [`RedactionMethod`] tag this output corresponds to. + pub fn method(&self) -> RedactionMethod { + match self { + Self::Text(t) => RedactionMethod::Text(match t { + TextRedactionOutput::Mask { .. } => TextRedactionMethod::Mask, + TextRedactionOutput::Replace { .. } => TextRedactionMethod::Replace, + TextRedactionOutput::Hash { .. } => TextRedactionMethod::Hash, + TextRedactionOutput::Encrypt { .. } => TextRedactionMethod::Encrypt, + TextRedactionOutput::Remove => TextRedactionMethod::Remove, + TextRedactionOutput::Synthesize { .. } => TextRedactionMethod::Synthesize, + TextRedactionOutput::Pseudonymize { .. } => TextRedactionMethod::Pseudonymize, + TextRedactionOutput::Tokenize { .. } => TextRedactionMethod::Tokenize, + TextRedactionOutput::Aggregate { .. } => TextRedactionMethod::Aggregate, + TextRedactionOutput::Generalize { .. } => TextRedactionMethod::Generalize, + TextRedactionOutput::DateShift { .. } => TextRedactionMethod::DateShift, + }), + Self::Image(i) => RedactionMethod::Image(match i { + ImageRedactionOutput::Blur { .. } => ImageRedactionMethod::Blur, + ImageRedactionOutput::Block { .. } => ImageRedactionMethod::Block, + ImageRedactionOutput::Pixelate { .. } => ImageRedactionMethod::Pixelate, + ImageRedactionOutput::Synthesize => ImageRedactionMethod::Synthesize, + }), + Self::Audio(a) => RedactionMethod::Audio(match a { + AudioRedactionOutput::Silence => AudioRedactionMethod::Silence, + AudioRedactionOutput::Remove => AudioRedactionMethod::Remove, + AudioRedactionOutput::Synthesize => AudioRedactionMethod::Synthesize, + }), + } + } +} diff --git a/crates/nvisy-ontology/src/redaction/review.rs b/crates/nvisy-ontology/src/redaction/review.rs new file mode 100644 index 0000000..d2e25fd --- /dev/null +++ b/crates/nvisy-ontology/src/redaction/review.rs @@ -0,0 +1,35 @@ +//! Human-in-the-loop review types. + +use jiff::Timestamp; +use serde::{Deserialize, Serialize}; + +/// Status of a human review on a redaction decision. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum ReviewStatus { + /// Awaiting human review. + Pending, + /// A human reviewer approved the redaction. + Approved, + /// A human reviewer rejected the redaction. + Rejected, + /// Automatically approved by policy (no human review required). + AutoApproved, +} + +/// A review decision recorded against a redaction. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct ReviewDecision { + /// Outcome of the review. + pub status: ReviewStatus, + /// Identifier of the reviewer (human or service account). + pub reviewer_id: String, + /// When the review decision was made. + #[cfg_attr(feature = "jsonschema", schemars(with = "String"))] + pub timestamp: Timestamp, + /// Optional reason for the decision. + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, +} diff --git a/crates/nvisy-ontology/src/redaction/spec.rs b/crates/nvisy-ontology/src/redaction/spec.rs new file mode 100644 index 0000000..5c7a039 --- /dev/null +++ b/crates/nvisy-ontology/src/redaction/spec.rs @@ -0,0 +1,168 @@ +//! Data-carrying redaction specifications submitted to the engine. +//! +//! A [`RedactionSpec`] describes *how* to redact — which method to apply and +//! the configuration parameters it needs (mask char, blur sigma, encryption +//! key id, etc.). Used on [`PolicyRule`](crate::policy::PolicyRule) and +//! [`Policy`](crate::policy::Policy). + +use derive_more::From; +use serde::{Deserialize, Serialize}; + +use super::method::{ + AudioRedactionMethod, ImageRedactionMethod, RedactionMethod, TextRedactionMethod, +}; + +/// Text redaction specification with method-specific configuration. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(tag = "method", rename_all = "snake_case")] +pub enum TextRedactionSpec { + /// Replace characters with a mask character. + Mask { + /// Character used for masking (default `'*'`). + #[serde(default = "default_mask_char")] + mask_char: char, + }, + /// Substitute with a fixed placeholder string. + Replace { + /// Template for the replacement (supports `{entityType}`, `{category}`, `{value}`). + #[serde(default)] + placeholder: String, + }, + /// Replace with a one-way hash. + Hash, + /// Encrypt the value; recoverable with the referenced key. + Encrypt { + /// Identifier of the encryption key to use. + key_id: String, + }, + /// Remove the value entirely. + Remove, + /// Replace with a synthetically generated value. + Synthesize, + /// Replace with a consistent pseudonym. + Pseudonymize, + /// Replace with a vault-backed reversible token. + Tokenize { + /// Identifier of the token vault. + #[serde(default)] + vault_id: Option, + }, + /// Aggregate into a range or bucket. + Aggregate, + /// Generalize to a less precise value. + Generalize { + /// Generalization level (1 = city, 2 = state, etc.). + #[serde(default)] + level: Option, + }, + /// Shift dates by a consistent offset. + DateShift { + /// Fixed offset in days (0 = engine picks a random offset). + #[serde(default)] + offset_days: i64, + }, +} + +fn default_mask_char() -> char { + '*' +} + +/// Image redaction specification with method-specific configuration. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(tag = "method", rename_all = "snake_case")] +pub enum ImageRedactionSpec { + /// Apply a gaussian blur. + Blur { + /// Blur sigma value. + #[serde(default = "default_sigma")] + sigma: f32, + }, + /// Overlay an opaque block. + Block { + /// RGBA color for the block. + #[serde(default = "default_block_color")] + color: [u8; 4], + }, + /// Apply pixelation (mosaic). + Pixelate { + /// Pixel block size. + #[serde(default = "default_block_size")] + block_size: u32, + }, + /// Replace with a synthetic region. + Synthesize, +} + +fn default_sigma() -> f32 { + 15.0 +} +fn default_block_color() -> [u8; 4] { + [0, 0, 0, 255] +} +fn default_block_size() -> u32 { + 10 +} + +/// Audio redaction specification. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(tag = "method", rename_all = "snake_case")] +pub enum AudioRedactionSpec { + /// Replace with silence. + Silence, + /// Remove the segment entirely. + Remove, + /// Replace with synthetic audio. + Synthesize, +} + +/// Unified redaction specification submitted to the engine. +/// +/// Carries the method to apply and its configuration parameters. +/// Used on [`PolicyRule`](crate::policy::PolicyRule) and +/// [`Policy`](crate::policy::Policy). +#[derive(Debug, Clone, PartialEq, From, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum RedactionSpec { + /// Text/tabular redaction specification. + Text(TextRedactionSpec), + /// Image/video redaction specification. + Image(ImageRedactionSpec), + /// Audio redaction specification. + Audio(AudioRedactionSpec), +} + +impl RedactionSpec { + /// Returns the [`RedactionMethod`] tag this spec corresponds to. + pub fn method(&self) -> RedactionMethod { + match self { + Self::Text(t) => RedactionMethod::Text(match t { + TextRedactionSpec::Mask { .. } => TextRedactionMethod::Mask, + TextRedactionSpec::Replace { .. } => TextRedactionMethod::Replace, + TextRedactionSpec::Hash => TextRedactionMethod::Hash, + TextRedactionSpec::Encrypt { .. } => TextRedactionMethod::Encrypt, + TextRedactionSpec::Remove => TextRedactionMethod::Remove, + TextRedactionSpec::Synthesize => TextRedactionMethod::Synthesize, + TextRedactionSpec::Pseudonymize => TextRedactionMethod::Pseudonymize, + TextRedactionSpec::Tokenize { .. } => TextRedactionMethod::Tokenize, + TextRedactionSpec::Aggregate => TextRedactionMethod::Aggregate, + TextRedactionSpec::Generalize { .. } => TextRedactionMethod::Generalize, + TextRedactionSpec::DateShift { .. } => TextRedactionMethod::DateShift, + }), + Self::Image(i) => RedactionMethod::Image(match i { + ImageRedactionSpec::Blur { .. } => ImageRedactionMethod::Blur, + ImageRedactionSpec::Block { .. } => ImageRedactionMethod::Block, + ImageRedactionSpec::Pixelate { .. } => ImageRedactionMethod::Pixelate, + ImageRedactionSpec::Synthesize => ImageRedactionMethod::Synthesize, + }), + Self::Audio(a) => RedactionMethod::Audio(match a { + AudioRedactionSpec::Silence => AudioRedactionMethod::Silence, + AudioRedactionSpec::Remove => AudioRedactionMethod::Remove, + AudioRedactionSpec::Synthesize => AudioRedactionMethod::Synthesize, + }), + } + } +} diff --git a/crates/nvisy-ontology/src/redaction/summary.rs b/crates/nvisy-ontology/src/redaction/summary.rs new file mode 100644 index 0000000..5246387 --- /dev/null +++ b/crates/nvisy-ontology/src/redaction/summary.rs @@ -0,0 +1,18 @@ +//! Per-source redaction summary. + +use serde::{Deserialize, Serialize}; + +use nvisy_core::path::ContentSource; + +/// Summary of redactions applied to a single content source. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +pub struct RedactionSummary { + /// The content source these counts apply to. + #[serde(flatten)] + pub source: ContentSource, + /// Number of redactions successfully applied. + pub redactions_applied: usize, + /// Number of redactions skipped (e.g. due to review holds or errors). + pub redactions_skipped: usize, +} diff --git a/crates/nvisy-pattern/Cargo.toml b/crates/nvisy-pattern/Cargo.toml new file mode 100644 index 0000000..f51b7c9 --- /dev/null +++ b/crates/nvisy-pattern/Cargo.toml @@ -0,0 +1,30 @@ +# https://doc.rust-lang.org/cargo/reference/manifest.html + +[package] +name = "nvisy-pattern" +description = "Built-in regex patterns and dictionaries for PII/PHI detection" +keywords = ["nvisy", "pattern", "pii", "dictionary"] +categories = ["text-processing"] + +version = { workspace = true } +rust-version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } + +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +# Internal crates +nvisy-ontology = { workspace = true, features = [] } + +# (De)serialization +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = [] } diff --git a/crates/nvisy-pattern/assets/dictionaries/first_names.txt b/crates/nvisy-pattern/assets/dictionaries/first_names.txt new file mode 100644 index 0000000..08fee00 --- /dev/null +++ b/crates/nvisy-pattern/assets/dictionaries/first_names.txt @@ -0,0 +1,50 @@ +James +Mary +Robert +Patricia +John +Jennifer +Michael +Linda +David +Elizabeth +William +Barbara +Richard +Susan +Joseph +Jessica +Thomas +Sarah +Christopher +Karen +Charles +Lisa +Daniel +Nancy +Matthew +Betty +Anthony +Margaret +Mark +Sandra +Donald +Ashley +Steven +Dorothy +Paul +Kimberly +Andrew +Emily +Joshua +Donna +Kenneth +Michelle +Kevin +Carol +Brian +Amanda +George +Melissa +Timothy +Deborah diff --git a/crates/nvisy-pattern/assets/dictionaries/last_names.txt b/crates/nvisy-pattern/assets/dictionaries/last_names.txt new file mode 100644 index 0000000..161ccd1 --- /dev/null +++ b/crates/nvisy-pattern/assets/dictionaries/last_names.txt @@ -0,0 +1,50 @@ +Smith +Johnson +Williams +Brown +Jones +Garcia +Miller +Davis +Rodriguez +Martinez +Hernandez +Lopez +Gonzalez +Wilson +Anderson +Thomas +Taylor +Moore +Jackson +Martin +Lee +Perez +Thompson +White +Harris +Sanchez +Clark +Ramirez +Lewis +Robinson +Walker +Young +Allen +King +Wright +Scott +Torres +Nguyen +Hill +Flores +Green +Adams +Nelson +Baker +Hall +Rivera +Campbell +Mitchell +Carter +Roberts diff --git a/crates/nvisy-pattern/assets/dictionaries/medical_terms.txt b/crates/nvisy-pattern/assets/dictionaries/medical_terms.txt new file mode 100644 index 0000000..e0a5d04 --- /dev/null +++ b/crates/nvisy-pattern/assets/dictionaries/medical_terms.txt @@ -0,0 +1,50 @@ +diabetes +hypertension +asthma +cancer +HIV +AIDS +hepatitis +tuberculosis +epilepsy +schizophrenia +bipolar +depression +anxiety +COPD +pneumonia +bronchitis +arthritis +osteoporosis +Alzheimer +Parkinson +dementia +leukemia +lymphoma +melanoma +carcinoma +chemotherapy +radiation +dialysis +transplant +amputation +prosthetic +insulin +metformin +lisinopril +atorvastatin +metoprolol +omeprazole +amlodipine +gabapentin +hydrocodone +oxycodone +morphine +fentanyl +naloxone +prednisone +warfarin +heparin +diagnosis +prognosis +pathology diff --git a/crates/nvisy-pattern/assets/patterns.json b/crates/nvisy-pattern/assets/patterns.json new file mode 100644 index 0000000..a5ae239 --- /dev/null +++ b/crates/nvisy-pattern/assets/patterns.json @@ -0,0 +1,74 @@ +[ + { + "name": "ssn", + "category": "pii", + "entity_type": "ssn", + "pattern": "\\b(\\d{3})-(\\d{2})-(\\d{4})\\b", + "confidence": 0.9, + "validator": "ssn" + }, + { + "name": "email", + "category": "pii", + "entity_type": "email", + "pattern": "\\b[a-zA-Z0-9._%+\\-]+@[a-zA-Z0-9.\\-]+\\.[a-zA-Z]{2,}\\b", + "confidence": 0.95 + }, + { + "name": "phone", + "category": "pii", + "entity_type": "phone", + "pattern": "(?:\\+\\d{1,3}[\\s.\\-]?)?\\(?\\d{2,4}\\)?[\\s.\\-]?\\d{3,4}[\\s.\\-]?\\d{4}\\b", + "confidence": 0.8 + }, + { + "name": "credit-card", + "category": "financial", + "entity_type": "credit_card", + "pattern": "\\b(?:\\d[ \\-]*?){13,19}\\b", + "confidence": 0.85, + "validator": "luhn" + }, + { + "name": "aws-key", + "category": "credentials", + "entity_type": "aws_access_key", + "pattern": "\\bAKIA[0-9A-Z]{16}\\b", + "confidence": 0.95 + }, + { + "name": "github-token", + "category": "credentials", + "entity_type": "github_token", + "pattern": "\\bgh[pousr]_[a-zA-Z0-9]{36}\\b", + "confidence": 0.95 + }, + { + "name": "stripe-key", + "category": "credentials", + "entity_type": "stripe_key", + "pattern": "\\bsk_(live|test)_[a-zA-Z0-9]{24,}\\b", + "confidence": 0.95 + }, + { + "name": "generic-api-key", + "category": "credentials", + "entity_type": "api_key", + "pattern": "(?i)(?:api[_\\-]?key|api[_\\-]?secret|access[_\\-]?token|secret[_\\-]?key|bearer)\\s*[:=]\\s*[\"']?([a-zA-Z0-9_\\-]{20,})[\"']?", + "confidence": 0.7 + }, + { + "name": "ipv4", + "category": "pii", + "entity_type": "ip_address", + "pattern": "\\b(?:(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.){3}(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\b", + "confidence": 0.75 + }, + { + "name": "ipv6", + "category": "pii", + "entity_type": "ip_address", + "pattern": "\\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\b|(?:[0-9a-fA-F]{1,4}:){1,7}:|::(?:[0-9a-fA-F]{1,4}:){0,5}[0-9a-fA-F]{1,4}\\b", + "confidence": 0.75 + } +] diff --git a/crates/nvisy-pattern/src/dictionaries/mod.rs b/crates/nvisy-pattern/src/dictionaries/mod.rs new file mode 100644 index 0000000..4ec9e16 --- /dev/null +++ b/crates/nvisy-pattern/src/dictionaries/mod.rs @@ -0,0 +1,41 @@ +//! Built-in dictionary data for name and term matching. +//! +//! Dictionaries are embedded at compile time via `include_str!()` and +//! loaded lazily on first access. + +use std::sync::LazyLock; + +static FIRST_NAMES: LazyLock> = LazyLock::new(|| { + parse_dictionary(include_str!("../../assets/dictionaries/first_names.txt")) +}); + +static LAST_NAMES: LazyLock> = LazyLock::new(|| { + parse_dictionary(include_str!("../../assets/dictionaries/last_names.txt")) +}); + +static MEDICAL_TERMS: LazyLock> = LazyLock::new(|| { + parse_dictionary(include_str!("../../assets/dictionaries/medical_terms.txt")) +}); + +/// Load a built-in dictionary by name. +/// +/// Names are prefixed with `"builtin:"` — e.g. `"builtin:first_names"`, +/// `"builtin:last_names"`, `"builtin:medical_terms"`. +/// +/// Returns `None` if the name is not recognized. +pub fn get_builtin(name: &str) -> Option<&'static [String]> { + match name { + "builtin:first_names" => Some(&FIRST_NAMES), + "builtin:last_names" => Some(&LAST_NAMES), + "builtin:medical_terms" => Some(&MEDICAL_TERMS), + _ => None, + } +} + +fn parse_dictionary(text: &str) -> Vec { + text.lines() + .map(|l| l.trim()) + .filter(|l| !l.is_empty()) + .map(|l| l.to_string()) + .collect() +} diff --git a/crates/nvisy-pattern/src/lib.rs b/crates/nvisy-pattern/src/lib.rs new file mode 100644 index 0000000..e927e45 --- /dev/null +++ b/crates/nvisy-pattern/src/lib.rs @@ -0,0 +1,16 @@ +//! Built-in regex patterns and dictionaries for PII/PHI detection. +//! +//! This crate provides the embedded pattern definitions (compiled from +//! `assets/patterns.json`) and dictionary data (first names, last names, +//! medical terms) used by the nvisy pipeline's detection actions. + +#![forbid(unsafe_code)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +/// Built-in regex pattern definitions and validation helpers. +pub mod patterns; +/// Built-in dictionary data for name and term matching. +pub mod dictionaries; + +#[doc(hidden)] +pub mod prelude; diff --git a/crates/nvisy-pattern/src/patterns/mod.rs b/crates/nvisy-pattern/src/patterns/mod.rs new file mode 100644 index 0000000..f4b79f2 --- /dev/null +++ b/crates/nvisy-pattern/src/patterns/mod.rs @@ -0,0 +1,110 @@ +//! Built-in regex pattern definitions and validation helpers. +//! +//! Patterns are loaded at startup from the embedded `assets/patterns.json` +//! file and compiled into a static registry keyed by pattern name. + +/// Checksum and format validators used by pattern definitions. +pub mod validators; + +use std::collections::HashMap; +use std::sync::LazyLock; + +use nvisy_ontology::entity::EntityCategory; + +/// JSON representation of a pattern loaded from disk. +#[derive(Debug, Clone, serde::Deserialize)] +struct PatternJson { + /// Human-readable pattern name (used as the registry key). + name: String, + /// Category string (e.g. `"pii"`, `"phi"`, `"financial"`). + category: String, + /// The entity type tag emitted when this pattern matches. + entity_type: String, + /// The regex pattern string. + pattern: String, + /// Base confidence score assigned to matches. + confidence: f64, + /// Optional validator name resolved at load time (e.g. `"ssn"`, `"luhn"`). + #[serde(default)] + validator: Option, +} + +/// A compiled regex-based detection pattern with optional post-match validation. +pub struct PatternDefinition { + /// Unique name identifying this pattern in the registry. + pub name: String, + /// The entity category (PII, PHI, Financial, etc.). + pub category: EntityCategory, + /// The entity type tag emitted on match (e.g. `"ssn"`, `"credit_card"`). + pub entity_type: String, + /// The raw regex pattern string. + pub pattern_str: String, + /// Base confidence score assigned to matches of this pattern. + pub confidence: f64, + /// Optional validation function applied after a regex match succeeds. + pub validate: Option bool>, +} + +/// Maps a category string from `patterns.json` to its [`EntityCategory`] variant. +fn parse_category(s: &str) -> EntityCategory { + match s { + "pii" => EntityCategory::Pii, + "phi" => EntityCategory::Phi, + "financial" => EntityCategory::Financial, + "credentials" => EntityCategory::Credentials, + other => EntityCategory::Custom(other.to_string()), + } +} + +/// Resolves a validator name string to its corresponding validation function. +fn resolve_validator(name: &str) -> Option bool> { + match name { + "ssn" => Some(validators::validate_ssn), + "luhn" => Some(validators::luhn_check), + _ => None, + } +} + +/// Deserializes and compiles all patterns from the embedded `patterns.json` asset. +fn load_patterns() -> Vec { + let json_bytes = include_bytes!("../../assets/patterns.json"); + let raw: Vec = + serde_json::from_slice(json_bytes).expect("Failed to parse patterns.json"); + + raw.into_iter() + .map(|p| PatternDefinition { + category: parse_category(&p.category), + validate: p.validator.as_deref().and_then(resolve_validator), + name: p.name, + entity_type: p.entity_type, + pattern_str: p.pattern, + confidence: p.confidence, + }) + .collect() +} + +static PATTERNS: LazyLock> = LazyLock::new(load_patterns); + +static REGISTRY: LazyLock> = + LazyLock::new(|| { + let mut map = HashMap::new(); + for p in PATTERNS.iter() { + map.insert(p.name.as_str(), p); + } + map + }); + +/// Look up a built-in pattern by name. +pub fn get_pattern(name: &str) -> Option<&'static PatternDefinition> { + REGISTRY.get(name).copied() +} + +/// Get all built-in patterns. +pub fn get_all_patterns() -> Vec<&'static PatternDefinition> { + REGISTRY.values().copied().collect() +} + +/// Get all built-in pattern names. +pub fn get_all_pattern_names() -> Vec<&'static str> { + REGISTRY.keys().copied().collect() +} diff --git a/crates/nvisy-pattern/src/patterns/validators.rs b/crates/nvisy-pattern/src/patterns/validators.rs new file mode 100644 index 0000000..b18baad --- /dev/null +++ b/crates/nvisy-pattern/src/patterns/validators.rs @@ -0,0 +1,47 @@ +//! Checksum and format validators for detected entity values. +//! +//! These functions are referenced by pattern definitions in `patterns.json` +//! and are also used directly by the checksum detection action. + +/// Validate a US Social Security Number. +pub fn validate_ssn(value: &str) -> bool { + let parts: Vec<&str> = value.split('-').collect(); + if parts.len() != 3 { + return false; + } + let area: u32 = match parts[0].parse() { + Ok(v) => v, + Err(_) => return false, + }; + let group: u32 = match parts[1].parse() { + Ok(v) => v, + Err(_) => return false, + }; + let serial: u32 = match parts[2].parse() { + Ok(v) => v, + Err(_) => return false, + }; + area > 0 && area < 900 && area != 666 && group > 0 && serial > 0 +} + +/// Luhn check algorithm for credit card validation. +pub fn luhn_check(num: &str) -> bool { + let digits: String = num.chars().filter(|c| c.is_ascii_digit()).collect(); + if digits.is_empty() { + return false; + } + let mut sum = 0u32; + let mut alternate = false; + for ch in digits.chars().rev() { + let mut n = ch.to_digit(10).unwrap_or(0); + if alternate { + n *= 2; + if n > 9 { + n -= 9; + } + } + sum += n; + alternate = !alternate; + } + sum % 10 == 0 +} diff --git a/crates/nvisy-pattern/src/prelude.rs b/crates/nvisy-pattern/src/prelude.rs new file mode 100644 index 0000000..2a352cd --- /dev/null +++ b/crates/nvisy-pattern/src/prelude.rs @@ -0,0 +1,4 @@ +//! Convenience re-exports for common nvisy-pattern types. + +pub use crate::patterns::{PatternDefinition, get_all_pattern_names, get_all_patterns, get_pattern}; +pub use crate::dictionaries::get_builtin; diff --git a/crates/nvisy-pipeline/Cargo.toml b/crates/nvisy-pipeline/Cargo.toml new file mode 100644 index 0000000..368e7ba --- /dev/null +++ b/crates/nvisy-pipeline/Cargo.toml @@ -0,0 +1,62 @@ +# https://doc.rust-lang.org/cargo/reference/manifest.html + +[package] +name = "nvisy-pipeline" +description = "Pipeline action/provider traits with detection and redaction actions for Nvisy" +keywords = ["nvisy", "pipeline", "detection", "redaction"] +categories = ["text-processing"] + +version = { workspace = true } +rust-version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } + +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[features] +default = ["image-redaction", "audio-redaction"] +# Image blur/block redaction via image + imageproc; enables nvisy-ingest/png for PngHandler +image-redaction = ["dep:image", "dep:imageproc", "nvisy-ingest/png"] +# Audio redaction pass-through; enables nvisy-ingest/wav for WavHandler +audio-redaction = ["nvisy-ingest/wav"] + +[dependencies] +# Internal crates +nvisy-core = { workspace = true, features = [] } +nvisy-ontology = { workspace = true, features = [] } +nvisy-ingest = { workspace = true, features = [] } +nvisy-pattern = { workspace = true, features = [] } + +# (De)serialization +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = [] } + +# Async runtime +tokio = { workspace = true, features = ["sync"] } +async-trait = { workspace = true, features = [] } + +# Primitive datatypes +uuid = { workspace = true, features = ["v4"] } +bytes = { workspace = true, features = [] } + +# Time +jiff = { workspace = true, features = [] } + +# Text processing +regex = { workspace = true, features = [] } +aho-corasick = { workspace = true, features = [] } + +# Observability +tracing = { workspace = true, features = [] } + +# Image processing (feature-gated) +image = { workspace = true, optional = true, features = [] } +imageproc = { workspace = true, optional = true, features = [] } diff --git a/crates/nvisy-pipeline/src/action.rs b/crates/nvisy-pipeline/src/action.rs new file mode 100644 index 0000000..584c5d9 --- /dev/null +++ b/crates/nvisy-pipeline/src/action.rs @@ -0,0 +1,35 @@ +//! The `Action` trait -- the fundamental processing unit in a pipeline. + +use serde::de::DeserializeOwned; + +use nvisy_core::error::Error; + +/// A processing step with typed input and output. +/// +/// Actions are the primary unit of work in a pipeline. Each action is +/// constructed via [`connect`](Action::connect), which validates and +/// stores parameters, then executed via [`execute`](Action::execute). +/// +/// Actions that need a provider client should hold it as a struct field +/// rather than receiving it as a parameter. +#[async_trait::async_trait] +pub trait Action: Sized + Send + Sync + 'static { + /// Strongly-typed parameters for this action. + type Params: DeserializeOwned + Send; + /// Typed input for this action. + type Input: Send; + /// Typed output for this action. + type Output: Send; + + /// Unique identifier for this action (e.g. "detect-regex"). + fn id(&self) -> &str; + + /// Validate parameters and construct a configured action instance. + /// + /// This is where parameter validation, regex compilation, automata + /// building, and other setup work happens. + async fn connect(params: Self::Params) -> Result; + + /// Execute the action with typed input, returning typed output. + async fn execute(&self, input: Self::Input) -> Result; +} diff --git a/crates/nvisy-pipeline/src/detection/checksum.rs b/crates/nvisy-pipeline/src/detection/checksum.rs new file mode 100644 index 0000000..f301e00 --- /dev/null +++ b/crates/nvisy-pipeline/src/detection/checksum.rs @@ -0,0 +1,107 @@ +//! Checksum-based entity validation action. + +use serde::Deserialize; + +use nvisy_ontology::entity::{DetectionMethod, Entity}; +use nvisy_core::error::Error; +use nvisy_pattern::patterns::validators::luhn_check; + +use crate::action::Action; + +/// Typed parameters for [`DetectChecksumAction`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetectChecksumParams { + /// Whether to discard entities that fail validation. + #[serde(default = "default_true")] + pub drop_invalid: bool, + /// Amount added to confidence on successful validation. + #[serde(default = "default_boost")] + pub confidence_boost: f64, +} + +fn default_true() -> bool { + true +} +fn default_boost() -> f64 { + 0.05 +} + +/// Validates previously detected entities using checksum algorithms. +/// +/// Entities whose type has a registered validator (e.g. Luhn for credit cards) +/// are verified. Valid matches receive a confidence boost and are re-emitted +/// with [`DetectionMethod::Checksum`]. Invalid matches can optionally be +/// dropped from the pipeline. +pub struct DetectChecksumAction { + params: DetectChecksumParams, +} + +#[async_trait::async_trait] +impl Action for DetectChecksumAction { + type Params = DetectChecksumParams; + type Input = Vec; + type Output = Vec; + + fn id(&self) -> &str { + "detect-checksum" + } + + async fn connect(params: Self::Params) -> Result { + Ok(Self { params }) + } + + async fn execute( + &self, + entities: Self::Input, + ) -> Result, Error> { + let drop_invalid = self.params.drop_invalid; + let confidence_boost = self.params.confidence_boost; + + let mut result = Vec::new(); + + for entity in entities { + let validator = get_validator(&entity.entity_type); + + if let Some(validate) = validator { + let is_valid = validate(&entity.value); + + if !is_valid && drop_invalid { + continue; + } + + if is_valid { + let mut boosted = Entity::new( + entity.category.clone(), + &entity.entity_type, + &entity.value, + DetectionMethod::Checksum, + (entity.confidence + confidence_boost).min(1.0), + ); + boosted.text_location = entity.text_location.clone(); + boosted.image_location = entity.image_location.clone(); + boosted.tabular_location = entity.tabular_location.clone(); + boosted.audio_location = entity.audio_location.clone(); + boosted.video_location = entity.video_location.clone(); + boosted.source.set_parent_id(entity.source.parent_id()); + + result.push(boosted); + continue; + } + } + + // No validator or not valid but not dropping -- pass through + result.push(entity); + } + + Ok(result) + } +} + +/// Returns the checksum validator function for a given entity type, if one exists. +fn get_validator(entity_type: &str) -> Option bool> { + match entity_type { + "credit_card" => Some(luhn_check), + _ => None, + } +} diff --git a/crates/nvisy-pipeline/src/detection/classify.rs b/crates/nvisy-pipeline/src/detection/classify.rs new file mode 100644 index 0000000..0e4b0d4 --- /dev/null +++ b/crates/nvisy-pipeline/src/detection/classify.rs @@ -0,0 +1,75 @@ +//! Sensitivity classification action. + +pub use nvisy_ontology::detection::ClassificationResult; +use nvisy_ontology::detection::{Sensitivity, SensitivityLevel}; +use nvisy_ontology::entity::Entity; +use nvisy_core::error::Error; + +use crate::action::Action; + +/// Assigns a sensitivity level based on detected entities. +/// +/// The action inspects the entities, computes a [`Sensitivity`] assessment, +/// and returns a [`ClassificationResult`]. +pub struct ClassifyAction; + +#[async_trait::async_trait] +impl Action for ClassifyAction { + type Params = (); + type Input = Vec; + type Output = ClassificationResult; + + fn id(&self) -> &str { + "classify" + } + + async fn connect(_params: Self::Params) -> Result { + Ok(Self) + } + + async fn execute( + &self, + entities: Self::Input, + ) -> Result { + let total_entities = entities.len(); + let level = compute_sensitivity_level(&entities); + + Ok(ClassificationResult { + sensitivity: Sensitivity { + level, + risk_score: None, + }, + total_entities, + }) + } +} + +/// Computes a sensitivity level from a set of detected entities. +/// +/// The heuristic is: +/// - [`Public`](SensitivityLevel::Public) — no entities. +/// - [`Restricted`](SensitivityLevel::Restricted) — at least one high-confidence (>= 0.9) credential, SSN, or credit card. +/// - [`Confidential`](SensitivityLevel::Confidential) — any critical type present, or more than 10 entities total. +/// - [`Internal`](SensitivityLevel::Internal) — 1–10 non-critical entities. +fn compute_sensitivity_level(entities: &[Entity]) -> SensitivityLevel { + if entities.is_empty() { + return SensitivityLevel::Public; + } + + let has_high_confidence = entities.iter().any(|e| e.confidence >= 0.9); + let has_critical_types = entities.iter().any(|e| { + matches!( + e.category, + nvisy_ontology::entity::EntityCategory::Credentials + ) || e.entity_type == "ssn" + || e.entity_type == "credit_card" + }); + + if has_critical_types && has_high_confidence { + return SensitivityLevel::Restricted; + } + if has_critical_types || entities.len() > 10 { + return SensitivityLevel::Confidential; + } + SensitivityLevel::Internal +} diff --git a/crates/nvisy-pipeline/src/detection/dictionary.rs b/crates/nvisy-pipeline/src/detection/dictionary.rs new file mode 100644 index 0000000..e653724 --- /dev/null +++ b/crates/nvisy-pipeline/src/detection/dictionary.rs @@ -0,0 +1,186 @@ +//! Aho-Corasick dictionary-based entity detection action. + +use aho_corasick::AhoCorasick; +use serde::Deserialize; + +use nvisy_ingest::handler::{TxtHandler, CsvHandler}; +use nvisy_ingest::document::Document; +use nvisy_ontology::entity::{ + DetectionMethod, Entity, EntityCategory, TabularLocation, TextLocation, +}; +use nvisy_core::error::{Error, ErrorKind}; +use nvisy_pattern::dictionaries; + +use crate::action::Action; + +/// Definition of a single dictionary for matching. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DictionaryDef { + /// Dictionary name — `"builtin:first_names"` for built-in, or a custom name. + pub name: String, + /// Entity category for matches from this dictionary. + pub category: EntityCategory, + /// Entity type label for matches (e.g. `"first_name"`, `"medical_term"`). + pub entity_type: String, + /// Custom values — empty when using a builtin dictionary. + #[serde(default)] + pub values: Vec, + /// Whether matching should be case-sensitive. + #[serde(default)] + pub case_sensitive: bool, +} + +/// Typed parameters for [`DetectDictionaryAction`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetectDictionaryParams { + /// One or more dictionaries to match against. + pub dictionaries: Vec, + /// Confidence score assigned to dictionary matches. + #[serde(default = "default_confidence")] + pub confidence: f64, +} + +fn default_confidence() -> f64 { + 0.85 +} + +/// Scans document text and tabular cells against Aho-Corasick automata +/// built from user-provided word lists and/or built-in gazetteers. +pub struct DetectDictionaryAction { + params: DetectDictionaryParams, + automata: Vec<(DictionaryDef, AhoCorasick, Vec)>, +} + +#[async_trait::async_trait] +impl Action for DetectDictionaryAction { + type Params = DetectDictionaryParams; + type Input = (Vec>, Vec>); + type Output = Vec; + + fn id(&self) -> &str { + "detect-dictionary" + } + + async fn connect(params: Self::Params) -> Result { + if params.dictionaries.is_empty() { + return Err(Error::new( + ErrorKind::Validation, + "at least one dictionary definition is required", + )); + } + let automata = build_automata(¶ms.dictionaries)?; + Ok(Self { params, automata }) + } + + async fn execute( + &self, + input: Self::Input, + ) -> Result, Error> { + let (text_docs, tabular_docs) = input; + let confidence = self.params.confidence; + let mut entities = Vec::new(); + + // Text content matching + for doc in &text_docs { + let lines = doc.handler().lines(); + let mut content = lines.join("\n"); + if doc.handler().trailing_newline() { + content.push('\n'); + } + + for (def, ac, values) in &self.automata { + for mat in ac.find_iter(&content) { + let value = &values[mat.pattern().as_usize()]; + let entity = Entity::new( + def.category.clone(), + &def.entity_type, + value.as_str(), + DetectionMethod::Dictionary, + confidence, + ) + .with_text_location(TextLocation { + start_offset: mat.start(), + end_offset: mat.end(), + context_start_offset: None, + context_end_offset: None, + element_id: None, + page_number: None, + }) + .with_parent(&doc.source); + entities.push(entity); + } + } + } + + // Tabular content matching + for doc in &tabular_docs { + for (row_idx, row) in doc.handler().rows().iter().enumerate() { + for (col_idx, cell) in row.iter().enumerate() { + if cell.is_empty() { + continue; + } + for (def, ac, values) in &self.automata { + for mat in ac.find_iter(cell) { + let value = &values[mat.pattern().as_usize()]; + let entity = Entity::new( + def.category.clone(), + &def.entity_type, + value.as_str(), + DetectionMethod::Dictionary, + confidence, + ) + .with_tabular_location(TabularLocation { + row_index: row_idx, + column_index: col_idx, + start_offset: Some(mat.start()), + end_offset: Some(mat.end()), + }) + .with_parent(&doc.source); + entities.push(entity); + } + } + } + } + } + + Ok(entities) + } +} + +/// Resolve dictionary values (builtin or custom) and build Aho-Corasick automata. +fn build_automata( + defs: &[DictionaryDef], +) -> Result)>, Error> { + let mut result = Vec::with_capacity(defs.len()); + + for def in defs { + let values: Vec = if def.name.starts_with("builtin:") { + let builtin = dictionaries::get_builtin(&def.name).ok_or_else(|| { + Error::new( + ErrorKind::Validation, + format!("unknown builtin dictionary: {}", def.name), + ) + })?; + builtin.to_vec() + } else { + def.values.clone() + }; + + if values.is_empty() { + continue; + } + + let ac = aho_corasick::AhoCorasickBuilder::new() + .ascii_case_insensitive(!def.case_sensitive) + .build(&values) + .map_err(|e| { + Error::new(ErrorKind::Runtime, format!("failed to build automaton: {e}")) + })?; + + result.push((def.clone(), ac, values)); + } + + Ok(result) +} diff --git a/crates/nvisy-pipeline/src/detection/manual.rs b/crates/nvisy-pipeline/src/detection/manual.rs new file mode 100644 index 0000000..80f89d2 --- /dev/null +++ b/crates/nvisy-pipeline/src/detection/manual.rs @@ -0,0 +1,74 @@ +//! Manual annotation detection action. +//! +//! Converts user-provided inclusion [`Annotation`]s into full [`Entity`] objects. + +use serde::Deserialize; + +use nvisy_ontology::entity::{DetectionMethod, Entity}; +use nvisy_ontology::detection::{Annotation, AnnotationKind}; +use nvisy_core::error::Error; + +use crate::action::Action; + +/// Typed parameters for [`DetectManualAction`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetectManualParams {} + +/// Converts each inclusion [`Annotation`] into a full [`Entity`] with +/// `DetectionMethod::Manual` and confidence 1.0. +pub struct DetectManualAction; + +#[async_trait::async_trait] +impl Action for DetectManualAction { + type Params = DetectManualParams; + type Input = Vec; + type Output = Vec; + + fn id(&self) -> &str { + "detect-manual" + } + + async fn connect(_params: Self::Params) -> Result { + Ok(Self) + } + + async fn execute( + &self, + annotations: Self::Input, + ) -> Result, Error> { + let mut entities = Vec::new(); + + for ann in &annotations { + if ann.kind != AnnotationKind::Inclusion { + continue; + } + let category = match &ann.category { + Some(c) => c.clone(), + None => continue, + }; + let entity_type = match &ann.entity_type { + Some(t) => t.clone(), + None => continue, + }; + let value = ann.value.clone().unwrap_or_default(); + + let mut entity = Entity::new( + category, + entity_type, + value, + DetectionMethod::Manual, + 1.0, + ); + entity.text_location = ann.text_location.clone(); + entity.image_location = ann.image_location.clone(); + entity.tabular_location = ann.tabular_location.clone(); + entity.audio_location = ann.audio_location.clone(); + entity.video_location = ann.video_location.clone(); + + entities.push(entity); + } + + Ok(entities) + } +} diff --git a/crates/nvisy-pipeline/src/detection/mod.rs b/crates/nvisy-pipeline/src/detection/mod.rs new file mode 100644 index 0000000..88c6648 --- /dev/null +++ b/crates/nvisy-pipeline/src/detection/mod.rs @@ -0,0 +1,20 @@ +//! Entity detection actions. +//! +//! Each sub-module exposes a single [`Action`](crate::action::Action) +//! that produces [`Entity`](nvisy_ontology::entity::Entity) values from +//! document content. + +/// Validates detected entities using checksum algorithms (e.g. Luhn). +pub mod checksum; +/// Computes a sensitivity classification for each blob based on detected entities. +pub mod classify; +/// Aho-Corasick dictionary-based entity detection. +pub mod dictionary; +/// Converts user-provided manual annotations into entities. +pub mod manual; +/// AI-powered named-entity recognition (text + image). +pub mod ner; +/// Scans document text with compiled regex patterns to detect PII/PHI entities. +pub mod regex; +/// Column-based rule matching for tabular data. +pub mod tabular; diff --git a/crates/nvisy-pipeline/src/detection/ner.rs b/crates/nvisy-pipeline/src/detection/ner.rs new file mode 100644 index 0000000..305e7c8 --- /dev/null +++ b/crates/nvisy-pipeline/src/detection/ner.rs @@ -0,0 +1,64 @@ +//! AI-powered named-entity recognition (NER) detection action. + +use serde::Deserialize; + +use nvisy_ingest::document::Document; +use nvisy_ingest::handler::TxtHandler; +use nvisy_ontology::entity::Entity; +use nvisy_core::error::Error; + +#[cfg(feature = "image-redaction")] +use nvisy_ingest::handler::PngHandler; + +use crate::action::Action; + +fn default_confidence() -> f64 { + 0.5 +} + +/// Typed parameters for [`DetectNerAction`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetectNerParams { + /// Entity types to detect (empty = all). + #[serde(default)] + pub entity_types: Vec, + /// Minimum confidence score for returned entities. + #[serde(default = "default_confidence")] + pub confidence_threshold: f64, +} + +/// Typed input for [`DetectNerAction`]. +pub struct DetectNerInput { + /// Text documents to scan for named entities. + pub text_docs: Vec>, + /// Image documents to scan for named entities (feature-gated). + #[cfg(feature = "image-redaction")] + pub image_docs: Vec>, +} + +/// AI NER detection stub — delegates to an NER model provider at runtime. +pub struct DetectNerAction; + +#[async_trait::async_trait] +impl Action for DetectNerAction { + type Params = DetectNerParams; + type Input = DetectNerInput; + type Output = Vec; + + fn id(&self) -> &str { + "detect-ner" + } + + async fn connect(_params: Self::Params) -> Result { + Ok(Self) + } + + async fn execute( + &self, + _input: Self::Input, + ) -> Result, Error> { + // Stub: real implementation will call an NER model provider. + Ok(Vec::new()) + } +} diff --git a/crates/nvisy-pipeline/src/detection/regex.rs b/crates/nvisy-pipeline/src/detection/regex.rs new file mode 100644 index 0000000..79326c9 --- /dev/null +++ b/crates/nvisy-pipeline/src/detection/regex.rs @@ -0,0 +1,112 @@ +//! Regex-based PII/PHI entity detection action. + +use regex::Regex; +use serde::Deserialize; + +use nvisy_ingest::handler::TxtHandler; +use nvisy_ingest::document::Document; +use nvisy_ontology::entity::{DetectionMethod, Entity, TextLocation}; +use nvisy_core::error::Error; +use nvisy_pattern::patterns::{self, PatternDefinition}; + +use crate::action::Action; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetectRegexParams { + #[serde(default)] + pub confidence_threshold: f64, + #[serde(default)] + pub patterns: Option>, +} + +pub struct DetectRegexAction { + params: DetectRegexParams, +} + +#[async_trait::async_trait] +impl Action for DetectRegexAction { + type Params = DetectRegexParams; + type Input = Vec>; + type Output = Vec; + + fn id(&self) -> &str { + "detect-regex" + } + + async fn connect(params: Self::Params) -> Result { + Ok(Self { params }) + } + + async fn execute( + &self, + documents: Self::Input, + ) -> Result, Error> { + let confidence_threshold = self.params.confidence_threshold; + let requested_patterns = &self.params.patterns; + + let active_patterns = resolve_patterns(requested_patterns); + + let compiled: Vec<(&PatternDefinition, Regex)> = active_patterns + .iter() + .filter_map(|p| Regex::new(&p.pattern_str).ok().map(|r| (*p, r))) + .collect(); + + let mut entities = Vec::new(); + + for doc in &documents { + let lines = doc.handler().lines(); + let mut content = lines.join("\n"); + if doc.handler().trailing_newline() { + content.push('\n'); + } + + for (pattern, regex) in &compiled { + for mat in regex.find_iter(&content) { + let value = mat.as_str(); + + if let Some(validate) = pattern.validate { + if !validate(value) { + continue; + } + } + + if pattern.confidence < confidence_threshold { + continue; + } + + let entity = Entity::new( + pattern.category.clone(), + &pattern.entity_type, + value, + DetectionMethod::Regex, + pattern.confidence, + ) + .with_text_location(TextLocation { + start_offset: mat.start(), + end_offset: mat.end(), + context_start_offset: None, + context_end_offset: None, + element_id: None, + page_number: None, + }) + .with_parent(&doc.source); + + entities.push(entity); + } + } + } + + Ok(entities) + } +} + +fn resolve_patterns(requested: &Option>) -> Vec<&'static PatternDefinition> { + match requested { + Some(names) if !names.is_empty() => names + .iter() + .filter_map(|n| patterns::get_pattern(n)) + .collect(), + _ => patterns::get_all_patterns(), + } +} diff --git a/crates/nvisy-pipeline/src/detection/tabular.rs b/crates/nvisy-pipeline/src/detection/tabular.rs new file mode 100644 index 0000000..92bcfcf --- /dev/null +++ b/crates/nvisy-pipeline/src/detection/tabular.rs @@ -0,0 +1,122 @@ +//! Column-based rule matching for tabular data. + +use regex::Regex; +use serde::Deserialize; + +use nvisy_ingest::handler::CsvHandler; +use nvisy_ingest::document::Document; +use nvisy_ontology::entity::{ + DetectionMethod, Entity, EntityCategory, TabularLocation, +}; +use nvisy_core::error::{Error, ErrorKind}; + +use crate::action::Action; + +/// A rule that matches column headers to classify entire columns. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ColumnRule { + /// Regex pattern to match against column names. + pub column_name_pattern: String, + /// Entity category for matches in the column. + pub category: EntityCategory, + /// Entity type label for matches. + pub entity_type: String, +} + +/// Typed parameters for [`DetectTabularAction`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetectTabularParams { + /// Column-matching rules. + pub column_rules: Vec, +} + +/// Matches column headers against rules and marks every non-empty cell +/// in matched columns as an entity. +pub struct DetectTabularAction { + compiled_rules: Vec<(Regex, ColumnRule)>, +} + +#[async_trait::async_trait] +impl Action for DetectTabularAction { + type Params = DetectTabularParams; + type Input = Vec>; + type Output = Vec; + + fn id(&self) -> &str { + "detect-tabular" + } + + async fn connect(params: Self::Params) -> Result { + let compiled_rules = params + .column_rules + .iter() + .map(|r| { + let re = Regex::new(&r.column_name_pattern).map_err(|e| { + Error::new( + ErrorKind::Validation, + format!( + "invalid column_name_pattern '{}': {e}", + r.column_name_pattern + ), + ) + })?; + Ok((re, r.clone())) + }) + .collect::, Error>>()?; + Ok(Self { compiled_rules }) + } + + async fn execute( + &self, + documents: Self::Input, + ) -> Result, Error> { + let mut entities = Vec::new(); + + for doc in &documents { + let headers = match doc.handler().headers() { + Some(h) => h, + None => continue, + }; + + for (col_idx, col_name) in headers.iter().enumerate() { + for (regex, rule) in &self.compiled_rules { + if !regex.is_match(col_name) { + continue; + } + + for (row_idx, row) in doc.handler().rows().iter().enumerate() { + if let Some(cell) = row.get(col_idx) { + if cell.is_empty() { + continue; + } + + let entity = Entity::new( + rule.category.clone(), + &rule.entity_type, + cell.as_str(), + DetectionMethod::Composite, + 0.9, + ) + .with_tabular_location(TabularLocation { + row_index: row_idx, + column_index: col_idx, + start_offset: Some(0), + end_offset: Some(cell.len()), + }) + .with_parent(&doc.source); + + entities.push(entity); + } + } + + // Only apply first matching rule per column + break; + } + } + } + + Ok(entities) + } +} diff --git a/crates/nvisy-pipeline/src/generation/mod.rs b/crates/nvisy-pipeline/src/generation/mod.rs new file mode 100644 index 0000000..be6c759 --- /dev/null +++ b/crates/nvisy-pipeline/src/generation/mod.rs @@ -0,0 +1,14 @@ +//! Content generation actions. +//! +//! Each sub-module exposes a single [`Action`](crate::action::Action) +//! that generates derived content (text, entities, or replacement values) +//! from documents. + +/// OCR text extraction from image documents. +#[cfg(feature = "image-redaction")] +pub mod ocr; +/// Synthetic replacement value generation for Synthesize redactions. +pub mod synthetic; +/// Speech-to-text transcription from audio documents. +#[cfg(feature = "audio-redaction")] +pub mod transcribe; diff --git a/crates/nvisy-pipeline/src/generation/ocr.rs b/crates/nvisy-pipeline/src/generation/ocr.rs new file mode 100644 index 0000000..9fbc022 --- /dev/null +++ b/crates/nvisy-pipeline/src/generation/ocr.rs @@ -0,0 +1,81 @@ +//! OCR text extraction action — generates text entities with bounding boxes +//! from image documents. + +use serde::Deserialize; + +use nvisy_ingest::document::Document; +use nvisy_ingest::handler::{PngHandler, TxtHandler}; +use nvisy_ontology::entity::Entity; +use nvisy_core::error::Error; + +use crate::action::Action; + +fn default_language() -> String { + "eng".into() +} + +fn default_engine() -> String { + "tesseract".into() +} + +fn default_confidence() -> f64 { + 0.5 +} + +/// Typed parameters for [`GenerateOcrAction`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GenerateOcrParams { + /// OCR language code (ISO 639-3). + #[serde(default = "default_language")] + pub language: String, + /// OCR engine identifier. + #[serde(default = "default_engine")] + pub engine: String, + /// Minimum confidence score for returned entities. + #[serde(default = "default_confidence")] + pub confidence_threshold: f64, +} + +/// Typed input for [`GenerateOcrAction`]. +pub struct GenerateOcrInput { + /// Image documents to extract text from. + pub image_docs: Vec>, +} + +/// Typed output for [`GenerateOcrAction`]. +pub struct GenerateOcrOutput { + /// Detected text entities with bounding-box locations. + pub entities: Vec, + /// Extracted text as new text documents. + pub text_docs: Vec>, +} + +/// OCR generation stub — delegates to an OCR engine provider at runtime. +pub struct GenerateOcrAction; + +#[async_trait::async_trait] +impl Action for GenerateOcrAction { + type Params = GenerateOcrParams; + type Input = GenerateOcrInput; + type Output = GenerateOcrOutput; + + fn id(&self) -> &str { + "generate-ocr" + } + + async fn connect(_params: Self::Params) -> Result { + Ok(Self) + } + + async fn execute( + &self, + _input: Self::Input, + ) -> Result { + // Stub: real implementation will call an OCR engine provider. + Ok(GenerateOcrOutput { + entities: Vec::new(), + text_docs: Vec::new(), + }) + } +} diff --git a/crates/nvisy-pipeline/src/generation/synthetic.rs b/crates/nvisy-pipeline/src/generation/synthetic.rs new file mode 100644 index 0000000..d6200b4 --- /dev/null +++ b/crates/nvisy-pipeline/src/generation/synthetic.rs @@ -0,0 +1,59 @@ +//! Synthetic data generation action — fills in realistic replacement values +//! for redactions marked with `Synthesize`. + +use serde::Deserialize; + +use nvisy_ontology::entity::Entity; +use nvisy_ontology::redaction::Redaction; +use nvisy_core::error::Error; + +use crate::action::Action; + +fn default_locale() -> String { + "en-US".into() +} + +/// Typed parameters for [`GenerateSyntheticAction`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GenerateSyntheticParams { + /// BCP-47 locale for synthetic value generation. + #[serde(default = "default_locale")] + pub locale: String, +} + +/// Typed input for [`GenerateSyntheticAction`]. +pub struct GenerateSyntheticInput { + /// The entities whose redactions need synthetic values. + pub entities: Vec, + /// The redaction instructions (some may have `Synthesize` outputs). + pub redactions: Vec, +} + +/// Synthetic data generation stub — fills `Synthesize` redaction outputs +/// with realistic replacement values at runtime. +pub struct GenerateSyntheticAction; + +#[async_trait::async_trait] +impl Action for GenerateSyntheticAction { + type Params = GenerateSyntheticParams; + type Input = GenerateSyntheticInput; + type Output = Vec; + + fn id(&self) -> &str { + "generate-synthetic" + } + + async fn connect(_params: Self::Params) -> Result { + Ok(Self) + } + + async fn execute( + &self, + input: Self::Input, + ) -> Result, Error> { + // Stub: returns redactions unchanged. Real implementation will fill + // Synthesize variants with generated replacement values. + Ok(input.redactions) + } +} diff --git a/crates/nvisy-pipeline/src/generation/transcribe.rs b/crates/nvisy-pipeline/src/generation/transcribe.rs new file mode 100644 index 0000000..3620e1b --- /dev/null +++ b/crates/nvisy-pipeline/src/generation/transcribe.rs @@ -0,0 +1,77 @@ +//! Speech-to-text transcription action — generates text entities with audio +//! locations and transcript documents from audio input. + +use serde::Deserialize; + +use nvisy_ingest::document::Document; +use nvisy_ingest::handler::{WavHandler, TxtHandler}; +use nvisy_ontology::entity::Entity; +use nvisy_core::error::Error; + +use crate::action::Action; + +fn default_language() -> String { + "en".into() +} + +fn default_confidence() -> f64 { + 0.5 +} + +/// Typed parameters for [`GenerateTranscribeAction`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GenerateTranscribeParams { + /// BCP-47 language tag for transcription. + #[serde(default = "default_language")] + pub language: String, + /// Whether to perform speaker diarization. + #[serde(default)] + pub enable_speaker_diarization: bool, + /// Minimum confidence score for returned entities. + #[serde(default = "default_confidence")] + pub confidence_threshold: f64, +} + +/// Typed input for [`GenerateTranscribeAction`]. +pub struct GenerateTranscribeInput { + /// Audio documents to transcribe. + pub audio_docs: Vec>, +} + +/// Typed output for [`GenerateTranscribeAction`]. +pub struct GenerateTranscribeOutput { + /// Detected entities with [`AudioLocation`](nvisy_ontology::entity::AudioLocation). + pub entities: Vec, + /// Transcripts as new text documents. + pub text_docs: Vec>, +} + +/// Speech-to-text stub — delegates to a transcription provider at runtime. +pub struct GenerateTranscribeAction; + +#[async_trait::async_trait] +impl Action for GenerateTranscribeAction { + type Params = GenerateTranscribeParams; + type Input = GenerateTranscribeInput; + type Output = GenerateTranscribeOutput; + + fn id(&self) -> &str { + "generate-transcribe" + } + + async fn connect(_params: Self::Params) -> Result { + Ok(Self) + } + + async fn execute( + &self, + _input: Self::Input, + ) -> Result { + // Stub: real implementation will call a speech-to-text provider. + Ok(GenerateTranscribeOutput { + entities: Vec::new(), + text_docs: Vec::new(), + }) + } +} diff --git a/crates/nvisy-pipeline/src/lib.rs b/crates/nvisy-pipeline/src/lib.rs new file mode 100644 index 0000000..b5618c3 --- /dev/null +++ b/crates/nvisy-pipeline/src/lib.rs @@ -0,0 +1,23 @@ +//! Pipeline action/provider traits with detection, redaction, and generation actions. +//! +//! This crate consolidates the processing pipeline: the [`Action`] and +//! [`Provider`] traits, entity detection (regex, dictionary, checksum, +//! tabular, manual, NER), policy evaluation, content redaction +//! (text/image/tabular/audio), content generation (OCR, transcription, +//! synthetic data), and audit-trail emission. + +#![forbid(unsafe_code)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +/// The `Action` trait — the fundamental processing unit in a pipeline. +pub mod action; +/// The `Provider` trait — factory for authenticated client connections. +pub mod provider; +/// Entity detection actions. +pub mod detection; +/// Redaction actions (policy evaluation, apply, audit). +pub mod redaction; +/// Content generation actions (OCR, transcription, synthetic data). +pub mod generation; +#[doc(hidden)] +pub mod prelude; diff --git a/crates/nvisy-pipeline/src/prelude.rs b/crates/nvisy-pipeline/src/prelude.rs new file mode 100644 index 0000000..4387f8c --- /dev/null +++ b/crates/nvisy-pipeline/src/prelude.rs @@ -0,0 +1,31 @@ +//! Convenience re-exports for common nvisy-pipeline types. + +pub use crate::action::Action; +pub use crate::provider::{ConnectedInstance, Provider}; + +pub use crate::detection::regex::{DetectRegexAction, DetectRegexParams}; +pub use crate::detection::dictionary::{DetectDictionaryAction, DetectDictionaryParams, DictionaryDef}; +pub use crate::detection::tabular::{DetectTabularAction, DetectTabularParams, ColumnRule}; +pub use crate::detection::manual::{DetectManualAction, DetectManualParams}; +pub use crate::detection::checksum::{DetectChecksumAction, DetectChecksumParams}; +pub use crate::detection::ner::{DetectNerAction, DetectNerParams, DetectNerInput}; +pub use crate::detection::classify::{ClassifyAction, ClassificationResult}; +pub use crate::redaction::evaluate_policy::{EvaluatePolicyAction, EvaluatePolicyParams}; +pub use crate::redaction::apply::{ + ApplyRedactionAction, ApplyRedactionParams, ApplyRedactionInput, ApplyRedactionOutput, +}; +pub use crate::redaction::emit_audit::{EmitAuditAction, EmitAuditParams}; +pub use crate::generation::synthetic::{ + GenerateSyntheticAction, GenerateSyntheticParams, GenerateSyntheticInput, +}; + +#[cfg(feature = "image-redaction")] +pub use crate::generation::ocr::{ + GenerateOcrAction, GenerateOcrParams, GenerateOcrInput, GenerateOcrOutput, +}; + +#[cfg(feature = "audio-redaction")] +pub use crate::generation::transcribe::{ + GenerateTranscribeAction, GenerateTranscribeParams, GenerateTranscribeInput, + GenerateTranscribeOutput, +}; diff --git a/crates/nvisy-pipeline/src/provider.rs b/crates/nvisy-pipeline/src/provider.rs new file mode 100644 index 0000000..b14ee26 --- /dev/null +++ b/crates/nvisy-pipeline/src/provider.rs @@ -0,0 +1,44 @@ +//! Provider trait for creating authenticated client connections. + +use std::future::Future; +use std::pin::Pin; + +use serde::de::DeserializeOwned; + +use nvisy_core::error::Error; + +/// A connected provider instance holding a typed client and an +/// optional async disconnect callback. +pub struct ConnectedInstance { + /// Typed client handle. + pub client: C, + /// Optional cleanup function called when the connection is no longer needed. + pub disconnect: Option Pin + Send>> + Send>>, +} + +/// Factory for creating authenticated connections to an external service. +/// +/// Implementations handle credential validation, connectivity verification, +/// and client construction for a specific provider (e.g. S3, OpenAI). +#[async_trait::async_trait] +pub trait Provider: Send + Sync + 'static { + /// Strongly-typed credentials for this provider. + type Credentials: DeserializeOwned + Send; + /// The client type produced by [`connect`](Self::connect). + type Client: Send + 'static; + + /// Unique identifier (e.g. "s3", "openai"). + fn id(&self) -> &str; + + /// Validate credentials shape without connecting. + fn validate_credentials(&self, creds: &Self::Credentials) -> Result<(), Error>; + + /// Verify credentials by attempting a lightweight connection. + async fn verify(&self, creds: &Self::Credentials) -> Result<(), Error>; + + /// Create a connected instance. + async fn connect( + &self, + creds: &Self::Credentials, + ) -> Result, Error>; +} diff --git a/crates/nvisy-pipeline/src/redaction/apply.rs b/crates/nvisy-pipeline/src/redaction/apply.rs new file mode 100644 index 0000000..7927c9a --- /dev/null +++ b/crates/nvisy-pipeline/src/redaction/apply.rs @@ -0,0 +1,427 @@ +//! Unified redaction action -- applies text, image, tabular, and audio redactions. + +use std::collections::HashMap; +use uuid::Uuid; +use serde::Deserialize; + +use nvisy_ingest::handler::{TxtHandler, TxtData, CsvHandler}; +use nvisy_ingest::document::Document; +use nvisy_ontology::entity::Entity; +use nvisy_ontology::redaction::{Redaction, RedactionOutput, TextRedactionOutput}; +use nvisy_core::error::Error; + +#[cfg(feature = "image-redaction")] +use bytes::Bytes; +#[cfg(feature = "image-redaction")] +use nvisy_ingest::handler::PngHandler; +#[cfg(feature = "image-redaction")] +use nvisy_ontology::entity::BoundingBox; +#[cfg(feature = "image-redaction")] +use nvisy_ontology::redaction::ImageRedactionOutput; +#[cfg(feature = "image-redaction")] +use nvisy_core::error::ErrorKind; + +#[cfg(feature = "audio-redaction")] +use nvisy_ingest::handler::WavHandler; + +use crate::action::Action; + +/// Typed parameters for [`ApplyRedactionAction`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ApplyRedactionParams { + /// Default mask character for text [`Mask`](nvisy_ontology::redaction::TextRedactionOutput::Mask) redactions. + #[serde(default = "default_mask_char")] + pub mask_char: char, + /// Sigma value for gaussian blur (image redaction). + #[cfg(feature = "image-redaction")] + #[serde(default = "default_sigma")] + pub blur_sigma: f32, + /// RGBA color for block overlays (image redaction). + #[cfg(feature = "image-redaction")] + #[serde(default = "default_block_color")] + pub block_color: [u8; 4], + /// Pixel block size for pixelation/mosaic (image redaction). + #[cfg(feature = "image-redaction")] + #[serde(default = "default_pixelate_block_size")] + pub pixelate_block_size: u32, + /// Duration in seconds to crossfade at silence boundaries (audio redaction). + #[cfg(feature = "audio-redaction")] + #[serde(default = "default_crossfade_secs")] + pub crossfade_secs: f64, +} + +fn default_mask_char() -> char { + '*' +} +#[cfg(feature = "image-redaction")] +fn default_sigma() -> f32 { + 15.0 +} +#[cfg(feature = "image-redaction")] +fn default_block_color() -> [u8; 4] { + [0, 0, 0, 255] +} +#[cfg(feature = "image-redaction")] +fn default_pixelate_block_size() -> u32 { + 10 +} +#[cfg(feature = "audio-redaction")] +fn default_crossfade_secs() -> f64 { + 0.05 +} + +/// Typed input for [`ApplyRedactionAction`]. +pub struct ApplyRedactionInput { + /// Text documents to redact. + pub text_docs: Vec>, + /// Image documents to redact (feature-gated). + #[cfg(feature = "image-redaction")] + pub image_docs: Vec>, + /// Audio documents to redact (feature-gated). + #[cfg(feature = "audio-redaction")] + pub audio_docs: Vec>, + /// Tabular documents to redact. + pub tabular_docs: Vec>, + /// Detected entities referenced by redaction instructions. + pub entities: Vec, + /// Redaction instructions to apply. + pub redactions: Vec, +} + +/// Typed output for [`ApplyRedactionAction`]. +pub struct ApplyRedactionOutput { + /// Redacted text documents. + pub text_docs: Vec>, + /// Redacted image documents (feature-gated). + #[cfg(feature = "image-redaction")] + pub image_docs: Vec>, + /// Redacted audio documents (feature-gated). + #[cfg(feature = "audio-redaction")] + pub audio_docs: Vec>, + /// Redacted tabular documents. + pub tabular_docs: Vec>, +} + +/// Applies pending [`Redaction`] instructions to document content. +/// +/// Dispatches per-document based on content type: +/// - **Text documents**: byte-offset replacement +/// - **Image documents**: blur/block overlay (feature-gated) +/// - **Audio documents**: stub pass-through (feature-gated) +/// - **Tabular documents**: cell-level redaction +pub struct ApplyRedactionAction { + params: ApplyRedactionParams, +} + +/// A single text replacement that has been resolved but not yet applied. +struct PendingRedaction { + /// Byte offset where the redaction starts in the original text. + start_offset: usize, + /// Byte offset where the redaction ends (exclusive) in the original text. + end_offset: usize, + /// The string that will replace the original span. + replacement_value: String, +} + +#[async_trait::async_trait] +impl Action for ApplyRedactionAction { + type Params = ApplyRedactionParams; + type Input = ApplyRedactionInput; + type Output = ApplyRedactionOutput; + + fn id(&self) -> &str { + "apply-redaction" + } + + async fn connect(params: Self::Params) -> Result { + Ok(Self { params }) + } + + async fn execute( + &self, + input: Self::Input, + ) -> Result { + let entity_map: HashMap = + input.entities.iter().map(|e| (e.source.as_uuid(), e)).collect(); + let redaction_map: HashMap = input.redactions + .iter() + .filter(|r| !r.applied) + .map(|r| (r.entity_id, r)) + .collect(); + + // Text documents + let mut result_text = Vec::new(); + for doc in &input.text_docs { + let redacted = apply_text_doc(doc, &entity_map, &redaction_map, &self.params); + result_text.push(redacted); + } + + // Image documents + #[cfg(feature = "image-redaction")] + let mut result_image = Vec::new(); + #[cfg(feature = "image-redaction")] + for doc in &input.image_docs { + let redacted = apply_image_doc( + doc, + &input.entities, + &redaction_map, + self.params.blur_sigma, + self.params.block_color, + )?; + result_image.push(redacted); + } + + // Audio documents + #[cfg(feature = "audio-redaction")] + let mut result_audio = Vec::new(); + #[cfg(feature = "audio-redaction")] + for doc in &input.audio_docs { + let redacted = apply_audio_doc(doc); + result_audio.push(redacted); + } + + // Tabular documents + let mut result_tabular = Vec::new(); + for doc in &input.tabular_docs { + let redacted = apply_tabular_doc(doc, &input.entities, &redaction_map, &self.params); + result_tabular.push(redacted); + } + + Ok(ApplyRedactionOutput { + text_docs: result_text, + #[cfg(feature = "image-redaction")] + image_docs: result_image, + #[cfg(feature = "audio-redaction")] + audio_docs: result_audio, + tabular_docs: result_tabular, + }) + } +} + +// --------------------------------------------------------------------------- +// Text redaction +// --------------------------------------------------------------------------- + +fn apply_text_doc( + doc: &Document, + entity_map: &HashMap, + redaction_map: &HashMap, + params: &ApplyRedactionParams, +) -> Document { + let lines = doc.handler().lines(); + let mut content = lines.join("\n"); + if doc.handler().trailing_newline() { + content.push('\n'); + } + + let mut pending: Vec = Vec::new(); + + for (entity_id, redaction) in redaction_map { + let entity = match entity_map.get(entity_id) { + Some(e) => e, + None => continue, + }; + + // Check entity belongs to this document + let belongs = entity.source.parent_id() == Some(doc.source.as_uuid()); + if !belongs { + continue; + } + + let (start_offset, end_offset) = match &entity.text_location { + Some(loc) => (loc.start_offset, loc.end_offset), + None => continue, + }; + + let replacement_value = match redaction.output.replacement_value() { + Some(v) => v.to_string(), + None => { + let span_len = end_offset.saturating_sub(start_offset); + params.mask_char.to_string().repeat(span_len) + } + }; + + pending.push(PendingRedaction { + start_offset, + end_offset, + replacement_value, + }); + } + + if pending.is_empty() { + return doc.clone(); + } + + let redacted_content = apply_text_redactions(&content, &mut pending); + + let trailing_newline = redacted_content.ends_with('\n'); + let new_lines: Vec = redacted_content.lines().map(String::from).collect(); + let handler = TxtHandler::new(TxtData { + lines: new_lines, + trailing_newline, + }); + let mut result = Document::new(handler); + result.source.set_parent_id(Some(doc.source.as_uuid())); + result +} + +/// Applies a set of pending redactions to `text`, returning the redacted result. +/// +/// Replacements are applied right-to-left (descending start offset) so that +/// earlier byte offsets remain valid after each substitution. +fn apply_text_redactions(text: &str, pending: &mut [PendingRedaction]) -> String { + // Sort by start offset descending (right-to-left) to preserve positions + pending.sort_by(|a, b| b.start_offset.cmp(&a.start_offset)); + + let mut result = text.to_string(); + for redaction in pending.iter() { + let start = redaction.start_offset.min(result.len()); + let end = redaction.end_offset.min(result.len()); + if start >= end { + continue; + } + + result = format!( + "{}{}{}", + &result[..start], + redaction.replacement_value, + &result[end..] + ); + } + result +} + +// --------------------------------------------------------------------------- +// Image redaction (feature-gated) +// --------------------------------------------------------------------------- + +#[cfg(feature = "image-redaction")] +fn apply_image_doc( + doc: &Document, + entities: &[Entity], + redaction_map: &HashMap, + blur_sigma: f32, + block_color: [u8; 4], +) -> Result, Error> { + use crate::redaction::render::{blur, block}; + + let image_bytes = doc.handler().bytes(); + + let mut blur_regions: Vec = Vec::new(); + let mut block_regions: Vec = Vec::new(); + + for entity in entities { + if let Some(ref img_loc) = entity.image_location { + let bbox = &img_loc.bounding_box; + if let Some(redaction) = redaction_map.get(&entity.source.as_uuid()) { + match &redaction.output { + RedactionOutput::Image(ImageRedactionOutput::Blur { .. }) => { + blur_regions.push(bbox.clone()) + } + RedactionOutput::Image(ImageRedactionOutput::Block { .. }) => { + block_regions.push(bbox.clone()) + } + _ => block_regions.push(bbox.clone()), + } + } + } + } + + if blur_regions.is_empty() && block_regions.is_empty() { + return Ok(doc.clone()); + } + + let dyn_img = image::load_from_memory(image_bytes).map_err(|e| { + Error::new(ErrorKind::Runtime, format!("image decode failed: {e}")) + })?; + + let mut result = dyn_img; + if !blur_regions.is_empty() { + result = blur::apply_gaussian_blur(&result, &blur_regions, blur_sigma); + } + if !block_regions.is_empty() { + let color = image::Rgba(block_color); + result = block::apply_block_overlay(&result, &block_regions, color); + } + + // Encode back to PNG + let mut buf = std::io::Cursor::new(Vec::new()); + result + .write_to(&mut buf, image::ImageFormat::Png) + .map_err(|e| { + Error::new(ErrorKind::Runtime, format!("image encode failed: {e}")) + })?; + + let new_doc = Document::new(PngHandler::new(Bytes::from(buf.into_inner()))); + Ok(new_doc) +} + +// --------------------------------------------------------------------------- +// Audio redaction (feature-gated) +// --------------------------------------------------------------------------- + +#[cfg(feature = "audio-redaction")] +fn apply_audio_doc(doc: &Document) -> Document { + tracing::warn!("audio redaction not yet implemented"); + doc.clone() +} + +// --------------------------------------------------------------------------- +// Tabular redaction +// --------------------------------------------------------------------------- + +fn apply_tabular_doc( + doc: &Document, + entities: &[Entity], + redaction_map: &HashMap, + params: &ApplyRedactionParams, +) -> Document { + let mut result = doc.clone(); + + for entity in entities { + if let Some(ref tab_loc) = entity.tabular_location { + let (row_idx, col_idx) = (tab_loc.row_index, tab_loc.column_index); + if let Some(redaction) = redaction_map.get(&entity.source.as_uuid()) { + if let Some(row) = result.handler_mut().rows_mut().get_mut(row_idx) { + if let Some(cell) = row.get_mut(col_idx) { + *cell = apply_cell_redaction(cell, &redaction.output, params.mask_char); + } + } + } + } + } + + result +} + +fn apply_cell_redaction(cell: &str, output: &RedactionOutput, default_mask: char) -> String { + match output { + RedactionOutput::Text(TextRedactionOutput::Mask { mask_char, .. }) => { + if cell.len() > 4 { + format!( + "{}{}", + mask_char.to_string().repeat(cell.len() - 4), + &cell[cell.len() - 4..] + ) + } else { + mask_char.to_string().repeat(cell.len()) + } + } + RedactionOutput::Text(TextRedactionOutput::Remove) => String::new(), + RedactionOutput::Text(TextRedactionOutput::Hash { .. }) => { + format!("[HASH:{:x}]", hash_string(cell)) + } + _ => output + .replacement_value() + .map(|v| v.to_string()) + .unwrap_or_else(|| default_mask.to_string().repeat(cell.len())), + } +} + +fn hash_string(s: &str) -> u64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + s.hash(&mut hasher); + hasher.finish() +} diff --git a/crates/nvisy-pipeline/src/redaction/emit_audit.rs b/crates/nvisy-pipeline/src/redaction/emit_audit.rs new file mode 100644 index 0000000..0c9a7f1 --- /dev/null +++ b/crates/nvisy-pipeline/src/redaction/emit_audit.rs @@ -0,0 +1,76 @@ +//! Audit trail emission action. + +use jiff::Timestamp; +use serde::Deserialize; +use uuid::Uuid; + +use nvisy_core::error::Error; +use nvisy_core::path::ContentSource; +use nvisy_ontology::audit::{Audit, AuditAction}; +use nvisy_ontology::redaction::Redaction; + +use crate::action::Action; + +/// Typed parameters for [`EmitAuditAction`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EmitAuditParams { + /// Pipeline run identifier to attach. + #[serde(default)] + pub run_id: Option, + /// Human or service identity to record. + #[serde(default)] + pub actor: Option, +} + +/// Emits an [`Audit`] record for every [`Redaction`] provided. +/// +/// Each audit entry captures the redaction method, replacement value, and +/// (when available) the originating policy rule ID. +pub struct EmitAuditAction { + params: EmitAuditParams, +} + +#[async_trait::async_trait] +impl Action for EmitAuditAction { + type Params = EmitAuditParams; + type Input = Vec; + type Output = Vec; + + fn id(&self) -> &str { + "emit-audit" + } + + async fn connect(params: Self::Params) -> Result { + Ok(Self { params }) + } + + async fn execute( + &self, + redactions: Self::Input, + ) -> Result, Error> { + let mut audits = Vec::new(); + + for redaction in &redactions { + let mut source = ContentSource::new(); + source.set_parent_id(Some(redaction.source.as_uuid())); + + let audit = Audit { + source, + action: AuditAction::Redaction, + timestamp: Timestamp::now(), + entity_id: Some(redaction.entity_id), + redaction_id: Some(redaction.source.as_uuid()), + policy_id: None, + source_id: None, + run_id: self.params.run_id, + actor: self.params.actor.clone(), + explanation: None, + }; + + audits.push(audit); + } + + Ok(audits) + } +} diff --git a/crates/nvisy-pipeline/src/redaction/evaluate_policy.rs b/crates/nvisy-pipeline/src/redaction/evaluate_policy.rs new file mode 100644 index 0000000..92e61d0 --- /dev/null +++ b/crates/nvisy-pipeline/src/redaction/evaluate_policy.rs @@ -0,0 +1,226 @@ +//! Policy evaluation action that maps detected entities to redaction instructions. + +use serde::Deserialize; + +use nvisy_ontology::entity::Entity; +use nvisy_ontology::policy::PolicyRule; +use nvisy_ontology::redaction::{ + AudioRedactionOutput, AudioRedactionSpec, ImageRedactionOutput, ImageRedactionSpec, Redaction, + RedactionOutput, RedactionSpec, TextRedactionOutput, TextRedactionSpec, +}; +use nvisy_core::error::Error; + +use crate::action::Action; + +/// Typed parameters for [`EvaluatePolicyAction`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EvaluatePolicyParams { + /// Ordered policy rules to evaluate. + #[serde(default)] + pub rules: Vec, + /// Fallback redaction specification when no rule matches. + #[serde(default = "default_spec")] + pub default_spec: RedactionSpec, + /// Fallback confidence threshold. + #[serde(default = "default_threshold")] + pub default_confidence_threshold: f64, +} + +fn default_spec() -> RedactionSpec { + RedactionSpec::Text(TextRedactionSpec::Mask { mask_char: '*' }) +} +fn default_threshold() -> f64 { + 0.5 +} + +/// Evaluates policy rules against detected entities and produces [`Redaction`] instructions. +/// +/// For each entity the action finds the first matching rule (sorted by priority), +/// applies its redaction spec and replacement template, and creates a +/// [`Redaction`]. Entities that fall below the confidence threshold are skipped. +pub struct EvaluatePolicyAction { + params: EvaluatePolicyParams, +} + +#[async_trait::async_trait] +impl Action for EvaluatePolicyAction { + type Params = EvaluatePolicyParams; + type Input = Vec; + type Output = Vec; + + fn id(&self) -> &str { + "evaluate-policy" + } + + async fn connect(params: Self::Params) -> Result { + Ok(Self { params }) + } + + async fn execute( + &self, + entities: Self::Input, + ) -> Result, Error> { + let default_spec = &self.params.default_spec; + let default_threshold = self.params.default_confidence_threshold; + + let mut sorted_rules = self.params.rules.clone(); + sorted_rules.sort_by_key(|r| r.priority); + + let mut redactions = Vec::new(); + + for entity in &entities { + let rule = find_matching_rule(entity, &sorted_rules); + let spec = rule.map(|r| &r.spec).unwrap_or(default_spec); + + if rule.is_none() && entity.confidence < default_threshold { + continue; + } + + let output = if let Some(r) = rule { + build_output_from_template(spec, &r.replacement_template, entity) + } else { + build_default_output(entity, spec) + }; + + let mut redaction = Redaction::new(entity.source.as_uuid(), output); + redaction = redaction.with_original_value(&entity.value); + if let Some(r) = rule { + redaction = redaction.with_policy_rule_id(r.id); + } + redaction.source.set_parent_id(Some(entity.source.as_uuid())); + + redactions.push(redaction); + } + + Ok(redactions) + } +} + +/// Returns the first enabled rule whose [`EntitySelector`] matches the given entity, +/// or `None` if no rule applies. +fn find_matching_rule<'a>(entity: &Entity, rules: &'a [PolicyRule]) -> Option<&'a PolicyRule> { + for rule in rules { + if rule.selector.matches(&entity.category, &entity.entity_type, entity.confidence) { + return Some(rule); + } + } + None +} + +/// Expands a replacement template using entity metadata. +/// +/// Supported placeholders: `{entityType}`, `{category}`, `{value}`. +fn apply_template(template: &str, entity: &Entity) -> String { + template + .replace("{entityType}", &entity.entity_type) + .replace( + "{category}", + &format!("{:?}", entity.category).to_lowercase(), + ) + .replace("{value}", &entity.value) +} + +/// Builds a [`RedactionOutput`] from a spec and a policy rule's replacement template. +fn build_output_from_template( + spec: &RedactionSpec, + template: &str, + entity: &Entity, +) -> RedactionOutput { + let replacement = apply_template(template, entity); + build_output_with_replacement(spec, replacement) +} + +/// Generates a [`RedactionOutput`] for an entity using the given default redaction spec. +fn build_default_output(entity: &Entity, spec: &RedactionSpec) -> RedactionOutput { + match spec { + RedactionSpec::Text(text) => { + let replacement = match text { + TextRedactionSpec::Mask { mask_char } => { + mask_char.to_string().repeat(entity.value.len()) + } + TextRedactionSpec::Replace { placeholder } => { + if placeholder.is_empty() { + format!("[{}]", entity.entity_type.to_uppercase()) + } else { + apply_template(placeholder, entity) + } + } + TextRedactionSpec::Remove => String::new(), + TextRedactionSpec::Hash => format!("[HASH:{}]", entity.entity_type), + TextRedactionSpec::Encrypt { .. } => format!("[ENC:{}]", entity.entity_type), + TextRedactionSpec::Synthesize => format!("[SYNTH:{}]", entity.entity_type), + TextRedactionSpec::Pseudonymize => format!("[PSEUDO:{}]", entity.entity_type), + TextRedactionSpec::Tokenize { .. } => format!("[TOKEN:{}]", entity.entity_type), + TextRedactionSpec::Aggregate => format!("[AGG:{}]", entity.entity_type), + TextRedactionSpec::Generalize { .. } => format!("[GEN:{}]", entity.entity_type), + TextRedactionSpec::DateShift { .. } => format!("[SHIFTED:{}]", entity.entity_type), + }; + build_output_with_replacement(spec, replacement) + } + RedactionSpec::Image(img) => RedactionOutput::Image(match img { + ImageRedactionSpec::Blur { sigma } => ImageRedactionOutput::Blur { sigma: *sigma }, + ImageRedactionSpec::Block { color } => ImageRedactionOutput::Block { color: *color }, + ImageRedactionSpec::Pixelate { block_size } => { + ImageRedactionOutput::Pixelate { block_size: *block_size } + } + ImageRedactionSpec::Synthesize => ImageRedactionOutput::Synthesize, + }), + RedactionSpec::Audio(audio) => RedactionOutput::Audio(match audio { + AudioRedactionSpec::Silence => AudioRedactionOutput::Silence, + AudioRedactionSpec::Remove => AudioRedactionOutput::Remove, + AudioRedactionSpec::Synthesize => AudioRedactionOutput::Synthesize, + }), + } +} + +/// Builds a [`RedactionOutput`] from a spec and a replacement string. +fn build_output_with_replacement(spec: &RedactionSpec, replacement: String) -> RedactionOutput { + match spec { + RedactionSpec::Text(text) => RedactionOutput::Text(match text { + TextRedactionSpec::Mask { mask_char } => TextRedactionOutput::Mask { + replacement, + mask_char: *mask_char, + }, + TextRedactionSpec::Replace { .. } => TextRedactionOutput::Replace { replacement }, + TextRedactionSpec::Hash => TextRedactionOutput::Hash { + hash_value: replacement, + }, + TextRedactionSpec::Encrypt { key_id } => TextRedactionOutput::Encrypt { + ciphertext: replacement, + key_id: key_id.clone(), + }, + TextRedactionSpec::Remove => TextRedactionOutput::Remove, + TextRedactionSpec::Synthesize => TextRedactionOutput::Synthesize { replacement }, + TextRedactionSpec::Pseudonymize => TextRedactionOutput::Pseudonymize { + pseudonym: replacement, + }, + TextRedactionSpec::Tokenize { vault_id } => TextRedactionOutput::Tokenize { + token: replacement, + vault_id: vault_id.clone(), + }, + TextRedactionSpec::Aggregate => TextRedactionOutput::Aggregate { replacement }, + TextRedactionSpec::Generalize { level } => TextRedactionOutput::Generalize { + replacement, + level: *level, + }, + TextRedactionSpec::DateShift { offset_days } => TextRedactionOutput::DateShift { + replacement, + offset_days: *offset_days, + }, + }), + RedactionSpec::Image(img) => RedactionOutput::Image(match img { + ImageRedactionSpec::Blur { sigma } => ImageRedactionOutput::Blur { sigma: *sigma }, + ImageRedactionSpec::Block { color } => ImageRedactionOutput::Block { color: *color }, + ImageRedactionSpec::Pixelate { block_size } => { + ImageRedactionOutput::Pixelate { block_size: *block_size } + } + ImageRedactionSpec::Synthesize => ImageRedactionOutput::Synthesize, + }), + RedactionSpec::Audio(audio) => RedactionOutput::Audio(match audio { + AudioRedactionSpec::Silence => AudioRedactionOutput::Silence, + AudioRedactionSpec::Remove => AudioRedactionOutput::Remove, + AudioRedactionSpec::Synthesize => AudioRedactionOutput::Synthesize, + }), + } +} diff --git a/crates/nvisy-pipeline/src/redaction/mod.rs b/crates/nvisy-pipeline/src/redaction/mod.rs new file mode 100644 index 0000000..66ccd46 --- /dev/null +++ b/crates/nvisy-pipeline/src/redaction/mod.rs @@ -0,0 +1,14 @@ +//! Redaction actions. +//! +//! Each sub-module exposes a single [`Action`](crate::action::Action) +//! that evaluates, applies, or records redaction decisions. + +/// Applies pending redactions to document content (text, image, tabular, audio). +pub mod apply; +/// Image rendering primitives for redaction overlays. +#[cfg(feature = "image-redaction")] +pub mod render; +/// Emits audit trail records for every applied redaction. +pub mod emit_audit; +/// Evaluates policy rules against detected entities and produces redaction instructions. +pub mod evaluate_policy; diff --git a/crates/nvisy-pipeline/src/redaction/render/block.rs b/crates/nvisy-pipeline/src/redaction/render/block.rs new file mode 100644 index 0000000..05d69a3 --- /dev/null +++ b/crates/nvisy-pipeline/src/redaction/render/block.rs @@ -0,0 +1,36 @@ +//! Solid color block overlay for image regions. + +use image::{DynamicImage, Rgba, RgbaImage}; +use nvisy_ontology::entity::BoundingBox; + +/// Apply a solid color block overlay to the specified regions of an image. +/// +/// Each [`BoundingBox`] describes a rectangular region (in pixel coordinates) +/// that will be covered with an opaque rectangle of the given `color`. +pub fn apply_block_overlay( + image: &DynamicImage, + regions: &[BoundingBox], + color: Rgba, +) -> DynamicImage { + let mut result = image.to_rgba8(); + let img_w = result.width(); + let img_h = result.height(); + + for region in regions { + let x = region.x.round() as u32; + let y = region.y.round() as u32; + let w = region.width.round() as u32; + let h = region.height.round() as u32; + + if x >= img_w || y >= img_h { + continue; + } + let w = w.min(img_w - x); + let h = h.min(img_h - y); + + let block = RgbaImage::from_pixel(w, h, color); + image::imageops::overlay(&mut result, &block, x as i64, y as i64); + } + + DynamicImage::ImageRgba8(result) +} diff --git a/crates/nvisy-pipeline/src/redaction/render/blur.rs b/crates/nvisy-pipeline/src/redaction/render/blur.rs new file mode 100644 index 0000000..468c49e --- /dev/null +++ b/crates/nvisy-pipeline/src/redaction/render/blur.rs @@ -0,0 +1,43 @@ +//! Gaussian blur for image regions. + +use image::DynamicImage; +use imageproc::filter::gaussian_blur_f32; +use nvisy_ontology::entity::BoundingBox; + +/// Apply gaussian blur to the specified regions of an image. +/// +/// Each [`BoundingBox`] describes a rectangular region (in pixel coordinates) +/// that will be blurred with the given `sigma` value. +pub fn apply_gaussian_blur( + image: &DynamicImage, + regions: &[BoundingBox], + sigma: f32, +) -> DynamicImage { + let mut result = image.clone(); + + for region in regions { + let x = region.x.round() as u32; + let y = region.y.round() as u32; + let w = region.width.round() as u32; + let h = region.height.round() as u32; + + // Clamp to image bounds + let img_w = result.width(); + let img_h = result.height(); + if x >= img_w || y >= img_h { + continue; + } + let w = w.min(img_w - x); + let h = h.min(img_h - y); + if w == 0 || h == 0 { + continue; + } + + // Crop the region, blur it, paste it back + let sub = result.crop_imm(x, y, w, h); + let blurred = DynamicImage::ImageRgba8(gaussian_blur_f32(&sub.to_rgba8(), sigma)); + image::imageops::overlay(&mut result, &blurred, x as i64, y as i64); + } + + result +} diff --git a/crates/nvisy-pipeline/src/redaction/render/mod.rs b/crates/nvisy-pipeline/src/redaction/render/mod.rs new file mode 100644 index 0000000..1796d48 --- /dev/null +++ b/crates/nvisy-pipeline/src/redaction/render/mod.rs @@ -0,0 +1,6 @@ +//! Image rendering primitives for redaction overlays. + +/// Gaussian blur for image regions. +pub mod blur; +/// Solid color block overlay for image regions. +pub mod block; diff --git a/crates/nvisy-python/Cargo.toml b/crates/nvisy-python/Cargo.toml new file mode 100644 index 0000000..089355f --- /dev/null +++ b/crates/nvisy-python/Cargo.toml @@ -0,0 +1,49 @@ +# https://doc.rust-lang.org/cargo/reference/manifest.html + +[package] +name = "nvisy-python" +description = "PyO3 bridge for AI NER detection via embedded Python" +keywords = ["nvisy", "python", "pyo3", "ner"] +categories = ["api-bindings"] + +version = { workspace = true } +rust-version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } + +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +# Internal crates +nvisy-core = { workspace = true, features = [] } +nvisy-ontology = { workspace = true, features = [] } +nvisy-pipeline = { workspace = true, features = [] } +nvisy-ingest = { workspace = true, features = [] } + +# (De)serialization +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = [] } + +# Async runtime +tokio = { workspace = true, features = ["sync", "rt"] } +async-trait = { workspace = true, features = [] } + +# Primitive datatypes +uuid = { workspace = true, features = ["v4"] } + +# Python interop +pyo3 = { workspace = true, features = ["auto-initialize", "serde"] } + +# Error handling +thiserror = { workspace = true, features = [] } + +# Observability +tracing = { workspace = true, features = [] } diff --git a/crates/nvisy-python/README.md b/crates/nvisy-python/README.md new file mode 100644 index 0000000..fece235 --- /dev/null +++ b/crates/nvisy-python/README.md @@ -0,0 +1,3 @@ +# nvisy-python + +PyO3 bridge plugin for the Nvisy runtime. Embeds a Python interpreter to run AI-powered named entity recognition (NER) models for text and image detection, exposing them as native Nvisy actions. diff --git a/crates/nvisy-python/src/actions/mod.rs b/crates/nvisy-python/src/actions/mod.rs new file mode 100644 index 0000000..cd5e2b1 --- /dev/null +++ b/crates/nvisy-python/src/actions/mod.rs @@ -0,0 +1,196 @@ +//! Pipeline actions that perform AI-powered named-entity recognition and OCR. +//! +//! Three actions are provided: +//! - [`DetectNerAction`] -- runs NER over text documents. +//! - [`DetectNerImageAction`] -- runs NER over images (OCR + entity detection). +//! - [`OcrDetectAction`] -- performs OCR on images to extract text regions. + +/// OCR detection pipeline action. +pub mod ocr; + +use serde::Deserialize; + +use nvisy_ingest::handler::{FormatHandler, TxtHandler}; +use nvisy_ingest::document::Document; +use nvisy_ingest::document::data::*; +use nvisy_ontology::entity::Entity; +use nvisy_core::error::Error; +use nvisy_core::io::ContentData; +use nvisy_pipeline::action::Action; +use crate::bridge::PythonBridge; +use crate::ner::{self, NerConfig}; + +/// Typed parameters for NER actions. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetectNerParams { + /// Entity type labels to detect (e.g., `["PERSON", "SSN"]`). + #[serde(default)] + pub entity_types: Vec, + /// Minimum confidence score to include a detection (0.0 -- 1.0). + #[serde(default = "default_confidence_threshold")] + pub confidence_threshold: f64, + /// Sampling temperature forwarded to the AI model. + #[serde(default)] + pub temperature: f64, + /// API key for the AI provider. + #[serde(default)] + pub api_key: String, + /// Model identifier (e.g., `"gpt-4"`). + #[serde(default = "default_model")] + pub model: String, + /// AI provider name (e.g., `"openai"`). + #[serde(default = "default_provider")] + pub provider: String, +} + +fn default_confidence_threshold() -> f64 { 0.5 } +fn default_model() -> String { "gpt-4".to_string() } +fn default_provider() -> String { "openai".to_string() } + +/// Pipeline action that detects named entities in text documents. +/// +/// Each document's text is sent through the NER model. If no documents are +/// provided, the raw content is interpreted as UTF-8 text. Detected entities +/// are returned directly. +pub struct DetectNerAction { + /// Python bridge used to call the NER model. + pub bridge: PythonBridge, + params: DetectNerParams, +} + +impl DetectNerAction { + /// Replace the default bridge with a pre-configured one. + pub fn with_bridge(mut self, bridge: PythonBridge) -> Self { + self.bridge = bridge; + self + } +} + +#[async_trait::async_trait] +impl Action for DetectNerAction { + type Params = DetectNerParams; + type Input = (ContentData, Vec>); + type Output = Vec; + + fn id(&self) -> &str { "detect-ner" } + + async fn connect(params: Self::Params) -> Result { + Ok(Self { bridge: PythonBridge::default(), params }) + } + + async fn execute( + &self, + input: Self::Input, + ) -> Result { + let (content, documents) = input; + let config = ner_config_from_params(&self.params); + + let docs = if documents.is_empty() { + let text = content.as_str() + .map_err(|e| Error::runtime( + format!("Content is not valid UTF-8: {}", e), + "python/ner", + false, + ))?; + vec![Document::new( + FormatHandler::Txt(TxtHandler), + DocumentData::Text(TextData { text: text.to_string() }), + )] + } else { + documents + }; + + let mut all_entities = Vec::new(); + for doc in &docs { + if let Some(content) = doc.text() { + let entities = ner::detect_ner(&self.bridge, content, &config).await?; + all_entities.extend(entities); + } + } + + Ok(all_entities) + } +} + +/// Pipeline action that detects named entities in images. +/// +/// Each image is processed individually through NER. If no images are +/// provided, the raw content is treated as a single image whose MIME type +/// is inferred from the content metadata. Detected entities are returned +/// directly. +pub struct DetectNerImageAction { + /// Python bridge used to call the NER model. + pub bridge: PythonBridge, + params: DetectNerParams, +} + +impl DetectNerImageAction { + /// Replace the default bridge with a pre-configured one. + pub fn with_bridge(mut self, bridge: PythonBridge) -> Self { + self.bridge = bridge; + self + } +} + +#[async_trait::async_trait] +impl Action for DetectNerImageAction { + type Params = DetectNerParams; + type Input = (ContentData, Vec>); + type Output = Vec; + + fn id(&self) -> &str { "detect-ner-image" } + + async fn connect(params: Self::Params) -> Result { + Ok(Self { bridge: PythonBridge::default(), params }) + } + + async fn execute( + &self, + input: Self::Input, + ) -> Result { + let (content, images) = input; + let config = ner_config_from_params(&self.params); + + let mut all_entities = Vec::new(); + + if images.is_empty() { + let mime_type = content.content_type() + .unwrap_or("application/octet-stream") + .to_string(); + let entities = ner::detect_ner_image( + &self.bridge, + content.as_bytes(), + &mime_type, + &config, + ).await?; + all_entities.extend(entities); + } else { + for doc in &images { + if let Some(image) = doc.image() { + let entities = ner::detect_ner_image( + &self.bridge, + &image.bytes, + &image.mime_type, + &config, + ).await?; + all_entities.extend(entities); + } + } + } + + Ok(all_entities) + } +} + +/// Convert [`DetectNerParams`] into the internal [`NerConfig`]. +fn ner_config_from_params(params: &DetectNerParams) -> NerConfig { + NerConfig { + entity_types: params.entity_types.clone(), + confidence_threshold: params.confidence_threshold, + temperature: params.temperature, + api_key: params.api_key.clone(), + model: params.model.clone(), + provider: params.provider.clone(), + } +} diff --git a/crates/nvisy-python/src/actions/ocr.rs b/crates/nvisy-python/src/actions/ocr.rs new file mode 100644 index 0000000..9d54e17 --- /dev/null +++ b/crates/nvisy-python/src/actions/ocr.rs @@ -0,0 +1,123 @@ +//! OCR detection pipeline action. + +use serde::Deserialize; + +use nvisy_ingest::handler::{FormatHandler, TxtHandler}; +use nvisy_ingest::document::Document; +use nvisy_ingest::document::data::*; +use nvisy_ontology::entity::Entity; +use nvisy_core::error::Error; +use nvisy_core::io::ContentData; +use nvisy_pipeline::action::Action; +use crate::bridge::PythonBridge; +use crate::ocr::{self, OcrConfig}; + +/// Typed parameters for [`OcrDetectAction`]. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OcrDetectParams { + /// Language hint (default `"eng"`). + #[serde(default = "default_language")] + pub language: String, + /// OCR engine to use. + #[serde(default = "default_engine")] + pub engine: String, + /// Minimum confidence threshold. + #[serde(default = "default_confidence")] + pub confidence_threshold: f64, +} + +fn default_language() -> String { + "eng".to_string() +} +fn default_engine() -> String { + "tesseract".to_string() +} +fn default_confidence() -> f64 { + 0.5 +} + +/// Pipeline action that performs OCR on images and produces entities +/// with bounding boxes, plus `Document` artifacts from concatenated +/// OCR text so downstream regex/dictionary/NER can process it. +pub struct OcrDetectAction { + /// Python bridge used to call the OCR backend. + pub bridge: PythonBridge, + params: OcrDetectParams, +} + +impl OcrDetectAction { + /// Replace the default bridge with a pre-configured one. + pub fn with_bridge(mut self, bridge: PythonBridge) -> Self { + self.bridge = bridge; + self + } +} + +#[async_trait::async_trait] +impl Action for OcrDetectAction { + type Params = OcrDetectParams; + type Input = (ContentData, Vec>); + type Output = (Vec, Vec>); + + fn id(&self) -> &str { + "detect-ocr" + } + + async fn connect(params: Self::Params) -> Result { + Ok(Self { bridge: PythonBridge::default(), params }) + } + + async fn execute( + &self, + input: Self::Input, + ) -> Result { + let (content, images) = input; + let config = OcrConfig { + language: self.params.language.clone(), + engine: self.params.engine.clone(), + confidence_threshold: self.params.confidence_threshold, + }; + + let mut all_entities = Vec::new(); + let mut all_ocr_text = Vec::new(); + + if images.is_empty() { + // Treat content as a single image + let mime_type = content + .content_type() + .unwrap_or("application/octet-stream") + .to_string(); + let entities = + ocr::detect_ocr(&self.bridge, content.as_bytes(), &mime_type, &config).await?; + for entity in &entities { + all_ocr_text.push(entity.value.clone()); + } + all_entities.extend(entities); + } else { + for doc in &images { + if let Some(image) = doc.image() { + let entities = + ocr::detect_ocr(&self.bridge, &image.bytes, &image.mime_type, &config) + .await?; + for entity in &entities { + all_ocr_text.push(entity.value.clone()); + } + all_entities.extend(entities); + } + } + } + + // Create a Document from concatenated OCR text for downstream processing + let mut documents = Vec::new(); + if !all_ocr_text.is_empty() { + let ocr_doc = Document::new( + FormatHandler::Txt(TxtHandler), + DocumentData::Text(TextData { text: all_ocr_text.join("\n") }), + ); + documents.push(ocr_doc); + } + + Ok((all_entities, documents)) + } +} diff --git a/crates/nvisy-python/src/bridge/mod.rs b/crates/nvisy-python/src/bridge/mod.rs new file mode 100644 index 0000000..d785427 --- /dev/null +++ b/crates/nvisy-python/src/bridge/mod.rs @@ -0,0 +1,45 @@ +//! Lightweight handle to a Python module loaded via PyO3. + +use pyo3::prelude::*; +use nvisy_core::error::Error; +use crate::error::from_pyerr; + +/// Lightweight handle to a Python NER module. +/// +/// The bridge does **not** hold the GIL or any Python objects; it simply +/// remembers which module to `import` when a detection function is called. +/// The default module name is `"nvisy_ai"`. +#[derive(Clone)] +pub struct PythonBridge { + /// Dotted Python module name to import (e.g., `"nvisy_ai"`). + module_name: String, +} + +impl PythonBridge { + /// Create a new bridge that will load the given Python module. + pub fn new(module_name: impl Into) -> Self { + Self { + module_name: module_name.into(), + } + } + + /// Initialize Python and verify the module can be imported. + pub fn init(&self) -> Result<(), Error> { + Python::with_gil(|py| { + py.import(&self.module_name) + .map_err(from_pyerr)?; + Ok(()) + }) + } + + /// Get the module name. + pub fn module_name(&self) -> &str { + &self.module_name + } +} + +impl Default for PythonBridge { + fn default() -> Self { + Self::new("nvisy_ai") + } +} diff --git a/crates/nvisy-python/src/error/mod.rs b/crates/nvisy-python/src/error/mod.rs new file mode 100644 index 0000000..ed3176f --- /dev/null +++ b/crates/nvisy-python/src/error/mod.rs @@ -0,0 +1,19 @@ +//! Conversion utilities from Python errors to [`Error`]. + +use nvisy_core::error::Error; +use pyo3::PyErr; +use pyo3::types::PyTracebackMethods; + +/// Convert a [`PyErr`] into an [`Error`], preserving the Python traceback when available. +pub fn from_pyerr(err: PyErr) -> Error { + pyo3::Python::with_gil(|py| { + let traceback = err + .traceback(py) + .map(|tb| tb.format().unwrap_or_default()); + let msg = match traceback { + Some(tb) => format!("{}\n{}", err, tb), + None => err.to_string(), + }; + Error::python(msg) + }) +} diff --git a/crates/nvisy-python/src/lib.rs b/crates/nvisy-python/src/lib.rs new file mode 100644 index 0000000..00f0411 --- /dev/null +++ b/crates/nvisy-python/src/lib.rs @@ -0,0 +1,21 @@ +//! Python/PyO3 bridge for AI-powered NER detection. +//! +//! This crate embeds a CPython interpreter via PyO3 and delegates named-entity +//! recognition (NER) to a Python module (`nvisy_ai`). It exposes pipeline +//! [`Action`](nvisy_pipeline::action::Action) implementations as well as a +//! [`Provider`](nvisy_pipeline::provider::Provider) for the +//! `"ai"` provider. + +#![deny(unsafe_code)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![doc = include_str!("../README.md")] + +pub mod actions; +pub mod bridge; +pub mod error; +pub mod ner; +pub mod ocr; +pub mod provider; + +#[doc(hidden)] +pub mod prelude; diff --git a/crates/nvisy-python/src/ner/mod.rs b/crates/nvisy-python/src/ner/mod.rs new file mode 100644 index 0000000..e16d5fd --- /dev/null +++ b/crates/nvisy-python/src/ner/mod.rs @@ -0,0 +1,182 @@ +//! Named-entity recognition (NER) detection via a Python AI backend. +//! +//! Functions in this module acquire the GIL, call into the Python `nvisy_ai` +//! module, and convert the returned list of dicts into [`Entity`] values. + +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList}; + +use nvisy_ontology::entity::{DetectionMethod, Entity, EntityCategory, TextLocation}; +use nvisy_core::error::Error; +use crate::bridge::PythonBridge; +use crate::error::from_pyerr; + +/// Configuration for NER detection passed to the Python backend. +#[derive(Debug, Clone)] +pub struct NerConfig { + /// Entity type labels to detect (e.g., `["PERSON", "SSN"]`). + pub entity_types: Vec, + /// Minimum confidence score to include a detection (0.0 -- 1.0). + pub confidence_threshold: f64, + /// Sampling temperature forwarded to the AI model. + pub temperature: f64, + /// API key for the AI provider. + pub api_key: String, + /// Model identifier (e.g., `"gpt-4"`). + pub model: String, + /// AI provider name (e.g., `"openai"`). + pub provider: String, +} + +/// Call Python detect_ner function via GIL + spawn_blocking. +pub async fn detect_ner( + bridge: &PythonBridge, + text: &str, + config: &NerConfig, +) -> Result, Error> { + let module_name = bridge.module_name().to_string(); + let text = text.to_string(); + let config = config.clone(); + + tokio::task::spawn_blocking(move || { + Python::with_gil(|py| { + let module = py.import(&module_name).map_err(from_pyerr)?; + + let kwargs = PyDict::new(py); + kwargs.set_item("text", &text).map_err(from_pyerr)?; + kwargs.set_item("entity_types", &config.entity_types).map_err(from_pyerr)?; + kwargs.set_item("confidence_threshold", config.confidence_threshold).map_err(from_pyerr)?; + kwargs.set_item("temperature", config.temperature).map_err(from_pyerr)?; + kwargs.set_item("api_key", &config.api_key).map_err(from_pyerr)?; + kwargs.set_item("model", &config.model).map_err(from_pyerr)?; + kwargs.set_item("provider", &config.provider).map_err(from_pyerr)?; + + let result = module + .call_method("detect_ner", (), Some(&kwargs)) + .map_err(from_pyerr)?; + + parse_python_entities(py, result) + }) + }) + .await + .map_err(|e| Error::python(format!("Task join error: {}", e)))? +} + +/// Call Python detect_ner_image function via GIL + spawn_blocking. +pub async fn detect_ner_image( + bridge: &PythonBridge, + image_data: &[u8], + mime_type: &str, + config: &NerConfig, +) -> Result, Error> { + let module_name = bridge.module_name().to_string(); + let image_data = image_data.to_vec(); + let mime_type = mime_type.to_string(); + let config = config.clone(); + + tokio::task::spawn_blocking(move || { + Python::with_gil(|py| { + let module = py.import(&module_name).map_err(from_pyerr)?; + + let kwargs = PyDict::new(py); + kwargs.set_item("image_bytes", &image_data[..]).map_err(from_pyerr)?; + kwargs.set_item("mime_type", &mime_type).map_err(from_pyerr)?; + kwargs.set_item("entity_types", &config.entity_types).map_err(from_pyerr)?; + kwargs.set_item("confidence_threshold", config.confidence_threshold).map_err(from_pyerr)?; + kwargs.set_item("api_key", &config.api_key).map_err(from_pyerr)?; + kwargs.set_item("model", &config.model).map_err(from_pyerr)?; + kwargs.set_item("provider", &config.provider).map_err(from_pyerr)?; + + let result = module + .call_method("detect_ner_image", (), Some(&kwargs)) + .map_err(from_pyerr)?; + + parse_python_entities(py, result) + }) + }) + .await + .map_err(|e| Error::python(format!("Task join error: {}", e)))? +} + +/// Parse Python list[dict] response into Vec. +fn parse_python_entities(_py: Python<'_>, result: Bound<'_, PyAny>) -> Result, Error> { + let list: &Bound<'_, PyList> = result.downcast().map_err(|e| { + Error::python(format!("Expected list from Python: {}", e)) + })?; + + let mut entities = Vec::new(); + + for item in list.iter() { + let dict: &Bound<'_, PyDict> = item.downcast().map_err(|e| { + Error::python(format!("Expected dict in list: {}", e)) + })?; + + let category_str: String = dict + .get_item("category") + .map_err(from_pyerr)? + .ok_or_else(|| Error::python("Missing 'category'"))? + .extract() + .map_err(from_pyerr)?; + + let category = match category_str.as_str() { + "pii" => EntityCategory::Pii, + "phi" => EntityCategory::Phi, + "financial" => EntityCategory::Financial, + "credentials" => EntityCategory::Credentials, + other => EntityCategory::Custom(other.to_string()), + }; + + let entity_type: String = dict + .get_item("entity_type") + .map_err(from_pyerr)? + .ok_or_else(|| Error::python("Missing 'entity_type'"))? + .extract() + .map_err(from_pyerr)?; + + let value: String = dict + .get_item("value") + .map_err(from_pyerr)? + .ok_or_else(|| Error::python("Missing 'value'"))? + .extract() + .map_err(from_pyerr)?; + + let confidence: f64 = dict + .get_item("confidence") + .map_err(from_pyerr)? + .ok_or_else(|| Error::python("Missing 'confidence'"))? + .extract() + .map_err(from_pyerr)?; + + let start_offset: usize = dict + .get_item("start_offset") + .map_err(from_pyerr)? + .and_then(|v| v.extract().ok()) + .unwrap_or(0); + + let end_offset: usize = dict + .get_item("end_offset") + .map_err(from_pyerr)? + .and_then(|v| v.extract().ok()) + .unwrap_or(0); + + let entity = Entity::new( + category, + entity_type, + value, + DetectionMethod::Ner, + confidence, + ) + .with_text_location(TextLocation { + start_offset, + end_offset, + context_start_offset: None, + context_end_offset: None, + element_id: None, + page_number: None, + }); + + entities.push(entity); + } + + Ok(entities) +} diff --git a/crates/nvisy-python/src/ocr/mod.rs b/crates/nvisy-python/src/ocr/mod.rs new file mode 100644 index 0000000..e2fd1a8 --- /dev/null +++ b/crates/nvisy-python/src/ocr/mod.rs @@ -0,0 +1,147 @@ +//! OCR text extraction via the Python backend. +//! +//! Calls `nvisy_ai.detect_ocr()` through the Python bridge to perform +//! optical character recognition on images, returning text regions with +//! bounding boxes. + +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList}; + +use nvisy_ontology::entity::{ + BoundingBox, DetectionMethod, Entity, EntityCategory, ImageLocation, +}; +use nvisy_core::error::Error; +use crate::bridge::PythonBridge; +use crate::error::from_pyerr; + +/// Configuration for OCR detection. +#[derive(Debug, Clone)] +pub struct OcrConfig { + /// Language hint (e.g. `"eng"` for English). + pub language: String, + /// OCR engine to use (`"tesseract"`, `"google-vision"`, `"aws-textract"`). + pub engine: String, + /// Minimum confidence threshold for OCR results. + pub confidence_threshold: f64, +} + +/// Call Python `detect_ocr()` via GIL + `spawn_blocking`. +/// +/// Returns a list of entities with `DetectionMethod::Ocr`, each carrying +/// a bounding box indicating where the text was found in the image. +pub async fn detect_ocr( + bridge: &PythonBridge, + image_data: &[u8], + mime_type: &str, + config: &OcrConfig, +) -> Result, Error> { + let module_name = bridge.module_name().to_string(); + let image_data = image_data.to_vec(); + let mime_type = mime_type.to_string(); + let config = config.clone(); + + tokio::task::spawn_blocking(move || { + Python::with_gil(|py| { + let module = py.import(&module_name).map_err(from_pyerr)?; + + let kwargs = PyDict::new(py); + kwargs.set_item("image_bytes", &image_data[..]).map_err(from_pyerr)?; + kwargs.set_item("mime_type", &mime_type).map_err(from_pyerr)?; + kwargs.set_item("language", &config.language).map_err(from_pyerr)?; + kwargs.set_item("engine", &config.engine).map_err(from_pyerr)?; + kwargs.set_item("confidence_threshold", config.confidence_threshold).map_err(from_pyerr)?; + + let result = module + .call_method("detect_ocr", (), Some(&kwargs)) + .map_err(from_pyerr)?; + + parse_ocr_results(result) + }) + }) + .await + .map_err(|e| Error::python(format!("Task join error: {}", e)))? +} + +/// Parse Python list[dict] OCR response into Vec. +/// +/// Expected Python response format: +/// ```python +/// [ +/// { +/// "text": "John Doe", +/// "x": 100.0, +/// "y": 200.0, +/// "width": 150.0, +/// "height": 30.0, +/// "confidence": 0.95 +/// }, +/// ... +/// ] +/// ``` +fn parse_ocr_results(result: Bound<'_, PyAny>) -> Result, Error> { + let list: &Bound<'_, PyList> = result.downcast().map_err(|e| { + Error::python(format!("Expected list from Python OCR: {}", e)) + })?; + + let mut entities = Vec::new(); + + for item in list.iter() { + let dict: &Bound<'_, PyDict> = item.downcast().map_err(|e| { + Error::python(format!("Expected dict in OCR list: {}", e)) + })?; + + let text: String = dict + .get_item("text") + .map_err(from_pyerr)? + .ok_or_else(|| Error::python("Missing 'text' in OCR result"))? + .extract() + .map_err(from_pyerr)?; + + let x: f64 = dict + .get_item("x") + .map_err(from_pyerr)? + .and_then(|v| v.extract().ok()) + .unwrap_or(0.0); + + let y: f64 = dict + .get_item("y") + .map_err(from_pyerr)? + .and_then(|v| v.extract().ok()) + .unwrap_or(0.0); + + let width: f64 = dict + .get_item("width") + .map_err(from_pyerr)? + .and_then(|v| v.extract().ok()) + .unwrap_or(0.0); + + let height: f64 = dict + .get_item("height") + .map_err(from_pyerr)? + .and_then(|v| v.extract().ok()) + .unwrap_or(0.0); + + let confidence: f64 = dict + .get_item("confidence") + .map_err(from_pyerr)? + .and_then(|v| v.extract().ok()) + .unwrap_or(0.0); + + let entity = Entity::new( + EntityCategory::Pii, + "ocr_text", + &text, + DetectionMethod::Ocr, + confidence, + ) + .with_image_location(ImageLocation { + bounding_box: BoundingBox { x, y, width, height }, + image_id: None, + page_number: None, + }); + + entities.push(entity); + } + + Ok(entities) +} diff --git a/crates/nvisy-python/src/prelude.rs b/crates/nvisy-python/src/prelude.rs new file mode 100644 index 0000000..325beb1 --- /dev/null +++ b/crates/nvisy-python/src/prelude.rs @@ -0,0 +1,5 @@ +//! Convenience re-exports. +pub use crate::actions::{DetectNerAction, DetectNerImageAction}; +pub use crate::actions::ocr::OcrDetectAction; +pub use crate::bridge::PythonBridge; +pub use crate::provider::AiProvider; diff --git a/crates/nvisy-python/src/provider/mod.rs b/crates/nvisy-python/src/provider/mod.rs new file mode 100644 index 0000000..efae0f7 --- /dev/null +++ b/crates/nvisy-python/src/provider/mod.rs @@ -0,0 +1,52 @@ +//! AI provider factory for the Python NER bridge. +//! +//! Registers itself as the `"ai"` provider and yields a [`PythonBridge`] +//! instance upon connection. + +use serde::Deserialize; + +use nvisy_core::error::Error; +use nvisy_pipeline::provider::{ConnectedInstance, Provider}; +use crate::bridge::PythonBridge; + +/// Typed credentials for the AI provider. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AiCredentials { + /// API key forwarded to the AI model provider. + pub api_key: String, +} + +/// Factory that creates [`PythonBridge`] instances from typed credentials. +/// +/// The Python interpreter is **not** initialized at connection time; it is +/// lazily loaded on the first NER call. +pub struct AiProvider; + +#[async_trait::async_trait] +impl Provider for AiProvider { + type Credentials = AiCredentials; + type Client = PythonBridge; + + fn id(&self) -> &str { "ai" } + + fn validate_credentials(&self, _creds: &Self::Credentials) -> Result<(), Error> { + // api_key is required by the struct, so if we got here it's present. + Ok(()) + } + + async fn verify(&self, creds: &Self::Credentials) -> Result<(), Error> { + self.validate_credentials(creds) + } + + async fn connect(&self, _creds: &Self::Credentials) -> Result, Error> { + let bridge = PythonBridge::default(); + // Don't init here — Python might not be available at connect time + // Init happens lazily when detect_ner is called + + Ok(ConnectedInstance { + client: bridge, + disconnect: None, + }) + } +} diff --git a/docker/Dockerfile b/docker/Dockerfile index 96fcfbf..12e14c6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,34 +1,52 @@ -FROM node:22-alpine AS base +FROM rust:1.85-bookworm AS builder + +RUN apt-get update && apt-get install -y python3-dev python3-pip && rm -rf /var/lib/apt/lists/* + WORKDIR /app -FROM base AS deps -COPY package.json package-lock.json ./ -COPY packages/nvisy-core/package.json packages/nvisy-core/package.json -COPY packages/nvisy-runtime/package.json packages/nvisy-runtime/package.json -COPY packages/nvisy-server/package.json packages/nvisy-server/package.json -RUN npm ci - -FROM base AS build -COPY --from=deps /app/node_modules ./node_modules -COPY --from=deps /app/packages/nvisy-core/node_modules ./packages/nvisy-core/node_modules -COPY --from=deps /app/packages/nvisy-runtime/node_modules ./packages/nvisy-runtime/node_modules -COPY --from=deps /app/packages/nvisy-server/node_modules ./packages/nvisy-server/node_modules +# Copy manifests first to cache dependency builds +COPY Cargo.toml Cargo.lock ./ +COPY crates/nvisy-core/Cargo.toml crates/nvisy-core/Cargo.toml +COPY crates/nvisy-detect/Cargo.toml crates/nvisy-detect/Cargo.toml +COPY crates/nvisy-engine/Cargo.toml crates/nvisy-engine/Cargo.toml +COPY crates/nvisy-object/Cargo.toml crates/nvisy-object/Cargo.toml +COPY crates/nvisy-python/Cargo.toml crates/nvisy-python/Cargo.toml +COPY crates/nvisy-server/Cargo.toml crates/nvisy-server/Cargo.toml + +# Create empty src files to satisfy cargo's manifest checks +RUN for crate in nvisy-core nvisy-detect nvisy-engine nvisy-object nvisy-python; do \ + mkdir -p crates/$crate/src && echo "" > crates/$crate/src/lib.rs; \ + done && \ + mkdir -p crates/nvisy-server/src && echo "fn main() {}" > crates/nvisy-server/src/main.rs + +# Create stub READMEs for crates that use doc = include_str!("../README.md") +RUN for crate in nvisy-core nvisy-detect nvisy-engine nvisy-object nvisy-python nvisy-server; do \ + touch crates/$crate/README.md; \ + done + +ENV PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 + +# Cache dependency build +RUN cargo build --release 2>/dev/null || true + +# Copy full source and build COPY . . -RUN npm run build - -FROM base AS runtime -ENV NODE_ENV=production -COPY --from=deps /app/node_modules ./node_modules -COPY --from=deps /app/packages/nvisy-core/node_modules ./packages/nvisy-core/node_modules -COPY --from=deps /app/packages/nvisy-runtime/node_modules ./packages/nvisy-runtime/node_modules -COPY --from=deps /app/packages/nvisy-server/node_modules ./packages/nvisy-server/node_modules -COPY --from=build /app/packages/nvisy-core/dist ./packages/nvisy-core/dist -COPY --from=build /app/packages/nvisy-core/package.json ./packages/nvisy-core/package.json -COPY --from=build /app/packages/nvisy-runtime/dist ./packages/nvisy-runtime/dist -COPY --from=build /app/packages/nvisy-runtime/package.json ./packages/nvisy-runtime/package.json -COPY --from=build /app/packages/nvisy-server/dist ./packages/nvisy-server/dist -COPY --from=build /app/packages/nvisy-server/package.json ./packages/nvisy-server/package.json -COPY package.json package-lock.json ./ +RUN cargo build --release + +# Runtime stage +FROM debian:bookworm-slim AS runtime + +RUN apt-get update && apt-get install -y \ + python3 python3-pip python3-venv ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Install Python packages +COPY packages/ /opt/nvisy/packages/ +RUN python3 -m pip install --break-system-packages \ + /opt/nvisy/packages/nvisy-ai \ + /opt/nvisy/packages/nvisy-exif + +COPY --from=builder /app/target/release/nvisy-server /usr/local/bin/nvisy-server EXPOSE 8080 -CMD ["node", "packages/nvisy-server/dist/main.js"] +CMD ["nvisy-server"] diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md deleted file mode 100644 index e1da7d2..0000000 --- a/docs/ARCHITECTURE.md +++ /dev/null @@ -1,339 +0,0 @@ -# Nvisy Runtime — Architecture - -**Technical architecture specification for the Nvisy Runtime ETL platform.** - ---- - -## 1. Overview - -Nvisy Runtime is a TypeScript-native, DAG-based ETL platform for AI data workloads. It is structured as a set of composable packages that can be consumed as a library or deployed as a long-lived server. - -This document defines the system architecture: package boundaries, data flow, execution model, connector interface, graph compilation, scheduling, error handling, and observability. It is intended as the authoritative reference for implementation. - ---- - -## 2. Package Structure - -The system is organized as a monorepo of npm packages under the `@nvisy` scope. - -``` -packages/ - nvisy-core/ Primitives, type system, validation, errors, - base interfaces for sources, sinks, and actions, - and core observability (structured logging, metrics, tracing) - nvisy-plugin-sql/ SQL connectors (PostgreSQL, MySQL) - nvisy-plugin-object/ Object store and file format connectors (S3, GCS, Parquet, JSONL, CSV) - nvisy-plugin-vector/ Vector database connectors (Pinecone, Qdrant, Milvus, Weaviate, pgvector) - nvisy-runtime/ Graph definition, JSON parser, DAG compiler, execution engine, - task runner, retry logic, concurrency control, - runtime-level observability (run metrics, node tracing) - nvisy-server/ HTTP server (Hono), REST API, cron scheduler, dashboard backend, - server-level observability (request logging, health endpoints) -``` - -### Dependency graph - -``` - nvisy-server - / \ - ▼ ▼ - nvisy-runtime nvisy-plugin-{sql,ai,object,vector} - \ / - ▼ ▼ - nvisy-core -``` - -Every package depends on `nvisy-core`. The plugin packages (`nvisy-plugin-sql`, `nvisy-plugin-ai`, `nvisy-plugin-object`, `nvisy-plugin-vector`) are siblings — they depend on `nvisy-core` for the base source, sink, and action interfaces, but are independent of each other and independent of `nvisy-runtime`. The server (or any application) imports both the runtime and the desired plugins, then registers each plugin with the engine at startup. `nvisy-runtime` is the central package that owns graph definition, compilation, and execution — but has no compile-time dependency on any plugin. No circular dependencies are permitted. Packages communicate through typed interfaces, never through implementation details. - -### Observability distribution - -There is no dedicated observability package. Instead, observability is distributed across three layers: - -- **`nvisy-core`** defines the observability primitives: structured log format, metric types, trace span interface, and lineage record structure. It also exports the logging, metrics, and tracing utilities that all other packages use. -- **`nvisy-runtime`** emits runtime observability: graph run duration, node execution times, primitives processed/failed, connector call counts, rate limit wait times. Each graph run produces an OpenTelemetry-compatible trace with nodes as spans. -- **`nvisy-server`** emits server-level observability: HTTP request logging, health check endpoints, and metric export endpoints (Prometheus, OpenTelemetry). - ---- - -## 3. Idiomatic Modern JavaScript with Effection - -The platform is built on idiomatic modern JavaScript — `async`/`await`, `AsyncIterable`, native `Promise`, and generator functions — with **Effection** providing structured concurrency for the runtime's DAG executor. - -### 3.1 Design philosophy - -Traditional TypeScript ETL code suffers from scattered try/catch blocks, manual resource cleanup, ad-hoc retry logic, and opaque concurrency. Nvisy addresses these problems with standard language features and a minimal set of libraries: - -- **Typed errors.** A structured error hierarchy with machine-readable tags enables programmatic error handling. TypeScript discriminated unions and Zod schemas enforce correctness at both compile time and runtime. -- **Resource safety.** Connector lifecycle (connect, use, disconnect) is managed with explicit `try`/`finally` blocks in the node executor. Cleanup runs regardless of success, failure, or cancellation. -- **Structured concurrency.** The runtime's DAG executor uses Effection — a structured concurrency library built on generators. Spawned tasks are scoped to their parent, so halting a graph run automatically cancels all in-flight nodes without manual bookkeeping. -- **Streaming.** Data flows between nodes via the native `AsyncIterable` protocol. Plugin interfaces (sources, sinks, actions) accept and return `AsyncIterable` — no special streaming library required. -- **Validation.** Zod schemas provide runtime validation, TypeScript type derivation, and structured parse errors in a single definition. - -### 3.2 Effection in the runtime - -Effection is used exclusively in `nvisy-runtime` for DAG execution. It provides `spawn` (launch concurrent tasks), `race` (timeout handling), `sleep` (backoff delays), and `call` (bridge async functions into generator-based operations). The runtime wraps graph execution in an Effection task; each node runs as a spawned child operation. Plugin code never touches Effection — sources, sinks, and actions are plain `async` functions and `AsyncIterable` generators. - ---- - -## 4. Core (`nvisy-core`) - -The core package serves three purposes: it defines the **primitive type system** (the data model), it provides the **base interfaces** for building sources, sinks, and actions, and it houses the **observability primitives** used by all other packages. - -### 4.1 Primitive type hierarchy - -All data flowing through a graph is represented as a **Primitive**. Primitives are immutable, serializable, and carry both payload and metadata. - -The primitive type discriminant covers the full AI data surface: `embedding`, `completion`, `structured_output`, `tool_call_trace`, `image`, `audio`, `fine_tune_sample`, and `raw` (an escape hatch for untyped data). Each type maps to a specific payload shape — for example, an embedding payload contains the vector, its dimensionality, the producing model, source text, and a content hash. - -Primitives are validated at construction time. The factory enforces payload conformance and rejects malformed data with structured errors before it enters the graph. - -### 4.2 Metadata envelope - -Every primitive carries a standard metadata envelope: creation timestamp, producing source or node identifier, graph ID, run ID, user-defined tags, and an extensible custom fields map. This envelope enables correlation, filtering, and auditing across the entire data lifecycle. - -### 4.3 Lineage - -Each transformation appends a lineage record to the primitive: the node ID that performed the operation, the operation name, a timestamp, the IDs of input primitives, and the parameters used. This enables full forward and backward tracing — given a vector in a database, trace back to the source document and every transformation it passed through. - -### 4.4 Source, Sink, and Action interfaces - -`nvisy-core` exports the abstract interfaces that all connectors and actions must implement. This is the extension contract of the platform: - -- **Source** — reads primitives from an external system. Declares supported primitive types. Returns an `AsyncIterable` stream of primitives with resumption context. -- **Sink** — writes primitives to an external system. Declares capabilities (batch size, upsert support, rate limits). Returns a write function that accepts individual primitives. -- **Action** — transforms primitives via the `pipe` method: receives an `AsyncIterable` of input primitives and returns an `AsyncIterable` of output primitives. Actions may be stateless (map, filter) or stateful (deduplicate, aggregate). - -All three interfaces include lifecycle methods (connect, disconnect) and capability declarations. By housing these interfaces in `nvisy-core`, the connector packages and any community-contributed connectors share a single, versioned contract. All methods use standard `async`/`await` and `AsyncIterable` — no special runtime library is required. - -### 4.5 Error taxonomy - -The core package defines a structured error hierarchy with machine-readable tags. The hierarchy distinguishes connector errors, validation errors, rate limit errors, timeout errors, graph compilation errors, and node execution errors. Each error class carries a `retryable` flag so the runtime can distinguish transient failures from terminal ones without string matching. - -### 4.6 Observability primitives - -Core defines the foundational observability types: structured log schema (JSON, with correlation IDs for graph, run, and node), metric types (counters, histograms, gauges), trace span interface (OpenTelemetry-compatible), and lineage record structure. It also exports utility functions for logging, metric emission, and span creation that all packages use uniformly. - -### 4.7 Utilities - -Common utilities shared across packages live in core: ULID generation, content hashing, primitive serialization/deserialization, and type-safe builder helpers for constructing primitives. - ---- - -## 5. Connector Packages - -### 5.1 Package separation rationale - -Connectors are split into three domain-specific packages rather than a single monolithic package. This serves two goals: - -1. **Install footprint.** Each connector package carries peer dependencies on the relevant client libraries (e.g., `@qdrant/js-client-rest`, `@aws-sdk/client-s3`, `pg`). Users who only need vector database connectors should not be forced to install SQL drivers or S3 SDKs. - -2. **Release independence.** A breaking change in a vector database client library should not force a release of the SQL connector package. Domain-specific packages can be versioned and released independently. - -### 5.2 `nvisy-plugin-sql` - -Implements the Source and Sink interfaces for relational databases. Initial targets: PostgreSQL and MySQL. Connectors handle connection pooling, query generation, type mapping between SQL types and primitive payloads, and batch insert/upsert operations. - -### 5.3 `nvisy-plugin-object` - -Implements the Source and Sink interfaces for object stores and file formats. Initial targets: S3, GCS, Parquet, JSONL, and CSV. Object store connectors handle multipart uploads, streaming reads, and prefix-based listing. File format connectors handle serialization, deserialization, schema inference, and chunked reading for large files. - -### 5.4 `nvisy-plugin-vector` - -Implements the Source and Sink interfaces for vector databases. Initial targets: Pinecone, Qdrant, Milvus, Weaviate, and pgvector. Vector connectors handle collection/index management, upsert with metadata, batch operations, and dimensionality validation. - -### 5.5 Plugin registration - -Plugins are registered with the `Engine` at startup via `engine.register(plugin)`. Each plugin bundles providers, streams, and actions under a namespace (e.g. `"sql"`, `"ai"`). The graph compiler resolves references like `"sql/postgres"` or `"ai/embed"` against the registry at compilation time. Community plugins install as npm packages and export a `PluginInstance` implementing the standard interface from `nvisy-core`. - ---- - -## 6. Runtime (`nvisy-runtime`) - -The runtime package owns the full lifecycle of a graph: definition, compilation, and execution. It is the central package — plugin packages register their providers, streams, and actions into the runtime's `Engine`, and the server orchestrates this wiring at startup. - -### 6.1 Graph definition and JSON serializability - -The central design constraint is that every graph must be representable as a plain JSON document. This means graphs can be stored in a database, versioned in source control, transmitted over an API, diffed, and reconstructed without loss. The programmatic TypeScript API is a convenience layer that produces the same JSON structure. - -A graph is a directed acyclic graph of **nodes**. Each node declares its type (source, action, sink, branch, fanout, fanin), its configuration, its upstream dependencies, and optional execution policies (retry, timeout, concurrency). Nodes are connected by edges derived from the dependency declarations. - -The graph JSON schema is defined using **Zod**. This provides runtime validation, TypeScript type derivation, and structured parse errors in a single definition. - -### 6.2 Node types - -**Source** — References a registered source connector by name and provides connection and extraction configuration. - -**Action** — Applies a transformation to primitives. Actions are resolved by name from the registry. The runtime provides built-in generic actions (filter, map, batch, deduplicate, validate, convert). Domain-specific actions (embed, chunk, complete) are provided by plugin packages (`nvisy-plugin-ai`, etc.) and registered into the engine by the application at startup. - -**Sink** — References a registered sink connector by name and provides connection and load configuration. - -**Branch** — Routes each primitive to one of several downstream nodes based on a predicate. A default route handles unmatched primitives. - -**FanOut** — Duplicates each primitive to multiple downstream nodes for parallel processing (e.g., embedding the same text with multiple models simultaneously). - -**FanIn** — Collects primitives from multiple upstream nodes and merges them into a single stream. - -### 6.3 Registries - -The runtime uses three registries for resolving node references to implementations: - -- **SourceRegistry** — Maps source names to `DataSource` factories. Populated by plugin packages (`nvisy-plugin-sql`, `nvisy-plugin-vector`, etc.) via `engine.register()`. -- **SinkRegistry** — Maps sink names to `DataSink` factories. Same population model. -- **ActionRegistry** — Maps action names to `Action` factories. Pre-loaded with the runtime's built-in generic actions. Extended by plugin packages (`nvisy-plugin-ai`, etc.). - -Separation ensures type safety (each registry returns the correct interface without narrowing), independent testability (mock only what you need), and clear error messages ("unknown source: xyz" vs "unknown action: xyz"). Plugin packages export a `PluginInstance` that the application registers with the engine at startup — the runtime itself has no compile-time dependency on any plugin. - -### 6.4 Built-in actions - -The runtime owns a set of generic, data-plane actions that operate on primitives without external dependencies: - -- **filter** — Drop primitives that don't match a predicate. -- **map** — Transform primitive fields (rename, reshape, project). -- **batch** — Group N primitives into batches before forwarding downstream. -- **deduplicate** — Drop duplicates by content hash or user-defined key. -- **validate** — Assert primitives match a schema; route failures to DLQ. -- **convert** — Cast between primitive types (Row → Document, etc.). - -These are pre-registered in the ActionRegistry. Domain-specific actions (embed, chunk, complete, extract) live in their respective packages. - -### 6.5 Graph compilation - -The compiler pipeline transforms raw JSON into an executable plan in three stages: - -1. **Parse** (`compiler/parse.ts`) — Decode the raw JSON against the Zod graph schema. This produces a typed, validated graph structure or structured errors with JSON paths. - -2. **Validate** (`compiler/validate.ts`) — Structural DAG validation: cycle detection via topological sort, dangling node references, type compatibility between connected nodes (source output types match downstream action input types), and connector/action name resolution against the registries. - -3. **Plan** (`compiler/plan.ts`) — Build the `ExecutionPlan`: resolve all names to concrete implementations via the registries, compute the topological execution order, aggregate concurrency constraints, and wire rate limit policies. The `ExecutionPlan` is an immutable data structure. Compilation throws structured errors on failure. - -### 6.6 Execution model - -The engine executes an `ExecutionPlan` as an Effection task. It spawns each node as a child operation, with dependency tracking ensuring nodes wait for their upstream inputs before executing. Effection's structured concurrency guarantees that halting the top-level task automatically cancels all in-flight nodes. The executor maintains a **done store** of completed results (both successes and terminal failures) and coordinates data flow via inter-node queues. - -### 6.7 Data flow between nodes - -Each edge in the DAG is backed by an Effection queue. Producer nodes push primitives into the queue; consumer nodes pull from it. For action nodes that expect an `AsyncIterable` input, the runtime bridges Effection queues into a `ReadableStream` via a `TransformStream`, allowing actions to consume data with standard `for await...of` iteration. - -Data flows through the system using the native `AsyncIterable` protocol. Sources yield items, actions pipe one `AsyncIterable` to another, and sinks consume items via a write function. No special streaming library is required — the platform relies entirely on JavaScript's built-in async iteration. - -For nodes that require all upstream data before starting (e.g., deduplication across the full dataset), a **materialization barrier** can be configured. The barrier drains the upstream queue into an array before forwarding to the node. - -Fan-out nodes push each primitive to multiple downstream queues. Fan-in nodes pull from multiple upstream queues and merge into a single stream. - -### 6.8 Retry policy - -Each node can define a retry policy specifying maximum retries, backoff strategy (fixed, exponential, or jitter), initial and maximum delay, and an optional allowlist of retryable error codes. The runtime implements retries as a generator-based loop using Effection's `sleep` for backoff delays. It distinguishes between retryable errors (network timeouts, rate limits, transient API failures) and terminal errors (authentication failures, schema violations, invalid configuration). Terminal errors fail the node immediately. - -### 6.9 Rate limiting - -Rate limits are enforced per-connector. When a node issues a request that would exceed the rate limit, the operation is suspended until tokens are available. Rate limits are declared in connector capabilities and can be overridden in graph configuration. - -### 6.10 Concurrency control - -Global concurrency is bounded by a configurable limit (default: 10 permits). Per-node concurrency can be set individually. The runtime respects both limits simultaneously. The concurrency pool (`engine/pool.ts`) manages this. - -### 6.11 Runtime observability - -The runtime emits structured metrics and trace spans for every graph run. Each run is an OpenTelemetry trace; each node execution is a span within that trace. Metrics include run duration, run status (success, partial failure, failure), per-node execution time, primitives processed and failed, connector calls issued, and rate limit wait time. - ---- - -## 7. Server (`nvisy-server`) - -### 7.1 Role - -The Node.js server is a **stateless execution worker**. It accepts graph JSON, compiles and executes it via the runtime, and reports status of in-flight runs. It does not persist graph definitions, run history, or lineage — that responsibility belongs to a separate persistent server (written in Rust). This package is a thin HTTP interface over the runtime engine. - -### 7.2 HTTP layer - -The HTTP layer is built on **Hono**, a lightweight, edge-compatible web framework. Hono provides routing, middleware composition, and request validation with minimal overhead. The server entry point is `main.ts`, which starts a Node.js HTTP server via `@hono/node-server`. - -### 7.3 Middleware - -All requests pass through two middleware layers: - -- **Request ID** — Assigns a unique `X-Request-Id` header to every request for correlation. -- **Request logger** — Emits structured JSON logs (method, path, status, latency, request ID) for every request. - -### 7.4 REST API - -The API surface covers health checks, graph execution, validation, and in-flight run management. All endpoints accept and return JSON. Connectors are defined within the graph JSON schema, not managed separately. - -| Method | Path | Description | -|--------|------|-------------| -| `GET` | `/health` | Liveness probe | -| `GET` | `/ready` | Readiness probe | -| `POST` | `/api/v1/graphs/execute` | Submit a graph for execution; returns `{ runId }` immediately | -| `POST` | `/api/v1/graphs/validate` | Compile and validate a graph without executing | -| `GET` | `/api/v1/graphs` | List in-flight runs | -| `GET` | `/api/v1/graphs/:runId` | Get detailed status of a single in-flight run | -| `DELETE` | `/api/v1/graphs/:runId` | Cancel a running execution | - -### 7.5 Server observability - -The server layer emits HTTP request logs (structured JSON with method, path, status, latency, request ID) and exposes health check and readiness endpoints. - ---- - -## 8. Error Handling - -### 8.1 Error propagation - -When a node encounters an error, the runtime first checks retryability via the error's `retryable` flag. If the error is retryable and retries remain, the node is re-attempted with backoff. If the error is terminal or retries are exhausted, the node is marked as failed. Downstream nodes that depend on the failed node are marked as skipped. Independent branches of the DAG continue executing. The graph run is marked as `partial_failure` or `failure` depending on whether any terminal sink node succeeded. - -Errors are caught at the node executor boundary and recorded in the run result. The structured error hierarchy ensures consistent, machine-readable failure information at every layer. - -### 8.2 Dead letter queue - -Primitives that fail processing can be routed to a dead letter queue (DLQ) instead of failing the entire node. This allows the graph to continue processing valid data while capturing failures for later inspection and replay. - ---- - -## 9. Security - -### 9.1 Secret management - -Connector credentials are never stored in graph definitions. They are resolved at runtime from environment variables, a pluggable secret provider interface (supporting AWS Secrets Manager, HashiCorp Vault, and similar systems), or `.env` files in development. - -### 9.2 Network and access control - -In server mode, the REST API supports TLS termination, bearer token authentication, IP allowlisting, and CORS configuration via Hono middleware. - -### 9.3 Data handling - -Primitives may contain sensitive data (PII in completions, proprietary embeddings). The platform provides configurable data retention policies per graph, primitive redaction hooks for logging, and encryption at rest for server-mode persistence. - ---- - -## 10. Performance Considerations - -### 10.1 Memory management - -Primitives are processed in streaming fashion wherever possible using `AsyncIterable`. Nodes that must materialize full datasets (deduplication, sorting) use configurable memory limits and spill to disk when exceeded. - -### 10.2 Embedding vectors - -Embedding vectors use `Float32Array` for memory efficiency — a 1536-dimensional embedding occupies 6 KB vs. approximately 24 KB as a JSON number array. For large-scale embedding workloads, this 4x reduction is significant. - -### 10.3 Batching - -Connectors declare their optimal batch size. The runtime automatically batches primitives to match, reducing round trips to external systems. Batching respects rate limits — a batch that would exceed the rate limit is delayed, not split. - -### 10.4 Backpressure - -Effection queues between nodes provide natural backpressure. When a downstream node processes slower than its upstream, the connecting queue suspends the upstream operation until the downstream is ready. This prevents memory exhaustion in unbalanced graphs without manual flow control. - ---- - -## 11. Extension Points - -The platform is designed for extension at multiple levels: - -| Extension Point | Mechanism | Example | -|----------------|-----------|---------| -| Custom primitive types | Extend the primitive type union and implement a payload interface | Graph embedding type | -| Custom connectors | Implement Source or Sink from `nvisy-core` | Elasticsearch connector | -| Custom actions | Implement Action from `nvisy-core`, or provide an inline function | Custom chunking strategy | -| Custom secret providers | Implement the SecretProvider interface | Azure Key Vault integration | -| Custom metric exporters | Implement the MetricExporter interface | StatsD exporter | -| Custom storage backends | Implement the StorageBackend interface (server mode) | MongoDB storage | diff --git a/docs/COMPLIANCE.md b/docs/COMPLIANCE.md new file mode 100644 index 0000000..3bd0c1a --- /dev/null +++ b/docs/COMPLIANCE.md @@ -0,0 +1,80 @@ +# Compliance & Audit + +## 1. Overview + +Enterprises do not purchase redaction tools; they purchase compliance guarantees. The value of automated redaction is realized only when the organization can demonstrate — to regulators, auditors, and legal counsel — that sensitive data was identified, handled, and redacted in accordance with applicable policy. + +This requires two complementary capabilities: a policy engine that encodes regulatory and organizational rules into executable redaction policies, and an audit system that records every decision, action, and outcome with sufficient detail to reconstruct the chain of custody for any piece of content. + +## 2. Policy Engine + +### 2.1 Policy Definition + +The platform must provide a policy builder that enables administrators to define redaction rules without writing code. Policies should express conditions over entity types, document classifications, confidence thresholds, and organizational context. + +### 2.2 Regulation Packs + +Prebuilt policy packs aligned to common regulatory frameworks should be available out of the box: + +- **HIPAA**: Protected health information in medical records, communications, and claims. +- **GDPR**: Personal data of EU residents across all modalities. +- **PCI-DSS**: Payment card data in documents, images, and structured records. +- **CJIS**: Criminal justice information in law enforcement contexts. +- **CCPA**: Personal information of California residents, including the right to deletion and opt-out of sale. +- **FERPA**: Student educational records and related identifiers. + +### 2.3 Policy Simulation + +Before a policy is applied to production data, administrators must be able to simulate its effect — previewing what would be redacted across a representative sample. This "dry run" capability reduces the risk of over-redaction or under-redaction in production. + +### 2.4 Policy Versioning and Approval + +Policies must be versioned, with a full history of changes. Modifications to active policies should require approval through a configurable workflow before taking effect. + +## 3. Explainability + +Every redaction decision must be explainable. The system must record and surface: + +- **What was redacted**: The specific content span, region, or audio segment. +- **Why it was redacted**: The triggering rule, pattern, or model prediction. +- **Which model version**: The exact version of any ML model involved in the decision. +- **Confidence level**: The detection confidence associated with the decision. +- **Who reviewed it**: The identity of any human reviewer who approved, rejected, or modified the decision. +- **When it was processed**: Timestamps for each stage of the pipeline. + +## 4. Audit Trails + +### 4.1 Immutability + +Audit logs must be append-only and tamper-evident. Once a record is written, it cannot be modified or deleted. + +### 4.2 Chain of Custody + +The audit system must maintain a complete chain of custody for every piece of content: from ingestion, through detection and redaction, to export. Every access event — who viewed the content and when — must be recorded. + +### 4.3 Reporting + +The platform must generate compliance reports suitable for submission to regulators and internal audit teams. Reports should include: + +- Redaction statistics by entity type, document category, and time period +- Policy adherence metrics +- Reviewer activity and approval rates +- Exceptions and overrides + +### 4.4 SOC 2 Readiness + +Logging infrastructure must meet the requirements of SOC 2 Type II certification, including continuous monitoring, access controls, and retention policies. + +## 5. Data Retention Policies + +### 5.1 Original Content + +The platform must enforce configurable retention policies for original (pre-redaction) content. Organizations must be able to specify maximum retention periods after which originals are permanently deleted. Zero-retention mode — in which originals are discarded immediately after processing — must be available for environments where persistent storage of sensitive content is prohibited. + +### 5.2 Redacted Output + +Redacted artifacts may be retained independently of originals, subject to their own retention schedule. The platform must track the lifecycle of each artifact and enforce automated deletion at expiry. + +### 5.3 Audit Logs + +Audit log retention must be configurable separately from content retention. Regulatory frameworks often require audit records to be retained for longer periods than the underlying data (e.g., seven years for HIPAA, six years for SOX). Audit logs must never be deleted before their configured retention period expires, regardless of content deletion status. diff --git a/docs/DETECTION.md b/docs/DETECTION.md new file mode 100644 index 0000000..9a297d8 --- /dev/null +++ b/docs/DETECTION.md @@ -0,0 +1,87 @@ +# Sensitive Data Detection + +## 1. Overview + +The detection engine is the core intellectual property of the platform. It is responsible for identifying sensitive content across all supported modalities with high precision and recall. Detection must operate through multiple complementary strategies — deterministic pattern matching, learned models, and computer vision — to achieve robust coverage across diverse content types and regulatory categories. + +## 2. Language Coverage + +The platform must support detection across multiple languages and writing systems. Real-world data frequently contains non-English text, multilingual documents, and code-switched content (multiple languages within a single document or conversation). Detection models must handle at minimum the major European languages, CJK (Chinese, Japanese, Korean), and Arabic script. Deterministic patterns must be parameterized by locale — national identifier formats, date conventions, and address structures vary by jurisdiction. + +For audio, speech-to-text and subsequent NER must support the same language set, including language identification and mid-utterance language switching. + +## 3. Deterministic Detection + +Deterministic methods provide high-precision, low-latency detection for well-defined patterns: + +- **Regular expressions**: Pattern matching for structured identifiers such as Social Security numbers, credit card numbers, passport numbers, and other nationally defined formats. +- **Checksum validation**: Algorithmic verification (e.g., Luhn algorithm for credit card numbers) to reduce false positives from pattern matching alone. +- **Custom pattern libraries**: User-defined pattern sets that extend detection to organization-specific sensitive categories such as internal project identifiers, proprietary terms, or custom reference numbers. + +## 4. Machine Learning and NLP-Based Detection + +Learned models address the detection of sensitive content that cannot be captured by fixed patterns: + +- **Named entity recognition (NER)**: Identification of person names, locations, organizations, and other entity types in unstructured text. +- **Domain-specific entity models**: Specialized models trained on financial data, medical records (HIPAA-relevant entities), legal identifiers, and biometric references. +- **Contextual detection**: Inference of sensitivity from surrounding context rather than explicit entity presence. Phrases such as "the patient" or "my lawyer" may indicate sensitive content even in the absence of a named entity. This capability requires models that reason over discourse context rather than isolated tokens. + +## 5. Computer Vision Detection + +Visual content requires detection methods that operate on pixel-level and spatial features: + +- **Face detection and recognition**: Identification of human faces in images and video frames for subsequent obfuscation. +- **Document and identifier detection**: Recognition of identity documents, license plates, and other visual identifiers. +- **Handwritten text detection**: Extraction and analysis of handwritten content in scanned documents and images. +- **Screen capture analysis**: Detection of sensitive text rendered in screenshots, application windows, and other digital captures. + +## 6. Audio Detection + +Audio content introduces temporal and speaker-level dimensions to detection: + +- **Transcript-based NER**: Application of named entity recognition to speech-to-text output, with alignment back to audio timestamps. +- **Direct waveform redaction**: Replacement of sensitive audio segments with silence, tones, or noise at the waveform level. +- **Speaker-specific redaction**: Selective redaction of content from identified speakers while preserving contributions from others, enabled by speaker diarization. + +## 7. Policies + +Detection is governed by policies — declarative rule sets that define what to detect and how to handle it. A policy is the primary configuration surface through which administrators, compliance officers, and integrators express redaction intent without modifying detection code. + +### 7.1 Policy Structure + +A policy is a named, versioned collection of rules. Each rule specifies: + +- **What to detect**: An entity type, pattern, or semantic category (e.g., "SSN", "face", "medical diagnosis"). +- **Detection parameters**: Confidence thresholds, locale constraints, and any modality-specific settings. +- **Redaction action**: The redaction method to apply when the rule matches (mask, blur, replace, or suppress). +- **Additional context**: Free-form or structured metadata attached to a rule that provides guidance to downstream stages — for example, a justification string, a regulatory citation, or instructions for human reviewers. + +### 7.2 Policy Composition + +Policies may extend other policies. A child policy inherits all rules from its parent and may add new rules, override inherited rules, or narrow inherited thresholds. This composition model enables organizations to maintain a base compliance policy (e.g., "HIPAA") and extend it with organization-specific additions without duplicating the base rule set. + +### 7.3 Annotations + +Files submitted for processing may carry annotations — either provided by the user at submission time or attached as part of a broader context (e.g., a case management system that tags documents with classification labels). Annotations can include: + +- **Pre-identified regions**: Bounding boxes, text spans, or time ranges that the submitter has already marked as sensitive, bypassing or supplementing automated detection. +- **Classification labels**: Document-level or region-level labels (e.g., "contains PHI", "attorney-client privileged") that influence which policy rules apply. +- **Exclusion markers**: Regions or entities explicitly marked as non-sensitive, instructing the detection engine to skip them. + +The detection engine must consume annotations as first-class inputs alongside its own detection results, merging user-provided and machine-generated findings into a unified annotation set before redaction. + +## 8. Detection Orchestration + +Individual detection strategies — deterministic, ML-based, vision, and audio — must be composed into a coherent pipeline rather than operating in isolation. + +### 8.1 Tiered Execution + +Detection should proceed in tiers ordered by cost and specificity. Deterministic patterns (regex, checksums) execute first, providing high-precision results at minimal computational cost. ML and vision models execute subsequently, targeting content that deterministic methods cannot address. This tiered architecture avoids unnecessary GPU inference for content that can be resolved through pattern matching alone. + +### 8.2 Result Merging + +When multiple detection strategies identify overlapping or adjacent sensitive regions within the same content, the platform must merge results into a unified set of detection annotations. Overlapping detections should be consolidated rather than duplicated. Each merged annotation must retain provenance — which strategies contributed to the detection and at what confidence level. + +### 8.3 Conflict Resolution + +When detection strategies disagree — for example, a regex match identifies a number as a credit card while an NER model classifies the surrounding context as non-sensitive — the platform must apply configurable conflict resolution rules. Default behavior should favor the higher-confidence or higher-sensitivity classification, but administrators must be able to override this through policy. diff --git a/docs/DEVELOPER.md b/docs/DEVELOPER.md new file mode 100644 index 0000000..20776dc --- /dev/null +++ b/docs/DEVELOPER.md @@ -0,0 +1,65 @@ +# Developer Experience & Advanced Capabilities + +## 1. Overview + +Platform adoption scales with developer experience. Organizations evaluate redaction platforms not only on detection accuracy but on how quickly they can integrate the platform into existing systems, automate workflows, and extend capabilities to meet domain-specific requirements. A first-class developer experience reduces time-to-value and expands the platform's addressable market beyond compliance teams to engineering organizations. + +## 2. Core Interfaces + +### 2.1 REST API + +A comprehensive REST API must expose all platform capabilities — content submission, policy management, redaction retrieval, and audit log access — as documented, versioned endpoints. The API is the primary integration surface and must be treated as a first-class product. + +API versioning must follow a clear strategy (URI-based or header-based) with documented deprecation timelines. Breaking changes must not be introduced without a major version increment and a migration period. + +### 2.2 SDKs + +Official client libraries for Python and JavaScript (at minimum) should wrap the REST API with idiomatic interfaces, type safety, and built-in error handling. SDKs lower the integration barrier and reduce the likelihood of misuse. + +### 2.3 Authentication and Rate Limiting + +All API access must be authenticated. The platform should support API key authentication for machine-to-machine integrations and OAuth 2.0 for user-facing applications. API keys must be scoped to specific permissions and rotatable without downtime. + +Rate limiting must be enforced per client and per tenant to prevent abuse and ensure fair resource allocation. Rate limit headers must be included in API responses so that clients can implement backoff strategies. Configurable rate tiers should be available for different client classes (e.g., higher limits for batch processing clients, lower limits for interactive use). + +### 2.4 Webhooks and Events + +An event-driven notification system must allow consumers to subscribe to processing lifecycle events — content ingested, detection complete, redaction applied, review approved — without polling. Webhook delivery should be reliable, with retry logic and delivery confirmation. + +## 3. Tooling + +### 3.1 CLI + +A command-line interface should support all common operations — submitting content, querying status, downloading results, managing policies — for scripting, automation, and developer workflows. + +### 3.2 Infrastructure as Code + +Terraform modules (or equivalent) should be provided for provisioning and configuring the platform in cloud environments, enabling reproducible deployments managed through version-controlled infrastructure definitions. + +### 3.3 Sample Policies and Synthetic Data + +A library of sample redaction policies and a synthetic data generator should be available to accelerate development and testing. Developers should be able to exercise the full pipeline against realistic but non-sensitive data without access to production content. + +## 4. Advanced Capabilities + +The following capabilities extend the platform beyond standard redaction into a category-defining position. Each represents an opportunity to increase the platform's value density — reducing the distance between raw ingestion and actionable, compliant output. + +### 4.1 Risk Scoring + +Documents and datasets should be scored by aggregate privacy exposure level, enabling organizations to prioritize review effort and allocate resources toward the highest-risk content. + +### 4.2 Smart Redaction Suggestion + +Rather than applying maximal redaction, the platform should be capable of suggesting the minimal set of redactions required to satisfy a given regulatory standard. This preserves data utility while meeting compliance obligations. + +### 4.3 Data Lineage Visualization + +A visual representation of the processing pipeline — from ingestion through detection, redaction, and export — should be available for each piece of content. Data lineage supports debugging, audit preparation, and stakeholder communication. + +### 4.4 Semantic Redaction + +Beyond named entity redaction, the platform should support redaction of semantic categories — for example, references to rare diseases, specific legal proceedings, or proprietary methodologies — that carry sensitivity not through the presence of a specific identifier but through their meaning in context. + +### 4.5 Synthetic Data Replacement + +Rather than replacing sensitive content with black bars or placeholder tokens, the platform should support replacement with realistic synthetic alternatives — generated names, addresses, dates, and other values that preserve the statistical and structural properties of the original data while eliminating re-identification risk. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md deleted file mode 100644 index 17517d0..0000000 --- a/docs/DEVELOPMENT.md +++ /dev/null @@ -1,133 +0,0 @@ -# Nvisy Runtime — Development - -**Technology choices and development roadmap for the Nvisy Runtime platform.** - ---- - -## Technology Choices - -| Concern | Choice | Rationale | -|---------|--------|-----------| -| Language | TypeScript | Type safety for the primitive system, broad ecosystem | -| Module system | ESM only | Modern standard, native in Node.js, tree-shakeable | -| Runtime | Node.js | Async I/O suited for connector-heavy workloads, npm ecosystem | -| Structured concurrency | Effection | Generator-based structured concurrency for DAG execution | -| Validation | Zod | Runtime validation, TypeScript type derivation, structured parse errors | -| Graph library | Graphology | DAG construction, cycle detection, topological sort | -| Package manager | npm workspaces | Monorepo management without additional tooling | -| Build | tsup | Fast TypeScript compilation, ESM output, declaration generation | -| Testing | Vitest | Fast, TypeScript-native, ESM-compatible | -| Linting | Biome | Unified formatter and linter, high performance | -| HTTP framework | Hono | Lightweight, edge-compatible, fast routing, middleware composition | -| Cron | croner | Lightweight, timezone-aware scheduling | - ---- - -## Development Roadmap - -### Phase 1 — Foundation - -Core infrastructure and proof-of-concept connectors. - -- **`nvisy-core`** - - Primitive type system (embedding, completion, structured_output, tool_call_trace, image, audio, fine_tune_sample, raw) - - Zod-based validation and type derivation - - Error taxonomy with machine-readable tags and retryable flags - - Base Source, Sink, and Action interfaces (AsyncIterable-based) - - Observability primitives (structured logging, metrics, tracing) - - Utility library (ULID generation, content hashing, serialization) - -- **`nvisy-runtime`** - - Graph JSON schema definition (Zod) - - JSON parser and graph validator - - DAG compiler (cycle detection, dependency resolution, execution planning) - - Execution engine with Effection-based structured concurrency - - Retry policies (fixed, exponential, jitter backoff) - - Timeout policies (per-node execution limits) - - Concurrency control (global and per-node limits) - - Built-in generic actions (filter, map, batch, deduplicate, validate, convert) - - Runtime metrics and OpenTelemetry tracing - -- **`nvisy-plugin-object`** - - S3 source and sink (multipart upload, streaming read, prefix listing) - - JSONL source and sink (line-delimited JSON, schema inference) - -- **`nvisy-plugin-vector`** - - Qdrant source and sink (collection management, upsert with metadata, dimensionality validation) - -### Phase 2 — Breadth - -Expand connector coverage, add domain-specific actions. - -- **`nvisy-plugin-vector`** - - Pinecone connector - - Milvus connector - - Weaviate connector - - pgvector connector - -- **`nvisy-plugin-sql`** - - PostgreSQL source and sink (connection pooling, query generation, batch upsert) - - MySQL source and sink - - MSSQL source and sink - -- **`nvisy-plugin-object`** - - GCS source and sink - - Parquet source and sink (columnar read/write, schema mapping) - - CSV source and sink (header detection, type inference, chunked reading) - -- **`nvisy-plugin-ai`** - - Embedding action (multi-provider: OpenAI, Anthropic, Cohere, Gemini) - - Chunking actions (fixed-size, contextual, similarity-based) - - Completion action (structured output extraction) - - Enrichment action (metadata augmentation via LLM) - -- **Runtime additions** - - Dead letter queue support (per-node failure routing) - - Dry-run mode (compile and validate without executing) - - Resumable execution (checkpoint and resume from last successful context) - -### Phase 3 — Server - -HTTP server, scheduling, and operational tooling. - -- **`nvisy-server`** - - REST API (Hono) for graph execution, validation, and run management - - Cron scheduler (croner) for time-based pipeline triggers - - Webhook-based event triggers - - Request logging and structured observability - - Health and readiness endpoints - -- **Storage backends** - - SQLite for development and single-node deployments - - PostgreSQL for production deployments - -- **Web dashboard** - - Run monitoring and status visualization - - Lineage exploration (trace primitives through transformations) - - Failure inspection and replay - -### Phase 4 — Production Hardening - -Performance, security, and operational maturity. - -- **Performance** - - Backpressure tuning and memory management - - Disk spill for materialization nodes (deduplication, sorting over large datasets) - - Batching optimization (adaptive batch sizing based on connector feedback) - - Performance benchmarks and profiling - -- **Security** - - Secret provider integrations (AWS Secrets Manager, HashiCorp Vault, Azure Key Vault) - - TLS termination and certificate management - - Bearer token authentication and API key management - - IP allowlisting and CORS configuration - -- **Operational** - - Graceful shutdown and in-flight run draining - - Configuration hot-reload - - Structured alerting on pipeline failures - -- **Community** - - Plugin SDK documentation and examples - - Connector contribution guide - - Published npm packages with semantic versioning diff --git a/docs/INFRASTRUCTURE.md b/docs/INFRASTRUCTURE.md new file mode 100644 index 0000000..8bc2ac3 --- /dev/null +++ b/docs/INFRASTRUCTURE.md @@ -0,0 +1,81 @@ +# Infrastructure + +## 1. Overview + +The regulated industries that require multimodal redaction — healthcare, legal, government, and financial services — impose stringent requirements on where and how data is processed. The platform must accommodate diverse deployment models, scale to meet variable workloads, and maintain rigorous security controls throughout. + +## 2. Deployment Models + +### 2.1 Cloud and On-Premises + +The platform must support deployment across multiple environments: + +- **Cloud-hosted**: Managed deployment in the vendor's infrastructure for organizations that accept cloud processing. +- **VPC deployment**: Installation within the customer's own virtual private cloud, ensuring data never leaves their network boundary. +- **On-premises**: Full deployment on customer-owned hardware for organizations with strict data sovereignty requirements. +- **Air-gapped**: Operation without network connectivity, required by certain government and defense use cases. +- **Edge processing**: Lightweight deployment at the point of data capture, relevant for law enforcement body cameras, field operations, and other latency-sensitive scenarios. + +### 2.2 Architecture + +The platform must be API-first, supporting both batch and streaming processing modes. An API-first design ensures that all platform capabilities are accessible programmatically, enabling integration into existing enterprise workflows without dependence on the platform's own user interface. + +## 3. Performance and Scale + +### 3.1 Workload Requirements + +The platform must handle workloads that span orders of magnitude in volume and latency sensitivity: + +- Large document sets (thousands to millions of PDFs) +- Long-form video and audio files +- Real-time stream redaction with sub-second latency targets +- Concurrent processing across multiple tenants or projects + +### 3.2 Scaling + +Horizontal scaling must be supported, allowing compute capacity to expand proportionally with workload volume. GPU acceleration should be available for ML inference workloads where throughput or latency requirements exceed CPU capacity. + +### 3.3 Cost Optimization + +The platform should optimize processing cost by routing content through the appropriate detection tier. Simple deterministic pattern matches should not incur the computational cost of ML inference. A tiered processing architecture — regex first, ML models only when deterministic methods are insufficient — reduces cost without sacrificing detection coverage. + +## 4. Security + +### 4.1 Data Protection + +Given that the platform processes the most sensitive data an organization holds, security must be foundational rather than additive: + +- **Encryption**: All data must be encrypted in transit (TLS) and at rest (AES-256 or equivalent). Field-level encryption should be available for particularly sensitive attributes. +- **Key management**: Integration with enterprise key management systems (AWS KMS, Azure Key Vault, HashiCorp Vault) for encryption key lifecycle management. +- **Zero-retention processing**: An operating mode in which no content persists on the platform after processing is complete. Content is held in memory only for the duration of the pipeline execution. +- **Ephemeral compute**: Processing environments that are created for each job and destroyed upon completion, leaving no residual data on disk. + +### 4.2 Access Control + +- **Role-based access control (RBAC)**: Fine-grained permissions governing who can configure policies, submit content, review redactions, and access audit logs. +- **Single sign-on (SSO) and SCIM**: Integration with enterprise identity providers for authentication and automated user provisioning. +- **Data residency controls**: Configuration to ensure that content is processed and stored only within specified geographic regions, in compliance with data sovereignty requirements. + +## 5. Multi-Tenancy + +### 5.1 Tenant Isolation + +The platform must support multi-tenant deployment with strict isolation between tenants. Content, policies, audit logs, detection models, and configuration must be segregated such that no tenant can access another tenant's data or influence another tenant's processing. Isolation must be enforced at the data layer (separate storage namespaces or encryption keys per tenant), the compute layer (dedicated or partitioned processing resources), and the API layer (tenant-scoped authentication and authorization). + +### 5.2 Tenant-Specific Configuration + +Each tenant must be able to configure its own detection policies, redaction rules, retention periods, and export formats independently. Platform-wide defaults may be set by the operator, but tenants must be able to override them within their permitted scope. + +## 6. Observability + +### 6.1 Metrics + +The platform must expose operational metrics covering ingestion throughput, detection latency, redaction processing time, queue depth, error rates, and resource utilization. Metrics must be available in a format compatible with standard monitoring systems (Prometheus, OpenTelemetry, or equivalent). + +### 6.2 Distributed Tracing + +Each piece of content should carry a trace identifier through every stage of the pipeline — ingestion, detection, redaction, review, and export. Distributed tracing enables operators to diagnose latency bottlenecks, identify failed processing stages, and correlate events across services. + +### 6.3 Alerting + +Configurable alerts must be available for operational anomalies: elevated error rates, processing latency exceeding thresholds, queue backpressure, model inference failures, and storage capacity warnings. Alerts must be deliverable through standard channels (email, webhook, PagerDuty, or equivalent). diff --git a/docs/INGESTION.md b/docs/INGESTION.md new file mode 100644 index 0000000..3d2a208 --- /dev/null +++ b/docs/INGESTION.md @@ -0,0 +1,92 @@ +# Ingestion & Transformation + +## 1. Overview + +The ingestion layer is responsible for accepting content from heterogeneous sources and normalizing it into a unified internal representation suitable for downstream detection and redaction. The transformation layer handles the inverse concern: producing redacted output in the appropriate format while preserving the structural integrity of the original document. + +The quality of the ingestion layer is a critical success factor. Redaction platforms that cannot reliably parse and extract content from real-world documents — scanned forms, embedded tables, multi-speaker audio — will produce incomplete redaction results regardless of the sophistication of their detection models. + +## 2. Supported Input Formats + +The platform must support ingestion across multiple modalities. Formats are organized into tiers reflecting implementation priority and expected coverage at each stage of the product lifecycle. + +### Tier 1 — Core (launch requirement) + +These formats represent the most common inputs in regulated enterprise environments and must be supported at general availability: + +- **PDF**: Native (digitally authored) and scanned, including multi-page documents with mixed content (text, images, tables, forms). +- **Images**: JPG, PNG, TIFF — the dominant formats for scanned documents, photographs, and screenshots. +- **Plain text and markup**: TXT, HTML, and Markdown. +- **Structured data**: CSV and JSON. + +### Tier 2 — Extended (near-term) + +These formats are frequently encountered in enterprise workflows and should be supported shortly after launch: + +- **Office documents**: DOCX, XLSX, PPTX. +- **Audio**: WAV, MP3, and other common audio formats. +- **Video**: Standard container formats (MP4, MOV, AVI) with frame-level extraction. +- **Email**: EML and MSG formats, including inline content and attachments (recursively ingested). + +### Tier 3 — Specialized (roadmap) + +These formats address long-tail use cases in specific verticals or operational contexts: + +- **Communications**: Chat log exports from Slack, Teams, and WhatsApp. +- **Database connectors**: Direct ingestion from relational databases and message queues. +- **Archival and compound formats**: ZIP, TAR, and other container formats with recursive extraction of enclosed files. +- **Domain-specific**: DICOM (medical imaging), GeoTIFF (geospatial), and other vertical-specific formats as demand dictates. + +## 3. Extraction Capabilities + +Each modality requires specialized extraction techniques: + +- **Optical character recognition (OCR)**: Layout-aware OCR that preserves spatial relationships between text regions, table cells, headers, and form fields. +- **Speech-to-text**: Transcription with speaker diarization, enabling attribution of spoken content to individual speakers. +- **Video frame extraction**: Decomposition of video streams into individual frames for visual analysis, with temporal alignment to audio tracks. +- **Entity identification in images**: Detection and localization of entities within images — faces, persons, objects, text regions, documents, and other identifiable elements — producing bounding boxes or segmentation masks that downstream detection and redaction stages can operate on. +- **Entity tracking in video**: Persistent tracking of identified entities across video frames. When a face, person, or object is detected in one frame, the platform must maintain identity continuity across subsequent frames to enable consistent redaction without requiring independent detection on every frame. +- **Document structure parsing**: Identification of semantic document elements — headings, paragraphs, tables, lists, and form fields — beyond raw text extraction. +- **Metadata extraction**: Capture of authorship, timestamps, geolocation, and other embedded metadata that may itself constitute sensitive information. + +## 4. Transformation & Output + +Following redaction, the transformation layer must produce output that meets downstream requirements while maintaining fidelity to the original format. + +### 4.1 Format Preservation + +Redacted output should preserve the structural characteristics of the source document. Tables must remain aligned, page layouts must be maintained, and non-redacted content must remain unaltered. + +### 4.2 Output Formats + +The primary output of the transformation layer is a redacted file in the same format as the input — a PDF produces a redacted PDF, an image produces a redacted image, and so on. The platform must not alter the source format unless explicitly requested. + +In addition to the format-preserving primary output, the platform should produce supplementary outputs that serve downstream workflows: + +- **Redaction metadata (JSON)**: A structured manifest describing every redaction applied — entity type, location, triggering rule, confidence score, and reviewer disposition. This metadata enables programmatic consumption of redaction results by audit systems, analytics pipelines, and downstream integrations. +- **Masked structured data (CSV/JSON)**: For tabular or structured inputs, a masked variant in which sensitive cell values are replaced according to the active masking strategy, suitable for analytics or data science consumption. +- **Anonymized datasets**: Fully de-identified exports intended for secondary use (model training, statistical analysis) where no re-identification pathway should exist. + +### 4.3 Masking Strategies + +Multiple masking strategies should be available, selected according to the use case: + +- **Tokenization and pseudonymization**: Replacement of sensitive values with consistent tokens that preserve referential integrity across documents. +- **Reversible masking**: Vault-based masking where original values can be recovered by authorized parties through a secure key exchange. +- **De-identification with re-linking key**: Removal of direct identifiers with a separately stored mapping that enables re-identification under controlled conditions. + +## 5. Validation and Error Handling + +Ingestion must account for real-world content that is malformed, incomplete, or unsupported. + +### 5.1 Input Validation + +Before processing begins, the platform must validate that submitted content meets minimum requirements: supported file format, non-zero size, and absence of corruption indicators. Invalid submissions must be rejected with actionable error messages that identify the specific validation failure. + +### 5.2 Partial Extraction + +When a document is partially parseable — a multi-page PDF with a corrupt page, an audio file with a damaged segment, or an image with an unreadable region — the platform should extract what it can and flag the remainder as incomplete. Partial extraction results must be clearly annotated so that downstream detection operates only on successfully extracted content. + +### 5.3 Error Reporting + +Every ingestion failure must produce a structured error record that includes the content identifier, the failure type (unsupported format, corrupt data, extraction timeout, codec unavailable), and the processing stage at which the failure occurred. These records must be available through the same audit infrastructure described in [COMPLIANCE.md](COMPLIANCE.md). diff --git a/docs/README.md b/docs/README.md index a7924e8..35a4546 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,118 +1,53 @@ -# Nvisy Runtime - -**An open-source ETL platform purpose-built for LLM and AI data pipelines.** - ---- +# Multimodal Redaction & Privacy Platform ## Abstract -The proliferation of large language models and embedding-based retrieval systems has created a new category of data engineering problem. Teams building AI-powered products must continuously move, transform, and validate data across a fragmented ecosystem of vector databases, model APIs, object stores, relational databases, and file formats — each with its own schema conventions, rate limits, and failure modes. - -Existing ETL platforms were designed for tabular, row-oriented data. They lack first-class support for the primitives that define AI workloads: high-dimensional embeddings, completion traces, structured outputs, tool-call logs, audio and image payloads, and fine-tuning datasets. Engineers are left stitching together ad-hoc scripts, battling impedance mismatches between systems that were never designed to interoperate. - -**Nvisy Runtime** addresses this gap. It is an open-source, TypeScript-native ETL platform that treats AI data as a first-class citizen. It provides a DAG-based execution engine, a typed primitive system for AI workloads, a broad connector ecosystem, and a declarative graph definition language — all designed to make AI data engineering reliable, observable, and composable. - ---- - -## Problem Statement - -### 1. AI data is structurally different from traditional data - -An embedding is not a row. A completion trace carries metadata (model, temperature, token counts, latency, cost) that has no analog in a traditional ETL schema. Fine-tuning datasets impose strict structural contracts. Tool-call sequences are trees, not tables. Current ETL platforms force these structures into tabular representations, losing semantic information and making transformations error-prone. - -### 2. The connector ecosystem is fragmented and immature - -Vector databases (Pinecone, Qdrant, Milvus, Weaviate, pgvector) expose incompatible APIs for upsert, query, and metadata filtering. Model provider APIs (OpenAI, Anthropic, Cohere, local inference servers) differ in authentication, rate limiting, batching, and response structure. Object stores, relational databases, and file formats each add their own integration surface. Teams rewrite the same connector logic project after project. - -### 3. Pipeline orchestration for AI workloads has unique requirements - -AI pipelines are not simple source-to-sink flows. They involve conditional branching (route data based on classification), fan-out (embed the same text with multiple models), rate-limited external calls (respect API quotas), idempotent retries (avoid duplicate embeddings), and cost tracking (monitor spend per pipeline run). General-purpose orchestrators like Airflow or Prefect can model these patterns, but they provide no native abstractions for them. - -### 4. Observability is an afterthought - -When an embedding pipeline fails at 3 AM, engineers need to know: which records failed, at which stage, with what error, and whether retrying is safe. They need lineage — the ability to trace a vector in a database back to the source document, through every transformation it passed through. Current tooling provides none of this out of the box. - ---- - -## Design Principles - -### AI-native type system - -Every data object flowing through a Nvisy graph is a typed AI primitive: `Embedding`, `Completion`, `StructuredOutput`, `ToolCallTrace`, `ImagePayload`, `AudioPayload`, `FineTuneSample`, or a user-defined extension. Primitives carry domain-specific metadata and enforce structural contracts at compile time (TypeScript) and runtime (validation). - -### DAG-based execution - -Graphs are directed acyclic graphs of nodes. The runtime resolves dependencies, manages parallelism, handles retries, and tracks execution state. This model supports conditional branching, fan-out/fan-in patterns, and partial re-execution of failed subgraphs — all essential for production AI workloads. - -### Declarative-first, code-escape-hatch - -Common operations (extract, map, filter, chunk, embed, deduplicate, load) are expressed declaratively in JSON graph definitions. For operations that require custom logic, users drop into TypeScript functions that receive and return typed primitives. The declarative layer compiles down to the same execution graph as hand-written code. Because graphs are plain JSON, they are trivially serializable, storable, versionable, and transmittable over the wire. - -### Idiomatic modern JavaScript with structured concurrency - -The platform is built on idiomatic modern JavaScript — `async`/`await`, `AsyncIterable`, and generator functions — with Effection providing structured concurrency for the runtime's DAG executor. This keeps the plugin interface simple (standard async code) while giving the execution engine automatic task cancellation, timeout handling, and resource cleanup. - -### Broad, pluggable connectors - -Connectors are organized into domain-specific packages — SQL, object storage, and vector databases — each installable independently. All connectors implement a standard source/sink interface defined in `nvisy-core`, making community contributions straightforward. Users install only the connector packages they need. - -### Library and server modes - -Nvisy Runtime can be embedded as an npm package for programmatic use or deployed as a long-lived server with a REST API, scheduler, and dashboard. The same graph definition works in both modes. - ---- - -## Core Concepts - -### Primitives - -A **primitive** is the unit of data in a Nvisy graph. Unlike raw JSON blobs, primitives are typed, validated, and carry metadata relevant to their domain. For example, an `Embedding` primitive contains the vector, its dimensionality, the model that produced it, the source text, and a content hash for deduplication. - -### Graphs - -A **graph** is a DAG of **nodes**. Each node is one of: - -- **Source** — reads data from an external system via a connector -- **Action** — applies a transformation or declarative operation to primitives -- **Sink** — writes data to an external system via a connector -- **Branch** — routes primitives to different downstream nodes based on a condition -- **FanOut / FanIn** — duplicates primitives across parallel subgraphs and merges results - -Graphs are defined as JSON structures. This makes them inherently serializable — they can be stored in a database, versioned in source control, transmitted over an API, and reconstructed without loss of fidelity. The programmatic TypeScript API produces the same JSON representation. - -### Connectors - -A **connector** is an adapter that knows how to read from or write to an external system. All connectors implement the source and sink interfaces defined in `nvisy-core`, and declare their capabilities (batch size, rate limits, supported primitive types). Connectors are organized into domain-specific packages: `nvisy-plugin-sql` for relational databases, `nvisy-plugin-object` for object stores and file formats, and `nvisy-plugin-vector` for vector databases. - -### Runtime - -The **runtime** is responsible for compiling and executing graphs. It parses JSON graph definitions, compiles them into execution plans, resolves node dependencies, manages concurrency limits, enforces rate limits on external calls, retries failed nodes with configurable backoff, and emits execution events for observability. +As organizations contend with an ever-growing volume of unstructured and multimodal data, the challenge of identifying and redacting sensitive information has become a critical concern. Regulatory frameworks such as GDPR, HIPAA, CCPA, and PCI-DSS impose strict obligations on how personally identifiable information (PII), protected health information (PHI), and other sensitive content must be handled across documents, images, audio, and video. ---- +This document series presents the architectural and functional requirements for a multimodal redaction platform capable of extracting content from heterogeneous sources, detecting sensitive data through deterministic and learned methods, applying context-aware redaction, and producing auditable evidence of compliance. -## Deployment Modes +The guiding principle is: **extract everything, understand context, redact precisely, prove compliance.** -| Mode | Use Case | Entry Point | -|------|----------|-------------| -| **Library** | Embedded in application code | `import { Engine } from "@nvisy/runtime"` | -| **Server** | Production scheduling, monitoring | `@nvisy/server` | +## Documents -The server mode exposes a REST API for graph management, a scheduler for cron and event-triggered execution, and a web dashboard for monitoring runs, inspecting lineage, and debugging failures. +| Document | Scope | +| --- | --- | +| [Ingestion & Transformation](INGESTION.md) | Multimodal content extraction and post-redaction output | +| [Detection](DETECTION.md) | Sensitive data detection across modalities | +| [Redaction & Review](REDACTION.md) | Context-aware redaction and human-in-the-loop workflows | +| [Compliance & Audit](COMPLIANCE.md) | Policy engine, explainability, and audit trails | +| [Infrastructure](INFRASTRUCTURE.md) | Deployment, performance, and security | +| [Developer Experience](DEVELOPER.md) | APIs, SDKs, tooling, and advanced capabilities | ---- +## Strategic Positioning -## Project Status +Three viable product directions exist for platforms in this space: -Nvisy Runtime is in the specification and design phase. This document serves as the product specification. Implementation will proceed according to the architecture defined in [ARCHITECTURE.md](./ARCHITECTURE.md). +1. **Compliance-first platform** — targets enterprise procurement cycles driven by regulatory mandates. +2. **Developer-first redaction API** — prioritizes integration speed, SDK quality, and self-serve adoption. +3. **AI-native multimodal privacy engine** — leads with model sophistication, context understanding, and semantic redaction. ---- +The strongest long-term defensibility lies in context-aware, explainable, policy-driven multimodal redaction — a convergence of all three directions. -## License +## Target Verticals -Apache License 2.0. See [LICENSE.txt](../LICENSE.txt). +The platform is designed to serve regulated industries where sensitive data handling is a legal and operational requirement: ---- +- **Healthcare**: HIPAA-governed medical records, clinical communications, insurance claims, and patient intake forms. +- **Legal**: Court filings, discovery documents, attorney-client communications, and case management systems. +- **Government and defense**: Law enforcement records, intelligence reports, FOIA responses, and classified material processing. +- **Financial services**: Transaction records, customer onboarding documents, fraud investigation files, and PCI-scoped payment data. +- **Education**: Student records, admissions documents, and FERPA-governed institutional data. -## Contributing +## Glossary -Contribution guidelines will be published once the core architecture stabilizes. The project is designed for community-contributed connectors and actions from the outset. +| Term | Definition | +| --- | --- | +| **PII** | Personally identifiable information — any data that can identify a specific individual | +| **PHI** | Protected health information — health data covered under HIPAA | +| **NER** | Named entity recognition — ML technique for identifying entities (names, locations, organizations) in text | +| **OCR** | Optical character recognition — extraction of text from images and scanned documents | +| **RBAC** | Role-based access control — permissions model based on user roles | +| **SSO** | Single sign-on — authentication mechanism allowing one set of credentials across multiple systems | +| **SCIM** | System for Cross-domain Identity Management — protocol for automating user provisioning | +| **KMS** | Key management service — system for managing cryptographic keys | diff --git a/docs/REDACTION.md b/docs/REDACTION.md new file mode 100644 index 0000000..6753e24 --- /dev/null +++ b/docs/REDACTION.md @@ -0,0 +1,61 @@ +# Redaction & Review + +## 1. Overview + +Detection identifies what is sensitive; redaction determines what to do about it. The distinction between a basic redaction tool and a production-grade platform lies in the ability to apply redaction with contextual awareness — understanding not just that a name appears in a document, but whose name it is, why it matters, and whether the surrounding regulatory and organizational policy requires its removal. + +Equally important is the human-in-the-loop review process. Automated redaction at scale demands human oversight to maintain trust, catch edge cases, and provide a feedback signal for continuous model improvement. + +## 2. Context-Aware Redaction + +### 2.1 Instance-Level Precision + +The platform must distinguish between occurrences of the same entity across different contexts. Redacting "John Smith" in one document should not require redacting every occurrence of the name across an entire corpus. Redaction decisions must be scoped to the relevant instance, document, or case. + +### 2.2 Role-Based and Conditional Redaction + +Redaction rules must support conditional logic: + +- **Role-based rules**: Redact all references to minors while preserving references to adults. +- **Document-type conditions**: Apply medical redaction policies only when the document type is classified as a health record. +- **Temporal conditions**: Redact specific time segments in audio or video content. + +### 2.3 Relationship-Aware Redaction + +Advanced redaction scenarios require reasoning over relationships between entities. For example, redacting all names associated with a specific case identifier, or redacting all communications involving a particular individual across a document set. + +### 2.4 Policy Templates + +Predefined redaction templates aligned to regulatory frameworks (HIPAA, GDPR, CCPA) enable rapid deployment and reduce the burden of manual policy configuration. + +## 3. Human-in-the-Loop Review + +### 3.1 Review Interface + +The platform must provide a review interface that enables human reviewers to inspect, approve, reject, or modify automated redaction decisions. This interface should present the original and redacted content side by side, with clear visual indicators of each redaction and its triggering rule or model. + +### 3.2 Confidence Scoring + +Each automated redaction decision should carry a confidence score derived from the underlying detection model. Reviewers can then prioritize their attention on low-confidence decisions, improving throughput without sacrificing accuracy. + +### 3.3 Bulk Operations + +For large document sets, the review interface must support bulk approval, rejection, and modification of redaction decisions, filtered by confidence threshold, entity type, or document category. + +### 3.4 Access Control + +Reviewer permissions must be configurable through role-based access control. Not all reviewers should have access to all document types or sensitivity levels. + +### 3.5 Active Learning + +Reviewer corrections — accepted, rejected, or modified redactions — should feed back into the detection models as training signal. Over time, this active learning loop reduces the volume of decisions requiring human review and improves model accuracy on organization-specific content. + +## 4. Redaction Versioning and Rollback + +### 4.1 Versioned Redaction State + +The platform must maintain versioned snapshots of redaction state for each piece of content. Each modification — whether automated or manual — produces a new version. Prior versions must remain accessible for comparison, audit, and rollback. + +### 4.2 Rollback + +Before export, any redaction decision must be reversible. Reviewers must be able to roll back individual redactions or restore an entire document to a previous redaction state. After export, rollback is no longer available — the exported artifact is final, and any corrections require re-processing from the original content. diff --git a/docs/TODO.md b/docs/TODO.md new file mode 100644 index 0000000..83f408d --- /dev/null +++ b/docs/TODO.md @@ -0,0 +1,11 @@ +# TODO + +## Engine + +- [ ] Implement `Engine` trait for the DAG runner +- [ ] Wire `EngineInput`/`EngineOutput` through the pipeline + +## Ontology + +- [ ] Add video document types (MP4, WebM, AVI) +- [ ] Add archive document types (ZIP, TAR, GZIP) diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index ccab769..0000000 --- a/package-lock.json +++ /dev/null @@ -1,7524 +0,0 @@ -{ - "name": "@nvisy/monorepo", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@nvisy/monorepo", - "workspaces": [ - "packages/*" - ], - "devDependencies": { - "@biomejs/biome": "^2.3.14", - "@types/node": "^25.2.0", - "@vitest/coverage-v8": "^4.0.18", - "rimraf": "^6.1.2", - "tsup": "^8.5.1", - "typescript": "^5.9.3", - "vitest": "^4.0.18" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "node_modules/@ai-sdk/anthropic": { - "version": "3.0.37", - "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.37.tgz", - "integrity": "sha512-tEgcJPw+a6obbF+SHrEiZsx3DNxOHqeY8bK4IpiNsZ8YPZD141R34g3lEAaQnmNN5mGsEJ8SXoEDabuzi8wFJQ==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "3.0.7", - "@ai-sdk/provider-utils": "4.0.13" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/@ai-sdk/gateway": { - "version": "3.0.36", - "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.36.tgz", - "integrity": "sha512-2r1Q6azvqMYxQ1hqfWZmWg4+8MajoldD/ty65XdhCaCoBfvDu7trcvxXDfTSU+3/wZ1JIDky46SWYFOHnTbsBw==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "3.0.7", - "@ai-sdk/provider-utils": "4.0.13", - "@vercel/oidc": "3.1.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/@ai-sdk/google": { - "version": "3.0.21", - "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-3.0.21.tgz", - "integrity": "sha512-qQuvcbDqDPZojtoT45UFCQVH2w3m6KJKKjqJduUsvhN5ZqOXste0h4HgHK8hwGuDfv96Jr9QQEpspbgp6iu5Uw==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "3.0.7", - "@ai-sdk/provider-utils": "4.0.13" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/@ai-sdk/openai": { - "version": "3.0.25", - "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.25.tgz", - "integrity": "sha512-DsaN46R98+D1W3lU3fKuPU3ofacboLaHlkAwxJPgJ8eup1AJHmPK1N1y10eJJbJcF6iby8Tf/vanoZxc9JPUfw==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "3.0.7", - "@ai-sdk/provider-utils": "4.0.13" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/@ai-sdk/provider": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.7.tgz", - "integrity": "sha512-VkPLrutM6VdA924/mG8OS+5frbVTcu6e046D2bgDo00tehBANR1QBJ/mPcZ9tXMFOsVcm6SQArOregxePzTFPw==", - "license": "Apache-2.0", - "dependencies": { - "json-schema": "^0.4.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@ai-sdk/provider-utils": { - "version": "4.0.13", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.13.tgz", - "integrity": "sha512-HHG72BN4d+OWTcq2NwTxOm/2qvk1duYsnhCDtsbYwn/h/4zeqURu1S0+Cn0nY2Ysq9a9HGKvrYuMn9bgFhR2Og==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "3.0.7", - "@standard-schema/spec": "^1.1.0", - "eventsource-parser": "^3.0.6" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/@asteasolutions/zod-to-openapi": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/@asteasolutions/zod-to-openapi/-/zod-to-openapi-8.4.0.tgz", - "integrity": "sha512-Ckp971tmTw4pnv+o7iK85ldBHBKk6gxMaoNyLn3c2Th/fKoTG8G3jdYuOanpdGqwlDB0z01FOjry2d32lfTqrA==", - "license": "MIT", - "dependencies": { - "openapi3-ts": "^4.1.2" - }, - "peerDependencies": { - "zod": "^4.0.0" - } - }, - "node_modules/@aws-crypto/crc32": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", - "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-crypto/crc32c": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-5.2.0.tgz", - "integrity": "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-crypto/sha1-browser": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-5.2.0.tgz", - "integrity": "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/supports-web-crypto": "^5.2.0", - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/is-array-buffer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", - "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/util-buffer-from": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", - "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/util-utf8": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", - "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-buffer-from": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha256-browser": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", - "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/sha256-js": "^5.2.0", - "@aws-crypto/supports-web-crypto": "^5.2.0", - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/is-array-buffer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", - "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", - "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", - "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-buffer-from": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha256-js": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", - "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-crypto/supports-web-crypto": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", - "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-crypto/util": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", - "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.222.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", - "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", - "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", - "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-buffer-from": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/client-s3": { - "version": "3.984.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.984.0.tgz", - "integrity": "sha512-7ny2Slr93Y+QniuluvcfWwyDi32zWQfznynL56Tk0vVh7bWrvS/odm8WP2nInKicRVNipcJHY2YInur6Q/9V0A==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/sha1-browser": "5.2.0", - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/credential-provider-node": "^3.972.5", - "@aws-sdk/middleware-bucket-endpoint": "^3.972.3", - "@aws-sdk/middleware-expect-continue": "^3.972.3", - "@aws-sdk/middleware-flexible-checksums": "^3.972.4", - "@aws-sdk/middleware-host-header": "^3.972.3", - "@aws-sdk/middleware-location-constraint": "^3.972.3", - "@aws-sdk/middleware-logger": "^3.972.3", - "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-sdk-s3": "^3.972.6", - "@aws-sdk/middleware-ssec": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.6", - "@aws-sdk/region-config-resolver": "^3.972.3", - "@aws-sdk/signature-v4-multi-region": "3.984.0", - "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.984.0", - "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.4", - "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", - "@smithy/eventstream-serde-browser": "^4.2.8", - "@smithy/eventstream-serde-config-resolver": "^4.3.8", - "@smithy/eventstream-serde-node": "^4.2.8", - "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/hash-blob-browser": "^4.2.9", - "@smithy/hash-node": "^4.2.8", - "@smithy/hash-stream-node": "^4.2.8", - "@smithy/invalid-dependency": "^4.2.8", - "@smithy/md5-js": "^4.2.8", - "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", - "@smithy/middleware-serde": "^4.2.9", - "@smithy/middleware-stack": "^4.2.8", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "@smithy/util-base64": "^4.3.0", - "@smithy/util-body-length-browser": "^4.2.0", - "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", - "@smithy/util-endpoints": "^3.2.8", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-retry": "^4.2.8", - "@smithy/util-stream": "^4.5.10", - "@smithy/util-utf8": "^4.2.0", - "@smithy/util-waiter": "^4.2.8", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/client-sso": { - "version": "3.982.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.982.0.tgz", - "integrity": "sha512-qJrIiivmvujdGqJ0ldSUvhN3k3N7GtPesoOI1BSt0fNXovVnMz4C/JmnkhZihU7hJhDvxJaBROLYTU+lpild4w==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/middleware-host-header": "^3.972.3", - "@aws-sdk/middleware-logger": "^3.972.3", - "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.6", - "@aws-sdk/region-config-resolver": "^3.972.3", - "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.982.0", - "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.4", - "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", - "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/hash-node": "^4.2.8", - "@smithy/invalid-dependency": "^4.2.8", - "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", - "@smithy/middleware-serde": "^4.2.9", - "@smithy/middleware-stack": "^4.2.8", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "@smithy/util-base64": "^4.3.0", - "@smithy/util-body-length-browser": "^4.2.0", - "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", - "@smithy/util-endpoints": "^3.2.8", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-retry": "^4.2.8", - "@smithy/util-utf8": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/client-sso/node_modules/@aws-sdk/util-endpoints": { - "version": "3.982.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.982.0.tgz", - "integrity": "sha512-M27u8FJP7O0Of9hMWX5dipp//8iglmV9jr7R8SR8RveU+Z50/8TqH68Tu6wUWBGMfXjzbVwn1INIAO5lZrlxXQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "@smithy/util-endpoints": "^3.2.8", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/core": { - "version": "3.973.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.973.6.tgz", - "integrity": "sha512-pz4ZOw3BLG0NdF25HoB9ymSYyPbMiIjwQJ2aROXRhAzt+b+EOxStfFv8s5iZyP6Kiw7aYhyWxj5G3NhmkoOTKw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@aws-sdk/xml-builder": "^3.972.4", - "@smithy/core": "^3.22.0", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/property-provider": "^4.2.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/signature-v4": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", - "@smithy/types": "^4.12.0", - "@smithy/util-base64": "^4.3.0", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-utf8": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/crc64-nvme": { - "version": "3.972.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/crc64-nvme/-/crc64-nvme-3.972.0.tgz", - "integrity": "sha512-ThlLhTqX68jvoIVv+pryOdb5coP1cX1/MaTbB9xkGDCbWbsqQcLqzPxuSoW1DCnAAIacmXCWpzUNOB9pv+xXQw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.4.tgz", - "integrity": "sha512-/8dnc7+XNMmViEom2xsNdArQxQPSgy4Z/lm6qaFPTrMFesT1bV3PsBhb19n09nmxHdrtQskYmViddUIjUQElXg==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/types": "^3.973.1", - "@smithy/property-provider": "^4.2.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.6.tgz", - "integrity": "sha512-5ERWqRljiZv44AIdvIRQ3k+EAV0Sq2WeJHvXuK7gL7bovSxOf8Al7MLH7Eh3rdovH4KHFnlIty7J71mzvQBl5Q==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/types": "^3.973.1", - "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/node-http-handler": "^4.4.8", - "@smithy/property-provider": "^4.2.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", - "@smithy/types": "^4.12.0", - "@smithy/util-stream": "^4.5.10", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.4.tgz", - "integrity": "sha512-eRUg+3HaUKuXWn/lEMirdiA5HOKmEl8hEHVuszIDt2MMBUKgVX5XNGmb3XmbgU17h6DZ+RtjbxQpjhz3SbTjZg==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/credential-provider-env": "^3.972.4", - "@aws-sdk/credential-provider-http": "^3.972.6", - "@aws-sdk/credential-provider-login": "^3.972.4", - "@aws-sdk/credential-provider-process": "^3.972.4", - "@aws-sdk/credential-provider-sso": "^3.972.4", - "@aws-sdk/credential-provider-web-identity": "^3.972.4", - "@aws-sdk/nested-clients": "3.982.0", - "@aws-sdk/types": "^3.973.1", - "@smithy/credential-provider-imds": "^4.2.8", - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.4.tgz", - "integrity": "sha512-nLGjXuvWWDlQAp505xIONI7Gam0vw2p7Qu3P6on/W2q7rjJXtYjtpHbcsaOjJ/pAju3eTvEQuSuRedcRHVQIAQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/nested-clients": "3.982.0", - "@aws-sdk/types": "^3.973.1", - "@smithy/property-provider": "^4.2.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.5.tgz", - "integrity": "sha512-VWXKgSISQCI2GKN3zakTNHSiZ0+mux7v6YHmmbLQp/o3fvYUQJmKGcLZZzg2GFA+tGGBStplra9VFNf/WwxpYg==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.4", - "@aws-sdk/credential-provider-http": "^3.972.6", - "@aws-sdk/credential-provider-ini": "^3.972.4", - "@aws-sdk/credential-provider-process": "^3.972.4", - "@aws-sdk/credential-provider-sso": "^3.972.4", - "@aws-sdk/credential-provider-web-identity": "^3.972.4", - "@aws-sdk/types": "^3.973.1", - "@smithy/credential-provider-imds": "^4.2.8", - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.4.tgz", - "integrity": "sha512-TCZpWUnBQN1YPk6grvd5x419OfXjHvhj5Oj44GYb84dOVChpg/+2VoEj+YVA4F4E/6huQPNnX7UYbTtxJqgihw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/types": "^3.973.1", - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.4.tgz", - "integrity": "sha512-wzsGwv9mKlwJ3vHLyembBvGE/5nPUIwRR2I51B1cBV4Cb4ql9nIIfpmHzm050XYTY5fqTOKJQnhLj7zj89VG8g==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/client-sso": "3.982.0", - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/token-providers": "3.982.0", - "@aws-sdk/types": "^3.973.1", - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.4.tgz", - "integrity": "sha512-hIzw2XzrG8jzsUSEatehmpkd5rWzASg5IHUfA+m01k/RtvfAML7ZJVVohuKdhAYx+wV2AThLiQJVzqn7F0khrw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/nested-clients": "3.982.0", - "@aws-sdk/types": "^3.973.1", - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.972.3.tgz", - "integrity": "sha512-fmbgWYirF67YF1GfD7cg5N6HHQ96EyRNx/rDIrTF277/zTWVuPI2qS/ZHgofwR1NZPe/NWvoppflQY01LrbVLg==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-arn-parser": "^3.972.2", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "@smithy/util-config-provider": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-expect-continue": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.972.3.tgz", - "integrity": "sha512-4msC33RZsXQpUKR5QR4HnvBSNCPLGHmB55oDiROqqgyOc+TOfVu2xgi5goA7ms6MdZLeEh2905UfWMnMMF4mRg==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.972.4.tgz", - "integrity": "sha512-xOxsUkF3O3BtIe3tf54OpPo94eZepjFm3z0Dd2TZKbsPxMiRTFXurC04wJ58o/wPW9YHVO9VqZik3MfoPfrKlw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/crc32": "5.2.0", - "@aws-crypto/crc32c": "5.2.0", - "@aws-crypto/util": "5.2.0", - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/crc64-nvme": "3.972.0", - "@aws-sdk/types": "^3.973.1", - "@smithy/is-array-buffer": "^4.2.0", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-stream": "^4.5.10", - "@smithy/util-utf8": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-host-header": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.3.tgz", - "integrity": "sha512-aknPTb2M+G3s+0qLCx4Li/qGZH8IIYjugHMv15JTYMe6mgZO8VBpYgeGYsNMGCqCZOcWzuf900jFBG5bopfzmA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-location-constraint": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.972.3.tgz", - "integrity": "sha512-nIg64CVrsXp67vbK0U1/Is8rik3huS3QkRHn2DRDx4NldrEFMgdkZGI/+cZMKD9k4YOS110Dfu21KZLHrFA/1g==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-logger": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.972.3.tgz", - "integrity": "sha512-Ftg09xNNRqaz9QNzlfdQWfpqMCJbsQdnZVJP55jfhbKi1+FTWxGuvfPoBhDHIovqWKjqbuiew3HuhxbJ0+OjgA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.972.3.tgz", - "integrity": "sha512-PY57QhzNuXHnwbJgbWYTrqIDHYSeOlhfYERTAuc16LKZpTZRJUjzBFokp9hF7u1fuGeE3D70ERXzdbMBOqQz7Q==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@aws/lambda-invoke-store": "^0.2.2", - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.6.tgz", - "integrity": "sha512-Xq7wM6kbgJN1UO++8dvH/efPb1nTwWqFCpZCR7RCLOETP7xAUAhVo7JmsCnML5Di/iC4Oo5VrJ4QmkYcMZniLw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-arn-parser": "^3.972.2", - "@smithy/core": "^3.22.0", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/signature-v4": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", - "@smithy/types": "^4.12.0", - "@smithy/util-config-provider": "^4.2.0", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-stream": "^4.5.10", - "@smithy/util-utf8": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-ssec": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.972.3.tgz", - "integrity": "sha512-dU6kDuULN3o3jEHcjm0c4zWJlY1zWVkjG9NPe9qxYLLpcbdj5kRYBS2DdWYD+1B9f910DezRuws7xDEqKkHQIg==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.972.6", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.6.tgz", - "integrity": "sha512-TehLN8W/kivl0U9HcS+keryElEWORROpghDXZBLfnb40DXM7hx/i+7OOjkogXQOF3QtUraJVRkHQ07bPhrWKlw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.982.0", - "@smithy/core": "^3.22.0", - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-user-agent/node_modules/@aws-sdk/util-endpoints": { - "version": "3.982.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.982.0.tgz", - "integrity": "sha512-M27u8FJP7O0Of9hMWX5dipp//8iglmV9jr7R8SR8RveU+Z50/8TqH68Tu6wUWBGMfXjzbVwn1INIAO5lZrlxXQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "@smithy/util-endpoints": "^3.2.8", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/nested-clients": { - "version": "3.982.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.982.0.tgz", - "integrity": "sha512-VVkaH27digrJfdVrT64rjkllvOp4oRiZuuJvrylLXAKl18ujToJR7AqpDldL/LS63RVne3QWIpkygIymxFtliQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/middleware-host-header": "^3.972.3", - "@aws-sdk/middleware-logger": "^3.972.3", - "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.6", - "@aws-sdk/region-config-resolver": "^3.972.3", - "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.982.0", - "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.4", - "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", - "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/hash-node": "^4.2.8", - "@smithy/invalid-dependency": "^4.2.8", - "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", - "@smithy/middleware-serde": "^4.2.9", - "@smithy/middleware-stack": "^4.2.8", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "@smithy/util-base64": "^4.3.0", - "@smithy/util-body-length-browser": "^4.2.0", - "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", - "@smithy/util-endpoints": "^3.2.8", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-retry": "^4.2.8", - "@smithy/util-utf8": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/nested-clients/node_modules/@aws-sdk/util-endpoints": { - "version": "3.982.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.982.0.tgz", - "integrity": "sha512-M27u8FJP7O0Of9hMWX5dipp//8iglmV9jr7R8SR8RveU+Z50/8TqH68Tu6wUWBGMfXjzbVwn1INIAO5lZrlxXQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "@smithy/util-endpoints": "^3.2.8", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/region-config-resolver": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.3.tgz", - "integrity": "sha512-v4J8qYAWfOMcZ4MJUyatntOicTzEMaU7j3OpkRCGGFSL2NgXQ5VbxauIyORA+pxdKZ0qQG2tCQjQjZDlXEC3Ow==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@smithy/config-resolver": "^4.4.6", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.984.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.984.0.tgz", - "integrity": "sha512-TaWbfYCwnuOSvDSrgs7QgoaoXse49E7LzUkVOUhoezwB7bkmhp+iojADm7UepCEu4021SquD7NG1xA+WCvmldA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/middleware-sdk-s3": "^3.972.6", - "@aws-sdk/types": "^3.973.1", - "@smithy/protocol-http": "^5.3.8", - "@smithy/signature-v4": "^5.3.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/token-providers": { - "version": "3.982.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.982.0.tgz", - "integrity": "sha512-v3M0KYp2TVHYHNBT7jHD9lLTWAdS9CaWJ2jboRKt0WAB65bA7iUEpR+k4VqKYtpQN4+8kKSc4w+K6kUNZkHKQw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.973.6", - "@aws-sdk/nested-clients": "3.982.0", - "@aws-sdk/types": "^3.973.1", - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/types": { - "version": "3.973.1", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.1.tgz", - "integrity": "sha512-DwHBiMNOB468JiX6+i34c+THsKHErYUdNQ3HexeXZvVn4zouLjgaS4FejiGSi2HyBuzuyHg7SuOPmjSvoU9NRg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/util-arn-parser": { - "version": "3.972.2", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.972.2.tgz", - "integrity": "sha512-VkykWbqMjlSgBFDyrY3nOSqupMc6ivXuGmvci6Q3NnLq5kC+mKQe2QBZ4nrWRE/jqOxeFP2uYzLtwncYYcvQDg==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/util-endpoints": { - "version": "3.984.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.984.0.tgz", - "integrity": "sha512-9ebjLA0hMKHeVvXEtTDCCOBtwjb0bOXiuUV06HNeVdgAjH6gj4x4Zwt4IBti83TiyTGOCl5YfZqGx4ehVsasbQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "@smithy/util-endpoints": "^3.2.8", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/util-locate-window": { - "version": "3.965.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.965.4.tgz", - "integrity": "sha512-H1onv5SkgPBK2P6JR2MjGgbOnttoNzSPIRoeZTNPZYyaplwGg50zS3amXvXqF0/qfXpWEC9rLWU564QTB9bSog==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.972.3.tgz", - "integrity": "sha512-JurOwkRUcXD/5MTDBcqdyQ9eVedtAsZgw5rBwktsPTN7QtPiS2Ld1jkJepNgYoCufz1Wcut9iup7GJDoIHp8Fw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@smithy/types": "^4.12.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.972.4.tgz", - "integrity": "sha512-3WFCBLiM8QiHDfosQq3Py+lIMgWlFWwFQliUHUqwEiRqLnKyhgbU3AKa7AWJF7lW2Oc/2kFNY4MlAYVnVc0i8A==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/middleware-user-agent": "^3.972.6", - "@aws-sdk/types": "^3.973.1", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "aws-crt": ">=1.0.0" - }, - "peerDependenciesMeta": { - "aws-crt": { - "optional": true - } - } - }, - "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.4.tgz", - "integrity": "sha512-0zJ05ANfYqI6+rGqj8samZBFod0dPPousBjLEqg8WdxSgbMAkRgLyn81lP215Do0rFJ/17LIXwr7q0yK24mP6Q==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "fast-xml-parser": "5.3.4", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws/lambda-invoke-store": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.3.tgz", - "integrity": "sha512-oLvsaPMTBejkkmHhjf09xTgk71mOqyr/409NKhRIL08If7AhVfUsJhVsx386uJaqNd42v9kWamQ9lFbkoC2dYw==", - "license": "Apache-2.0", - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure-rest/core-client": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/@azure-rest/core-client/-/core-client-2.5.1.tgz", - "integrity": "sha512-EHaOXW0RYDKS5CFffnixdyRPak5ytiCtU7uXDcP/uiY+A6jFRwNGzzJBiznkCzvi5EYpY+YWinieqHb0oY916A==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.10.0", - "@azure/core-rest-pipeline": "^1.22.0", - "@azure/core-tracing": "^1.3.0", - "@typespec/ts-http-runtime": "^0.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "license": "MIT", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-auth": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz", - "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-util": "^1.13.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/core-client": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.10.1.tgz", - "integrity": "sha512-Nh5PhEOeY6PrnxNPsEHRr9eimxLwgLlpmguQaHKBinFYA/RU9+kOYVOQqOrTsCL+KSxrLLl1gD8Dk5BFW/7l/w==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.10.0", - "@azure/core-rest-pipeline": "^1.22.0", - "@azure/core-tracing": "^1.3.0", - "@azure/core-util": "^1.13.0", - "@azure/logger": "^1.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/core-http-compat": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.3.2.tgz", - "integrity": "sha512-Tf6ltdKzOJEgxZeWLCjMxrxbodB/ZeCbzzA1A2qHbhzAjzjHoBVSUeSl/baT/oHAxhc4qdqVaDKnc2+iE932gw==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@azure/core-client": "^1.10.0", - "@azure/core-rest-pipeline": "^1.22.0" - } - }, - "node_modules/@azure/core-lro": { - "version": "2.7.2", - "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", - "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-util": "^1.2.0", - "@azure/logger": "^1.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-paging": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.6.2.tgz", - "integrity": "sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==", - "license": "MIT", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-rest-pipeline": { - "version": "1.22.2", - "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.22.2.tgz", - "integrity": "sha512-MzHym+wOi8CLUlKCQu12de0nwcq9k9Kuv43j4Wa++CsCpJwps2eeBQwD2Bu8snkxTtDKDx4GwjuR9E8yC8LNrg==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.10.0", - "@azure/core-tracing": "^1.3.0", - "@azure/core-util": "^1.13.0", - "@azure/logger": "^1.3.0", - "@typespec/ts-http-runtime": "^0.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/core-tracing": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz", - "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==", - "license": "MIT", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/core-util": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.13.1.tgz", - "integrity": "sha512-XPArKLzsvl0Hf0CaGyKHUyVgF7oDnhKoP85Xv6M4StF/1AhfORhZudHtOyf2s+FcbuQ9dPRAjB8J2KvRRMUK2A==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@typespec/ts-http-runtime": "^0.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/core-xml": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.5.0.tgz", - "integrity": "sha512-D/sdlJBMJfx7gqoj66PKVmhDDaU6TKA49ptcolxdas29X7AfvLTmfAGLjAcIMBK7UZ2o4lygHIqVckOlQU3xWw==", - "license": "MIT", - "dependencies": { - "fast-xml-parser": "^5.0.7", - "tslib": "^2.8.1" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/identity": { - "version": "4.13.0", - "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.13.0.tgz", - "integrity": "sha512-uWC0fssc+hs1TGGVkkghiaFkkS7NkTxfnCH+Hdg+yTehTpMcehpok4PgUKKdyCH+9ldu6FhiHRv84Ntqj1vVcw==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.9.0", - "@azure/core-client": "^1.9.2", - "@azure/core-rest-pipeline": "^1.17.0", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.11.0", - "@azure/logger": "^1.0.0", - "@azure/msal-browser": "^4.2.0", - "@azure/msal-node": "^3.5.0", - "open": "^10.1.0", - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/keyvault-common": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@azure/keyvault-common/-/keyvault-common-2.0.0.tgz", - "integrity": "sha512-wRLVaroQtOqfg60cxkzUkGKrKMsCP6uYXAOomOIysSMyt1/YM0eUn9LqieAWM8DLcU4+07Fio2YGpPeqUbpP9w==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.3.0", - "@azure/core-client": "^1.5.0", - "@azure/core-rest-pipeline": "^1.8.0", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.10.0", - "@azure/logger": "^1.1.4", - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/keyvault-keys": { - "version": "4.10.0", - "resolved": "https://registry.npmjs.org/@azure/keyvault-keys/-/keyvault-keys-4.10.0.tgz", - "integrity": "sha512-eDT7iXoBTRZ2n3fLiftuGJFD+yjkiB1GNqzU2KbY1TLYeXeSPVTVgn2eJ5vmRTZ11978jy2Kg2wI7xa9Tyr8ag==", - "license": "MIT", - "dependencies": { - "@azure-rest/core-client": "^2.3.3", - "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.9.0", - "@azure/core-http-compat": "^2.2.0", - "@azure/core-lro": "^2.7.2", - "@azure/core-paging": "^1.6.2", - "@azure/core-rest-pipeline": "^1.19.0", - "@azure/core-tracing": "^1.2.0", - "@azure/core-util": "^1.11.0", - "@azure/keyvault-common": "^2.0.0", - "@azure/logger": "^1.1.4", - "tslib": "^2.8.1" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/logger": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.3.0.tgz", - "integrity": "sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==", - "license": "MIT", - "dependencies": { - "@typespec/ts-http-runtime": "^0.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/msal-browser": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-4.28.1.tgz", - "integrity": "sha512-al2u2fTchbClq3L4C1NlqLm+vwKfhYCPtZN2LR/9xJVaQ4Mnrwf5vANvuyPSJHcGvw50UBmhuVmYUAhTEetTpA==", - "license": "MIT", - "dependencies": { - "@azure/msal-common": "15.14.1" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@azure/msal-common": { - "version": "15.14.1", - "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-15.14.1.tgz", - "integrity": "sha512-IkzF7Pywt6QKTS0kwdCv/XV8x8JXknZDvSjj/IccooxnP373T5jaadO3FnOrbWo3S0UqkfIDyZNTaQ/oAgRdXw==", - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@azure/msal-node": { - "version": "3.8.6", - "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-3.8.6.tgz", - "integrity": "sha512-XTmhdItcBckcVVTy65Xp+42xG4LX5GK+9AqAsXPXk4IqUNv+LyQo5TMwNjuFYBfAB2GTG9iSQGk+QLc03vhf3w==", - "license": "MIT", - "dependencies": { - "@azure/msal-common": "15.14.1", - "jsonwebtoken": "^9.0.0", - "uuid": "^8.3.0" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/@azure/storage-blob": { - "version": "12.30.0", - "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.30.0.tgz", - "integrity": "sha512-peDCR8blSqhsAKDbpSP/o55S4sheNwSrblvCaHUZ5xUI73XA7ieUGGwrONgD/Fng0EoDe1VOa3fAQ7+WGB3Ocg==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.9.0", - "@azure/core-client": "^1.9.3", - "@azure/core-http-compat": "^2.2.0", - "@azure/core-lro": "^2.2.0", - "@azure/core-paging": "^1.6.2", - "@azure/core-rest-pipeline": "^1.19.1", - "@azure/core-tracing": "^1.2.0", - "@azure/core-util": "^1.11.0", - "@azure/core-xml": "^1.4.5", - "@azure/logger": "^1.1.4", - "@azure/storage-common": "^12.2.0", - "events": "^3.0.0", - "tslib": "^2.8.1" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@azure/storage-common": { - "version": "12.3.0", - "resolved": "https://registry.npmjs.org/@azure/storage-common/-/storage-common-12.3.0.tgz", - "integrity": "sha512-/OFHhy86aG5Pe8dP5tsp+BuJ25JOAl9yaMU3WZbkeoiFMHFtJ7tu5ili7qEdBXNW9G5lDB19trwyI6V49F/8iQ==", - "license": "MIT", - "dependencies": { - "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.9.0", - "@azure/core-http-compat": "^2.2.0", - "@azure/core-rest-pipeline": "^1.19.1", - "@azure/core-tracing": "^1.2.0", - "@azure/core-util": "^1.11.0", - "@azure/logger": "^1.1.4", - "events": "^3.3.0", - "tslib": "^2.8.1" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/parser": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.0.tgz", - "integrity": "sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.29.0" - }, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/types": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", - "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.28.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@bcoe/v8-coverage": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", - "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@biomejs/biome": { - "version": "2.3.14", - "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.3.14.tgz", - "integrity": "sha512-QMT6QviX0WqXJCaiqVMiBUCr5WRQ1iFSjvOLoTk6auKukJMvnMzWucXpwZB0e8F00/1/BsS9DzcKgWH+CLqVuA==", - "dev": true, - "license": "MIT OR Apache-2.0", - "bin": { - "biome": "bin/biome" - }, - "engines": { - "node": ">=14.21.3" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/biome" - }, - "optionalDependencies": { - "@biomejs/cli-darwin-arm64": "2.3.14", - "@biomejs/cli-darwin-x64": "2.3.14", - "@biomejs/cli-linux-arm64": "2.3.14", - "@biomejs/cli-linux-arm64-musl": "2.3.14", - "@biomejs/cli-linux-x64": "2.3.14", - "@biomejs/cli-linux-x64-musl": "2.3.14", - "@biomejs/cli-win32-arm64": "2.3.14", - "@biomejs/cli-win32-x64": "2.3.14" - } - }, - "node_modules/@biomejs/cli-darwin-arm64": { - "version": "2.3.14", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.3.14.tgz", - "integrity": "sha512-UJGPpvWJMkLxSRtpCAKfKh41Q4JJXisvxZL8ChN1eNW3m/WlPFJ6EFDCE7YfUb4XS8ZFi3C1dFpxUJ0Ety5n+A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT OR Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-darwin-x64": { - "version": "2.3.14", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.3.14.tgz", - "integrity": "sha512-PNkLNQG6RLo8lG7QoWe/hhnMxJIt1tEimoXpGQjwS/dkdNiKBLPv4RpeQl8o3s1OKI3ZOR5XPiYtmbGGHAOnLA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT OR Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-linux-arm64": { - "version": "2.3.14", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.3.14.tgz", - "integrity": "sha512-KT67FKfzIw6DNnUNdYlBg+eU24Go3n75GWK6NwU4+yJmDYFe9i/MjiI+U/iEzKvo0g7G7MZqoyrhIYuND2w8QQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT OR Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-linux-arm64-musl": { - "version": "2.3.14", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.3.14.tgz", - "integrity": "sha512-LInRbXhYujtL3sH2TMCH/UBwJZsoGwfQjBrMfl84CD4hL/41C/EU5mldqf1yoFpsI0iPWuU83U+nB2TUUypWeg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT OR Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-linux-x64": { - "version": "2.3.14", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.3.14.tgz", - "integrity": "sha512-ZsZzQsl9U+wxFrGGS4f6UxREUlgHwmEfu1IrXlgNFrNnd5Th6lIJr8KmSzu/+meSa9f4rzFrbEW9LBBA6ScoMA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT OR Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-linux-x64-musl": { - "version": "2.3.14", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.3.14.tgz", - "integrity": "sha512-KQU7EkbBBuHPW3/rAcoiVmhlPtDSGOGRPv9js7qJVpYTzjQmVR+C9Rfcz+ti8YCH+zT1J52tuBybtP4IodjxZQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT OR Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-win32-arm64": { - "version": "2.3.14", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.3.14.tgz", - "integrity": "sha512-+IKYkj/pUBbnRf1G1+RlyA3LWiDgra1xpS7H2g4BuOzzRbRB+hmlw0yFsLprHhbbt7jUzbzAbAjK/Pn0FDnh1A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT OR Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@biomejs/cli-win32-x64": { - "version": "2.3.14", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.3.14.tgz", - "integrity": "sha512-oizCjdyQ3WJEswpb3Chdngeat56rIdSYK12JI3iI11Mt5T5EXcZ7WLuowzEaFPNJ3zmOQFliMN8QY1Pi+qsfdQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT OR Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=14.21.3" - } - }, - "node_modules/@colors/colors": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", - "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", - "license": "MIT", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/@dabh/diagnostics": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.8.tgz", - "integrity": "sha512-R4MSXTVnuMzGD7bzHdW2ZhhdPC/igELENcq5IjEverBvq5hn1SXCWcsi6eSsdWP0/Ur+SItRRjAktmdoX/8R/Q==", - "license": "MIT", - "dependencies": { - "@so-ric/colorspace": "^1.1.6", - "enabled": "2.0.x", - "kuler": "^2.0.0" - } - }, - "node_modules/@datastructures-js/deque": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@datastructures-js/deque/-/deque-1.0.8.tgz", - "integrity": "sha512-PSBhJ2/SmeRPRHuBv7i/fHWIdSC3JTyq56qb+Rq0wjOagi0/fdV5/B/3Md5zFZus/W6OkSPMaxMKKMNMrSmubg==", - "license": "MIT" - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", - "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", - "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", - "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", - "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", - "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", - "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", - "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", - "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", - "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", - "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", - "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", - "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", - "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", - "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", - "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", - "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz", - "integrity": "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", - "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", - "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", - "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", - "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", - "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", - "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", - "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", - "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", - "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@google-cloud/paginator": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", - "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", - "license": "Apache-2.0", - "dependencies": { - "arrify": "^2.0.0", - "extend": "^3.0.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@google-cloud/projectify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", - "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", - "license": "Apache-2.0", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@google-cloud/promisify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", - "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/@google-cloud/storage": { - "version": "7.19.0", - "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.19.0.tgz", - "integrity": "sha512-n2FjE7NAOYyshogdc7KQOl/VZb4sneqPjWouSyia9CMDdMhRX5+RIbqalNmC7LOLzuLAN89VlF2HvG8na9G+zQ==", - "license": "Apache-2.0", - "dependencies": { - "@google-cloud/paginator": "^5.0.0", - "@google-cloud/projectify": "^4.0.0", - "@google-cloud/promisify": "<4.1.0", - "abort-controller": "^3.0.0", - "async-retry": "^1.3.3", - "duplexify": "^4.1.3", - "fast-xml-parser": "^5.3.4", - "gaxios": "^6.0.2", - "google-auth-library": "^9.6.3", - "html-entities": "^2.5.2", - "mime": "^3.0.0", - "p-limit": "^3.0.1", - "retry-request": "^7.0.0", - "teeny-request": "^9.0.0", - "uuid": "^8.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@graphql-typed-document-node/core": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@graphql-typed-document-node/core/-/core-3.2.0.tgz", - "integrity": "sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==", - "license": "MIT", - "peerDependencies": { - "graphql": "^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" - } - }, - "node_modules/@grpc/grpc-js": { - "version": "1.14.3", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", - "integrity": "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==", - "license": "Apache-2.0", - "dependencies": { - "@grpc/proto-loader": "^0.8.0", - "@js-sdsl/ordered-map": "^4.4.2" - }, - "engines": { - "node": ">=12.10.0" - } - }, - "node_modules/@grpc/grpc-js/node_modules/@grpc/proto-loader": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.0.tgz", - "integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==", - "license": "Apache-2.0", - "dependencies": { - "lodash.camelcase": "^4.3.0", - "long": "^5.0.0", - "protobufjs": "^7.5.3", - "yargs": "^17.7.2" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/@grpc/proto-loader": { - "version": "0.7.15", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", - "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", - "license": "Apache-2.0", - "dependencies": { - "lodash.camelcase": "^4.3.0", - "long": "^5.0.0", - "protobufjs": "^7.2.5", - "yargs": "^17.7.2" - }, - "bin": { - "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/@hono/event-emitter": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@hono/event-emitter/-/event-emitter-2.0.0.tgz", - "integrity": "sha512-7/zg7hfPh9lncYCxU3avk40vGhiqP4D5NvmaNX+8QxXivIkrLckSia5P4Nz6PH+A1T8Aj3yFlONf4AiX5rqaEA==", - "license": "MIT", - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "hono": "*" - } - }, - "node_modules/@hono/node-server": { - "version": "1.19.9", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.9.tgz", - "integrity": "sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw==", - "license": "MIT", - "engines": { - "node": ">=18.14.1" - }, - "peerDependencies": { - "hono": "^4" - } - }, - "node_modules/@hono/node-ws": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@hono/node-ws/-/node-ws-1.3.0.tgz", - "integrity": "sha512-ju25YbbvLuXdqBCmLZLqnNYu1nbHIQjoyUqA8ApZOeL1k4skuiTcw5SW77/5SUYo2Xi2NVBJoVlfQurnKEp03Q==", - "license": "MIT", - "dependencies": { - "ws": "^8.17.0" - }, - "engines": { - "node": ">=18.14.1" - }, - "peerDependencies": { - "@hono/node-server": "^1.19.2", - "hono": "^4.6.0" - } - }, - "node_modules/@hono/otel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@hono/otel/-/otel-1.1.0.tgz", - "integrity": "sha512-3lXExGP+odVTF3W1kTHgRGw4d4xdiYpeRs8dnTwfnHfw5uGEXgUzmkB4/ZQd3tDxYRt7eUhnWuBk5ChV97eqkA==", - "license": "MIT", - "dependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/semantic-conventions": "^1.28.0" - }, - "peerDependencies": { - "hono": ">=4.0.0" - } - }, - "node_modules/@hono/zod-openapi": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@hono/zod-openapi/-/zod-openapi-1.2.1.tgz", - "integrity": "sha512-aZza4V8wkqpdHBWFNPiCeWd0cGOXbYuQW9AyezHs/jwQm5p67GkUyXwfthAooAwnG7thTpvOJkThZpCoY6us8w==", - "license": "MIT", - "dependencies": { - "@asteasolutions/zod-to-openapi": "^8.1.0", - "@hono/zod-validator": "^0.7.6", - "openapi3-ts": "^4.5.0" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "hono": ">=4.3.6", - "zod": "^4.0.0" - } - }, - "node_modules/@hono/zod-validator": { - "version": "0.7.6", - "resolved": "https://registry.npmjs.org/@hono/zod-validator/-/zod-validator-0.7.6.tgz", - "integrity": "sha512-Io1B6d011Gj1KknV4rXYz4le5+5EubcWEU/speUjuw9XMMIaP3n78yXLhjd2A3PXaXaUwEAluOiAyLqhBEJgsw==", - "license": "MIT", - "peerDependencies": { - "hono": ">=3.9.0", - "zod": "^3.25.0 || ^4.0.0" - } - }, - "node_modules/@isaacs/balanced-match": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", - "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/brace-expansion": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.1.tgz", - "integrity": "sha512-WMz71T1JS624nWj2n2fnYAuPovhv7EUhk69R6i9dsVyzxt5eM3bjwvgk9L+APE1TRscGysAVMANkB0jh0LQZrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@isaacs/balanced-match": "^4.0.1" - }, - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.31", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", - "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, - "node_modules/@js-joda/core": { - "version": "5.7.0", - "resolved": "https://registry.npmjs.org/@js-joda/core/-/core-5.7.0.tgz", - "integrity": "sha512-WBu4ULVVxySLLzK1Ppq+OdfP+adRS4ntmDQT915rzDJ++i95gc2jZkM5B6LWEAwN3lGXpfie3yPABozdD3K3Vg==", - "license": "BSD-3-Clause" - }, - "node_modules/@js-sdsl/ordered-map": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", - "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/js-sdsl" - } - }, - "node_modules/@logtape/hono": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@logtape/hono/-/hono-2.0.2.tgz", - "integrity": "sha512-dCIfdFnpEguVvd0cLeo7BOMXXBZ/e0dTcOiB9rn46tMsILZlmiIVWz+O0q1wOqSkooOau6zCkw+Rt58UT5nvPQ==", - "funding": [ - "https://github.com/sponsors/dahlia" - ], - "license": "MIT", - "peerDependencies": { - "@logtape/logtape": "^2.0.2", - "hono": "^4.0.0" - } - }, - "node_modules/@logtape/logtape": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@logtape/logtape/-/logtape-2.0.2.tgz", - "integrity": "sha512-cveUBLbCMFkvkLycP/2vNWvywl47JcJbazHIju94/QNGboZ/jyYAgZIm0ZXezAOx3eIz8OG1EOZ5CuQP3+2FQg==", - "funding": [ - "https://github.com/sponsors/dahlia" - ], - "license": "MIT" - }, - "node_modules/@logtape/pretty": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@logtape/pretty/-/pretty-2.0.2.tgz", - "integrity": "sha512-WMKoHuaEZvJgDeciIM/OL+joDDlheuoSpkfJOuKncdim7eV6GfIh0BUxLt0Td4JJsljzt5dAttxaX0kXqE0N9Q==", - "funding": [ - "https://github.com/sponsors/dahlia" - ], - "license": "MIT", - "peerDependencies": { - "@logtape/logtape": "^2.0.2" - } - }, - "node_modules/@logtape/redaction": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@logtape/redaction/-/redaction-2.0.2.tgz", - "integrity": "sha512-NSURYmPLk2E3H1VgxSqG1P65qScVWXntChS3pvXthku6v2E2bHWp+BvDYksoA15SNT7KtTXYgfsV2O3LCMJQrw==", - "funding": [ - "https://github.com/sponsors/dahlia" - ], - "license": "MIT", - "peerDependencies": { - "@logtape/logtape": "^2.0.2" - } - }, - "node_modules/@nvisy/core": { - "resolved": "packages/nvisy-core", - "link": true - }, - "node_modules/@nvisy/plugin-ai": { - "resolved": "packages/nvisy-plugin-ai", - "link": true - }, - "node_modules/@nvisy/plugin-object": { - "resolved": "packages/nvisy-plugin-object", - "link": true - }, - "node_modules/@nvisy/plugin-pandoc": { - "resolved": "packages/nvisy-plugin-pandoc", - "link": true - }, - "node_modules/@nvisy/plugin-sql": { - "resolved": "packages/nvisy-plugin-sql", - "link": true - }, - "node_modules/@nvisy/plugin-vector": { - "resolved": "packages/nvisy-plugin-vector", - "link": true - }, - "node_modules/@nvisy/runtime": { - "resolved": "packages/nvisy-runtime", - "link": true - }, - "node_modules/@nvisy/server": { - "resolved": "packages/nvisy-server", - "link": true - }, - "node_modules/@opentelemetry/api": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", - "license": "Apache-2.0", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.39.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.39.0.tgz", - "integrity": "sha512-R5R9tb2AXs2IRLNKLBJDynhkfmx7mX0vi8NkhZb3gUkPWHn6HXk5J8iQ/dql0U3ApfWym4kXXmBDRGO+oeOfjg==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/@petamoriken/float16": { - "version": "3.9.3", - "resolved": "https://registry.npmjs.org/@petamoriken/float16/-/float16-3.9.3.tgz", - "integrity": "sha512-8awtpHXCx/bNpFt4mt2xdkgtgVvKqty8VbjHI/WWWQuEw+KLzFot3f4+LkQY9YmOtq7A5GdOnqoIC8Pdygjk2g==", - "license": "MIT" - }, - "node_modules/@pinecone-database/pinecone": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-4.1.0.tgz", - "integrity": "sha512-WoVsbvmCgvZfjm/nCasJXuQ/tw0es5BpedLHvRScAm6xJ/nL07s3B0TrsM8m8rACTiUgbdYsdLY1W6cEBhS9xA==", - "license": "Apache-2.0", - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", - "license": "BSD-3-Clause" - }, - "node_modules/@qdrant/js-client-rest": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/@qdrant/js-client-rest/-/js-client-rest-1.16.2.tgz", - "integrity": "sha512-Zm4wEZURrZ24a+Hmm4l1QQYjiz975Ep3vF0yzWR7ICGcxittNz47YK2iBOk8kb8qseCu8pg7WmO1HOIsO8alvw==", - "license": "Apache-2.0", - "dependencies": { - "@qdrant/openapi-typescript-fetch": "1.2.6", - "undici": "^6.0.0" - }, - "engines": { - "node": ">=18.17.0", - "pnpm": ">=8" - }, - "peerDependencies": { - "typescript": ">=4.7" - } - }, - "node_modules/@qdrant/openapi-typescript-fetch": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@qdrant/openapi-typescript-fetch/-/openapi-typescript-fetch-1.2.6.tgz", - "integrity": "sha512-oQG/FejNpItrxRHoyctYvT3rwGZOnK4jr3JdppO/c78ktDvkWiPXPHNsrDf33K9sZdRb6PR7gi4noIapu5q4HA==", - "license": "MIT", - "engines": { - "node": ">=18.0.0", - "pnpm": ">=8" - } - }, - "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.57.1.tgz", - "integrity": "sha512-A6ehUVSiSaaliTxai040ZpZ2zTevHYbvu/lDoeAteHI8QnaosIzm4qwtezfRg1jOYaUmnzLX1AOD6Z+UJjtifg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@rollup/rollup-android-arm64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.57.1.tgz", - "integrity": "sha512-dQaAddCY9YgkFHZcFNS/606Exo8vcLHwArFZ7vxXq4rigo2bb494/xKMMwRRQW6ug7Js6yXmBZhSBRuBvCCQ3w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.57.1.tgz", - "integrity": "sha512-crNPrwJOrRxagUYeMn/DZwqN88SDmwaJ8Cvi/TN1HnWBU7GwknckyosC2gd0IqYRsHDEnXf328o9/HC6OkPgOg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.57.1.tgz", - "integrity": "sha512-Ji8g8ChVbKrhFtig5QBV7iMaJrGtpHelkB3lsaKzadFBe58gmjfGXAOfI5FV0lYMH8wiqsxKQ1C9B0YTRXVy4w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.57.1.tgz", - "integrity": "sha512-R+/WwhsjmwodAcz65guCGFRkMb4gKWTcIeLy60JJQbXrJ97BOXHxnkPFrP+YwFlaS0m+uWJTstrUA9o+UchFug==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] - }, - "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.57.1.tgz", - "integrity": "sha512-IEQTCHeiTOnAUC3IDQdzRAGj3jOAYNr9kBguI7MQAAZK3caezRrg0GxAb6Hchg4lxdZEI5Oq3iov/w/hnFWY9Q==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] - }, - "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.57.1.tgz", - "integrity": "sha512-F8sWbhZ7tyuEfsmOxwc2giKDQzN3+kuBLPwwZGyVkLlKGdV1nvnNwYD0fKQ8+XS6hp9nY7B+ZeK01EBUE7aHaw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.57.1.tgz", - "integrity": "sha512-rGfNUfn0GIeXtBP1wL5MnzSj98+PZe/AXaGBCRmT0ts80lU5CATYGxXukeTX39XBKsxzFpEeK+Mrp9faXOlmrw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.57.1.tgz", - "integrity": "sha512-MMtej3YHWeg/0klK2Qodf3yrNzz6CGjo2UntLvk2RSPlhzgLvYEB3frRvbEF2wRKh1Z2fDIg9KRPe1fawv7C+g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.57.1.tgz", - "integrity": "sha512-1a/qhaaOXhqXGpMFMET9VqwZakkljWHLmZOX48R0I/YLbhdxr1m4gtG1Hq7++VhVUmf+L3sTAf9op4JlhQ5u1Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.57.1.tgz", - "integrity": "sha512-QWO6RQTZ/cqYtJMtxhkRkidoNGXc7ERPbZN7dVW5SdURuLeVU7lwKMpo18XdcmpWYd0qsP1bwKPf7DNSUinhvA==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.57.1.tgz", - "integrity": "sha512-xpObYIf+8gprgWaPP32xiN5RVTi/s5FCR+XMXSKmhfoJjrpRAjCuuqQXyxUa/eJTdAE6eJ+KDKaoEqjZQxh3Gw==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.57.1.tgz", - "integrity": "sha512-4BrCgrpZo4hvzMDKRqEaW1zeecScDCR+2nZ86ATLhAoJ5FQ+lbHVD3ttKe74/c7tNT9c6F2viwB3ufwp01Oh2w==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.57.1.tgz", - "integrity": "sha512-NOlUuzesGauESAyEYFSe3QTUguL+lvrN1HtwEEsU2rOwdUDeTMJdO5dUYl/2hKf9jWydJrO9OL/XSSf65R5+Xw==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.57.1.tgz", - "integrity": "sha512-ptA88htVp0AwUUqhVghwDIKlvJMD/fmL/wrQj99PRHFRAG6Z5nbWoWG4o81Nt9FT+IuqUQi+L31ZKAFeJ5Is+A==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.57.1.tgz", - "integrity": "sha512-S51t7aMMTNdmAMPpBg7OOsTdn4tySRQvklmL3RpDRyknk87+Sp3xaumlatU+ppQ+5raY7sSTcC2beGgvhENfuw==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.57.1.tgz", - "integrity": "sha512-Bl00OFnVFkL82FHbEqy3k5CUCKH6OEJL54KCyx2oqsmZnFTR8IoNqBF+mjQVcRCT5sB6yOvK8A37LNm/kPJiZg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.57.1.tgz", - "integrity": "sha512-ABca4ceT4N+Tv/GtotnWAeXZUZuM/9AQyCyKYyKnpk4yoA7QIAuBt6Hkgpw8kActYlew2mvckXkvx0FfoInnLg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.57.1.tgz", - "integrity": "sha512-HFps0JeGtuOR2convgRRkHCekD7j+gdAuXM+/i6kGzQtFhlCtQkpwtNzkNj6QhCDp7DRJ7+qC/1Vg2jt5iSOFw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.57.1.tgz", - "integrity": "sha512-H+hXEv9gdVQuDTgnqD+SQffoWoc0Of59AStSzTEj/feWTBAnSfSD3+Dql1ZruJQxmykT/JVY0dE8Ka7z0DH1hw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ] - }, - "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.57.1.tgz", - "integrity": "sha512-4wYoDpNg6o/oPximyc/NG+mYUejZrCU2q+2w6YZqrAs2UcNUChIZXjtafAiiZSUc7On8v5NyNj34Kzj/Ltk6dQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ] - }, - "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.57.1.tgz", - "integrity": "sha512-O54mtsV/6LW3P8qdTcamQmuC990HDfR71lo44oZMZlXU4tzLrbvTii87Ni9opq60ds0YzuAlEr/GNwuNluZyMQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.57.1.tgz", - "integrity": "sha512-P3dLS+IerxCT/7D2q2FYcRdWRl22dNbrbBEtxdWhXrfIMPP9lQhb5h4Du04mdl5Woq05jVCDPCMF7Ub0NAjIew==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.57.1.tgz", - "integrity": "sha512-VMBH2eOOaKGtIJYleXsi2B8CPVADrh+TyNxJ4mWPnKfLB/DBUmzW+5m1xUrcwWoMfSLagIRpjUFeW5CO5hyciQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.57.1.tgz", - "integrity": "sha512-mxRFDdHIWRxg3UfIIAwCm6NzvxG0jDX/wBN6KsQFTvKFqqg9vTrWUE68qEjHt19A5wwx5X5aUi2zuZT7YR0jrA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@scalar/core": { - "version": "0.3.37", - "resolved": "https://registry.npmjs.org/@scalar/core/-/core-0.3.37.tgz", - "integrity": "sha512-cQWMHsGD9jCiYHi91acR3tOsj+qGk+dRQ2W+N5+au1NZ/GkUNT5TUEufekn/sj1S8af+lOnn3y0xXoTI34jCog==", - "license": "MIT", - "dependencies": { - "@scalar/types": "0.6.2" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@scalar/helpers": { - "version": "0.2.11", - "resolved": "https://registry.npmjs.org/@scalar/helpers/-/helpers-0.2.11.tgz", - "integrity": "sha512-Y7DLt1bIZF9dvHzJwSJTcC1lpSr1Tbf4VBhHOCRIHu23Rr7/lhQnddRxFmPV1tZXwEQKz7F7yRrubwCfKPCucw==", - "license": "MIT", - "engines": { - "node": ">=20" - } - }, - "node_modules/@scalar/hono-api-reference": { - "version": "0.9.40", - "resolved": "https://registry.npmjs.org/@scalar/hono-api-reference/-/hono-api-reference-0.9.40.tgz", - "integrity": "sha512-0tQOxyEwuu1QGcoA5aCJg2eSmNfF35mxeGx13TND9ud5ZBeuOqli8jyfykgkqV3gFTnDDlQYgQcOvB6Rgk2beA==", - "license": "MIT", - "dependencies": { - "@scalar/core": "0.3.37" - }, - "engines": { - "node": ">=20" - }, - "peerDependencies": { - "hono": "^4.11.5" - } - }, - "node_modules/@scalar/types": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/@scalar/types/-/types-0.6.2.tgz", - "integrity": "sha512-VWfY/z9R5NT8PpKVmvmIj6QSh56MMcl8x3JsGiNxR+w7txGQEq+QzEl35aU56uSBFmLfPk1oyInoaHhkosKooA==", - "license": "MIT", - "dependencies": { - "@scalar/helpers": "0.2.11", - "nanoid": "^5.1.6", - "type-fest": "^5.3.1", - "zod": "^4.3.5" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@scalar/types/node_modules/nanoid": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.6.tgz", - "integrity": "sha512-c7+7RQ+dMB5dPwwCp4ee1/iV/q2P6aK1mTZcfr1BTuVlyW9hJYiMPybJCcnBlQtuSmTIWNeazm/zqNoZSSElBg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.js" - }, - "engines": { - "node": "^18 || >=20" - } - }, - "node_modules/@smithy/abort-controller": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-4.2.8.tgz", - "integrity": "sha512-peuVfkYHAmS5ybKxWcfraK7WBBP0J+rkfUcbHJJKQ4ir3UAUNQI+Y4Vt/PqSzGqgloJ5O1dk7+WzNL8wcCSXbw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/chunked-blob-reader": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-5.2.0.tgz", - "integrity": "sha512-WmU0TnhEAJLWvfSeMxBNe5xtbselEO8+4wG0NtZeL8oR21WgH1xiO37El+/Y+H/Ie4SCwBy3MxYWmOYaGgZueA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/chunked-blob-reader-native": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-4.2.1.tgz", - "integrity": "sha512-lX9Ay+6LisTfpLid2zZtIhSEjHMZoAR5hHCR4H7tBz/Zkfr5ea8RcQ7Tk4mi0P76p4cN+Btz16Ffno7YHpKXnQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-base64": "^4.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/config-resolver": { - "version": "4.4.6", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.4.6.tgz", - "integrity": "sha512-qJpzYC64kaj3S0fueiu3kXm8xPrR3PcXDPEgnaNMRn0EjNSZFoFjvbUp0YUDsRhN1CB90EnHJtbxWKevnH99UQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/node-config-provider": "^4.3.8", - "@smithy/types": "^4.12.0", - "@smithy/util-config-provider": "^4.2.0", - "@smithy/util-endpoints": "^3.2.8", - "@smithy/util-middleware": "^4.2.8", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/core": { - "version": "3.22.1", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.22.1.tgz", - "integrity": "sha512-x3ie6Crr58MWrm4viHqqy2Du2rHYZjwu8BekasrQx4ca+Y24dzVAwq3yErdqIbc2G3I0kLQA13PQ+/rde+u65g==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/middleware-serde": "^4.2.9", - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "@smithy/util-base64": "^4.3.0", - "@smithy/util-body-length-browser": "^4.2.0", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-stream": "^4.5.11", - "@smithy/util-utf8": "^4.2.0", - "@smithy/uuid": "^1.1.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/credential-provider-imds": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.8.tgz", - "integrity": "sha512-FNT0xHS1c/CPN8upqbMFP83+ul5YgdisfCfkZ86Jh2NSmnqw/AJ6x5pEogVCTVvSm7j9MopRU89bmDelxuDMYw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/node-config-provider": "^4.3.8", - "@smithy/property-provider": "^4.2.8", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/eventstream-codec": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-4.2.8.tgz", - "integrity": "sha512-jS/O5Q14UsufqoGhov7dHLOPCzkYJl9QDzusI2Psh4wyYx/izhzvX9P4D69aTxcdfVhEPhjK+wYyn/PzLjKbbw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/crc32": "5.2.0", - "@smithy/types": "^4.12.0", - "@smithy/util-hex-encoding": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-browser": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-4.2.8.tgz", - "integrity": "sha512-MTfQT/CRQz5g24ayXdjg53V0mhucZth4PESoA5IhvaWVDTOQLfo8qI9vzqHcPsdd2v6sqfTYqF5L/l+pea5Uyw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/eventstream-serde-universal": "^4.2.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-config-resolver": { - "version": "4.3.8", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-4.3.8.tgz", - "integrity": "sha512-ah12+luBiDGzBruhu3efNy1IlbwSEdNiw8fOZksoKoWW1ZHvO/04MQsdnws/9Aj+5b0YXSSN2JXKy/ClIsW8MQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-node": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-4.2.8.tgz", - "integrity": "sha512-cYpCpp29z6EJHa5T9WL0KAlq3SOKUQkcgSoeRfRVwjGgSFl7Uh32eYGt7IDYCX20skiEdRffyDpvF2efEZPC0A==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/eventstream-serde-universal": "^4.2.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-universal": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-4.2.8.tgz", - "integrity": "sha512-iJ6YNJd0bntJYnX6s52NC4WFYcZeKrPUr1Kmmr5AwZcwCSzVpS7oavAmxMR7pMq7V+D1G4s9F5NJK0xwOsKAlQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/eventstream-codec": "^4.2.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/fetch-http-handler": { - "version": "5.3.9", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.3.9.tgz", - "integrity": "sha512-I4UhmcTYXBrct03rwzQX1Y/iqQlzVQaPxWjCjula++5EmWq9YGBrx6bbGqluGc1f0XEfhSkiY4jhLgbsJUMKRA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.8", - "@smithy/querystring-builder": "^4.2.8", - "@smithy/types": "^4.12.0", - "@smithy/util-base64": "^4.3.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/hash-blob-browser": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-4.2.9.tgz", - "integrity": "sha512-m80d/iicI7DlBDxyQP6Th7BW/ejDGiF0bgI754+tiwK0lgMkcaIBgvwwVc7OFbY4eUzpGtnig52MhPAEJ7iNYg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/chunked-blob-reader": "^5.2.0", - "@smithy/chunked-blob-reader-native": "^4.2.1", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/hash-node": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.2.8.tgz", - "integrity": "sha512-7ZIlPbmaDGxVoxErDZnuFG18WekhbA/g2/i97wGj+wUBeS6pcUeAym8u4BXh/75RXWhgIJhyC11hBzig6MljwA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "@smithy/util-buffer-from": "^4.2.0", - "@smithy/util-utf8": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/hash-stream-node": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-4.2.8.tgz", - "integrity": "sha512-v0FLTXgHrTeheYZFGhR+ehX5qUm4IQsjAiL9qehad2cyjMWcN2QG6/4mSwbSgEQzI7jwfoXj7z4fxZUx/Mhj2w==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "@smithy/util-utf8": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/invalid-dependency": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.2.8.tgz", - "integrity": "sha512-N9iozRybwAQ2dn9Fot9kI6/w9vos2oTXLhtK7ovGqwZjlOcxu6XhPlpLpC+INsxktqHinn5gS2DXDjDF2kG5sQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/is-array-buffer": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-4.2.0.tgz", - "integrity": "sha512-DZZZBvC7sjcYh4MazJSGiWMI2L7E0oCiRHREDzIxi/M2LY79/21iXt6aPLHge82wi5LsuRF5A06Ds3+0mlh6CQ==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/md5-js": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-4.2.8.tgz", - "integrity": "sha512-oGMaLj4tVZzLi3itBa9TCswgMBr7k9b+qKYowQ6x1rTyTuO1IU2YHdHUa+891OsOH+wCsH7aTPRsTJO3RMQmjQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "@smithy/util-utf8": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/middleware-content-length": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.8.tgz", - "integrity": "sha512-RO0jeoaYAB1qBRhfVyq0pMgBoUK34YEJxVxyjOWYZiOKOq2yMZ4MnVXMZCUDenpozHue207+9P5ilTV1zeda0A==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/middleware-endpoint": { - "version": "4.4.13", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.13.tgz", - "integrity": "sha512-x6vn0PjYmGdNuKh/juUJJewZh7MoQ46jYaJ2mvekF4EesMuFfrl4LaW/k97Zjf8PTCPQmPgMvwewg7eNoH9n5w==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/core": "^3.22.1", - "@smithy/middleware-serde": "^4.2.9", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "@smithy/util-middleware": "^4.2.8", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/middleware-retry": { - "version": "4.4.30", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.30.tgz", - "integrity": "sha512-CBGyFvN0f8hlnqKH/jckRDz78Snrp345+PVk8Ux7pnkUCW97Iinse59lY78hBt04h1GZ6hjBN94BRwZy1xC8Bg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/node-config-provider": "^4.3.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/service-error-classification": "^4.2.8", - "@smithy/smithy-client": "^4.11.2", - "@smithy/types": "^4.12.0", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-retry": "^4.2.8", - "@smithy/uuid": "^1.1.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/middleware-serde": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.2.9.tgz", - "integrity": "sha512-eMNiej0u/snzDvlqRGSN3Vl0ESn3838+nKyVfF2FKNXFbi4SERYT6PR392D39iczngbqqGG0Jl1DlCnp7tBbXQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/middleware-stack": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.2.8.tgz", - "integrity": "sha512-w6LCfOviTYQjBctOKSwy6A8FIkQy7ICvglrZFl6Bw4FmcQ1Z420fUtIhxaUZZshRe0VCq4kvDiPiXrPZAe8oRA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/node-config-provider": { - "version": "4.3.8", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.3.8.tgz", - "integrity": "sha512-aFP1ai4lrbVlWjfpAfRSL8KFcnJQYfTl5QxLJXY32vghJrDuFyPZ6LtUL+JEGYiFRG1PfPLHLoxj107ulncLIg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/property-provider": "^4.2.8", - "@smithy/shared-ini-file-loader": "^4.4.3", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/node-http-handler": { - "version": "4.4.9", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.4.9.tgz", - "integrity": "sha512-KX5Wml5mF+luxm1szW4QDz32e3NObgJ4Fyw+irhph4I/2geXwUy4jkIMUs5ZPGflRBeR6BUkC2wqIab4Llgm3w==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/abort-controller": "^4.2.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/querystring-builder": "^4.2.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/property-provider": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.2.8.tgz", - "integrity": "sha512-EtCTbyIveCKeOXDSWSdze3k612yCPq1YbXsbqX3UHhkOSW8zKsM9NOJG5gTIya0vbY2DIaieG8pKo1rITHYL0w==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/protocol-http": { - "version": "5.3.8", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.3.8.tgz", - "integrity": "sha512-QNINVDhxpZ5QnP3aviNHQFlRogQZDfYlCkQT+7tJnErPQbDhysondEjhikuANxgMsZrkGeiAxXy4jguEGsDrWQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/querystring-builder": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-4.2.8.tgz", - "integrity": "sha512-Xr83r31+DrE8CP3MqPgMJl+pQlLLmOfiEUnoyAlGzzJIrEsbKsPy1hqH0qySaQm4oWrCBlUqRt+idEgunKB+iw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "@smithy/util-uri-escape": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/querystring-parser": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.2.8.tgz", - "integrity": "sha512-vUurovluVy50CUlazOiXkPq40KGvGWSdmusa3130MwrR1UNnNgKAlj58wlOe61XSHRpUfIIh6cE0zZ8mzKaDPA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/service-error-classification": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.2.8.tgz", - "integrity": "sha512-mZ5xddodpJhEt3RkCjbmUQuXUOaPNTkbMGR0bcS8FE0bJDLMZlhmpgrvPNCYglVw5rsYTpSnv19womw9WWXKQQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/shared-ini-file-loader": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.4.3.tgz", - "integrity": "sha512-DfQjxXQnzC5UbCUPeC3Ie8u+rIWZTvuDPAGU/BxzrOGhRvgUanaP68kDZA+jaT3ZI+djOf+4dERGlm9mWfFDrg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/signature-v4": { - "version": "5.3.8", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.3.8.tgz", - "integrity": "sha512-6A4vdGj7qKNRF16UIcO8HhHjKW27thsxYci+5r/uVRkdcBEkOEiY8OMPuydLX4QHSrJqGHPJzPRwwVTqbLZJhg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^4.2.0", - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "@smithy/util-hex-encoding": "^4.2.0", - "@smithy/util-middleware": "^4.2.8", - "@smithy/util-uri-escape": "^4.2.0", - "@smithy/util-utf8": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/smithy-client": { - "version": "4.11.2", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.11.2.tgz", - "integrity": "sha512-SCkGmFak/xC1n7hKRsUr6wOnBTJ3L22Qd4e8H1fQIuKTAjntwgU8lrdMe7uHdiT2mJAOWA/60qaW9tiMu69n1A==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/core": "^3.22.1", - "@smithy/middleware-endpoint": "^4.4.13", - "@smithy/middleware-stack": "^4.2.8", - "@smithy/protocol-http": "^5.3.8", - "@smithy/types": "^4.12.0", - "@smithy/util-stream": "^4.5.11", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/types": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.12.0.tgz", - "integrity": "sha512-9YcuJVTOBDjg9LWo23Qp0lTQ3D7fQsQtwle0jVfpbUHy9qBwCEgKuVH4FqFB3VYu0nwdHKiEMA+oXz7oV8X1kw==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/url-parser": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.2.8.tgz", - "integrity": "sha512-NQho9U68TGMEU639YkXnVMV3GEFFULmmaWdlu1E9qzyIePOHsoSnagTGSDv1Zi8DCNN6btxOSdgmy5E/hsZwhA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/querystring-parser": "^4.2.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-base64": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.3.0.tgz", - "integrity": "sha512-GkXZ59JfyxsIwNTWFnjmFEI8kZpRNIBfxKjv09+nkAWPt/4aGaEWMM04m4sxgNVWkbt2MdSvE3KF/PfX4nFedQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-buffer-from": "^4.2.0", - "@smithy/util-utf8": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-body-length-browser": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.2.0.tgz", - "integrity": "sha512-Fkoh/I76szMKJnBXWPdFkQJl2r9SjPt3cMzLdOB6eJ4Pnpas8hVoWPYemX/peO0yrrvldgCUVJqOAjUrOLjbxg==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-body-length-node": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.2.1.tgz", - "integrity": "sha512-h53dz/pISVrVrfxV1iqXlx5pRg3V2YWFcSQyPyXZRrZoZj4R4DeWRDo1a7dd3CPTcFi3kE+98tuNyD2axyZReA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-buffer-from": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-4.2.0.tgz", - "integrity": "sha512-kAY9hTKulTNevM2nlRtxAG2FQ3B2OR6QIrPY3zE5LqJy1oxzmgBGsHLWTcNhWXKchgA0WHW+mZkQrng/pgcCew==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-config-provider": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.2.0.tgz", - "integrity": "sha512-YEjpl6XJ36FTKmD+kRJJWYvrHeUvm5ykaUS5xK+6oXffQPHeEM4/nXlZPe+Wu0lsgRUcNZiliYNh/y7q9c2y6Q==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-defaults-mode-browser": { - "version": "4.3.29", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.29.tgz", - "integrity": "sha512-nIGy3DNRmOjaYaaKcQDzmWsro9uxlaqUOhZDHQed9MW/GmkBZPtnU70Pu1+GT9IBmUXwRdDuiyaeiy9Xtpn3+Q==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/property-provider": "^4.2.8", - "@smithy/smithy-client": "^4.11.2", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-defaults-mode-node": { - "version": "4.2.32", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.32.tgz", - "integrity": "sha512-7dtFff6pu5fsjqrVve0YMhrnzJtccCWDacNKOkiZjJ++fmjGExmmSu341x+WU6Oc1IccL7lDuaUj7SfrHpWc5Q==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/config-resolver": "^4.4.6", - "@smithy/credential-provider-imds": "^4.2.8", - "@smithy/node-config-provider": "^4.3.8", - "@smithy/property-provider": "^4.2.8", - "@smithy/smithy-client": "^4.11.2", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-endpoints": { - "version": "3.2.8", - "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.2.8.tgz", - "integrity": "sha512-8JaVTn3pBDkhZgHQ8R0epwWt+BqPSLCjdjXXusK1onwJlRuN69fbvSK66aIKKO7SwVFM6x2J2ox5X8pOaWcUEw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/node-config-provider": "^4.3.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-hex-encoding": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-4.2.0.tgz", - "integrity": "sha512-CCQBwJIvXMLKxVbO88IukazJD9a4kQ9ZN7/UMGBjBcJYvatpWk+9g870El4cB8/EJxfe+k+y0GmR9CAzkF+Nbw==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-middleware": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.2.8.tgz", - "integrity": "sha512-PMqfeJxLcNPMDgvPbbLl/2Vpin+luxqTGPpW3NAQVLbRrFRzTa4rNAASYeIGjRV9Ytuhzny39SpyU04EQreF+A==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-retry": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.2.8.tgz", - "integrity": "sha512-CfJqwvoRY0kTGe5AkQokpURNCT1u/MkRzMTASWMPPo2hNSnKtF1D45dQl3DE2LKLr4m+PW9mCeBMJr5mCAVThg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/service-error-classification": "^4.2.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-stream": { - "version": "4.5.11", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.11.tgz", - "integrity": "sha512-lKmZ0S/3Qj2OF5H1+VzvDLb6kRxGzZHq6f3rAsoSu5cTLGsn3v3VQBA8czkNNXlLjoFEtVu3OQT2jEeOtOE2CA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/node-http-handler": "^4.4.9", - "@smithy/types": "^4.12.0", - "@smithy/util-base64": "^4.3.0", - "@smithy/util-buffer-from": "^4.2.0", - "@smithy/util-hex-encoding": "^4.2.0", - "@smithy/util-utf8": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-uri-escape": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-4.2.0.tgz", - "integrity": "sha512-igZpCKV9+E/Mzrpq6YacdTQ0qTiLm85gD6N/IrmyDvQFA4UnU3d5g3m8tMT/6zG/vVkWSU+VxeUyGonL62DuxA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-utf8": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.2.0.tgz", - "integrity": "sha512-zBPfuzoI8xyBtR2P6WQj63Rz8i3AmfAaJLuNG8dWsfvPe8lO4aCPYLn879mEgHndZH1zQ2oXmG8O1GGzzaoZiw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-buffer-from": "^4.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-waiter": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-4.2.8.tgz", - "integrity": "sha512-n+lahlMWk+aejGuax7DPWtqav8HYnWxQwR+LCG2BgCUmaGcTe9qZCFsmw8TMg9iG75HOwhrJCX9TCJRLH+Yzqg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/abort-controller": "^4.2.8", - "@smithy/types": "^4.12.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/uuid": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@smithy/uuid/-/uuid-1.1.0.tgz", - "integrity": "sha512-4aUIteuyxtBUhVdiQqcDhKFitwfd9hqoSDYY2KRXiWtgoWJ9Bmise+KfEPDiVHWeJepvF8xJO9/9+WDIciMFFw==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@so-ric/colorspace": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@so-ric/colorspace/-/colorspace-1.1.6.tgz", - "integrity": "sha512-/KiKkpHNOBgkFJwu9sh48LkHSMYGyuTcSFK/qMBdnOAlrRJzRSXAOFB5qwzaVQuDl8wAvHVMkaASQDReTahxuw==", - "license": "MIT", - "dependencies": { - "color": "^5.0.2", - "text-hex": "1.0.x" - } - }, - "node_modules/@standard-schema/spec": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", - "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", - "license": "MIT" - }, - "node_modules/@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, - "node_modules/@types/caseless": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", - "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", - "license": "MIT" - }, - "node_modules/@types/chai": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", - "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/deep-eql": "*", - "assertion-error": "^2.0.1" - } - }, - "node_modules/@types/deep-eql": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", - "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/estree": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", - "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/node": { - "version": "25.2.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.2.1.tgz", - "integrity": "sha512-CPrnr8voK8vC6eEtyRzvMpgp3VyVRhgclonE7qYi6P9sXwYb59ucfrnmFBTaP0yUi8Gk4yZg/LlTJULGxvTNsg==", - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/@types/pg": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.16.0.tgz", - "integrity": "sha512-RmhMd/wD+CF8Dfo+cVIy3RR5cl8CyfXQ0tGgW6XBL8L4LM/UTEbNXYRbLwU6w+CgrKBNbrQWt4FUtTfaU5jSYQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, - "node_modules/@types/readable-stream": { - "version": "4.0.23", - "resolved": "https://registry.npmjs.org/@types/readable-stream/-/readable-stream-4.0.23.tgz", - "integrity": "sha512-wwXrtQvbMHxCbBgjHaMGEmImFTQxxpfMOR/ZoQnXxB1woqkUbdLGFDgauo00Py9IudiaqSeiBiulSV9i6XIPig==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/request": { - "version": "2.48.13", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.13.tgz", - "integrity": "sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==", - "license": "MIT", - "dependencies": { - "@types/caseless": "*", - "@types/node": "*", - "@types/tough-cookie": "*", - "form-data": "^2.5.5" - } - }, - "node_modules/@types/tough-cookie": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", - "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", - "license": "MIT" - }, - "node_modules/@types/triple-beam": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", - "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", - "license": "MIT" - }, - "node_modules/@typespec/ts-http-runtime": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.3.tgz", - "integrity": "sha512-91fp6CAAJSRtH5ja95T1FHSKa8aPW9/Zw6cta81jlZTUw/+Vq8jM/AfF/14h2b71wwR84JUTW/3Y8QPhDAawFA==", - "license": "MIT", - "dependencies": { - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@vercel/oidc": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.1.0.tgz", - "integrity": "sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w==", - "license": "Apache-2.0", - "engines": { - "node": ">= 20" - } - }, - "node_modules/@vitest/coverage-v8": { - "version": "4.0.18", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.0.18.tgz", - "integrity": "sha512-7i+N2i0+ME+2JFZhfuz7Tg/FqKtilHjGyGvoHYQ6iLV0zahbsJ9sljC9OcFcPDbhYKCet+sG8SsVqlyGvPflZg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.0.18", - "ast-v8-to-istanbul": "^0.3.10", - "istanbul-lib-coverage": "^3.2.2", - "istanbul-lib-report": "^3.0.1", - "istanbul-reports": "^3.2.0", - "magicast": "^0.5.1", - "obug": "^2.1.1", - "std-env": "^3.10.0", - "tinyrainbow": "^3.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@vitest/browser": "4.0.18", - "vitest": "4.0.18" - }, - "peerDependenciesMeta": { - "@vitest/browser": { - "optional": true - } - } - }, - "node_modules/@vitest/expect": { - "version": "4.0.18", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.18.tgz", - "integrity": "sha512-8sCWUyckXXYvx4opfzVY03EOiYVxyNrHS5QxX3DAIi5dpJAAkyJezHCP77VMX4HKA2LDT/Jpfo8i2r5BE3GnQQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@standard-schema/spec": "^1.0.0", - "@types/chai": "^5.2.2", - "@vitest/spy": "4.0.18", - "@vitest/utils": "4.0.18", - "chai": "^6.2.1", - "tinyrainbow": "^3.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/mocker": { - "version": "4.0.18", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.0.18.tgz", - "integrity": "sha512-HhVd0MDnzzsgevnOWCBj5Otnzobjy5wLBe4EdeeFGv8luMsGcYqDuFRMcttKWZA5vVO8RFjexVovXvAM4JoJDQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/spy": "4.0.18", - "estree-walker": "^3.0.3", - "magic-string": "^0.30.21" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0-0" - }, - "peerDependenciesMeta": { - "msw": { - "optional": true - }, - "vite": { - "optional": true - } - } - }, - "node_modules/@vitest/pretty-format": { - "version": "4.0.18", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.0.18.tgz", - "integrity": "sha512-P24GK3GulZWC5tz87ux0m8OADrQIUVDPIjjj65vBXYG17ZeU3qD7r+MNZ1RNv4l8CGU2vtTRqixrOi9fYk/yKw==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyrainbow": "^3.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/runner": { - "version": "4.0.18", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.0.18.tgz", - "integrity": "sha512-rpk9y12PGa22Jg6g5M3UVVnTS7+zycIGk9ZNGN+m6tZHKQb7jrP7/77WfZy13Y/EUDd52NDsLRQhYKtv7XfPQw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/utils": "4.0.18", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/snapshot": { - "version": "4.0.18", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.0.18.tgz", - "integrity": "sha512-PCiV0rcl7jKQjbgYqjtakly6T1uwv/5BQ9SwBLekVg/EaYeQFPiXcgrC2Y7vDMA8dM1SUEAEV82kgSQIlXNMvA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.0.18", - "magic-string": "^0.30.21", - "pathe": "^2.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/spy": { - "version": "4.0.18", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.0.18.tgz", - "integrity": "sha512-cbQt3PTSD7P2OARdVW3qWER5EGq7PHlvE+QfzSC0lbwO+xnt7+XH06ZzFjFRgzUX//JmpxrCu92VdwvEPlWSNw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@vitest/utils": { - "version": "4.0.18", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.0.18.tgz", - "integrity": "sha512-msMRKLMVLWygpK3u2Hybgi4MNjcYJvwTb0Ru09+fOyCXIgT5raYP041DRRdiJiI3k/2U6SEbAETB3YtBrUkCFA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/pretty-format": "4.0.18", - "tinyrainbow": "^3.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/@zilliz/milvus2-sdk-node": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/@zilliz/milvus2-sdk-node/-/milvus2-sdk-node-2.6.9.tgz", - "integrity": "sha512-qOaVIpQ3E4w6Dp4lp9QIIuGedpE5dWRhK9SRX+y9WXcq4EXYvcdfR2aG/Vb5tWBPQwcMrGb5z8gRfFy7/gRbIw==", - "license": "Apache-2.0", - "dependencies": { - "@grpc/grpc-js": "1.7.3", - "@grpc/proto-loader": "^0.7.10", - "@opentelemetry/api": "^1.9.0", - "@petamoriken/float16": "^3.8.6", - "dayjs": "^1.11.7", - "generic-pool": "^3.9.0", - "lru-cache": "^9.1.2", - "protobufjs": "^7.2.6", - "winston": "^3.9.0" - } - }, - "node_modules/@zilliz/milvus2-sdk-node/node_modules/lru-cache": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-9.1.2.tgz", - "integrity": "sha512-ERJq3FOzJTxBbFjZ7iDs+NiK4VI9Wz+RdrrAB8dio1oV+YvdPzUEE4QNiT2VD51DkIbCYRUUzCRkssXCHqSnKQ==", - "license": "ISC", - "engines": { - "node": "14 || >=16.14" - } - }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "license": "MIT", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, - "node_modules/abort-controller-x": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/abort-controller-x/-/abort-controller-x-0.5.0.tgz", - "integrity": "sha512-yTt9CI0x+nRfX6BFMenEGP8ooPvErGH6AbFz20C2IeOLIlDsrw/VHpgne3GsCEuTA410IiFiaLVFKmgM4bKEPQ==", - "license": "MIT" - }, - "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", - "dev": true, - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/ai": { - "version": "6.0.73", - "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.73.tgz", - "integrity": "sha512-p2/ICXIjAM4+bIFHEkAB+l58zq+aTmxAkotsb6doNt/CEms72zt6gxv2ky1fQDwU4ecMOcmMh78VJUSEKECzlg==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/gateway": "3.0.36", - "@ai-sdk/provider": "3.0.7", - "@ai-sdk/provider-utils": "4.0.13", - "@opentelemetry/api": "1.9.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/ansi-styles/node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/ansi-styles/node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, - "node_modules/any-promise": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", - "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", - "dev": true, - "license": "MIT" - }, - "node_modules/arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/assertion-error": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", - "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - } - }, - "node_modules/ast-v8-to-istanbul": { - "version": "0.3.11", - "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-0.3.11.tgz", - "integrity": "sha512-Qya9fkoofMjCBNVdWINMjB5KZvkYfaO9/anwkWnjxibpWUxo5iHl2sOdP7/uAqaRuUYuoo8rDwnbaaKVFxoUvw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.31", - "estree-walker": "^3.0.3", - "js-tokens": "^10.0.0" - } - }, - "node_modules/async": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", - "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", - "license": "MIT" - }, - "node_modules/async-retry": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "license": "MIT", - "dependencies": { - "retry": "0.13.1" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/aws-ssl-profiles": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz", - "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==", - "license": "MIT", - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/bignumber.js": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", - "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/bl": { - "version": "6.1.6", - "resolved": "https://registry.npmjs.org/bl/-/bl-6.1.6.tgz", - "integrity": "sha512-jLsPgN/YSvPUg9UX0Kd73CXpm2Psg9FxMeCSXnk3WBO3CMT10JMwijubhGfHCnFu6TPn1ei3b975dxv7K2pWVg==", - "license": "MIT", - "dependencies": { - "@types/readable-stream": "^4.0.0", - "buffer": "^6.0.3", - "inherits": "^2.0.4", - "readable-stream": "^4.2.0" - } - }, - "node_modules/bl/node_modules/readable-stream": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", - "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", - "license": "MIT", - "dependencies": { - "abort-controller": "^3.0.0", - "buffer": "^6.0.3", - "events": "^3.3.0", - "process": "^0.11.10", - "string_decoder": "^1.3.0" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - } - }, - "node_modules/bowser": { - "version": "2.13.1", - "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.13.1.tgz", - "integrity": "sha512-OHawaAbjwx6rqICCKgSG0SAnT05bzd7ppyKLVUITZpANBaaMFBAsaNkto3LoQ31tyFP5kNujE8Cdx85G9VzOkw==", - "license": "MIT" - }, - "node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, - "node_modules/buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", - "license": "BSD-3-Clause" - }, - "node_modules/bundle-name": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", - "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", - "license": "MIT", - "dependencies": { - "run-applescript": "^7.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/bundle-require": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/bundle-require/-/bundle-require-5.1.0.tgz", - "integrity": "sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==", - "dev": true, - "license": "MIT", - "dependencies": { - "load-tsconfig": "^0.2.3" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "peerDependencies": { - "esbuild": ">=0.18" - } - }, - "node_modules/cac": { - "version": "6.7.14", - "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", - "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/chai": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", - "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/chokidar": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", - "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "readdirp": "^4.0.1" - }, - "engines": { - "node": ">= 14.16.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/color": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/color/-/color-5.0.3.tgz", - "integrity": "sha512-ezmVcLR3xAVp8kYOm4GS45ZLLgIE6SPAFoduLr6hTDajwb3KZ2F46gulK3XpcwRFb5KKGCSezCBAY4Dw4HsyXA==", - "license": "MIT", - "dependencies": { - "color-convert": "^3.1.3", - "color-string": "^2.1.3" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/color-convert": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-3.1.3.tgz", - "integrity": "sha512-fasDH2ont2GqF5HpyO4w0+BcewlhHEZOFn9c1ckZdHpJ56Qb7MHhH/IcJZbBGgvdtwdwNbLvxiBEdg336iA9Sg==", - "license": "MIT", - "dependencies": { - "color-name": "^2.0.0" - }, - "engines": { - "node": ">=14.6" - } - }, - "node_modules/color-name": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-2.1.0.tgz", - "integrity": "sha512-1bPaDNFm0axzE4MEAzKPuqKWeRaT43U/hyxKPBdqTfmPF+d6n7FSoTFxLVULUJOmiLp01KjhIPPH+HrXZJN4Rg==", - "license": "MIT", - "engines": { - "node": ">=12.20" - } - }, - "node_modules/color-string": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-2.1.4.tgz", - "integrity": "sha512-Bb6Cq8oq0IjDOe8wJmi4JeNn763Xs9cfrBcaylK1tPypWzyoy2G3l90v9k64kjphl/ZJjPIShFztenRomi8WTg==", - "license": "MIT", - "dependencies": { - "color-name": "^2.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/commander": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", - "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, - "node_modules/confbox": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz", - "integrity": "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==", - "dev": true, - "license": "MIT" - }, - "node_modules/consola": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz", - "integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.18.0 || >=16.10.0" - } - }, - "node_modules/cross-fetch": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.2.0.tgz", - "integrity": "sha512-Q+xVJLoGOeIMXZmbUK4HYk+69cQH6LudR0Vu/pRm2YlU/hDV9CiS0gKUMaWY5f2NeUH9C1nV3bsTlCo0FsTV1Q==", - "license": "MIT", - "dependencies": { - "node-fetch": "^2.7.0" - } - }, - "node_modules/dayjs": { - "version": "1.11.19", - "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.19.tgz", - "integrity": "sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw==", - "license": "MIT" - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/default-browser": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.5.0.tgz", - "integrity": "sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==", - "license": "MIT", - "dependencies": { - "bundle-name": "^4.1.0", - "default-browser-id": "^5.0.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/default-browser-id": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz", - "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/define-lazy-prop": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", - "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/denque": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", - "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.10" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/duplexify": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", - "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", - "license": "MIT", - "dependencies": { - "end-of-stream": "^1.4.1", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "stream-shift": "^1.0.2" - } - }, - "node_modules/ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "license": "Apache-2.0", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, - "node_modules/effection": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/effection/-/effection-4.0.2.tgz", - "integrity": "sha512-O8WMGP10nPuJDwbNGILcaCNWS+CvDYjcdsUSD79nWZ+WtUQ8h1MEV7JJwCSZCSeKx8+TdEaZ/8r6qPTR2o/o8w==", - "license": "MIT", - "engines": { - "node": ">= 16" - } - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/enabled": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", - "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==", - "license": "MIT" - }, - "node_modules/end-of-stream": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", - "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "license": "MIT", - "dependencies": { - "once": "^1.4.0" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-module-lexer": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", - "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", - "dev": true, - "license": "MIT" - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/esbuild": { - "version": "0.27.3", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", - "integrity": "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.3", - "@esbuild/android-arm": "0.27.3", - "@esbuild/android-arm64": "0.27.3", - "@esbuild/android-x64": "0.27.3", - "@esbuild/darwin-arm64": "0.27.3", - "@esbuild/darwin-x64": "0.27.3", - "@esbuild/freebsd-arm64": "0.27.3", - "@esbuild/freebsd-x64": "0.27.3", - "@esbuild/linux-arm": "0.27.3", - "@esbuild/linux-arm64": "0.27.3", - "@esbuild/linux-ia32": "0.27.3", - "@esbuild/linux-loong64": "0.27.3", - "@esbuild/linux-mips64el": "0.27.3", - "@esbuild/linux-ppc64": "0.27.3", - "@esbuild/linux-riscv64": "0.27.3", - "@esbuild/linux-s390x": "0.27.3", - "@esbuild/linux-x64": "0.27.3", - "@esbuild/netbsd-arm64": "0.27.3", - "@esbuild/netbsd-x64": "0.27.3", - "@esbuild/openbsd-arm64": "0.27.3", - "@esbuild/openbsd-x64": "0.27.3", - "@esbuild/openharmony-arm64": "0.27.3", - "@esbuild/sunos-x64": "0.27.3", - "@esbuild/win32-arm64": "0.27.3", - "@esbuild/win32-ia32": "0.27.3", - "@esbuild/win32-x64": "0.27.3" - } - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/estree-walker": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", - "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0" - } - }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "license": "MIT", - "engines": { - "node": ">=0.8.x" - } - }, - "node_modules/eventsource-parser": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", - "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", - "license": "MIT", - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/expect-type": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", - "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, - "node_modules/fast-xml-parser": { - "version": "5.3.4", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.4.tgz", - "integrity": "sha512-EFd6afGmXlCx8H8WTZHhAoDaWaGyuIBoZJ2mknrNxug+aZKjkp0a0dlars9Izl+jF+7Gu1/5f/2h68cQpe0IiA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "strnum": "^2.1.0" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, - "node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, - "node_modules/fecha": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", - "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", - "license": "MIT" - }, - "node_modules/fix-dts-default-cjs-exports": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/fix-dts-default-cjs-exports/-/fix-dts-default-cjs-exports-1.0.1.tgz", - "integrity": "sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "magic-string": "^0.30.17", - "mlly": "^1.7.4", - "rollup": "^4.34.8" - } - }, - "node_modules/fn.name": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", - "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==", - "license": "MIT" - }, - "node_modules/form-data": { - "version": "2.5.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.5.tgz", - "integrity": "sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.35", - "safe-buffer": "^5.2.1" - }, - "engines": { - "node": ">= 0.12" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gaxios": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", - "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gaxios/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/gcp-metadata": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", - "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", - "license": "Apache-2.0", - "dependencies": { - "gaxios": "^6.1.1", - "google-logging-utils": "^0.0.2", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/generate-function": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", - "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", - "license": "MIT", - "dependencies": { - "is-property": "^1.0.2" - } - }, - "node_modules/generic-pool": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", - "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/glob": { - "version": "13.0.1", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.1.tgz", - "integrity": "sha512-B7U/vJpE3DkJ5WXTgTpTRN63uV42DseiXXKMwG14LQBXmsdeIoHAPbU/MEo6II0k5ED74uc2ZGTC6MwHFQhF6w==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "minimatch": "^10.1.2", - "minipass": "^7.1.2", - "path-scurry": "^2.0.0" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/google-auth-library": { - "version": "9.15.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", - "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", - "license": "Apache-2.0", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^6.1.1", - "gcp-metadata": "^6.1.0", - "gtoken": "^7.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/google-logging-utils": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", - "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/graphology": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/graphology/-/graphology-0.26.0.tgz", - "integrity": "sha512-8SSImzgUUYC89Z042s+0r/vMibY7GX/Emz4LDO5e7jYXhuoWfHISPFJYjpRLUSJGq6UQ6xlenvX1p/hJdfXuXg==", - "license": "MIT", - "dependencies": { - "events": "^3.3.0" - }, - "peerDependencies": { - "graphology-types": ">=0.24.0" - } - }, - "node_modules/graphology-dag": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/graphology-dag/-/graphology-dag-0.4.1.tgz", - "integrity": "sha512-3ch9oOAnHZDoT043vyg7ukmSkKJ505nFzaHaYOn0IF2PgGo5VtIavyVK4UpbIa4tli3hhGm1ZTdBsubTmaxu/w==", - "license": "MIT", - "dependencies": { - "graphology-utils": "^2.4.1", - "mnemonist": "^0.39.0" - }, - "peerDependencies": { - "graphology-types": ">=0.19.0" - } - }, - "node_modules/graphology-types": { - "version": "0.24.8", - "resolved": "https://registry.npmjs.org/graphology-types/-/graphology-types-0.24.8.tgz", - "integrity": "sha512-hDRKYXa8TsoZHjgEaysSRyPdT6uB78Ci8WnjgbStlQysz7xR52PInxNsmnB7IBOM1BhikxkNyCVEFgmPKnpx3Q==", - "license": "MIT" - }, - "node_modules/graphology-utils": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/graphology-utils/-/graphology-utils-2.5.2.tgz", - "integrity": "sha512-ckHg8MXrXJkOARk56ZaSCM1g1Wihe2d6iTmz1enGOz4W/l831MBCKSayeFQfowgF8wd+PQ4rlch/56Vs/VZLDQ==", - "license": "MIT", - "peerDependencies": { - "graphology-types": ">=0.23.0" - } - }, - "node_modules/graphql": { - "version": "16.12.0", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.12.0.tgz", - "integrity": "sha512-DKKrynuQRne0PNpEbzuEdHlYOMksHSUI8Zc9Unei5gTsMNA2/vMpoMz/yKba50pejK56qj98qM0SjYxAKi13gQ==", - "license": "MIT", - "engines": { - "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" - } - }, - "node_modules/graphql-request": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/graphql-request/-/graphql-request-6.1.0.tgz", - "integrity": "sha512-p+XPfS4q7aIpKVcgmnZKhMNqhltk20hfXtkaIkTfjjmiKMJ5xrt5c743cL03y/K7y1rg3WrIC49xGiEQ4mxdNw==", - "license": "MIT", - "dependencies": { - "@graphql-typed-document-node/core": "^3.2.0", - "cross-fetch": "^3.1.5" - }, - "peerDependencies": { - "graphql": "14 - 16" - } - }, - "node_modules/gtoken": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", - "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", - "license": "MIT", - "dependencies": { - "gaxios": "^6.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/hono": { - "version": "4.11.8", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.11.8.tgz", - "integrity": "sha512-eVkB/CYCCei7K2WElZW9yYQFWssG0DhaDhVvr7wy5jJ22K+ck8fWW0EsLpB0sITUTvPnc97+rrbQqIr5iqiy9Q==", - "license": "MIT", - "engines": { - "node": ">=16.9.0" - } - }, - "node_modules/html-entities": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz", - "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/mdevils" - }, - { - "type": "patreon", - "url": "https://patreon.com/mdevils" - } - ], - "license": "MIT" - }, - "node_modules/html-escaper": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", - "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", - "dev": true, - "license": "MIT" - }, - "node_modules/http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/iconv-lite": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", - "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, - "node_modules/is-docker": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", - "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", - "license": "MIT", - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-inside-container": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", - "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", - "license": "MIT", - "dependencies": { - "is-docker": "^3.0.0" - }, - "bin": { - "is-inside-container": "cli.js" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-property": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", - "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==", - "license": "MIT" - }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-wsl": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.0.tgz", - "integrity": "sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==", - "license": "MIT", - "dependencies": { - "is-inside-container": "^1.0.0" - }, - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/istanbul-lib-coverage": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", - "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-report": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", - "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "istanbul-lib-coverage": "^3.0.0", - "make-dir": "^4.0.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-reports": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", - "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "html-escaper": "^2.0.0", - "istanbul-lib-report": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/joycon": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", - "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/js-md4": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/js-md4/-/js-md4-0.3.2.tgz", - "integrity": "sha512-/GDnfQYsltsjRswQhN9fhv3EMw2sCpUdrdxyWDOUK7eyD++r3gRhzgiQgc/x4MAv2i1iuQ4lxO5mvqM3vj4bwA==", - "license": "MIT" - }, - "node_modules/js-tokens": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", - "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "license": "MIT", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, - "node_modules/json-schema": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", - "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", - "license": "(AFL-2.1 OR BSD-3-Clause)" - }, - "node_modules/jsonwebtoken": { - "version": "9.0.3", - "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", - "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==", - "license": "MIT", - "dependencies": { - "jws": "^4.0.1", - "lodash.includes": "^4.3.0", - "lodash.isboolean": "^3.0.3", - "lodash.isinteger": "^4.0.4", - "lodash.isnumber": "^3.0.3", - "lodash.isplainobject": "^4.0.6", - "lodash.isstring": "^4.0.1", - "lodash.once": "^4.0.0", - "ms": "^2.1.1", - "semver": "^7.5.4" - }, - "engines": { - "node": ">=12", - "npm": ">=6" - } - }, - "node_modules/jwa": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", - "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", - "license": "MIT", - "dependencies": { - "buffer-equal-constant-time": "^1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/jws": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", - "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", - "license": "MIT", - "dependencies": { - "jwa": "^2.0.1", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/kuler": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", - "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", - "license": "MIT" - }, - "node_modules/kysely": { - "version": "0.28.11", - "resolved": "https://registry.npmjs.org/kysely/-/kysely-0.28.11.tgz", - "integrity": "sha512-zpGIFg0HuoC893rIjYX1BETkVWdDnzTzF5e0kWXJFg5lE0k1/LfNWBejrcnOFu8Q2Rfq/hTDTU7XLUM8QOrpzg==", - "license": "MIT", - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/lilconfig": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", - "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/antonk52" - } - }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, - "license": "MIT" - }, - "node_modules/load-tsconfig": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/load-tsconfig/-/load-tsconfig-0.2.5.tgz", - "integrity": "sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - } - }, - "node_modules/lodash.camelcase": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", - "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", - "license": "MIT" - }, - "node_modules/lodash.includes": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", - "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", - "license": "MIT" - }, - "node_modules/lodash.isboolean": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", - "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", - "license": "MIT" - }, - "node_modules/lodash.isinteger": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", - "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", - "license": "MIT" - }, - "node_modules/lodash.isnumber": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", - "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", - "license": "MIT" - }, - "node_modules/lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", - "license": "MIT" - }, - "node_modules/lodash.isstring": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", - "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", - "license": "MIT" - }, - "node_modules/lodash.once": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", - "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", - "license": "MIT" - }, - "node_modules/logform": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.7.0.tgz", - "integrity": "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ==", - "license": "MIT", - "dependencies": { - "@colors/colors": "1.6.0", - "@types/triple-beam": "^1.3.2", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "safe-stable-stringify": "^2.3.1", - "triple-beam": "^1.3.0" - }, - "engines": { - "node": ">= 12.0.0" - } - }, - "node_modules/long": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", - "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "license": "Apache-2.0" - }, - "node_modules/lru-cache": { - "version": "11.2.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.5.tgz", - "integrity": "sha512-vFrFJkWtJvJnD5hg+hJvVE8Lh/TcMzKnTgCWmtBipwI5yLX/iX+5UB2tfuyODF5E7k9xEzMdYgGqaSb1c0c5Yw==", - "dev": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/lru.min": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.4.tgz", - "integrity": "sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==", - "license": "MIT", - "engines": { - "bun": ">=1.0.0", - "deno": ">=1.30.0", - "node": ">=8.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wellwelwel" - } - }, - "node_modules/magic-bytes.js": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/magic-bytes.js/-/magic-bytes.js-1.13.0.tgz", - "integrity": "sha512-afO2mnxW7GDTXMm5/AoN1WuOcdoKhtgXjIvHmobqTD1grNplhGdv3PFOyjCVmrnOZBIT/gD/koDKpYG+0mvHcg==", - "license": "MIT" - }, - "node_modules/magic-string": { - "version": "0.30.21", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", - "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.5" - } - }, - "node_modules/magicast": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.2.tgz", - "integrity": "sha512-E3ZJh4J3S9KfwdjZhe2afj6R9lGIN5Pher1pF39UGrXRqq/VDaGVIGN13BjHd2u8B61hArAGOnso7nBOouW3TQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.29.0", - "@babel/types": "^7.29.0", - "source-map-js": "^1.2.1" - } - }, - "node_modules/make-dir": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", - "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", - "dev": true, - "license": "MIT", - "dependencies": { - "semver": "^7.5.3" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/mime": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", - "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/minimatch": { - "version": "10.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.2.tgz", - "integrity": "sha512-fu656aJ0n2kcXwsnwnv9g24tkU5uSmOlTjd6WyyaKm2Z+h1qmY6bAjrcaIxF/BslFqbZ8UBtbJi7KgQOZD2PTw==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/brace-expansion": "^5.0.1" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/mlly": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.0.tgz", - "integrity": "sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "acorn": "^8.15.0", - "pathe": "^2.0.3", - "pkg-types": "^1.3.1", - "ufo": "^1.6.1" - } - }, - "node_modules/mnemonist": { - "version": "0.39.8", - "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.8.tgz", - "integrity": "sha512-vyWo2K3fjrUw8YeeZ1zF0fy6Mu59RHokURlld8ymdUPjMlD9EC9ov1/YPqTgqRvUN9nTr3Gqfz29LYAmu0PHPQ==", - "license": "MIT", - "dependencies": { - "obliterator": "^2.0.1" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/mysql2": { - "version": "3.16.3", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.16.3.tgz", - "integrity": "sha512-+3XhQEt4FEFuvGV0JjIDj4eP2OT/oIj/54dYvqhblnSzlfcxVOuj+cd15Xz6hsG4HU1a+A5+BA9gm0618C4z7A==", - "license": "MIT", - "dependencies": { - "aws-ssl-profiles": "^1.1.2", - "denque": "^2.1.0", - "generate-function": "^2.3.1", - "iconv-lite": "^0.7.2", - "long": "^5.3.2", - "lru.min": "^1.1.3", - "named-placeholders": "^1.1.6", - "seq-queue": "^0.0.5", - "sqlstring": "^2.3.3" - }, - "engines": { - "node": ">= 8.0" - } - }, - "node_modules/mz": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", - "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "any-promise": "^1.0.0", - "object-assign": "^4.0.1", - "thenify-all": "^1.0.0" - } - }, - "node_modules/named-placeholders": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.6.tgz", - "integrity": "sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==", - "license": "MIT", - "dependencies": { - "lru.min": "^1.1.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/nanoid": { - "version": "3.3.11", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", - "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "node_modules/native-duplexpair": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/native-duplexpair/-/native-duplexpair-1.0.0.tgz", - "integrity": "sha512-E7QQoM+3jvNtlmyfqRZ0/U75VFgCls+fSkbml2MpgWkWyz3ox8Y58gNhfuziuQYGNNQAbFZJQck55LHCnCK6CA==", - "license": "MIT" - }, - "node_modules/nice-grpc": { - "version": "2.1.14", - "resolved": "https://registry.npmjs.org/nice-grpc/-/nice-grpc-2.1.14.tgz", - "integrity": "sha512-GK9pKNxlvnU5FAdaw7i2FFuR9CqBspcE+if2tqnKXBcE0R8525wj4BZvfcwj7FjvqbssqKxRHt2nwedalbJlww==", - "license": "MIT", - "dependencies": { - "@grpc/grpc-js": "^1.14.0", - "abort-controller-x": "^0.4.0", - "nice-grpc-common": "^2.0.2" - } - }, - "node_modules/nice-grpc-client-middleware-retry": { - "version": "3.1.13", - "resolved": "https://registry.npmjs.org/nice-grpc-client-middleware-retry/-/nice-grpc-client-middleware-retry-3.1.13.tgz", - "integrity": "sha512-Q9I/wm5lYkDTveKFirrTHBkBY137yavXZ4xQDXTPIycUp7aLXD8xPTHFhqtAFWUw05aS91uffZZRgdv3HS0y/g==", - "license": "MIT", - "dependencies": { - "abort-controller-x": "^0.4.0", - "nice-grpc-common": "^2.0.2" - } - }, - "node_modules/nice-grpc-client-middleware-retry/node_modules/abort-controller-x": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/abort-controller-x/-/abort-controller-x-0.4.3.tgz", - "integrity": "sha512-VtUwTNU8fpMwvWGn4xE93ywbogTYsuT+AUxAXOeelbXuQVIwNmC5YLeho9sH4vZ4ITW8414TTAOG1nW6uIVHCA==", - "license": "MIT" - }, - "node_modules/nice-grpc-common": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/nice-grpc-common/-/nice-grpc-common-2.0.2.tgz", - "integrity": "sha512-7RNWbls5kAL1QVUOXvBsv1uO0wPQK3lHv+cY1gwkTzirnG1Nop4cBJZubpgziNbaVc/bl9QJcyvsf/NQxa3rjQ==", - "license": "MIT", - "dependencies": { - "ts-error": "^1.0.6" - } - }, - "node_modules/nice-grpc/node_modules/abort-controller-x": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/abort-controller-x/-/abort-controller-x-0.4.3.tgz", - "integrity": "sha512-VtUwTNU8fpMwvWGn4xE93ywbogTYsuT+AUxAXOeelbXuQVIwNmC5YLeho9sH4vZ4ITW8414TTAOG1nW6uIVHCA==", - "license": "MIT" - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/obliterator": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.5.tgz", - "integrity": "sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==", - "license": "MIT" - }, - "node_modules/obug": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", - "integrity": "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==", - "dev": true, - "funding": [ - "https://github.com/sponsors/sxzz", - "https://opencollective.com/debug" - ], - "license": "MIT" - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/one-time": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", - "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", - "license": "MIT", - "dependencies": { - "fn.name": "1.x.x" - } - }, - "node_modules/open": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", - "integrity": "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==", - "license": "MIT", - "dependencies": { - "default-browser": "^5.2.1", - "define-lazy-prop": "^3.0.0", - "is-inside-container": "^1.0.0", - "wsl-utils": "^0.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/openapi3-ts": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/openapi3-ts/-/openapi3-ts-4.5.0.tgz", - "integrity": "sha512-jaL+HgTq2Gj5jRcfdutgRGLosCy/hT8sQf6VOy+P+g36cZOjI1iukdPnijC+4CmeRzg/jEllJUboEic2FhxhtQ==", - "license": "MIT", - "dependencies": { - "yaml": "^2.8.0" - } - }, - "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/package-json-from-dist": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", - "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "dev": true, - "license": "BlueOak-1.0.0" - }, - "node_modules/path-scurry": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", - "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, - "node_modules/pg": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.18.0.tgz", - "integrity": "sha512-xqrUDL1b9MbkydY/s+VZ6v+xiMUmOUk7SS9d/1kpyQxoJ6U9AO1oIJyUWVZojbfe5Cc/oluutcgFG4L9RDP1iQ==", - "license": "MIT", - "dependencies": { - "pg-connection-string": "^2.11.0", - "pg-pool": "^3.11.0", - "pg-protocol": "^1.11.0", - "pg-types": "2.2.0", - "pgpass": "1.0.5" - }, - "engines": { - "node": ">= 16.0.0" - }, - "optionalDependencies": { - "pg-cloudflare": "^1.3.0" - }, - "peerDependencies": { - "pg-native": ">=3.0.1" - }, - "peerDependenciesMeta": { - "pg-native": { - "optional": true - } - } - }, - "node_modules/pg-cloudflare": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", - "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", - "license": "MIT", - "optional": true - }, - "node_modules/pg-connection-string": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz", - "integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==", - "license": "MIT" - }, - "node_modules/pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", - "license": "ISC", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-pool": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.11.0.tgz", - "integrity": "sha512-MJYfvHwtGp870aeusDh+hg9apvOe2zmpZJpyt+BMtzUWlVqbhFmMK6bOBXLBUPd7iRtIF9fZplDc7KrPN3PN7w==", - "license": "MIT", - "peerDependencies": { - "pg": ">=8.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.11.0.tgz", - "integrity": "sha512-pfsxk2M9M3BuGgDOfuy37VNRRX3jmKgMjcvAcWqNDpZSf4cUmv8HSOl5ViRQFsfARFn0KuUQTgLxVMbNq5NW3g==", - "license": "MIT" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/pgpass": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", - "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", - "license": "MIT", - "dependencies": { - "split2": "^4.1.0" - } - }, - "node_modules/pgvector": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/pgvector/-/pgvector-0.2.1.tgz", - "integrity": "sha512-nKaQY9wtuiidwLMdVIce1O3kL0d+FxrigCVzsShnoqzOSaWWWOvuctb/sYwlai5cTwwzRSNa+a/NtN2kVZGNJw==", - "license": "MIT", - "engines": { - "node": ">= 18" - } - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pirates": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", - "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, - "node_modules/pkg-types": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.3.1.tgz", - "integrity": "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "confbox": "^0.1.8", - "mlly": "^1.7.4", - "pathe": "^2.0.1" - } - }, - "node_modules/postcss": { - "version": "8.5.6", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", - "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "nanoid": "^3.3.11", - "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } - }, - "node_modules/postcss-load-config": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", - "integrity": "sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "lilconfig": "^3.1.1" - }, - "engines": { - "node": ">= 18" - }, - "peerDependencies": { - "jiti": ">=1.21.0", - "postcss": ">=8.0.9", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "jiti": { - "optional": true - }, - "postcss": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } - } - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", - "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/process": { - "version": "0.11.10", - "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", - "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", - "license": "MIT", - "engines": { - "node": ">= 0.6.0" - } - }, - "node_modules/protobufjs": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", - "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/readdirp": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", - "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14.18.0" - }, - "funding": { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/retry-request": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", - "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", - "license": "MIT", - "dependencies": { - "@types/request": "^2.48.8", - "extend": "^3.0.2", - "teeny-request": "^9.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/rimraf": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-6.1.2.tgz", - "integrity": "sha512-cFCkPslJv7BAXJsYlK1dZsbP8/ZNLkCAQ0bi1hf5EKX2QHegmDFEFA6QhuYJlk7UDdc+02JjO80YSOrWPpw06g==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "glob": "^13.0.0", - "package-json-from-dist": "^1.0.1" - }, - "bin": { - "rimraf": "dist/esm/bin.mjs" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/rollup": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.57.1.tgz", - "integrity": "sha512-oQL6lgK3e2QZeQ7gcgIkS2YZPg5slw37hYufJ3edKlfQSGGm8ICoxswK15ntSzF/a8+h7ekRy7k7oWc3BQ7y8A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "1.0.8" - }, - "bin": { - "rollup": "dist/bin/rollup" - }, - "engines": { - "node": ">=18.0.0", - "npm": ">=8.0.0" - }, - "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.57.1", - "@rollup/rollup-android-arm64": "4.57.1", - "@rollup/rollup-darwin-arm64": "4.57.1", - "@rollup/rollup-darwin-x64": "4.57.1", - "@rollup/rollup-freebsd-arm64": "4.57.1", - "@rollup/rollup-freebsd-x64": "4.57.1", - "@rollup/rollup-linux-arm-gnueabihf": "4.57.1", - "@rollup/rollup-linux-arm-musleabihf": "4.57.1", - "@rollup/rollup-linux-arm64-gnu": "4.57.1", - "@rollup/rollup-linux-arm64-musl": "4.57.1", - "@rollup/rollup-linux-loong64-gnu": "4.57.1", - "@rollup/rollup-linux-loong64-musl": "4.57.1", - "@rollup/rollup-linux-ppc64-gnu": "4.57.1", - "@rollup/rollup-linux-ppc64-musl": "4.57.1", - "@rollup/rollup-linux-riscv64-gnu": "4.57.1", - "@rollup/rollup-linux-riscv64-musl": "4.57.1", - "@rollup/rollup-linux-s390x-gnu": "4.57.1", - "@rollup/rollup-linux-x64-gnu": "4.57.1", - "@rollup/rollup-linux-x64-musl": "4.57.1", - "@rollup/rollup-openbsd-x64": "4.57.1", - "@rollup/rollup-openharmony-arm64": "4.57.1", - "@rollup/rollup-win32-arm64-msvc": "4.57.1", - "@rollup/rollup-win32-ia32-msvc": "4.57.1", - "@rollup/rollup-win32-x64-gnu": "4.57.1", - "@rollup/rollup-win32-x64-msvc": "4.57.1", - "fsevents": "~2.3.2" - } - }, - "node_modules/run-applescript": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz", - "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/safe-stable-stringify": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", - "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, - "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/seq-queue": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz", - "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==" - }, - "node_modules/siginfo": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", - "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", - "dev": true, - "license": "ISC" - }, - "node_modules/source-map": { - "version": "0.7.6", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", - "integrity": "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">= 12" - } - }, - "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/split2": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", - "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", - "license": "ISC", - "engines": { - "node": ">= 10.x" - } - }, - "node_modules/sprintf-js": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", - "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", - "license": "BSD-3-Clause" - }, - "node_modules/sqlstring": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", - "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/stackback": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", - "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", - "dev": true, - "license": "MIT" - }, - "node_modules/std-env": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", - "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", - "dev": true, - "license": "MIT" - }, - "node_modules/stream-events": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", - "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", - "license": "MIT", - "dependencies": { - "stubs": "^3.0.0" - } - }, - "node_modules/stream-shift": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", - "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", - "license": "MIT" - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strnum": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.2.tgz", - "integrity": "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT" - }, - "node_modules/stubs": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", - "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", - "license": "MIT" - }, - "node_modules/sucrase": { - "version": "3.35.1", - "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz", - "integrity": "sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.2", - "commander": "^4.0.0", - "lines-and-columns": "^1.1.6", - "mz": "^2.7.0", - "pirates": "^4.0.1", - "tinyglobby": "^0.2.11", - "ts-interface-checker": "^0.1.9" - }, - "bin": { - "sucrase": "bin/sucrase", - "sucrase-node": "bin/sucrase-node" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/tagged-tag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/tagged-tag/-/tagged-tag-1.0.0.tgz", - "integrity": "sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==", - "license": "MIT", - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/tarn": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", - "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/tedious": { - "version": "19.2.0", - "resolved": "https://registry.npmjs.org/tedious/-/tedious-19.2.0.tgz", - "integrity": "sha512-2dDjX0KP54riDvJPiiIozv0WRS/giJb3/JG2lWpa2dgM0Gha7mLAxbTR3ltPkGzfoS6M3oDnhYnWuzeaZibHuQ==", - "license": "MIT", - "dependencies": { - "@azure/core-auth": "^1.7.2", - "@azure/identity": "^4.2.1", - "@azure/keyvault-keys": "^4.4.0", - "@js-joda/core": "^5.6.5", - "@types/node": ">=18", - "bl": "^6.1.4", - "iconv-lite": "^0.7.0", - "js-md4": "^0.3.2", - "native-duplexpair": "^1.0.0", - "sprintf-js": "^1.1.3" - }, - "engines": { - "node": ">=18.17" - } - }, - "node_modules/teeny-request": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", - "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", - "license": "Apache-2.0", - "dependencies": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^2.6.9", - "stream-events": "^1.0.5", - "uuid": "^9.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/teeny-request/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/teeny-request/node_modules/http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "license": "MIT", - "dependencies": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/teeny-request/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/teeny-request/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/text-hex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", - "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", - "license": "MIT" - }, - "node_modules/thenify": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", - "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", - "dev": true, - "license": "MIT", - "dependencies": { - "any-promise": "^1.0.0" - } - }, - "node_modules/thenify-all": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", - "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", - "dev": true, - "license": "MIT", - "dependencies": { - "thenify": ">= 3.1.0 < 4" - }, - "engines": { - "node": ">=0.8" - } - }, - "node_modules/tinybench": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", - "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", - "dev": true, - "license": "MIT" - }, - "node_modules/tinyexec": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", - "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", - "dev": true, - "license": "MIT" - }, - "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "fdir": "^6.5.0", - "picomatch": "^4.0.3" - }, - "engines": { - "node": ">=12.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/SuperchupuDev" - } - }, - "node_modules/tinyrainbow": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.0.3.tgz", - "integrity": "sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, - "node_modules/tree-kill": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", - "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", - "dev": true, - "license": "MIT", - "bin": { - "tree-kill": "cli.js" - } - }, - "node_modules/triple-beam": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", - "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", - "license": "MIT", - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/ts-error": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/ts-error/-/ts-error-1.0.6.tgz", - "integrity": "sha512-tLJxacIQUM82IR7JO1UUkKlYuUTmoY9HBJAmNWFzheSlDS5SPMcNIepejHJa4BpPQLAcbRhRf3GDJzyj6rbKvA==", - "license": "MIT" - }, - "node_modules/ts-interface-checker": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", - "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", - "dev": true, - "license": "Apache-2.0" - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" - }, - "node_modules/tsup": { - "version": "8.5.1", - "resolved": "https://registry.npmjs.org/tsup/-/tsup-8.5.1.tgz", - "integrity": "sha512-xtgkqwdhpKWr3tKPmCkvYmS9xnQK3m3XgxZHwSUjvfTjp7YfXe5tT3GgWi0F2N+ZSMsOeWeZFh7ZZFg5iPhing==", - "dev": true, - "license": "MIT", - "dependencies": { - "bundle-require": "^5.1.0", - "cac": "^6.7.14", - "chokidar": "^4.0.3", - "consola": "^3.4.0", - "debug": "^4.4.0", - "esbuild": "^0.27.0", - "fix-dts-default-cjs-exports": "^1.0.0", - "joycon": "^3.1.1", - "picocolors": "^1.1.1", - "postcss-load-config": "^6.0.1", - "resolve-from": "^5.0.0", - "rollup": "^4.34.8", - "source-map": "^0.7.6", - "sucrase": "^3.35.0", - "tinyexec": "^0.3.2", - "tinyglobby": "^0.2.11", - "tree-kill": "^1.2.2" - }, - "bin": { - "tsup": "dist/cli-default.js", - "tsup-node": "dist/cli-node.js" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@microsoft/api-extractor": "^7.36.0", - "@swc/core": "^1", - "postcss": "^8.4.12", - "typescript": ">=4.5.0" - }, - "peerDependenciesMeta": { - "@microsoft/api-extractor": { - "optional": true - }, - "@swc/core": { - "optional": true - }, - "postcss": { - "optional": true - }, - "typescript": { - "optional": true - } - } - }, - "node_modules/type-fest": { - "version": "5.4.3", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.4.3.tgz", - "integrity": "sha512-AXSAQJu79WGc79/3e9/CR77I/KQgeY1AhNvcShIH4PTcGYyC4xv6H4R4AUOwkPS5799KlVDAu8zExeCrkGquiA==", - "license": "(MIT OR CC0-1.0)", - "dependencies": { - "tagged-tag": "^1.0.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/ufo": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.3.tgz", - "integrity": "sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/undici": { - "version": "6.23.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-6.23.0.tgz", - "integrity": "sha512-VfQPToRA5FZs/qJxLIinmU59u0r7LXqoJkCzinq3ckNJp3vKEh7jTWN589YQ5+aoAC/TGRLyJLCPKcLQbM8r9g==", - "license": "MIT", - "engines": { - "node": ">=18.17" - } - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "license": "MIT" - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "license": "MIT" - }, - "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/vite": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.1.tgz", - "integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==", - "dev": true, - "license": "MIT", - "dependencies": { - "esbuild": "^0.27.0", - "fdir": "^6.5.0", - "picomatch": "^4.0.3", - "postcss": "^8.5.6", - "rollup": "^4.43.0", - "tinyglobby": "^0.2.15" - }, - "bin": { - "vite": "bin/vite.js" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "funding": { - "url": "https://github.com/vitejs/vite?sponsor=1" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - }, - "peerDependencies": { - "@types/node": "^20.19.0 || >=22.12.0", - "jiti": ">=1.21.0", - "less": "^4.0.0", - "lightningcss": "^1.21.0", - "sass": "^1.70.0", - "sass-embedded": "^1.70.0", - "stylus": ">=0.54.8", - "sugarss": "^5.0.0", - "terser": "^5.16.0", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "jiti": { - "optional": true - }, - "less": { - "optional": true - }, - "lightningcss": { - "optional": true - }, - "sass": { - "optional": true - }, - "sass-embedded": { - "optional": true - }, - "stylus": { - "optional": true - }, - "sugarss": { - "optional": true - }, - "terser": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } - } - }, - "node_modules/vitest": { - "version": "4.0.18", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.0.18.tgz", - "integrity": "sha512-hOQuK7h0FGKgBAas7v0mSAsnvrIgAvWmRFjmzpJ7SwFHH3g1k2u37JtYwOwmEKhK6ZO3v9ggDBBm0La1LCK4uQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/expect": "4.0.18", - "@vitest/mocker": "4.0.18", - "@vitest/pretty-format": "4.0.18", - "@vitest/runner": "4.0.18", - "@vitest/snapshot": "4.0.18", - "@vitest/spy": "4.0.18", - "@vitest/utils": "4.0.18", - "es-module-lexer": "^1.7.0", - "expect-type": "^1.2.2", - "magic-string": "^0.30.21", - "obug": "^2.1.1", - "pathe": "^2.0.3", - "picomatch": "^4.0.3", - "std-env": "^3.10.0", - "tinybench": "^2.9.0", - "tinyexec": "^1.0.2", - "tinyglobby": "^0.2.15", - "tinyrainbow": "^3.0.3", - "vite": "^6.0.0 || ^7.0.0", - "why-is-node-running": "^2.3.0" - }, - "bin": { - "vitest": "vitest.mjs" - }, - "engines": { - "node": "^20.0.0 || ^22.0.0 || >=24.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "@edge-runtime/vm": "*", - "@opentelemetry/api": "^1.9.0", - "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.0.18", - "@vitest/browser-preview": "4.0.18", - "@vitest/browser-webdriverio": "4.0.18", - "@vitest/ui": "4.0.18", - "happy-dom": "*", - "jsdom": "*" - }, - "peerDependenciesMeta": { - "@edge-runtime/vm": { - "optional": true - }, - "@opentelemetry/api": { - "optional": true - }, - "@types/node": { - "optional": true - }, - "@vitest/browser-playwright": { - "optional": true - }, - "@vitest/browser-preview": { - "optional": true - }, - "@vitest/browser-webdriverio": { - "optional": true - }, - "@vitest/ui": { - "optional": true - }, - "happy-dom": { - "optional": true - }, - "jsdom": { - "optional": true - } - } - }, - "node_modules/vitest/node_modules/tinyexec": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.2.tgz", - "integrity": "sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/weaviate-client": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/weaviate-client/-/weaviate-client-3.11.0.tgz", - "integrity": "sha512-pEO+V8OZ84KUKz9ftQnSuooCT4Fdh3SjkDj6FPfxI3Iy6qc+PTTAMFFO0wL2prnf71DIs0tdNw/1RFX4kJkE/w==", - "license": "BSD-3-Clause", - "dependencies": { - "@datastructures-js/deque": "^1.0.8", - "abort-controller-x": "^0.5.0", - "graphql": "^16.12.0", - "graphql-request": "^6.1.0", - "long": "^5.3.2", - "nice-grpc": "^2.1.14", - "nice-grpc-client-middleware-retry": "^3.1.13", - "nice-grpc-common": "^2.0.2", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/weaviate-client/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/why-is-node-running": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", - "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", - "dev": true, - "license": "MIT", - "dependencies": { - "siginfo": "^2.0.0", - "stackback": "0.0.2" - }, - "bin": { - "why-is-node-running": "cli.js" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/winston": { - "version": "3.19.0", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz", - "integrity": "sha512-LZNJgPzfKR+/J3cHkxcpHKpKKvGfDZVPS4hfJCc4cCG0CgYzvlD6yE/S3CIL/Yt91ak327YCpiF/0MyeZHEHKA==", - "license": "MIT", - "dependencies": { - "@colors/colors": "^1.6.0", - "@dabh/diagnostics": "^2.0.8", - "async": "^3.2.3", - "is-stream": "^2.0.0", - "logform": "^2.7.0", - "one-time": "^1.0.0", - "readable-stream": "^3.4.0", - "safe-stable-stringify": "^2.3.1", - "stack-trace": "0.0.x", - "triple-beam": "^1.3.0", - "winston-transport": "^4.9.0" - }, - "engines": { - "node": ">= 12.0.0" - } - }, - "node_modules/winston-transport": { - "version": "4.9.0", - "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.9.0.tgz", - "integrity": "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A==", - "license": "MIT", - "dependencies": { - "logform": "^2.7.0", - "readable-stream": "^3.6.2", - "triple-beam": "^1.3.0" - }, - "engines": { - "node": ">= 12.0.0" - } - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC" - }, - "node_modules/ws": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", - "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/wsl-utils": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.1.0.tgz", - "integrity": "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==", - "license": "MIT", - "dependencies": { - "is-wsl": "^3.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "license": "MIT", - "engines": { - "node": ">=0.4" - } - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yaml": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", - "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", - "license": "ISC", - "bin": { - "yaml": "bin.mjs" - }, - "engines": { - "node": ">= 14.6" - }, - "funding": { - "url": "https://github.com/sponsors/eemeli" - } - }, - "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "license": "MIT", - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/zod": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", - "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "packages/nvisy-core": { - "name": "@nvisy/core", - "version": "0.1.0", - "dependencies": { - "@logtape/logtape": "^2.0.2", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "packages/nvisy-plugin-ai": { - "name": "@nvisy/plugin-ai", - "version": "0.1.0", - "dependencies": { - "@ai-sdk/anthropic": "^3.0.36", - "@ai-sdk/google": "^3.0.20", - "@ai-sdk/openai": "^3.0.25", - "@logtape/logtape": "^2.0.2", - "@nvisy/core": "*", - "ai": "^6.0.69", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "packages/nvisy-plugin-object": { - "name": "@nvisy/plugin-object", - "version": "0.1.0", - "dependencies": { - "@aws-sdk/client-s3": "^3.750.0", - "@azure/storage-blob": "^12.26.0", - "@google-cloud/storage": "^7.15.0", - "@logtape/logtape": "^2.0.2", - "@nvisy/core": "*", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "packages/nvisy-plugin-pandoc": { - "name": "@nvisy/plugin-pandoc", - "version": "0.1.0", - "dependencies": { - "@logtape/logtape": "^2.0.2", - "@nvisy/core": "*", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "packages/nvisy-plugin-sql": { - "name": "@nvisy/plugin-sql", - "version": "0.1.0", - "dependencies": { - "@logtape/logtape": "^2.0.2", - "@nvisy/core": "*", - "kysely": "^0.28.11", - "mysql2": "^3.16.3", - "pg": "^8.18.0", - "tarn": "^3.0.2", - "tedious": "^19.2.0", - "zod": "^4.3.6" - }, - "devDependencies": { - "@types/pg": "^8.16.0" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "packages/nvisy-plugin-vector": { - "name": "@nvisy/plugin-vector", - "version": "0.1.0", - "dependencies": { - "@logtape/logtape": "^2.0.2", - "@nvisy/core": "*", - "@pinecone-database/pinecone": "^4.0.0", - "@qdrant/js-client-rest": "^1.13.0", - "@zilliz/milvus2-sdk-node": "^2.5.0", - "pg": "^8.13.0", - "pgvector": "^0.2.0", - "weaviate-client": "^3.5.0", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "packages/nvisy-runtime": { - "name": "@nvisy/runtime", - "version": "0.1.0", - "dependencies": { - "@logtape/logtape": "^2.0.2", - "@nvisy/core": "*", - "effection": "^4.0.2", - "graphology": "^0.26.0", - "graphology-dag": "^0.4.1", - "graphology-types": "^0.24.8", - "magic-bytes.js": "^1.13.0", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } - }, - "packages/nvisy-server": { - "name": "@nvisy/server", - "version": "0.1.0", - "dependencies": { - "@hono/event-emitter": "^2.0.0", - "@hono/node-server": "^1.19.9", - "@hono/node-ws": "^1.3.0", - "@hono/otel": "^1.1.0", - "@hono/zod-openapi": "^1.2.1", - "@hono/zod-validator": "^0.7.6", - "@logtape/hono": "^2.0.2", - "@logtape/logtape": "^2.0.2", - "@logtape/pretty": "^2.0.2", - "@logtape/redaction": "^2.0.2", - "@nvisy/core": "*", - "@nvisy/plugin-ai": "*", - "@nvisy/plugin-object": "*", - "@nvisy/plugin-pandoc": "*", - "@nvisy/plugin-sql": "*", - "@nvisy/plugin-vector": "*", - "@nvisy/runtime": "*", - "@scalar/hono-api-reference": "^0.9.40", - "hono": "^4.11.7", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } - } - } -} diff --git a/package.json b/package.json deleted file mode 100644 index 3efa9d9..0000000 --- a/package.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "name": "@nvisy/monorepo", - "private": true, - "type": "module", - "engines": { - "node": ">=22.0.0" - }, - "workspaces": [ - "packages/*" - ], - "scripts": { - "build": "npm run build --workspaces", - "test": "vitest", - "test:coverage": "vitest --coverage", - "lint": "biome lint .", - "lint:fix": "biome lint --write .", - "format": "biome format --write .", - "format:check": "biome format .", - "check": "biome check .", - "check:fix": "biome check --write .", - "typecheck": "npm run typecheck --workspaces", - "clean": "npm run clean --workspaces && rimraf node_modules" - }, - "overrides": { - "@zilliz/milvus2-sdk-node": { - "@grpc/grpc-js": "^1.8.22" - } - }, - "devDependencies": { - "@biomejs/biome": "^2.3.14", - "@types/node": "^25.2.0", - "@vitest/coverage-v8": "^4.0.18", - "rimraf": "^6.1.2", - "tsup": "^8.5.1", - "typescript": "^5.9.3", - "vitest": "^4.0.18" - } -} diff --git a/packages/README.md b/packages/README.md deleted file mode 100644 index 0b3b5ba..0000000 --- a/packages/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Packages - -[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/build.yml) - -The monorepo is organized around a shared core, a runtime engine, a set of -plugin packages, and an HTTP server. `nvisy-core` is the shared foundation -with no internal dependencies. - -## nvisy-core - -Shared primitives, type system, validation, error taxonomy, and base -interfaces for sources, sinks, and actions. Also houses core observability -utilities (structured logging, metrics, tracing). Every other package depends -on this one. - -## nvisy-runtime - -Graph definition, JSON parser, DAG compiler, and execution engine. Parses JSON -graph definitions into an immutable `ExecutionPlan`, then executes it — -walking the DAG in topological order, managing concurrency via Effection -structured concurrency, enforcing per-connector rate limits, retrying failed nodes with -configurable backoff, and emitting runtime metrics and OpenTelemetry traces. - -## nvisy-plugin-sql - -Source and Sink implementations for relational databases. Targets PostgreSQL -and MySQL. Handles connection pooling, query generation, type mapping, and -batch insert/upsert operations. - -## nvisy-plugin-object - -Source and Sink implementations for object stores and file formats. Targets S3, -GCS, Parquet, JSONL, and CSV. Handles multipart uploads, streaming reads, -prefix-based listing, schema inference, and chunked reading. - -## nvisy-plugin-vector - -Source and Sink implementations for vector databases. Targets Pinecone, Qdrant, -Milvus, Weaviate, and pgvector. Handles collection/index management, upsert -with metadata, batch operations, and dimensionality validation. - -## nvisy-server - -HTTP server built on Hono. Exposes a REST API for graph CRUD, run -management, connector health checks, and lineage queries. Includes a cron -scheduler, webhook-based event triggers, and server-level observability -(request logging, health endpoints, metric export). - diff --git a/packages/nvisy-ai/pyproject.toml b/packages/nvisy-ai/pyproject.toml new file mode 100644 index 0000000..0065a46 --- /dev/null +++ b/packages/nvisy-ai/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "nvisy-ai" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [ + "openai>=1.0", + "anthropic>=0.30", + "google-generativeai>=0.7", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.backends" + +[tool.hatch.build.targets.wheel] +packages = ["src/nvisy_ai"] diff --git a/packages/nvisy-ai/src/nvisy_ai/__init__.py b/packages/nvisy-ai/src/nvisy_ai/__init__.py new file mode 100644 index 0000000..c1bf9cd --- /dev/null +++ b/packages/nvisy-ai/src/nvisy_ai/__init__.py @@ -0,0 +1,5 @@ +"""Nvisy AI NER detection package.""" + +from nvisy_ai.ner import detect_ner, detect_ner_image + +__all__ = ["detect_ner", "detect_ner_image"] diff --git a/packages/nvisy-ai/src/nvisy_ai/ner.py b/packages/nvisy-ai/src/nvisy_ai/ner.py new file mode 100644 index 0000000..3e4076c --- /dev/null +++ b/packages/nvisy-ai/src/nvisy_ai/ner.py @@ -0,0 +1,145 @@ +"""NER detection functions called from Rust via PyO3.""" + +import json +from typing import Optional + +from .prompts import NER_SYSTEM_PROMPT, NER_IMAGE_SYSTEM_PROMPT +from .providers.base import CompletionClient + + +def _get_client(provider: str, api_key: str, model: str) -> CompletionClient: + """Create a completion client for the given provider.""" + if provider == "openai": + from .providers.openai import OpenAIClient + return OpenAIClient(api_key=api_key, model=model) + elif provider == "anthropic": + from .providers.anthropic import AnthropicClient + return AnthropicClient(api_key=api_key, model=model) + elif provider == "gemini": + from .providers.gemini import GeminiClient + return GeminiClient(api_key=api_key, model=model) + else: + raise ValueError(f"Unknown provider: {provider}") + + +def _parse_entities(response_text: str) -> list[dict]: + """Parse the JSON response from the LLM into entity dicts.""" + text = response_text.strip() + # Strip markdown code fences if present + if text.startswith("```"): + lines = text.split("\n") + lines = [l for l in lines if not l.startswith("```")] + text = "\n".join(lines) + + try: + entities = json.loads(text) + except json.JSONDecodeError: + return [] + + if not isinstance(entities, list): + return [] + + return entities + + +def detect_ner( + text: str, + entity_types: Optional[list[str]] = None, + confidence_threshold: float = 0.5, + temperature: float = 0.0, + api_key: str = "", + model: str = "gpt-4", + provider: str = "openai", +) -> list[dict]: + """Detect named entities in text using an LLM. + + Called from Rust via PyO3. + + Args: + text: The text to analyze. + entity_types: Optional list of entity types to detect. + confidence_threshold: Minimum confidence to include. + temperature: LLM temperature parameter. + api_key: API key for the provider. + model: Model name to use. + provider: Provider name ("openai", "anthropic", "gemini"). + + Returns: + List of entity dicts with keys: category, entity_type, value, + confidence, start_offset, end_offset. + """ + import asyncio + + client = _get_client(provider, api_key, model) + + user_prompt = f"Analyze the following text for sensitive data:\n\n{text}" + if entity_types: + user_prompt += f"\n\nOnly detect these entity types: {', '.join(entity_types)}" + + loop = asyncio.new_event_loop() + try: + response = loop.run_until_complete( + client.complete(NER_SYSTEM_PROMPT, user_prompt, temperature) + ) + finally: + loop.close() + + entities = _parse_entities(response) + + # Filter by confidence threshold + return [ + e for e in entities + if e.get("confidence", 0) >= confidence_threshold + ] + + +def detect_ner_image( + image_bytes: bytes, + mime_type: str, + entity_types: Optional[list[str]] = None, + confidence_threshold: float = 0.5, + temperature: float = 0.0, + api_key: str = "", + model: str = "gpt-4", + provider: str = "openai", +) -> list[dict]: + """Detect named entities in an image using a multimodal LLM. + + Called from Rust via PyO3. + + Args: + image_bytes: Raw image bytes. + mime_type: MIME type of the image. + entity_types: Optional list of entity types to detect. + confidence_threshold: Minimum confidence to include. + api_key: API key for the provider. + model: Model name to use. + provider: Provider name ("openai", "anthropic", "gemini"). + + Returns: + List of entity dicts. + """ + import asyncio + + client = _get_client(provider, api_key, model) + + user_prompt = "Analyze this image for any visible sensitive data." + if entity_types: + user_prompt += f"\n\nOnly detect these entity types: {', '.join(entity_types)}" + + loop = asyncio.new_event_loop() + try: + response = loop.run_until_complete( + client.complete_with_image( + NER_IMAGE_SYSTEM_PROMPT, image_bytes, mime_type, user_prompt, temperature + ) + ) + finally: + loop.close() + + entities = _parse_entities(response) + + return [ + e for e in entities + if e.get("confidence", 0) >= confidence_threshold + ] diff --git a/packages/nvisy-ai/src/nvisy_ai/prompts.py b/packages/nvisy-ai/src/nvisy_ai/prompts.py new file mode 100644 index 0000000..fc608cc --- /dev/null +++ b/packages/nvisy-ai/src/nvisy_ai/prompts.py @@ -0,0 +1,37 @@ +"""NER system prompts for AI-based entity detection.""" + +NER_SYSTEM_PROMPT = """You are a Named Entity Recognition (NER) system specialized in detecting sensitive data. + +Given text, identify all instances of sensitive data including: +- PII: names, addresses, dates of birth, Social Security numbers, phone numbers, email addresses +- PHI: medical record numbers, health plan IDs, diagnoses, medications +- Financial: credit card numbers, bank account numbers, tax IDs +- Credentials: API keys, passwords, tokens, secret keys + +For each entity found, provide: +1. category: one of "pii", "phi", "financial", "credentials", "custom" +2. entity_type: specific type (e.g. "name", "ssn", "email", "credit_card") +3. value: the exact text matched +4. confidence: float 0-1 indicating detection confidence +5. start_offset: character offset where entity starts in the text +6. end_offset: character offset where entity ends in the text + +Return a JSON array of objects. If no entities found, return []. +Only return the JSON array, no additional text.""" + +NER_IMAGE_SYSTEM_PROMPT = """You are a Named Entity Recognition (NER) system that analyzes images for sensitive data. + +Examine the provided image and identify any visible sensitive data including: +- PII: names, addresses, dates of birth, Social Security numbers, phone numbers, email addresses +- PHI: medical record numbers, health plan IDs, diagnoses, medications +- Financial: credit card numbers, bank account numbers, tax IDs +- Credentials: API keys, passwords, tokens, secret keys + +For each entity found, provide: +1. category: one of "pii", "phi", "financial", "credentials", "custom" +2. entity_type: specific type (e.g. "name", "ssn", "email", "credit_card") +3. value: the exact text detected +4. confidence: float 0-1 indicating detection confidence + +Return a JSON array of objects. If no entities found, return []. +Only return the JSON array, no additional text.""" diff --git a/packages/nvisy-ai/src/nvisy_ai/providers/__init__.py b/packages/nvisy-ai/src/nvisy_ai/providers/__init__.py new file mode 100644 index 0000000..f40c051 --- /dev/null +++ b/packages/nvisy-ai/src/nvisy_ai/providers/__init__.py @@ -0,0 +1 @@ +"""AI provider implementations.""" diff --git a/packages/nvisy-ai/src/nvisy_ai/providers/anthropic.py b/packages/nvisy-ai/src/nvisy_ai/providers/anthropic.py new file mode 100644 index 0000000..0c741c5 --- /dev/null +++ b/packages/nvisy-ai/src/nvisy_ai/providers/anthropic.py @@ -0,0 +1,61 @@ +"""Anthropic completion provider.""" + +import base64 +from anthropic import Anthropic +from .base import CompletionClient + + +class AnthropicClient(CompletionClient): + """Anthropic-based completion client.""" + + def __init__(self, api_key: str, model: str = "claude-sonnet-4-5-20250929"): + self._client = Anthropic(api_key=api_key) + self._model = model + + async def complete( + self, + system_prompt: str, + user_prompt: str, + temperature: float = 0.0, + ) -> str: + response = self._client.messages.create( + model=self._model, + max_tokens=4096, + temperature=temperature, + system=system_prompt, + messages=[{"role": "user", "content": user_prompt}], + ) + return response.content[0].text if response.content else "" + + async def complete_with_image( + self, + system_prompt: str, + image_bytes: bytes, + mime_type: str, + user_prompt: str, + temperature: float = 0.0, + ) -> str: + b64 = base64.b64encode(image_bytes).decode("utf-8") + response = self._client.messages.create( + model=self._model, + max_tokens=4096, + temperature=temperature, + system=system_prompt, + messages=[ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": mime_type, + "data": b64, + }, + }, + {"type": "text", "text": user_prompt}, + ], + }, + ], + ) + return response.content[0].text if response.content else "" diff --git a/packages/nvisy-ai/src/nvisy_ai/providers/base.py b/packages/nvisy-ai/src/nvisy_ai/providers/base.py new file mode 100644 index 0000000..51021ec --- /dev/null +++ b/packages/nvisy-ai/src/nvisy_ai/providers/base.py @@ -0,0 +1,30 @@ +"""Abstract base class for AI completion providers.""" + +from abc import ABC, abstractmethod +from typing import Any + + +class CompletionClient(ABC): + """Abstract completion client for LLM providers.""" + + @abstractmethod + async def complete( + self, + system_prompt: str, + user_prompt: str, + temperature: float = 0.0, + ) -> str: + """Send a completion request and return the response text.""" + ... + + @abstractmethod + async def complete_with_image( + self, + system_prompt: str, + image_bytes: bytes, + mime_type: str, + user_prompt: str, + temperature: float = 0.0, + ) -> str: + """Send a multimodal completion request with an image.""" + ... diff --git a/packages/nvisy-ai/src/nvisy_ai/providers/gemini.py b/packages/nvisy-ai/src/nvisy_ai/providers/gemini.py new file mode 100644 index 0000000..ac0a8c5 --- /dev/null +++ b/packages/nvisy-ai/src/nvisy_ai/providers/gemini.py @@ -0,0 +1,39 @@ +"""Google Gemini completion provider.""" + +import google.generativeai as genai +from .base import CompletionClient + + +class GeminiClient(CompletionClient): + """Google Gemini-based completion client.""" + + def __init__(self, api_key: str, model: str = "gemini-1.5-pro"): + genai.configure(api_key=api_key) + self._model = genai.GenerativeModel(model) + + async def complete( + self, + system_prompt: str, + user_prompt: str, + temperature: float = 0.0, + ) -> str: + response = self._model.generate_content( + f"{system_prompt}\n\n{user_prompt}", + generation_config=genai.types.GenerationConfig(temperature=temperature), + ) + return response.text or "" + + async def complete_with_image( + self, + system_prompt: str, + image_bytes: bytes, + mime_type: str, + user_prompt: str, + temperature: float = 0.0, + ) -> str: + image_part = {"mime_type": mime_type, "data": image_bytes} + response = self._model.generate_content( + [f"{system_prompt}\n\n{user_prompt}", image_part], + generation_config=genai.types.GenerationConfig(temperature=temperature), + ) + return response.text or "" diff --git a/packages/nvisy-ai/src/nvisy_ai/providers/openai.py b/packages/nvisy-ai/src/nvisy_ai/providers/openai.py new file mode 100644 index 0000000..60bd501 --- /dev/null +++ b/packages/nvisy-ai/src/nvisy_ai/providers/openai.py @@ -0,0 +1,59 @@ +"""OpenAI completion provider.""" + +import base64 +from openai import OpenAI +from .base import CompletionClient + + +class OpenAIClient(CompletionClient): + """OpenAI-based completion client.""" + + def __init__(self, api_key: str, model: str = "gpt-4"): + self._client = OpenAI(api_key=api_key) + self._model = model + + async def complete( + self, + system_prompt: str, + user_prompt: str, + temperature: float = 0.0, + ) -> str: + response = self._client.chat.completions.create( + model=self._model, + temperature=temperature, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + ) + return response.choices[0].message.content or "" + + async def complete_with_image( + self, + system_prompt: str, + image_bytes: bytes, + mime_type: str, + user_prompt: str, + temperature: float = 0.0, + ) -> str: + b64 = base64.b64encode(image_bytes).decode("utf-8") + response = self._client.chat.completions.create( + model=self._model, + temperature=temperature, + messages=[ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:{mime_type};base64,{b64}", + }, + }, + {"type": "text", "text": user_prompt}, + ], + }, + ], + ) + return response.choices[0].message.content or "" diff --git a/packages/nvisy-core/README.md b/packages/nvisy-core/README.md deleted file mode 100644 index 29464c2..0000000 --- a/packages/nvisy-core/README.md +++ /dev/null @@ -1,124 +0,0 @@ -# @nvisy/core - -[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/build.yml) - -Core primitives and abstractions for the Nvisy runtime platform. - -## Features - -- **Data types**: `Document`, `Embedding`, `Blob`, and `Entry` for pipeline data -- **Module system**: bundle providers, streams, and actions under a namespace -- **Provider abstraction**: connection lifecycle management with credential validation -- **Stream contracts**: resumable sources and sinks for external systems -- **Action contracts**: stream transforms with optional client dependencies -- **Error taxonomy**: `RuntimeError`, `ValidationError`, `ConnectionError`, `CancellationError` - -## Overview - -This package defines the foundational abstractions that all Nvisy modules implement: - -- **Data types** (`Data`, `Document`, `Embedding`, `Blob`, `Entry`): immutable data containers that flow through pipelines. -- **Modules** (`Module.define`): namespace for grouping providers, streams, and actions. -- **Providers** (`Provider.withAuthentication`, `Provider.withoutAuthentication`): external client lifecycle management. -- **Streams** (`Stream.createSource`, `Stream.createTarget`): data I/O layer for reading from and writing to external systems. -- **Actions** (`Action.withClient`, `Action.withoutClient`): stream transforms that process data between sources and targets. - -## Usage - -### Defining a Provider - -```ts -import { Provider } from "@nvisy/core"; -import { z } from "zod"; - -const myProvider = Provider.withAuthentication("my-provider", { - credentials: z.object({ - apiKey: z.string(), - endpoint: z.string().url(), - }), - connect: async (creds) => { - const client = await createClient(creds); - return { - client, - disconnect: () => client.close(), - }; - }, -}); -``` - -### Defining a Stream Source - -```ts -import { Stream, Entry } from "@nvisy/core"; -import { z } from "zod"; - -const mySource = Stream.createSource("my-source", MyClient, { - types: [Entry, z.object({ cursor: z.string().optional() }), z.object({ limit: z.number() })], - reader: async function* (client, ctx, params) { - for await (const item of client.list({ cursor: ctx.cursor, limit: params.limit })) { - yield { data: new Entry(item), context: { cursor: item.id } }; - } - }, -}); -``` - -### Defining a Stream Target - -```ts -import { Stream, Entry } from "@nvisy/core"; -import { z } from "zod"; - -const myTarget = Stream.createTarget("my-target", MyClient, { - types: [Entry, z.object({ collection: z.string() })], - writer: (client, params) => async (item) => { - await client.insert(params.collection, item.fields); - }, -}); -``` - -### Defining an Action - -```ts -import { Action, Entry } from "@nvisy/core"; -import { z } from "zod"; - -const myFilter = Action.withoutClient("my-filter", { - types: [Entry], - params: z.object({ minValue: z.number() }), - transform: async function* (stream, params) { - for await (const entry of stream) { - if ((entry.get("value") as number) >= params.minValue) { - yield entry; - } - } - }, -}); -``` - -### Bundling into a Module - -```ts -import { Module } from "@nvisy/core"; - -const myModule = Module.define("my-module") - .withProviders(myProvider) - .withStreams(mySource, myTarget) - .withActions(myFilter); - -// Register with the engine -registry.load(myModule); -``` - -## Changelog - -See [CHANGELOG.md](../../CHANGELOG.md) for release notes and version history. - -## License - -Apache 2.0 License - see [LICENSE.txt](../../LICENSE.txt) - -## Support - -- **Documentation**: [docs.nvisy.com](https://docs.nvisy.com) -- **Issues**: [GitHub Issues](https://github.com/nvisycom/runtime/issues) -- **Email**: [support@nvisy.com](mailto:support@nvisy.com) diff --git a/packages/nvisy-core/package.json b/packages/nvisy-core/package.json deleted file mode 100644 index 565b7b6..0000000 --- a/packages/nvisy-core/package.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "@nvisy/core", - "version": "0.1.0", - "description": "Core data types, errors, and utilities for the Nvisy runtime", - "type": "module", - "exports": { - ".": { - "source": "./src/index.ts", - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsup", - "build:watch": "tsup --watch", - "clean": "rimraf dist", - "typecheck": "tsc -b" - }, - "dependencies": { - "@logtape/logtape": "^2.0.2", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } -} diff --git a/packages/nvisy-core/src/action.ts b/packages/nvisy-core/src/action.ts deleted file mode 100644 index e9ade5a..0000000 --- a/packages/nvisy-core/src/action.ts +++ /dev/null @@ -1,235 +0,0 @@ -import type { z } from "zod"; -import type { Data } from "./datatypes/data.js"; -import type { ClassRef } from "./types.js"; - -/** - * Stream transform that operates without a provider client. - * - * @template TIn - Input data type consumed by the transform. - * @template TOut - Output data type produced by the transform. - * @template TParam - Configuration parameters for the transform. - */ -export type ClientlessTransformFn< - TIn extends Data, - TOut extends Data, - TParam, -> = (stream: AsyncIterable, params: TParam) => AsyncIterable; - -/** - * Stream transform that requires a provider client. - * - * @template TClient - Provider client type (e.g. database connection). - * @template TIn - Input data type consumed by the transform. - * @template TOut - Output data type produced by the transform. - * @template TParam - Configuration parameters for the transform. - */ -export type ClientTransformFn< - TClient, - TIn extends Data, - TOut extends Data, - TParam, -> = ( - stream: AsyncIterable, - params: TParam, - client: TClient, -) => AsyncIterable; - -/** - * Configuration for creating an action that does not require a provider client. - * - * @template TIn - Input data type consumed by the action. - * @template TOut - Output data type produced by the action. - * @template TParam - Configuration parameters for the action. - */ -export interface ClientlessActionConfig< - TIn extends Data, - TOut extends Data, - TParam, -> { - /** Input/output type classes. Single-element array means input equals output. */ - readonly types: - | [inputClass: ClassRef, outputClass: ClassRef] - | [inputClass: ClassRef]; - /** Zod schema for validating action parameters. */ - readonly params: z.ZodType; - /** The transform function that processes the stream. */ - readonly transform: ClientlessTransformFn; -} - -/** - * Configuration for creating an action that requires a provider client. - * - * @template TClient - Provider client type required by the action. - * @template TIn - Input data type consumed by the action. - * @template TOut - Output data type produced by the action. - * @template TParam - Configuration parameters for the action. - */ -export interface ClientActionConfig< - TClient, - TIn extends Data, - TOut extends Data, - TParam, -> { - /** Input/output type classes. Single-element array means input equals output. */ - readonly types: - | [inputClass: ClassRef, outputClass: ClassRef] - | [inputClass: ClassRef]; - /** Zod schema for validating action parameters. */ - readonly params: z.ZodType; - /** The transform function that processes the stream with client access. */ - readonly transform: ClientTransformFn; -} - -/** - * A registered action instance that can transform data streams. - * - * Actions are the intermediate processing steps in a pipeline, - * transforming data between sources and targets. - * - * @template TClient - Provider client type (void if no client needed). - * @template TIn - Input data type consumed by the action. - * @template TOut - Output data type produced by the action. - * @template TParam - Configuration parameters for the action. - */ -export interface ActionInstance< - TClient = void, - TIn extends Data = Data, - TOut extends Data = Data, - TParam = unknown, -> { - /** Unique identifier for this action. */ - readonly id: string; - /** Client class required by this action (undefined if clientless). */ - readonly clientClass?: ClassRef; - /** Class reference for validating input data type. */ - readonly inputClass: ClassRef; - /** Class reference for validating output data type. */ - readonly outputClass: ClassRef; - /** Zod schema for validating action parameters. */ - readonly schema: z.ZodType; - /** Transform an input stream into an output stream. */ - pipe( - stream: AsyncIterable, - params: TParam, - client: TClient, - ): AsyncIterable; -} - -class ActionImpl - implements ActionInstance -{ - readonly id: string; - readonly clientClass?: ClassRef; - readonly inputClass: ClassRef; - readonly outputClass: ClassRef; - readonly schema: z.ZodType; - readonly #transform: ClientTransformFn; - - constructor(config: { - id: string; - clientClass?: ClassRef; - inputClass: ClassRef; - outputClass: ClassRef; - schema: z.ZodType; - transform: ClientTransformFn; - }) { - this.id = config.id; - if (config.clientClass) this.clientClass = config.clientClass; - this.inputClass = config.inputClass; - this.outputClass = config.outputClass; - this.schema = config.schema; - this.#transform = config.transform; - } - - pipe( - stream: AsyncIterable, - params: TParam, - client: TClient, - ): AsyncIterable { - return this.#transform(stream, params, client); - } -} - -/** Factory for creating action instances. */ -export const Action: { - /** - * Create an action that does not require a provider client. - * - * @param id - Unique identifier for the action. - * @param config - Action configuration including types and transform. - */ - withoutClient( - id: string, - config: ClientlessActionConfig, - ): ActionInstance; - withoutClient( - id: string, - config: ClientlessActionConfig, - ): ActionInstance; - - /** - * Create an action that requires a provider client. - * - * @param id - Unique identifier for the action. - * @param clientClass - Class reference for the required provider client. - * @param config - Action configuration including types and transform. - */ - withClient( - id: string, - clientClass: ClassRef, - config: ClientActionConfig, - ): ActionInstance; - withClient( - id: string, - clientClass: ClassRef, - config: ClientActionConfig, - ): ActionInstance; -} = { - withoutClient( - id: string, - config: { - types: [ClassRef] | [ClassRef, ClassRef]; - params: z.ZodType; - transform: (...args: never[]) => AsyncIterable; - }, - ): ActionInstance { - const [inputClass, outputClass] = config.types; - return new ActionImpl({ - id, - inputClass, - outputClass: outputClass ?? inputClass, - schema: config.params, - transform: (stream, params, _client) => - ( - config.transform as ( - stream: AsyncIterable, - params: unknown, - ) => AsyncIterable - )(stream, params), - }); - }, - - withClient( - id: string, - clientClass: ClassRef, - config: { - types: [ClassRef] | [ClassRef, ClassRef]; - params: z.ZodType; - transform: (...args: never[]) => AsyncIterable; - }, - ): ActionInstance { - const [inputClass, outputClass] = config.types; - return new ActionImpl({ - id, - clientClass, - inputClass, - outputClass: outputClass ?? inputClass, - schema: config.params, - transform: config.transform as ( - stream: AsyncIterable, - params: unknown, - client: unknown, - ) => AsyncIterable, - }); - }, -}; diff --git a/packages/nvisy-core/src/datatypes/blob.test.ts b/packages/nvisy-core/src/datatypes/blob.test.ts deleted file mode 100644 index da330ec..0000000 --- a/packages/nvisy-core/src/datatypes/blob.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { Blob } from "./blob.js"; - -describe("Blob", () => { - it("stores path and data", () => { - const data = Buffer.from("hello world"); - const blob = new Blob("uploads/file.txt", data); - expect(blob.path).toBe("uploads/file.txt"); - expect(blob.data).toBe(data); - expect(blob.data.toString()).toBe("hello world"); - }); - - it("contentType is optional and defaults to undefined", () => { - const blob = new Blob("file.bin", Buffer.from([0x00, 0x01])); - expect(blob.contentType).toBeUndefined(); - }); - - it("accepts contentType in constructor", () => { - const blob = new Blob( - "report.pdf", - Buffer.from("pdf content"), - "application/pdf", - ); - expect(blob.contentType).toBe("application/pdf"); - }); - - it("size returns byte length of data", () => { - const blob = new Blob("file.txt", Buffer.from("abc")); - expect(blob.size).toBe(3); - }); - - it("size handles empty buffer", () => { - const blob = new Blob("empty.bin", Buffer.alloc(0)); - expect(blob.size).toBe(0); - }); - - it("size handles binary data correctly", () => { - const binaryData = Buffer.from([0x00, 0xff, 0x10, 0x20, 0x30]); - const blob = new Blob("binary.bin", binaryData); - expect(blob.size).toBe(5); - }); - - it("extends Data and has id, parentId, metadata", () => { - const blob = new Blob("file.txt", Buffer.from("content")); - expect(blob.id).toMatch( - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, - ); - expect(blob.parentId).toBeNull(); - expect(blob.metadata).toBeNull(); - }); - - it("supports deriveFrom for lineage", () => { - const parent = new Blob("parent.txt", Buffer.from("parent")); - const child = new Blob("child.txt", Buffer.from("child")).deriveFrom( - parent, - ); - expect(child.parentId).toBe(parent.id); - expect(child.isDerived).toBe(true); - }); - - it("supports withMetadata", () => { - const blob = new Blob("file.txt", Buffer.from("content")).withMetadata({ - source: "s3", - bucket: "my-bucket", - }); - expect(blob.metadata).toEqual({ source: "s3", bucket: "my-bucket" }); - }); - - it("handles various path formats", () => { - const s3Blob = new Blob("s3://bucket/key/file.pdf", Buffer.from("")); - expect(s3Blob.path).toBe("s3://bucket/key/file.pdf"); - - const gcsBlob = new Blob("gs://bucket/object", Buffer.from("")); - expect(gcsBlob.path).toBe("gs://bucket/object"); - - const localBlob = new Blob("/var/data/file.txt", Buffer.from("")); - expect(localBlob.path).toBe("/var/data/file.txt"); - }); -}); diff --git a/packages/nvisy-core/src/datatypes/blob.ts b/packages/nvisy-core/src/datatypes/blob.ts deleted file mode 100644 index 891616c..0000000 --- a/packages/nvisy-core/src/datatypes/blob.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { Data } from "./data.js"; - -/** - * A file or binary blob retrieved from object storage (S3, GCS, Dropbox, etc.). - * - * Wraps raw bytes together with their storage path and MIME type so - * downstream processors can decide how to parse the content. - * - * @example - * ```ts - * const obj = new Blob("uploads/report.pdf", Buffer.from(pdfBytes)); - * console.log(obj.size); // byte length - * ``` - */ -export class Blob extends Data { - readonly #path: string; - readonly #data: Buffer; - readonly #contentType: string | undefined; - - constructor(path: string, data: Buffer, contentType?: string) { - super(); - this.#path = path; - this.#data = data; - this.#contentType = contentType; - } - - /** Storage path or key (e.g. `"s3://bucket/file.pdf"`). */ - get path(): string { - return this.#path; - } - - /** Raw binary content. */ - get data(): Buffer { - return this.#data; - } - - /** MIME type of the content (e.g. `"application/pdf"`). */ - get contentType(): string | undefined { - return this.#contentType; - } - - /** Size of the raw data in bytes. */ - get size(): number { - return this.#data.byteLength; - } -} diff --git a/packages/nvisy-core/src/datatypes/data.test.ts b/packages/nvisy-core/src/datatypes/data.test.ts deleted file mode 100644 index 2d98a8e..0000000 --- a/packages/nvisy-core/src/datatypes/data.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { Data } from "./data.js"; - -class TestData extends Data {} - -describe("Data", () => { - it("auto-generates a UUID id", () => { - const a = new TestData(); - const b = new TestData(); - expect(a.id).toMatch( - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, - ); - expect(a.id).not.toBe(b.id); - }); - - it("defaults: parentId is null, metadata is null, isDerived is false", () => { - const data = new TestData(); - expect(data.parentId).toBeNull(); - expect(data.metadata).toBeNull(); - expect(data.isDerived).toBe(false); - }); - - describe("deriveFrom", () => { - it("copies parentId and metadata from parent", () => { - const parent = new TestData().withMetadata({ key: "value" }); - const child = new TestData().deriveFrom(parent); - expect(child.parentId).toBe(parent.id); - expect(child.metadata).toEqual({ key: "value" }); - expect(child.isDerived).toBe(true); - }); - - it("copies null metadata from parent", () => { - const parent = new TestData(); - const child = new TestData().deriveFrom(parent); - expect(child.parentId).toBe(parent.id); - expect(child.metadata).toBeNull(); - }); - - it("returns this for chaining", () => { - const parent = new TestData(); - const child = new TestData(); - expect(child.deriveFrom(parent)).toBe(child); - }); - }); - - describe("withParent", () => { - it("sets parentId", () => { - const data = new TestData().withParent("parent-123"); - expect(data.parentId).toBe("parent-123"); - expect(data.isDerived).toBe(true); - }); - - it("accepts null to clear", () => { - const data = new TestData().withParent("p-1").withParent(null); - expect(data.parentId).toBeNull(); - expect(data.isDerived).toBe(false); - }); - - it("returns this for chaining", () => { - const data = new TestData(); - expect(data.withParent("x")).toBe(data); - }); - }); - - describe("withMetadata", () => { - it("sets metadata", () => { - const data = new TestData().withMetadata({ key: "value" }); - expect(data.metadata).toEqual({ key: "value" }); - }); - - it("accepts null to clear", () => { - const data = new TestData().withMetadata({ a: 1 }).withMetadata(null); - expect(data.metadata).toBeNull(); - }); - - it("returns this for chaining", () => { - const data = new TestData(); - expect(data.withMetadata({ a: 1 })).toBe(data); - }); - }); - - it("deriveFrom then withMetadata overrides metadata", () => { - const parent = new TestData().withMetadata({ old: 1 }); - const child = new TestData().deriveFrom(parent).withMetadata({ new: 2 }); - expect(child.parentId).toBe(parent.id); - expect(child.metadata).toEqual({ new: 2 }); - }); -}); diff --git a/packages/nvisy-core/src/datatypes/data.ts b/packages/nvisy-core/src/datatypes/data.ts deleted file mode 100644 index e07851b..0000000 --- a/packages/nvisy-core/src/datatypes/data.ts +++ /dev/null @@ -1,81 +0,0 @@ -/** - * A JSON-compatible value. - * - * Mirrors the types that `JSON.parse` can return and `JSON.stringify` - * can accept, making it safe for serialisation boundaries (APIs, - * databases, message queues). - */ -export type JsonValue = - | string - | number - | boolean - | null - | JsonValue[] - | { [key: string]: JsonValue }; - -/** - * Key-value metadata bag attached to {@link Data} items. - * - * All values must be JSON-serialisable so metadata can travel across - * process boundaries without lossy conversion. - */ -export type Metadata = Record; - -/** - * Abstract base class for all data types flowing through the pipeline. - * - * Every piece of data in the system — documents, embeddings, database rows, - * storage objects — extends this class, guaranteeing a unique {@link id} and - * optional key-value {@link metadata}. - * - * Use {@link deriveFrom} to set lineage and copy metadata from a parent in - * one call. Use {@link withParent} and {@link withMetadata} for manual - * control. All fluent setters return `this` for chaining. - */ -export abstract class Data { - readonly #id: string = crypto.randomUUID(); - #parentId: string | null = null; - #metadata: Metadata | null = null; - - /** Unique identifier for this data item. */ - get id(): string { - return this.#id; - } - - /** ID of the parent data item this was derived from. `null` when this is a root item. */ - get parentId(): string | null { - return this.#parentId; - } - - /** `true` when this item was derived from another (i.e. {@link parentId} is set). */ - get isDerived(): boolean { - return this.#parentId !== null; - } - - /** Key-value metadata attached to this data item. `null` when unset. */ - get metadata(): Metadata | null { - return this.#metadata; - } - - /** - * Mark this item as derived from `parent`, copying its {@link id} as - * {@link parentId} and its {@link metadata}. Returns `this` for chaining. - */ - deriveFrom(parent: Data): this { - this.#parentId = parent.#id; - this.#metadata = parent.#metadata; - return this; - } - - /** Set the parent ID for lineage tracking. Returns `this` for chaining. */ - withParent(id: string | null): this { - this.#parentId = id; - return this; - } - - /** Set or replace metadata. Returns `this` for chaining. */ - withMetadata(metadata: Metadata | null): this { - this.#metadata = metadata; - return this; - } -} diff --git a/packages/nvisy-core/src/datatypes/document.test.ts b/packages/nvisy-core/src/datatypes/document.test.ts deleted file mode 100644 index 4846366..0000000 --- a/packages/nvisy-core/src/datatypes/document.test.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { DocumentPage, DocumentSection } from "./document.js"; -import { Document } from "./document.js"; - -describe("Document", () => { - it("stores content and has no pages by default", () => { - const doc = new Document("hello world"); - expect(doc.content).toBe("hello world"); - expect(doc.pages).toBeUndefined(); - expect(doc.flatElements).toEqual([]); - }); - - it("constructor accepts pages in options", () => { - const pages: DocumentPage[] = [ - { - pageNumber: 1, - sections: [ - { - title: "Intro", - elements: [{ type: "paragraph", text: "Hello" }], - }, - ], - }, - ]; - const doc = new Document("Hello", { pages }); - expect(doc.content).toBe("Hello"); - expect(doc.pages).toEqual(pages); - }); - - describe("fromPages", () => { - it("derives content from element texts joined with \\n\\n", () => { - const pages: DocumentPage[] = [ - { - pageNumber: 1, - sections: [ - { - elements: [ - { type: "heading", text: "Title", level: 1 }, - { type: "paragraph", text: "First paragraph." }, - ], - }, - ], - }, - { - pageNumber: 2, - sections: [ - { - elements: [{ type: "paragraph", text: "Second page content." }], - }, - ], - }, - ]; - - const doc = Document.fromPages(pages); - expect(doc.content).toBe( - "Title\n\nFirst paragraph.\n\nSecond page content.", - ); - expect(doc.pages).toEqual(pages); - }); - - it("produces empty content from empty pages array", () => { - const doc = Document.fromPages([]); - expect(doc.content).toBe(""); - expect(doc.pages).toEqual([]); - }); - - it("preserves sourceType", () => { - const pages: DocumentPage[] = [ - { - pageNumber: 1, - sections: [ - { - elements: [{ type: "paragraph", text: "text" }], - }, - ], - }, - ]; - const doc = Document.fromPages(pages, { sourceType: "pdf" }); - expect(doc.sourceType).toBe("pdf"); - expect(doc.pages).toHaveLength(1); - }); - }); - - describe("flatElements", () => { - it("traverses pages -> sections recursively in document order", () => { - const pages: DocumentPage[] = [ - { - pageNumber: 1, - sections: [ - { - title: "S1", - elements: [ - { type: "heading", text: "H1", level: 1 }, - { type: "paragraph", text: "P1" }, - ], - children: [ - { - title: "S1.1", - elements: [{ type: "paragraph", text: "P1.1" }], - }, - ], - }, - ], - }, - { - pageNumber: 2, - sections: [ - { - elements: [{ type: "table", text: "T1" }], - }, - ], - }, - ]; - - const doc = new Document("ignored", { pages }); - expect(doc.flatElements.map((e) => e.text)).toEqual([ - "H1", - "P1", - "P1.1", - "T1", - ]); - }); - - it("handles deeply nested sections (3+ levels)", () => { - const deepSection: DocumentSection = { - title: "L1", - elements: [{ type: "paragraph", text: "Level 1" }], - children: [ - { - title: "L2", - elements: [{ type: "paragraph", text: "Level 2" }], - children: [ - { - title: "L3", - elements: [{ type: "paragraph", text: "Level 3" }], - children: [ - { - title: "L4", - elements: [{ type: "code", text: "Level 4" }], - }, - ], - }, - ], - }, - ], - }; - const pages: DocumentPage[] = [ - { pageNumber: 1, sections: [deepSection] }, - ]; - const doc = new Document("x", { pages }); - expect(doc.flatElements.map((e) => e.text)).toEqual([ - "Level 1", - "Level 2", - "Level 3", - "Level 4", - ]); - }); - - it("handles sections without titles", () => { - const pages: DocumentPage[] = [ - { - pageNumber: 1, - sections: [ - { - elements: [{ type: "paragraph", text: "No title section" }], - }, - ], - }, - ]; - const doc = new Document("x", { pages }); - expect(doc.flatElements).toHaveLength(1); - expect(doc.flatElements[0]!.text).toBe("No title section"); - }); - - it("includes elements with empty text strings", () => { - const pages: DocumentPage[] = [ - { - pageNumber: 1, - sections: [ - { - elements: [ - { type: "image", text: "" }, - { type: "paragraph", text: "after" }, - ], - }, - ], - }, - ]; - const doc = new Document("x", { pages }); - expect(doc.flatElements).toHaveLength(2); - expect(doc.flatElements[0]!.text).toBe(""); - expect(doc.flatElements[0]!.type).toBe("image"); - }); - }); -}); diff --git a/packages/nvisy-core/src/datatypes/document.ts b/packages/nvisy-core/src/datatypes/document.ts deleted file mode 100644 index 7d918e9..0000000 --- a/packages/nvisy-core/src/datatypes/document.ts +++ /dev/null @@ -1,134 +0,0 @@ -import type { Metadata } from "./data.js"; -import { Data } from "./data.js"; - -/** The kind of structural element within a document. */ -export type ElementType = - | "paragraph" - | "heading" - | "table" - | "list" - | "image" - | "code"; - -/** A single structural element within a {@link DocumentSection}. */ -export interface DocumentElement { - readonly type: ElementType; - readonly text: string; - /** Heading level (1-6). Only meaningful when `type` is `"heading"`. */ - readonly level?: number; - /** Element-scoped metadata (e.g. table caption, alt text). */ - readonly metadata?: Metadata; -} - -/** A titled section containing elements and optional nested sub-sections. */ -export interface DocumentSection { - readonly title?: string; - readonly elements: readonly DocumentElement[]; - readonly children?: readonly DocumentSection[]; -} - -/** A single page of a document. */ -export interface DocumentPage { - /** 1-based page number. */ - readonly pageNumber: number; - readonly sections: readonly DocumentSection[]; -} - -/** Options for constructing a {@link Document}. */ -export interface DocumentOptions { - readonly sourceType?: string; - readonly pages?: readonly DocumentPage[]; -} - -/** - * A parsed human-readable text representation of a document. - * - * Represents extracted text from a partition step — the raw bytes have - * already been converted into plain text that can be chunked, enriched, - * or embedded. - * - * @example - * ```ts - * const doc = new Document("Quarterly Report\n\nRevenue increased…", { - * sourceType: "pdf", - * }); - * ``` - */ -export class Document extends Data { - readonly #content: string; - readonly #sourceType?: string | undefined; - readonly #pages?: readonly DocumentPage[] | undefined; - - constructor(content: string, options?: DocumentOptions) { - super(); - this.#content = content; - this.#sourceType = options?.sourceType; - this.#pages = options?.pages; - } - - /** Text content of the document. */ - get content(): string { - return this.#content; - } - - /** Origin format (e.g. "pdf", "markdown", "docx", "html", "transcript", "database"). */ - get sourceType(): string | undefined { - return this.#sourceType; - } - - /** Optional hierarchical page structure. */ - get pages(): readonly DocumentPage[] | undefined { - return this.#pages; - } - - /** All elements across all pages and sections, flattened in document order. */ - get flatElements(): DocumentElement[] { - if (this.#pages == null) return []; - return collectElements(this.#pages); - } - - /** - * Create a Document by deriving `content` from the element texts in the given pages. - * - * Element texts are joined with `\n\n` separators. - */ - static fromPages( - pages: readonly DocumentPage[], - options?: Omit, - ): Document { - const content = flattenPagesToText(pages); - return new Document(content, { ...options, pages }); - } -} - -/** Collect all elements from a page tree in document order. */ -function collectElements(pages: readonly DocumentPage[]): DocumentElement[] { - const out: DocumentElement[] = []; - for (const page of pages) { - for (const section of page.sections) { - flattenSection(section, out); - } - } - return out; -} - -/** Recursively collect elements from a section and its children. */ -function flattenSection( - section: DocumentSection, - out: DocumentElement[], -): void { - for (const el of section.elements) { - out.push(el); - } - if (section.children) { - for (const child of section.children) { - flattenSection(child, out); - } - } -} - -/** Derive plain text content from a page tree. */ -function flattenPagesToText(pages: readonly DocumentPage[]): string { - const elements = collectElements(pages); - return elements.map((el) => el.text).join("\n\n"); -} diff --git a/packages/nvisy-core/src/datatypes/embedding.ts b/packages/nvisy-core/src/datatypes/embedding.ts deleted file mode 100644 index 007b83e..0000000 --- a/packages/nvisy-core/src/datatypes/embedding.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Data } from "./data.js"; - -/** - * A dense vector embedding produced by an embedding model. - * - * Stores the vector as a `Float32Array` for memory efficiency and fast - * math operations. Use {@link dimensions} to inspect the vector size - * without accessing the underlying array. - * - * @example - * ```ts - * const e = new Embedding([0.12, -0.34, 0.56]); - * console.log(e.dimensions); // 3 - * ``` - */ -export class Embedding extends Data { - readonly #vector: Float32Array; - - constructor(vector: Float32Array | number[]) { - super(); - this.#vector = - vector instanceof Float32Array ? vector : new Float32Array(vector); - } - - /** The dense embedding vector. */ - get vector(): Float32Array { - return this.#vector; - } - - /** Dimensionality of the embedding vector. */ - get dimensions(): number { - return this.#vector.length; - } -} diff --git a/packages/nvisy-core/src/datatypes/index.ts b/packages/nvisy-core/src/datatypes/index.ts deleted file mode 100644 index 4042864..0000000 --- a/packages/nvisy-core/src/datatypes/index.ts +++ /dev/null @@ -1,55 +0,0 @@ -/** - * @module datatypes - * - * Base data model and built-in types for the Nvisy pipeline. - */ - -export { Blob } from "./blob.js"; -export type { JsonValue, Metadata } from "./data.js"; -export { Data } from "./data.js"; -export type { - DocumentElement, - DocumentOptions, - DocumentPage, - DocumentSection, - ElementType, -} from "./document.js"; -export { Document } from "./document.js"; -export { Embedding } from "./embedding.js"; - -import type { ClassRef } from "../types.js"; -import type { Data } from "./data.js"; - -/** - * A custom data type registered by a plugin. - * - * Plugins use this to extend the type system with new {@link Data} - * subclasses without modifying nvisy-core. - */ -export interface Datatype { - /** Unique identifier for this data type (e.g. "audio", "image"). */ - readonly id: string; - /** Class reference for the custom data type. */ - readonly dataClass: ClassRef; -} - -/** Factory for creating data type entries. */ -export const Datatypes = { - /** Create a Datatype for registering a custom data type with a plugin. */ - define(id: string, dataClass: ClassRef): Datatype { - return { id, dataClass }; - }, -} as const; - -import { Blob } from "./blob.js"; -import { Document } from "./document.js"; -import { Embedding } from "./embedding.js"; - -/** Pre-defined Document datatype entry. */ -export const documentDatatype = Datatypes.define("document", Document); - -/** Pre-defined Blob datatype entry. */ -export const blobDatatype = Datatypes.define("blob", Blob); - -/** Pre-defined Embedding datatype entry. */ -export const embeddingDatatype = Datatypes.define("embedding", Embedding); diff --git a/packages/nvisy-core/src/errors/cancellation.ts b/packages/nvisy-core/src/errors/cancellation.ts deleted file mode 100644 index 3730d25..0000000 --- a/packages/nvisy-core/src/errors/cancellation.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { RuntimeError, type RuntimeErrorOptions } from "./runtime.js"; - -/** - * The operation was explicitly cancelled. - * - * Cancellation errors are not retryable by default since the - * cancellation was intentional. - * - * @example - * ```ts - * if (signal.aborted) { - * throw new CancellationError("Operation cancelled by user"); - * } - * ``` - */ -export class CancellationError extends RuntimeError { - constructor(message = "Operation cancelled", options?: RuntimeErrorOptions) { - super(message, { retryable: false, ...options }); - } -} diff --git a/packages/nvisy-core/src/errors/connection.ts b/packages/nvisy-core/src/errors/connection.ts deleted file mode 100644 index 881d102..0000000 --- a/packages/nvisy-core/src/errors/connection.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { type ErrorContext, RuntimeError } from "./runtime.js"; - -/** - * Could not reach an external service, storage backend, or database. - * - * Also covers missing or unregistered connections. Connection errors - * are retryable by default since network issues are often transient. - * - * @example - * ```ts - * throw new ConnectionError("Database connection timeout", { - * source: "postgres", - * details: { host: "db.example.com", port: 5432 }, - * }); - * - * // Wrap provider connection failures - * catch (error) { - * throw ConnectionError.wrap(error, { source: "postgres" }); - * } - * ``` - */ -export class ConnectionError extends RuntimeError { - /** - * Wrap an unknown error as a ConnectionError. - * - * If the error is already a ConnectionError, returns it unchanged. - * Otherwise, creates a new ConnectionError with the original as cause. - * - * @param error - The error to wrap. - * @param context - Optional context (source, details). - */ - static override wrap( - error: unknown, - context?: ErrorContext, - ): ConnectionError { - if (error instanceof ConnectionError) return error; - const message = error instanceof Error ? error.message : String(error); - const cause = error instanceof Error ? error : undefined; - return new ConnectionError(`Connection failed: ${message}`, { - ...context, - ...(cause && { cause }), - }); - } - - /** - * Create a connection error for missing connections. - * - * @param connectionId - The connection ID that wasn't found. - * @param source - The component that raised the error. - */ - static notFound(connectionId: string, source?: string): ConnectionError { - return new ConnectionError(`Connection not found: ${connectionId}`, { - ...(source && { source }), - retryable: false, - }); - } -} diff --git a/packages/nvisy-core/src/errors/index.ts b/packages/nvisy-core/src/errors/index.ts deleted file mode 100644 index 0e240e0..0000000 --- a/packages/nvisy-core/src/errors/index.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Structured error hierarchy for the Nvisy runtime. - * - * All errors extend {@link RuntimeError} which provides: - * - `source` — component that raised the error - * - `details` — machine-readable context - * - `retryable` — whether the operation can be retried - * - * Default retryability: - * - {@link RuntimeError} — `true` (transient failures) - * - {@link ValidationError} — `false` (bad input won't fix itself) - * - {@link ConnectionError} — `true` (network issues are transient) - * - {@link CancellationError} — `false` (intentional cancellation) - * - * @module - */ - -export { CancellationError } from "./cancellation.js"; -export { ConnectionError } from "./connection.js"; -export type { ErrorContext, RuntimeErrorOptions } from "./runtime.js"; -export { RuntimeError } from "./runtime.js"; -export { ValidationError } from "./validation.js"; diff --git a/packages/nvisy-core/src/errors/runtime.ts b/packages/nvisy-core/src/errors/runtime.ts deleted file mode 100644 index bff2788..0000000 --- a/packages/nvisy-core/src/errors/runtime.ts +++ /dev/null @@ -1,82 +0,0 @@ -/** Structured context attached to runtime errors. */ -export interface ErrorContext { - /** Which component or subsystem produced the error. */ - readonly source?: string; - /** Machine-readable details about the failure (IDs, paths, limits, etc.). */ - readonly details?: Record; -} - -/** Options for constructing a RuntimeError. */ -export interface RuntimeErrorOptions extends ErrorContext { - /** Whether the caller may safely retry the operation. */ - readonly retryable?: boolean; - /** The underlying error that caused this one. */ - readonly cause?: Error; -} - -/** - * Base class for all Nvisy runtime errors. - * - * Every error carries structured context for logging and debugging. - * Extends the built-in `Error` so `instanceof RuntimeError` works everywhere. - * - * By default, runtime errors are retryable. Subclasses like `ValidationError` - * override this to `false` since validation failures won't succeed on retry. - * - * @example - * ```ts - * throw new RuntimeError("Operation failed", { - * source: "engine", - * details: { nodeId: "abc" }, - * }); - * - * // Wrap unknown errors - * catch (error) { - * throw RuntimeError.wrap(error, { source: "provider" }); - * } - * ``` - */ -export class RuntimeError extends Error { - readonly #source: string | undefined; - readonly #details: Record | undefined; - readonly #retryable: boolean; - - constructor(message: string, options?: RuntimeErrorOptions) { - super(message, options?.cause ? { cause: options.cause } : undefined); - this.name = this.constructor.name; - this.#source = options?.source; - this.#details = options?.details; - this.#retryable = options?.retryable ?? true; - } - - /** Which component or subsystem produced the error. */ - get source(): string | undefined { - return this.#source; - } - - /** Machine-readable details about the failure (IDs, paths, limits, etc.). */ - get details(): Record | undefined { - return this.#details; - } - - /** Whether the caller may safely retry the operation. */ - get retryable(): boolean { - return this.#retryable; - } - - /** - * Wrap an unknown error as a RuntimeError. - * - * If the error is already a RuntimeError, returns it unchanged. - * Otherwise, creates a new RuntimeError with the original as cause. - */ - static wrap(error: unknown, context?: ErrorContext): RuntimeError { - if (error instanceof RuntimeError) return error; - const message = error instanceof Error ? error.message : String(error); - const cause = error instanceof Error ? error : undefined; - return new RuntimeError(message, { - ...context, - ...(cause && { cause }), - }); - } -} diff --git a/packages/nvisy-core/src/errors/validation.ts b/packages/nvisy-core/src/errors/validation.ts deleted file mode 100644 index 44e693a..0000000 --- a/packages/nvisy-core/src/errors/validation.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { RuntimeError, type RuntimeErrorOptions } from "./runtime.js"; - -/** - * Input did not pass schema or business rules. - * - * Also covers invalid workflow and pipeline definitions. - * Validation errors are not retryable by default since the same - * input will fail validation again. - * - * @example - * ```ts - * throw new ValidationError("Invalid graph definition", { source: "compiler" }); - * - * // Use factory for common "not found" pattern - * throw ValidationError.notFound("myAction", "action", "registry"); - * ``` - */ -export class ValidationError extends RuntimeError { - constructor(message: string, options?: RuntimeErrorOptions) { - super(message, { retryable: false, ...options }); - } - - /** - * Create a "not found" validation error. - * - * @param name - The name that wasn't found. - * @param type - The type of thing (e.g., "action", "provider", "stream"). - * @param source - The component that raised the error. - */ - static notFound( - name: string, - type: string, - source?: string, - ): ValidationError { - return new ValidationError(`Unknown ${type}: ${name}`, { - ...(source && { source }), - }); - } - - /** - * Create a validation error for parse failures. - * - * @param message - The parse error message. - * @param source - The component that raised the error. - */ - static parse(message: string, source?: string): ValidationError { - return new ValidationError(`Parse error: ${message}`, { - ...(source && { source }), - }); - } -} diff --git a/packages/nvisy-core/src/index.ts b/packages/nvisy-core/src/index.ts deleted file mode 100644 index e105aa8..0000000 --- a/packages/nvisy-core/src/index.ts +++ /dev/null @@ -1,82 +0,0 @@ -/** - * @module @nvisy/core - * - * Public API surface for the nvisy core library. - */ - -export type { ActionInstance } from "./action.js"; -export { Action } from "./action.js"; -export type { - Datatype, - DocumentElement, - DocumentOptions, - DocumentPage, - DocumentSection, - ElementType, - JsonValue, - Metadata, -} from "./datatypes/index.js"; -export { - Blob, - blobDatatype, - Data, - Datatypes, - Document, - documentDatatype, - Embedding, - embeddingDatatype, -} from "./datatypes/index.js"; -export type { ErrorContext } from "./errors/index.js"; -export { - CancellationError, - ConnectionError, - RuntimeError, - ValidationError, -} from "./errors/index.js"; -export type { - LoaderConfig, - LoaderInstance, - LoadFn, - PlaintextParams, -} from "./loaders/index.js"; -export { - Loader, - plaintextLoader, - plaintextParamsSchema, -} from "./loaders/index.js"; -export type { - AnyActionInstance, - AnyLoaderInstance, - AnyProviderFactory, - AnyStreamSource, - AnyStreamTarget, - PluginInstance, -} from "./plugin.js"; -export { Plugin } from "./plugin.js"; -export type { - ConnectedInstance, - ProviderFactory, - ProviderInstance, -} from "./provider.js"; -export { Provider } from "./provider.js"; -export type { - Resumable, - StreamSource, - StreamTarget, - WriterFn, -} from "./stream.js"; -export { Stream } from "./stream.js"; -export type { ClassRef } from "./types.js"; - -import { - blobDatatype, - documentDatatype, - embeddingDatatype, -} from "./datatypes/index.js"; -import { plaintextLoader } from "./loaders/index.js"; -import { Plugin } from "./plugin.js"; - -/** Built-in core plugin that registers the Document, Blob, and Embedding datatypes, and plaintext loader. */ -export const corePlugin = Plugin.define("core") - .withDatatypes(documentDatatype, blobDatatype, embeddingDatatype) - .withLoaders(plaintextLoader); diff --git a/packages/nvisy-core/src/loaders/index.ts b/packages/nvisy-core/src/loaders/index.ts deleted file mode 100644 index e4b475f..0000000 --- a/packages/nvisy-core/src/loaders/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type { LoaderConfig, LoaderInstance, LoadFn } from "./loader.js"; -export { Loader } from "./loader.js"; -export type { PlaintextParams } from "./plaintext.js"; -export { plaintextLoader, plaintextParamsSchema } from "./plaintext.js"; diff --git a/packages/nvisy-core/src/loaders/loader.ts b/packages/nvisy-core/src/loaders/loader.ts deleted file mode 100644 index 882c658..0000000 --- a/packages/nvisy-core/src/loaders/loader.ts +++ /dev/null @@ -1,99 +0,0 @@ -import type { z } from "zod"; -import type { Blob } from "../datatypes/blob.js"; -import type { Document } from "../datatypes/document.js"; - -/** - * Function that transforms a Blob into one or more Documents. - * - * @template TParam - Configuration parameters for the loader. - */ -export type LoadFn = ( - blob: Blob, - params: TParam, -) => AsyncIterable; - -/** - * Configuration for creating a loader. - * - * @template TParam - Configuration parameters for the loader. - */ -export interface LoaderConfig { - /** File extensions this loader handles (e.g. [".pdf"], [".md", ".markdown"]). */ - readonly extensions: string[]; - /** MIME types this loader handles (e.g. ["application/pdf"], ["text/plain"]). */ - readonly contentTypes: string[]; - /** Zod schema for validating loader parameters. */ - readonly params: z.ZodType; - /** The load function that transforms a Blob into Documents. */ - readonly load: LoadFn; -} - -/** - * A registered loader instance that transforms Blobs into Documents. - * - * Loaders are specialized transforms that convert binary objects - * (files from object storage) into structured Document instances - * that can be processed by the pipeline. - * - * @template TParam - Configuration parameters for the loader. - */ -export interface LoaderInstance { - /** Unique identifier for this loader (e.g. "pdf", "docx"). */ - readonly id: string; - /** File extensions this loader handles. */ - readonly extensions: readonly string[]; - /** MIME types this loader handles. */ - readonly contentTypes: readonly string[]; - /** Zod schema for validating loader parameters. */ - readonly schema: z.ZodType; - /** Transform a Blob into one or more Documents. */ - load(blob: Blob, params: TParam): AsyncIterable; -} - -class LoaderImpl implements LoaderInstance { - readonly id: string; - readonly extensions: readonly string[]; - readonly contentTypes: readonly string[]; - readonly schema: z.ZodType; - readonly #load: LoadFn; - - constructor(config: { - id: string; - extensions: string[]; - contentTypes: string[]; - schema: z.ZodType; - load: LoadFn; - }) { - this.id = config.id; - this.extensions = config.extensions; - this.contentTypes = config.contentTypes; - this.schema = config.schema; - this.#load = config.load; - } - - load(blob: Blob, params: TParam): AsyncIterable { - return this.#load(blob, params); - } -} - -/** Factory for creating loader instances. */ -export const Loader = { - /** - * Create a loader that transforms Blobs into Documents. - * - * @param id - Unique identifier for the loader (e.g. "pdf", "docx"). - * @param config - Loader configuration including match criteria and load function. - */ - define( - id: string, - config: LoaderConfig, - ): LoaderInstance { - return new LoaderImpl({ - id, - extensions: config.extensions, - contentTypes: config.contentTypes, - schema: config.params, - load: config.load, - }); - }, -} as const; diff --git a/packages/nvisy-core/src/loaders/plaintext.test.ts b/packages/nvisy-core/src/loaders/plaintext.test.ts deleted file mode 100644 index f143c93..0000000 --- a/packages/nvisy-core/src/loaders/plaintext.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { Blob } from "../datatypes/blob.js"; -import { plaintextLoader } from "./plaintext.js"; - -async function collectDocs( - iter: AsyncIterable, -) { - const docs = []; - for await (const doc of iter) { - docs.push(doc); - } - return docs; -} - -describe("plaintextLoader", () => { - it("has id 'plaintext'", () => { - expect(plaintextLoader.id).toBe("plaintext"); - }); - - it("matches .txt extension", () => { - expect(plaintextLoader.extensions).toContain(".txt"); - }); - - it("matches text/plain content type", () => { - expect(plaintextLoader.contentTypes).toContain("text/plain"); - }); - - it("converts utf-8 text blob to document", async () => { - const blob = new Blob("readme.txt", Buffer.from("Hello, world!")); - const docs = await collectDocs( - plaintextLoader.load(blob, { encoding: "utf-8" }), - ); - - expect(docs).toHaveLength(1); - expect(docs[0]!.content).toBe("Hello, world!"); - expect(docs[0]!.sourceType).toBe("text"); - }); - - it("derives document from blob (sets parentId)", async () => { - const blob = new Blob("file.txt", Buffer.from("content")); - const docs = await collectDocs( - plaintextLoader.load(blob, { encoding: "utf-8" }), - ); - - expect(docs[0]!.parentId).toBe(blob.id); - expect(docs[0]!.isDerived).toBe(true); - }); - - it("handles empty file", async () => { - const blob = new Blob("empty.txt", Buffer.alloc(0)); - const docs = await collectDocs( - plaintextLoader.load(blob, { encoding: "utf-8" }), - ); - - expect(docs).toHaveLength(1); - expect(docs[0]!.content).toBe(""); - }); - - it("handles multiline content", async () => { - const content = "Line 1\nLine 2\nLine 3"; - const blob = new Blob("multi.txt", Buffer.from(content)); - const docs = await collectDocs( - plaintextLoader.load(blob, { encoding: "utf-8" }), - ); - - expect(docs[0]!.content).toBe(content); - }); - - it("supports ascii encoding", async () => { - const blob = new Blob("ascii.txt", Buffer.from("ASCII text", "ascii")); - const docs = await collectDocs( - plaintextLoader.load(blob, { encoding: "ascii" }), - ); - - expect(docs[0]!.content).toBe("ASCII text"); - }); - - it("supports latin1 encoding", async () => { - const blob = new Blob("latin.txt", Buffer.from("café", "latin1")); - const docs = await collectDocs( - plaintextLoader.load(blob, { encoding: "latin1" }), - ); - - expect(docs[0]!.content).toBe("café"); - }); - - it("defaults to utf-8 when encoding not specified", async () => { - const blob = new Blob("utf8.txt", Buffer.from("Unicode: 你好")); - const params = plaintextLoader.schema.parse({}); - const docs = await collectDocs(plaintextLoader.load(blob, params)); - - expect(docs[0]!.content).toBe("Unicode: 你好"); - }); - - it("schema validates encoding enum", () => { - expect(() => - plaintextLoader.schema.parse({ encoding: "invalid" }), - ).toThrow(); - }); - - it("schema rejects unknown properties", () => { - expect(() => - plaintextLoader.schema.parse({ encoding: "utf-8", extra: "field" }), - ).toThrow(); - }); -}); diff --git a/packages/nvisy-core/src/loaders/plaintext.ts b/packages/nvisy-core/src/loaders/plaintext.ts deleted file mode 100644 index 812820b..0000000 --- a/packages/nvisy-core/src/loaders/plaintext.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { z } from "zod"; -import { Document } from "../datatypes/document.js"; -import { Loader } from "./loader.js"; - -/** Schema for plaintext loader parameters. */ -export const plaintextParamsSchema = z - .object({ - /** Character encoding of the blob data. Defaults to "utf-8". */ - encoding: z - .enum(["utf-8", "ascii", "latin1", "utf16le"]) - .optional() - .default("utf-8"), - }) - .strict(); - -export type PlaintextParams = z.infer; - -/** - * Loader that converts plaintext blobs (.txt files) into Documents. - * - * Reads the blob data as text using the specified encoding and - * creates a Document with the text content. - */ -export const plaintextLoader = Loader.define("plaintext", { - extensions: [".txt"], - contentTypes: ["text/plain"], - params: plaintextParamsSchema, - async *load(blob, params) { - const content = blob.data.toString(params.encoding); - const doc = new Document(content, { sourceType: "text" }); - doc.deriveFrom(blob); - yield doc; - }, -}); diff --git a/packages/nvisy-core/src/plugin.ts b/packages/nvisy-core/src/plugin.ts deleted file mode 100644 index afbf766..0000000 --- a/packages/nvisy-core/src/plugin.ts +++ /dev/null @@ -1,104 +0,0 @@ -import type { ActionInstance } from "./action.js"; -import type { Datatype } from "./datatypes/index.js"; -import type { LoaderInstance } from "./loaders/loader.js"; -import type { ProviderFactory } from "./provider.js"; -import type { StreamSource, StreamTarget } from "./stream.js"; - -// biome-ignore lint/suspicious/noExplicitAny: existential type alias -export type AnyProviderFactory = ProviderFactory; - -// biome-ignore lint/suspicious/noExplicitAny: existential type alias -export type AnyActionInstance = ActionInstance; - -// biome-ignore lint/suspicious/noExplicitAny: existential type alias -export type AnyLoaderInstance = LoaderInstance; - -// biome-ignore lint/suspicious/noExplicitAny: existential type alias -export type AnyStreamSource = StreamSource; - -// biome-ignore lint/suspicious/noExplicitAny: existential type alias -export type AnyStreamTarget = StreamTarget; - -/** - * A plugin bundles providers, streams, actions, and loaders under a namespace. - * - * Plugins are the unit of registration with the engine. All entries - * are namespaced as `"pluginId/name"` to avoid collisions. - */ -export interface PluginInstance { - /** Unique identifier for the plugin (e.g. "sql", "openai"). */ - readonly id: string; - /** Provider factories keyed by their ID. */ - readonly providers: Readonly>; - /** Stream sources and targets keyed by their ID. */ - readonly streams: Readonly>; - /** Actions keyed by their ID. */ - readonly actions: Readonly>; - /** Loaders keyed by their ID. */ - readonly loaders: Readonly>; - /** Custom data types keyed by their ID. */ - readonly datatypes: Readonly>; -} - -class PluginBuilder implements PluginInstance { - readonly id: string; - readonly providers: Readonly> = {}; - readonly streams: Readonly< - Record - > = {}; - readonly actions: Readonly> = {}; - readonly loaders: Readonly> = {}; - readonly datatypes: Readonly> = {}; - - constructor(id: string) { - this.id = id; - } - - /** Add providers to this plugin. */ - withProviders(...providers: AnyProviderFactory[]): this { - const record = { ...this.providers }; - for (const p of providers) record[p.id] = p; - (this as { providers: typeof record }).providers = record; - return this; - } - - /** Add streams to this plugin. */ - withStreams(...streams: (AnyStreamSource | AnyStreamTarget)[]): this { - const record = { ...this.streams }; - for (const s of streams) record[s.id] = s; - (this as { streams: typeof record }).streams = record; - return this; - } - - /** Add actions to this plugin. */ - withActions(...actions: AnyActionInstance[]): this { - const record = { ...this.actions }; - for (const a of actions) record[a.id] = a; - (this as { actions: typeof record }).actions = record; - return this; - } - - /** Add loaders to this plugin. */ - withLoaders(...loaders: AnyLoaderInstance[]): this { - const record = { ...this.loaders }; - for (const l of loaders) record[l.id] = l; - (this as { loaders: typeof record }).loaders = record; - return this; - } - - /** Add custom data types to this plugin. */ - withDatatypes(...datatypes: Datatype[]): this { - const record = { ...this.datatypes }; - for (const d of datatypes) record[d.id] = d; - (this as { datatypes: typeof record }).datatypes = record; - return this; - } -} - -/** Factory for creating plugin definitions. */ -export const Plugin = { - /** Create a new plugin with the given ID. */ - define(id: string): PluginBuilder { - return new PluginBuilder(id); - }, -} as const; diff --git a/packages/nvisy-core/src/provider.ts b/packages/nvisy-core/src/provider.ts deleted file mode 100644 index e76e5e2..0000000 --- a/packages/nvisy-core/src/provider.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { z } from "zod"; -import { ConnectionError } from "./errors/index.js"; - -const logger = getLogger(["nvisy", "provider"]); - -/** - * Configuration for creating a provider that requires credentials. - * - * @template TCred - Credential type required for authentication. - * @template TClient - Client type returned after successful connection. - */ -export interface AuthenticatedProviderConfig { - /** Zod schema for validating credentials. */ - readonly credentials: z.ZodType; - /** Verify connectivity without establishing a persistent connection. */ - readonly verify?: (credentials: TCred) => Promise; - /** Factory function that establishes a connection using credentials. */ - readonly connect: (credentials: TCred) => Promise>; -} - -/** - * Configuration for creating a provider that does not require credentials. - * - * @template TClient - Client type returned after successful connection. - */ -export interface UnauthenticatedProviderConfig { - /** Factory function that establishes a connection. */ - readonly connect: () => Promise>; -} - -/** - * Raw provider instance returned from a connect function. - * - * This is the internal representation before wrapping with lifecycle management. - * - * @template TClient - Client type provided by this instance. - */ -export interface ProviderInstance { - /** The connected client ready for use. */ - readonly client: TClient; - /** Optional cleanup function called when disconnecting. */ - disconnect?(): Promise; -} - -/** - * A connected provider instance with lifecycle management. - * - * Wraps a raw {@link ProviderInstance} with idempotent disconnect handling - * and logging. - * - * @template TClient - Client type provided by this instance. - */ -export interface ConnectedInstance { - /** Identifier of the provider that created this instance. */ - readonly id: string; - /** The connected client ready for use. */ - readonly client: TClient; - /** Disconnect and release resources (idempotent). */ - disconnect(): Promise; -} - -/** - * Factory for creating provider connections. - * - * Providers manage the lifecycle of external clients (databases, APIs, etc.). - * Each provider defines a credential schema and a connect function that - * returns a managed {@link ConnectedInstance}. - * - * @template TCred - Credential type required for authentication. - * @template TClient - Client type returned after successful connection. - */ -export interface ProviderFactory { - /** Unique identifier for this provider. */ - readonly id: string; - /** Zod schema for validating credentials. */ - readonly credentialSchema: z.ZodType; - /** Verify connectivity without establishing a persistent connection. */ - verify(credentials: TCred): Promise; - /** Create a new connection using the provided credentials. */ - connect(credentials: TCred): Promise>; -} - -const noop = async () => {}; - -class ConnectedInstanceImpl implements ConnectedInstance { - readonly id: string; - readonly client: TClient; - readonly #disconnect: () => Promise; - #disconnected = false; - - constructor(id: string, instance: ProviderInstance) { - this.id = id; - this.client = instance.client; - this.#disconnect = instance.disconnect ?? noop; - } - - async disconnect(): Promise { - if (this.#disconnected) return; - this.#disconnected = true; - - try { - await this.#disconnect(); - logger.debug("Provider disconnected", { provider: this.id }); - } catch (error) { - logger.warn("Provider disconnect failed", { - provider: this.id, - error: String(error), - }); - throw error; - } - } -} - -class ProviderFactoryImpl - implements ProviderFactory -{ - readonly id: string; - readonly credentialSchema: z.ZodType; - readonly #connect: (credentials: TCred) => Promise>; - readonly #verify: (credentials: TCred) => Promise; - - constructor( - id: string, - credentialSchema: z.ZodType, - connect: (credentials: TCred) => Promise>, - verify?: (credentials: TCred) => Promise, - ) { - this.id = id; - this.credentialSchema = credentialSchema; - this.#connect = connect; - this.#verify = verify ?? noop; - } - - async verify(credentials: TCred): Promise { - await this.#verify(credentials); - } - - async connect(credentials: TCred): Promise> { - try { - const instance = await this.#connect(credentials); - logger.debug("Provider connected", { provider: this.id }); - return new ConnectedInstanceImpl(this.id, instance); - } catch (error) { - logger.warn("Provider connection failed", { - provider: this.id, - error: error instanceof Error ? error.message : String(error), - }); - throw ConnectionError.wrap(error, { source: this.id }); - } - } -} - -/** Factory for creating provider definitions. */ -export const Provider = { - /** - * Create a provider that requires authentication credentials. - * - * @param id - Unique identifier for the provider. - * @param config - Provider configuration including credential schema and connect function. - */ - withAuthentication( - id: string, - config: AuthenticatedProviderConfig, - ): ProviderFactory { - return new ProviderFactoryImpl( - id, - config.credentials, - config.connect, - config.verify, - ); - }, - - /** - * Create a provider that does not require authentication. - * - * @param id - Unique identifier for the provider. - * @param config - Provider configuration including connect function. - */ - withoutAuthentication( - id: string, - config: UnauthenticatedProviderConfig, - ): ProviderFactory { - return new ProviderFactoryImpl(id, z.void(), () => config.connect()); - }, -}; diff --git a/packages/nvisy-core/src/stream.ts b/packages/nvisy-core/src/stream.ts deleted file mode 100644 index 6aacecb..0000000 --- a/packages/nvisy-core/src/stream.ts +++ /dev/null @@ -1,253 +0,0 @@ -import type { z } from "zod"; -import type { Data } from "./datatypes/index.js"; -import type { ClassRef } from "./types.js"; - -/** - * A data item paired with resumption context. - * - * Stream sources emit resumables so that the engine can persist - * context after each item, enabling crash recovery. - * - * @template TData - The data type being streamed. - * @template TCtx - Context type for resumption (e.g. cursor, offset). - */ -export interface Resumable { - /** The data item being streamed. */ - readonly data: TData; - /** Context for resuming from this point. */ - readonly context: TCtx; -} - -/** - * Function that reads data from an external system. - * - * @template TClient - Provider client type for connecting to the source. - * @template TData - Data type produced by the reader. - * @template TCtx - Context type for resumption (e.g. cursor, offset). - * @template TParam - Configuration parameters for the reader. - */ -export type ReaderFn = ( - client: TClient, - ctx: TCtx, - params: TParam, -) => AsyncIterable>; - -/** - * Function that returns a writer for persisting data items. - * - * @template TClient - Provider client type for connecting to the target. - * @template TData - Data type consumed by the writer. - * @template TParam - Configuration parameters for the writer. - */ -export type WriterFn = ( - client: TClient, - params: TParam, -) => (item: TData) => Promise; - -/** - * Configuration for creating a stream source. - * - * @template TClient - Provider client type for connecting to the source. - * @template TData - Data type produced by the source. - * @template TCtx - Context type for resumption. - * @template TParam - Configuration parameters for the source. - */ -export interface SourceConfig { - /** Type information: data class, context schema, and param schema. */ - readonly types: [ - dataClass: ClassRef, - contextSchema: z.ZodType, - paramSchema: z.ZodType, - ]; - /** The reader function that produces data items. */ - readonly reader: ReaderFn; -} - -/** - * Configuration for creating a stream target. - * - * @template TClient - Provider client type for connecting to the target. - * @template TData - Data type consumed by the target. - * @template TParam - Configuration parameters for the target. - */ -export interface TargetConfig { - /** Type information: data class and param schema. */ - readonly types: [dataClass: ClassRef, paramSchema: z.ZodType]; - /** The writer function that persists data items. */ - readonly writer: WriterFn; -} - -/** - * A stream source that reads data from an external system. - * - * Sources are the entry points of a pipeline, producing data items - * that flow through actions to targets. - * - * @template TClient - Provider client type for connecting to the source. - * @template TData - Data type produced by the source. - * @template TCtx - Context type for resumption. - * @template TParam - Configuration parameters for the source. - */ -export interface StreamSource< - TClient, - TData extends Data, - TCtx, - TParam = void, -> { - /** Discriminator for runtime type checking. */ - readonly kind: "source"; - /** Unique identifier for this stream source. */ - readonly id: string; - /** Class reference for the required provider client. */ - readonly clientClass: ClassRef; - /** Class reference for the data type produced. */ - readonly dataClass: ClassRef; - /** Zod schema for validating and parsing resumption context. */ - readonly contextSchema: z.ZodType; - /** Zod schema for validating stream parameters. */ - readonly paramSchema: z.ZodType; - /** Read data from the source, yielding resumable items. */ - read( - client: TClient, - ctx: TCtx, - params: TParam, - ): AsyncIterable>; -} - -/** - * A stream target that writes data to an external system. - * - * Targets are the exit points of a pipeline, persisting data items - * that have flowed from sources through actions. - * - * @template TClient - Provider client type for connecting to the target. - * @template TData - Data type consumed by the target. - * @template TParam - Configuration parameters for the target. - */ -export interface StreamTarget { - /** Discriminator for runtime type checking. */ - readonly kind: "target"; - /** Unique identifier for this stream target. */ - readonly id: string; - /** Class reference for the required provider client. */ - readonly clientClass: ClassRef; - /** Class reference for the data type consumed. */ - readonly dataClass: ClassRef; - /** Zod schema for validating stream parameters. */ - readonly paramSchema: z.ZodType; - /** Create a writer function for persisting items. */ - write(client: TClient, params: TParam): (item: TData) => Promise; -} - -class StreamSourceImpl - implements StreamSource -{ - readonly kind = "source" as const; - readonly id: string; - readonly clientClass: ClassRef; - readonly dataClass: ClassRef; - readonly contextSchema: z.ZodType; - readonly paramSchema: z.ZodType; - readonly #read: ReaderFn; - - constructor(config: { - id: string; - clientClass: ClassRef; - dataClass: ClassRef; - contextSchema: z.ZodType; - paramSchema: z.ZodType; - read: ReaderFn; - }) { - this.id = config.id; - this.clientClass = config.clientClass; - this.dataClass = config.dataClass; - this.contextSchema = config.contextSchema; - this.paramSchema = config.paramSchema; - this.#read = config.read; - } - - read( - client: TClient, - ctx: TCtx, - params: TParam, - ): AsyncIterable> { - return this.#read(client, ctx, params); - } -} - -class StreamTargetImpl - implements StreamTarget -{ - readonly kind = "target" as const; - readonly id: string; - readonly clientClass: ClassRef; - readonly dataClass: ClassRef; - readonly paramSchema: z.ZodType; - readonly #writer: WriterFn; - - constructor(config: { - id: string; - clientClass: ClassRef; - dataClass: ClassRef; - paramSchema: z.ZodType; - writer: WriterFn; - }) { - this.id = config.id; - this.clientClass = config.clientClass; - this.dataClass = config.dataClass; - this.paramSchema = config.paramSchema; - this.#writer = config.writer; - } - - write(client: TClient, params: TParam): (item: TData) => Promise { - return this.#writer(client, params); - } -} - -/** Factory for creating stream sources and targets. */ -export const Stream = { - /** - * Create a stream source for reading data from an external system. - * - * @param id - Unique identifier for the stream source. - * @param clientClass - Class reference for the required provider client. - * @param config - Source configuration including types and reader function. - */ - createSource( - id: string, - clientClass: ClassRef, - config: SourceConfig, - ): StreamSource { - const [dataClass, contextSchema, paramSchema] = config.types; - return new StreamSourceImpl({ - id, - clientClass, - dataClass, - contextSchema, - paramSchema, - read: config.reader, - }); - }, - - /** - * Create a stream target for writing data to an external system. - * - * @param id - Unique identifier for the stream target. - * @param clientClass - Class reference for the required provider client. - * @param config - Target configuration including types and writer function. - */ - createTarget( - id: string, - clientClass: ClassRef, - config: TargetConfig, - ): StreamTarget { - const [dataClass, paramSchema] = config.types; - return new StreamTargetImpl({ - id, - clientClass, - dataClass, - paramSchema, - writer: config.writer, - }); - }, -} as const; diff --git a/packages/nvisy-core/src/types.ts b/packages/nvisy-core/src/types.ts deleted file mode 100644 index 799363a..0000000 --- a/packages/nvisy-core/src/types.ts +++ /dev/null @@ -1,2 +0,0 @@ -/** Constructor reference for runtime `instanceof` checks and generic type inference. */ -export type ClassRef = abstract new (...args: never[]) => T; diff --git a/packages/nvisy-core/test/action.fixtures.ts b/packages/nvisy-core/test/action.fixtures.ts deleted file mode 100644 index 241ef6c..0000000 --- a/packages/nvisy-core/test/action.fixtures.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { z } from "zod"; -import { Action } from "../src/action.js"; -import type { JsonValue } from "../src/datatypes/data.js"; -import { Data } from "../src/datatypes/data.js"; - -/** Minimal row-like data type for testing. */ -export class TestRow extends Data { - readonly #columns: Readonly>; - - constructor(columns: Record) { - super(); - this.#columns = columns; - } - - get columns(): Readonly> { - return this.#columns; - } - - get(column: string): JsonValue | undefined { - return this.#columns[column]; - } -} - -export const FilterParams = z.object({ - column: z.string(), - value: z.string(), -}); -export type FilterParams = z.infer; - -export const ExampleFilter = Action.withoutClient("filter", { - types: [TestRow], - params: FilterParams, - transform: async function* (stream, params) { - for await (const row of stream) { - if (row.get(params.column) === params.value) yield row; - } - }, -}); - -export const MapParams = z.object({ - column: z.string(), - fn: z.enum(["uppercase", "lowercase"]), -}); -export type MapParams = z.infer; - -export const ExampleMap = Action.withoutClient("map", { - types: [TestRow], - params: MapParams, - transform: async function* (stream, params) { - for await (const row of stream) { - const val = row.get(params.column); - if (typeof val !== "string") { - yield row; - } else { - const mapped = - params.fn === "uppercase" ? val.toUpperCase() : val.toLowerCase(); - yield new TestRow({ - ...row.columns, - [params.column]: mapped, - }).deriveFrom(row); - } - } - }, -}); diff --git a/packages/nvisy-core/test/action.test.ts b/packages/nvisy-core/test/action.test.ts deleted file mode 100644 index 5b7d433..0000000 --- a/packages/nvisy-core/test/action.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { ActionInstance } from "../src/action.js"; -import type { Data } from "../src/datatypes/data.js"; -import { ExampleFilter, ExampleMap, TestRow } from "./action.fixtures.js"; - -async function collect(iter: AsyncIterable): Promise { - const result: T[] = []; - for await (const item of iter) result.push(item); - return result; -} - -async function* fromArray(items: ReadonlyArray): AsyncIterable { - yield* items; -} - -async function runAction( - // biome-ignore lint/suspicious/noExplicitAny: generic test helper - action: ActionInstance, - items: ReadonlyArray, - params: unknown, -): Promise> { - const stream = action.pipe(fromArray(items), params, undefined as undefined); - return collect(stream); -} - -const rows = [ - new TestRow({ id: "1", name: "Alice" }), - new TestRow({ id: "2", name: "Bob" }), - new TestRow({ id: "3", name: "Charlie" }), -]; - -describe("ExampleFilter", () => { - it("keeps rows matching the predicate", async () => { - const result = await runAction(ExampleFilter, rows, { - column: "name", - value: "Bob", - }); - - expect(result).toHaveLength(1); - expect(result[0]!.get("name")).toBe("Bob"); - }); - - it("returns empty array when nothing matches", async () => { - const result = await runAction(ExampleFilter, rows, { - column: "name", - value: "Nobody", - }); - - expect(result).toHaveLength(0); - }); -}); - -describe("ExampleMap", () => { - it("transforms column values to uppercase", async () => { - const result = await runAction(ExampleMap, rows, { - column: "name", - fn: "uppercase", - }); - - expect(result).toHaveLength(3); - expect(result[0]!.get("name")).toBe("ALICE"); - expect(result[1]!.get("name")).toBe("BOB"); - expect(result[2]!.get("name")).toBe("CHARLIE"); - }); - - it("transforms column values to lowercase", async () => { - const result = await runAction(ExampleMap, rows, { - column: "name", - fn: "lowercase", - }); - - expect(result[0]!.get("name")).toBe("alice"); - }); - - it("leaves non-string columns unchanged", async () => { - const result = await runAction(ExampleMap, rows, { - column: "id", - fn: "uppercase", - }); - - expect(result[0]!.get("id")).toBe("1"); - expect(result[0]!.get("name")).toBe("Alice"); - }); -}); diff --git a/packages/nvisy-core/test/provider.fixtures.ts b/packages/nvisy-core/test/provider.fixtures.ts deleted file mode 100644 index 4c1c299..0000000 --- a/packages/nvisy-core/test/provider.fixtures.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { z } from "zod"; -import type { JsonValue } from "../src/datatypes/data.js"; -import { Data } from "../src/datatypes/data.js"; -import { Provider } from "../src/provider.js"; -import type { Resumable } from "../src/stream.js"; -import { Stream } from "../src/stream.js"; - -/** Minimal row-like data type for testing. */ -export class TestRow extends Data { - readonly #columns: Readonly>; - - constructor(columns: Record) { - super(); - this.#columns = columns; - } - - get columns(): Readonly> { - return this.#columns; - } - - get(column: string): JsonValue | undefined { - return this.#columns[column]; - } -} - -export const Credentials = z.object({ - host: z.string(), - port: z.number(), -}); -export type Credentials = z.infer; - -export const Params = z.object({ - table: z.string(), -}); -export type Params = z.infer; - -export const Cursor = z.object({ - offset: z.number(), -}); -export type Cursor = z.infer; - -export class ExampleClient { - readonly rows: ReadonlyArray> = [ - { id: "1", name: "Alice" }, - { id: "2", name: "Bob" }, - { id: "3", name: "Charlie" }, - ]; -} - -async function* readStream( - client: ExampleClient, - ctx: Cursor, - _params: Params, -): AsyncIterable> { - const items = client.rows.slice(ctx.offset).map((row, i) => ({ - data: new TestRow(row), - context: { offset: ctx.offset + i + 1 }, - })); - yield* items; -} - -export const ExampleProvider = Provider.withAuthentication("example", { - credentials: Credentials, - connect: async (_credentials) => ({ - client: new ExampleClient(), - disconnect: async () => {}, - }), -}); - -export const ExampleSource = Stream.createSource("read", ExampleClient, { - types: [TestRow, Cursor, Params], - reader: (client, ctx, params) => readStream(client, ctx, params), -}); - -export const ExampleTarget = Stream.createTarget("write", ExampleClient, { - types: [TestRow, Params], - writer: (_client, _params) => async (_item) => {}, -}); diff --git a/packages/nvisy-core/test/provider.test.ts b/packages/nvisy-core/test/provider.test.ts deleted file mode 100644 index 60df76a..0000000 --- a/packages/nvisy-core/test/provider.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { beforeAll, describe, expect, it } from "vitest"; -import { - ExampleClient, - ExampleProvider, - ExampleSource, - ExampleTarget, - TestRow, -} from "./provider.fixtures.js"; - -async function collect(iter: AsyncIterable): Promise { - const result: T[] = []; - for await (const item of iter) result.push(item); - return result; -} - -describe("ExampleProvider", () => { - it("connect returns a managed instance with a client", async () => { - const instance = await ExampleProvider.connect({ - host: "localhost", - port: 5432, - }); - expect(instance.id).toBe("example"); - expect(instance.client).toBeInstanceOf(ExampleClient); - }); -}); - -describe("ExampleSource", () => { - let client: ExampleClient; - - beforeAll(async () => { - const instance = await ExampleProvider.connect({ - host: "localhost", - port: 5432, - }); - client = instance.client; - }); - - it("reads all rows from offset 0", async () => { - const collected = await collect( - ExampleSource.read(client, { offset: 0 }, { table: "users" }), - ); - - expect(collected).toHaveLength(3); - expect(collected[0]!.data.columns).toEqual({ id: "1", name: "Alice" }); - expect(collected[2]!.data.columns).toEqual({ id: "3", name: "Charlie" }); - }); - - it("resumes from a given offset", async () => { - const collected = await collect( - ExampleSource.read(client, { offset: 2 }, { table: "users" }), - ); - - expect(collected).toHaveLength(1); - expect(collected[0]!.data.columns).toEqual({ id: "3", name: "Charlie" }); - }); - - it("yields correct resumption context", async () => { - const collected = await collect( - ExampleSource.read(client, { offset: 0 }, { table: "users" }), - ); - const contexts = collected.map((r) => r.context); - - expect(contexts).toEqual([{ offset: 1 }, { offset: 2 }, { offset: 3 }]); - }); -}); - -describe("ExampleTarget", () => { - let client: ExampleClient; - - beforeAll(async () => { - const instance = await ExampleProvider.connect({ - host: "localhost", - port: 5432, - }); - client = instance.client; - }); - - it("writes a row without error", async () => { - const row = new TestRow({ id: "4", name: "Diana" }); - const writer = ExampleTarget.write(client, { table: "users" }); - await writer(row); - }); -}); diff --git a/packages/nvisy-core/tsconfig.json b/packages/nvisy-core/tsconfig.json deleted file mode 100644 index 054a6c8..0000000 --- a/packages/nvisy-core/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - /* Emit */ - "outDir": "./dist", - "rootDir": "./src", - "composite": true - }, - /* Scope */ - "include": ["src/**/*"], - "exclude": ["node_modules", "dist", "src/**/*.test.ts", "src/**/*.spec.ts"] -} diff --git a/packages/nvisy-core/tsup.config.ts b/packages/nvisy-core/tsup.config.ts deleted file mode 100644 index d68a5db..0000000 --- a/packages/nvisy-core/tsup.config.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { defineConfig } from "tsup"; - -export default defineConfig({ - /* Entry */ - entry: ["src/index.ts"], - format: ["esm"], - - /* Output */ - outDir: "dist", - dts: { compilerOptions: { composite: false } }, - sourcemap: true, - clean: true, - - /* Optimization */ - splitting: false, - treeshake: true, - skipNodeModulesBundle: true, - - /* Environment */ - platform: "node", - target: "es2024", -}); diff --git a/packages/nvisy-exif/pyproject.toml b/packages/nvisy-exif/pyproject.toml new file mode 100644 index 0000000..0cbec69 --- /dev/null +++ b/packages/nvisy-exif/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "nvisy-exif" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [ + "Pillow>=10.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.backends" + +[tool.hatch.build.targets.wheel] +packages = ["src/nvisy_exif"] diff --git a/packages/nvisy-exif/src/nvisy_exif/__init__.py b/packages/nvisy-exif/src/nvisy_exif/__init__.py new file mode 100644 index 0000000..dd6cbd8 --- /dev/null +++ b/packages/nvisy-exif/src/nvisy_exif/__init__.py @@ -0,0 +1 @@ +from .exif import read_exif, strip_exif diff --git a/packages/nvisy-exif/src/nvisy_exif/exif.py b/packages/nvisy-exif/src/nvisy_exif/exif.py new file mode 100644 index 0000000..952d804 --- /dev/null +++ b/packages/nvisy-exif/src/nvisy_exif/exif.py @@ -0,0 +1,58 @@ +"""EXIF metadata reading and stripping for images. + +Uses Pillow for EXIF handling. Supports JPEG, PNG, and TIFF formats. +These functions are designed to be callable from Rust via PyO3. +""" + +from __future__ import annotations + +import io + +from PIL import Image +from PIL.ExifTags import TAGS + + +def read_exif(image_bytes: bytes) -> dict: + """Read EXIF metadata from image bytes. + + Args: + image_bytes: Raw image bytes (JPEG, PNG, or TIFF). + + Returns: + Dictionary mapping human-readable tag names to their values. + Binary or complex values are converted to strings. + """ + img = Image.open(io.BytesIO(image_bytes)) + exif_data = img.getexif() + + result: dict[str, object] = {} + for tag_id, value in exif_data.items(): + tag_name = TAGS.get(tag_id, str(tag_id)) + # Convert bytes to hex string for JSON compatibility + if isinstance(value, bytes): + value = value.hex() + result[tag_name] = value + + return result + + +def strip_exif(image_bytes: bytes) -> bytes: + """Remove all EXIF metadata from image bytes. + + Args: + image_bytes: Raw image bytes (JPEG, PNG, or TIFF). + + Returns: + Image bytes with all EXIF metadata removed, preserving the + original format. + """ + img = Image.open(io.BytesIO(image_bytes)) + fmt = img.format or "JPEG" + + # Create a clean copy without EXIF data + clean = Image.new(img.mode, img.size) + clean.putdata(list(img.getdata())) + + buf = io.BytesIO() + clean.save(buf, format=fmt) + return buf.getvalue() diff --git a/packages/nvisy-plugin-ai/README.md b/packages/nvisy-plugin-ai/README.md deleted file mode 100644 index 573ca1f..0000000 --- a/packages/nvisy-plugin-ai/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# @nvisy/plugin-ai - -[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/build.yml) - -AI provider plugin for the Nvisy runtime, backed by the [Vercel AI SDK](https://sdk.vercel.ai). - -## Features - -- **Embedding generation** — batch-embed documents into vectors -- **Chunking** — character, section, page, embedding-similarity, and LLM-contextual strategies -- **Partitioning** — extract text from blobs and documents (auto-detect or regex rules) -- **Enrichment** — metadata extraction, NER, image/table description, and table-to-HTML via LLM - -## Overview - -Provides LLM and embedding model integrations for AI-powered data pipelines. The plugin exposes: - -- **Providers**: - - `ai/openai-completion` — OpenAI completion (language model) - - `ai/openai-embedding` — OpenAI embedding - - `ai/anthropic-completion` — Anthropic completion - - `ai/gemini-completion` — Gemini completion - - `ai/gemini-embedding` — Gemini embedding -- **Actions**: - - `ai/embed` — generate embeddings from documents (batched) - - `ai/chunk` — split documents by character, section, or page boundaries - - `ai/chunk_similarity` — split using embedding cosine-similarity thresholds - - `ai/chunk_contextual` — split using an LLM to find natural boundaries - - `ai/partition` — extract text from blobs/documents (auto or regex rules) - - `ai/partition_contextual` — AI-based contextual partitioning (stub, not yet implemented) - - `ai/enrich` — extract metadata, entities, image/table descriptions, or convert tables to HTML via LLM - -## Usage - -```ts -import { aiPlugin } from "@nvisy/plugin-ai"; - -engine.register(aiPlugin); -``` - -## Changelog - -See [CHANGELOG.md](../../CHANGELOG.md) for release notes and version history. - -## License - -Apache 2.0 License - see [LICENSE.txt](../../LICENSE.txt) - -## Support - -- **Documentation**: [docs.nvisy.com](https://docs.nvisy.com) -- **Issues**: [GitHub Issues](https://github.com/nvisycom/runtime/issues) -- **Email**: [support@nvisy.com](mailto:support@nvisy.com) diff --git a/packages/nvisy-plugin-ai/package.json b/packages/nvisy-plugin-ai/package.json deleted file mode 100644 index 35dc224..0000000 --- a/packages/nvisy-plugin-ai/package.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "@nvisy/plugin-ai", - "version": "0.1.0", - "description": "AI provider integrations for the Nvisy platform", - "type": "module", - "exports": { - ".": { - "source": "./src/index.ts", - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsup", - "build:watch": "tsup --watch", - "clean": "rimraf dist", - "typecheck": "tsc -b" - }, - "dependencies": { - "@ai-sdk/anthropic": "^3.0.36", - "@ai-sdk/google": "^3.0.20", - "@ai-sdk/openai": "^3.0.25", - "@logtape/logtape": "^2.0.2", - "@nvisy/core": "*", - "ai": "^6.0.69", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } -} diff --git a/packages/nvisy-plugin-ai/src/actions/chunk-contextual.ts b/packages/nvisy-plugin-ai/src/actions/chunk-contextual.ts deleted file mode 100644 index 41f6849..0000000 --- a/packages/nvisy-plugin-ai/src/actions/chunk-contextual.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { Action, Document } from "@nvisy/core"; -import { z } from "zod"; -import { Chunk } from "../datatypes/index.js"; -import { AICompletionClient } from "../providers/client.js"; - -const ChunkContextualParams = z.object({ - /** Maximum characters per chunk. Defaults to 2000. */ - maxChunkSize: z.number().default(2000), -}); - -/** - * Split documents into semantically meaningful chunks using an LLM. - * - * Uses a language model to determine natural chunk boundaries. - */ -export const chunkContextual = Action.withClient( - "chunk_contextual", - AICompletionClient, - { - types: [Document, Chunk], - params: ChunkContextualParams, - transform: transformChunkContextual, - }, -); - -async function* transformChunkContextual( - stream: AsyncIterable, - params: z.infer, - client: AICompletionClient, -) { - for await (const doc of stream) { - const texts = await chunkByContext(doc.content, params, client); - - for (let i = 0; i < texts.length; i++) { - yield new Chunk(texts[i]!, { - chunkIndex: i, - chunkTotal: texts.length, - }).deriveFrom(doc); - } - } -} - -/** Use an LLM to determine natural chunk boundaries. */ -async function chunkByContext( - text: string, - params: { maxChunkSize: number }, - client: AICompletionClient, -): Promise { - const result = await client.complete({ - messages: [ - { - role: "system", - content: `You are a text segmentation assistant. Split the following text into semantically coherent chunks. Each chunk should be at most ${params.maxChunkSize} characters. Return ONLY a JSON array of strings, where each string is one chunk. Do not add any explanation.`, - }, - { - role: "user", - content: text, - }, - ], - }); - - try { - const parsed = JSON.parse(result.content) as unknown; - if (Array.isArray(parsed) && parsed.every((c) => typeof c === "string")) { - return parsed as string[]; - } - } catch { - // Fall back to returning the whole text as a single chunk - } - - return [text]; -} diff --git a/packages/nvisy-plugin-ai/src/actions/chunk-similarity.ts b/packages/nvisy-plugin-ai/src/actions/chunk-similarity.ts deleted file mode 100644 index 297662d..0000000 --- a/packages/nvisy-plugin-ai/src/actions/chunk-similarity.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { Action, Document } from "@nvisy/core"; -import { z } from "zod"; -import { Chunk } from "../datatypes/index.js"; -import { EmbeddingClient } from "../providers/client.js"; - -const ChunkSimilarityParams = z.object({ - /** Cosine similarity threshold for splitting (0-1). Defaults to 0.5. */ - threshold: z.number().min(0).max(1).default(0.5), -}); - -/** - * Split documents into semantically meaningful chunks using embedding similarity. - * - * Computes embeddings for sentences and splits where cosine - * similarity drops below a threshold. - */ -export const chunkSimilarity = Action.withClient( - "chunk_similarity", - EmbeddingClient, - { - types: [Document, Chunk], - params: ChunkSimilarityParams, - transform: transformChunkSimilarity, - }, -); - -async function* transformChunkSimilarity( - stream: AsyncIterable, - params: z.infer, - client: EmbeddingClient, -) { - for await (const doc of stream) { - const texts = await chunkBySimilarity(doc.content, params, client); - - for (let i = 0; i < texts.length; i++) { - yield new Chunk(texts[i]!, { - chunkIndex: i, - chunkTotal: texts.length, - }).deriveFrom(doc); - } - } -} - -/** Split sentences into semantic groups by embedding similarity. */ -async function chunkBySimilarity( - text: string, - params: { threshold: number }, - client: EmbeddingClient, -): Promise { - const sentences = splitSentences(text); - if (sentences.length <= 1) return [text]; - - const vectors = await client.embed(sentences, {}); - - const chunks: string[] = []; - let current: string[] = [sentences[0]!]; - - for (let i = 1; i < sentences.length; i++) { - const sim = cosineSimilarity(vectors[i - 1]!, vectors[i]!); - if (sim < params.threshold) { - chunks.push(current.join(" ")); - current = []; - } - current.push(sentences[i]!); - } - if (current.length > 0) { - chunks.push(current.join(" ")); - } - - return chunks; -} - -function splitSentences(text: string): string[] { - return text - .split(/(?<=[.!?])\s+/) - .map((s) => s.trim()) - .filter((s) => s.length > 0); -} - -function cosineSimilarity(a: Float32Array, b: Float32Array): number { - let dot = 0; - let normA = 0; - let normB = 0; - for (let i = 0; i < a.length; i++) { - dot += a[i]! * b[i]!; - normA += a[i]! * a[i]!; - normB += b[i]! * b[i]!; - } - const denom = Math.sqrt(normA) * Math.sqrt(normB); - return denom === 0 ? 0 : dot / denom; -} diff --git a/packages/nvisy-plugin-ai/src/actions/chunk.ts b/packages/nvisy-plugin-ai/src/actions/chunk.ts deleted file mode 100644 index d126537..0000000 --- a/packages/nvisy-plugin-ai/src/actions/chunk.ts +++ /dev/null @@ -1,174 +0,0 @@ -import type { DocumentPage, DocumentSection } from "@nvisy/core"; -import { Action, Document } from "@nvisy/core"; -import { z } from "zod"; -import { Chunk } from "../datatypes/index.js"; - -const CharacterStrategy = z.object({ - strategy: z.literal("character"), - /** Maximum chunk size in characters. */ - size: z.number(), - /** Number of overlapping characters between chunks. */ - overlap: z.number().default(0), -}); - -const SectionStrategy = z.object({ - strategy: z.literal("section"), - /** Heading level to split on (1-6). Defaults to 2. */ - level: z.number().min(1).max(6).default(2), -}); - -const PageStrategy = z.object({ - strategy: z.literal("page"), -}); - -const ChunkParams = z.discriminatedUnion("strategy", [ - CharacterStrategy, - SectionStrategy, - PageStrategy, -]); - -/** - * Split documents into smaller chunks using various strategies. - * - * - `"character"`: fixed-size character splitting with optional overlap - * - `"section"`: split on markdown headings at a given level - * - `"page"`: split on page boundary markers in content - */ -export const chunk = Action.withoutClient("chunk", { - types: [Document, Chunk], - params: ChunkParams, - transform: transformChunk, -}); - -async function* transformChunk( - stream: AsyncIterable, - params: z.infer, -) { - for await (const doc of stream) { - switch (params.strategy) { - case "page": { - if (doc.pages != null && doc.pages.length > 0) { - for (let i = 0; i < doc.pages.length; i++) { - const page = doc.pages[i]!; - yield new Chunk(Document.fromPages([page]).content, { - chunkIndex: i, - chunkTotal: doc.pages.length, - }).deriveFrom(doc); - } - continue; - } - break; - } - case "section": { - if (doc.pages != null && doc.pages.length > 0) { - const sections = chunkSectionsByLevel(doc.pages, params.level); - for (let i = 0; i < sections.length; i++) { - const sec = sections[i]!; - yield new Chunk( - Document.fromPages([{ pageNumber: 1, sections: [sec] }]).content, - { chunkIndex: i, chunkTotal: sections.length }, - ).deriveFrom(doc); - } - continue; - } - break; - } - } - - // Fallback: string-based chunking - let texts: string[]; - switch (params.strategy) { - case "character": - texts = chunkByCharacter(doc.content, params.size, params.overlap); - break; - case "section": - texts = chunkBySection(doc.content, params.level); - break; - case "page": - texts = chunkByPage(doc.content); - break; - } - - for (let i = 0; i < texts.length; i++) { - yield new Chunk(texts[i]!, { - chunkIndex: i, - chunkTotal: texts.length, - }).deriveFrom(doc); - } - } -} - -function chunkByCharacter( - text: string, - size: number, - overlap: number, -): string[] { - const chunks: string[] = []; - let start = 0; - while (start < text.length) { - chunks.push(text.slice(start, start + size)); - start += size - overlap; - if (size - overlap <= 0) break; - } - return chunks; -} - -function chunkBySection(text: string, level: number): string[] { - const prefix = "#".repeat(level); - const pattern = new RegExp(`^${prefix}\\s`, "m"); - const parts = text.split(pattern); - - const chunks: string[] = []; - for (let i = 0; i < parts.length; i++) { - const part = parts[i]!.trim(); - if (part.length === 0) continue; - // Re-add the heading prefix for sections after the first - chunks.push(i > 0 ? `${prefix} ${part}` : part); - } - return chunks.length > 0 ? chunks : [text]; -} - -function chunkByPage(text: string): string[] { - // Split on common page break markers - const pages = text.split(/\f|\n---\n|\n\*\*\*\n/); - const chunks: string[] = []; - for (const page of pages) { - const trimmed = page.trim(); - if (trimmed.length > 0) { - chunks.push(trimmed); - } - } - return chunks.length > 0 ? chunks : [text]; -} - -/** Walk the page->section tree, collecting sections at the target depth level. */ -function chunkSectionsByLevel( - pages: readonly DocumentPage[], - targetLevel: number, -): DocumentSection[] { - const out: DocumentSection[] = []; - for (const page of pages) { - for (const section of page.sections) { - collectSectionsAtLevel(section, 1, targetLevel, out); - } - } - return out; -} - -/** Recursively traverse the section tree, collecting sections at the target depth. */ -function collectSectionsAtLevel( - section: DocumentSection, - currentLevel: number, - targetLevel: number, - out: DocumentSection[], -): void { - if (currentLevel === targetLevel) { - out.push(section); - return; - } - if (section.children) { - for (const child of section.children) { - collectSectionsAtLevel(child, currentLevel + 1, targetLevel, out); - } - } -} diff --git a/packages/nvisy-plugin-ai/src/actions/embed.ts b/packages/nvisy-plugin-ai/src/actions/embed.ts deleted file mode 100644 index e5ba913..0000000 --- a/packages/nvisy-plugin-ai/src/actions/embed.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { Action, Document, Embedding } from "@nvisy/core"; -import { z } from "zod"; -import { EmbeddingClient } from "../providers/client.js"; - -const EmbedParams = z.object({ - /** Desired embedding dimensions (if supported by the model). */ - dimensions: z.number().optional(), - /** Number of documents to embed per API call. */ - batchSize: z.number().default(64), -}); - -/** - * Generate embeddings for documents using an AI provider. - * - * Consumes {@link Document} items, batches their content, calls the - * provider's embedding API, and yields one {@link Embedding} per document. - */ -export const embed = Action.withClient("embed", EmbeddingClient, { - types: [Document, Embedding], - params: EmbedParams, - transform: transformEmbed, -}); - -async function* transformEmbed( - stream: AsyncIterable, - params: z.infer, - client: EmbeddingClient, -) { - let batch: Document[] = []; - - for await (const doc of stream) { - batch.push(doc); - if (batch.length >= params.batchSize) { - yield* emitBatch(batch, client, params.dimensions); - batch = []; - } - } - - if (batch.length > 0) { - yield* emitBatch(batch, client, params.dimensions); - } -} - -async function* emitBatch( - batch: Document[], - client: EmbeddingClient, - dimensions: number | undefined, -): AsyncIterable { - const texts = batch.map((doc) => doc.content); - - const vectors = await client.embed(texts, { - ...(dimensions != null ? { dimensions } : {}), - }); - - for (let i = 0; i < batch.length; i++) { - const doc = batch[i]!; - const vector = vectors[i]!; - yield new Embedding(vector).deriveFrom(doc); - } -} diff --git a/packages/nvisy-plugin-ai/src/actions/enrich.ts b/packages/nvisy-plugin-ai/src/actions/enrich.ts deleted file mode 100644 index 221ace3..0000000 --- a/packages/nvisy-plugin-ai/src/actions/enrich.ts +++ /dev/null @@ -1,182 +0,0 @@ -import type { Metadata } from "@nvisy/core"; -import { Action, Document } from "@nvisy/core"; -import { z } from "zod"; -import { AICompletionClient } from "../providers/client.js"; - -const MetadataType = z.object({ - type: z.literal("metadata"), - /** Field names to extract from the document. */ - fields: z.array(z.string()), -}); - -const NerType = z.object({ - type: z.literal("ner"), - /** Entity types to extract (e.g. ["PERSON", "ORG"]). If omitted, extract all. */ - entityTypes: z.array(z.string()).optional(), -}); - -const ImageDescriptionType = z.object({ - type: z.literal("image_description"), -}); - -const TableDescriptionType = z.object({ - type: z.literal("table_description"), -}); - -const TableToHtmlType = z.object({ - type: z.literal("table_to_html"), -}); - -const EnrichParams = z.discriminatedUnion("type", [ - MetadataType, - NerType, - ImageDescriptionType, - TableDescriptionType, - TableToHtmlType, -]); - -/** - * Enrich documents with AI-extracted metadata. - * - * - `"metadata"`: extract structured fields from content - * - `"ner"`: named entity recognition - * - `"image_description"`: describe image content - * - `"table_description"`: describe table content - * - `"table_to_html"`: convert table content to HTML - */ -export const enrich = Action.withClient("enrich", AICompletionClient, { - types: [Document], - params: EnrichParams, - transform: transformEnrich, -}); - -async function* transformEnrich( - stream: AsyncIterable, - params: z.infer, - client: AICompletionClient, -) { - for await (const doc of stream) { - let enrichedMeta: Metadata; - - switch (params.type) { - case "metadata": - enrichedMeta = await extractMetadata( - doc.content, - params.fields, - client, - ); - break; - case "ner": - enrichedMeta = await extractEntities( - doc.content, - params.entityTypes, - client, - ); - break; - case "image_description": - enrichedMeta = await describeContent(doc.content, "image", client); - break; - case "table_description": - enrichedMeta = await describeContent(doc.content, "table", client); - break; - case "table_to_html": - enrichedMeta = await convertTableToHtml(doc.content, client); - break; - } - - yield new Document(doc.content, { - ...(doc.sourceType != null ? { sourceType: doc.sourceType } : {}), - ...(doc.pages != null ? { pages: doc.pages } : {}), - }) - .deriveFrom(doc) - .withMetadata({ ...(doc.metadata ?? {}), ...enrichedMeta }); - } -} - -async function extractMetadata( - text: string, - fields: string[], - client: AICompletionClient, -): Promise { - const result = await client.complete({ - messages: [ - { - role: "system", - content: `Extract the following fields from the document: ${fields.join(", ")}. Return ONLY a JSON object with these fields as keys. If a field cannot be determined, set it to null.`, - }, - { role: "user", content: text }, - ], - }); - return parseJsonResponse(result.content, "extracted"); -} - -async function extractEntities( - text: string, - entityTypes: string[] | undefined, - client: AICompletionClient, -): Promise { - const typeClause = entityTypes - ? `Focus on these entity types: ${entityTypes.join(", ")}.` - : "Extract all entity types you can identify."; - - const result = await client.complete({ - messages: [ - { - role: "system", - content: `Perform named entity recognition on the following text. ${typeClause} Return ONLY a JSON object where keys are entity types and values are arrays of extracted entities.`, - }, - { role: "user", content: text }, - ], - }); - return { entities: parseJsonResponse(result.content, "entities") }; -} - -async function describeContent( - text: string, - contentKind: "image" | "table", - client: AICompletionClient, -): Promise { - const result = await client.complete({ - messages: [ - { - role: "system", - content: `Describe the following ${contentKind} content in detail. Return ONLY a JSON object with a "description" field containing your description.`, - }, - { role: "user", content: text }, - ], - }); - return parseJsonResponse(result.content, "description"); -} - -async function convertTableToHtml( - text: string, - client: AICompletionClient, -): Promise { - const result = await client.complete({ - messages: [ - { - role: "system", - content: - 'Convert the following table content into clean HTML. Return ONLY a JSON object with an "html" field containing the HTML table markup.', - }, - { role: "user", content: text }, - ], - }); - return parseJsonResponse(result.content, "tableHtml"); -} - -function parseJsonResponse(content: string, fallbackKey: string): Metadata { - try { - const parsed = JSON.parse(content) as Record; - if ( - typeof parsed === "object" && - parsed !== null && - !Array.isArray(parsed) - ) { - return parsed as Metadata; - } - } catch { - // If JSON parsing fails, store the raw response - } - return { [fallbackKey]: content }; -} diff --git a/packages/nvisy-plugin-ai/src/actions/index.ts b/packages/nvisy-plugin-ai/src/actions/index.ts deleted file mode 100644 index bf7d74e..0000000 --- a/packages/nvisy-plugin-ai/src/actions/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export { chunk } from "./chunk.js"; -export { chunkContextual } from "./chunk-contextual.js"; -export { chunkSimilarity } from "./chunk-similarity.js"; -export { embed } from "./embed.js"; -export { enrich } from "./enrich.js"; -export { partition } from "./partition.js"; -export { partitionContextual } from "./partition-contextual.js"; diff --git a/packages/nvisy-plugin-ai/src/actions/partition-contextual.ts b/packages/nvisy-plugin-ai/src/actions/partition-contextual.ts deleted file mode 100644 index 51b7233..0000000 --- a/packages/nvisy-plugin-ai/src/actions/partition-contextual.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Action, Document, RuntimeError } from "@nvisy/core"; -import { z } from "zod"; -import { AICompletionClient } from "../providers/client.js"; - -const PartitionContextualParams = z.object({}); - -/** - * Partition documents and blobs using an AI model for contextual analysis. - * - * This action is a placeholder — it throws "not yet implemented" - * until AI-based contextual partitioning support is added. - */ -export const partitionContextual = Action.withClient( - "partition_contextual", - AICompletionClient, - { - types: [Document], - params: PartitionContextualParams, - transform: transformPartitionContextual, - }, -); - -// biome-ignore lint/correctness/useYield: stub action throws before yielding -async function* transformPartitionContextual( - _stream: AsyncIterable, - _params: z.infer, - _client: AICompletionClient, -) { - throw new RuntimeError("partition_contextual is not yet implemented", { - source: "ai/partition_contextual", - retryable: false, - }); -} diff --git a/packages/nvisy-plugin-ai/src/actions/partition.ts b/packages/nvisy-plugin-ai/src/actions/partition.ts deleted file mode 100644 index fd5e275..0000000 --- a/packages/nvisy-plugin-ai/src/actions/partition.ts +++ /dev/null @@ -1,75 +0,0 @@ -import type { Metadata } from "@nvisy/core"; -import { Action, Document } from "@nvisy/core"; -import { z } from "zod"; - -const AutoStrategy = z.object({ - strategy: z.literal("auto"), -}); - -const RuleStrategy = z.object({ - strategy: z.literal("rule"), - /** Regex pattern to split content on. */ - pattern: z.string(), - /** Whether to include the delimiter in chunks. Defaults to false. */ - includeDelimiter: z.boolean().default(false), -}); - -const PartitionParams = z.discriminatedUnion("strategy", [ - AutoStrategy, - RuleStrategy, -]); - -/** - * Partition documents into structured documents. - * - * - `"auto"`: pass through document content as-is - * - `"rule"`: split content using a regex pattern - */ -export const partition = Action.withoutClient("partition", { - types: [Document, Document], - params: PartitionParams, - transform: transformPartition, -}); - -async function* transformPartition( - stream: AsyncIterable, - params: z.infer, -) { - for await (const item of stream) { - const text = item.content; - const sourceId = item.id; - const baseMeta = item.metadata; - - let parts: string[]; - - switch (params.strategy) { - case "auto": - parts = [text]; - break; - case "rule": { - const regex = new RegExp(params.pattern, "g"); - if (params.includeDelimiter) { - parts = text.split(regex).filter((p) => p.length > 0); - } else { - parts = text.split(regex).filter((p) => p.length > 0); - } - break; - } - } - - for (let i = 0; i < parts.length; i++) { - const metadata: Metadata = { - ...(baseMeta ?? {}), - partIndex: i, - partTotal: parts.length, - }; - yield new Document(parts[i]!, { - ...(params.strategy === "auto" && item.pages != null - ? { pages: item.pages } - : {}), - }) - .withParent(sourceId) - .withMetadata(metadata); - } - } -} diff --git a/packages/nvisy-plugin-ai/src/datatypes/chunk.ts b/packages/nvisy-plugin-ai/src/datatypes/chunk.ts deleted file mode 100644 index aef1b85..0000000 --- a/packages/nvisy-plugin-ai/src/datatypes/chunk.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Data } from "@nvisy/core"; - -/** Options for constructing a {@link Chunk}. */ -export interface ChunkOptions { - readonly chunkIndex?: number; - readonly chunkTotal?: number; -} - -/** - * A text segment produced by a chunking step. - * - * Represents a portion of a larger {@link Document} after splitting. - * Carries optional provenance fields ({@link chunkIndex}, - * {@link chunkTotal}) so downstream steps can trace chunks back to their - * origin. Use {@link Data.withParent | withParent} to set the source document ID. - * - * @example - * ```ts - * const chunk = new Chunk("First paragraph…", { - * chunkIndex: 0, - * chunkTotal: 5, - * }).deriveFrom(doc); - * ``` - */ -export class Chunk extends Data { - readonly #content: string; - readonly #chunkIndex?: number | undefined; - readonly #chunkTotal?: number | undefined; - - constructor(content: string, options?: ChunkOptions) { - super(); - this.#content = content; - this.#chunkIndex = options?.chunkIndex; - this.#chunkTotal = options?.chunkTotal; - } - - /** Text content of this chunk. */ - get content(): string { - return this.#content; - } - - /** Zero-based index of this chunk within the source document. */ - get chunkIndex(): number | undefined { - return this.#chunkIndex; - } - - /** Total number of chunks the source document was split into. */ - get chunkTotal(): number | undefined { - return this.#chunkTotal; - } -} diff --git a/packages/nvisy-plugin-ai/src/datatypes/index.ts b/packages/nvisy-plugin-ai/src/datatypes/index.ts deleted file mode 100644 index 17b2f57..0000000 --- a/packages/nvisy-plugin-ai/src/datatypes/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export type { ChunkOptions } from "./chunk.js"; -export { Chunk } from "./chunk.js"; diff --git a/packages/nvisy-plugin-ai/src/index.ts b/packages/nvisy-plugin-ai/src/index.ts deleted file mode 100644 index e8b8a49..0000000 --- a/packages/nvisy-plugin-ai/src/index.ts +++ /dev/null @@ -1,60 +0,0 @@ -/** - * @module @nvisy/plugin-ai - * - * AI provider plugin for the Nvisy runtime. - * - * Exposes LLM providers (OpenAI, Anthropic, Gemini), embedding generation, - * chunking, partitioning, and enrichment actions for AI-powered pipelines. - * - * Backed by the Vercel AI SDK for unified provider access. - * - * @example - * ```ts - * import { aiPlugin } from "@nvisy/plugin-ai"; - * - * engine.register(aiPlugin); - * ``` - */ - -import { Datatypes, Plugin } from "@nvisy/core"; -import { - chunk, - chunkContextual, - chunkSimilarity, - embed, - enrich, - partition, - partitionContextual, -} from "./actions/index.js"; -import { Chunk } from "./datatypes/index.js"; -import { - anthropicCompletion, - geminiCompletion, - geminiEmbedding, - openaiCompletion, - openaiEmbedding, -} from "./providers/index.js"; - -/** The AI plugin: register this with the runtime to enable all AI providers and actions. */ -export const aiPlugin = Plugin.define("ai") - .withProviders( - openaiCompletion, - openaiEmbedding, - anthropicCompletion, - geminiCompletion, - geminiEmbedding, - ) - .withActions( - embed, - chunk, - chunkSimilarity, - chunkContextual, - partition, - partitionContextual, - enrich, - ) - .withDatatypes(Datatypes.define("chunk", Chunk)); - -export { Embedding } from "@nvisy/core"; -export type { ChunkOptions } from "./datatypes/index.js"; -export { Chunk } from "./datatypes/index.js"; diff --git a/packages/nvisy-plugin-ai/src/providers/anthropic.ts b/packages/nvisy-plugin-ai/src/providers/anthropic.ts deleted file mode 100644 index fb46c5e..0000000 --- a/packages/nvisy-plugin-ai/src/providers/anthropic.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { createAnthropic } from "@ai-sdk/anthropic"; -import { generateText } from "ai"; -import { - makeProvider, - type ProviderConnection, - VercelCompletionClient, -} from "./client.js"; - -function makeAnthropic(credentials: ProviderConnection) { - return createAnthropic({ - apiKey: credentials.apiKey, - ...(credentials.baseUrl != null ? { baseURL: credentials.baseUrl } : {}), - }); -} - -/** Anthropic completion provider factory backed by the Vercel AI SDK. */ -export const anthropicCompletion = makeProvider({ - id: "anthropic-completion", - createClient: (credentials) => - new VercelCompletionClient({ - languageModel: makeAnthropic(credentials)(credentials.model), - }), - verify: async (credentials) => { - const provider = makeAnthropic(credentials); - await generateText({ - model: provider(credentials.model), - prompt: "hi", - maxOutputTokens: 1, - }); - }, -}); diff --git a/packages/nvisy-plugin-ai/src/providers/client.ts b/packages/nvisy-plugin-ai/src/providers/client.ts deleted file mode 100644 index 0d92e04..0000000 --- a/packages/nvisy-plugin-ai/src/providers/client.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { ConnectionError, Provider, type ProviderFactory } from "@nvisy/core"; -import type { EmbeddingModel, LanguageModel } from "ai"; -import { embedMany, generateText } from "ai"; -import type { ProviderConnection } from "./schemas.js"; -import { ProviderConnection as ApiKeyCredentialsSchema } from "./schemas.js"; - -export type { ProviderConnection } from "./schemas.js"; - -const logger = getLogger(["nvisy", "ai"]); - -/** A single message in a chat conversation. */ -export interface ChatMessage { - readonly role: "system" | "user" | "assistant"; - readonly content: string; -} - -/** Options for a completion request. */ -export interface CompletionOptions { - /** Messages comprising the conversation. */ - readonly messages: ReadonlyArray; - /** Sampling temperature (0-2). */ - readonly temperature?: number; - /** Maximum tokens to generate. */ - readonly maxTokens?: number; -} - -/** Result of a completion request. */ -export interface CompletionResult { - /** Generated text content. */ - readonly content: string; - /** Token usage statistics. */ - readonly usage?: { - readonly promptTokens: number; - readonly completionTokens: number; - }; -} - -/** Options for an embedding request. */ -export interface EmbedOptions { - /** Desired embedding dimensions (if supported by the model). */ - readonly dimensions?: number; -} - -/** - * Abstract AI client with completion capability. - */ -export abstract class AICompletionClient { - abstract complete(options: CompletionOptions): Promise; -} - -/** - * Abstract AI client with embedding capability. - */ -export abstract class EmbeddingClient { - abstract embed( - input: string[], - options: EmbedOptions, - ): Promise; -} - -/** Build the generateText call options from our CompletionOptions. */ -function buildGenerateTextArgs( - model: LanguageModel, - options: CompletionOptions, -) { - const systemParts = options.messages - .filter((m) => m.role === "system") - .map((m) => m.content); - - const systemText = - systemParts.length > 0 ? systemParts.join("\n\n") : undefined; - - return { - model, - ...(systemText != null ? { system: systemText } : {}), - messages: options.messages - .filter((m) => m.role !== "system") - .map((m) => ({ - role: m.role as "user" | "assistant", - content: m.content, - })), - ...(options.temperature != null - ? { temperature: options.temperature } - : {}), - ...(options.maxTokens != null - ? { maxOutputTokens: options.maxTokens } - : {}), - }; -} - -/** Map AI SDK usage to our CompletionResult format. */ -function mapUsage( - usage: - | { inputTokens: number | undefined; outputTokens: number | undefined } - | undefined, -): CompletionResult["usage"] { - if (!usage || usage.inputTokens == null || usage.outputTokens == null) { - return undefined; - } - return { - promptTokens: usage.inputTokens, - completionTokens: usage.outputTokens, - }; -} - -/** - * Embedding-only AI client backed by the Vercel AI SDK. - * - * Uses {@link embedMany} from the `ai` package, - * delegating model creation to provider-specific factories. - */ -export class VercelEmbeddingClient extends EmbeddingClient { - readonly #model: EmbeddingModel; - - constructor(config: { embeddingModel: EmbeddingModel }) { - super(); - this.#model = config.embeddingModel; - } - - async embed( - input: string[], - _options: EmbedOptions, - ): Promise { - const result = await embedMany({ - model: this.#model, - values: input, - }); - - return result.embeddings.map((e) => new Float32Array(e)); - } -} - -/** - * Completion-only AI client backed by the Vercel AI SDK. - */ -export class VercelCompletionClient extends AICompletionClient { - readonly #model: LanguageModel; - - constructor(config: { languageModel: LanguageModel }) { - super(); - this.#model = config.languageModel; - } - - async complete(options: CompletionOptions): Promise { - const result = await generateText( - buildGenerateTextArgs(this.#model, options), - ); - const usage = mapUsage(result.usage); - return { - content: result.text, - ...(usage != null ? { usage } : {}), - }; - } -} - -/** Normalise an unknown throw into a {@link ConnectionError}. */ -function toConnectionError(error: unknown, source: string): ConnectionError { - if (error instanceof ConnectionError) return error; - logger.error("Connection to {provider} failed: {error}", { - provider: source, - error: error instanceof Error ? error.message : String(error), - }); - return ConnectionError.wrap(error, { source }); -} - -/** Configuration for {@link makeProvider}. */ -export interface ProviderConfig { - /** Unique provider identifier, e.g. `"openai"`, `"anthropic"`, `"gemini"`. */ - readonly id: string; - /** Factory that creates an AI client from validated credentials. */ - readonly createClient: (credentials: ProviderConnection) => TClient; - /** Verify the connection is live (called once during connect). */ - readonly verify: (credentials: ProviderConnection) => Promise; -} - -/** - * Create an AI {@link ProviderFactory} parameterised by a client constructor. - * - * The returned factory validates {@link ProviderConnection} at parse time, then - * creates the client on connect after verifying connectivity. - */ -export const makeProvider = ( - config: ProviderConfig, -): ProviderFactory => - Provider.withAuthentication(config.id, { - credentials: ApiKeyCredentialsSchema, - verify: config.verify, - connect: async (credentials) => { - try { - const client = config.createClient(credentials); - logger.info("Connected to {provider}", { provider: config.id }); - return { - client, - disconnect: async () => { - logger.debug("Disconnected from {provider}", { - provider: config.id, - }); - }, - }; - } catch (error) { - throw toConnectionError(error, config.id); - } - }, - }); diff --git a/packages/nvisy-plugin-ai/src/providers/gemini.ts b/packages/nvisy-plugin-ai/src/providers/gemini.ts deleted file mode 100644 index 22e1286..0000000 --- a/packages/nvisy-plugin-ai/src/providers/gemini.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createGoogleGenerativeAI } from "@ai-sdk/google"; -import { embedMany, generateText } from "ai"; -import { - makeProvider, - type ProviderConnection, - VercelCompletionClient, - VercelEmbeddingClient, -} from "./client.js"; - -function makeGemini(credentials: ProviderConnection) { - return createGoogleGenerativeAI({ - apiKey: credentials.apiKey, - ...(credentials.baseUrl != null ? { baseURL: credentials.baseUrl } : {}), - }); -} - -/** Gemini completion provider factory backed by the Vercel AI SDK. */ -export const geminiCompletion = makeProvider({ - id: "gemini-completion", - createClient: (credentials) => - new VercelCompletionClient({ - languageModel: makeGemini(credentials)(credentials.model), - }), - verify: async (credentials) => { - const provider = makeGemini(credentials); - await generateText({ - model: provider(credentials.model), - prompt: "hi", - maxOutputTokens: 1, - }); - }, -}); - -/** Gemini embedding provider factory backed by the Vercel AI SDK. */ -export const geminiEmbedding = makeProvider({ - id: "gemini-embedding", - createClient: (credentials) => - new VercelEmbeddingClient({ - embeddingModel: makeGemini(credentials).embeddingModel(credentials.model), - }), - verify: async (credentials) => { - const provider = makeGemini(credentials); - await embedMany({ - model: provider.embeddingModel(credentials.model), - values: ["test"], - }); - }, -}); diff --git a/packages/nvisy-plugin-ai/src/providers/index.ts b/packages/nvisy-plugin-ai/src/providers/index.ts deleted file mode 100644 index 711b3f1..0000000 --- a/packages/nvisy-plugin-ai/src/providers/index.ts +++ /dev/null @@ -1,18 +0,0 @@ -export { anthropicCompletion } from "./anthropic.js"; -export type { - ChatMessage, - CompletionOptions, - CompletionResult, - EmbedOptions, - ProviderConfig, - ProviderConnection, -} from "./client.js"; -export { - AICompletionClient, - EmbeddingClient, - makeProvider, - VercelCompletionClient, - VercelEmbeddingClient, -} from "./client.js"; -export { geminiCompletion, geminiEmbedding } from "./gemini.js"; -export { openaiCompletion, openaiEmbedding } from "./openai.js"; diff --git a/packages/nvisy-plugin-ai/src/providers/openai.ts b/packages/nvisy-plugin-ai/src/providers/openai.ts deleted file mode 100644 index 9862a17..0000000 --- a/packages/nvisy-plugin-ai/src/providers/openai.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createOpenAI } from "@ai-sdk/openai"; -import { embedMany, generateText } from "ai"; -import { - makeProvider, - type ProviderConnection, - VercelCompletionClient, - VercelEmbeddingClient, -} from "./client.js"; - -function makeOpenAI(credentials: ProviderConnection) { - return createOpenAI({ - apiKey: credentials.apiKey, - ...(credentials.baseUrl != null ? { baseURL: credentials.baseUrl } : {}), - }); -} - -/** OpenAI completion provider factory backed by the Vercel AI SDK. */ -export const openaiCompletion = makeProvider({ - id: "openai-completion", - createClient: (credentials) => - new VercelCompletionClient({ - languageModel: makeOpenAI(credentials)(credentials.model), - }), - verify: async (credentials) => { - const provider = makeOpenAI(credentials); - await generateText({ - model: provider(credentials.model), - prompt: "hi", - maxOutputTokens: 1, - }); - }, -}); - -/** OpenAI embedding provider factory backed by the Vercel AI SDK. */ -export const openaiEmbedding = makeProvider({ - id: "openai-embedding", - createClient: (credentials) => - new VercelEmbeddingClient({ - embeddingModel: makeOpenAI(credentials).embedding(credentials.model), - }), - verify: async (credentials) => { - const provider = makeOpenAI(credentials); - await embedMany({ - model: provider.embedding(credentials.model), - values: ["test"], - }); - }, -}); diff --git a/packages/nvisy-plugin-ai/src/providers/schemas.ts b/packages/nvisy-plugin-ai/src/providers/schemas.ts deleted file mode 100644 index 10c6f7b..0000000 --- a/packages/nvisy-plugin-ai/src/providers/schemas.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { z } from "zod"; - -/** - * API key credentials shared by all AI providers. - * - * Validated at graph parse time before any connection is attempted. - */ -export const ProviderConnection = z.object({ - /** API key for authentication. */ - apiKey: z.string(), - /** Optional custom base URL for the API. */ - baseUrl: z.string().optional(), - /** Model identifier bound to this connection. */ - model: z.string(), -}); -export type ProviderConnection = z.infer; diff --git a/packages/nvisy-plugin-ai/tsconfig.json b/packages/nvisy-plugin-ai/tsconfig.json deleted file mode 100644 index c91a2dd..0000000 --- a/packages/nvisy-plugin-ai/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - /* Emit */ - "outDir": "./dist", - "rootDir": "./src", - "composite": true - }, - /* Scope */ - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"], - "references": [{ "path": "../nvisy-core" }] -} diff --git a/packages/nvisy-plugin-ai/tsup.config.ts b/packages/nvisy-plugin-ai/tsup.config.ts deleted file mode 100644 index d68a5db..0000000 --- a/packages/nvisy-plugin-ai/tsup.config.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { defineConfig } from "tsup"; - -export default defineConfig({ - /* Entry */ - entry: ["src/index.ts"], - format: ["esm"], - - /* Output */ - outDir: "dist", - dts: { compilerOptions: { composite: false } }, - sourcemap: true, - clean: true, - - /* Optimization */ - splitting: false, - treeshake: true, - skipNodeModulesBundle: true, - - /* Environment */ - platform: "node", - target: "es2024", -}); diff --git a/packages/nvisy-plugin-object/package.json b/packages/nvisy-plugin-object/package.json deleted file mode 100644 index cdb6e8c..0000000 --- a/packages/nvisy-plugin-object/package.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "@nvisy/plugin-object", - "version": "0.1.0", - "description": "Object store integrations for the Nvisy platform", - "type": "module", - "exports": { - ".": { - "source": "./src/index.ts", - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsup", - "build:watch": "tsup --watch", - "clean": "rimraf dist", - "typecheck": "tsc -b" - }, - "dependencies": { - "@aws-sdk/client-s3": "^3.750.0", - "@azure/storage-blob": "^12.26.0", - "@google-cloud/storage": "^7.15.0", - "@logtape/logtape": "^2.0.2", - "@nvisy/core": "*", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } -} diff --git a/packages/nvisy-plugin-object/src/index.ts b/packages/nvisy-plugin-object/src/index.ts deleted file mode 100644 index 60f8f54..0000000 --- a/packages/nvisy-plugin-object/src/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -/** - * @module @nvisy/plugin-object - * - * Object store plugin for the Nvisy runtime. - * - * Exposes S3, GCS, and Azure Blob providers, plus read/write streams - * that list, get, and put objects as {@link Blob}s. - * - * @example - * ```ts - * import { objectPlugin } from "@nvisy/plugin-object"; - * - * // Register with the runtime - * runtime.register(objectPlugin); - * ``` - */ - -import { Plugin } from "@nvisy/core"; -import { azure, gcs, s3 } from "./providers/index.js"; -import { read, write } from "./streams/index.js"; - -/** The Object plugin: register this with the runtime to enable object store providers and streams. */ -export const objectPlugin = Plugin.define("object") - .withProviders(s3, gcs, azure) - .withStreams(read, write); - -export type { ListResult } from "./providers/index.js"; -export { ObjectStoreClient } from "./providers/index.js"; diff --git a/packages/nvisy-plugin-object/src/providers/azure.ts b/packages/nvisy-plugin-object/src/providers/azure.ts deleted file mode 100644 index 0ceef9e..0000000 --- a/packages/nvisy-plugin-object/src/providers/azure.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { - type BlobHTTPHeaders, - BlobServiceClient, - type BlockBlobUploadOptions, - type ContainerClient, - StorageSharedKeyCredential, -} from "@azure/storage-blob"; -import { getLogger } from "@logtape/logtape"; -import { z } from "zod"; -import { - type ListResult, - makeObjectProvider, - ObjectStoreClient, - ObjectStoreProvider, -} from "./client.js"; - -const logger = getLogger(["nvisy", "object"]); - -/** - * Credentials for connecting to Azure Blob Storage. - */ -export const AzureCredentials = z.object({ - /** Azure storage account name. */ - accountName: z.string(), - /** Azure Blob container name. */ - containerName: z.string(), - /** Storage account key (provide this or `connectionString`). */ - accountKey: z.string().optional(), - /** Full connection string (provide this or `accountKey`). */ - connectionString: z.string().optional(), -}); -export type AzureCredentials = z.infer; - -class AzureObjectStoreClient extends ObjectStoreClient { - readonly #container: ContainerClient; - - constructor(container: ContainerClient) { - super(); - this.#container = container; - } - - async list(prefix: string, cursor?: string): Promise { - const keys: string[] = []; - const iter = cursor - ? this.#container - .listBlobsFlat({ prefix }) - .byPage({ continuationToken: cursor }) - : this.#container.listBlobsFlat({ prefix }).byPage(); - - const page = await iter.next(); - if (!page.done) { - for (const blob of page.value.segment.blobItems) { - keys.push(blob.name); - } - const token = page.value.continuationToken; - if (token) { - return { keys, nextCursor: token }; - } - } - return { keys }; - } - - async get(key: string): Promise<{ data: Buffer; contentType?: string }> { - const blobClient = this.#container.getBlobClient(key); - const response = await blobClient.download(); - const body = response.readableStreamBody; - if (!body) throw new Error(`Empty response body for blob "${key}"`); - - const chunks: Buffer[] = []; - for await (const chunk of body) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - } - const contentType = response.contentType; - if (contentType) { - return { data: Buffer.concat(chunks), contentType }; - } - return { data: Buffer.concat(chunks) }; - } - - async put(key: string, data: Buffer, contentType?: string): Promise { - const blockClient = this.#container.getBlockBlobClient(key); - const opts: BlockBlobUploadOptions = {}; - if (contentType) { - const headers: BlobHTTPHeaders = { blobContentType: contentType }; - opts.blobHTTPHeaders = headers; - } - await blockClient.upload(data, data.byteLength, opts); - } -} - -function createContainerClient(creds: AzureCredentials): ContainerClient { - if (creds.connectionString) { - return BlobServiceClient.fromConnectionString( - creds.connectionString, - ).getContainerClient(creds.containerName); - } - if (creds.accountKey) { - const sharedKey = new StorageSharedKeyCredential( - creds.accountName, - creds.accountKey, - ); - const service = new BlobServiceClient( - `https://${creds.accountName}.blob.core.windows.net`, - sharedKey, - ); - return service.getContainerClient(creds.containerName); - } - throw new Error( - "Azure credentials must include either accountKey or connectionString", - ); -} - -/** Azure Blob Storage provider. */ -export const azure = makeObjectProvider( - "azure", - AzureCredentials, - async (creds) => { - logger.debug( - "Connecting to Azure container {containerName} in account {accountName}", - { containerName: creds.containerName, accountName: creds.accountName }, - ); - - const container = createContainerClient(creds); - - return new ObjectStoreProvider( - new AzureObjectStoreClient(container), - "azure", - ); - }, -); diff --git a/packages/nvisy-plugin-object/src/providers/client.ts b/packages/nvisy-plugin-object/src/providers/client.ts deleted file mode 100644 index 30d23c7..0000000 --- a/packages/nvisy-plugin-object/src/providers/client.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { - Provider, - type ProviderFactory, - type ProviderInstance, -} from "@nvisy/core"; -import type { z } from "zod"; - -const logger = getLogger(["nvisy", "object"]); - -/** - * Result of listing objects under a prefix. - */ -export interface ListResult { - /** Object keys returned in this page. */ - readonly keys: string[]; - /** Opaque cursor for fetching the next page, or `undefined` if exhausted. */ - readonly nextCursor?: string; -} - -/** - * Abstract client that object-store streams use for I/O. - * - * Each provider (S3, GCS, Azure) supplies a concrete subclass. - * The class reference is required by {@link Stream.createSource} and - * {@link Stream.createTarget} for runtime client-type matching. - */ -export abstract class ObjectStoreClient { - /** List object keys under `prefix`, optionally resuming from `cursor`. */ - abstract list(prefix: string, cursor?: string): Promise; - - /** Retrieve a single object by key. */ - abstract get(key: string): Promise<{ data: Buffer; contentType?: string }>; - - /** Write a single object by key. */ - abstract put(key: string, data: Buffer, contentType?: string): Promise; -} - -/** - * Connected object-store provider instance. - * - * Holds an {@link ObjectStoreClient} and manages teardown on - * {@link disconnect}. - */ -export class ObjectStoreProvider - implements ProviderInstance -{ - readonly client: ObjectStoreClient; - readonly #id: string; - readonly #disconnect: (() => Promise) | undefined; - - constructor( - client: ObjectStoreClient, - id: string, - disconnect?: () => Promise, - ) { - this.client = client; - this.#id = id; - this.#disconnect = disconnect; - } - - async disconnect(): Promise { - await this.#disconnect?.(); - logger.debug("Disconnected from {provider}", { provider: this.#id }); - } -} - -/** - * Create an object-store {@link ProviderFactory} from a credential schema - * and a connect function. - * - * This mirrors {@link makeSqlProvider} but is generic over credentials - * so that S3, GCS, and Azure can each supply their own schema. - */ -export const makeObjectProvider = ( - id: string, - credentials: z.ZodType, - connect: (creds: TCred) => Promise>, -): ProviderFactory => - Provider.withAuthentication(id, { - credentials, - connect, - }); diff --git a/packages/nvisy-plugin-object/src/providers/gcs.ts b/packages/nvisy-plugin-object/src/providers/gcs.ts deleted file mode 100644 index 20c6c63..0000000 --- a/packages/nvisy-plugin-object/src/providers/gcs.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { Storage, type StorageOptions } from "@google-cloud/storage"; -import { getLogger } from "@logtape/logtape"; -import { z } from "zod"; -import { - type ListResult, - makeObjectProvider, - ObjectStoreClient, - ObjectStoreProvider, -} from "./client.js"; - -const logger = getLogger(["nvisy", "object"]); - -/** - * Credentials for connecting to Google Cloud Storage. - */ -export const GcsCredentials = z.object({ - /** GCP project ID. */ - projectId: z.string(), - /** GCS bucket name. */ - bucket: z.string(), - /** Path to a service-account key file (optional if running on GCE). */ - keyFilename: z.string().optional(), -}); -export type GcsCredentials = z.infer; - -class GcsObjectStoreClient extends ObjectStoreClient { - readonly #storage: Storage; - readonly #bucket: string; - - constructor(storage: Storage, bucket: string) { - super(); - this.#storage = storage; - this.#bucket = bucket; - } - - async list(prefix: string, cursor?: string): Promise { - const options: { prefix: string; startOffset?: string } = { prefix }; - if (cursor) { - options.startOffset = cursor; - } - - const [files] = await this.#storage.bucket(this.#bucket).getFiles(options); - // When resuming, GCS startOffset is inclusive — skip the cursor key itself - const keys = files.map((f) => f.name).filter((name) => name !== cursor); - return { keys }; - } - - async get(key: string): Promise<{ data: Buffer; contentType?: string }> { - const file = this.#storage.bucket(this.#bucket).file(key); - const [contents] = await file.download(); - const [metadata] = await file.getMetadata(); - const contentType = metadata.contentType as string | undefined; - if (contentType) { - return { data: contents, contentType }; - } - return { data: contents }; - } - - async put(key: string, data: Buffer, contentType?: string): Promise { - const file = this.#storage.bucket(this.#bucket).file(key); - if (contentType) { - await file.save(data, { contentType }); - } else { - await file.save(data); - } - } -} - -/** Google Cloud Storage provider. */ -export const gcs = makeObjectProvider("gcs", GcsCredentials, async (creds) => { - logger.debug("Connecting to GCS bucket {bucket} in project {projectId}", { - bucket: creds.bucket, - projectId: creds.projectId, - }); - - const opts: StorageOptions = { projectId: creds.projectId }; - if (creds.keyFilename) { - opts.keyFilename = creds.keyFilename; - } - const storage = new Storage(opts); - - return new ObjectStoreProvider( - new GcsObjectStoreClient(storage, creds.bucket), - "gcs", - ); -}); diff --git a/packages/nvisy-plugin-object/src/providers/index.ts b/packages/nvisy-plugin-object/src/providers/index.ts deleted file mode 100644 index 795aa8b..0000000 --- a/packages/nvisy-plugin-object/src/providers/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export { type AzureCredentials, azure } from "./azure.js"; -export { - type ListResult, - makeObjectProvider, - ObjectStoreClient, - ObjectStoreProvider, -} from "./client.js"; -export { type GcsCredentials, gcs } from "./gcs.js"; -export { type S3Credentials, s3 } from "./s3.js"; diff --git a/packages/nvisy-plugin-object/src/providers/s3.ts b/packages/nvisy-plugin-object/src/providers/s3.ts deleted file mode 100644 index 5f315d9..0000000 --- a/packages/nvisy-plugin-object/src/providers/s3.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { - GetObjectCommand, - ListObjectsV2Command, - PutObjectCommand, - S3Client, - type S3ClientConfig, -} from "@aws-sdk/client-s3"; -import { getLogger } from "@logtape/logtape"; -import { z } from "zod"; -import { - type ListResult, - makeObjectProvider, - ObjectStoreClient, - ObjectStoreProvider, -} from "./client.js"; - -const logger = getLogger(["nvisy", "object"]); - -/** - * Credentials for connecting to Amazon S3. - */ -export const S3Credentials = z.object({ - /** AWS region (e.g. `"us-east-1"`). */ - region: z.string(), - /** S3 bucket name. */ - bucket: z.string(), - /** AWS access key ID. */ - accessKeyId: z.string(), - /** AWS secret access key. */ - secretAccessKey: z.string(), - /** Optional custom endpoint for S3-compatible stores (e.g. MinIO). */ - endpoint: z.string().optional(), -}); -export type S3Credentials = z.infer; - -class S3ObjectStoreClient extends ObjectStoreClient { - readonly #client: S3Client; - readonly #bucket: string; - - constructor(client: S3Client, bucket: string) { - super(); - this.#client = client; - this.#bucket = bucket; - } - - async list(prefix: string, cursor?: string): Promise { - const response = await this.#client.send( - new ListObjectsV2Command({ - Bucket: this.#bucket, - Prefix: prefix, - StartAfter: cursor, - }), - ); - const keys = (response.Contents ?? []) - .map((o) => o.Key) - .filter((k): k is string => k != null); - const lastKey = response.IsTruncated - ? response.Contents?.at(-1)?.Key - : undefined; - if (lastKey) { - return { keys, nextCursor: lastKey }; - } - return { keys }; - } - - async get(key: string): Promise<{ data: Buffer; contentType?: string }> { - const response = await this.#client.send( - new GetObjectCommand({ Bucket: this.#bucket, Key: key }), - ); - const bytes = await response.Body!.transformToByteArray(); - const contentType = response.ContentType; - if (contentType) { - return { data: Buffer.from(bytes), contentType }; - } - return { data: Buffer.from(bytes) }; - } - - async put(key: string, data: Buffer, contentType?: string): Promise { - await this.#client.send( - new PutObjectCommand({ - Bucket: this.#bucket, - Key: key, - Body: data, - ContentType: contentType, - }), - ); - } -} - -/** Amazon S3 provider. */ -export const s3 = makeObjectProvider("s3", S3Credentials, async (creds) => { - logger.debug("Connecting to S3 bucket {bucket} in {region}", { - bucket: creds.bucket, - region: creds.region, - }); - - const config: S3ClientConfig = { - region: creds.region, - credentials: { - accessKeyId: creds.accessKeyId, - secretAccessKey: creds.secretAccessKey, - }, - }; - if (creds.endpoint) { - config.endpoint = creds.endpoint; - } - - const client = new S3Client(config); - - return new ObjectStoreProvider( - new S3ObjectStoreClient(client, creds.bucket), - "s3", - async () => client.destroy(), - ); -}); diff --git a/packages/nvisy-plugin-object/src/streams/index.ts b/packages/nvisy-plugin-object/src/streams/index.ts deleted file mode 100644 index a37fd24..0000000 --- a/packages/nvisy-plugin-object/src/streams/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { read } from "./read.js"; -export { write } from "./write.js"; diff --git a/packages/nvisy-plugin-object/src/streams/read.ts b/packages/nvisy-plugin-object/src/streams/read.ts deleted file mode 100644 index 48a6181..0000000 --- a/packages/nvisy-plugin-object/src/streams/read.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { Blob, type Resumable, RuntimeError, Stream } from "@nvisy/core"; -import { z } from "zod"; -import { ObjectStoreClient } from "../providers/client.js"; - -const logger = getLogger(["nvisy", "object"]); - -/** - * Per-node parameters for the object-store read stream. - */ -export const ObjectParams = z.object({ - /** Key prefix to list objects under (e.g. `"uploads/2024/"`). */ - prefix: z.string().default(""), - /** Maximum keys to fetch per list page. */ - batchSize: z.number().default(100), -}); -export type ObjectParams = z.infer; - -/** - * Keyset pagination cursor for resumable object reads. - * - * `lastKey` is `null` on the very first page. - */ -export const ObjectCursor = z.object({ - /** The last key successfully yielded, or `null` before the first page. */ - lastKey: z.string().nullable().default(null), -}); -export type ObjectCursor = z.infer; - -/** - * Source stream that lists objects under a prefix and yields each as - * a {@link Blob}. Pagination uses the last-key cursor from the store's - * list API. - */ -export const read = Stream.createSource("read", ObjectStoreClient, { - types: [Blob, ObjectCursor, ObjectParams], - reader: (client, cursor, params) => readStream(client, cursor, params), -}); - -async function* readStream( - client: ObjectStoreClient, - cursor: ObjectCursor, - params: ObjectParams, -): AsyncIterable> { - const { prefix, batchSize } = params; - - logger.debug("Read stream opened on prefix {prefix}", { prefix, batchSize }); - - let nextCursor: string | undefined = cursor.lastKey ?? undefined; - let totalObjects = 0; - - while (true) { - let keys: readonly string[]; - let pageCursor: string | undefined; - - try { - const result = await client.list(prefix, nextCursor); - keys = result.keys; - pageCursor = result.nextCursor; - logger.debug("List returned {count} keys", { count: keys.length }); - } catch (error) { - logger.error("List failed for prefix {prefix}: {error}", { - prefix, - error: error instanceof Error ? error.message : String(error), - }); - throw RuntimeError.wrap(error, { source: "object/read" }); - } - - for (const key of keys) { - try { - const { data, contentType } = await client.get(key); - totalObjects++; - yield { - data: new Blob(key, data, contentType), - context: { lastKey: key } as ObjectCursor, - }; - } catch (error) { - logger.error("Get failed for key {key}: {error}", { - key, - error: error instanceof Error ? error.message : String(error), - }); - throw RuntimeError.wrap(error, { source: "object/read" }); - } - } - - if (keys.length < batchSize || !pageCursor) break; - nextCursor = pageCursor; - } - - logger.debug("Read stream closed, {totalObjects} objects yielded", { - totalObjects, - }); -} diff --git a/packages/nvisy-plugin-object/src/streams/write.ts b/packages/nvisy-plugin-object/src/streams/write.ts deleted file mode 100644 index 2553438..0000000 --- a/packages/nvisy-plugin-object/src/streams/write.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { Blob, RuntimeError, Stream } from "@nvisy/core"; -import { z } from "zod"; -import { ObjectStoreClient } from "../providers/client.js"; - -const logger = getLogger(["nvisy", "object"]); - -/** - * Per-node parameters for the object-store write stream. - */ -export const WriteParams = z.object({ - /** Key prefix to prepend to each blob path on write. */ - prefix: z.string().default(""), -}); -export type WriteParams = z.infer; - -/** - * Target stream that writes each {@link Blob} to the object store - * via the provider client's `put` method. - */ -export const write = Stream.createTarget("write", ObjectStoreClient, { - types: [Blob, WriteParams], - writer: (client, params) => async (item: Blob) => { - const key = params.prefix ? `${params.prefix}${item.path}` : item.path; - try { - await client.put(key, item.data, item.contentType); - logger.debug("Put object {key} ({size} bytes)", { - key, - size: item.size, - }); - } catch (error) { - logger.error("Put failed for {key}: {error}", { - key, - error: error instanceof Error ? error.message : String(error), - }); - throw RuntimeError.wrap(error, { source: "object/write" }); - } - }, -}); diff --git a/packages/nvisy-plugin-object/tsconfig.json b/packages/nvisy-plugin-object/tsconfig.json deleted file mode 100644 index c91a2dd..0000000 --- a/packages/nvisy-plugin-object/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - /* Emit */ - "outDir": "./dist", - "rootDir": "./src", - "composite": true - }, - /* Scope */ - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"], - "references": [{ "path": "../nvisy-core" }] -} diff --git a/packages/nvisy-plugin-object/tsup.config.ts b/packages/nvisy-plugin-object/tsup.config.ts deleted file mode 100644 index d68a5db..0000000 --- a/packages/nvisy-plugin-object/tsup.config.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { defineConfig } from "tsup"; - -export default defineConfig({ - /* Entry */ - entry: ["src/index.ts"], - format: ["esm"], - - /* Output */ - outDir: "dist", - dts: { compilerOptions: { composite: false } }, - sourcemap: true, - clean: true, - - /* Optimization */ - splitting: false, - treeshake: true, - skipNodeModulesBundle: true, - - /* Environment */ - platform: "node", - target: "es2024", -}); diff --git a/packages/nvisy-plugin-pandoc/package.json b/packages/nvisy-plugin-pandoc/package.json deleted file mode 100644 index 3dfdc72..0000000 --- a/packages/nvisy-plugin-pandoc/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@nvisy/plugin-pandoc", - "version": "0.1.0", - "description": "Pandoc document conversion plugin for the Nvisy platform", - "type": "module", - "exports": { - ".": { - "source": "./src/index.ts", - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsup", - "build:watch": "tsup --watch", - "clean": "rimraf dist", - "typecheck": "tsc -b" - }, - "dependencies": { - "@logtape/logtape": "^2.0.2", - "@nvisy/core": "*", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } -} diff --git a/packages/nvisy-plugin-pandoc/src/index.ts b/packages/nvisy-plugin-pandoc/src/index.ts deleted file mode 100644 index 201a3d8..0000000 --- a/packages/nvisy-plugin-pandoc/src/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** - * @module @nvisy/plugin-pandoc - * - * Pandoc document conversion plugin for the Nvisy runtime. - * - * Provides actions for converting documents between formats using Pandoc. - */ - -import { Plugin } from "@nvisy/core"; - -/** Pandoc plugin instance. */ -export const pandocPlugin = Plugin.define("pandoc"); diff --git a/packages/nvisy-plugin-pandoc/tsconfig.json b/packages/nvisy-plugin-pandoc/tsconfig.json deleted file mode 100644 index c91a2dd..0000000 --- a/packages/nvisy-plugin-pandoc/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - /* Emit */ - "outDir": "./dist", - "rootDir": "./src", - "composite": true - }, - /* Scope */ - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"], - "references": [{ "path": "../nvisy-core" }] -} diff --git a/packages/nvisy-plugin-pandoc/tsup.config.ts b/packages/nvisy-plugin-pandoc/tsup.config.ts deleted file mode 100644 index d68a5db..0000000 --- a/packages/nvisy-plugin-pandoc/tsup.config.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { defineConfig } from "tsup"; - -export default defineConfig({ - /* Entry */ - entry: ["src/index.ts"], - format: ["esm"], - - /* Output */ - outDir: "dist", - dts: { compilerOptions: { composite: false } }, - sourcemap: true, - clean: true, - - /* Optimization */ - splitting: false, - treeshake: true, - skipNodeModulesBundle: true, - - /* Environment */ - platform: "node", - target: "es2024", -}); diff --git a/packages/nvisy-plugin-sql/README.md b/packages/nvisy-plugin-sql/README.md deleted file mode 100644 index b994591..0000000 --- a/packages/nvisy-plugin-sql/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# @nvisy/plugin-sql - -[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/build.yml) - -SQL provider plugin for the Nvisy runtime. - -## Features - -- **Postgres, MySQL, and MSSQL** providers with credential validation and connection lifecycle management -- **Keyset-paginated reads** for efficient, resumable streaming over large tables -- **Per-item writes** via Kysely INSERT for batch pipeline sinks -- **Row-level transforms**: filter, project, rename, and coerce columns in the pipeline - -## Overview - -Provides Postgres, MySQL, and MSSQL integrations through a unified Kysely-based client. The plugin exposes: - -- **Providers** (`sql/postgres`, `sql/mysql`, `sql/mssql`): connection lifecycle management with credential validation. -- **Streams** (`sql/read`, `sql/write`): keyset-paginated source and per-item insert sink. -- **Actions** (`sql/filter`, `sql/project`, `sql/rename`, `sql/coerce`): row-level transforms applied in the pipeline. - -## Usage - -```ts -import { sqlPlugin } from "@nvisy/plugin-sql"; - -registry.load(sqlPlugin); -``` - -## Changelog - -See [CHANGELOG.md](../../CHANGELOG.md) for release notes and version history. - -## License - -Apache 2.0 License - see [LICENSE.txt](../../LICENSE.txt) - -## Support - -- **Documentation**: [docs.nvisy.com](https://docs.nvisy.com) -- **Issues**: [GitHub Issues](https://github.com/nvisycom/runtime/issues) -- **Email**: [support@nvisy.com](mailto:support@nvisy.com) diff --git a/packages/nvisy-plugin-sql/package.json b/packages/nvisy-plugin-sql/package.json deleted file mode 100644 index 6e65044..0000000 --- a/packages/nvisy-plugin-sql/package.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "name": "@nvisy/plugin-sql", - "version": "0.1.0", - "description": "SQL provider integrations for the Nvisy platform", - "type": "module", - "exports": { - ".": { - "source": "./src/index.ts", - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsup", - "build:watch": "tsup --watch", - "clean": "rimraf dist", - "typecheck": "tsc -b" - }, - "dependencies": { - "@logtape/logtape": "^2.0.2", - "@nvisy/core": "*", - "kysely": "^0.28.11", - "mysql2": "^3.16.3", - "pg": "^8.18.0", - "tarn": "^3.0.2", - "tedious": "^19.2.0", - "zod": "^4.3.6" - }, - "devDependencies": { - "@types/pg": "^8.16.0" - }, - "engines": { - "node": ">=22.0.0" - } -} diff --git a/packages/nvisy-plugin-sql/src/actions/coerce.ts b/packages/nvisy-plugin-sql/src/actions/coerce.ts deleted file mode 100644 index 81f30e5..0000000 --- a/packages/nvisy-plugin-sql/src/actions/coerce.ts +++ /dev/null @@ -1,63 +0,0 @@ -import type { JsonValue } from "@nvisy/core"; -import { Action } from "@nvisy/core"; -import { z } from "zod"; -import { Row } from "../datatypes/index.js"; - -/** Allowed target types for column coercion. */ -const CoerceTarget = z.enum(["string", "number", "boolean"]); - -/** - * Parameters for the `sql/coerce` action. - * - * `columns` maps column names to a target type. Columns not listed - * are passed through unchanged. - */ -const CoerceParams = z.object({ - columns: z.record(z.string(), CoerceTarget), -}); - -/** - * Cast a single value to the requested type. - * - * - `null` / `undefined` -> `null` regardless of target. - * - `"number"` on a non-numeric string -> `null`. - */ -function coerceValue( - value: JsonValue | undefined, - target: "string" | "number" | "boolean", -): JsonValue { - if (value === null || value === undefined) return null; - - switch (target) { - case "string": - return String(value); - case "number": { - const n = Number(value); - return Number.isNaN(n) ? null : n; - } - case "boolean": - return Boolean(value); - } -} - -/** - * Coerce column values to a target type (`string`, `number`, or `boolean`). - * - * Null values remain null regardless of the target. Non-numeric strings - * coerced to `number` become `null`. Row identity and metadata are preserved. - */ -export const coerce = Action.withoutClient("coerce", { - types: [Row], - params: CoerceParams, - transform: async function* (stream, params) { - for await (const row of stream) { - const result: Record = { ...row.columns }; - - for (const [column, target] of Object.entries(params.columns)) { - result[column] = coerceValue(result[column], target); - } - - yield new Row(result).deriveFrom(row); - } - }, -}); diff --git a/packages/nvisy-plugin-sql/src/actions/filter.ts b/packages/nvisy-plugin-sql/src/actions/filter.ts deleted file mode 100644 index a972ea2..0000000 --- a/packages/nvisy-plugin-sql/src/actions/filter.ts +++ /dev/null @@ -1,111 +0,0 @@ -import type { JsonValue } from "@nvisy/core"; -import { Action } from "@nvisy/core"; -import { z } from "zod"; -import { Row } from "../datatypes/index.js"; - -/** Supported comparison operators for a single filter condition. */ -const Operator = z.enum([ - "eq", - "neq", - "gt", - "gte", - "lt", - "lte", - "in", - "notIn", - "isNull", - "isNotNull", -]); -type Operator = z.infer; - -/** A single predicate: `column value`. */ -const FilterCondition = z.object({ - column: z.string(), - op: Operator, - value: z.unknown().optional(), -}); - -/** - * Parameters for the `sql/filter` action. - * - * @param conditions Array of predicates applied to each row. - * @param mode Combine with `"and"` (default) or `"or"`. - */ -const FilterParams = z.object({ - conditions: z.array(FilterCondition), - mode: z.enum(["and", "or"]).optional(), -}); - -/** Evaluate a single {@link FilterCondition} against a row. */ -function matchCondition( - row: Row, - condition: { column: string; op: Operator; value?: unknown }, -): boolean { - const val = row.get(condition.column); - - switch (condition.op) { - case "eq": - return val === condition.value; - case "neq": - return val !== condition.value; - case "gt": - return ( - typeof val === "number" && - typeof condition.value === "number" && - val > condition.value - ); - case "gte": - return ( - typeof val === "number" && - typeof condition.value === "number" && - val >= condition.value - ); - case "lt": - return ( - typeof val === "number" && - typeof condition.value === "number" && - val < condition.value - ); - case "lte": - return ( - typeof val === "number" && - typeof condition.value === "number" && - val <= condition.value - ); - case "in": - return ( - Array.isArray(condition.value) && - (condition.value as JsonValue[]).includes(val as JsonValue) - ); - case "notIn": - return ( - Array.isArray(condition.value) && - !(condition.value as JsonValue[]).includes(val as JsonValue) - ); - case "isNull": - return val === null || val === undefined; - case "isNotNull": - return val !== null && val !== undefined; - } -} - -/** - * Filter rows by a set of column-level predicates. - * - * Conditions are combined with AND (default) or OR. Supports equality, - * comparison, set membership, and null checks. - */ -export const filter = Action.withoutClient("filter", { - types: [Row], - params: FilterParams, - transform: async function* (stream, params) { - const mode = params.mode ?? "and"; - for await (const row of stream) { - const match = - mode === "and" - ? params.conditions.every((c) => matchCondition(row, c)) - : params.conditions.some((c) => matchCondition(row, c)); - if (match) yield row; - } - }, -}); diff --git a/packages/nvisy-plugin-sql/src/actions/index.ts b/packages/nvisy-plugin-sql/src/actions/index.ts deleted file mode 100644 index 8aa2d94..0000000 --- a/packages/nvisy-plugin-sql/src/actions/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export { coerce } from "./coerce.js"; -export { filter } from "./filter.js"; -export { project } from "./project.js"; -export { rename } from "./rename.js"; diff --git a/packages/nvisy-plugin-sql/src/actions/project.ts b/packages/nvisy-plugin-sql/src/actions/project.ts deleted file mode 100644 index a676632..0000000 --- a/packages/nvisy-plugin-sql/src/actions/project.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type { JsonValue } from "@nvisy/core"; -import { Action } from "@nvisy/core"; -import { z } from "zod"; -import { Row } from "../datatypes/index.js"; - -/** - * Parameters for the `sql/project` action. - * - * Provide **either** `keep` (include only these columns) or `drop` - * (exclude these columns). Columns not present in the row are ignored. - */ -const ProjectParams = z.union([ - z.object({ keep: z.array(z.string()) }), - z.object({ drop: z.array(z.string()) }), -]); - -/** - * Project (select / exclude) columns from each row. - * - * Use `{ keep: [...] }` to retain only named columns, or - * `{ drop: [...] }` to remove named columns. Row identity and - * metadata are preserved. - */ -export const project = Action.withoutClient("project", { - types: [Row], - params: ProjectParams, - transform: async function* (stream, params) { - for await (const row of stream) { - const cols = row.columns; - let projected: Record; - - if ("keep" in params) { - projected = {}; - for (const key of params.keep) { - if (key in cols) { - projected[key] = cols[key]!; - } - } - } else { - const dropSet = new Set(params.drop); - projected = {}; - for (const [key, val] of Object.entries(cols)) { - if (!dropSet.has(key)) { - projected[key] = val; - } - } - } - - yield new Row(projected).deriveFrom(row); - } - }, -}); diff --git a/packages/nvisy-plugin-sql/src/actions/rename.ts b/packages/nvisy-plugin-sql/src/actions/rename.ts deleted file mode 100644 index 9a9f40d..0000000 --- a/packages/nvisy-plugin-sql/src/actions/rename.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { JsonValue } from "@nvisy/core"; -import { Action } from "@nvisy/core"; -import { z } from "zod"; -import { Row } from "../datatypes/index.js"; - -/** - * Parameters for the `sql/rename` action. - * - * `mapping` is a `{ oldName: newName }` record. Columns not present - * in the mapping are passed through unchanged. - */ -const RenameParams = z.object({ - mapping: z.record(z.string(), z.string()), -}); - -/** - * Rename columns according to a key mapping. - * - * Each entry in `mapping` renames `oldKey -> newKey`. Columns not in - * the mapping are preserved as-is. Row identity and metadata are kept. - */ -export const rename = Action.withoutClient("rename", { - types: [Row], - params: RenameParams, - transform: async function* (stream, params) { - for await (const row of stream) { - const result: Record = {}; - - for (const [key, val] of Object.entries(row.columns)) { - const newKey = params.mapping[key] ?? key; - result[newKey] = val; - } - - yield new Row(result).deriveFrom(row); - } - }, -}); diff --git a/packages/nvisy-plugin-sql/src/datatypes/index.ts b/packages/nvisy-plugin-sql/src/datatypes/index.ts deleted file mode 100644 index 6dce926..0000000 --- a/packages/nvisy-plugin-sql/src/datatypes/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { Row } from "./row.js"; diff --git a/packages/nvisy-plugin-sql/src/datatypes/row.ts b/packages/nvisy-plugin-sql/src/datatypes/row.ts deleted file mode 100644 index 9d1f48c..0000000 --- a/packages/nvisy-plugin-sql/src/datatypes/row.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { JsonValue } from "@nvisy/core"; -import { Data } from "@nvisy/core"; - -/** - * A row from a relational database. - * - * Maps column names to JSON-compatible values. Use the {@link get} helper - * for safe column access that returns `undefined` on missing keys rather - * than throwing. - * - * @example - * ```ts - * const row = new Row({ name: "Alice", age: 30, active: true }); - * row.get("name"); // "Alice" - * row.get("missing"); // undefined - * ``` - */ -export class Row extends Data { - readonly #columns: Readonly>; - - constructor(columns: Record) { - super(); - this.#columns = columns; - } - - /** Column name -> value mapping. */ - get columns(): Readonly> { - return this.#columns; - } - - /** Get a column value by name, or `undefined` if missing. */ - get(column: string): JsonValue | undefined { - return this.#columns[column]; - } -} diff --git a/packages/nvisy-plugin-sql/src/index.ts b/packages/nvisy-plugin-sql/src/index.ts deleted file mode 100644 index 52749e2..0000000 --- a/packages/nvisy-plugin-sql/src/index.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * @module @nvisy/plugin-sql - * - * SQL provider plugin for the Nvisy runtime. - * - * Exposes Postgres, MySQL, and MSSQL providers (client lifecycle only), - * read/write streams (keyset-paginated source + batch-insert sink), and - * row-level transform actions (filter, project, rename, coerce). - * - * @example - * ```ts - * import { sqlPlugin } from "@nvisy/plugin-sql"; - * - * // Register with the runtime - * runtime.register(sqlPlugin); - * ``` - */ - -import { Datatypes, Plugin } from "@nvisy/core"; -import { coerce, filter, project, rename } from "./actions/index.js"; -import { Row } from "./datatypes/index.js"; -import { mssql, mysql, postgres } from "./providers/index.js"; -import { read, write } from "./streams/index.js"; - -/** The SQL plugin: register this with the runtime to enable all SQL providers, streams, and actions. */ -export const sqlPlugin = Plugin.define("sql") - .withProviders(postgres, mysql, mssql) - .withStreams(read, write) - .withActions(filter, project, rename, coerce) - .withDatatypes(Datatypes.define("row", Row)); - -export { Row } from "./datatypes/index.js"; diff --git a/packages/nvisy-plugin-sql/src/providers/client.ts b/packages/nvisy-plugin-sql/src/providers/client.ts deleted file mode 100644 index db52503..0000000 --- a/packages/nvisy-plugin-sql/src/providers/client.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { - ConnectionError, - Provider, - type ProviderFactory, - type ProviderInstance, -} from "@nvisy/core"; -import { type Dialect, Kysely, sql } from "kysely"; -import { SqlCredentials } from "./schemas.js"; - -export type { SqlCredentials } from "./schemas.js"; - -const logger = getLogger(["nvisy", "sql"]); - -/** A database with an unknown schema: any table, any column, unknown values. */ -type DynamicDatabase = Record>; - -/** - * Wrapper around a {@link Kysely} instance that serves as the concrete - * class reference required by {@link Stream.createSource} and - * {@link Stream.createTarget} for runtime client-type matching. - * - * The underlying instance is schema-agnostic ({@link DynamicDatabase}) - * because table structures are not known at compile time. - */ -export class KyselyClient { - #db: Kysely; - - constructor(db: Kysely) { - this.#db = db; - } - - /** The underlying Kysely instance used for query building and execution. */ - get db(): Kysely { - return this.#db; - } -} - -/** Configuration for {@link makeSqlProvider}. */ -export interface SqlProviderConfig { - /** Unique provider identifier, e.g. `"postgres"`, `"mysql"`, `"mssql"`. */ - readonly id: string; - /** Build a Kysely {@link Dialect} from validated connection credentials. */ - readonly createDialect: (creds: SqlCredentials) => Dialect; -} - -/** - * Connected SQL provider instance returned by {@link makeSqlProvider}. - * - * Holds a {@link KyselyClient} and manages teardown of the underlying - * Kysely connection pool on {@link disconnect}. - */ -export class SqlProvider implements ProviderInstance { - readonly client: KyselyClient; - #id: string; - - constructor(client: KyselyClient, id: string) { - this.client = client; - this.#id = id; - } - - async disconnect(): Promise { - await this.client.db.destroy(); - logger.debug("Disconnected from {provider}", { provider: this.#id }); - } -} - -/** Instantiate a Kysely dialect and wrap it in a {@link KyselyClient}. */ -function createClient( - config: SqlProviderConfig, - credentials: SqlCredentials, -): KyselyClient { - logger.debug("Connecting to {provider} at {host}:{port}/{database}", { - provider: config.id, - host: credentials.host, - port: credentials.port, - database: credentials.database, - }); - return new KyselyClient( - new Kysely({ dialect: config.createDialect(credentials) }), - ); -} - -/** Run `SELECT 1` to verify the connection is live. */ -async function verifyConnection( - config: SqlProviderConfig, - credentials: SqlCredentials, -): Promise { - const client = createClient(config, credentials); - try { - await sql`SELECT 1`.execute(client.db); - logger.info("Verified {provider} at {host}:{port}/{database}", { - provider: config.id, - host: credentials.host, - port: credentials.port, - database: credentials.database, - }); - } finally { - await client.db.destroy(); - } -} - -/** Normalise an unknown throw into a {@link ConnectionError}, re-throwing as-is if already one. */ -function toConnectionError(error: unknown, source: string): ConnectionError { - if (error instanceof ConnectionError) return error; - logger.error("Connection to {provider} failed: {error}", { - provider: source, - error: error instanceof Error ? error.message : String(error), - }); - return ConnectionError.wrap(error, { source }); -} - -/** - * Create a SQL {@link ProviderFactory} parameterised by a dialect constructor. - * - * The returned factory validates {@link SqlCredentials} at parse time, then - * opens a {@link KyselyClient} on connect and tears it down on disconnect. - * Actual data I/O is handled by the stream layer, not the provider. - */ -export const makeSqlProvider = ( - config: SqlProviderConfig, -): ProviderFactory => - Provider.withAuthentication(config.id, { - credentials: SqlCredentials, - verify: async (credentials) => { - try { - await verifyConnection(config, credentials); - } catch (error) { - throw toConnectionError(error, config.id); - } - }, - connect: async (credentials) => { - try { - const client = createClient(config, credentials); - return new SqlProvider(client, config.id); - } catch (error) { - throw toConnectionError(error, config.id); - } - }, - }); diff --git a/packages/nvisy-plugin-sql/src/providers/index.ts b/packages/nvisy-plugin-sql/src/providers/index.ts deleted file mode 100644 index 049883b..0000000 --- a/packages/nvisy-plugin-sql/src/providers/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export type { SqlProviderConfig } from "./client.js"; -export { KyselyClient, makeSqlProvider, SqlProvider } from "./client.js"; -export { mssql } from "./mssql.js"; -export { mysql } from "./mysql.js"; -export { postgres } from "./postgres.js"; -export { SqlCredentials } from "./schemas.js"; diff --git a/packages/nvisy-plugin-sql/src/providers/mssql.ts b/packages/nvisy-plugin-sql/src/providers/mssql.ts deleted file mode 100644 index ab4ddc4..0000000 --- a/packages/nvisy-plugin-sql/src/providers/mssql.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { type Dialect, MssqlDialect } from "kysely"; -import * as Tarn from "tarn"; -import * as Tedious from "tedious"; -import { makeSqlProvider } from "./client.js"; -import type { SqlCredentials } from "./schemas.js"; - -/** Create a `tedious` {@link Tedious.Connection} from credentials. */ -function createConnection(creds: SqlCredentials): Tedious.Connection { - return new Tedious.Connection({ - server: creds.host, - authentication: { - options: { - userName: creds.username, - password: creds.password, - }, - type: "default", - }, - options: { - database: creds.database, - port: creds.port, - trustServerCertificate: true, - }, - }); -} - -/** Create an MSSQL dialect backed by `tedious` with a `tarn` connection pool. */ -function createDialect(creds: SqlCredentials): Dialect { - return new MssqlDialect({ - tarn: { - ...Tarn, - options: { min: 0, max: 10 }, - }, - tedious: { - ...Tedious, - connectionFactory: () => createConnection(creds), - }, - }); -} - -/** Microsoft SQL Server provider. Keyset-paginated source and batch-insert sink via kysely + `tedious`. */ -export const mssql = makeSqlProvider({ id: "mssql", createDialect }); diff --git a/packages/nvisy-plugin-sql/src/providers/mysql.ts b/packages/nvisy-plugin-sql/src/providers/mysql.ts deleted file mode 100644 index 0c0de8d..0000000 --- a/packages/nvisy-plugin-sql/src/providers/mysql.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { type Dialect, MysqlDialect } from "kysely"; -import { createPool } from "mysql2/promise"; -import { makeSqlProvider } from "./client.js"; -import type { SqlCredentials } from "./schemas.js"; - -/** Create a MySQL dialect backed by a `mysql2` connection pool. */ -function createDialect(creds: SqlCredentials): Dialect { - return new MysqlDialect({ - pool: createPool({ - host: creds.host, - port: creds.port, - database: creds.database, - user: creds.username, - password: creds.password, - }), - }); -} - -/** MySQL provider. Keyset-paginated source and batch-insert sink via kysely + `mysql2`. */ -export const mysql = makeSqlProvider({ id: "mysql", createDialect }); diff --git a/packages/nvisy-plugin-sql/src/providers/postgres.ts b/packages/nvisy-plugin-sql/src/providers/postgres.ts deleted file mode 100644 index 7c7fdb3..0000000 --- a/packages/nvisy-plugin-sql/src/providers/postgres.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { type Dialect, PostgresDialect } from "kysely"; -import pg from "pg"; -import { makeSqlProvider } from "./client.js"; -import type { SqlCredentials } from "./schemas.js"; - -/** Create a PostgreSQL dialect backed by a `pg.Pool`. */ -function createDialect(creds: SqlCredentials): Dialect { - return new PostgresDialect({ - pool: new pg.Pool({ - host: creds.host, - port: creds.port, - database: creds.database, - user: creds.username, - password: creds.password, - }), - }); -} - -/** PostgreSQL provider. Keyset-paginated source and batch-insert sink via kysely + `pg`. */ -export const postgres = makeSqlProvider({ id: "postgres", createDialect }); diff --git a/packages/nvisy-plugin-sql/src/providers/schemas.ts b/packages/nvisy-plugin-sql/src/providers/schemas.ts deleted file mode 100644 index df1b2bd..0000000 --- a/packages/nvisy-plugin-sql/src/providers/schemas.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { z } from "zod"; - -/** - * Connection credentials shared by all SQL providers. - * - * Validated at graph parse time before any connection is attempted. - */ -export const SqlCredentials = z.object({ - /** Database server hostname or IP address. */ - host: z.string(), - /** Database server port. */ - port: z.number(), - /** Target database name. */ - database: z.string(), - /** Authentication username. */ - username: z.string(), - /** Authentication password. */ - password: z.string(), -}); -export type SqlCredentials = z.infer; diff --git a/packages/nvisy-plugin-sql/src/streams/index.ts b/packages/nvisy-plugin-sql/src/streams/index.ts deleted file mode 100644 index b319f8d..0000000 --- a/packages/nvisy-plugin-sql/src/streams/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { read } from "./read.js"; -export { SqlCursor, SqlParams } from "./schemas.js"; -export { write } from "./write.js"; diff --git a/packages/nvisy-plugin-sql/src/streams/read.ts b/packages/nvisy-plugin-sql/src/streams/read.ts deleted file mode 100644 index a3a8e32..0000000 --- a/packages/nvisy-plugin-sql/src/streams/read.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import type { JsonValue, Resumable } from "@nvisy/core"; -import { RuntimeError, Stream } from "@nvisy/core"; -import { type SqlBool, sql } from "kysely"; -import { Row } from "../datatypes/index.js"; -import { KyselyClient } from "../providers/client.js"; -import { SqlCursor, SqlParams } from "./schemas.js"; - -const logger = getLogger(["nvisy", "sql"]); - -/** - * Keyset-paginated source stream that yields one {@link Row} at a time. - * - * Pages are fetched using a composite `(idColumn, tiebreaker)` cursor - * for stable ordering across batches. The stream terminates when a - * batch returns fewer rows than `batchSize`. - */ -export const read = Stream.createSource("read", KyselyClient, { - types: [Row, SqlCursor, SqlParams], - reader: (client, cursor, params) => readStream(client, cursor, params), -}); - -async function* readStream( - client: KyselyClient, - cursor: SqlCursor, - params: SqlParams, -): AsyncIterable> { - const { table, columns, idColumn, tiebreaker, batchSize } = params; - const { ref } = client.db.dynamic; - - logger.debug("Read stream opened on {table}", { - table, - idColumn, - tiebreaker, - batchSize, - }); - - let lastId = cursor.lastId; - let lastTiebreaker = cursor.lastTiebreaker; - let totalRows = 0; - - while (true) { - let rows: ReadonlyArray>; - - try { - let query = client.db - .selectFrom(table) - .orderBy(ref(idColumn), "asc") - .orderBy(ref(tiebreaker), "asc") - .limit(batchSize); - - if (columns.length > 0) { - query = query.select(columns.map((c) => ref(c))); - } else { - query = query.selectAll(); - } - - if (lastId !== null && lastTiebreaker !== null) { - query = query.where( - sql`(${sql.ref(idColumn)}, ${sql.ref(tiebreaker)}) > (${lastId}, ${lastTiebreaker})`, - ); - } - - rows = await query.execute(); - logger.debug("Read batch returned {count} rows from {table}", { - count: rows.length, - table, - }); - } catch (error) { - logger.error("Read failed on {table}: {error}", { - table, - error: error instanceof Error ? error.message : String(error), - }); - throw RuntimeError.wrap(error, { source: "sql/read" }); - } - - for (const row of rows) { - totalRows++; - lastId = (row[idColumn] as string | number) ?? null; - lastTiebreaker = (row[tiebreaker] as string | number) ?? null; - yield { - data: new Row(row as Record), - context: { lastId, lastTiebreaker } as SqlCursor, - }; - } - - if (rows.length < batchSize) break; - } - - logger.debug("Read stream closed on {table}, {totalRows} rows yielded", { - table, - totalRows, - }); -} diff --git a/packages/nvisy-plugin-sql/src/streams/schemas.ts b/packages/nvisy-plugin-sql/src/streams/schemas.ts deleted file mode 100644 index faed9df..0000000 --- a/packages/nvisy-plugin-sql/src/streams/schemas.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { z } from "zod"; - -/** - * Per-node parameters that describe what to read from or write to. - * - * Attached to each provider node in the workflow graph. - */ -export const SqlParams = z.object({ - /** Target table name. */ - table: z.string(), - /** Columns to select (empty array = `SELECT *`). */ - columns: z.array(z.string()), - /** Primary sort column for keyset pagination (must be sequential / monotonic). */ - idColumn: z.string(), - /** Secondary sort column for stable ordering when `idColumn` values collide. */ - tiebreaker: z.string(), - /** Maximum rows per page during keyset pagination. */ - batchSize: z.number(), -}); -export type SqlParams = z.infer; - -/** - * Keyset pagination cursor for resumable reads. - * - * Both fields are `null` on the very first page and are updated after - * each yielded row. - */ -export const SqlCursor = z.object({ - /** Last seen value of the `idColumn`, or `null` for the first page. */ - lastId: z.union([z.number(), z.string(), z.null()]), - /** Last seen value of the `tiebreaker` column, or `null` for the first page. */ - lastTiebreaker: z.union([z.number(), z.string(), z.null()]), -}); -export type SqlCursor = z.infer; diff --git a/packages/nvisy-plugin-sql/src/streams/write.ts b/packages/nvisy-plugin-sql/src/streams/write.ts deleted file mode 100644 index accd9d9..0000000 --- a/packages/nvisy-plugin-sql/src/streams/write.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { RuntimeError, Stream } from "@nvisy/core"; -import { Row } from "../datatypes/index.js"; -import { KyselyClient } from "../providers/client.js"; -import { SqlParams } from "./schemas.js"; - -const logger = getLogger(["nvisy", "sql"]); - -/** - * Per-item insert target stream. - * - * Extracts the column map from each {@link Row} and writes it via - * a Kysely INSERT. Each element piped through the writer triggers - * an individual INSERT statement. - */ -export const write = Stream.createTarget("write", KyselyClient, { - types: [Row, SqlParams], - writer: (client, params) => async (item: Row) => { - const record = item.columns as Record; - if (Object.keys(record).length === 0) return; - - try { - await client.db.insertInto(params.table).values(record).execute(); - logger.debug("Inserted row into {table}", { table: params.table }); - } catch (error) { - logger.error("Write failed on {table}: {error}", { - table: params.table, - error: error instanceof Error ? error.message : String(error), - }); - throw RuntimeError.wrap(error, { source: "sql/write" }); - } - }, -}); diff --git a/packages/nvisy-plugin-sql/tsconfig.json b/packages/nvisy-plugin-sql/tsconfig.json deleted file mode 100644 index c91a2dd..0000000 --- a/packages/nvisy-plugin-sql/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - /* Emit */ - "outDir": "./dist", - "rootDir": "./src", - "composite": true - }, - /* Scope */ - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"], - "references": [{ "path": "../nvisy-core" }] -} diff --git a/packages/nvisy-plugin-sql/tsup.config.ts b/packages/nvisy-plugin-sql/tsup.config.ts deleted file mode 100644 index d68a5db..0000000 --- a/packages/nvisy-plugin-sql/tsup.config.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { defineConfig } from "tsup"; - -export default defineConfig({ - /* Entry */ - entry: ["src/index.ts"], - format: ["esm"], - - /* Output */ - outDir: "dist", - dts: { compilerOptions: { composite: false } }, - sourcemap: true, - clean: true, - - /* Optimization */ - splitting: false, - treeshake: true, - skipNodeModulesBundle: true, - - /* Environment */ - platform: "node", - target: "es2024", -}); diff --git a/packages/nvisy-plugin-vector/README.md b/packages/nvisy-plugin-vector/README.md deleted file mode 100644 index e4247b8..0000000 --- a/packages/nvisy-plugin-vector/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# @nvisy/plugin-vector - -[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/build.yml) - -Vector database plugin for the Nvisy runtime. - -## Features - -- **Pinecone, Qdrant, Milvus, Weaviate, and pgvector** providers with credential validation and connection lifecycle management -- **Vector upsert streams** for writing embeddings to vector databases -- **Vector search streams** for similarity queries -- **Metadata filtering** actions for vector search results - -## Overview - -Provides vector database integrations for embedding storage and similarity search. The plugin exposes: - -- **Providers** (`vector/pinecone`, `vector/qdrant`, `vector/milvus`, `vector/weaviate`, `vector/pgvector`): connection lifecycle management with credential validation. -- **Streams** (`vector/upsert`, `vector/search`): vector write and similarity search streams. -- **Actions** (`vector/filter`, `vector/rerank`): post-processing transforms for search results. - -## Usage - -```ts -import { vectorPlugin } from "@nvisy/plugin-vector"; - -registry.load(vectorPlugin); -``` - -## Changelog - -See [CHANGELOG.md](../../CHANGELOG.md) for release notes and version history. - -## License - -Apache 2.0 License - see [LICENSE.txt](../../LICENSE.txt) - -## Support - -- **Documentation**: [docs.nvisy.com](https://docs.nvisy.com) -- **Issues**: [GitHub Issues](https://github.com/nvisycom/runtime/issues) -- **Email**: [support@nvisy.com](mailto:support@nvisy.com) diff --git a/packages/nvisy-plugin-vector/package.json b/packages/nvisy-plugin-vector/package.json deleted file mode 100644 index abfe938..0000000 --- a/packages/nvisy-plugin-vector/package.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "@nvisy/plugin-vector", - "version": "0.1.0", - "description": "Vector database integrations for the Nvisy platform", - "type": "module", - "exports": { - ".": { - "source": "./src/index.ts", - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsup", - "build:watch": "tsup --watch", - "clean": "rimraf dist", - "typecheck": "tsc -b" - }, - "dependencies": { - "@logtape/logtape": "^2.0.2", - "@nvisy/core": "*", - "@pinecone-database/pinecone": "^4.0.0", - "@qdrant/js-client-rest": "^1.13.0", - "@zilliz/milvus2-sdk-node": "^2.5.0", - "pg": "^8.13.0", - "pgvector": "^0.2.0", - "weaviate-client": "^3.5.0", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } -} diff --git a/packages/nvisy-plugin-vector/src/index.ts b/packages/nvisy-plugin-vector/src/index.ts deleted file mode 100644 index 2b67dee..0000000 --- a/packages/nvisy-plugin-vector/src/index.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * @module @nvisy/plugin-vector - * - * Vector database plugin for the Nvisy runtime. - * - * Exposes vector database providers (Pinecone, Qdrant, Milvus, Weaviate, pgvector) - * and an upsert target stream for writing embeddings to vector stores. - * - * @example - * ```ts - * import { vectorPlugin } from "@nvisy/plugin-vector"; - * - * engine.register(vectorPlugin); - * ``` - */ - -import { Plugin } from "@nvisy/core"; -import { - milvus, - pgvectorProvider, - pinecone, - qdrant, - weaviateProvider, -} from "./providers/index.js"; -import { upsert } from "./streams/index.js"; - -/** The Vector plugin: register this with the runtime to enable vector store providers and streams. */ -export const vectorPlugin = Plugin.define("vector") - .withProviders(pinecone, qdrant, milvus, weaviateProvider, pgvectorProvider) - .withStreams(upsert); - -export type { UpsertVector } from "./providers/index.js"; -export { VectorClient } from "./providers/index.js"; diff --git a/packages/nvisy-plugin-vector/src/providers/client.ts b/packages/nvisy-plugin-vector/src/providers/client.ts deleted file mode 100644 index 47efaa4..0000000 --- a/packages/nvisy-plugin-vector/src/providers/client.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { - Provider, - type ProviderFactory, - type ProviderInstance, -} from "@nvisy/core"; -import type { z } from "zod"; - -const logger = getLogger(["nvisy", "vector"]); - -/** - * A single vector to upsert into the vector store. - */ -export interface UpsertVector { - /** Unique identifier for this vector. */ - readonly id: string; - /** The dense embedding vector. */ - readonly vector: Float32Array | number[]; - /** Optional metadata to store alongside the vector. */ - readonly metadata?: Record | undefined; -} - -/** - * Abstract client that vector-store streams use for I/O. - * - * Each provider (Pinecone, Qdrant, Milvus, Weaviate, pgvector) supplies a - * concrete subclass. The class reference is required by - * {@link Stream.createTarget} for runtime client-type matching. - */ -export abstract class VectorClient { - /** Upsert one or more vectors into the store. */ - abstract upsert(vectors: UpsertVector[]): Promise; -} - -/** - * Connected vector-store provider instance. - * - * Holds a {@link VectorClient} and manages teardown on {@link disconnect}. - */ -export class VectorProvider implements ProviderInstance { - readonly client: VectorClient; - readonly #id: string; - readonly #disconnect: (() => Promise) | undefined; - - constructor( - client: VectorClient, - id: string, - disconnect?: () => Promise, - ) { - this.client = client; - this.#id = id; - this.#disconnect = disconnect; - } - - async disconnect(): Promise { - await this.#disconnect?.(); - logger.debug("Disconnected from {provider}", { provider: this.#id }); - } -} - -/** - * Create a vector-store {@link ProviderFactory} from a credential schema - * and a connect function. - */ -export const makeVectorProvider = ( - id: string, - credentials: z.ZodType, - connect: (creds: TCred) => Promise>, -): ProviderFactory => - Provider.withAuthentication(id, { - credentials, - connect, - }); diff --git a/packages/nvisy-plugin-vector/src/providers/index.ts b/packages/nvisy-plugin-vector/src/providers/index.ts deleted file mode 100644 index 32d6383..0000000 --- a/packages/nvisy-plugin-vector/src/providers/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -export { - makeVectorProvider, - type UpsertVector, - VectorClient, - VectorProvider, -} from "./client.js"; -export { type MilvusCredentials, milvus } from "./milvus.js"; -export { type PgvectorCredentials, pgvectorProvider } from "./pgvector.js"; -export { type PineconeCredentials, pinecone } from "./pinecone.js"; -export { type QdrantCredentials, qdrant } from "./qdrant.js"; -export { type WeaviateCredentials, weaviateProvider } from "./weaviate.js"; diff --git a/packages/nvisy-plugin-vector/src/providers/milvus.ts b/packages/nvisy-plugin-vector/src/providers/milvus.ts deleted file mode 100644 index 1a3db18..0000000 --- a/packages/nvisy-plugin-vector/src/providers/milvus.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { MilvusClient } from "@zilliz/milvus2-sdk-node"; -import { z } from "zod"; -import { - makeVectorProvider, - type UpsertVector, - VectorClient, - VectorProvider, -} from "./client.js"; - -const logger = getLogger(["nvisy", "vector"]); - -/** - * Credentials for connecting to Milvus / Zilliz. - */ -export const MilvusCredentials = z.object({ - /** Milvus server address (e.g. `"localhost:19530"`). */ - address: z.string(), - /** Optional authentication token. */ - token: z.string().optional(), - /** Name of the Milvus collection. */ - collectionName: z.string(), -}); -export type MilvusCredentials = z.infer; - -class MilvusVectorClient extends VectorClient { - readonly #client: MilvusClient; - readonly #collectionName: string; - - constructor(client: MilvusClient, collectionName: string) { - super(); - this.#client = client; - this.#collectionName = collectionName; - } - - async upsert(vectors: UpsertVector[]): Promise { - await this.#client.upsert({ - collection_name: this.#collectionName, - data: vectors.map((v) => ({ - id: v.id, - vector: [...v.vector], - ...v.metadata, - })), - }); - } -} - -/** Milvus / Zilliz vector database provider. */ -export const milvus = makeVectorProvider( - "milvus", - MilvusCredentials, - async (creds) => { - logger.debug( - "Connecting to Milvus at {address} collection {collectionName}", - { - address: creds.address, - collectionName: creds.collectionName, - }, - ); - - const config: ConstructorParameters[0] = { - address: creds.address, - }; - if (creds.token) { - config.token = creds.token; - } - - const client = new MilvusClient(config); - - return new VectorProvider( - new MilvusVectorClient(client, creds.collectionName), - "milvus", - ); - }, -); diff --git a/packages/nvisy-plugin-vector/src/providers/pgvector.ts b/packages/nvisy-plugin-vector/src/providers/pgvector.ts deleted file mode 100644 index f8e2499..0000000 --- a/packages/nvisy-plugin-vector/src/providers/pgvector.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import pg from "pg"; -import pgvector from "pgvector"; -import { z } from "zod"; -import { - makeVectorProvider, - type UpsertVector, - VectorClient, - VectorProvider, -} from "./client.js"; - -const logger = getLogger(["nvisy", "vector"]); - -/** - * Credentials for connecting to PostgreSQL with pgvector. - */ -export const PgvectorCredentials = z.object({ - /** PostgreSQL connection string (e.g. `"postgresql://user:pass@host/db"`). */ - connectionString: z.string(), - /** Table name to store vectors in. */ - tableName: z.string(), -}); -export type PgvectorCredentials = z.infer; - -class PgVectorClient extends VectorClient { - readonly #pool: pg.Pool; - readonly #tableName: string; - - constructor(pool: pg.Pool, tableName: string) { - super(); - this.#pool = pool; - this.#tableName = tableName; - } - - async upsert(vectors: UpsertVector[]): Promise { - const client = await this.#pool.connect(); - try { - for (const v of vectors) { - const embedding = pgvector.toSql([...v.vector]); - await client.query( - `INSERT INTO ${this.#tableName} (id, embedding, metadata) - VALUES ($1, $2, $3) - ON CONFLICT (id) DO UPDATE SET embedding = $2, metadata = $3`, - [v.id, embedding, JSON.stringify(v.metadata ?? {})], - ); - } - } finally { - client.release(); - } - } -} - -/** PostgreSQL + pgvector provider. */ -export const pgvectorProvider = makeVectorProvider( - "pgvector", - PgvectorCredentials, - async (creds) => { - logger.debug("Connecting to pgvector table {tableName}", { - tableName: creds.tableName, - }); - - const pool = new pg.Pool({ connectionString: creds.connectionString }); - await pool.query("CREATE EXTENSION IF NOT EXISTS vector"); - - return new VectorProvider( - new PgVectorClient(pool, creds.tableName), - "pgvector", - async () => { - await pool.end(); - }, - ); - }, -); diff --git a/packages/nvisy-plugin-vector/src/providers/pinecone.ts b/packages/nvisy-plugin-vector/src/providers/pinecone.ts deleted file mode 100644 index 76e90f7..0000000 --- a/packages/nvisy-plugin-vector/src/providers/pinecone.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { Pinecone } from "@pinecone-database/pinecone"; -import { z } from "zod"; -import { - makeVectorProvider, - type UpsertVector, - VectorClient, - VectorProvider, -} from "./client.js"; - -const logger = getLogger(["nvisy", "vector"]); - -/** - * Credentials for connecting to Pinecone. - */ -export const PineconeCredentials = z.object({ - /** Pinecone API key. */ - apiKey: z.string(), - /** Name of the Pinecone index. */ - indexName: z.string(), -}); -export type PineconeCredentials = z.infer; - -class PineconeVectorClient extends VectorClient { - readonly #index: ReturnType; - - constructor(index: ReturnType) { - super(); - this.#index = index; - } - - async upsert(vectors: UpsertVector[]): Promise { - await this.#index.upsert( - vectors.map((v) => ({ - id: v.id, - values: [...v.vector], - metadata: v.metadata as Record, - })), - ); - } -} - -/** Pinecone vector database provider. */ -export const pinecone = makeVectorProvider( - "pinecone", - PineconeCredentials, - async (creds) => { - logger.debug("Connecting to Pinecone index {indexName}", { - indexName: creds.indexName, - }); - - const client = new Pinecone({ apiKey: creds.apiKey }); - const index = client.index(creds.indexName); - - return new VectorProvider(new PineconeVectorClient(index), "pinecone"); - }, -); diff --git a/packages/nvisy-plugin-vector/src/providers/qdrant.ts b/packages/nvisy-plugin-vector/src/providers/qdrant.ts deleted file mode 100644 index 16fc149..0000000 --- a/packages/nvisy-plugin-vector/src/providers/qdrant.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { QdrantClient } from "@qdrant/js-client-rest"; -import { z } from "zod"; -import { - makeVectorProvider, - type UpsertVector, - VectorClient, - VectorProvider, -} from "./client.js"; - -const logger = getLogger(["nvisy", "vector"]); - -/** - * Credentials for connecting to Qdrant. - */ -export const QdrantCredentials = z.object({ - /** Qdrant server URL. */ - url: z.string(), - /** Optional API key. */ - apiKey: z.string().optional(), - /** Name of the Qdrant collection. */ - collectionName: z.string(), -}); -export type QdrantCredentials = z.infer; - -class QdrantVectorClient extends VectorClient { - readonly #client: QdrantClient; - readonly #collectionName: string; - - constructor(client: QdrantClient, collectionName: string) { - super(); - this.#client = client; - this.#collectionName = collectionName; - } - - async upsert(vectors: UpsertVector[]): Promise { - await this.#client.upsert(this.#collectionName, { - points: vectors.map((v) => ({ - id: v.id, - vector: [...v.vector], - payload: v.metadata ?? {}, - })), - }); - } -} - -/** Qdrant vector database provider. */ -export const qdrant = makeVectorProvider( - "qdrant", - QdrantCredentials, - async (creds) => { - logger.debug("Connecting to Qdrant at {url} collection {collectionName}", { - url: creds.url, - collectionName: creds.collectionName, - }); - - const config: ConstructorParameters[0] = { - url: creds.url, - }; - if (creds.apiKey) { - config.apiKey = creds.apiKey; - } - - const client = new QdrantClient(config); - - return new VectorProvider( - new QdrantVectorClient(client, creds.collectionName), - "qdrant", - ); - }, -); diff --git a/packages/nvisy-plugin-vector/src/providers/weaviate.ts b/packages/nvisy-plugin-vector/src/providers/weaviate.ts deleted file mode 100644 index 4755c8f..0000000 --- a/packages/nvisy-plugin-vector/src/providers/weaviate.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import weaviate, { type WeaviateClient } from "weaviate-client"; -import { z } from "zod"; -import { - makeVectorProvider, - type UpsertVector, - VectorClient, - VectorProvider, -} from "./client.js"; - -const logger = getLogger(["nvisy", "vector"]); - -/** - * Credentials for connecting to Weaviate. - */ -export const WeaviateCredentials = z.object({ - /** Weaviate host (e.g. `"localhost:8080"`). */ - host: z.string(), - /** Weaviate gRPC port (defaults to 50051). */ - grpcPort: z.number().default(50051), - /** Optional API key. */ - apiKey: z.string().optional(), - /** Name of the Weaviate collection (class). */ - collectionName: z.string(), -}); -export type WeaviateCredentials = z.infer; - -class WeaviateVectorClient extends VectorClient { - readonly #client: WeaviateClient; - readonly #collectionName: string; - - constructor(client: WeaviateClient, collectionName: string) { - super(); - this.#client = client; - this.#collectionName = collectionName; - } - - async upsert(vectors: UpsertVector[]): Promise { - const collection = this.#client.collections.get(this.#collectionName); - await collection.data.insertMany( - vectors.map((v) => ({ - properties: (v.metadata ?? {}) as Record, - vectors: [...v.vector], - })), - ); - } -} - -/** Weaviate vector database provider. */ -export const weaviateProvider = makeVectorProvider( - "weaviate", - WeaviateCredentials, - async (creds) => { - logger.debug( - "Connecting to Weaviate at {host} collection {collectionName}", - { - host: creds.host, - collectionName: creds.collectionName, - }, - ); - - const connectOpts: Parameters[0] = { - host: creds.host, - grpcPort: creds.grpcPort, - }; - if (creds.apiKey) { - connectOpts.authCredentials = new weaviate.ApiKey(creds.apiKey); - } - const client = await weaviate.connectToLocal(connectOpts); - - return new VectorProvider( - new WeaviateVectorClient(client, creds.collectionName), - "weaviate", - async () => client.close(), - ); - }, -); diff --git a/packages/nvisy-plugin-vector/src/streams/index.ts b/packages/nvisy-plugin-vector/src/streams/index.ts deleted file mode 100644 index 8f7b108..0000000 --- a/packages/nvisy-plugin-vector/src/streams/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { upsert } from "./upsert.js"; diff --git a/packages/nvisy-plugin-vector/src/streams/upsert.ts b/packages/nvisy-plugin-vector/src/streams/upsert.ts deleted file mode 100644 index e12da23..0000000 --- a/packages/nvisy-plugin-vector/src/streams/upsert.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { Embedding, RuntimeError, Stream } from "@nvisy/core"; -import { z } from "zod"; -import { VectorClient } from "../providers/client.js"; - -const logger = getLogger(["nvisy", "vector"]); - -/** - * Per-node parameters for the vector upsert stream. - */ -export const UpsertParams = z.object({}); -export type UpsertParams = z.infer; - -/** - * Target stream that upserts each {@link Embedding} into the vector store - * via the provider client's `upsert` method. - */ -export const upsert = Stream.createTarget("upsert", VectorClient, { - types: [Embedding, UpsertParams], - writer: - (client: VectorClient, _params: UpsertParams) => - async (item: Embedding) => { - try { - await client.upsert([ - { - id: item.id, - vector: item.vector, - metadata: item.metadata ?? undefined, - }, - ]); - logger.debug("Upserted vector {id} ({dims} dims)", { - id: item.id, - dims: item.dimensions, - }); - } catch (error) { - logger.error("Upsert failed for {id}: {error}", { - id: item.id, - error: error instanceof Error ? error.message : String(error), - }); - throw RuntimeError.wrap(error, { source: "vector/upsert" }); - } - }, -}); diff --git a/packages/nvisy-plugin-vector/tsconfig.json b/packages/nvisy-plugin-vector/tsconfig.json deleted file mode 100644 index c91a2dd..0000000 --- a/packages/nvisy-plugin-vector/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - /* Emit */ - "outDir": "./dist", - "rootDir": "./src", - "composite": true - }, - /* Scope */ - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"], - "references": [{ "path": "../nvisy-core" }] -} diff --git a/packages/nvisy-plugin-vector/tsup.config.ts b/packages/nvisy-plugin-vector/tsup.config.ts deleted file mode 100644 index d68a5db..0000000 --- a/packages/nvisy-plugin-vector/tsup.config.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { defineConfig } from "tsup"; - -export default defineConfig({ - /* Entry */ - entry: ["src/index.ts"], - format: ["esm"], - - /* Output */ - outDir: "dist", - dts: { compilerOptions: { composite: false } }, - sourcemap: true, - clean: true, - - /* Optimization */ - splitting: false, - treeshake: true, - skipNodeModulesBundle: true, - - /* Environment */ - platform: "node", - target: "es2024", -}); diff --git a/packages/nvisy-runtime/README.md b/packages/nvisy-runtime/README.md deleted file mode 100644 index 2c842fa..0000000 --- a/packages/nvisy-runtime/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# @nvisy/runtime - -[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/build.yml) - -Graph definition, DAG compiler, and execution engine for the Nvisy platform. - -## Features - -- **Graph schema**: JSON-based pipeline definitions with source, action, and target nodes -- **DAG compiler**: validates graph structure, detects cycles, and produces execution plans -- **Execution engine**: runs pipelines with Effection-based structured concurrency -- **Retry policies**: configurable backoff strategies for transient failures -- **Timeout policies**: per-node execution time limits - -## Overview - -Parses JSON graph definitions into an immutable execution plan, then runs it — walking the DAG in topological order with Effection-based concurrency, retry policies, and timeout handling. - -- **Schema** (`Graph`, `SourceNode`, `ActionNode`, `TargetNode`): Zod schemas for validating pipeline definitions. -- **Compiler** (`compile`): transforms a graph into an execution plan with resolved registry entries. -- **Engine** (`Engine`): validates, compiles, and executes graphs with connection management. - -## Usage - -### Registering Plugins - -```ts -import { Engine } from "@nvisy/runtime"; -import { sqlPlugin } from "@nvisy/plugin-sql"; - -const engine = new Engine().register(sqlPlugin); -``` - -### Validating a Graph - -```ts -const result = engine.validate(graphDefinition, connections); - -if (!result.valid) { - console.error(result.errors); -} -``` - -### Executing a Graph - -```ts -const result = await engine.execute(graphDefinition, connections, { - signal: abortController.signal, - onContextUpdate: (nodeId, credId, ctx) => { - // Persist resumption context - }, -}); - -console.log(result.status); // "success" | "partial_failure" | "failure" -``` - -## Changelog - -See [CHANGELOG.md](../../CHANGELOG.md) for release notes and version history. - -## License - -Apache 2.0 License - see [LICENSE.txt](../../LICENSE.txt) - -## Support - -- **Documentation**: [docs.nvisy.com](https://docs.nvisy.com) -- **Issues**: [GitHub Issues](https://github.com/nvisycom/runtime/issues) -- **Email**: [support@nvisy.com](mailto:support@nvisy.com) diff --git a/packages/nvisy-runtime/package.json b/packages/nvisy-runtime/package.json deleted file mode 100644 index f45d89a..0000000 --- a/packages/nvisy-runtime/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "@nvisy/runtime", - "version": "0.1.0", - "description": "Graph definition, DAG compilation, and execution engine for the Nvisy platform", - "type": "module", - "exports": { - ".": { - "source": "./src/index.ts", - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "files": [ - "dist" - ], - "scripts": { - "build": "tsup", - "build:watch": "tsup --watch", - "clean": "rimraf dist", - "typecheck": "tsc -b" - }, - "dependencies": { - "@logtape/logtape": "^2.0.2", - "@nvisy/core": "*", - "effection": "^4.0.2", - "graphology": "^0.26.0", - "graphology-dag": "^0.4.1", - "graphology-types": "^0.24.8", - "magic-bytes.js": "^1.13.0", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } -} diff --git a/packages/nvisy-runtime/src/compiler/index.ts b/packages/nvisy-runtime/src/compiler/index.ts deleted file mode 100644 index 25d4ebb..0000000 --- a/packages/nvisy-runtime/src/compiler/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import type { Registry } from "../registry.js"; -import { parseGraph } from "./parse.js"; -import type { ExecutionPlan } from "./plan.js"; -import { buildPlan } from "./plan.js"; - -const logger = getLogger(["nvisy", "compiler"]); - -export type { - ExecutionPlan, - ResolvedActionNode, - ResolvedNode, - ResolvedSourceNode, - ResolvedTargetNode, -} from "./plan.js"; - -/** Compile a graph definition into an execution plan. */ -export function compile(input: unknown, registry: Registry): ExecutionPlan { - logger.info("Compiling graph"); - const parsed = parseGraph(input); - const plan = buildPlan(parsed, registry); - logger.info("Graph {graphId} compiled successfully", { - graphId: plan.definition.id, - }); - return plan; -} diff --git a/packages/nvisy-runtime/src/compiler/parse.ts b/packages/nvisy-runtime/src/compiler/parse.ts deleted file mode 100644 index 5dec74c..0000000 --- a/packages/nvisy-runtime/src/compiler/parse.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { ValidationError } from "@nvisy/core"; -import { DirectedGraph } from "graphology"; -import { Graph, type GraphNode } from "../schema.js"; - -const logger = getLogger(["nvisy", "compiler"]); - -/** Node attributes stored in the runtime graph. */ -export interface RuntimeNodeAttrs { - readonly schema: GraphNode; -} - -/** Graphology directed graph with typed node and edge attributes. */ -export type RuntimeGraph = DirectedGraph; - -/** Result of parsing a graph definition. */ -export interface ParsedGraph { - readonly definition: Graph; - readonly graph: RuntimeGraph; -} - -/** Convert a parsed Graph into a graphology DirectedGraph. */ -function buildRuntimeGraph(def: Graph): RuntimeGraph { - const graph: RuntimeGraph = new DirectedGraph(); - - for (const node of def.nodes) { - graph.addNode(node.id, { schema: node }); - } - - for (const edge of def.edges) { - graph.addEdgeWithKey(`${edge.from}->${edge.to}`, edge.from, edge.to); - } - - return graph; -} - -/** Parse and validate a graph definition from unknown input. */ -export function parseGraph(input: unknown): ParsedGraph { - const result = Graph.safeParse(input); - if (!result.success) { - logger.warn("Graph parse failed: {error}", { error: result.error.message }); - throw new ValidationError(`Graph parse error: ${result.error.message}`, { - source: "compiler", - retryable: false, - }); - } - - const definition = result.data; - logger.debug( - "Graph parsed: {graphId} ({nodeCount} nodes, {edgeCount} edges)", - { - graphId: definition.id, - nodeCount: definition.nodes.length, - edgeCount: definition.edges.length, - }, - ); - return { - definition, - graph: buildRuntimeGraph(definition), - }; -} diff --git a/packages/nvisy-runtime/src/compiler/plan.ts b/packages/nvisy-runtime/src/compiler/plan.ts deleted file mode 100644 index ecd1381..0000000 --- a/packages/nvisy-runtime/src/compiler/plan.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import type { - AnyActionInstance, - AnyProviderFactory, - AnyStreamSource, - AnyStreamTarget, -} from "@nvisy/core"; -import { ValidationError } from "@nvisy/core"; -import { hasCycle, topologicalSort } from "graphology-dag"; -import type { Registry } from "../registry.js"; -import type { Graph, GraphNode } from "../schema.js"; -import type { ParsedGraph, RuntimeGraph } from "./parse.js"; - -const logger = getLogger(["nvisy", "compiler"]); - -/** Resolved source node with provider and stream references. */ -export interface ResolvedSourceNode { - readonly type: "source"; - readonly provider: AnyProviderFactory; - readonly stream: AnyStreamSource; - readonly connection: string; - readonly params: Readonly>; -} - -/** Resolved action node with action reference. */ -export interface ResolvedActionNode { - readonly type: "action"; - readonly action: AnyActionInstance; - readonly provider?: AnyProviderFactory; - readonly connection?: string; - readonly params: Readonly>; -} - -/** Resolved target node with provider and stream references. */ -export interface ResolvedTargetNode { - readonly type: "target"; - readonly provider: AnyProviderFactory; - readonly stream: AnyStreamTarget; - readonly connection: string; - readonly params: Readonly>; -} - -/** A resolved registry entry carried with the node for execution. */ -export type ResolvedNode = - | ResolvedSourceNode - | ResolvedActionNode - | ResolvedTargetNode; - -/** Compiled graph ready for execution. */ -export interface ExecutionPlan { - readonly graph: RuntimeGraph; - readonly definition: Graph; - readonly order: ReadonlyArray; - readonly resolved: ReadonlyMap; -} - -/** Build an execution plan from a parsed graph. */ -export function buildPlan( - parsed: ParsedGraph, - registry: Registry, -): ExecutionPlan { - const { definition, graph } = parsed; - - if (hasCycle(graph)) { - logger.warn("Graph contains a cycle", { graphId: definition.id }); - throw new ValidationError("Graph contains a cycle", { - source: "compiler", - retryable: false, - }); - } - - const resolved = resolveAllNodes(definition.nodes, registry, definition.id); - const order = topologicalSort(graph); - - logger.debug("Execution plan built", { - graphId: definition.id, - order: order.join(" → "), - }); - - return { graph, definition, order, resolved }; -} - -function resolveAllNodes( - nodes: ReadonlyArray, - registry: Registry, - graphId: string, -): Map { - const resolved = new Map(); - const unresolved: string[] = []; - - for (const node of nodes) { - const entry = resolveNode(node, registry, unresolved); - if (entry) { - resolved.set(node.id, entry); - } - } - - if (unresolved.length > 0) { - logger.warn("Unresolved names: {names}", { - graphId, - names: unresolved.join(", "), - }); - throw new ValidationError(`Unresolved names: ${unresolved.join(", ")}`, { - source: "compiler", - retryable: false, - }); - } - - return resolved; -} - -function resolveNode( - node: GraphNode, - registry: Registry, - unresolved: string[], -): ResolvedNode | undefined { - switch (node.type) { - case "source": { - const provider = registry.findProvider(node.provider); - const stream = registry.findStream(node.stream); - if (!provider) { - unresolved.push(`provider "${node.provider}" (node ${node.id})`); - } - if (!stream) { - unresolved.push(`stream "${node.stream}" (node ${node.id})`); - } else if (stream.kind !== "source") { - unresolved.push( - `stream "${node.stream}" is not a source (node ${node.id})`, - ); - } - if (provider && stream?.kind === "source") { - return { - type: "source", - provider, - stream, - connection: node.connection, - params: node.params as Readonly>, - }; - } - return undefined; - } - case "target": { - const provider = registry.findProvider(node.provider); - const stream = registry.findStream(node.stream); - if (!provider) { - unresolved.push(`provider "${node.provider}" (node ${node.id})`); - } - if (!stream) { - unresolved.push(`stream "${node.stream}" (node ${node.id})`); - } else if (stream.kind !== "target") { - unresolved.push( - `stream "${node.stream}" is not a target (node ${node.id})`, - ); - } - if (provider && stream?.kind === "target") { - return { - type: "target", - provider, - stream, - connection: node.connection, - params: node.params as Readonly>, - }; - } - return undefined; - } - case "action": { - const action = registry.findAction(node.action); - if (!action) { - unresolved.push(`action "${node.action}" (node ${node.id})`); - return undefined; - } - - if (node.provider) { - const provider = registry.findProvider(node.provider); - if (!provider) { - unresolved.push(`provider "${node.provider}" (node ${node.id})`); - return undefined; - } - - return { - type: "action", - action, - provider, - ...(node.connection != null ? { connection: node.connection } : {}), - params: node.params as Readonly>, - }; - } - - return { - type: "action", - action, - params: node.params as Readonly>, - }; - } - } -} diff --git a/packages/nvisy-runtime/src/engine/bridge.ts b/packages/nvisy-runtime/src/engine/bridge.ts deleted file mode 100644 index de52ce8..0000000 --- a/packages/nvisy-runtime/src/engine/bridge.ts +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Loader bridge for automatic Blob → Document conversion. - * - * When a source node produces Blobs but downstream nodes expect Documents, - * this bridge automatically detects and applies the appropriate loader. - * Converted documents are cached by blob ID to avoid duplicate conversions - * when a source has multiple downstream consumers. - */ - -import { getLogger } from "@logtape/logtape"; -import { Blob, type Data, type Document, RuntimeError } from "@nvisy/core"; -import type { Registry } from "../registry.js"; - -const logger = getLogger(["nvisy", "bridge"]); - -/** Cache for converted documents, shared across an execution run. */ -export type LoaderCache = Map; - -/** Creates a new empty loader cache for an execution run. */ -export function createLoaderCache(): LoaderCache { - return new Map(); -} - -/** - * Wraps an async iterable to automatically convert Blobs to Documents. - * - * The bridge inspects each data item: - * - If it's a Blob, looks up the appropriate loader and converts it - * - If it's already a Document (or other type), passes it through unchanged - * - * Converted documents are cached by blob.id using the shared cache - * to avoid redundant conversions when the same blob is consumed - * by multiple downstream nodes. - */ -export interface BridgeOptions { - /** When true, skip blobs with no matching loader instead of throwing. */ - readonly ignoreUnsupported?: boolean; -} - -export async function* applyLoaderBridge( - stream: AsyncIterable, - registry: Registry, - cache: LoaderCache, - options?: BridgeOptions, -): AsyncIterable { - for await (const item of stream) { - if (!(item instanceof Blob)) { - yield item; - continue; - } - - const cached = cache.get(item.id); - if (cached) { - logger.debug("Using cached documents for blob {id}", { id: item.id }); - for (const doc of cached) { - yield doc; - } - continue; - } - - const loader = registry.findLoaderForBlob({ - path: item.path, - data: item.data, - ...(item.contentType && { contentType: item.contentType }), - }); - - if (!loader) { - if (options?.ignoreUnsupported) { - logger.warn("No loader found for blob {path}, skipping", { - path: item.path, - }); - continue; - } - throw new RuntimeError( - `No loader found for blob: ${item.path} (contentType: ${item.contentType ?? "unknown"})`, - { source: "bridge", retryable: false }, - ); - } - - logger.debug("Converting blob {path} using loader {loader}", { - path: item.path, - loader: loader.id, - }); - - const docs: Document[] = []; - const params = loader.schema.parse({}); - for await (const doc of loader.load(item, params)) { - docs.push(doc); - yield doc; - } - cache.set(item.id, docs); - } -} diff --git a/packages/nvisy-runtime/src/engine/connections.ts b/packages/nvisy-runtime/src/engine/connections.ts deleted file mode 100644 index 7eb8086..0000000 --- a/packages/nvisy-runtime/src/engine/connections.ts +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Connection validation and types. - * - * Validates connection credentials against provider schemas before - * execution begins, ensuring all connections are valid upfront. - */ - -import type { AnyProviderFactory } from "@nvisy/core"; -import { ValidationError } from "@nvisy/core"; -import { z } from "zod"; -import type { ExecutionPlan } from "../compiler/index.js"; -import type { - ResolvedActionNode, - ResolvedNode, - ResolvedSourceNode, - ResolvedTargetNode, -} from "../compiler/plan.js"; - -/** Schema for a single connection entry. */ -export const ConnectionSchema = z.object({ - /** Provider type identifier (e.g., "postgres", "s3"). */ - type: z.string(), - /** Provider-specific credentials (validated against provider schema at runtime). */ - credentials: z.unknown(), - /** Optional resumption context for crash recovery. */ - context: z.unknown(), -}); - -/** Schema for the connections map (UUID keys). */ -export const ConnectionsSchema = z.record(z.uuid(), ConnectionSchema); - -/** A connection entry with credentials for a specific provider. */ -export type Connection = z.infer; - -/** - * Map of connection ID (UUID) to connection configuration. - * - * Connections are referenced by nodes in the graph definition. - * Each connection specifies credentials that are validated against - * the provider's credential schema before execution. - */ -export type Connections = z.infer; - -/** - * A connection with validated credentials. - * - * Created during upfront validation, credentials have been parsed - * against the provider's schema and are ready for use. - */ -export interface ValidatedConnection { - readonly provider: AnyProviderFactory; - readonly credentials: unknown; - readonly context: unknown; -} - -interface ResolvedWithConnection { - readonly provider: AnyProviderFactory; - readonly connection: string; -} - -function hasConnection( - resolved: ResolvedNode, -): resolved is (ResolvedSourceNode | ResolvedTargetNode | ResolvedActionNode) & - ResolvedWithConnection { - return "connection" in resolved && resolved.connection !== undefined; -} - -/** - * Validate all connections referenced by the execution plan. - * - * Performs upfront validation of credentials against provider schemas. - * This ensures all connections are valid before execution begins, - * avoiding partial execution failures due to credential issues. - */ -export function validateConnections( - plan: ExecutionPlan, - connections: Connections, -): Map { - const validated = new Map(); - const errors: string[] = []; - - for (const nodeId of plan.order) { - const resolved = plan.resolved.get(nodeId); - if (!resolved || !hasConnection(resolved)) continue; - - const connId = resolved.connection; - if (validated.has(connId)) continue; - - const conn = connections[connId]; - if (!conn) { - errors.push(`Missing connection "${connId}" for node ${nodeId}`); - continue; - } - - const result = resolved.provider.credentialSchema.safeParse( - conn.credentials, - ); - if (!result.success) { - errors.push( - `Invalid credentials for connection "${connId}": ${result.error.message}`, - ); - continue; - } - - validated.set(connId, { - provider: resolved.provider, - credentials: result.data, - context: conn.context, - }); - } - - if (errors.length > 0) { - throw new ValidationError(errors.join("; "), { - source: "engine", - retryable: false, - details: { errors }, - }); - } - - return validated; -} diff --git a/packages/nvisy-runtime/src/engine/context.ts b/packages/nvisy-runtime/src/engine/context.ts deleted file mode 100644 index 3520d36..0000000 --- a/packages/nvisy-runtime/src/engine/context.ts +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Execution context and edge graph construction. - * - * Provides the runtime context that carries validated state through - * execution, and builds edge queues for data flow between nodes. - */ - -import type { Data } from "@nvisy/core"; -import { ValidationError } from "@nvisy/core"; -import { createQueue, type Queue } from "effection"; -import type { ExecutionPlan } from "../compiler/index.js"; -import type { ResolvedNode } from "../compiler/plan.js"; -import type { Registry } from "../registry.js"; -import type { GraphNode } from "../schema.js"; -import type { LoaderCache } from "./bridge.js"; -import type { ValidatedConnection } from "./connections.js"; -import type { ExecuteOptions } from "./executor.js"; - -/** - * An edge in the execution graph. - * - * Edges connect nodes and carry data via an Effection queue. - * The queue enables backpressure-aware streaming between nodes. - */ -export interface Edge { - readonly from: string; - readonly to: string; - readonly queue: Queue; -} - -/** - * Context passed through the execution of a graph. - * - * Provides access to the execution plan, validated connections, - * edge queues, and helper methods for retrieving node information. - */ -export interface ExecutionContext { - readonly runId: string; - readonly plan: ExecutionPlan; - readonly connections: ReadonlyMap; - readonly inEdges: ReadonlyMap; - readonly outEdges: ReadonlyMap; - readonly options: ExecuteOptions | undefined; - readonly registry: Registry; - readonly loaderCache: LoaderCache; - - getNode(nodeId: string): GraphNode; - getResolved(nodeId: string): ResolvedNode; - getConnection(nodeId: string): ValidatedConnection; -} - -/** Configuration for creating an execution context. */ -export interface ContextConfig { - readonly runId: string; - readonly plan: ExecutionPlan; - readonly connections: ReadonlyMap; - readonly inEdges: ReadonlyMap; - readonly outEdges: ReadonlyMap; - readonly registry: Registry; - readonly loaderCache: LoaderCache; - readonly options?: ExecuteOptions; -} - -/** - * Build edge maps from the execution plan. - * - * Creates Effection queues for each edge in the graph. - * These queues enable backpressure-aware streaming between nodes. - */ -export function buildEdges(plan: ExecutionPlan): { - inEdges: Map; - outEdges: Map; -} { - const inEdges = new Map(); - const outEdges = new Map(); - - for (const id of plan.order) { - inEdges.set(id, []); - outEdges.set(id, []); - } - - for (const entry of plan.graph.edgeEntries()) { - const edge: Edge = { - from: entry.source, - to: entry.target, - queue: createQueue(), - }; - outEdges.get(entry.source)!.push(edge); - inEdges.get(entry.target)!.push(edge); - } - - return { inEdges, outEdges }; -} - -/** Create an execution context for a graph run. */ -export function createContext(config: ContextConfig): ExecutionContext { - const { - runId, - plan, - connections, - inEdges, - outEdges, - registry, - loaderCache, - options, - } = config; - - return { - runId, - plan, - connections, - inEdges, - outEdges, - options, - registry, - loaderCache, - - getNode(nodeId: string): GraphNode { - const node = plan.graph.getNodeAttributes(nodeId).schema; - if (!node) { - throw new ValidationError(`Node not found: ${nodeId}`, { - source: "engine", - retryable: false, - }); - } - return node; - }, - - getResolved(nodeId: string): ResolvedNode { - const resolved = plan.resolved.get(nodeId); - if (!resolved) { - throw new ValidationError(`Resolved node not found: ${nodeId}`, { - source: "engine", - retryable: false, - }); - } - return resolved; - }, - - getConnection(nodeId: string): ValidatedConnection { - const resolved = plan.resolved.get(nodeId); - if (!resolved || !("connection" in resolved) || !resolved.connection) { - throw new ValidationError(`Node ${nodeId} has no connection`, { - source: "engine", - retryable: false, - }); - } - const conn = connections.get(resolved.connection); - if (!conn) { - throw new ValidationError( - `Connection not found: ${resolved.connection}`, - { source: "engine", retryable: false }, - ); - } - return conn; - }, - }; -} diff --git a/packages/nvisy-runtime/src/engine/engine.ts b/packages/nvisy-runtime/src/engine/engine.ts deleted file mode 100644 index 19b857d..0000000 --- a/packages/nvisy-runtime/src/engine/engine.ts +++ /dev/null @@ -1,162 +0,0 @@ -/** - * Primary runtime entry point. - * - * Coordinates plugin registration, graph validation, and execution. - * Delegates actual graph execution to the executor module and run - * tracking to the RunManager. - * - * @example - * ```ts - * const engine = new Engine(); - * engine.register(sqlPlugin); - * const runId = engine.execute(graph, connections); - * const state = engine.getRun(runId); - * ``` - */ - -import type { PluginInstance } from "@nvisy/core"; -import { corePlugin, ValidationError } from "@nvisy/core"; -import { compile, type ExecutionPlan } from "../compiler/index.js"; -import { Registry, type RegistrySchema } from "../registry.js"; -import { - type Connections, - ConnectionsSchema, - validateConnections, -} from "./connections.js"; -import { type ExecuteOptions, execute, type RunResult } from "./executor.js"; -import { - RunManager, - type RunState, - type RunStatus, - type RunSummary, -} from "./runs.js"; - -/** Result of graph validation. */ -export interface ValidationResult { - readonly valid: boolean; - readonly errors: ReadonlyArray; -} - -export class Engine { - readonly #registry = new Registry(); - readonly #runs = new RunManager(); - - constructor() { - this.#registry.load(corePlugin); - } - - /** Snapshot of all registered actions and providers with their schemas. */ - get schema(): RegistrySchema { - return this.#registry.schema; - } - - /** Register a plugin's providers, actions, and streams. */ - register(plugin: PluginInstance): this { - this.#registry.load(plugin); - return this; - } - - /** - * Validate a graph definition and connections without executing. - * - * Checks graph structure (parse, cycles, dangling edges, name resolution) - * and validates each connection's credentials against its provider schema. - */ - validate(graph: unknown, connections: Connections): ValidationResult { - const errors: string[] = []; - - const shapeResult = ConnectionsSchema.safeParse(connections); - if (!shapeResult.success) { - errors.push(...shapeResult.error.issues.map((i) => i.message)); - } - - let plan: ExecutionPlan | null = null; - try { - plan = compile(graph, this.#registry); - } catch (e) { - errors.push(e instanceof Error ? e.message : String(e)); - } - - if (plan) { - try { - validateConnections(plan, connections); - } catch (e) { - // biome-ignore lint/complexity/useLiteralKeys: index signature requires bracket access - if (e instanceof ValidationError && e.details?.["errors"]) { - // biome-ignore lint/complexity/useLiteralKeys: index signature requires bracket access - errors.push(...(e.details["errors"] as string[])); - } else { - errors.push(e instanceof Error ? e.message : String(e)); - } - } - } - - return { valid: errors.length === 0, errors }; - } - - /** - * Execute a graph in the background. - * - * Returns immediately with a runId for tracking progress, - * retrieving results, or cancelling execution. - */ - execute( - graph: unknown, - connections: Connections, - options?: ExecuteOptions, - ): string { - const plan = this.#compile(graph, connections); - return this.#runs.submit({ - runId: crypto.randomUUID(), - plan, - connections, - registry: this.#registry, - executor: execute, - ...(options && { options }), - }); - } - - /** - * Execute a graph synchronously. - * - * Blocks until execution completes. For background execution, use {@link execute}. - */ - async executeSync( - graph: unknown, - connections: Connections, - options?: ExecuteOptions, - ): Promise { - const plan = this.#compile(graph, connections); - return execute(plan, connections, this.#registry, options); - } - - /** Get the current state of a run by its ID. */ - getRun(runId: string): RunState | undefined { - return this.#runs.get(runId); - } - - /** List all runs, optionally filtered by status. */ - listRuns(status?: RunStatus): RunSummary[] { - return this.#runs.list(status); - } - - /** Cancel a running execution. */ - cancelRun(runId: string): boolean { - return this.#runs.cancel(runId); - } - - #compile(graph: unknown, connections: Connections): ExecutionPlan { - const validation = this.validate(graph, connections); - if (!validation.valid) { - throw new ValidationError( - `Graph validation failed: ${validation.errors.join("; ")}`, - { - source: "engine", - retryable: false, - details: { errors: validation.errors }, - }, - ); - } - return compile(graph, this.#registry); - } -} diff --git a/packages/nvisy-runtime/src/engine/executor.ts b/packages/nvisy-runtime/src/engine/executor.ts deleted file mode 100644 index 4256bf3..0000000 --- a/packages/nvisy-runtime/src/engine/executor.ts +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Graph execution orchestration. - * - * Executes compiled graphs using Effection for structured concurrency. - * Nodes are spawned as concurrent tasks that communicate via edge queues. - * - * The execution model: - * 1. All nodes are spawned concurrently in topological order - * 2. Each node waits for its dependencies (incoming edges) to complete - * 3. Data flows through edges via backpressure-aware queues - * 4. Node failures are isolated - other branches can continue - */ - -import { getLogger } from "@logtape/logtape"; -import { CancellationError } from "@nvisy/core"; -import { - run as effectionRun, - type Operation, - spawn, - type WithResolvers, - withResolvers, -} from "effection"; -import type { ExecutionPlan } from "../compiler/index.js"; -import type { Registry } from "../registry.js"; -import { createLoaderCache } from "./bridge.js"; -import type { Connections, ValidatedConnection } from "./connections.js"; -import { validateConnections } from "./connections.js"; -import { buildEdges, createContext, type ExecutionContext } from "./context.js"; -import { executeNode, type NodeResult } from "./nodes.js"; - -const logger = getLogger(["nvisy", "executor"]); - -/** Options for graph execution. */ -export interface ExecuteOptions { - readonly signal?: AbortSignal; - readonly onContextUpdate?: ( - nodeId: string, - connectionId: string, - context: unknown, - ) => void; -} - -/** Result of executing a complete graph. */ -export interface RunResult { - readonly runId: string; - readonly status: "success" | "partial_failure" | "failure"; - readonly nodes: ReadonlyArray; -} - -/** - * Spawn a node as a concurrent task. - * - * Waits for all dependencies to complete before executing. - * Closes outgoing edge queues when done (success or failure). - */ -function* spawnNode( - ctx: ExecutionContext, - nodeId: string, - completions: ReadonlyMap>, -): Operation { - const deps = ctx.plan.graph.inNeighbors(nodeId); - const completion = completions.get(nodeId); - if (!completion) return; - - const outEdges = ctx.outEdges.get(nodeId) ?? []; - - yield* spawn(function* () { - for (const dep of deps) { - const depCompletion = completions.get(dep); - if (depCompletion) yield* depCompletion.operation; - } - - try { - const result = yield* executeNode(ctx, nodeId); - completion.resolve(result); - } catch (error) { - logger.error("Node {nodeId} failed: {error}", { - nodeId, - error: error instanceof Error ? error.message : String(error), - }); - completion.resolve({ - nodeId, - status: "failure", - error: error instanceof Error ? error : new Error(String(error)), - itemsProcessed: 0, - }); - } finally { - for (const edge of outEdges) { - edge.queue.close(); - } - } - }); -} - -/** - * Execute a graph within Effection structured concurrency. - * - * Spawns all nodes concurrently and collects results. - * Determines overall status based on individual node results. - */ -function* runGraph( - plan: ExecutionPlan, - validatedConnections: Map, - registry: Registry, - options?: ExecuteOptions, -): Operation { - const runId = crypto.randomUUID(); - logger.info("Run {runId} started ({count} nodes)", { - runId, - count: plan.order.length, - }); - - const { inEdges, outEdges } = buildEdges(plan); - const completions = new Map>(); - for (const id of plan.order) { - completions.set(id, withResolvers()); - } - - const ctx = createContext({ - runId, - plan, - connections: validatedConnections, - inEdges, - outEdges, - registry, - loaderCache: createLoaderCache(), - ...(options && { options }), - }); - - for (const id of plan.order) { - yield* spawnNode(ctx, id, completions); - } - - const results: NodeResult[] = []; - for (const id of plan.order) { - const completion = completions.get(id); - if (completion) results.push(yield* completion.operation); - } - - const hasFailure = results.some((r) => r.status === "failure"); - const allFailure = results.every((r) => r.status === "failure"); - const status = allFailure - ? "failure" - : hasFailure - ? "partial_failure" - : "success"; - - logger.info("Run {runId} completed ({status})", { runId, status }); - return { runId, status, nodes: results }; -} - -/** - * Execute a compiled execution plan. - * - * This is the main entry point for graph execution. It: - * 1. Validates all connections upfront - * 2. Runs the graph using Effection structured concurrency - * 3. Handles cancellation via AbortSignal - */ -export async function execute( - plan: ExecutionPlan, - connections: Connections, - registry: Registry, - options?: ExecuteOptions, -): Promise { - const signal = options?.signal; - - if (signal?.aborted) { - throw new CancellationError("Execution cancelled"); - } - - const validatedConnections = validateConnections(plan, connections); - - const task = effectionRun(() => - runGraph(plan, validatedConnections, registry, options), - ); - - if (!signal) { - return task; - } - - const onAbort = () => void task.halt(); - signal.addEventListener("abort", onAbort, { once: true }); - - try { - return await task; - } finally { - signal.removeEventListener("abort", onAbort); - } -} diff --git a/packages/nvisy-runtime/src/engine/index.ts b/packages/nvisy-runtime/src/engine/index.ts deleted file mode 100644 index e4455fb..0000000 --- a/packages/nvisy-runtime/src/engine/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -export type { - ActionDescriptor, - ProviderDescriptor, - RegistrySchema, -} from "../registry.js"; -export type { Connection, Connections } from "./connections.js"; -export { Engine, type ValidationResult } from "./engine.js"; -export type { ExecuteOptions, RunResult } from "./executor.js"; -export type { NodeResult } from "./nodes.js"; -export type { NodeProgress, RunState, RunStatus, RunSummary } from "./runs.js"; diff --git a/packages/nvisy-runtime/src/engine/nodes.ts b/packages/nvisy-runtime/src/engine/nodes.ts deleted file mode 100644 index 579a383..0000000 --- a/packages/nvisy-runtime/src/engine/nodes.ts +++ /dev/null @@ -1,307 +0,0 @@ -/** - * Node execution logic for source, action, and target nodes. - * - * Each node type has a dedicated executor that handles: - * - Provider connection management - * - Parameter validation - * - Data streaming through edges - */ - -import { getLogger } from "@logtape/logtape"; -import type { Data } from "@nvisy/core"; -import { RuntimeError, ValidationError } from "@nvisy/core"; -import { call, type Operation, spawn } from "effection"; -import type { - ResolvedActionNode, - ResolvedNode, - ResolvedSourceNode, - ResolvedTargetNode, -} from "../compiler/plan.js"; -import type { GraphNode } from "../schema.js"; -import { applyLoaderBridge } from "./bridge.js"; -import type { Edge, ExecutionContext } from "./context.js"; -import { withRetry, withTimeout } from "./policies.js"; - -const logger = getLogger(["nvisy", "nodes"]); - -/** Result of executing a single node. */ -export interface NodeResult { - readonly nodeId: string; - readonly status: "success" | "failure" | "skipped"; - readonly error?: Error; - readonly itemsProcessed: number; -} - -/** Validate parameters against a Zod-like schema. */ -function validateParams( - schema: { - safeParse: ( - v: unknown, - ) => - | { success: true; data: T } - | { success: false; error: { message: string } }; - }, - params: unknown, - nodeId: string, -): T { - const result = schema.safeParse(params); - if (!result.success) { - throw new ValidationError( - `Invalid params for node ${nodeId}: ${result.error.message}`, - { source: "engine", retryable: false }, - ); - } - return result.data; -} - -/** - * Convert edge queues to a ReadableStream. - * - * Spawns a background task that drains each edge queue sequentially - * and writes items to a TransformStream. The readable side is returned - * for consumption by action nodes. - */ -function* edgesToStream(edges: Edge[]): Operation> { - const { readable, writable } = new TransformStream(); - - yield* spawn(function* () { - const writer = writable.getWriter(); - try { - for (const edge of edges) { - for ( - let next = yield* edge.queue.next(); - !next.done; - next = yield* edge.queue.next() - ) { - yield* call(() => writer.write(next.value)); - } - } - } finally { - yield* call(() => writer.close()); - } - }); - - return readable; -} - -/** - * Execute a source node: read data from an external system. - * - * Connects to the provider, reads items via the stream, and pushes - * each item to all outgoing edges. Emits context updates for - * crash recovery. - */ -function* executeSource( - ctx: ExecutionContext, - node: GraphNode, - resolved: ResolvedSourceNode, - outEdges: Edge[], -): Operation { - const conn = ctx.getConnection(node.id); - const params = validateParams( - resolved.stream.paramSchema, - resolved.params, - node.id, - ); - - const instance = yield* call(() => conn.provider.connect(conn.credentials)); - let count = 0; - - try { - const initialCtx = resolved.stream.contextSchema.parse(conn.context ?? {}); - const readable = resolved.stream.read(instance.client, initialCtx, params); - - yield* call(async () => { - for await (const resumable of readable) { - for (const edge of outEdges) { - edge.queue.add(resumable.data); - } - count++; - ctx.options?.onContextUpdate?.( - node.id, - resolved.connection, - resumable.context, - ); - } - }); - } finally { - yield* call(() => instance.disconnect()); - } - - return count; -} - -/** - * Execute an action node: transform data through a processing function. - * - * Optionally connects to a provider if the action requires a client. - * Reads from incoming edges, applies the transformation, and writes - * to outgoing edges. - */ -function* executeAction( - ctx: ExecutionContext, - node: GraphNode, - resolved: ResolvedActionNode, - inEdges: Edge[], - outEdges: Edge[], -): Operation { - const params = validateParams( - resolved.action.schema, - resolved.params, - node.id, - ); - - let client: unknown; - let disconnect: (() => Promise) | undefined; - - if (resolved.provider && resolved.connection) { - const conn = ctx.connections.get(resolved.connection); - if (!conn) { - throw new ValidationError( - `Connection not found: ${resolved.connection}`, - { source: "engine", retryable: false }, - ); - } - const instance = yield* call(() => conn.provider.connect(conn.credentials)); - client = instance.client; - disconnect = () => instance.disconnect(); - - if ( - resolved.action.clientClass && - !(client instanceof resolved.action.clientClass) - ) { - throw new ValidationError( - `Provider "${resolved.provider.id}" client is not compatible with action "${resolved.action.id}"`, - { source: "engine", retryable: false }, - ); - } - } - - try { - const rawInputStream = yield* edgesToStream(inEdges); - const inputStream = applyLoaderBridge( - rawInputStream, - ctx.registry, - ctx.loaderCache, - ); - const outputStream = resolved.action.pipe(inputStream, params, client); - let count = 0; - - yield* call(async () => { - for await (const item of outputStream) { - for (const edge of outEdges) { - edge.queue.add(item); - } - count++; - } - }); - - return count; - } finally { - if (disconnect) yield* call(disconnect); - } -} - -/** - * Execute a target node: write data to an external system. - * - * Connects to the provider and writes each item from incoming edges - * using the stream's writer function. - */ -function* executeTarget( - ctx: ExecutionContext, - node: GraphNode, - resolved: ResolvedTargetNode, - inEdges: Edge[], -): Operation { - const conn = ctx.getConnection(node.id); - const params = validateParams( - resolved.stream.paramSchema, - resolved.params, - node.id, - ); - - const instance = yield* call(() => conn.provider.connect(conn.credentials)); - let count = 0; - - try { - const writeFn = resolved.stream.write(instance.client, params); - for (const edge of inEdges) { - for ( - let next = yield* edge.queue.next(); - !next.done; - next = yield* edge.queue.next() - ) { - yield* call(() => writeFn(next.value)); - count++; - } - } - } finally { - yield* call(() => instance.disconnect()); - } - - return count; -} - -/** Dispatch to the appropriate node executor based on type. */ -function* dispatchNode( - ctx: ExecutionContext, - node: GraphNode, - resolved: ResolvedNode, - inEdges: Edge[], - outEdges: Edge[], -): Operation { - switch (resolved.type) { - case "source": - return yield* executeSource(ctx, node, resolved, outEdges); - case "action": - return yield* executeAction(ctx, node, resolved, inEdges, outEdges); - case "target": - return yield* executeTarget(ctx, node, resolved, inEdges); - } -} - -/** - * Execute a single node with retry and timeout policies. - * - * Wraps the node execution with configurable retry logic and timeout. - * Returns a NodeResult indicating success or failure. - */ -export function* executeNode( - ctx: ExecutionContext, - nodeId: string, -): Operation { - const node = ctx.getNode(nodeId); - const resolved = ctx.getResolved(nodeId); - const inEdges = ctx.inEdges.get(nodeId) ?? []; - const outEdges = ctx.outEdges.get(nodeId) ?? []; - - logger.debug("Executing node {nodeId} ({type})", { - nodeId, - type: resolved.type, - }); - - function* base(): Operation { - const count = yield* dispatchNode(ctx, node, resolved, inEdges, outEdges); - - logger.debug("Node {nodeId} completed ({count} items)", { nodeId, count }); - return { nodeId, status: "success", itemsProcessed: count }; - } - - const timeoutMs = node.timeout?.nodeTimeoutMs; - const timeoutFallback: NodeResult = { - nodeId, - status: "failure", - error: new RuntimeError(`Node ${nodeId} timed out after ${timeoutMs}ms`, { - source: "engine", - retryable: true, - }), - itemsProcessed: 0, - }; - - return yield* withTimeout( - () => withRetry(base, node.retry, nodeId), - timeoutMs, - timeoutFallback, - ); -} diff --git a/packages/nvisy-runtime/src/engine/policies.ts b/packages/nvisy-runtime/src/engine/policies.ts deleted file mode 100644 index 6279092..0000000 --- a/packages/nvisy-runtime/src/engine/policies.ts +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Execution policies for retry and timeout handling. - * - * These policies wrap Effection operations to provide: - * - Retry with configurable backoff strategies - * - Timeout with fallback values - * - * Policies are composable and respect Effection structured concurrency. - */ - -import { getLogger } from "@logtape/logtape"; -import { RuntimeError } from "@nvisy/core"; -import { type Operation, race, sleep } from "effection"; -import type { RetryPolicy } from "../schema.js"; - -const logger = getLogger(["nvisy", "engine"]); - -/** - * Wrap an operation with retry logic. - * - * Retries are only attempted for errors that are marked as retryable. - * Non-retryable errors (RuntimeError with `retryable: false`) are - * thrown immediately without retry. - * - * @param fn - Operation factory to retry. - * @param policy - Retry configuration (maxRetries, backoff strategy, delays). - * @param nodeId - Node ID for logging. - * @returns The operation result if successful. - * @throws The last error if all retries are exhausted. - * - * @example - * ```ts - * const result = yield* withRetry( - * () => fetchData(), - * { maxRetries: 3, backoff: "exponential", initialDelayMs: 100, maxDelayMs: 5000 }, - * "node-123" - * ); - * ``` - */ -export function* withRetry( - fn: () => Operation, - policy: RetryPolicy | undefined, - nodeId: string, -): Operation { - if (!policy) return yield* fn(); - - const { maxRetries, backoff, initialDelayMs, maxDelayMs } = policy; - let lastError: unknown; - - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - return yield* fn(); - } catch (error) { - lastError = error; - - // Non-retryable errors fail immediately - if (error instanceof RuntimeError && error.retryable === false) { - throw error; - } - - logger.warn("Node {nodeId} attempt {attempt} failed: {error}", { - nodeId, - attempt: attempt + 1, - maxRetries, - error: error instanceof Error ? error.message : String(error), - }); - - if (attempt < maxRetries) { - const delay = computeDelay( - backoff, - initialDelayMs, - maxDelayMs, - attempt, - ); - yield* sleep(delay); - } - } - } - - throw lastError; -} - -/** - * Compute delay for a retry attempt based on backoff strategy. - * - * Strategies: - * - `fixed`: Always use initialDelayMs - * - `exponential`: Double the delay each attempt (capped at maxDelayMs) - * - `jitter`: Random delay between 0 and exponential delay (good for avoiding thundering herd) - */ -function computeDelay( - backoff: RetryPolicy["backoff"], - initialDelayMs: number, - maxDelayMs: number, - attempt: number, -): number { - const exponentialDelay = Math.min(initialDelayMs * 2 ** attempt, maxDelayMs); - switch (backoff) { - case "fixed": - return initialDelayMs; - case "exponential": - return exponentialDelay; - case "jitter": - // Full jitter: random value between 0 and the exponential delay - // This provides good collision avoidance while maintaining backoff growth - return Math.floor(Math.random() * exponentialDelay); - } -} - -/** - * Wrap an operation with a timeout. - * - * Uses Effection's `race` to run the operation against a timer. - * If the timeout expires first, returns the fallback value. - * The original operation is automatically cancelled by Effection. - * - * @param fn - Operation factory to execute. - * @param timeoutMs - Maximum time to wait (undefined = no timeout). - * @param fallback - Value to return if timeout expires. - * @returns The operation result or fallback. - * - * @example - * ```ts - * const result = yield* withTimeout( - * () => slowOperation(), - * 5000, - * { status: "timeout" } - * ); - * ``` - */ -export function* withTimeout( - fn: () => Operation, - timeoutMs: number | undefined, - fallback: T, -): Operation { - if (!timeoutMs) return yield* fn(); - - return yield* race([ - fn(), - (function* (): Operation { - yield* sleep(timeoutMs); - return fallback; - })(), - ]); -} diff --git a/packages/nvisy-runtime/src/engine/runs.ts b/packages/nvisy-runtime/src/engine/runs.ts deleted file mode 100644 index 4328124..0000000 --- a/packages/nvisy-runtime/src/engine/runs.ts +++ /dev/null @@ -1,295 +0,0 @@ -/** - * Run management for background graph executions. - * - * Provides: - * - Tracking of in-flight and completed runs - * - Progress monitoring at the node level - * - Cancellation support via AbortController - * - Automatic cleanup of completed runs after TTL - */ - -import { getLogger } from "@logtape/logtape"; -import type { ExecutionPlan } from "../compiler/index.js"; -import type { Registry } from "../registry.js"; -import type { Connections } from "./connections.js"; -import type { ExecuteOptions, RunResult } from "./executor.js"; -import type { NodeResult } from "./nodes.js"; - -const logger = getLogger(["nvisy", "runs"]); - -/** - * Lifecycle status of a background execution run. - * - * Transitions: pending → running → completed | failed | cancelled - */ -export type RunStatus = - | "pending" - | "running" - | "completed" - | "failed" - | "cancelled"; - -/** Progress of a single node within a run. */ -export interface NodeProgress { - readonly nodeId: string; - readonly status: "pending" | "running" | "completed" | "failed"; - readonly itemsProcessed: number; - readonly error?: Error; -} - -/** - * Complete state of an execution run. - * - * Includes per-node progress for monitoring long-running executions. - */ -export interface RunState { - readonly runId: string; - readonly status: RunStatus; - readonly startedAt: Date; - readonly completedAt?: Date; - readonly nodeProgress: ReadonlyMap; - readonly result?: RunResult; - readonly error?: Error; -} - -/** Summary of a run for listing (without full progress details). */ -export interface RunSummary { - readonly runId: string; - readonly status: RunStatus; - readonly startedAt: Date; - readonly completedAt?: Date; -} - -/** Function signature for executing a plan. */ -export type PlanExecutor = ( - plan: ExecutionPlan, - connections: Connections, - registry: Registry, - options?: ExecuteOptions, -) => Promise; - -/** Configuration for submitting a graph execution. */ -export interface SubmitConfig { - readonly runId: string; - readonly plan: ExecutionPlan; - readonly connections: Connections; - readonly registry: Registry; - readonly executor: PlanExecutor; - readonly options?: ExecuteOptions; -} - -interface MutableRun { - runId: string; - status: RunStatus; - startedAt: Date; - completedAt: Date | null; - nodeProgress: Map; - result: RunResult | null; - error: Error | null; - abort: AbortController; -} - -function createRunState(run: MutableRun): RunState { - return { - runId: run.runId, - status: run.status, - startedAt: run.startedAt, - nodeProgress: new Map(run.nodeProgress), - ...(run.completedAt && { completedAt: run.completedAt }), - ...(run.result && { result: run.result }), - ...(run.error && { error: run.error }), - }; -} - -function createRunSummary(run: MutableRun): RunSummary { - return { - runId: run.runId, - status: run.status, - startedAt: run.startedAt, - ...(run.completedAt && { completedAt: run.completedAt }), - }; -} - -function createNodeProgress(nodeId: string, result?: NodeResult): NodeProgress { - return { - nodeId, - status: result - ? result.status === "success" - ? "completed" - : "failed" - : "pending", - itemsProcessed: result?.itemsProcessed ?? 0, - ...(result?.error && { error: result.error }), - }; -} - -/** - * Manages in-flight and recently completed graph executions. - * - * @example - * ```ts - * const manager = new RunManager({ ttlMs: 5 * 60 * 1000 }); - * const runId = manager.submit({ runId: id, plan, connections, registry, executor }); - * - * const state = manager.get(runId); - * console.log(state?.status, state?.nodeProgress); - * - * manager.cancel(runId); - * ``` - */ -export class RunManager { - readonly #runs = new Map(); - readonly #ttlMs: number; - - constructor(options?: { ttlMs?: number }) { - this.#ttlMs = options?.ttlMs ?? 5 * 60 * 1000; - } - - /** - * Submit a graph for background execution. - * - * Starts execution immediately and returns the run ID. - * Use {@link get} to monitor progress or {@link cancel} to abort. - */ - submit(config: SubmitConfig): string { - const { runId, plan, connections, registry, executor, options } = config; - - const run: MutableRun = { - runId, - status: "pending", - startedAt: new Date(), - completedAt: null, - nodeProgress: new Map( - plan.order.map((id) => [id, createNodeProgress(id)]), - ), - result: null, - error: null, - abort: new AbortController(), - }; - - this.#runs.set(runId, run); - logger.info("Run submitted: {runId}", { runId }); - - this.#executeInBackground(run, { - plan, - connections, - registry, - executor, - ...(options && { options }), - }); - - return runId; - } - - /** Get the current state of a run. */ - get(runId: string): RunState | undefined { - const run = this.#runs.get(runId); - return run ? createRunState(run) : undefined; - } - - /** List all runs, optionally filtered by status. */ - list(status?: RunStatus): RunSummary[] { - const summaries: RunSummary[] = []; - for (const run of this.#runs.values()) { - if (!status || run.status === status) { - summaries.push(createRunSummary(run)); - } - } - return summaries; - } - - /** - * Request cancellation of a running execution. - * - * @returns True if cancellation was requested, false if run not found or already completed. - */ - cancel(runId: string): boolean { - const run = this.#runs.get(runId); - if (!run || (run.status !== "pending" && run.status !== "running")) { - return false; - } - - run.abort.abort(); - logger.info("Run cancellation requested: {runId}", { runId }); - return true; - } - - /** Check if a run exists. */ - has(runId: string): boolean { - return this.#runs.has(runId); - } - - async #executeInBackground( - run: MutableRun, - config: Omit, - ): Promise { - const { plan, connections, registry, executor, options } = config; - - run.status = "running"; - logger.info("Run started: {runId}", { runId: run.runId }); - - const signal = options?.signal - ? AbortSignal.any([options.signal, run.abort.signal]) - : run.abort.signal; - - try { - const result = await executor(plan, connections, registry, { - ...options, - signal, - onContextUpdate: (nodeId, connectionId, context) => { - const progress = run.nodeProgress.get(nodeId); - if (progress) { - run.nodeProgress.set(nodeId, { - ...progress, - status: "running", - itemsProcessed: progress.itemsProcessed + 1, - }); - } - options?.onContextUpdate?.(nodeId, connectionId, context); - }, - }); - - run.status = "completed"; - run.completedAt = new Date(); - run.result = result; - - for (const nodeResult of result.nodes) { - run.nodeProgress.set( - nodeResult.nodeId, - createNodeProgress(nodeResult.nodeId, nodeResult), - ); - } - - logger.info("Run completed: {runId} (status={status})", { - runId: run.runId, - status: result.status, - }); - } catch (error) { - run.completedAt = new Date(); - - if (run.abort.signal.aborted) { - run.status = "cancelled"; - logger.info("Run cancelled: {runId}", { runId: run.runId }); - } else { - run.status = "failed"; - run.error = error instanceof Error ? error : new Error(String(error)); - logger.error("Run failed: {runId} (error={error})", { - runId: run.runId, - error: run.error.message, - }); - } - } - - this.#scheduleCleanup(run.runId); - } - - #scheduleCleanup(runId: string): void { - setTimeout(() => { - const run = this.#runs.get(runId); - if (run?.completedAt) { - this.#runs.delete(runId); - logger.debug("Run cleaned up: {runId}", { runId }); - } - }, this.#ttlMs); - } -} diff --git a/packages/nvisy-runtime/src/index.ts b/packages/nvisy-runtime/src/index.ts deleted file mode 100644 index cd75c6e..0000000 --- a/packages/nvisy-runtime/src/index.ts +++ /dev/null @@ -1,37 +0,0 @@ -/** - * @module @nvisy/runtime - * - * Pipeline execution engine for the Nvisy runtime. - * - * Compiles graph definitions into execution plans, validates connections, - * and runs pipelines with retry, timeout, and cancellation support. - */ - -export type { - ActionDescriptor, - Connection, - Connections, - ExecuteOptions, - NodeProgress, - NodeResult, - ProviderDescriptor, - RegistrySchema, - RunResult, - RunState, - RunStatus, - RunSummary, - ValidationResult, -} from "./engine/index.js"; -export { Engine } from "./engine/index.js"; -export type { - ActionNode, - BackoffStrategy, - ConcurrencyPolicy, - Graph, - GraphEdge, - GraphNode, - RetryPolicy, - SourceNode, - TargetNode, - TimeoutPolicy, -} from "./schema.js"; diff --git a/packages/nvisy-runtime/src/registry.ts b/packages/nvisy-runtime/src/registry.ts deleted file mode 100644 index b5fbd22..0000000 --- a/packages/nvisy-runtime/src/registry.ts +++ /dev/null @@ -1,253 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import type { - AnyActionInstance, - AnyLoaderInstance, - AnyProviderFactory, - AnyStreamSource, - AnyStreamTarget, - Datatype, - PluginInstance, -} from "@nvisy/core"; -import { ValidationError } from "@nvisy/core"; -import { filetypeinfo } from "magic-bytes.js"; - -const logger = getLogger(["nvisy", "registry"]); - -/** Describes a single registered action for schema generation. */ -export interface ActionDescriptor { - readonly name: string; - readonly configSchema: AnyActionInstance["schema"]; -} - -/** Describes a single registered provider for schema generation. */ -export interface ProviderDescriptor { - readonly name: string; - readonly credentialSchema: AnyProviderFactory["credentialSchema"]; -} - -/** - * Complete snapshot of everything currently registered, - * suitable for generating an OpenAPI spec alongside the static - * graph/node/edge schemas. - */ -export interface RegistrySchema { - readonly actions: ReadonlyArray; - readonly providers: ReadonlyArray; - readonly streams: number; - readonly loaders: number; - readonly datatypes: number; -} - -/** - * Unified registry that stores providers, actions, and loaders contributed by - * {@link PluginInstance} objects. - * - * All entries are keyed as `"pluginId/name"`. - */ -export class Registry { - readonly #actions = new Map(); - readonly #loaders = new Map(); - readonly #providers = new Map(); - readonly #streams = new Map(); - readonly #datatypes = new Map(); - readonly #plugins = new Set(); - - /** Snapshot of all registered actions and providers with their schemas. */ - get schema(): RegistrySchema { - const actions: ActionDescriptor[] = []; - for (const [name, action] of this.#actions) { - actions.push({ name, configSchema: action.schema }); - } - - const providers: ProviderDescriptor[] = []; - for (const [name, factory] of this.#providers) { - providers.push({ name, credentialSchema: factory.credentialSchema }); - } - - return { - actions, - providers, - streams: this.#streams.size, - loaders: this.#loaders.size, - datatypes: this.#datatypes.size, - }; - } - - /** Load all providers, actions, loaders, and streams declared by a plugin. */ - load(plugin: PluginInstance): void { - if (this.#plugins.has(plugin.id)) { - throw new ValidationError(`Plugin already loaded: ${plugin.id}`, { - source: "registry", - retryable: false, - details: { pluginId: plugin.id }, - }); - } - - const maps = [ - ["provider", this.#providers, plugin.providers], - ["action", this.#actions, plugin.actions], - ["loader", this.#loaders, plugin.loaders], - ["stream", this.#streams, plugin.streams], - ["datatype", this.#datatypes, plugin.datatypes], - ] as const; - - // Check for collisions across all maps - const collisions: string[] = []; - for (const [kind, map, entries] of maps) { - for (const name of Object.keys(entries)) { - const key = `${plugin.id}/${name}`; - if (map.has(key)) { - collisions.push(`${kind} "${key}"`); - } - } - } - - if (collisions.length > 0) { - logger.error( - "Registry collision loading plugin {pluginId}: {collisions}", - { pluginId: plugin.id, collisions: collisions.join(", ") }, - ); - throw new ValidationError( - `Registry collision: ${collisions.join(", ")}`, - { - source: "registry", - retryable: false, - details: { pluginId: plugin.id, collisions }, - }, - ); - } - - // Register all entries - this.#plugins.add(plugin.id); - const loaded: Record = {}; - for (const [kind, map, entries] of maps) { - const names: string[] = []; - for (const [name, value] of Object.entries(entries)) { - (map as Map).set(`${plugin.id}/${name}`, value); - names.push(name); - } - loaded[kind] = names; - } - - const counts: Record = {}; - for (const [kind, names] of Object.entries(loaded)) { - if (names.length > 0) { - counts[`${kind}s`] = names.length; - } - } - logger.debug("Plugin loaded: {plugin}", { plugin: plugin.id, ...counts }); - } - - /** Look up an action by name. */ - getAction(name: string): AnyActionInstance { - return this.#getOrThrow(this.#actions, name, "action"); - } - - /** Look up a provider factory by name. */ - getProvider(name: string): AnyProviderFactory { - return this.#getOrThrow(this.#providers, name, "provider"); - } - - /** Look up a stream by name. */ - getStream(name: string): AnyStreamSource | AnyStreamTarget { - return this.#getOrThrow(this.#streams, name, "stream"); - } - - /** Look up a loader by name. */ - getLoader(name: string): AnyLoaderInstance { - return this.#getOrThrow(this.#loaders, name, "loader"); - } - - /** Look up a data type by name. */ - getDataType(name: string): Datatype { - return this.#getOrThrow(this.#datatypes, name, "datatype"); - } - - /** Look up an action by name, returning undefined if not found. */ - findAction(name: string): AnyActionInstance | undefined { - return this.#actions.get(name); - } - - /** Look up a loader by name, returning undefined if not found. */ - findLoader(name: string): AnyLoaderInstance | undefined { - return this.#loaders.get(name); - } - - /** Look up a provider factory by name, returning undefined if not found. */ - findProvider(name: string): AnyProviderFactory | undefined { - return this.#providers.get(name); - } - - /** Look up a stream by name, returning undefined if not found. */ - findStream(name: string): (AnyStreamSource | AnyStreamTarget) | undefined { - return this.#streams.get(name); - } - - /** Look up a data type by name, returning undefined if not found. */ - findDataType(name: string): Datatype | undefined { - return this.#datatypes.get(name); - } - - /** - * Find a loader that matches the given blob by content type, magic bytes, or extension. - * - * Matching priority: - * 1. If blob has contentType, match by contentType first - * 2. Detect file type from magic bytes and match by extension - * 3. Fall back to file extension from blob.path - */ - findLoaderForBlob(blob: { - path: string; - contentType?: string; - data?: Uint8Array; - }): AnyLoaderInstance | undefined { - if (blob.contentType) { - for (const loader of this.#loaders.values()) { - if (loader.contentTypes.includes(blob.contentType)) { - return loader; - } - } - } - - if (blob.data) { - const detected = filetypeinfo(blob.data); - const first = detected[0]; - if (first?.extension) { - const ext = `.${first.extension}`; - for (const loader of this.#loaders.values()) { - if (loader.extensions.includes(ext)) { - return loader; - } - } - } - } - - const ext = this.#getExtension(blob.path); - if (ext) { - for (const loader of this.#loaders.values()) { - if (loader.extensions.includes(ext)) { - return loader; - } - } - } - - return undefined; - } - - #getOrThrow(map: Map, name: string, kind: string): T { - const entry = map.get(name); - if (!entry) { - logger.warn(`${kind} not found: ${name}`, { [kind]: name }); - throw ValidationError.notFound(name, kind, "registry"); - } - return entry; - } - - #getExtension(path: string): string | undefined { - const lastDot = path.lastIndexOf("."); - if (lastDot === -1 || lastDot === path.length - 1) { - return undefined; - } - return path.slice(lastDot).toLowerCase(); - } -} diff --git a/packages/nvisy-runtime/src/schema.ts b/packages/nvisy-runtime/src/schema.ts deleted file mode 100644 index 81f3625..0000000 --- a/packages/nvisy-runtime/src/schema.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { z } from "zod"; - -/** Strategy for calculating delay between retry attempts. */ -export const BackoffStrategy = z.enum(["fixed", "exponential", "jitter"]); - -/** Controls how failed operations are retried. */ -export const RetryPolicy = z.object({ - /** Maximum number of retry attempts after the initial failure. */ - maxRetries: z.number().default(3), - /** Strategy for calculating delay between attempts. */ - backoff: BackoffStrategy.default("exponential"), - /** Delay before the first retry in milliseconds. */ - initialDelayMs: z.number().default(1000), - /** Maximum delay between retries in milliseconds. */ - maxDelayMs: z.number().default(30_000), -}); - -/** Controls execution time limits for nodes and graphs. */ -export const TimeoutPolicy = z.object({ - /** Maximum execution time for a single node in milliseconds. */ - nodeTimeoutMs: z.number().optional(), - /** Maximum execution time for the entire graph in milliseconds. */ - graphTimeoutMs: z.number().optional(), -}); - -/** Controls parallel execution limits. */ -export const ConcurrencyPolicy = z.object({ - /** Maximum number of nodes executing concurrently across the graph. */ - maxGlobal: z.number().default(10), - /** Maximum concurrent operations within a single node. */ - maxPerNode: z.number().optional(), -}); - -export type BackoffStrategy = z.infer; -export type RetryPolicy = z.infer; -export type TimeoutPolicy = z.infer; -export type ConcurrencyPolicy = z.infer; - -/** Common properties shared by all node types. */ -const NodeBase = z.object({ - /** Unique identifier for the node. */ - id: z.uuid(), - /** Retry policy for this node. Overrides graph-level policy. */ - retry: RetryPolicy.optional(), - /** Timeout policy for this node. Overrides graph-level policy. */ - timeout: TimeoutPolicy.optional(), - /** Concurrency policy for this node. Overrides graph-level policy. */ - concurrency: ConcurrencyPolicy.optional(), -}); - -/** A source node reads data from an external system. */ -export const SourceNode = NodeBase.extend({ - /** Discriminator for source nodes. */ - type: z.literal("source"), - /** Provider identifier in "module/name" format. */ - provider: z.string(), - /** Stream identifier in "module/name" format. */ - stream: z.string(), - /** UUID reference to a connection in the connections map. */ - connection: z.uuid(), - /** Stream-specific configuration parameters. */ - params: z.record(z.string(), z.unknown()), -}); - -/** An action node transforms data flowing through the graph. */ -export const ActionNode = NodeBase.extend({ - /** Discriminator for action nodes. */ - type: z.literal("action"), - /** Action identifier in "module/name" format. */ - action: z.string(), - /** Provider identifier for client-bound actions (optional). */ - provider: z.string().optional(), - /** UUID reference to a connection for client-bound actions (optional). */ - connection: z.uuid().optional(), - /** Action-specific configuration parameters. */ - params: z.record(z.string(), z.unknown()).default({}), -}); - -/** A target node writes data to an external system. */ -export const TargetNode = NodeBase.extend({ - /** Discriminator for target nodes. */ - type: z.literal("target"), - /** Provider identifier in "module/name" format. */ - provider: z.string(), - /** Stream identifier in "module/name" format. */ - stream: z.string(), - /** UUID reference to a connection in the connections map. */ - connection: z.uuid(), - /** Stream-specific configuration parameters. */ - params: z.record(z.string(), z.unknown()), -}); - -/** A node in the execution graph. Can be a source, action, or target. */ -export const GraphNode = z.discriminatedUnion("type", [ - SourceNode, - ActionNode, - TargetNode, -]); - -/** A directed edge connecting two nodes in the graph. */ -export const GraphEdge = z.object({ - /** UUID of the source node. */ - from: z.uuid(), - /** UUID of the target node. */ - to: z.uuid(), -}); - -export type SourceNode = z.infer; -export type ActionNode = z.infer; -export type TargetNode = z.infer; -export type GraphNode = z.infer; -export type GraphEdge = z.infer; - -/** - * A complete graph definition describing a data pipeline. - * - * The graph is a directed acyclic graph (DAG) where source nodes produce data, - * action nodes transform data, target nodes consume data, and edges define - * data flow between nodes. - */ -export const Graph = z.object({ - /** Unique identifier for the graph. */ - id: z.uuid(), - /** Human-readable name for the graph. */ - name: z.string().optional(), - /** Description of what the graph does. */ - description: z.string().optional(), - /** Nodes in the graph. */ - nodes: z.array(GraphNode), - /** Edges connecting nodes. Defines data flow direction. */ - edges: z.array(GraphEdge).default([]), - /** Graph-level concurrency policy. Can be overridden per-node. */ - concurrency: ConcurrencyPolicy.optional(), - /** Graph-level timeout policy. Can be overridden per-node. */ - timeout: TimeoutPolicy.optional(), - /** Arbitrary metadata attached to the graph. */ - metadata: z.record(z.string(), z.unknown()).default({}), -}); - -export type Graph = z.infer; diff --git a/packages/nvisy-runtime/test/compile.test.ts b/packages/nvisy-runtime/test/compile.test.ts deleted file mode 100644 index 414fe5d..0000000 --- a/packages/nvisy-runtime/test/compile.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { compile } from "../src/compiler/index.js"; -import { - ACTION_ID, - diamondGraph, - GRAPH_ID, - linearGraph, - makeTestRegistry, - SOURCE_ID, - TARGET_ID, -} from "./fixtures.js"; - -describe("compile", () => { - it("compiles a valid linear graph end-to-end", () => { - const registry = makeTestRegistry(); - const plan = compile(linearGraph(), registry); - - expect(plan.definition.id).toBe(GRAPH_ID); - expect(plan.order).toEqual([SOURCE_ID, ACTION_ID, TARGET_ID]); - expect(plan.graph.order).toBe(3); - expect(plan.graph.size).toBe(2); - }); - - it("compiles a diamond graph end-to-end", () => { - const registry = makeTestRegistry(); - const plan = compile(diamondGraph(), registry); - - expect(plan.order[0]).toBe(SOURCE_ID); - expect(plan.order[plan.order.length - 1]).toBe(TARGET_ID); - expect(plan.order).toHaveLength(4); - }); - - it("resolves all nodes during compilation", () => { - const registry = makeTestRegistry(); - const plan = compile(linearGraph(), registry); - - for (const id of plan.order) { - expect(plan.resolved.get(id)).toBeDefined(); - } - }); - - it("rejects invalid input", () => { - const registry = makeTestRegistry(); - - expect(() => compile("not a graph", registry)).toThrow("Graph parse error"); - }); - - it("rejects graphs with cycles through full pipeline", () => { - const registry = makeTestRegistry(); - const cyclic = { - id: GRAPH_ID, - nodes: [ - { id: SOURCE_ID, type: "action", action: "test/noop", config: {} }, - { id: ACTION_ID, type: "action", action: "test/noop", config: {} }, - ], - edges: [ - { from: SOURCE_ID, to: ACTION_ID }, - { from: ACTION_ID, to: SOURCE_ID }, - ], - }; - - expect(() => compile(cyclic, registry)).toThrow("Graph contains a cycle"); - }); - - it("rejects unresolved names through full pipeline", () => { - const registry = makeTestRegistry(); - const unresolved = { - id: GRAPH_ID, - nodes: [ - { id: SOURCE_ID, type: "action", action: "missing/action", config: {} }, - ], - }; - - expect(() => compile(unresolved, registry)).toThrow("Unresolved names"); - }); - - it("preserves concurrency policy in definition", () => { - const registry = makeTestRegistry(); - const input = { - ...linearGraph(), - concurrency: { maxGlobal: 5 }, - }; - - const plan = compile(input, registry); - - expect(plan.definition.concurrency?.maxGlobal).toBe(5); - }); - - it("rejects source node without connection field", () => { - const registry = makeTestRegistry(); - const input = { - id: GRAPH_ID, - nodes: [ - { - id: SOURCE_ID, - type: "source", - provider: "test/testdb", - stream: "test/read", - // missing connection - params: { table: "t" }, - }, - ], - }; - - expect(() => compile(input, registry)).toThrow("Graph parse error"); - }); - - it("rejects non-UUID connection field", () => { - const registry = makeTestRegistry(); - const input = { - id: GRAPH_ID, - nodes: [ - { - id: SOURCE_ID, - type: "source", - provider: "test/testdb", - stream: "test/read", - connection: "not-a-uuid", - params: { table: "t" }, - }, - ], - }; - - expect(() => compile(input, registry)).toThrow("Graph parse error"); - }); -}); diff --git a/packages/nvisy-runtime/test/engine.test.ts b/packages/nvisy-runtime/test/engine.test.ts deleted file mode 100644 index 885b14d..0000000 --- a/packages/nvisy-runtime/test/engine.test.ts +++ /dev/null @@ -1,495 +0,0 @@ -import { - Action, - CancellationError, - Document, - Plugin, - Provider, - RuntimeError, - ValidationError, -} from "@nvisy/core"; -import { beforeEach, describe, expect, it } from "vitest"; -import { z } from "zod"; -import type { Connections } from "../src/engine/connections.js"; -import { - CRED_ID, - diamondGraph, - linearGraph, - makeTestEngine, - SOURCE_ID, - sourceEntries, - testConnections, - writtenItems, -} from "./fixtures.js"; - -beforeEach(() => { - writtenItems.length = 0; -}); - -describe("validate", () => { - it("valid graph returns { valid: true, errors: [] }", () => { - const engine = makeTestEngine(); - const result = engine.validate(linearGraph(), testConnections()); - - expect(result.valid).toBe(true); - expect(result.errors).toEqual([]); - }); - - it("invalid graph returns errors", () => { - const engine = makeTestEngine(); - const result = engine.validate("not a graph", testConnections()); - - expect(result.valid).toBe(false); - expect(result.errors.length).toBeGreaterThan(0); - expect(result.errors[0]).toContain("Graph parse error"); - }); - - it("missing connections returns errors", () => { - const engine = makeTestEngine(); - const result = engine.validate(linearGraph(), {}); - - expect(result.valid).toBe(false); - expect(result.errors.some((e) => e.includes("Missing connection"))).toBe( - true, - ); - }); - - it("invalid credentials returns errors", () => { - const engine = makeTestEngine(); - const connections: Connections = { - [CRED_ID]: { - type: "testdb", - credentials: { wrong: "field" }, - context: {}, - }, - }; - const result = engine.validate(linearGraph(), connections); - - expect(result.valid).toBe(false); - expect(result.errors.some((e) => e.includes("Invalid credentials"))).toBe( - true, - ); - }); - - it("incompatible provider client rejected at execution time", async () => { - // Action requires this client class - abstract class EmbeddingClient { - abstract embed(input: string[]): Promise; - } - - // Provider produces a plain object — not an EmbeddingClient - class CompletionOnlyClient {} - - const incompatProvider = Provider.withAuthentication("incompatdb", { - credentials: z.object({ key: z.string() }), - connect: async () => ({ client: new CompletionOnlyClient() }), - }); - - const embeddingAction = Action.withClient("needs_embed", EmbeddingClient, { - types: [Document], - params: z.object({}), - transform: (stream) => stream, - }); - - const capPlugin = Plugin.define("cap") - .withProviders(incompatProvider) - .withActions(embeddingAction); - - const engine = makeTestEngine(); - engine.register(capPlugin); - - const credId = "00000000-0000-4000-8000-0000000000d0"; - const graph = { - id: "00000000-0000-4000-8000-000000000030", - nodes: [ - { - id: "00000000-0000-4000-8000-000000000031", - type: "action" as const, - action: "cap/needs_embed", - provider: "cap/incompatdb", - connection: credId, - params: {}, - }, - ], - edges: [], - }; - - const connections: Connections = { - [credId]: { - type: "incompatdb", - credentials: { key: "test" }, - context: {}, - }, - }; - - const result = await engine.executeSync(graph, connections); - expect(result.status).toBe("failure"); - expect( - result.nodes.some( - (n) => - n.status === "failure" && n.error?.message.includes("not compatible"), - ), - ).toBe(true); - }); -}); - -describe("execute", () => { - it("linear pipeline: source -> action -> target", async () => { - const engine = makeTestEngine(); - const result = await engine.executeSync(linearGraph(), testConnections()); - - expect(result.status).toBe("success"); - expect(result.nodes).toHaveLength(3); - for (const node of result.nodes) { - expect(node.status).toBe("success"); - } - expect(writtenItems).toHaveLength(sourceEntries.length); - }); - - it("diamond graph: source -> 2 actions -> target", async () => { - const engine = makeTestEngine(); - const result = await engine.executeSync(diamondGraph(), testConnections()); - - expect(result.status).toBe("success"); - expect(result.nodes).toHaveLength(4); - for (const node of result.nodes) { - expect(node.status).toBe("success"); - } - // Source fans out to 2 actions, each forwards all items to target - // Target sees items from both action branches - expect(writtenItems).toHaveLength(sourceEntries.length * 2); - }); - - it("empty source: 0 items, all nodes succeed", async () => { - const engine = makeTestEngine(); - // Override source entries to empty for this test - const original = [...sourceEntries]; - sourceEntries.length = 0; - try { - const result = await engine.executeSync(linearGraph(), testConnections()); - - expect(result.status).toBe("success"); - for (const node of result.nodes) { - expect(node.status).toBe("success"); - } - expect(writtenItems).toHaveLength(0); - } finally { - sourceEntries.push(...original); - } - }); - - it("cancellation via AbortSignal (pre-aborted)", async () => { - const engine = makeTestEngine(); - const controller = new AbortController(); - controller.abort(); - - await expect( - engine.executeSync(linearGraph(), testConnections(), { - signal: controller.signal, - }), - ).rejects.toThrow(CancellationError); - }); - - it("cancellation via AbortSignal (abort during execution)", async () => { - const engine = makeTestEngine(); - const controller = new AbortController(); - - // Abort after a short delay - setTimeout(() => controller.abort(), 5); - - // The execution should either complete normally (if fast enough) - // or be halted. Either outcome is acceptable — we just verify - // it doesn't hang. - const result = await engine.executeSync(linearGraph(), testConnections(), { - signal: controller.signal, - }); - // If we get here, execution completed before abort — that's fine - expect(result).toBeDefined(); - }); - - it("non-retryable error stops immediately", async () => { - const { Action, Document, Plugin, Provider, Stream } = await import( - "@nvisy/core" - ); - const { z } = await import("zod"); - - class FailClient {} - - const failProvider = Provider.withAuthentication("faildb", { - credentials: z.object({ host: z.string() }), - connect: async () => ({ - client: new FailClient(), - }), - }); - - const failSource = Stream.createSource("read", FailClient, { - types: [ - Document, - z.object({}).default({}), - z.record(z.string(), z.unknown()), - ], - // biome-ignore lint/correctness/useYield: intentionally throws before yielding to test error handling - reader: async function* () { - throw new RuntimeError("Non-retryable failure", { - retryable: false, - }); - }, - }); - - const failTarget = Stream.createTarget("write", FailClient, { - types: [Document, z.record(z.string(), z.unknown())], - writer: () => async () => {}, - }); - - const failPlugin = Plugin.define("fail") - .withActions( - Action.withoutClient("noop", { - types: [Document], - params: z.object({}), - transform: (stream) => stream, - }), - ) - .withProviders(failProvider) - .withStreams(failSource, failTarget); - - const engine = makeTestEngine(); - engine.register(failPlugin); - - const failCredId = "00000000-0000-4000-8000-0000000000f1"; - const graph = { - id: "00000000-0000-4000-8000-000000000010", - nodes: [ - { - id: "00000000-0000-4000-8000-000000000011", - type: "source" as const, - provider: "fail/faildb", - stream: "fail/read", - connection: failCredId, - params: {}, - retry: { - maxRetries: 3, - backoff: "fixed" as const, - initialDelayMs: 1, - maxDelayMs: 1, - }, - }, - { - id: "00000000-0000-4000-8000-000000000012", - type: "target" as const, - provider: "fail/faildb", - stream: "fail/write", - connection: failCredId, - params: {}, - }, - ], - edges: [ - { - from: "00000000-0000-4000-8000-000000000011", - to: "00000000-0000-4000-8000-000000000012", - }, - ], - }; - - const connections: Connections = { - ...testConnections(), - [failCredId]: { - type: "faildb", - credentials: { host: "localhost" }, - context: {}, - }, - }; - - const result = await engine.executeSync(graph, connections); - - // The source node should fail (non-retryable skips retries) - const sourceNode = result.nodes.find( - (n) => n.nodeId === "00000000-0000-4000-8000-000000000011", - ); - expect(sourceNode?.status).toBe("failure"); - expect(sourceNode?.error?.message).toContain("Non-retryable failure"); - }); - - it("retryable error triggers retry", async () => { - const { Action, Document, Plugin, Provider, Stream } = await import( - "@nvisy/core" - ); - const { z } = await import("zod"); - - let attempts = 0; - - class RetryClient {} - - const retryProvider = Provider.withAuthentication("retrydb", { - credentials: z.object({ host: z.string() }), - connect: async () => ({ - client: new RetryClient(), - }), - }); - - const retrySource = Stream.createSource("read", RetryClient, { - types: [ - Document, - z.object({}).default({}), - z.record(z.string(), z.unknown()), - ], - reader: async function* () { - attempts++; - if (attempts < 3) { - throw new RuntimeError("Transient failure", { retryable: true }); - } - yield { - data: new Document("recovered"), - context: {}, - }; - }, - }); - - const retryTarget = Stream.createTarget("write", RetryClient, { - types: [Document, z.record(z.string(), z.unknown())], - writer: () => async () => {}, - }); - - const retryPlugin = Plugin.define("retry") - .withActions( - Action.withoutClient("noop", { - types: [Document], - params: z.object({}), - transform: (stream) => stream, - }), - ) - .withProviders(retryProvider) - .withStreams(retrySource, retryTarget); - - const engine = makeTestEngine(); - engine.register(retryPlugin); - - const retryCredId = "00000000-0000-4000-8000-0000000000f2"; - const graph = { - id: "00000000-0000-4000-8000-000000000020", - nodes: [ - { - id: "00000000-0000-4000-8000-000000000021", - type: "source" as const, - provider: "retry/retrydb", - stream: "retry/read", - connection: retryCredId, - params: {}, - retry: { - maxRetries: 5, - backoff: "fixed" as const, - initialDelayMs: 1, - maxDelayMs: 1, - }, - }, - { - id: "00000000-0000-4000-8000-000000000022", - type: "target" as const, - provider: "retry/retrydb", - stream: "retry/write", - connection: retryCredId, - params: {}, - }, - ], - edges: [ - { - from: "00000000-0000-4000-8000-000000000021", - to: "00000000-0000-4000-8000-000000000022", - }, - ], - }; - - const connections: Connections = { - ...testConnections(), - [retryCredId]: { - type: "retrydb", - credentials: { host: "localhost" }, - context: {}, - }, - }; - - const result = await engine.executeSync(graph, connections); - - expect(result.status).toBe("success"); - expect(attempts).toBe(3); // Failed twice, succeeded on third - }); - - it("calls onContextUpdate for each yielded resumable", async () => { - const engine = makeTestEngine(); - const updates: Array<{ - nodeId: string; - connectionId: string; - context: unknown; - }> = []; - - const result = await engine.executeSync(linearGraph(), testConnections(), { - onContextUpdate: (nodeId, connectionId, context) => { - updates.push({ nodeId, connectionId, context }); - }, - }); - - expect(result.status).toBe("success"); - expect(updates).toHaveLength(sourceEntries.length); - for (const update of updates) { - expect(update.nodeId).toBe(SOURCE_ID); - expect(update.connectionId).toBe(CRED_ID); - expect(update.context).toHaveProperty("cursor"); - } - }); -}); - -describe("credential validation", () => { - it("rejects malformed connections map (non-UUID keys)", async () => { - const engine = makeTestEngine(); - const connections = { - "not-a-uuid": { - type: "testdb", - credentials: { host: "localhost" }, - context: {}, - }, - }; - - await expect( - engine.executeSync(linearGraph(), connections), - ).rejects.toThrow(ValidationError); - }); - - it("rejects missing connection entry at execution time", async () => { - const engine = makeTestEngine(); - - // Provide empty connections — the node references CRED_ID which won't be found - await expect(engine.executeSync(linearGraph(), {})).rejects.toThrow( - ValidationError, - ); - }); - - it("rejects credentials that don't match provider schema", async () => { - const engine = makeTestEngine(); - const connections: Connections = { - [CRED_ID]: { - type: "testdb", - credentials: { wrong: "field" }, - context: {}, - }, - }; - - await expect( - engine.executeSync(linearGraph(), connections), - ).rejects.toThrow(ValidationError); - }); - - it("accepts valid connections map with extra entries", async () => { - const engine = makeTestEngine(); - const extraCredId = "00000000-0000-4000-8000-0000000000e0"; - const connections: Connections = { - ...testConnections(), - [extraCredId]: { - type: "other", - credentials: { unused: true }, - context: {}, - }, - }; - - const result = await engine.executeSync(linearGraph(), connections); - - expect(result.status).toBe("success"); - }); -}); diff --git a/packages/nvisy-runtime/test/fixtures.ts b/packages/nvisy-runtime/test/fixtures.ts deleted file mode 100644 index 6b09fac..0000000 --- a/packages/nvisy-runtime/test/fixtures.ts +++ /dev/null @@ -1,233 +0,0 @@ -import type { JsonValue, Resumable } from "@nvisy/core"; -import { Action, Data, Plugin, Provider, Stream } from "@nvisy/core"; -import { z } from "zod"; -import type { Connections } from "../src/engine/connections.js"; -import { Engine } from "../src/engine/engine.js"; -import { Registry } from "../src/registry.js"; - -/** Minimal row-like data type for testing. */ -export class TestRow extends Data { - readonly #columns: Readonly>; - - constructor(columns: Record) { - super(); - this.#columns = columns; - } - - get columns(): Readonly> { - return this.#columns; - } - - get(column: string): JsonValue | undefined { - return this.#columns[column]; - } -} - -export const GRAPH_ID = "00000000-0000-4000-8000-000000000000"; -export const SOURCE_ID = "00000000-0000-4000-8000-000000000001"; -export const ACTION_ID = "00000000-0000-4000-8000-000000000002"; -export const TARGET_ID = "00000000-0000-4000-8000-000000000003"; -export const EXTRA_ID = "00000000-0000-4000-8000-000000000004"; -export const CRED_ID = "00000000-0000-4000-8000-0000000000c0"; - -const NoopParams = z.object({}); - -export const noopAction = Action.withoutClient("noop", { - types: [TestRow], - params: NoopParams, - transform: (stream, _params) => stream, -}); - -const TestCredentials = z.object({ host: z.string() }); - -class TestClient {} - -export const testProvider = Provider.withAuthentication("testdb", { - credentials: TestCredentials, - connect: async (_creds) => ({ - client: new TestClient(), - }), -}); - -const TestContext = z.object({ - cursor: z.string().nullable().default(null), -}); - -const TestParams = z.record(z.string(), z.unknown()); - -/** - * Items produced by the mock source stream. - * Exposed so tests can assert on them. - */ -export const sourceEntries: TestRow[] = [ - new TestRow({ name: "Alice", age: 30 }), - new TestRow({ name: "Bob", age: 25 }), - new TestRow({ name: "Carol", age: 35 }), -]; - -export const testSourceStream = Stream.createSource("read", TestClient, { - types: [TestRow, TestContext, TestParams], - reader: async function* (_client, _ctx, _params) { - for (const row of sourceEntries) { - yield { data: row, context: { cursor: row.id } } as Resumable< - TestRow, - z.infer - >; - } - }, -}); - -/** - * Items written to the mock target stream. - * Tests can inspect this array after execution. - */ -export const writtenItems: Data[] = []; - -export const testTargetStream = Stream.createTarget("write", TestClient, { - types: [TestRow, TestParams], - writer: (_client, _params) => { - return async (item: TestRow) => { - writtenItems.push(item); - }; - }, -}); - -export const testPlugin = Plugin.define("test") - .withActions(noopAction) - .withProviders(testProvider) - .withStreams(testSourceStream, testTargetStream); - -/** - * Create a Registry pre-loaded with the test plugin. - */ -export function makeTestRegistry(): Registry { - const registry = new Registry(); - registry.load(testPlugin); - return registry; -} - -/** - * Create an Engine pre-loaded with the test plugin. - */ -export function makeTestEngine(): Engine { - return new Engine().register(testPlugin); -} - -/** - * Default credential map matching the test provider's schema. - */ -export function testCredentials() { - return { [CRED_ID]: { host: "localhost" } }; -} - -/** - * Default connections map matching the test provider's schema. - */ -export function testConnections(): Connections { - return { - [CRED_ID]: { - type: "testdb", - credentials: { host: "localhost" }, - context: {}, - }, - }; -} - -export function linearGraph() { - return { - id: GRAPH_ID, - nodes: [ - { - id: SOURCE_ID, - type: "source" as const, - provider: "test/testdb", - stream: "test/read", - connection: CRED_ID, - params: { table: "users" }, - }, - { - id: ACTION_ID, - type: "action" as const, - action: "test/noop", - params: {}, - }, - { - id: TARGET_ID, - type: "target" as const, - provider: "test/testdb", - stream: "test/write", - connection: CRED_ID, - params: { table: "output" }, - }, - ], - edges: [ - { from: SOURCE_ID, to: ACTION_ID }, - { from: ACTION_ID, to: TARGET_ID }, - ], - }; -} - -export function isolatedNodesGraph() { - return { - id: GRAPH_ID, - nodes: [ - { - id: SOURCE_ID, - type: "source" as const, - provider: "test/testdb", - stream: "test/read", - connection: CRED_ID, - params: { table: "users" }, - }, - { - id: ACTION_ID, - type: "action" as const, - action: "test/noop", - params: {}, - }, - ], - edges: [], - }; -} - -export function diamondGraph() { - return { - id: GRAPH_ID, - nodes: [ - { - id: SOURCE_ID, - type: "source" as const, - provider: "test/testdb", - stream: "test/read", - connection: CRED_ID, - params: { table: "users" }, - }, - { - id: ACTION_ID, - type: "action" as const, - action: "test/noop", - params: {}, - }, - { - id: EXTRA_ID, - type: "action" as const, - action: "test/noop", - params: {}, - }, - { - id: TARGET_ID, - type: "target" as const, - provider: "test/testdb", - stream: "test/write", - connection: CRED_ID, - params: { table: "output" }, - }, - ], - edges: [ - { from: SOURCE_ID, to: ACTION_ID }, - { from: SOURCE_ID, to: EXTRA_ID }, - { from: ACTION_ID, to: TARGET_ID }, - { from: EXTRA_ID, to: TARGET_ID }, - ], - }; -} diff --git a/packages/nvisy-runtime/test/parse.test.ts b/packages/nvisy-runtime/test/parse.test.ts deleted file mode 100644 index 79f0184..0000000 --- a/packages/nvisy-runtime/test/parse.test.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { parseGraph } from "../src/compiler/parse.js"; -import { - ACTION_ID, - CRED_ID, - GRAPH_ID, - linearGraph, - SOURCE_ID, - TARGET_ID, -} from "./fixtures.js"; - -describe("parseGraph", () => { - it("parses a valid linear graph", () => { - const result = parseGraph(linearGraph()); - - expect(result.definition.id).toBe(GRAPH_ID); - expect(result.definition.nodes).toHaveLength(3); - expect(result.definition.edges).toHaveLength(2); - expect(result.graph.order).toBe(3); - expect(result.graph.size).toBe(2); - }); - - it("returns a RuntimeGraph with correct node attributes", () => { - const result = parseGraph(linearGraph()); - - const attrs = result.graph.getNodeAttributes(SOURCE_ID); - expect(attrs.schema.type).toBe("source"); - }); - - it("creates edge keys in from->to format", () => { - const result = parseGraph(linearGraph()); - - expect(result.graph.hasEdge(`${SOURCE_ID}->${ACTION_ID}`)).toBe(true); - expect(result.graph.hasEdge(`${ACTION_ID}->${TARGET_ID}`)).toBe(true); - }); - - it("rejects input missing required fields", () => { - expect(() => parseGraph({})).toThrow("Graph parse error"); - }); - - it("rejects non-UUID node IDs", () => { - const bad = { - id: GRAPH_ID, - nodes: [ - { id: "not-a-uuid", type: "action", action: "test/noop", params: {} }, - ], - }; - - expect(() => parseGraph(bad)).toThrow("Graph parse error"); - }); - - it("rejects non-UUID graph ID", () => { - const bad = { - id: "bad-graph-id", - nodes: [], - }; - - expect(() => parseGraph(bad)).toThrow("Graph parse error"); - }); - - it("defaults edges to empty array", () => { - const input = { - id: GRAPH_ID, - nodes: [ - { - id: SOURCE_ID, - type: "source", - provider: "x", - stream: "x/read", - connection: CRED_ID, - params: { key: "val" }, - }, - ], - }; - - const result = parseGraph(input); - expect(result.definition.edges).toEqual([]); - expect(result.graph.size).toBe(0); - }); - - it("defaults metadata to empty object", () => { - const input = { - id: GRAPH_ID, - nodes: [], - }; - - const result = parseGraph(input); - expect(result.definition.metadata).toEqual({}); - }); - - it("rejects duplicate node IDs (caught by graphology during parse)", () => { - const input = { - id: GRAPH_ID, - nodes: [ - { - id: SOURCE_ID, - type: "source", - provider: "test/testdb", - stream: "test/read", - connection: CRED_ID, - params: { table: "t" }, - }, - { id: SOURCE_ID, type: "action", action: "test/noop", params: {} }, - ], - }; - - expect(() => parseGraph(input)).toThrow("already exist"); - }); - - it("rejects dangling edge.from references (caught by graphology during parse)", () => { - const input = { - id: GRAPH_ID, - nodes: [ - { - id: SOURCE_ID, - type: "source", - provider: "test/testdb", - stream: "test/read", - connection: CRED_ID, - params: { table: "t" }, - }, - ], - edges: [{ from: ACTION_ID, to: SOURCE_ID }], - }; - - expect(() => parseGraph(input)).toThrow("not found"); - }); - - it("rejects dangling edge.to references (caught by graphology during parse)", () => { - const input = { - id: GRAPH_ID, - nodes: [ - { - id: SOURCE_ID, - type: "source", - provider: "test/testdb", - stream: "test/read", - connection: CRED_ID, - params: { table: "t" }, - }, - ], - edges: [{ from: SOURCE_ID, to: ACTION_ID }], - }; - - expect(() => parseGraph(input)).toThrow("not found"); - }); -}); - -describe("buildRuntimeGraph", () => { - it("builds a graph matching the definition", () => { - const { graph } = parseGraph({ - id: GRAPH_ID, - nodes: [ - { - id: SOURCE_ID, - type: "source" as const, - provider: "x", - stream: "x/read", - connection: CRED_ID, - params: { k: "v" }, - }, - { id: ACTION_ID, type: "action" as const, action: "y", params: {} }, - ], - edges: [{ from: SOURCE_ID, to: ACTION_ID }], - }); - - expect(graph.order).toBe(2); - expect(graph.size).toBe(1); - expect(graph.hasNode(SOURCE_ID)).toBe(true); - expect(graph.hasNode(ACTION_ID)).toBe(true); - expect(graph.hasEdge(`${SOURCE_ID}->${ACTION_ID}`)).toBe(true); - expect(graph.source(`${SOURCE_ID}->${ACTION_ID}`)).toBe(SOURCE_ID); - expect(graph.target(`${SOURCE_ID}->${ACTION_ID}`)).toBe(ACTION_ID); - }); -}); diff --git a/packages/nvisy-runtime/test/plan.test.ts b/packages/nvisy-runtime/test/plan.test.ts deleted file mode 100644 index 86b0b79..0000000 --- a/packages/nvisy-runtime/test/plan.test.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { parseGraph } from "../src/compiler/parse.js"; -import { buildPlan } from "../src/compiler/plan.js"; -import { - ACTION_ID, - CRED_ID, - diamondGraph, - EXTRA_ID, - GRAPH_ID, - isolatedNodesGraph, - linearGraph, - makeTestRegistry, - SOURCE_ID, - TARGET_ID, -} from "./fixtures.js"; - -describe("buildPlan", () => { - it("produces a topological order for a linear graph", () => { - const registry = makeTestRegistry(); - const parsed = parseGraph(linearGraph()); - const plan = buildPlan(parsed, registry); - - expect(plan.order).toEqual([SOURCE_ID, ACTION_ID, TARGET_ID]); - }); - - it("produces a valid topological order for a diamond graph", () => { - const registry = makeTestRegistry(); - const parsed = parseGraph(diamondGraph()); - const plan = buildPlan(parsed, registry); - - // Source must come first, sink must come last - expect(plan.order[0]).toBe(SOURCE_ID); - expect(plan.order[plan.order.length - 1]).toBe(TARGET_ID); - // Both middle nodes must appear between source and sink - expect(plan.order).toContain(ACTION_ID); - expect(plan.order).toContain(EXTRA_ID); - expect(plan.order.indexOf(ACTION_ID)).toBeGreaterThan(0); - expect(plan.order.indexOf(EXTRA_ID)).toBeGreaterThan(0); - expect(plan.order.indexOf(ACTION_ID)).toBeLessThan(plan.order.length - 1); - expect(plan.order.indexOf(EXTRA_ID)).toBeLessThan(plan.order.length - 1); - }); - - it("handles isolated nodes (no edges)", () => { - const registry = makeTestRegistry(); - const parsed = parseGraph(isolatedNodesGraph()); - const plan = buildPlan(parsed, registry); - - expect(plan.order).toHaveLength(2); - expect(plan.order).toContain(SOURCE_ID); - expect(plan.order).toContain(ACTION_ID); - }); - - it("stores resolved entries in the resolved map", () => { - const registry = makeTestRegistry(); - const parsed = parseGraph(linearGraph()); - const plan = buildPlan(parsed, registry); - - const sourceResolved = plan.resolved.get(SOURCE_ID); - expect(sourceResolved).toBeDefined(); - expect(sourceResolved!.type).toBe("source"); - - const actionResolved = plan.resolved.get(ACTION_ID); - expect(actionResolved).toBeDefined(); - expect(actionResolved!.type).toBe("action"); - - const targetResolved = plan.resolved.get(TARGET_ID); - expect(targetResolved).toBeDefined(); - expect(targetResolved!.type).toBe("target"); - }); - - it("exposes the graph definition on the plan", () => { - const registry = makeTestRegistry(); - const parsed = parseGraph(linearGraph()); - const plan = buildPlan(parsed, registry); - - expect(plan.definition.id).toBe(GRAPH_ID); - expect(plan.definition.nodes).toHaveLength(3); - }); - - it("plan.graph is the same RuntimeGraph instance", () => { - const registry = makeTestRegistry(); - const parsed = parseGraph(linearGraph()); - const plan = buildPlan(parsed, registry); - - // Verify graph structure matches definition - expect(plan.graph.order).toBe(plan.definition.nodes.length); - expect(plan.graph.size).toBe(plan.definition.edges.length); - }); - - it("rejects graphs with cycles", () => { - const registry = makeTestRegistry(); - const input = { - id: GRAPH_ID, - nodes: [ - { id: SOURCE_ID, type: "action", action: "test/noop", params: {} }, - { id: ACTION_ID, type: "action", action: "test/noop", params: {} }, - { id: TARGET_ID, type: "action", action: "test/noop", params: {} }, - ], - edges: [ - { from: SOURCE_ID, to: ACTION_ID }, - { from: ACTION_ID, to: TARGET_ID }, - { from: TARGET_ID, to: SOURCE_ID }, - ], - }; - - const parsed = parseGraph(input); - expect(() => buildPlan(parsed, registry)).toThrow("Graph contains a cycle"); - }); - - it("rejects unresolved action names", () => { - const registry = makeTestRegistry(); - const input = { - id: GRAPH_ID, - nodes: [ - { - id: ACTION_ID, - type: "action", - action: "nonexistent/action", - params: {}, - }, - ], - }; - - const parsed = parseGraph(input); - expect(() => buildPlan(parsed, registry)).toThrow("Unresolved names"); - }); - - it("rejects unresolved provider names", () => { - const registry = makeTestRegistry(); - const input = { - id: GRAPH_ID, - nodes: [ - { - id: SOURCE_ID, - type: "source", - provider: "nonexistent/provider", - stream: "test/read", - connection: CRED_ID, - params: { key: "val" }, - }, - ], - }; - - const parsed = parseGraph(input); - expect(() => buildPlan(parsed, registry)).toThrow("Unresolved names"); - }); - - it("rejects unresolved stream names", () => { - const registry = makeTestRegistry(); - const input = { - id: GRAPH_ID, - nodes: [ - { - id: SOURCE_ID, - type: "source", - provider: "test/testdb", - stream: "test/nonexistent", - connection: CRED_ID, - params: { key: "val" }, - }, - ], - }; - - const parsed = parseGraph(input); - expect(() => buildPlan(parsed, registry)).toThrow("Unresolved names"); - }); - - it("passes for an empty graph", () => { - const registry = makeTestRegistry(); - const input = { id: GRAPH_ID, nodes: [] }; - - const parsed = parseGraph(input); - const plan = buildPlan(parsed, registry); - - expect(plan.definition.nodes).toHaveLength(0); - expect(plan.graph.order).toBe(0); - expect(plan.resolved.size).toBe(0); - }); -}); diff --git a/packages/nvisy-runtime/test/registry.test.ts b/packages/nvisy-runtime/test/registry.test.ts deleted file mode 100644 index 8f4f114..0000000 --- a/packages/nvisy-runtime/test/registry.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { Datatypes, Document, Plugin, ValidationError } from "@nvisy/core"; -import { describe, expect, it } from "vitest"; -import { Registry } from "../src/registry.js"; -import { - makeTestRegistry, - noopAction, - testPlugin, - testProvider, - testSourceStream, - testTargetStream, -} from "./fixtures.js"; - -describe("Registry", () => { - it("loads a plugin and resolves its entries by qualified name", () => { - const registry = makeTestRegistry(); - - expect(registry.getAction("test/noop")).toBe(noopAction); - expect(registry.getProvider("test/testdb")).toBe(testProvider); - expect(registry.getStream("test/read")).toBe(testSourceStream); - expect(registry.getStream("test/write")).toBe(testTargetStream); - }); - - it("find* returns undefined for missing entries", () => { - const registry = makeTestRegistry(); - - expect(registry.findAction("missing/action")).toBeUndefined(); - expect(registry.findProvider("missing/provider")).toBeUndefined(); - expect(registry.findStream("missing/stream")).toBeUndefined(); - expect(registry.findDataType("missing/type")).toBeUndefined(); - }); - - it("get* throws ValidationError for missing entries", () => { - const registry = makeTestRegistry(); - - expect(() => registry.getAction("missing/action")).toThrow(ValidationError); - expect(() => registry.getProvider("missing/provider")).toThrow( - ValidationError, - ); - expect(() => registry.getStream("missing/stream")).toThrow(ValidationError); - expect(() => registry.getDataType("missing/type")).toThrow(ValidationError); - }); - - it("rejects loading the same plugin twice", () => { - const registry = makeTestRegistry(); - - expect(() => registry.load(testPlugin)).toThrow("Plugin already loaded"); - }); - - it("loads datatypes and resolves them", () => { - const registry = new Registry(); - const plugin = Plugin.define("dt").withDatatypes( - Datatypes.define("document", Document), - ); - registry.load(plugin); - - const entry = registry.getDataType("dt/document"); - expect(entry.id).toBe("document"); - expect(entry.dataClass).toBe(Document); - }); - - it("schema snapshot includes actions and providers", () => { - const registry = makeTestRegistry(); - const schema = registry.schema; - - expect(schema.actions).toHaveLength(1); - expect(schema.actions[0]!.name).toBe("test/noop"); - expect(schema.providers).toHaveLength(1); - expect(schema.providers[0]!.name).toBe("test/testdb"); - expect(schema.streams).toBe(2); - expect(schema.loaders).toBe(0); - expect(schema.datatypes).toBe(0); - }); - - it("schema snapshot is empty for a fresh registry", () => { - const registry = new Registry(); - const schema = registry.schema; - - expect(schema.actions).toHaveLength(0); - expect(schema.providers).toHaveLength(0); - expect(schema.streams).toBe(0); - expect(schema.loaders).toBe(0); - expect(schema.datatypes).toBe(0); - }); -}); diff --git a/packages/nvisy-runtime/tsconfig.json b/packages/nvisy-runtime/tsconfig.json deleted file mode 100644 index c91a2dd..0000000 --- a/packages/nvisy-runtime/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - /* Emit */ - "outDir": "./dist", - "rootDir": "./src", - "composite": true - }, - /* Scope */ - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"], - "references": [{ "path": "../nvisy-core" }] -} diff --git a/packages/nvisy-runtime/tsup.config.ts b/packages/nvisy-runtime/tsup.config.ts deleted file mode 100644 index d68a5db..0000000 --- a/packages/nvisy-runtime/tsup.config.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { defineConfig } from "tsup"; - -export default defineConfig({ - /* Entry */ - entry: ["src/index.ts"], - format: ["esm"], - - /* Output */ - outDir: "dist", - dts: { compilerOptions: { composite: false } }, - sourcemap: true, - clean: true, - - /* Optimization */ - splitting: false, - treeshake: true, - skipNodeModulesBundle: true, - - /* Environment */ - platform: "node", - target: "es2024", -}); diff --git a/packages/nvisy-server/README.md b/packages/nvisy-server/README.md deleted file mode 100644 index e993cc6..0000000 --- a/packages/nvisy-server/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# @nvisy/server - -[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/build.yml) - -HTTP server for the Nvisy Runtime platform, built on Hono. - -## Features - -- **REST API**: graph lifecycle management, run execution, and monitoring -- **Connector health checks**: verify provider connections before execution -- **Cron scheduling**: time-based pipeline triggers -- **Webhook events**: HTTP-based pipeline triggers - -## Overview - -Exposes a REST API for graph lifecycle management, run execution and monitoring, connector health checks, and lineage queries. Includes cron-based scheduling and webhook event triggers. - -## Usage - -```ts -import { createServer } from "@nvisy/server"; - -const server = createServer({ - engine, - port: 3000, -}); - -await server.start(); -``` - -## Changelog - -See [CHANGELOG.md](../../CHANGELOG.md) for release notes and version history. - -## License - -Apache 2.0 License - see [LICENSE.txt](../../LICENSE.txt) - -## Support - -- **Documentation**: [docs.nvisy.com](https://docs.nvisy.com) -- **Issues**: [GitHub Issues](https://github.com/nvisycom/runtime/issues) -- **Email**: [support@nvisy.com](mailto:support@nvisy.com) diff --git a/packages/nvisy-server/package.json b/packages/nvisy-server/package.json deleted file mode 100644 index 5d5ee2a..0000000 --- a/packages/nvisy-server/package.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "name": "@nvisy/server", - "version": "0.1.0", - "private": true, - "description": "HTTP execution worker for the Nvisy runtime", - "type": "module", - "scripts": { - "build": "tsup", - "build:watch": "tsup --watch", - "clean": "rimraf dist", - "dev": "node --watch dist/main.js", - "start": "node dist/main.js", - "typecheck": "tsc -b" - }, - "dependencies": { - "@hono/event-emitter": "^2.0.0", - "@hono/node-server": "^1.19.9", - "@hono/node-ws": "^1.3.0", - "@hono/otel": "^1.1.0", - "@hono/zod-openapi": "^1.2.1", - "@hono/zod-validator": "^0.7.6", - "@logtape/hono": "^2.0.2", - "@logtape/logtape": "^2.0.2", - "@logtape/pretty": "^2.0.2", - "@logtape/redaction": "^2.0.2", - "@nvisy/core": "*", - "@nvisy/plugin-ai": "*", - "@nvisy/plugin-object": "*", - "@nvisy/plugin-pandoc": "*", - "@nvisy/plugin-sql": "*", - "@nvisy/plugin-vector": "*", - "@nvisy/runtime": "*", - "@scalar/hono-api-reference": "^0.9.40", - "hono": "^4.11.7", - "zod": "^4.3.6" - }, - "engines": { - "node": ">=22.0.0" - } -} diff --git a/packages/nvisy-server/src/app.ts b/packages/nvisy-server/src/app.ts deleted file mode 100644 index 95b267e..0000000 --- a/packages/nvisy-server/src/app.ts +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Application building blocks for the Nvisy HTTP server. - * - * Exports the Hono app factory ({@link createApp}) and a plain - * server lifecycle ({@link startServer}). These are composed - * into a running service by `main.ts`. - * - * @module - */ - -import { serve } from "@hono/node-server"; -import { createNodeWebSocket } from "@hono/node-ws"; -import { OpenAPIHono } from "@hono/zod-openapi"; -import { getLogger } from "@logtape/logtape"; -import type { ServerConfig } from "./config.js"; -import { registerHandlers } from "./handler/index.js"; -import { engineMiddleware, registerMiddleware } from "./middleware/index.js"; -import { createEngine } from "./service/index.js"; - -const logger = getLogger(["nvisy", "server"]); - -/** Build a fully configured OpenAPIHono application with middleware and routes. */ -export function createApp(config: ServerConfig) { - const app = new OpenAPIHono(); - const { injectWebSocket, upgradeWebSocket } = createNodeWebSocket({ app }); - - const engine = createEngine(); - app.use("*", engineMiddleware(engine)); - - logger.debug("Registering middleware"); - registerMiddleware(app, config); - logger.debug("Registering route handlers"); - registerHandlers(app, config); - logger.debug("App initialised (mode={mode}, cors={cors})", { - mode: config.isDevelopment ? "development" : "production", - cors: config.corsOrigin, - }); - - return { app, injectWebSocket, upgradeWebSocket }; -} - -export interface StartServerOptions { - app: OpenAPIHono; - host: string; - port: number; - injectWebSocket: (server: ReturnType) => void; -} - -/** - * Start the Node.js HTTP server. - * - * Returns a cleanup function that closes the server gracefully. - */ -export function startServer(opts: StartServerOptions): { - server: ReturnType; - close: () => void; -} { - const server = serve({ - fetch: opts.app.fetch, - hostname: opts.host, - port: opts.port, - }); - opts.injectWebSocket(server); - - logger.info("Server started on {host}:{port}", { - host: opts.host, - port: opts.port, - }); - - return { - server, - close: () => server.close(), - }; -} diff --git a/packages/nvisy-server/src/config.ts b/packages/nvisy-server/src/config.ts deleted file mode 100644 index e2490f0..0000000 --- a/packages/nvisy-server/src/config.ts +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Typed server configuration loaded from environment variables. - * - * | Variable | Type | Default | - * |----------------------|--------|-----------------| - * | `PORT` | number | `8080` | - * | `HOST` | string | `"0.0.0.0"` | - * | `CORS_ORIGIN` | string | `"*"` | - * | `BODY_LIMIT_BYTES` | number | `1048576` (1MB) | - * | `REQUEST_TIMEOUT_MS` | number | `30000` (30s) | - * | `NODE_ENV` | string | `"development"` | - * - * `isDevelopment` is derived from `NODE_ENV` — `true` unless - * `NODE_ENV` is explicitly set to `"production"`. - */ - -import { AsyncLocalStorage } from "node:async_hooks"; -import { - configure, - getConsoleSink, - jsonLinesFormatter, -} from "@logtape/logtape"; -import { prettyFormatter } from "@logtape/pretty"; -import { redactByField } from "@logtape/redaction"; -import { z } from "zod"; - -const EnvSchema = z.object({ - PORT: z.coerce.number().default(8080), - HOST: z.string().default("0.0.0.0"), - CORS_ORIGIN: z.string().default("*"), - BODY_LIMIT_BYTES: z.coerce.number().default(1024 * 1024), - REQUEST_TIMEOUT_MS: z.coerce.number().default(30_000), - NODE_ENV: z.string().default("development"), -}); - -export interface ServerConfig { - readonly port: number; - readonly host: string; - readonly corsOrigin: string; - readonly bodyLimitBytes: number; - readonly requestTimeoutMs: number; - readonly isDevelopment: boolean; -} - -export function loadConfig(): ServerConfig { - const env = EnvSchema.parse(process.env); - - return { - port: env.PORT, - host: env.HOST, - corsOrigin: env.CORS_ORIGIN, - bodyLimitBytes: env.BODY_LIMIT_BYTES, - requestTimeoutMs: env.REQUEST_TIMEOUT_MS, - isDevelopment: env.NODE_ENV !== "production", - }; -} - -/** - * Configure LogTape logging. - * - * - **development** — human-readable, coloured console output via `@logtape/pretty`. - * - **production** — JSON Lines (machine-parseable) via the built-in `jsonLinesFormatter`. - * - * Sensitive fields are automatically redacted by `@logtape/redaction`. - * Per-request `requestId` is propagated to every log call via - * `AsyncLocalStorage` — see `middleware/index.ts`. - */ -export async function configureLogging(config: ServerConfig): Promise { - const consoleSink = config.isDevelopment - ? getConsoleSink({ formatter: prettyFormatter }) - : getConsoleSink({ formatter: jsonLinesFormatter }); - - await configure({ - contextLocalStorage: new AsyncLocalStorage(), - sinks: { console: redactByField(consoleSink) }, - loggers: [ - { - category: ["logtape", "meta"], - lowestLevel: "warning", - sinks: ["console"], - }, - { - category: ["nvisy"], - lowestLevel: config.isDevelopment ? "debug" : "info", - sinks: ["console"], - }, - ], - }); -} diff --git a/packages/nvisy-server/src/handler/graphs-routes.ts b/packages/nvisy-server/src/handler/graphs-routes.ts deleted file mode 100644 index 4588e4f..0000000 --- a/packages/nvisy-server/src/handler/graphs-routes.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { createRoute, z } from "@hono/zod-openapi"; -import { - CancelResponseSchema, - ErrorResponseSchema, - ExecuteRequestSchema, - ExecuteResponseSchema, - RunDetailSchema, - RunIdParamSchema, - RunSummarySchema, - ValidateRequestSchema, - ValidateResponseSchema, -} from "./graphs-schema.js"; - -export const executeRoute = createRoute({ - method: "post", - path: "/api/v1/graphs/execute", - tags: ["Graphs"], - summary: "Execute a graph", - description: - "Submit a graph for execution. Returns immediately with a run ID.", - request: { - body: { - content: { "application/json": { schema: ExecuteRequestSchema } }, - }, - }, - responses: { - 202: { - description: "Graph execution started", - content: { "application/json": { schema: ExecuteResponseSchema } }, - }, - }, -}); - -export const validateRoute = createRoute({ - method: "post", - path: "/api/v1/graphs/validate", - tags: ["Graphs"], - summary: "Validate a graph", - description: "Compile and validate a graph definition without executing it.", - request: { - body: { - content: { "application/json": { schema: ValidateRequestSchema } }, - }, - }, - responses: { - 200: { - description: "Validation result", - content: { "application/json": { schema: ValidateResponseSchema } }, - }, - }, -}); - -export const listRunsRoute = createRoute({ - method: "get", - path: "/api/v1/graphs", - tags: ["Graphs"], - summary: "List in-flight runs", - responses: { - 200: { - description: "List of currently executing runs", - content: { "application/json": { schema: z.array(RunSummarySchema) } }, - }, - }, -}); - -export const getRunRoute = createRoute({ - method: "get", - path: "/api/v1/graphs/{runId}", - tags: ["Graphs"], - summary: "Get run status", - description: "Get detailed status of a single in-flight run.", - request: { - params: RunIdParamSchema, - }, - responses: { - 200: { - description: "Run details", - content: { "application/json": { schema: RunDetailSchema } }, - }, - 404: { - description: "Run not found", - content: { "application/json": { schema: ErrorResponseSchema } }, - }, - }, -}); - -export const cancelRunRoute = createRoute({ - method: "delete", - path: "/api/v1/graphs/{runId}", - tags: ["Graphs"], - summary: "Cancel a run", - description: "Cancel a running graph execution.", - request: { - params: RunIdParamSchema, - }, - responses: { - 200: { - description: "Run cancelled", - content: { "application/json": { schema: CancelResponseSchema } }, - }, - 404: { - description: "Run not found or already completed", - content: { "application/json": { schema: ErrorResponseSchema } }, - }, - }, -}); diff --git a/packages/nvisy-server/src/handler/graphs-schema.ts b/packages/nvisy-server/src/handler/graphs-schema.ts deleted file mode 100644 index 6b18b01..0000000 --- a/packages/nvisy-server/src/handler/graphs-schema.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { z } from "@hono/zod-openapi"; - -export const ErrorResponseSchema = z.object({ - status: z.number(), - error: z.string(), - requestId: z.string().optional(), -}); - -export const ConnectionSchema = z.object({ - type: z.string(), - credentials: z.unknown(), - context: z.unknown(), -}); - -export const ConnectionsSchema = z.record(z.uuid(), ConnectionSchema); - -export const GraphSchema = z.record(z.string(), z.unknown()); - -export const ExecuteRequestSchema = z.object({ - graph: GraphSchema, - connections: ConnectionsSchema, -}); - -export const ValidateRequestSchema = z.object({ - graph: GraphSchema, - connections: ConnectionsSchema, -}); - -export const ExecuteResponseSchema = z.object({ - runId: z.string(), -}); - -export const ValidateResponseSchema = z.object({ - valid: z.boolean(), - errors: z.array(z.string()), -}); - -export const RunStatusSchema = z.enum([ - "pending", - "running", - "completed", - "failed", - "cancelled", -]); - -export const RunSummarySchema = z.object({ - runId: z.string(), - status: RunStatusSchema, - startedAt: z.string(), - completedAt: z.string().optional(), -}); - -export const NodeProgressSchema = z.object({ - nodeId: z.string(), - status: z.enum(["pending", "running", "completed", "failed"]), - itemsProcessed: z.number(), - error: z.string().optional(), -}); - -export const NodeResultSchema = z.object({ - nodeId: z.string(), - status: z.enum(["success", "failure", "skipped"]), - itemsProcessed: z.number(), - error: z.string().optional(), -}); - -export const RunResultSchema = z.object({ - runId: z.string(), - status: z.enum(["success", "partial_failure", "failure"]), - nodes: z.array(NodeResultSchema), -}); - -export const RunDetailSchema = z.object({ - runId: z.string(), - status: RunStatusSchema, - startedAt: z.string(), - completedAt: z.string().optional(), - nodeProgress: z.record(z.string(), NodeProgressSchema), - result: RunResultSchema.optional(), - error: z.string().optional(), -}); - -export const CancelResponseSchema = z.object({ - runId: z.string(), - cancelled: z.boolean(), -}); - -export const RunIdParamSchema = z.object({ - runId: z.uuid().openapi({ param: { name: "runId", in: "path" } }), -}); diff --git a/packages/nvisy-server/src/handler/graphs.ts b/packages/nvisy-server/src/handler/graphs.ts deleted file mode 100644 index c261df8..0000000 --- a/packages/nvisy-server/src/handler/graphs.ts +++ /dev/null @@ -1,146 +0,0 @@ -import type { OpenAPIHono } from "@hono/zod-openapi"; -import { getLogger } from "@logtape/logtape"; -import { getEngine } from "../middleware/index.js"; -import { - cancelRunRoute, - executeRoute, - getRunRoute, - listRunsRoute, - validateRoute, -} from "./graphs-routes.js"; - -const logger = getLogger(["nvisy", "server"]); - -/** - * Graph execution endpoints. - * - * POST /api/v1/graphs/execute — Submit a graph for execution, returns { runId } - * POST /api/v1/graphs/validate — Compile and validate a graph without executing - * GET /api/v1/graphs — List in-flight runs - * GET /api/v1/graphs/:runId — Get detailed status of a single in-flight run - * DELETE /api/v1/graphs/:runId — Cancel a running execution - */ -export function registerGraphHandler(app: OpenAPIHono): void { - app.openapi(executeRoute, async (c) => { - const { graph, connections } = c.req.valid("json"); - const engine = getEngine(c); - - const runId = engine.execute(graph, connections); - logger.info("Graph execution submitted: {runId}", { runId }); - - return c.json({ runId }, 202); - }); - - app.openapi(validateRoute, async (c) => { - const { graph, connections } = c.req.valid("json"); - const engine = getEngine(c); - - logger.debug("Graph validation requested"); - const result = engine.validate(graph, connections); - - return c.json({ valid: result.valid, errors: [...result.errors] }, 200); - }); - - app.openapi(listRunsRoute, async (c) => { - const engine = getEngine(c); - - logger.debug("Listing runs"); - const runs = engine.listRuns(); - - return c.json( - runs.map((run) => ({ - runId: run.runId, - status: run.status, - startedAt: run.startedAt.toISOString(), - completedAt: run.completedAt?.toISOString(), - })), - 200, - ); - }); - - app.openapi(getRunRoute, async (c) => { - const { runId } = c.req.valid("param"); - const engine = getEngine(c); - - logger.debug("Run status requested: {runId}", { runId }); - const run = engine.getRun(runId); - - if (!run) { - const requestId = c.get("requestId") as string | undefined; - return c.json( - { status: 404, error: `Run not found: ${runId}`, requestId }, - 404, - ); - } - - const nodeProgress: Record< - string, - { - nodeId: string; - status: "pending" | "running" | "completed" | "failed"; - itemsProcessed: number; - error?: string; - } - > = {}; - for (const [nodeId, progress] of run.nodeProgress) { - nodeProgress[nodeId] = { - nodeId: progress.nodeId, - status: progress.status, - itemsProcessed: progress.itemsProcessed, - ...(progress.error && { error: progress.error.message }), - }; - } - - return c.json( - { - runId: run.runId, - status: run.status, - startedAt: run.startedAt.toISOString(), - completedAt: run.completedAt?.toISOString(), - nodeProgress, - result: run.result - ? { - runId: run.result.runId, - status: run.result.status, - nodes: run.result.nodes.map((n) => ({ - nodeId: n.nodeId, - status: n.status, - itemsProcessed: n.itemsProcessed, - ...(n.error && { error: n.error.message }), - })), - } - : undefined, - error: run.error?.message, - }, - 200, - ); - }); - - app.openapi(cancelRunRoute, async (c) => { - const { runId } = c.req.valid("param"); - const engine = getEngine(c); - - logger.info("Run cancellation requested: {runId}", { runId }); - const cancelled = engine.cancelRun(runId); - - if (!cancelled) { - const requestId = c.get("requestId") as string | undefined; - return c.json( - { - status: 404, - error: `Run not found or already completed: ${runId}`, - requestId, - }, - 404, - ); - } - - return c.json({ runId, cancelled: true }, 200); - }); - - logger.debug(" POST {route}", { route: "/api/v1/graphs/execute" }); - logger.debug(" POST {route}", { route: "/api/v1/graphs/validate" }); - logger.debug(" GET {route}", { route: "/api/v1/graphs" }); - logger.debug(" GET {route}", { route: "/api/v1/graphs/:runId" }); - logger.debug(" DEL {route}", { route: "/api/v1/graphs/:runId" }); -} diff --git a/packages/nvisy-server/src/handler/health-routes.ts b/packages/nvisy-server/src/handler/health-routes.ts deleted file mode 100644 index 2c8234c..0000000 --- a/packages/nvisy-server/src/handler/health-routes.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { createRoute } from "@hono/zod-openapi"; -import { HealthResponseSchema, ReadyResponseSchema } from "./health-schema.js"; - -export const healthRoute = createRoute({ - method: "get", - path: "/health", - tags: ["Health"], - summary: "Liveness probe", - responses: { - 200: { - description: "Server is alive", - content: { "application/json": { schema: HealthResponseSchema } }, - }, - }, -}); - -export const readyRoute = createRoute({ - method: "get", - path: "/ready", - tags: ["Health"], - summary: "Readiness probe", - responses: { - 200: { - description: "Server can accept work", - content: { "application/json": { schema: ReadyResponseSchema } }, - }, - 503: { - description: "Server is not ready", - content: { "application/json": { schema: ReadyResponseSchema } }, - }, - }, -}); diff --git a/packages/nvisy-server/src/handler/health-schema.ts b/packages/nvisy-server/src/handler/health-schema.ts deleted file mode 100644 index a2c1083..0000000 --- a/packages/nvisy-server/src/handler/health-schema.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { z } from "@hono/zod-openapi"; - -export const HealthResponseSchema = z.object({ - status: z.literal("ok"), -}); - -export const ReadyResponseSchema = z.object({ - status: z.enum(["ready", "unavailable"]), -}); diff --git a/packages/nvisy-server/src/handler/health.ts b/packages/nvisy-server/src/handler/health.ts deleted file mode 100644 index 4cbf8b2..0000000 --- a/packages/nvisy-server/src/handler/health.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { OpenAPIHono } from "@hono/zod-openapi"; -import { getLogger } from "@logtape/logtape"; -import { healthRoute, readyRoute } from "./health-routes.js"; - -const logger = getLogger(["nvisy", "server"]); - -/** - * Health and readiness endpoints. - * - * GET /health — Liveness probe. Always returns 200. - * GET /ready — Readiness probe. Returns 200 when the runtime can accept work. - */ -export function registerHealthHandler(app: OpenAPIHono): void { - app.openapi(healthRoute, (c) => { - return c.json({ status: "ok" as const }, 200); - }); - - app.openapi(readyRoute, (c) => { - // TODO: check whether the runtime can accept new graph executions - return c.json({ status: "ready" as const }, 200); - }); - - logger.debug(" GET {route}", { route: "/health" }); - logger.debug(" GET {route}", { route: "/ready" }); -} diff --git a/packages/nvisy-server/src/handler/index.ts b/packages/nvisy-server/src/handler/index.ts deleted file mode 100644 index 24488e9..0000000 --- a/packages/nvisy-server/src/handler/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { OpenAPIHono } from "@hono/zod-openapi"; -import { getLogger } from "@logtape/logtape"; -import type { ServerConfig } from "../config.js"; -import { registerGraphHandler } from "./graphs.js"; -import { registerHealthHandler } from "./health.js"; -import { registerOpenApiHandler } from "./openapi.js"; - -const logger = getLogger(["nvisy", "server"]); - -/** - * Register all route handlers on the given OpenAPIHono app. - * - * Registration order matters: health and graph handlers are registered - * first so that the OpenAPI spec includes all routes. - */ -export function registerHandlers(app: OpenAPIHono, config: ServerConfig): void { - logger.debug("Registering health handlers"); - registerHealthHandler(app); - logger.debug("Registering graph handlers"); - registerGraphHandler(app); - logger.debug("Registering OpenAPI handlers"); - registerOpenApiHandler(app, config); -} diff --git a/packages/nvisy-server/src/handler/openapi.ts b/packages/nvisy-server/src/handler/openapi.ts deleted file mode 100644 index 0ad912e..0000000 --- a/packages/nvisy-server/src/handler/openapi.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type { OpenAPIHono } from "@hono/zod-openapi"; -import { getLogger } from "@logtape/logtape"; -import { Scalar } from "@scalar/hono-api-reference"; -import type { ServerConfig } from "../config.js"; - -const logger = getLogger(["nvisy", "server"]); - -/** Path where the OpenAPI 3.1 JSON specification is served. */ -const SPEC_PATH = "/openapi.json"; - -/** Path where the Scalar API reference UI is served. */ -const DOCS_PATH = "/docs"; - -/** - * OpenAPI spec and Scalar API reference endpoints. - * - * GET /openapi.json — OpenAPI 3.1 JSON spec - * GET /docs — Scalar API reference UI - */ -export function registerOpenApiHandler( - app: OpenAPIHono, - config: ServerConfig, -): void { - app.doc(SPEC_PATH, { - openapi: "3.1.0", - info: { - title: "Nvisy Runtime", - version: "0.1.0", - description: "Stateless execution worker for Nvisy graph pipelines.", - license: { name: "Apache-2.0" }, - }, - servers: [{ url: `http://${config.host}:${config.port}` }], - }); - - app.get( - DOCS_PATH, - Scalar({ - url: SPEC_PATH, - theme: "default", - }), - ); - - logger.debug(" GET {spec}", { spec: SPEC_PATH }); - logger.debug(" GET {docs}", { docs: DOCS_PATH }); -} diff --git a/packages/nvisy-server/src/main.ts b/packages/nvisy-server/src/main.ts deleted file mode 100644 index 326d634..0000000 --- a/packages/nvisy-server/src/main.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Server entry point. - * - * Reads configuration from environment variables, configures - * logging via LogTape, builds the Hono app, and starts the - * HTTP server. - * - * @module - */ - -import { getLogger } from "@logtape/logtape"; -import { createApp, startServer } from "./app.js"; -import { configureLogging, loadConfig } from "./config.js"; - -const config = loadConfig(); -await configureLogging(config); - -const { app, injectWebSocket } = createApp(config); - -const { close } = startServer({ - app, - host: config.host, - port: config.port, - injectWebSocket, -}); - -const logger = getLogger(["nvisy", "server"]); - -function shutdown() { - logger.info("Shutting down"); - close(); - process.exit(0); -} - -process.on("SIGINT", shutdown); -process.on("SIGTERM", shutdown); diff --git a/packages/nvisy-server/src/middleware/error-handler.ts b/packages/nvisy-server/src/middleware/error-handler.ts deleted file mode 100644 index 7775fd2..0000000 --- a/packages/nvisy-server/src/middleware/error-handler.ts +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Global error and not-found handlers. - * - * - {@link createErrorHandler} — registered via `app.onError` - * - {@link createNotFoundHandler} — registered via `app.notFound` - * - * Both return the unified {@link ErrorResponse} JSON envelope so every - * error response — 4xx, 5xx, thrown `HTTPException`, unmatched route — - * has the same shape. - * - * `requestId` appears in log output automatically via LogTape implicit - * context (set by the `withContext` middleware in `index.ts`). It is - * still read from the Hono context for inclusion in the JSON body. - */ - -import { getLogger } from "@logtape/logtape"; -import type { Context, ErrorHandler, NotFoundHandler } from "hono"; -import { HTTPException } from "hono/http-exception"; - -/** Unified JSON error envelope returned by all error responses. */ -interface ErrorResponse { - status: number; - error: string; - requestId?: string; - stack?: string; -} - -const logger = getLogger(["nvisy", "server"]); - -/** - * Create the global `app.onError` handler. - * - * - `HTTPException` → logs at warn, returns the exception's status + message. - * - Anything else → logs at error, returns 500 with a generic message - * (or the real message + stack trace in development). - */ -export function createErrorHandler(opts: { - isDevelopment: boolean; -}): ErrorHandler { - return (error: Error, c: Context): Response => { - const requestId = c.get("requestId") as string | undefined; - - if (error instanceof HTTPException) { - const status = error.status; - logger.warn("HTTP {status} on {method} {path}: {message}", { - status, - method: c.req.method, - path: c.req.path, - message: error.message, - }); - const body: ErrorResponse = { - status, - error: error.message, - ...(requestId && { requestId }), - }; - return c.json(body, status); - } - - logger.error("Unhandled error on {method} {path}: {message}", { - method: c.req.method, - path: c.req.path, - message: error.message, - stack: error.stack, - }); - - const body: ErrorResponse = { - status: 500, - error: opts.isDevelopment ? error.message : "Internal server error", - ...(requestId && { requestId }), - ...(opts.isDevelopment && error.stack && { stack: error.stack }), - }; - return c.json(body, 500); - }; -} - -/** - * Create the global `app.notFound` handler. - * - * - **development** — includes the method and path in the error message. - * - **production** — returns a generic "Not found" message. - */ -export function createNotFoundHandler(opts: { - isDevelopment: boolean; -}): NotFoundHandler { - return (c) => { - const requestId = c.get("requestId") as string | undefined; - const body: ErrorResponse = { - status: 404, - error: opts.isDevelopment - ? `Not found: ${c.req.method} ${c.req.path}` - : "Not found", - ...(requestId && { requestId }), - }; - return c.json(body, 404); - }; -} diff --git a/packages/nvisy-server/src/middleware/hono-context.ts b/packages/nvisy-server/src/middleware/hono-context.ts deleted file mode 100644 index f9fad1e..0000000 --- a/packages/nvisy-server/src/middleware/hono-context.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { Engine } from "@nvisy/runtime"; -import type { Context, MiddlewareHandler } from "hono"; - -const ENGINE_KEY = "engine" as const; - -declare module "hono" { - interface ContextVariableMap { - [ENGINE_KEY]: Engine; - } -} - -/** Middleware that injects the Engine into Hono context. */ -export function engineMiddleware(engine: Engine): MiddlewareHandler { - return async (c, next) => { - c.set(ENGINE_KEY, engine); - await next(); - }; -} - -/** Retrieve the Engine from Hono context. */ -export function getEngine(c: Context): Engine { - return c.get(ENGINE_KEY); -} diff --git a/packages/nvisy-server/src/middleware/index.ts b/packages/nvisy-server/src/middleware/index.ts deleted file mode 100644 index 57fce7a..0000000 --- a/packages/nvisy-server/src/middleware/index.ts +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Global middleware registration. - * - * Middleware is applied in declaration order. The first three entries - * establish the per-request foundations that everything else relies on: - * - * 1. **OTel instrumentation** — wraps the request in an OpenTelemetry span. - * 2. **Request ID** — generates (or reads from `X-Request-Id`) a UUID. - * 3. **Implicit log context** — propagates `requestId` into every LogTape - * log call for the remainder of the request via `withContext`. - * - * After that, the request logger and standard security / transport - * middleware are registered. - */ - -import { httpInstrumentationMiddleware } from "@hono/otel"; -import type { OpenAPIHono } from "@hono/zod-openapi"; -import { withContext } from "@logtape/logtape"; -import { bodyLimit } from "hono/body-limit"; -import { compress } from "hono/compress"; -import { cors } from "hono/cors"; -import { csrf } from "hono/csrf"; -import { etag } from "hono/etag"; -import { requestId } from "hono/request-id"; -import { secureHeaders } from "hono/secure-headers"; -import { timeout } from "hono/timeout"; -import { timing } from "hono/timing"; -import type { ServerConfig } from "../config.js"; -import { createErrorHandler, createNotFoundHandler } from "./error-handler.js"; -import { createRequestLogger } from "./request-logger.js"; - -export { engineMiddleware, getEngine } from "./hono-context.js"; - -/** Register all global middleware on the given OpenAPIHono app. */ -export function registerMiddleware(app: OpenAPIHono, config: ServerConfig) { - app.onError(createErrorHandler({ isDevelopment: config.isDevelopment })); - app.notFound(createNotFoundHandler({ isDevelopment: config.isDevelopment })); - - app.use("*", httpInstrumentationMiddleware()); - app.use("*", requestId()); - app.use("*", async (c, next) => { - await withContext({ requestId: c.get("requestId") }, next); - }); - app.use("*", createRequestLogger({ isDevelopment: config.isDevelopment })); - - app.use("*", secureHeaders()); - app.use("*", csrf()); - app.use("*", compress()); - app.use("*", etag()); - app.use("*", bodyLimit({ maxSize: config.bodyLimitBytes })); - app.use("*", timeout(config.requestTimeoutMs)); - app.use("*", timing()); - app.use("*", cors({ origin: config.corsOrigin })); -} diff --git a/packages/nvisy-server/src/middleware/request-logger.ts b/packages/nvisy-server/src/middleware/request-logger.ts deleted file mode 100644 index 93326c5..0000000 --- a/packages/nvisy-server/src/middleware/request-logger.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** - * HTTP request logger backed by {@link https://jsr.io/@logtape/hono | @logtape/hono}. - * - * - **development** — human-readable one-liner: - * `GET /api/v1/graphs → 200 (1.2ms)` - * - **production** — structured object (consumed by `jsonLinesFormatter`): - * `{ method, path, status, responseTime }` - * - * The per-request `requestId` is attached automatically via LogTape - * implicit context — see the `withContext` middleware in `index.ts`. - */ - -import { type HonoContext, honoLogger } from "@logtape/hono"; -import type { MiddlewareHandler } from "hono"; - -/** Human-readable request summary for development console output. */ -const devFormat = (c: HonoContext, ms: number): string => - `${c.req.method} ${c.req.path} → ${c.res.status} (${ms.toFixed(1)}ms)`; - -/** Structured request properties for JSON Lines production output. */ -const prodFormat = (c: HonoContext, ms: number) => ({ - method: c.req.method, - path: c.req.path, - status: c.res.status, - responseTime: ms.toFixed(1), -}); - -/** Create request-logging middleware appropriate for the environment. */ -export function createRequestLogger(opts: { - isDevelopment: boolean; -}): MiddlewareHandler { - return honoLogger({ - category: ["nvisy", "server"], - format: opts.isDevelopment ? devFormat : prodFormat, - }); -} diff --git a/packages/nvisy-server/src/service/engine.ts b/packages/nvisy-server/src/service/engine.ts deleted file mode 100644 index fd7b4ce..0000000 --- a/packages/nvisy-server/src/service/engine.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { aiPlugin } from "@nvisy/plugin-ai"; -import { objectPlugin } from "@nvisy/plugin-object"; -import { pandocPlugin } from "@nvisy/plugin-pandoc"; -import { sqlPlugin } from "@nvisy/plugin-sql"; -import { vectorPlugin } from "@nvisy/plugin-vector"; -import { Engine } from "@nvisy/runtime"; - -const logger = getLogger(["nvisy", "engine"]); - -/** Create and initialize the Engine with all standard plugins. */ -export function createEngine(): Engine { - logger.info("Initializing engine"); - - try { - const engine = new Engine() - .register(aiPlugin) - .register(objectPlugin) - .register(pandocPlugin) - .register(sqlPlugin) - .register(vectorPlugin); - - const { actions, providers, streams, loaders, datatypes } = engine.schema; - logger.info("Engine initialized", { - providers: providers.length, - streams, - actions: actions.length, - loaders, - datatypes, - }); - - return engine; - } catch (error) { - logger.fatal("Failed to initialize engine: {error}", { error }); - throw error; - } -} diff --git a/packages/nvisy-server/src/service/index.ts b/packages/nvisy-server/src/service/index.ts deleted file mode 100644 index 3dcb0a4..0000000 --- a/packages/nvisy-server/src/service/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { createEngine } from "./engine.js"; diff --git a/packages/nvisy-server/tsconfig.json b/packages/nvisy-server/tsconfig.json deleted file mode 100644 index be2bcc4..0000000 --- a/packages/nvisy-server/tsconfig.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - /* Emit */ - "outDir": "./dist", - "rootDir": "./src", - "composite": true - }, - /* Scope */ - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"], - "references": [ - { "path": "../nvisy-core" }, - { "path": "../nvisy-runtime" }, - { "path": "../nvisy-plugin-ai" }, - { "path": "../nvisy-plugin-sql" }, - { "path": "../nvisy-plugin-vector" } - ] -} diff --git a/packages/nvisy-server/tsup.config.ts b/packages/nvisy-server/tsup.config.ts deleted file mode 100644 index d95ed71..0000000 --- a/packages/nvisy-server/tsup.config.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { defineConfig } from "tsup"; - -export default defineConfig({ - /* Entry */ - entry: ["src/main.ts"], - format: ["esm"], - - /* Output */ - outDir: "dist", - dts: false, - sourcemap: true, - clean: true, - - /* Optimization */ - splitting: false, - treeshake: true, - skipNodeModulesBundle: true, - - /* Environment */ - platform: "node", - target: "es2024", -}); diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..772f2b9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "nvisy-workspace" +version = "0.1.0" +requires-python = ">=3.11" + +[tool.uv.workspace] +members = ["packages/nvisy-ai", "packages/nvisy-exif"] diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..12a6950 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,6 @@ +# https://rust-lang.github.io/rustfmt + +group_imports = "StdExternalCrate" +imports_granularity = "Module" +reorder_impl_items = true +merge_derives = false diff --git a/tsconfig.json b/tsconfig.json deleted file mode 100644 index 0779260..0000000 --- a/tsconfig.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "compilerOptions": { - /* Language & Environment */ - "target": "ES2024", - "lib": ["ES2024"], - - /* Modules */ - "module": "NodeNext", - "moduleResolution": "nodenext", - "resolveJsonModule": true, - "isolatedModules": true, - "verbatimModuleSyntax": true, - - /* Type Checking */ - "strict": true, - "noUnusedLocals": true, - "noUnusedParameters": true, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": true, - "noUncheckedIndexedAccess": true, - "noImplicitOverride": true, - "noPropertyAccessFromIndexSignature": true, - "exactOptionalPropertyTypes": true, - - /* Emit */ - "declaration": true, - "declarationMap": true, - "sourceMap": true, - - /* Interop */ - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true - }, - "exclude": ["node_modules", "dist"] -} diff --git a/vitest.config.ts b/vitest.config.ts deleted file mode 100644 index 495bd0b..0000000 --- a/vitest.config.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { defineConfig } from "vitest/config"; - -/* Resolve the "source" export condition so workspace packages point to - ./src/index.ts instead of ./dist/index.js. Vitest runs in Vite's SSR - environment, so ssr.resolve.conditions is required here — the top-level - resolve.conditions only applies to the client environment. */ - -export default defineConfig({ - /* Module Resolution */ - ssr: { - resolve: { - conditions: ["source", "import", "default"], - }, - }, - - test: { - /* Environment */ - globals: true, - environment: "node", - - /* Discovery */ - include: [ - "packages/*/src/**/*.{test,spec}.ts", - "packages/*/test/**/*.{test,spec}.ts", - ], - exclude: ["node_modules", "dist", "**/*.d.ts"], - - /* Coverage */ - coverage: { - provider: "v8", - reporter: ["text", "json", "html", "lcov"], - exclude: [ - "node_modules/", - "dist/", - "coverage/", - "**/*.d.ts", - "**/*.config.*", - "**/index.ts", - ], - }, - - /* Timeouts */ - testTimeout: 15000, - }, - - /* Transform */ - esbuild: { target: "es2024" }, -});