diff --git a/.github/logos/logo-mark.svg b/.github/logos/logo-mark.svg new file mode 100644 index 00000000..12e38cc8 --- /dev/null +++ b/.github/logos/logo-mark.svg @@ -0,0 +1,3 @@ + + + diff --git a/README.md b/README.md index 1c5a6b2b..e4f87a44 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -

skilld

+

skilld

[![npm version](https://img.shields.io/npm/v/skilld?color=yellow)](https://npmjs.com/package/skilld) [![npm downloads](https://img.shields.io/npm/dm/skilld?color=yellow)](https://npm.chart.dev/skilld) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5aae414d..bd5662cb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,8 +7,8 @@ settings: catalogs: default: '@mariozechner/pi-ai': - specifier: ^0.63.1 - version: 0.63.1 + specifier: ^0.64.0 + version: 0.64.0 '@mdream/crawl': specifier: ^1.0.3 version: 1.0.3 @@ -89,8 +89,8 @@ catalogs: specifier: ^7.3.0 version: 7.3.0 sqlite-vec: - specifier: ^0.1.7 - version: 0.1.7 + specifier: ^0.1.8 + version: 0.1.8 tinyglobby: specifier: ^0.2.15 version: 0.2.15 @@ -134,7 +134,7 @@ importers: version: 3.8.1 '@mariozechner/pi-ai': specifier: 'catalog:' - version: 0.63.1(ws@8.19.0)(zod@4.3.6) + version: 0.64.0(ws@8.19.0)(zod@4.3.6) '@mdream/crawl': specifier: 'catalog:' version: 1.0.3(crawlee@3.16.0(@types/node@25.5.0))(magicast@0.5.2) @@ -185,13 +185,13 @@ importers: version: 2.0.3 retriv: specifier: 'catalog:' - version: 0.13.0(@huggingface/transformers@3.8.1)(ai@6.0.68(zod@4.3.6))(sqlite-vec@0.1.7)(typescript@6.0.2) + version: 0.13.0(@huggingface/transformers@3.8.1)(ai@6.0.68(zod@4.3.6))(sqlite-vec@0.1.8)(typescript@6.0.2) semver: specifier: 'catalog:' version: 7.7.4 sqlite-vec: specifier: catalog:deps - version: 0.1.7 + version: 0.1.8 std-env: specifier: 'catalog:' version: 4.0.0 @@ -203,7 +203,7 @@ importers: version: 6.0.2 unagent: specifier: 'catalog:' - version: 0.0.8(@huggingface/transformers@3.8.1)(ai@6.0.68(zod@4.3.6))(sqlite-vec@0.1.7) + version: 0.0.8(@huggingface/transformers@3.8.1)(ai@6.0.68(zod@4.3.6))(sqlite-vec@0.1.8) unist-util-visit: specifier: catalog:deps version: 5.1.0 @@ -1334,8 +1334,8 @@ packages: resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} engines: {node: '>=8'} - '@mariozechner/pi-ai@0.63.1': - resolution: {integrity: sha512-wjgwY+yfrFO6a9QdAfjWpH7iSrDean6GsKDDMohNcLCy6PreMxHOZvNM0NwJARL1tZoZovr7ikAQfLGFZbnjsw==} + '@mariozechner/pi-ai@0.64.0': + resolution: {integrity: sha512-Z/Jnf+JSVDPLRcxJsa8XhYTJKIqKekNueaCpBLGQHgizL1F9RQ1Rur3rIfZpfXkt2cLu/AIPtOs223ueuoWaWg==} engines: {node: '>=20.0.0'} hasBin: true @@ -4738,33 +4738,33 @@ packages: split@0.3.3: resolution: {integrity: sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==} - sqlite-vec-darwin-arm64@0.1.7: - resolution: {integrity: sha512-dQ7u4GKPdOPi3IfZ44K7HHdYup2JssM6fuKR9zgqRzW137uFOQmRhbYChNu+ZfW+yhJutsPgfNRFsuWKmy627w==} + sqlite-vec-darwin-arm64@0.1.8: + resolution: {integrity: sha512-EoHyjPgX8uhWWcYSWbLwJHU6j8AQvwYbwDXFaT1ocdNtJy5qAGtX6LLsBR3vuWOyosMnULF6+DExjldku23h7A==} cpu: [arm64] os: [darwin] - sqlite-vec-darwin-x64@0.1.7: - resolution: {integrity: sha512-MDoczft1BriQcGMEz+CqeSCkB0OsAf12ytZOapS6MaB7zgNzLLSLH6Sxe3yzcPWUyDuCWgK7WzyRIo8u1vAIVA==} + sqlite-vec-darwin-x64@0.1.8: + resolution: {integrity: sha512-IafNV502w90y1BvWQXaRZ2AyyqNINmZ+7LEvkfL45Gf7kC182EFKxn9vbCy6UrY6QYXN/XQo7aICpTO7MIZBqw==} cpu: [x64] os: [darwin] - sqlite-vec-linux-arm64@0.1.7: - resolution: {integrity: sha512-V429sYT/gwr9PgtT8rbjQd6ls7CFchFpiS45TKSf7rU7wxt9MBmCVorUcheD4kEZb4VeZ6PnFXXCqPMeaHkaUw==} + sqlite-vec-linux-arm64@0.1.8: + resolution: {integrity: sha512-R/zogzxQ4LEAgju3knhZuWs8MKdIqN6Bq+ORhLYz5S4eQ98F+rYqilHQ8vsE3Ouy347yxNB2HyldDMcwzgfTyQ==} cpu: [arm64] os: [linux] - sqlite-vec-linux-x64@0.1.7: - resolution: {integrity: sha512-wZL+lXeW7y63DLv6FYU6Q4nv2lP5F94cWt7bJCWNiHmZ6NdKIgz/p0QlyuJA/51b8TyoDvsTdusLVlZz9cIh5A==} + sqlite-vec-linux-x64@0.1.8: + resolution: {integrity: sha512-DBIf2d2mvDb1M//snAKljpIsNQNqXX2AXWNBua0YCj40F/ygFJJ8j3k1pbcJfaEoaP21B+VquH0Gc6RIwZvXig==} cpu: [x64] os: [linux] - sqlite-vec-windows-x64@0.1.7: - resolution: {integrity: sha512-FEZMjMT03irJxwqMQg+A+4hHCiFslxISOAkQ0eYn2lP7GdpppkgYveaT5Xnw/2V+GLq2MXOJb0nDGFNethHSkg==} + sqlite-vec-windows-x64@0.1.8: + resolution: {integrity: sha512-+fBb7kzTpSAvoO/06YMDefnM5PpRhnPVqAYiUvLfBnUCRILDq2F9lcjQQIRjhobrb3eoJWzsGHwnqdVfaIUNxQ==} cpu: [x64] os: [win32] - sqlite-vec@0.1.7: - resolution: {integrity: sha512-1Sge9uRc3B6wDKR4J6sGFi/E2ai9SAU5FenDki3OmhdP/a49PO2Juy1U5yQnx2bZP5t+C3BYJTkG+KkDi3q9Xg==} + sqlite-vec@0.1.8: + resolution: {integrity: sha512-L3xKhQUYQ7kcb3v31KPyPaEigE2upETMSx/5K3vTwm8HRsbci9PKGklXU+mxEYVogojpkenM0TZK5Sz/2FXTQw==} stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -6659,7 +6659,7 @@ snapshots: '@lukeed/ms@2.0.2': {} - '@mariozechner/pi-ai@0.63.1(ws@8.19.0)(zod@4.3.6)': + '@mariozechner/pi-ai@0.64.0(ws@8.19.0)(zod@4.3.6)': dependencies: '@anthropic-ai/sdk': 0.73.0(zod@4.3.6) '@aws-sdk/client-bedrock-runtime': 3.1014.0 @@ -10380,11 +10380,11 @@ snapshots: ret@0.5.0: {} - retriv@0.13.0(@huggingface/transformers@3.8.1)(ai@6.0.68(zod@4.3.6))(sqlite-vec@0.1.7)(typescript@6.0.2): + retriv@0.13.0(@huggingface/transformers@3.8.1)(ai@6.0.68(zod@4.3.6))(sqlite-vec@0.1.8)(typescript@6.0.2): optionalDependencies: '@huggingface/transformers': 3.8.1 ai: 6.0.68(zod@4.3.6) - sqlite-vec: 0.1.7 + sqlite-vec: 0.1.8 typescript: 6.0.2 retry@0.12.0: @@ -10661,28 +10661,28 @@ snapshots: through: 2.3.8 optional: true - sqlite-vec-darwin-arm64@0.1.7: + sqlite-vec-darwin-arm64@0.1.8: optional: true - sqlite-vec-darwin-x64@0.1.7: + sqlite-vec-darwin-x64@0.1.8: optional: true - sqlite-vec-linux-arm64@0.1.7: + sqlite-vec-linux-arm64@0.1.8: optional: true - sqlite-vec-linux-x64@0.1.7: + sqlite-vec-linux-x64@0.1.8: optional: true - sqlite-vec-windows-x64@0.1.7: + sqlite-vec-windows-x64@0.1.8: optional: true - sqlite-vec@0.1.7: + sqlite-vec@0.1.8: optionalDependencies: - sqlite-vec-darwin-arm64: 0.1.7 - sqlite-vec-darwin-x64: 0.1.7 - sqlite-vec-linux-arm64: 0.1.7 - sqlite-vec-linux-x64: 0.1.7 - sqlite-vec-windows-x64: 0.1.7 + sqlite-vec-darwin-arm64: 0.1.8 + sqlite-vec-darwin-x64: 0.1.8 + sqlite-vec-linux-arm64: 0.1.8 + sqlite-vec-linux-x64: 0.1.8 + sqlite-vec-windows-x64: 0.1.8 stackback@0.0.2: {} @@ -10928,7 +10928,7 @@ snapshots: uint8array-extras@1.5.0: {} - unagent@0.0.8(@huggingface/transformers@3.8.1)(ai@6.0.68(zod@4.3.6))(sqlite-vec@0.1.7): + unagent@0.0.8(@huggingface/transformers@3.8.1)(ai@6.0.68(zod@4.3.6))(sqlite-vec@0.1.8): dependencies: croner: 9.1.0 hookable: 6.0.1 @@ -10938,7 +10938,7 @@ snapshots: optionalDependencies: '@huggingface/transformers': 3.8.1 ai: 6.0.68(zod@4.3.6) - sqlite-vec: 0.1.7 + sqlite-vec: 0.1.8 unconfig-core@7.5.0: dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 777f96e8..87c6223c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -7,7 +7,7 @@ packages: overrides: global-agent: ^4.1.3 catalog: - '@mariozechner/pi-ai': ^0.63.1 + '@mariozechner/pi-ai': ^0.64.0 '@mdream/crawl': ^1.0.3 '@types/semver': ^7.7.1 bumpp: ^11.0.1 @@ -36,7 +36,7 @@ catalogs: mlly: ^1.8.2 oxc-parser: ^0.121.0 p-limit: ^7.3.0 - sqlite-vec: ^0.1.7 + sqlite-vec: ^0.1.8 tinyglobby: ^0.2.15 unist-util-visit: ^5.1.0 dev-build: diff --git a/src/retriv/index.ts b/src/retriv/index.ts index dade6bf7..7531322d 100644 --- a/src/retriv/index.ts +++ b/src/retriv/index.ts @@ -6,15 +6,47 @@ export type { ChunkEntity, Document, IndexConfig, IndexPhase, IndexProgress, Sea type RetrivInstance = Awaited> export class SearchDepsUnavailableError extends Error { - constructor(cause: unknown) { - super('Search dependencies unavailable (sqlite-vec or retriv not installed). Search indexing skipped.') + constructor(cause: unknown, message?: string) { + super(message ?? 'Search dependencies unavailable (sqlite-vec or retriv not installed). Search indexing skipped.') this.name = 'SearchDepsUnavailableError' this.cause = cause } } +let _fts5Available: boolean | null = null + +/** + * Probe whether SQLite FTS5 module is available. + * Windows Node.js binaries often ship without FTS5 compiled in. + */ +function checkFts5(): boolean { + if (_fts5Available !== null) + return _fts5Available + const nodeSqlite = globalThis.process?.getBuiltinModule?.('node:sqlite') as typeof import('node:sqlite') | undefined + if (!nodeSqlite) { + _fts5Available = false + return false + } + const db = new nodeSqlite.DatabaseSync(':memory:') + try { + db.exec('CREATE VIRTUAL TABLE _fts5_probe USING fts5(content)') + db.exec('DROP TABLE _fts5_probe') + _fts5Available = true + } + catch { + _fts5Available = false + } + finally { + db.close() + } + return _fts5Available +} + // Dynamic imports: retriv/chunkers/auto eagerly loads typescript which may not be installed (e.g. npx) export async function getDb(config: Pick) { + if (!checkFts5()) + throw new SearchDepsUnavailableError(new Error('FTS5 module not available'), 'SQLite FTS5 module not available. Search indexing skipped. On Windows, run from WSL where FTS5 is included.') + let createRetriv, autoChunker, sqliteMod, sqliteVec, transformersJs, cachedEmbeddings try { ;([