From d95578a5ba3a96c83c77200b39023654a77a96eb Mon Sep 17 00:00:00 2001 From: Meng Lin Date: Sun, 1 Mar 2026 21:16:05 +1100 Subject: [PATCH 1/6] feat: deploy large wasm duckdb --- .gitignore | 1 + .../benchmark/functions/assets/[[path]].ts | 33 ++++++ examples/benchmark/package.json | 4 +- examples/benchmark/public/_routes.json | 5 + examples/benchmark/src/components/sqlTest.tsx | 12 +- examples/benchmark/src/scripts/deployPages.ts | 110 ++++++++++++++++++ examples/benchmark/wrangler.toml | 8 ++ nx.json | 4 + package.json | 2 +- pnpm-lock.yaml | 4 +- 10 files changed, 172 insertions(+), 11 deletions(-) create mode 100644 examples/benchmark/functions/assets/[[path]].ts create mode 100644 examples/benchmark/public/_routes.json create mode 100644 examples/benchmark/src/scripts/deployPages.ts diff --git a/.gitignore b/.gitignore index d498422..4cf3662 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ coverage # build dist +.wrangler # nx .nx diff --git a/examples/benchmark/functions/assets/[[path]].ts b/examples/benchmark/functions/assets/[[path]].ts new file mode 100644 index 0000000..5a4a938 --- /dev/null +++ b/examples/benchmark/functions/assets/[[path]].ts @@ -0,0 +1,33 @@ +export async function onRequest(context: { + request: Request + // @ts-expect-error + env: { WASM_BUCKET: R2Bucket } + next: () => Promise +}) { + const { request, env, next } = context + + if (request.method !== 'GET' && request.method !== 'HEAD') { + return next() + } + + const url = new URL(request.url) + if (!url.pathname.endsWith('.wasm')) { + return next() + } + + const key = url.pathname.replace(/^\/+/, '') + const object = await env.WASM_BUCKET.get(key) + if (!object) { + return next() + } + + const headers = new Headers() + object.writeHttpMetadata(headers) + headers.set('etag', object.httpEtag) + if (key.endsWith('.wasm')) { + headers.set('Content-Type', 'application/wasm') + } + headers.set('Cache-Control', 'public, max-age=31536000, immutable') + + return new Response(object.body, { headers }) +} diff --git a/examples/benchmark/package.json b/examples/benchmark/package.json index 7d168cd..f72fffd 100644 --- a/examples/benchmark/package.json +++ b/examples/benchmark/package.json @@ -9,7 +9,7 @@ "test:lint": "biome lint --write", "test:type": "tsgo --noEmit", "build": "pnpm vite build", - "wrangler:deploy": "wrangler pages deploy" + "wrangler:deploy": "tsx ./src/scripts/deployPages.ts" }, "dependencies": { "@codemirror/commands": "^6.10.2", @@ -25,7 +25,7 @@ "@tanstack/solid-virtual": "^3.10.8", "@tursodatabase/database-wasm": "^0.4.4", "codemirror": "^6.0.2", - "solid-js": "^1.9.10", + "solid-js": "^1.9.11", "sqlocal": "^0.17.0", "zod": "^4.3.6", "zod-schema-faker": "^2.1.0" diff --git a/examples/benchmark/public/_routes.json b/examples/benchmark/public/_routes.json new file mode 100644 index 0000000..e5284d6 --- /dev/null +++ b/examples/benchmark/public/_routes.json @@ -0,0 +1,5 @@ +{ + "version": 1, + "include": ["/assets/*.wasm"], + "exclude": [] +} diff --git a/examples/benchmark/src/components/sqlTest.tsx b/examples/benchmark/src/components/sqlTest.tsx index fc18a14..663676b 100644 --- a/examples/benchmark/src/components/sqlTest.tsx +++ b/examples/benchmark/src/components/sqlTest.tsx @@ -1,6 +1,6 @@ import { createResource } from 'solid-js' -// import { DuckdbDB, duckdbFactory } from './database/duckdbDB.tsx' -// import { DuckdbSchemaMigrator } from './database/duckdbSchemaMigrator.tsx' +import { DuckdbDB, duckdbFactory } from './database/duckdbDB.tsx' +import { DuckdbSchemaMigrator } from './database/duckdbSchemaMigrator.tsx' import { PgliteDB, pgliteFactory } from './database/pgliteDB.tsx' import { PgliteSchemaMigrator } from './database/pgliteSchemaMigrator.tsx' import { SqliteDB, sqliteFactory } from './database/sqliteDB.tsx' @@ -10,7 +10,7 @@ import { StoolapSchemaMigrator } from './database/stoolapSchemaMigrator.tsx' import { TanstackDB, tanstackDbFactory } from './database/tanstackDB.tsx' import { TursoDB, tursoFactory } from './database/tursoDB.tsx' import { TursoSchemaMigrator } from './database/tursoSchemaMigrator.tsx' -// import { TestDuckdbQuery } from './test/testDuckdbQuery.tsx' +import { TestDuckdbQuery } from './test/testDuckdbQuery.tsx' import { TestPgliteDbIvm } from './test/testPgliteDbIvm.tsx' import { TestPgliteDbQuery } from './test/testPgliteDbQuery.tsx' import { TestSqliteQuery } from './test/testSqliteQuery.tsx' @@ -21,7 +21,7 @@ import { UsageMonitor } from './usageMonitor.tsx' export default function SqlTest(props: { query: string; rowCount: number }) { const [tursoQueryDb] = createResource(tursoFactory) - // const [duckdbQueryDb] = createResource(duckdbFactory) + const [duckdbQueryDb] = createResource(duckdbFactory) const [stoolapQueryDb] = createResource(stoolapFactory) return ( @@ -36,13 +36,13 @@ export default function SqlTest(props: { query: string; rowCount: number }) { - {/*
+
-
*/} +
diff --git a/examples/benchmark/src/scripts/deployPages.ts b/examples/benchmark/src/scripts/deployPages.ts new file mode 100644 index 0000000..8d56d0b --- /dev/null +++ b/examples/benchmark/src/scripts/deployPages.ts @@ -0,0 +1,110 @@ +import { spawnSync } from 'node:child_process' +import { readdirSync, readFileSync, statSync, unlinkSync } from 'node:fs' +import { join, relative, resolve } from 'node:path' + +const rootDir = resolve( + new URL('.', import.meta.url).pathname, + '..', + '..', + '..', +) +const distDir = 'dist' +const assetsDir = resolve(distDir, 'assets') +const wranglerToml = resolve(rootDir, 'wrangler.toml') + +function walk(dir: string) { + const entries = readdirSync(dir) + const files: string[] = [] + for (const entry of entries) { + const fullPath = join(dir, entry) + const stat = statSync(fullPath) + if (stat.isDirectory()) { + files.push(...walk(fullPath)) + continue + } + files.push(fullPath) + } + return files +} + +function run(command: string, args: string[]) { + const result = spawnSync(command, args, { stdio: 'inherit' }) + if (result.status !== 0) { + process.exit(result.status ?? 1) + } +} + +function runStatus(command: string, args: string[]) { + const result = spawnSync(command, args, { stdio: 'inherit' }) + return result.status ?? 1 +} + +function readBucketName() { + try { + const text = readFileSync(wranglerToml, 'utf8') + const match = text.match(/bucket_name\s*=\s*"([^"]+)"/) + if (match?.[1]) { + return match[1] + } + } catch { + // Ignore and fall back to env + } + return 'tanstack-db-interpreter-wasm' +} + +if (!statSync(distDir, { throwIfNoEntry: false })) { + console.error('dist/ not found. Run the build first.') + process.exit(1) +} + +const bucketName = readBucketName() +if (!bucketName) { + console.error( + 'R2 bucket name not found. Set bucket_name in wrangler.toml or WASM_BUCKET_NAME env var.', + ) + process.exit(1) +} + +let wasmFiles: string[] = [] +try { + wasmFiles = walk(assetsDir).filter((file) => file.endsWith('.wasm')) +} catch { + wasmFiles = [] +} + +if (wasmFiles.length > 0) { + console.log(`Uploading ${wasmFiles.length} WASM files to R2...`) + for (const file of wasmFiles) { + const key = relative(distDir, file).replaceAll('\\', '/') + const commonArgs = [ + 'r2', + 'object', + 'put', + '--remote', + '--file', + file, + '--content-type', + 'application/wasm', + ] + const combinedStatus = runStatus('wrangler', [ + ...commonArgs, + `${bucketName}/${key}`, + ]) + if (combinedStatus !== 0) { + const splitStatus = runStatus('wrangler', [ + ...commonArgs, + bucketName, + key, + ]) + if (splitStatus !== 0) { + process.exit(splitStatus) + } + } + unlinkSync(file) + } +} else { + console.log('No WASM files found in dist/assets.') +} + +const pagesArgs = ['pages', 'deploy', distDir, ...process.argv.slice(2)] +run('wrangler', pagesArgs) diff --git a/examples/benchmark/wrangler.toml b/examples/benchmark/wrangler.toml index 71e2b07..27cabe2 100644 --- a/examples/benchmark/wrangler.toml +++ b/examples/benchmark/wrangler.toml @@ -3,3 +3,11 @@ name = "tanstack-db-sql-interpreter-benchmark" pages_build_output_dir = "./dist" compatibility_date = "2025-02-28" + +[[r2_buckets]] +binding = "WASM_BUCKET" +bucket_name = "tanstack-db-interpreter-wasm" +preview_bucket_name = "tanstack-db-interpreter-wasm-preview" + +[vars] +WASM_PUBLIC_BASE = "https://" diff --git a/nx.json b/nx.json index 27d0b70..0e1e77b 100644 --- a/nx.json +++ b/nx.json @@ -7,6 +7,7 @@ "test:lint": { "inputs": [ "{projectRoot}/src/*", + "{projectRoot}/functions/*", "{projectRoot}/test/*", "{projectRoot}/package.json", "{workspaceRoot}/biome.json" @@ -17,6 +18,7 @@ "test:type": { "inputs": [ "{projectRoot}/src/*", + "{projectRoot}/functions/*", "{projectRoot}/test/*", "{projectRoot}/package.json", "{projectRoot}/tsconfig.json", @@ -28,6 +30,7 @@ "test:unit": { "inputs": [ "{projectRoot}/src/*", + "{projectRoot}/functions/*", "{projectRoot}/test/**/*", "{projectRoot}/package.json", "{projectRoot}/vitest.config.ts" @@ -37,6 +40,7 @@ "build": { "inputs": [ "{projectRoot}/src/**/*", + "{projectRoot}/functions/*", "{projectRoot}/esbuild.ts", "{projectRoot}/package.json", "{projectRoot}/vite.config.ts" diff --git a/package.json b/package.json index 0c060ec..63e7a80 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "devDependencies": { "@arethetypeswrong/cli": "^0.18.2", "@biomejs/biome": "2.4.4", - "@types/node": "^24.6.2", + "@types/node": "^24.11.0", "@typescript/native-preview": "7.0.0-dev.20260222.1", "@vitest/coverage-v8": "4.0.18", "esbuild": "^0.27.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c6867a9..de33468 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,7 +15,7 @@ importers: specifier: 2.4.4 version: 2.4.4 '@types/node': - specifier: ^24.6.2 + specifier: ^24.11.0 version: 24.11.0 '@typescript/native-preview': specifier: 7.0.0-dev.20260222.1 @@ -93,7 +93,7 @@ importers: specifier: ^6.0.2 version: 6.0.2 solid-js: - specifier: ^1.9.10 + specifier: ^1.9.11 version: 1.9.11 sqlocal: specifier: ^0.17.0 From b96d7c502c6d206bdc317a41a11ff7cab57c9cbf Mon Sep 17 00:00:00 2001 From: Meng Lin Date: Sun, 1 Mar 2026 22:14:43 +1100 Subject: [PATCH 2/6] greatly simplify script --- examples/benchmark/src/scripts/deployPages.ts | 136 +++++------------- 1 file changed, 32 insertions(+), 104 deletions(-) diff --git a/examples/benchmark/src/scripts/deployPages.ts b/examples/benchmark/src/scripts/deployPages.ts index 8d56d0b..5a8d4a9 100644 --- a/examples/benchmark/src/scripts/deployPages.ts +++ b/examples/benchmark/src/scripts/deployPages.ts @@ -1,110 +1,38 @@ -import { spawnSync } from 'node:child_process' -import { readdirSync, readFileSync, statSync, unlinkSync } from 'node:fs' -import { join, relative, resolve } from 'node:path' +import { execSync } from 'node:child_process' +import { readdirSync, readFileSync, rm, rmSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' -const rootDir = resolve( - new URL('.', import.meta.url).pathname, - '..', - '..', - '..', -) -const distDir = 'dist' -const assetsDir = resolve(distDir, 'assets') -const wranglerToml = resolve(rootDir, 'wrangler.toml') +const __filename = fileURLToPath(import.meta.url) +const __dirname = dirname(__filename) -function walk(dir: string) { - const entries = readdirSync(dir) - const files: string[] = [] - for (const entry of entries) { - const fullPath = join(dir, entry) - const stat = statSync(fullPath) - if (stat.isDirectory()) { - files.push(...walk(fullPath)) - continue - } - files.push(fullPath) - } - return files +const wranglerToml = join(__dirname, '../../wrangler.toml') +const readBucketName = () => { + const text = readFileSync(wranglerToml, 'utf8') + const match = text.match(/bucket_name\s*=\s*"([^"]+)"/) + if (match?.[1]) return match[1] + throw Error(`'bucket_name=BUCKET_NAME' cannot be found in wrangler.toml`) } - -function run(command: string, args: string[]) { - const result = spawnSync(command, args, { stdio: 'inherit' }) - if (result.status !== 0) { - process.exit(result.status ?? 1) - } -} - -function runStatus(command: string, args: string[]) { - const result = spawnSync(command, args, { stdio: 'inherit' }) - return result.status ?? 1 -} - -function readBucketName() { - try { - const text = readFileSync(wranglerToml, 'utf8') - const match = text.match(/bucket_name\s*=\s*"([^"]+)"/) - if (match?.[1]) { - return match[1] - } - } catch { - // Ignore and fall back to env - } - return 'tanstack-db-interpreter-wasm' -} - -if (!statSync(distDir, { throwIfNoEntry: false })) { - console.error('dist/ not found. Run the build first.') - process.exit(1) -} - const bucketName = readBucketName() -if (!bucketName) { - console.error( - 'R2 bucket name not found. Set bucket_name in wrangler.toml or WASM_BUCKET_NAME env var.', - ) - process.exit(1) -} -let wasmFiles: string[] = [] -try { - wasmFiles = walk(assetsDir).filter((file) => file.endsWith('.wasm')) -} catch { - wasmFiles = [] -} - -if (wasmFiles.length > 0) { - console.log(`Uploading ${wasmFiles.length} WASM files to R2...`) - for (const file of wasmFiles) { - const key = relative(distDir, file).replaceAll('\\', '/') - const commonArgs = [ - 'r2', - 'object', - 'put', - '--remote', - '--file', - file, - '--content-type', - 'application/wasm', - ] - const combinedStatus = runStatus('wrangler', [ - ...commonArgs, - `${bucketName}/${key}`, - ]) - if (combinedStatus !== 0) { - const splitStatus = runStatus('wrangler', [ - ...commonArgs, - bucketName, - key, - ]) - if (splitStatus !== 0) { - process.exit(splitStatus) - } - } - unlinkSync(file) - } -} else { - console.log('No WASM files found in dist/assets.') -} +const assetsDir = join(__dirname, '../../dist/assets') +const uploadWasmFuncs = readdirSync(assetsDir) + .filter((file) => file.endsWith('.wasm')) + .map( + (wasmFile) => + new Promise((resolve) => { + const key = join('assets', wasmFile) + const filePath = join(assetsDir, wasmFile) + console.info(`Putting '${wasmFile}' to r2 as '${key}'`) + execSync( + `wrangler r2 object put --remote --file "${filePath}" --content-type application/wasm ${bucketName}/${key}`, + ) + rmSync(filePath) + console.info(`Successfully put '${wasmFile}' to r2 as '${key}'\n`) + resolve() + }), + ) +await Promise.all(uploadWasmFuncs) -const pagesArgs = ['pages', 'deploy', distDir, ...process.argv.slice(2)] -run('wrangler', pagesArgs) +console.info(`Deploying static site to pages`) +execSync(`wrangler pages deploy`) From 572f7cf5cffda94fecf833c3899b28fec4101ed3 Mon Sep 17 00:00:00 2001 From: Meng Lin Date: Sun, 1 Mar 2026 22:17:31 +1100 Subject: [PATCH 3/6] simplify path function cloudflare worker --- .../benchmark/functions/assets/[[path]].ts | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/examples/benchmark/functions/assets/[[path]].ts b/examples/benchmark/functions/assets/[[path]].ts index 5a4a938..ec31694 100644 --- a/examples/benchmark/functions/assets/[[path]].ts +++ b/examples/benchmark/functions/assets/[[path]].ts @@ -5,28 +5,17 @@ export async function onRequest(context: { next: () => Promise }) { const { request, env, next } = context - - if (request.method !== 'GET' && request.method !== 'HEAD') { - return next() - } - const url = new URL(request.url) - if (!url.pathname.endsWith('.wasm')) { - return next() - } + if (request.method !== 'GET' && request.method !== 'HEAD') return next() + if (!url.pathname.endsWith('.wasm')) return next() - const key = url.pathname.replace(/^\/+/, '') + const key = url.pathname const object = await env.WASM_BUCKET.get(key) - if (!object) { - return next() - } + if (!object) return next() const headers = new Headers() object.writeHttpMetadata(headers) headers.set('etag', object.httpEtag) - if (key.endsWith('.wasm')) { - headers.set('Content-Type', 'application/wasm') - } headers.set('Cache-Control', 'public, max-age=31536000, immutable') return new Response(object.body, { headers }) From ae9983c862d7818bc0fefe341cecaac5d7c1af1d Mon Sep 17 00:00:00 2001 From: Meng Lin Date: Sun, 1 Mar 2026 22:17:59 +1100 Subject: [PATCH 4/6] cleanup --- examples/benchmark/src/scripts/deployPages.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmark/src/scripts/deployPages.ts b/examples/benchmark/src/scripts/deployPages.ts index 5a8d4a9..bc0a37d 100644 --- a/examples/benchmark/src/scripts/deployPages.ts +++ b/examples/benchmark/src/scripts/deployPages.ts @@ -1,5 +1,5 @@ import { execSync } from 'node:child_process' -import { readdirSync, readFileSync, rm, rmSync } from 'node:fs' +import { readdirSync, readFileSync, rmSync } from 'node:fs' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' From 6abdafccf1ba565be7acdf6358f76a58e69e60e9 Mon Sep 17 00:00:00 2001 From: Meng Lin Date: Sun, 1 Mar 2026 23:05:16 +1100 Subject: [PATCH 5/6] better logging deployment --- examples/benchmark/src/scripts/deployPages.ts | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/examples/benchmark/src/scripts/deployPages.ts b/examples/benchmark/src/scripts/deployPages.ts index bc0a37d..b7b18c5 100644 --- a/examples/benchmark/src/scripts/deployPages.ts +++ b/examples/benchmark/src/scripts/deployPages.ts @@ -16,23 +16,16 @@ const readBucketName = () => { const bucketName = readBucketName() const assetsDir = join(__dirname, '../../dist/assets') -const uploadWasmFuncs = readdirSync(assetsDir) +readdirSync(assetsDir) .filter((file) => file.endsWith('.wasm')) - .map( - (wasmFile) => - new Promise((resolve) => { - const key = join('assets', wasmFile) - const filePath = join(assetsDir, wasmFile) - console.info(`Putting '${wasmFile}' to r2 as '${key}'`) - execSync( - `wrangler r2 object put --remote --file "${filePath}" --content-type application/wasm ${bucketName}/${key}`, - ) - rmSync(filePath) - console.info(`Successfully put '${wasmFile}' to r2 as '${key}'\n`) - resolve() - }), - ) -await Promise.all(uploadWasmFuncs) + .forEach((wasmFile) => { + const key = join('assets', wasmFile) + const filePath = join(assetsDir, wasmFile) + execSync( + `wrangler r2 object put --remote --file "${filePath}" --content-type application/wasm ${bucketName}/${key}`, + { stdio: 'inherit' }, + ) + rmSync(filePath) + }) -console.info(`Deploying static site to pages`) -execSync(`wrangler pages deploy`) +execSync(`wrangler pages deploy`, { stdio: 'inherit' }) From 8211095173334991cb58a6e90e2a0ce4abea8408 Mon Sep 17 00:00:00 2001 From: Meng Lin Date: Sun, 1 Mar 2026 23:17:34 +1100 Subject: [PATCH 6/6] remove preview bucket --- examples/benchmark/wrangler.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/benchmark/wrangler.toml b/examples/benchmark/wrangler.toml index 27cabe2..2330990 100644 --- a/examples/benchmark/wrangler.toml +++ b/examples/benchmark/wrangler.toml @@ -7,7 +7,6 @@ compatibility_date = "2025-02-28" [[r2_buckets]] binding = "WASM_BUCKET" bucket_name = "tanstack-db-interpreter-wasm" -preview_bucket_name = "tanstack-db-interpreter-wasm-preview" [vars] WASM_PUBLIC_BASE = "https://"