From 40969cbc0e8d205ea3fd0747d4ed146788cd9cd2 Mon Sep 17 00:00:00 2001 From: Pete Bacon Darwin Date: Thu, 20 Jan 2022 11:40:05 +0000 Subject: [PATCH] fix: align publishing sites asset keys with Wrangler 1 - Use the same hashing strategy for asset keys (xxhash64) - Include the full path (from cwd) in the asset key - Match include and exclude patterns against full path (from cwd) - Validate that the asset key is not over 512 bytes long --- .changeset/quiet-steaks-smoke.md | 10 + .vscode/settings.json | 3 +- package-lock.json | 22 +- packages/wrangler/package.json | 3 +- packages/wrangler/scripts/bundle.ts | 8 +- .../wrangler/src/__tests__/publish.test.ts | 191 +++++++++++------- packages/wrangler/src/sites.tsx | 81 +++++--- 7 files changed, 211 insertions(+), 107 deletions(-) create mode 100644 .changeset/quiet-steaks-smoke.md diff --git a/.changeset/quiet-steaks-smoke.md b/.changeset/quiet-steaks-smoke.md new file mode 100644 index 0000000000..abc083a67e --- /dev/null +++ b/.changeset/quiet-steaks-smoke.md @@ -0,0 +1,10 @@ +--- +"wrangler": patch +--- + +fix: align publishing sites asset keys with Wrangler 1 + +- Use the same hashing strategy for asset keys (xxhash64) +- Include the full path (from cwd) in the asset key +- Match include and exclude patterns against full path (from cwd) +- Validate that the asset key is not over 512 bytes long diff --git a/.vscode/settings.json b/.vscode/settings.json index c3f1b285cf..9a22756538 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,7 +23,8 @@ "weakmap", "weakset", "webassemblymemory", - "websockets" + "websockets", + "xxhash" ], "cSpell.ignoreWords": ["yxxx"] } diff --git a/package-lock.json b/package-lock.json index c2ca21fe54..7d944227c8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,7 +30,7 @@ "typescript": "^4.5.2" }, "engines": { - "node": ">=16.0.0" + "node": ">=16.7.0" } }, "node_modules/@babel/code-frame": { @@ -15213,6 +15213,15 @@ "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" }, + "node_modules/xxhash-addon": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/xxhash-addon/-/xxhash-addon-1.4.0.tgz", + "integrity": "sha512-n3Ml0Vgvy7jMYJBlQIoFLjYxXNZQ5CbzW8E2Ynq2QCUpWMqCouooW7j02+7Oud5FijBuSrjQNuN/fCiz1SHN+w==", + "hasInstallScript": true, + "engines": { + "node": ">=8.6.0 <9.0.0 || >=10.0.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -15337,7 +15346,8 @@ "esbuild": "0.14.1", "miniflare": "2.2.0", "path-to-regexp": "^6.2.0", - "semiver": "^1.1.0" + "semiver": "^1.1.0", + "xxhash-addon": "^1.4.0" }, "bin": { "wrangler": "bin/wrangler.js", @@ -26863,7 +26873,7 @@ "find-up": "^6.2.0", "formdata-node": "^4.3.1", "fsevents": "~2.3.2", - "ignore": "*", + "ignore": "^5.2.0", "ink": "^3.2.0", "ink-select-input": "^4.2.1", "ink-table": "^3.0.0", @@ -26882,6 +26892,7 @@ "tmp-promise": "^3.0.3", "undici": "^4.11.1", "ws": "^8.3.0", + "xxhash-addon": "^1.4.0", "yargs": "^17.3.0" }, "dependencies": { @@ -27088,6 +27099,11 @@ "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" }, + "xxhash-addon": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/xxhash-addon/-/xxhash-addon-1.4.0.tgz", + "integrity": "sha512-n3Ml0Vgvy7jMYJBlQIoFLjYxXNZQ5CbzW8E2Ynq2QCUpWMqCouooW7j02+7Oud5FijBuSrjQNuN/fCiz1SHN+w==" + }, "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/packages/wrangler/package.json b/packages/wrangler/package.json index 95237cba2b..21d30cbd3a 100644 --- a/packages/wrangler/package.json +++ b/packages/wrangler/package.json @@ -39,7 +39,8 @@ "esbuild": "0.14.1", "miniflare": "2.2.0", "path-to-regexp": "^6.2.0", - "semiver": "^1.1.0" + "semiver": "^1.1.0", + "xxhash-addon": "^1.4.0" }, "optionalDependencies": { "fsevents": "~2.3.2" diff --git a/packages/wrangler/scripts/bundle.ts b/packages/wrangler/scripts/bundle.ts index 3e4a19f9eb..268ffdfcef 100644 --- a/packages/wrangler/scripts/bundle.ts +++ b/packages/wrangler/scripts/bundle.ts @@ -12,7 +12,13 @@ async function run() { platform: "node", format: "cjs", // minify: true, // TODO: enable this again - external: ["fsevents", "esbuild", "miniflare", "@miniflare/core"], // todo - bundle miniflare too + external: [ + "fsevents", + "esbuild", + "miniflare", + "@miniflare/core", + "xxhash-addon", + ], // todo - bundle miniflare too sourcemap: true, inject: [path.join(__dirname, "../import_meta_url.js")], define: { diff --git a/packages/wrangler/src/__tests__/publish.test.ts b/packages/wrangler/src/__tests__/publish.test.ts index 4a77503615..ec4ba6e4ed 100644 --- a/packages/wrangler/src/__tests__/publish.test.ts +++ b/packages/wrangler/src/__tests__/publish.test.ts @@ -62,16 +62,16 @@ describe("publish", () => { describe("asset upload", () => { it("should upload all the files in the directory specified by `config.site.bucket`", async () => { const assets = [ - { filePath: "file-1.txt", content: "Content of file-1" }, - { filePath: "file-2.txt", content: "Content of file-2" }, + { filePath: "assets/file-1.txt", content: "Content of file-1" }, + { filePath: "assets/file-2.txt", content: "Content of file-2" }, ]; const kvNamespace = { title: "__test-name_sites_assets", id: "__test-name_sites_assets-id", }; - writeWranglerToml("./index.js", "./assets"); + writeWranglerToml("./index.js", "assets"); writeEsmWorkerSource(); - writeAssets("./assets", assets); + writeAssets(assets); mockUploadWorkerRequest(); mockSubDomainRequest(); mockListKVNamespacesRequest(kvNamespace); @@ -80,8 +80,10 @@ describe("publish", () => { const { stdout, stderr, error } = await runWrangler("publish"); expect(stripTimings(stdout)).toMatchInlineSnapshot(` - "uploading assets/file-1.txt... - uploading assets/file-2.txt... + "reading assets/file-1.txt... + uploading as assets/file-1.2ca234f380.txt... + reading assets/file-2.txt... + uploading as assets/file-2.5938485188.txt... Uploaded test-name (TIMINGS) @@ -97,57 +99,58 @@ describe("publish", () => { it("should only upload files that are not already in the KV namespace", async () => { const assets = [ - { filePath: "file-1.txt", content: "Content of file-1" }, - { filePath: "file-2.txt", content: "Content of file-2" }, + { filePath: "assets/file-1.txt", content: "Content of file-1" }, + { filePath: "assets/file-2.txt", content: "Content of file-2" }, ]; const kvNamespace = { title: "__test-name_sites_assets", id: "__test-name_sites_assets-id", }; - writeWranglerToml("./index.js", "./assets"); + writeWranglerToml("./index.js", "assets"); writeEsmWorkerSource(); - writeAssets("./assets", assets); + writeAssets(assets); mockUploadWorkerRequest(); mockSubDomainRequest(); mockListKVNamespacesRequest(kvNamespace); // Put file-1 in the KV namespace - mockKeyListRequest(kvNamespace.id, [ - "file-1.c514defbb343fb04ad55183d8336ae0a5988616b.txt", - ]); + mockKeyListRequest(kvNamespace.id, ["assets/file-1.2ca234f380.txt"]); // Check we do not upload file-1 mockUploadAssetsToKVRequest( kvNamespace.id, - assets.filter((a) => a.filePath !== "file-1.txt") + assets.filter((a) => a.filePath !== "assets/file-1.txt") ); const { stdout, stderr, error } = await runWrangler("publish"); expect(stripTimings(stdout)).toMatchInlineSnapshot(` - "uploading assets/file-2.txt... - Uploaded - test-name - (TIMINGS) - Deployed - test-name - (TIMINGS) - - test-name.test-sub-domain.workers.dev" - `); + "reading assets/file-1.txt... + skipping - already uploaded + reading assets/file-2.txt... + uploading as assets/file-2.5938485188.txt... + Uploaded + test-name + (TIMINGS) + Deployed + test-name + (TIMINGS) + + test-name.test-sub-domain.workers.dev" + `); expect(stderr).toMatchInlineSnapshot(`""`); expect(error).toMatchInlineSnapshot(`undefined`); }); it("should only upload files that match the `site-include` arg", async () => { const assets = [ - { filePath: "file-1.txt", content: "Content of file-1" }, - { filePath: "file-2.txt", content: "Content of file-2" }, + { filePath: "assets/file-1.txt", content: "Content of file-1" }, + { filePath: "assets/file-2.txt", content: "Content of file-2" }, ]; const kvNamespace = { title: "__test-name_sites_assets", id: "__test-name_sites_assets-id", }; - writeWranglerToml("./index.js", "./assets"); + writeWranglerToml("./index.js", "assets"); writeEsmWorkerSource(); - writeAssets("./assets", assets); + writeAssets(assets); mockUploadWorkerRequest(); mockSubDomainRequest(); mockListKVNamespacesRequest(kvNamespace); @@ -155,14 +158,15 @@ describe("publish", () => { // Check we only upload file-1 mockUploadAssetsToKVRequest( kvNamespace.id, - assets.filter((a) => a.filePath === "file-1.txt") + assets.filter((a) => a.filePath === "assets/file-1.txt") ); const { stdout, stderr, error } = await runWrangler( "publish --site-include file-1.txt" ); expect(stripTimings(stdout)).toMatchInlineSnapshot(` - "uploading assets/file-1.txt... + "reading assets/file-1.txt... + uploading as assets/file-1.2ca234f380.txt... Uploaded test-name (TIMINGS) @@ -178,16 +182,16 @@ describe("publish", () => { it("should not upload files that match the `site-exclude` arg", async () => { const assets = [ - { filePath: "file-1.txt", content: "Content of file-1" }, - { filePath: "file-2.txt", content: "Content of file-2" }, + { filePath: "assets/file-1.txt", content: "Content of file-1" }, + { filePath: "assets/file-2.txt", content: "Content of file-2" }, ]; const kvNamespace = { title: "__test-name_sites_assets", id: "__test-name_sites_assets-id", }; - writeWranglerToml("./index.js", "./assets"); + writeWranglerToml("./index.js", "assets"); writeEsmWorkerSource(); - writeAssets("./assets", assets); + writeAssets(assets); mockUploadWorkerRequest(); mockSubDomainRequest(); mockListKVNamespacesRequest(kvNamespace); @@ -195,14 +199,15 @@ describe("publish", () => { // Check we only upload file-1 mockUploadAssetsToKVRequest( kvNamespace.id, - assets.filter((a) => a.filePath === "file-1.txt") + assets.filter((a) => a.filePath === "assets/file-1.txt") ); const { stdout, stderr, error } = await runWrangler( "publish --site-exclude file-2.txt" ); expect(stripTimings(stdout)).toMatchInlineSnapshot(` - "uploading assets/file-1.txt... + "reading assets/file-1.txt... + uploading as assets/file-1.2ca234f380.txt... Uploaded test-name (TIMINGS) @@ -218,16 +223,16 @@ describe("publish", () => { it("should only upload files that match the `site.include` config", async () => { const assets = [ - { filePath: "file-1.txt", content: "Content of file-1" }, - { filePath: "file-2.txt", content: "Content of file-2" }, + { filePath: "assets/file-1.txt", content: "Content of file-1" }, + { filePath: "assets/file-2.txt", content: "Content of file-2" }, ]; const kvNamespace = { title: "__test-name_sites_assets", id: "__test-name_sites_assets-id", }; - writeWranglerToml("./index.js", "./assets", ["file-1.txt"]); + writeWranglerToml("./index.js", "assets", ["file-1.txt"]); writeEsmWorkerSource(); - writeAssets("./assets", assets); + writeAssets(assets); mockUploadWorkerRequest(); mockSubDomainRequest(); mockListKVNamespacesRequest(kvNamespace); @@ -235,12 +240,13 @@ describe("publish", () => { // Check we only upload file-1 mockUploadAssetsToKVRequest( kvNamespace.id, - assets.filter((a) => a.filePath === "file-1.txt") + assets.filter((a) => a.filePath === "assets/file-1.txt") ); const { stdout, stderr, error } = await runWrangler("publish"); expect(stripTimings(stdout)).toMatchInlineSnapshot(` - "uploading assets/file-1.txt... + "reading assets/file-1.txt... + uploading as assets/file-1.2ca234f380.txt... Uploaded test-name (TIMINGS) @@ -256,16 +262,16 @@ describe("publish", () => { it("should not upload files that match the `site.exclude` config", async () => { const assets = [ - { filePath: "file-1.txt", content: "Content of file-1" }, - { filePath: "file-2.txt", content: "Content of file-2" }, + { filePath: "assets/file-1.txt", content: "Content of file-1" }, + { filePath: "assets/file-2.txt", content: "Content of file-2" }, ]; const kvNamespace = { title: "__test-name_sites_assets", id: "__test-name_sites_assets-id", }; - writeWranglerToml("./index.js", "./assets", undefined, ["file-2.txt"]); + writeWranglerToml("./index.js", "assets", undefined, ["file-2.txt"]); writeEsmWorkerSource(); - writeAssets("./assets", assets); + writeAssets(assets); mockUploadWorkerRequest(); mockSubDomainRequest(); mockListKVNamespacesRequest(kvNamespace); @@ -273,12 +279,13 @@ describe("publish", () => { // Check we only upload file-1 mockUploadAssetsToKVRequest( kvNamespace.id, - assets.filter((a) => a.filePath === "file-1.txt") + assets.filter((a) => a.filePath === "assets/file-1.txt") ); const { stdout, stderr, error } = await runWrangler("publish"); expect(stripTimings(stdout)).toMatchInlineSnapshot(` - "uploading assets/file-1.txt... + "reading assets/file-1.txt... + uploading as assets/file-1.2ca234f380.txt... Uploaded test-name (TIMINGS) @@ -294,16 +301,16 @@ describe("publish", () => { it("should use `site-include` arg over `site.include` config", async () => { const assets = [ - { filePath: "file-1.txt", content: "Content of file-1" }, - { filePath: "file-2.txt", content: "Content of file-2" }, + { filePath: "assets/file-1.txt", content: "Content of file-1" }, + { filePath: "assets/file-2.txt", content: "Content of file-2" }, ]; const kvNamespace = { title: "__test-name_sites_assets", id: "__test-name_sites_assets-id", }; - writeWranglerToml("./index.js", "./assets", ["file-2.txt"]); + writeWranglerToml("./index.js", "assets", ["file-2.txt"]); writeEsmWorkerSource(); - writeAssets("./assets", assets); + writeAssets(assets); mockUploadWorkerRequest(); mockSubDomainRequest(); mockListKVNamespacesRequest(kvNamespace); @@ -311,14 +318,15 @@ describe("publish", () => { // Check we only upload file-1 mockUploadAssetsToKVRequest( kvNamespace.id, - assets.filter((a) => a.filePath === "file-1.txt") + assets.filter((a) => a.filePath === "assets/file-1.txt") ); const { stdout, stderr, error } = await runWrangler( "publish --site-include file-1.txt" ); expect(stripTimings(stdout)).toMatchInlineSnapshot(` - "uploading assets/file-1.txt... + "reading assets/file-1.txt... + uploading as assets/file-1.2ca234f380.txt... Uploaded test-name (TIMINGS) @@ -334,16 +342,18 @@ describe("publish", () => { it("should use `site-exclude` arg over `site.exclude` config", async () => { const assets = [ - { filePath: "file-1.txt", content: "Content of file-1" }, - { filePath: "file-2.txt", content: "Content of file-2" }, + { filePath: "assets/file-1.txt", content: "Content of file-1" }, + { filePath: "assets/file-2.txt", content: "Content of file-2" }, ]; const kvNamespace = { title: "__test-name_sites_assets", id: "__test-name_sites_assets-id", }; - writeWranglerToml("./index.js", "./assets", undefined, ["file-1.txt"]); + writeWranglerToml("./index.js", "assets", undefined, [ + "assets/file-1.txt", + ]); writeEsmWorkerSource(); - writeAssets("./assets", assets); + writeAssets(assets); mockUploadWorkerRequest(); mockSubDomainRequest(); mockListKVNamespacesRequest(kvNamespace); @@ -351,14 +361,15 @@ describe("publish", () => { // Check we only upload file-1 mockUploadAssetsToKVRequest( kvNamespace.id, - assets.filter((a) => a.filePath === "file-1.txt") + assets.filter((a) => a.filePath.endsWith("assets/file-1.txt")) ); const { stdout, stderr, error } = await runWrangler( "publish --site-exclude file-2.txt" ); expect(stripTimings(stdout)).toMatchInlineSnapshot(` - "uploading assets/file-1.txt... + "reading assets/file-1.txt... + uploading as assets/file-1.2ca234f380.txt... Uploaded test-name (TIMINGS) @@ -375,12 +386,12 @@ describe("publish", () => { it("should error if the asset is over 25Mb", async () => { const assets = [ { - filePath: "large-file.txt", + filePath: "assets/large-file.txt", // This file is greater than 25MiB when base64 encoded but small enough to be uploaded. content: "X".repeat(25 * 1024 * 1024 * 0.8 + 1), }, { - filePath: "too-large-file.txt", + filePath: "assets/too-large-file.txt", content: "X".repeat(25 * 1024 * 1024 + 1), }, ]; @@ -388,9 +399,11 @@ describe("publish", () => { title: "__test-name_sites_assets", id: "__test-name_sites_assets-id", }; - writeWranglerToml("./index.js", "./assets", undefined, ["file-1.txt"]); + writeWranglerToml("./index.js", "assets", undefined, [ + "assets/file-1.txt", + ]); writeEsmWorkerSource(); - writeAssets("./assets", assets); + writeAssets(assets); mockUploadWorkerRequest(); mockSubDomainRequest(); mockListKVNamespacesRequest(kvNamespace); @@ -398,9 +411,10 @@ describe("publish", () => { const { stdout, stderr, error } = await runWrangler("publish"); - expect(stdout).toMatchInlineSnapshot( - `"uploading assets/large-file.txt..."` - ); + expect(stdout).toMatchInlineSnapshot(` + "reading assets/large-file.txt... + uploading as assets/large-file.0ea0637a45.txt..." + `); expect(stderr).toMatchInlineSnapshot(` "File assets/too-large-file.txt is too big, it should be under 25 MiB. See https://developers.cloudflare.com/workers/platform/limits#kv-limits @@ -411,6 +425,39 @@ describe("publish", () => { `[Error: File assets/too-large-file.txt is too big, it should be under 25 MiB. See https://developers.cloudflare.com/workers/platform/limits#kv-limits]` ); }); + + it("should error if the asset key is over 512 characters", async () => { + const longFilePathAsset = { + filePath: "assets/" + "folder/".repeat(100) + "file.txt", + content: "content of file", + }; + const kvNamespace = { + title: "__test-name_sites_assets", + id: "__test-name_sites_assets-id", + }; + writeWranglerToml("./index.js", "assets"); + writeEsmWorkerSource(); + writeAssets([longFilePathAsset]); + mockUploadWorkerRequest(); + mockSubDomainRequest(); + mockListKVNamespacesRequest(kvNamespace); + mockKeyListRequest(kvNamespace.id, []); + + const { stdout, stderr, error } = await runWrangler("publish"); + + expect(stdout).toMatchInlineSnapshot( + `"reading assets/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/file.txt..."` + ); + expect(stderr).toMatchInlineSnapshot(` + "The asset path key \\"assets/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/file.3da0d0cd12.txt\\" exceeds the maximum key size limit of 512. See https://developers.cloudflare.com/workers/platform/limits#kv-limits\\", + + %s + If you think this is a bug then please create an issue at https://github.com/cloudflare/wrangler2/issues/new." + `); + expect(error).toMatchInlineSnapshot( + `[Error: The asset path key "assets/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/folder/file.3da0d0cd12.txt" exceeds the maximum key size limit of 512. See https://developers.cloudflare.com/workers/platform/limits#kv-limits",]` + ); + }); }); }); @@ -453,14 +500,10 @@ function writeEsmWorkerSource() { } /** Write mock assets to the file system so they can be uploaded. */ -function writeAssets( - assetDir: string, - assets: { filePath: string; content: string }[] -) { +function writeAssets(assets: { filePath: string; content: string }[]) { for (const asset of assets) { - const filePath = path.join(assetDir, asset.filePath); - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - fs.writeFileSync(filePath, asset.content); + fs.mkdirSync(path.dirname(asset.filePath), { recursive: true }); + fs.writeFileSync(asset.filePath, asset.content); } } diff --git a/packages/wrangler/src/sites.tsx b/packages/wrangler/src/sites.tsx index fe50d41bf9..e11d5a4ab7 100644 --- a/packages/wrangler/src/sites.tsx +++ b/packages/wrangler/src/sites.tsx @@ -1,8 +1,7 @@ -import crypto from "node:crypto"; -import { createReadStream } from "node:fs"; import * as path from "node:path"; import { readdir, readFile, stat } from "node:fs/promises"; import ignore from "ignore"; +import { XXHash64 } from "xxhash-addon"; import { fetchResult } from "./cfetch"; import type { Config } from "./config"; import { listNamespaceKeys, listNamespaces, putBulkKeyValue } from "./kv"; @@ -10,6 +9,9 @@ import { listNamespaceKeys, listNamespaces, putBulkKeyValue } from "./kv"; async function* getFilesInFolder(dirPath: string): AsyncIterable { const files = await readdir(dirPath, { withFileTypes: true }); for (const file of files) { + // TODO: always ignore `node_modules` + // TODO: ignore hidden files (starting with .) but not .well-known?? + // TODO: follow symlinks?? if (file.isDirectory()) { yield* await getFilesInFolder(path.join(dirPath, file.name)); } else { @@ -18,27 +20,32 @@ async function* getFilesInFolder(dirPath: string): AsyncIterable { } } -async function hashFileContent(filePath: string): Promise { - return new Promise((resolve, reject) => { - const hash = crypto.createHash("sha1"); - const rs = createReadStream(filePath); - rs.on("error", reject); - rs.on("data", (chunk) => hash.update(chunk)); - rs.on("end", () => resolve(hash.digest("hex"))); - }); +/** + * Create a hash key for the given content using the xxhash algorithm. + * + * Note we only return the first 10 characters, since we will also include the file name in the asset manifest key + * the most important thing here is to detect changes of a single file to invalidate the cache and + * it's impossible to serve two different files with the same name + */ +function hashFileContent(content: string): string { + const hasher = new XXHash64(); + hasher.update(Buffer.from(content)); + const hash = hasher.digest(); + return hash.toString("hex").substring(0, 10); } -async function hashAsset(filePath: string): Promise<{ - assetKey: string; - hash: string; -}> { - const extName = path.extname(filePath); +/** + * Create a hashed asset key for the given asset. + * + * The key will change if the file path or content of the asset changes. + * The algorithm used here matches that of Wrangler 1. + */ +function hashAsset(filePath: string, content: string): string { + const extName = path.extname(filePath) || ""; const baseName = path.basename(filePath, extName); - const hash = await hashFileContent(filePath); - return { - assetKey: `${baseName}.${hash}${extName || ""}`, - hash, - }; + const directory = path.dirname(filePath); + const hash = hashFileContent(content); + return urlSafe(path.join(directory, `${baseName}.${hash}${extName}`)); } async function createKVNamespaceIfNotAlreadyExisting( @@ -117,29 +124,32 @@ export async function syncAssets( const include = createPatternMatcher(siteAssets.includePatterns, false); const exclude = createPatternMatcher(siteAssets.excludePatterns, true); - // TODO: this can be more efficient by parallelising for await (const file of getFilesInFolder(siteAssets.baseDirectory)) { - const relativePath = path.relative(siteAssets.baseDirectory, file); - if (!include(relativePath)) { + if (!include(file)) { continue; } - if (exclude(relativePath)) { + if (exclude(file)) { continue; } await validateAssetSize(file); + console.log(`reading ${file}...`); + const content = await readFile(file, "base64"); + + const assetKey = hashAsset(file, content); + validateAssetKey(assetKey); - const { assetKey } = await hashAsset(file); // now put each of the files into kv if (!keys.has(assetKey)) { - console.log(`uploading ${file}...`); - const content = await readFile(file, "base64"); + console.log(`uploading as ${assetKey}...`); upload.push({ key: assetKey, value: content, base64: true, }); + } else { + console.log(`skipping - already uploaded`); } manifest[path.relative(siteAssets.baseDirectory, file)] = assetKey; } @@ -168,6 +178,23 @@ async function validateAssetSize(filePath: string) { } } +function validateAssetKey(assetKey: string) { + if (assetKey.length > 512) { + throw new Error( + `The asset path key "${assetKey}" exceeds the maximum key size limit of 512. See https://developers.cloudflare.com/workers/platform/limits#kv-limits",` + ); + } +} + +/** + * Convert a filePath to be safe to use as a relative URL. + * + * Primarily this involves converting Windows backslashes to forward slashes. + */ +function urlSafe(filePath: string): string { + return filePath.replace(/\\/g, "/"); +} + /** * Information about the assets that should be uploaded */