From 9a692b2d61a7495f352a51fe596170093fd7cfc3 Mon Sep 17 00:00:00 2001 From: Saleh Abdel Motaal Date: Wed, 28 Aug 2024 13:47:22 -0400 Subject: [PATCH 1/8] chore(shims): Remove extraneous files and exports --- packages/shims/package.json | 2 -- packages/shims/src/apply-lockdown.mjs | 11 ----------- 2 files changed, 13 deletions(-) delete mode 100644 packages/shims/src/apply-lockdown.mjs diff --git a/packages/shims/package.json b/packages/shims/package.json index 120ae5f53..14689d028 100644 --- a/packages/shims/package.json +++ b/packages/shims/package.json @@ -10,9 +10,7 @@ "sideEffects": false, "type": "module", "exports": { - "./apply-lockdown": "./dist/apply-lockdown.mjs", "./endoify": "./dist/endoify.mjs", - "./eventual-send": "./dist/eventual-send.mjs", "./package.json": "./package.json" }, "main": "./dist/index.cjs", diff --git a/packages/shims/src/apply-lockdown.mjs b/packages/shims/src/apply-lockdown.mjs deleted file mode 100644 index e01804d3a..000000000 --- a/packages/shims/src/apply-lockdown.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import './ses.mjs'; - -lockdown({ - consoleTaming: 'unsafe', - dateTaming: 'unsafe', - domainTaming: 'unsafe', - errorTaming: 'unsafe', - mathTaming: 'unsafe', - overrideTaming: 'severe', - stackFiltering: 'verbose', -}); From 0ef130e5259011d80026b931af15ba9b28fe4a8b Mon Sep 17 00:00:00 2001 From: Saleh Abdel Motaal Date: Wed, 28 Aug 2024 13:48:12 -0400 Subject: [PATCH 2/8] chore(shims): Add `typescript` in `devDependencies` --- packages/shims/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/shims/package.json b/packages/shims/package.json index 14689d028..6b67d88a0 100644 --- a/packages/shims/package.json +++ b/packages/shims/package.json @@ -40,6 +40,7 @@ "@metamask/auto-changelog": "^3.4.4", "deepmerge": "^4.3.1", "rimraf": "^6.0.1", + "typescript": "~5.5.4", "vite": "^5.3.5", "vitest": "^2.0.5" }, From 78338f1ab99358fa8d4cdf75e3ac075950703d8e Mon Sep 17 00:00:00 2001 From: Saleh Abdel Motaal Date: Wed, 28 Aug 2024 14:00:09 -0400 Subject: [PATCH 3/8] chore: Update `yarn.lock` --- yarn.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn.lock b/yarn.lock index c60081c49..3f8df0378 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1324,6 +1324,7 @@ __metadata: deepmerge: "npm:^4.3.1" rimraf: "npm:^6.0.1" ses: "npm:^1.7.0" + typescript: "npm:~5.5.4" vite: "npm:^5.3.5" vitest: "npm:^2.0.5" languageName: unknown From e41b3d7217f07755b55b277fa0d945fe0571387b Mon Sep 17 00:00:00 2001 From: Saleh Abdel Motaal Date: Wed, 28 Aug 2024 14:27:41 -0400 Subject: [PATCH 4/8] refactor(shims): Optimize building and testing --- constraints.pro | 1 - packages/extension/src/background.ts | 2 +- packages/extension/vite.config.ts | 14 +- packages/shims/.eslintrc.cjs | 2 +- packages/shims/package.json | 4 +- packages/shims/scripts/bundle.js | 68 +++++----- ...plugin-endo-script-identifier-transform.js | 123 ++++++++++++++++++ .../shims/src/{endoify.mjs => endoify.js} | 0 packages/shims/src/endoify.test.ts | 8 +- .../shims/src/vitest-environment-endoified.ts | 26 ---- packages/shims/tsconfig.json | 4 +- packages/shims/vitest.config.ts | 3 - 12 files changed, 170 insertions(+), 85 deletions(-) create mode 100644 packages/shims/scripts/helpers/rollup-plugin-endo-script-identifier-transform.js rename packages/shims/src/{endoify.mjs => endoify.js} (100%) delete mode 100644 packages/shims/src/vitest-environment-endoified.ts diff --git a/constraints.pro b/constraints.pro index bfe1280c9..48aa60a24 100644 --- a/constraints.pro +++ b/constraints.pro @@ -309,7 +309,6 @@ gen_enforced_field(WorkspaceCwd, 'scripts.changelog:update', CorrectChangelogUpd % All non-root packages must have the same "test" script. gen_enforced_field(WorkspaceCwd, 'scripts.test', 'vitest run --config vitest.config.ts') :- - WorkspaceCwd \= 'packages/shims', WorkspaceCwd \= 'packages/test-utils', WorkspaceCwd \= '.'. diff --git a/packages/extension/src/background.ts b/packages/extension/src/background.ts index b899e06a7..eff0c6b8e 100644 --- a/packages/extension/src/background.ts +++ b/packages/extension/src/background.ts @@ -1,6 +1,6 @@ /* eslint-disable import-x/no-unassigned-import */ import './dev-console.js'; -import './endoify.mjs'; +import './endoify.js'; /* eslint-enable import-x/no-unassigned-import */ import type { ExtensionMessage } from './shared.js'; diff --git a/packages/extension/vite.config.ts b/packages/extension/vite.config.ts index 7d5e5d364..73c446442 100644 --- a/packages/extension/vite.config.ts +++ b/packages/extension/vite.config.ts @@ -14,10 +14,7 @@ const projectRoot = './src'; * Module specifiers that will be ignored by Rollup if imported, and therefore * not transformed. **Only applies to JavaScript and TypeScript files.** */ -const externalModules: readonly string[] = [ - './dev-console.js', - './endoify.mjs', -]; +const externalModules: readonly string[] = ['./dev-console.js', './endoify.js']; /** * Files that need to be statically copied to the destination directory. @@ -28,10 +25,7 @@ const staticCopyTargets: readonly string[] = [ 'manifest.json', // External modules 'dev-console.js', - '../../shims/dist/endoify.mjs', - // Dependencies of external modules - '../../shims/dist/eventual-send.mjs', - '../../../node_modules/ses/dist/ses.mjs', + '../../shims/dist/endoify.js', ]; // https://vitejs.dev/config/ @@ -73,12 +67,12 @@ export default defineConfig({ * @returns The Vite plugin. */ function endoifyHtmlFilesPlugin(): Plugin { - const endoifyElement = ''; + const endoifyElement = ''; return { name: 'externalize-plugin', async transformIndexHtml(htmlString): Promise { - if (htmlString.includes('endoify.mjs')) { + if (htmlString.includes('endoify.js')) { throw new Error( `HTML document already references endoify script:\n${htmlString}`, ); diff --git a/packages/shims/.eslintrc.cjs b/packages/shims/.eslintrc.cjs index 9c6c9b4df..6f33f2ef4 100644 --- a/packages/shims/.eslintrc.cjs +++ b/packages/shims/.eslintrc.cjs @@ -5,7 +5,7 @@ module.exports = { overrides: [ { - files: ['src/**/*.mjs'], + files: ['src/**/*.js'], globals: { lockdown: 'readonly' }, rules: { 'import-x/extensions': 'off', diff --git a/packages/shims/package.json b/packages/shims/package.json index 6b67d88a0..8ea2ac779 100644 --- a/packages/shims/package.json +++ b/packages/shims/package.json @@ -10,7 +10,7 @@ "sideEffects": false, "type": "module", "exports": { - "./endoify": "./dist/endoify.mjs", + "./endoify": "./dist/endoify.js", "./package.json": "./package.json" }, "main": "./dist/index.cjs", @@ -24,7 +24,7 @@ "changelog:validate": "../../scripts/validate-changelog.sh @ocap/shims", "clean": "rimraf --glob ./dist './*.tsbuildinfo'", "publish:preview": "yarn npm publish --tag preview", - "test": "yarn build && vitest run --config vitest.config.ts --passWithNoTests", + "test": "vitest run --config vitest.config.ts", "test:clean": "yarn test --no-cache --coverage.clean", "test:dev": "yarn test --coverage false", "test:verbose": "yarn test --reporter verbose", diff --git a/packages/shims/scripts/bundle.js b/packages/shims/scripts/bundle.js index b27a9f6d7..900f18661 100644 --- a/packages/shims/scripts/bundle.js +++ b/packages/shims/scripts/bundle.js @@ -2,47 +2,37 @@ import 'ses'; import '@endo/lockdown/commit.js'; import bundleSource from '@endo/bundle-source'; -import { createReadStream, createWriteStream } from 'fs'; -import { mkdir } from 'fs/promises'; -import path from 'path'; +import { mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { rimraf } from 'rimraf'; -import { Readable } from 'stream'; -import { fileURLToPath } from 'url'; + +import endoScriptIdentifierTransformPlugin from './helpers/rollup-plugin-endo-script-identifier-transform.js'; console.log('Bundling shims...'); const rootDir = fileURLToPath(new URL('..', import.meta.url)); -const src = path.resolve(rootDir, 'src'); -const dist = path.resolve(rootDir, 'dist'); - -await mkdir(dist, { recursive: true }); -await rimraf(`${dist}/*`, { glob: true }); - -/** - * Bundles the target file as endoScript and returns the content as a readable stream. - * - * @param {string} specifier - Import path to the file to bundle, e.g. `'@endo/eventual-send/shim.js'`. - * @returns {Promise} The bundled file contents as a Readable stream. - */ -const createEndoBundleReadStream = async (specifier) => { - const filePath = fileURLToPath(import.meta.resolve(specifier)); - const { source: bundle } = await bundleSource(filePath, { - format: 'endoScript', - }); - return Readable.from(bundle); -}; - -const sources = { - ses: createReadStream( - path.resolve(rootDir, '../../node_modules/ses/dist/ses.mjs'), - ), - eventualSend: await createEndoBundleReadStream('@endo/eventual-send/shim.js'), - shim: createReadStream(path.resolve(src, 'endoify.mjs')), -}; - -const target = createWriteStream(path.resolve(dist, 'endoify.mjs')); - -sources.ses.pipe(target, { end: false }); -sources.ses.on('end', () => sources.eventualSend.pipe(target, { end: false })); -sources.eventualSend.on('end', () => sources.shim.pipe(target, { end: true })); -sources.shim.on('end', () => console.log('Success!')); +const srcDir = path.resolve(rootDir, 'src'); +const distDir = path.resolve(rootDir, 'dist'); + +await mkdir(distDir, { recursive: true }); +await rimraf(`${distDir}/*`, { glob: true }); + +for (const [name, specifier] of Object.entries({ + endoify: path.resolve(srcDir, 'endoify.js'), +})) { + const outputPath = path.resolve(distDir, `${name}.js`); + const sourcePath = fileURLToPath(import.meta.resolve(specifier)); + + let { source } = await bundleSource(sourcePath, { format: 'endoScript' }); + + if (process.argv.includes('--with-zwj-rewrite')) { + source = endoScriptIdentifierTransformPlugin({ + scopedRoot: path.resolve(rootDir, '../..'), + }).transform(source, specifier).code; + } + + await writeFile(outputPath, source); +} + +console.log('Success!'); diff --git a/packages/shims/scripts/helpers/rollup-plugin-endo-script-identifier-transform.js b/packages/shims/scripts/helpers/rollup-plugin-endo-script-identifier-transform.js new file mode 100644 index 000000000..68056e122 --- /dev/null +++ b/packages/shims/scripts/helpers/rollup-plugin-endo-script-identifier-transform.js @@ -0,0 +1,123 @@ +/* eslint-disable jsdoc/require-returns-type */ +/* eslint-disable spaced-comment */ +/* eslint-disable jsdoc/valid-types */ +/* eslint-disable no-plusplus */ + +/** + * Quickly removes the normalized scope prefix from a normalized pathname. + * + * @param {string} normalizedPathname - The normalized pathname. + * @param {string} [normalizedScope] - The normalized scope. + * @returns {string} The scoped path. + */ +const scopedPath = (normalizedPathname, normalizedScope = '') => + normalizedPathname.slice( + normalizedPathname.indexOf(normalizedScope) + normalizedScope.length + 1 || + 0, + ); + +/** + * Rollup plugin to transform endoScript identifiers. + * + * @param {object} [options] - The plugin options. + * @param {(id: string) => boolean} [options.maybeEndoScript] - A function to determine if the script should be transformed. + * @param {string} [options.scopedRoot] - The root directory to scope the transformed script. + * @param {boolean} [options.timing] - Whether to log the transform time. + * @param {boolean} [options.debugging] - Whether to log the transform details. + * @param {boolean} [options.validation] - Whether to validate the transform. + * @returns The Rollup plugin. + */ +export default function endoScriptIdentifierTransformPlugin({ + maybeEndoScript, + scopedRoot, + timing = false, + debugging = false, + validation = false, +}) { + const zwjIdentifierMatcher = + /(? { + const replacement = `$${'\u034f'.repeat( + underscores.length, + )}${identifier}\u034f$`; + + if (validation) { + records[index] = { match, replacement, identifier }; + } + + replacements++; + return replacement; + }, + ); + + if (validation) { + for (const match of replacedCode.matchAll(cgjIdentifierMatcher)) { + if (match[0] !== records[match.index ?? -1]?.replacement) { + throw new Error( + `Mismatched replacement: ${match[0]} !== ${ + records[match.index ?? -1]?.replacement + }`, + ); + } + if (match[1] !== records[match.index ?? -1]?.identifier) { + throw new Error( + `Mismatched replacement: ${match[1]} !== ${ + records[match.index ?? -1]?.identifier + }`, + ); + } + } + } + + if (timing) { + console.timeEnd(`transform ${scopedId}`); + } + + if (debugging) { + console.dir( + { + transform: { + id: scopedId, + replacements, + delta: replacedCode.length - code.length, + }, + }, + { depth: 1, maxStringLength: 100, compact: true }, + ); + } + + return { + code: replacedCode, + moduleSideEffects: 'no-treeshake', + }; + }, + }; +} diff --git a/packages/shims/src/endoify.mjs b/packages/shims/src/endoify.js similarity index 100% rename from packages/shims/src/endoify.mjs rename to packages/shims/src/endoify.js diff --git a/packages/shims/src/endoify.test.ts b/packages/shims/src/endoify.test.ts index de8618c6c..e2069c343 100644 --- a/packages/shims/src/endoify.test.ts +++ b/packages/shims/src/endoify.test.ts @@ -1,4 +1,5 @@ -import { HandledPromise } from '@endo/eventual-send'; +import './endoify.js'; +import type { HandledPromiseConstructor } from '@endo/eventual-send'; import { describe, expect, it } from 'vitest'; describe('endoified', () => { @@ -10,3 +11,8 @@ describe('endoified', () => { expect(typeof HandledPromise).not.toBe('undefined'); // Due to eventual send }); }); + +declare global { + // eslint-disable-next-line no-var + var HandledPromise: HandledPromiseConstructor; +} diff --git a/packages/shims/src/vitest-environment-endoified.ts b/packages/shims/src/vitest-environment-endoified.ts deleted file mode 100644 index 1f0131bc6..000000000 --- a/packages/shims/src/vitest-environment-endoified.ts +++ /dev/null @@ -1,26 +0,0 @@ -import type { Context } from 'node:vm'; -import type { Environment } from 'vitest/environments'; - -export default { - name: 'endoified', - transformMode: 'ssr', - async setupVM() { - const vm = await import('node:vm'); - return { - getVmContext(): Context { - return vm.createContext({ - setTimeout, - clearTimeout, - }); - }, - teardown(): void { - return undefined; - }, - }; - }, - async setup() { - throw new Error( - 'endoified environment requires vitest option --pool=vmThreads or --pool=vmForks', - ); - }, -} as Environment; diff --git a/packages/shims/tsconfig.json b/packages/shims/tsconfig.json index 6f1d89de4..43b2f489c 100644 --- a/packages/shims/tsconfig.json +++ b/packages/shims/tsconfig.json @@ -1,7 +1,9 @@ { "extends": "../../tsconfig.packages.json", "compilerOptions": { - "baseUrl": "./" + "baseUrl": "./", + "allowJs": true, + "checkJs": true }, "references": [], "include": ["./src"] diff --git a/packages/shims/vitest.config.ts b/packages/shims/vitest.config.ts index 4d7480d27..6b2328fbd 100644 --- a/packages/shims/vitest.config.ts +++ b/packages/shims/vitest.config.ts @@ -12,12 +12,9 @@ const config = mergeConfig( defineConfig({ test: { pool: 'vmThreads', - environment: './vitest-environment-endoified.ts', - setupFiles: '../dist/endoify.mjs', }, }), ); -// @ts-expect-error We can and will delete this. delete config.test.coverage.thresholds; export default config; From caae2eb5d5466edc6c9eaae34daab7b3d1be2287 Mon Sep 17 00:00:00 2001 From: Saleh Abdel Motaal Date: Wed, 28 Aug 2024 14:31:05 -0400 Subject: [PATCH 5/8] chore: Update `packages/shims/src/endoify.js` --- packages/shims/src/endoify.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/shims/src/endoify.js b/packages/shims/src/endoify.js index 40124f6f0..2208eeac8 100644 --- a/packages/shims/src/endoify.js +++ b/packages/shims/src/endoify.js @@ -1,6 +1,5 @@ -/* eslint-disable import-x/unambiguous */ -// @inline './ses.mjs'; -// @inline './eventual-send.mjs'; +import 'ses'; +import '@endo/eventual-send/shim.js'; lockdown({ consoleTaming: 'unsafe', @@ -10,5 +9,3 @@ lockdown({ domainTaming: 'unsafe', overrideTaming: 'severe', }); - -/* eslint-enable import-x/unambiguous */ From 038c8a0313bfbd23a180ef0c0588d09d6de7fa20 Mon Sep 17 00:00:00 2001 From: Saleh Abdel Motaal Date: Wed, 28 Aug 2024 14:38:34 -0400 Subject: [PATCH 6/8] refactor(extension): Optimize building and testing --- packages/extension/src/endoify.test.ts | 47 +++++++++++++++++++ packages/extension/src/endoify.ts | 3 ++ packages/extension/src/iframe-manager.test.ts | 1 + packages/extension/src/shared.test.ts | 1 + packages/extension/vite.config.ts | 23 +++++++-- packages/extension/vitest.config.ts | 20 ++++++-- 6 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 packages/extension/src/endoify.test.ts create mode 100644 packages/extension/src/endoify.ts diff --git a/packages/extension/src/endoify.test.ts b/packages/extension/src/endoify.test.ts new file mode 100644 index 000000000..645882923 --- /dev/null +++ b/packages/extension/src/endoify.test.ts @@ -0,0 +1,47 @@ +import './endoify.js'; +// eslint-disable-next-line n/no-extraneous-import +import type { HandledPromiseConstructor } from '@endo/eventual-send'; +import { describe, expect, it } from 'vitest'; + +describe('endoified', () => { + it('calls lockdown', () => { + expect(Object.isFrozen(Array.prototype)).toBe(true); // Due to `lockdown()`, and therefore `ses` + }); + + it('loads eventual-send', () => { + expect(typeof HandledPromise).not.toBe('undefined'); // Due to eventual send + }); +}); + +describe(`endoify`, () => { + const assertions = [ + (): boolean => typeof globalThis === 'object', + (): boolean => typeof lockdown === 'function', + (): boolean => typeof repairIntrinsics === 'function', + (): boolean => typeof Compartment === 'function', + (): boolean => typeof assert === 'function', + (): boolean => typeof HandledPromise === 'function', + (): boolean => typeof harden === 'function', + (): boolean => typeof getStackString === 'function', + (): boolean => { + try { + return !Object.assign(harden({ a: 1 }), { b: 2 }); + } catch { + return true; + } + }, + ]; + + for (const assertion of assertions) { + it(`asserts ${String(assertion).replace(/^.*?=>\s*/u, '')}`, () => { + expect(assertion()).toBe(true); + }); + } +}); + +declare global { + // eslint-disable-next-line no-var + var getStackString: (error: Error) => string; + // eslint-disable-next-line no-var + var HandledPromise: HandledPromiseConstructor; +} diff --git a/packages/extension/src/endoify.ts b/packages/extension/src/endoify.ts new file mode 100644 index 000000000..11479b6c9 --- /dev/null +++ b/packages/extension/src/endoify.ts @@ -0,0 +1,3 @@ +/* eslint-disable import-x/extensions */ +/* eslint-disable import-x/no-unassigned-import */ +import '@ocap/shims/endoify'; diff --git a/packages/extension/src/iframe-manager.test.ts b/packages/extension/src/iframe-manager.test.ts index 6c3d4c9a1..b8dd6ef16 100644 --- a/packages/extension/src/iframe-manager.test.ts +++ b/packages/extension/src/iframe-manager.test.ts @@ -1,3 +1,4 @@ +import './endoify.js'; import * as snapsUtils from '@metamask/snaps-utils'; import { delay, makePromiseKitMock } from '@ocap/test-utils'; import { vi, describe, it, expect } from 'vitest'; diff --git a/packages/extension/src/shared.test.ts b/packages/extension/src/shared.test.ts index 2412a8716..d10870d46 100644 --- a/packages/extension/src/shared.test.ts +++ b/packages/extension/src/shared.test.ts @@ -1,3 +1,4 @@ +import './endoify.js'; import { delay } from '@ocap/test-utils'; import { vi, describe, it, expect } from 'vitest'; diff --git a/packages/extension/vite.config.ts b/packages/extension/vite.config.ts index 73c446442..36a591a40 100644 --- a/packages/extension/vite.config.ts +++ b/packages/extension/vite.config.ts @@ -50,6 +50,15 @@ export default defineConfig({ }, }, + resolve: { + alias: [ + { + find: '@ocap/shims/endoify', + replacement: './endoify.js', + }, + ], + }, + plugins: [ endoifyHtmlFilesPlugin(), viteStaticCopy({ @@ -72,20 +81,28 @@ function endoifyHtmlFilesPlugin(): Plugin { return { name: 'externalize-plugin', async transformIndexHtml(htmlString): Promise { - if (htmlString.includes('endoify.js')) { + const htmlDoc = loadHtml(htmlString); + + if (htmlDoc('script[src="endoify.ts"]').length > 0) { + throw new Error( + `HTML document should not reference "endoify.ts" directly:\n${htmlString}`, + ); + } + + if (htmlDoc('script[src="endoify.js"]').length > 0) { throw new Error( `HTML document already references endoify script:\n${htmlString}`, ); } - const htmlDoc = loadHtml(htmlString); - if (htmlDoc('head').length !== 1 || htmlDoc('head script').length < 1) { + if (htmlDoc('head').length !== 1 || htmlDoc('head > script').length < 1) { throw new Error( `Expected HTML document with a single containing at least one