diff --git a/constraints.pro b/constraints.pro index bfe1280c9..48aa60a24 100644 --- a/constraints.pro +++ b/constraints.pro @@ -309,7 +309,6 @@ gen_enforced_field(WorkspaceCwd, 'scripts.changelog:update', CorrectChangelogUpd % All non-root packages must have the same "test" script. gen_enforced_field(WorkspaceCwd, 'scripts.test', 'vitest run --config vitest.config.ts') :- - WorkspaceCwd \= 'packages/shims', WorkspaceCwd \= 'packages/test-utils', WorkspaceCwd \= '.'. diff --git a/packages/extension/src/background.ts b/packages/extension/src/background.ts index b899e06a7..eff0c6b8e 100644 --- a/packages/extension/src/background.ts +++ b/packages/extension/src/background.ts @@ -1,6 +1,6 @@ /* eslint-disable import-x/no-unassigned-import */ import './dev-console.js'; -import './endoify.mjs'; +import './endoify.js'; /* eslint-enable import-x/no-unassigned-import */ import type { ExtensionMessage } from './shared.js'; diff --git a/packages/extension/vite.config.ts b/packages/extension/vite.config.ts index 7d5e5d364..73c446442 100644 --- a/packages/extension/vite.config.ts +++ b/packages/extension/vite.config.ts @@ -14,10 +14,7 @@ const projectRoot = './src'; * Module specifiers that will be ignored by Rollup if imported, and therefore * not transformed. **Only applies to JavaScript and TypeScript files.** */ -const externalModules: readonly string[] = [ - './dev-console.js', - './endoify.mjs', -]; +const externalModules: readonly string[] = ['./dev-console.js', './endoify.js']; /** * Files that need to be statically copied to the destination directory. @@ -28,10 +25,7 @@ const staticCopyTargets: readonly string[] = [ 'manifest.json', // External modules 'dev-console.js', - '../../shims/dist/endoify.mjs', - // Dependencies of external modules - '../../shims/dist/eventual-send.mjs', - '../../../node_modules/ses/dist/ses.mjs', + '../../shims/dist/endoify.js', ]; // https://vitejs.dev/config/ @@ -73,12 +67,12 @@ export default defineConfig({ * @returns The Vite plugin. */ function endoifyHtmlFilesPlugin(): Plugin { - const endoifyElement = ''; + const endoifyElement = ''; return { name: 'externalize-plugin', async transformIndexHtml(htmlString): Promise { - if (htmlString.includes('endoify.mjs')) { + if (htmlString.includes('endoify.js')) { throw new Error( `HTML document already references endoify script:\n${htmlString}`, ); diff --git a/packages/shims/.eslintrc.cjs b/packages/shims/.eslintrc.cjs index 9c6c9b4df..209e64534 100644 --- a/packages/shims/.eslintrc.cjs +++ b/packages/shims/.eslintrc.cjs @@ -5,7 +5,7 @@ module.exports = { overrides: [ { - files: ['src/**/*.mjs'], + files: ['src/**/*.js', 'scripts/**/*.js'], globals: { lockdown: 'readonly' }, rules: { 'import-x/extensions': 'off', diff --git a/packages/shims/package.json b/packages/shims/package.json index 6b67d88a0..8ea2ac779 100644 --- a/packages/shims/package.json +++ b/packages/shims/package.json @@ -10,7 +10,7 @@ "sideEffects": false, "type": "module", "exports": { - "./endoify": "./dist/endoify.mjs", + "./endoify": "./dist/endoify.js", "./package.json": "./package.json" }, "main": "./dist/index.cjs", @@ -24,7 +24,7 @@ "changelog:validate": "../../scripts/validate-changelog.sh @ocap/shims", "clean": "rimraf --glob ./dist './*.tsbuildinfo'", "publish:preview": "yarn npm publish --tag preview", - "test": "yarn build && vitest run --config vitest.config.ts --passWithNoTests", + "test": "vitest run --config vitest.config.ts", "test:clean": "yarn test --no-cache --coverage.clean", "test:dev": "yarn test --coverage false", "test:verbose": "yarn test --reporter verbose", diff --git a/packages/shims/scripts/bundle.js b/packages/shims/scripts/bundle.js index b27a9f6d7..a4e8d0e50 100644 --- a/packages/shims/scripts/bundle.js +++ b/packages/shims/scripts/bundle.js @@ -1,48 +1,29 @@ +// @ts-check + import 'ses'; import '@endo/lockdown/commit.js'; -import bundleSource from '@endo/bundle-source'; -import { createReadStream, createWriteStream } from 'fs'; -import { mkdir } from 'fs/promises'; -import path from 'path'; +import { mkdir } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { rimraf } from 'rimraf'; -import { Readable } from 'stream'; -import { fileURLToPath } from 'url'; + +import { generateEndoScriptBundle } from './helpers/generate-endo-script-bundle.js'; console.log('Bundling shims...'); const rootDir = fileURLToPath(new URL('..', import.meta.url)); -const src = path.resolve(rootDir, 'src'); -const dist = path.resolve(rootDir, 'dist'); - -await mkdir(dist, { recursive: true }); -await rimraf(`${dist}/*`, { glob: true }); - -/** - * Bundles the target file as endoScript and returns the content as a readable stream. - * - * @param {string} specifier - Import path to the file to bundle, e.g. `'@endo/eventual-send/shim.js'`. - * @returns {Promise} The bundled file contents as a Readable stream. - */ -const createEndoBundleReadStream = async (specifier) => { - const filePath = fileURLToPath(import.meta.resolve(specifier)); - const { source: bundle } = await bundleSource(filePath, { - format: 'endoScript', - }); - return Readable.from(bundle); -}; - -const sources = { - ses: createReadStream( - path.resolve(rootDir, '../../node_modules/ses/dist/ses.mjs'), - ), - eventualSend: await createEndoBundleReadStream('@endo/eventual-send/shim.js'), - shim: createReadStream(path.resolve(src, 'endoify.mjs')), -}; - -const target = createWriteStream(path.resolve(dist, 'endoify.mjs')); - -sources.ses.pipe(target, { end: false }); -sources.ses.on('end', () => sources.eventualSend.pipe(target, { end: false })); -sources.eventualSend.on('end', () => sources.shim.pipe(target, { end: true })); -sources.shim.on('end', () => console.log('Success!')); +const srcDir = path.resolve(rootDir, 'src'); +const distDir = path.resolve(rootDir, 'dist'); +const argv = Object.freeze([...process.argv]); + +await mkdir(distDir, { recursive: true }); +await rimraf(`${distDir}/*`, { glob: true }); + +generateEndoScriptBundle( + path.resolve(srcDir, 'endoify.js'), + path.resolve(distDir, `endoify.js`), + { argv }, +); + +console.log('Success!'); diff --git a/packages/shims/scripts/helpers/generate-endo-script-bundle.js b/packages/shims/scripts/helpers/generate-endo-script-bundle.js new file mode 100644 index 000000000..9c0a5e323 --- /dev/null +++ b/packages/shims/scripts/helpers/generate-endo-script-bundle.js @@ -0,0 +1,73 @@ +// @ts-check + +import bundleSource from '@endo/bundle-source'; +import { writeFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; + +import endoScriptIdentifierTransformPlugin from './rollup-plugin-endo-script-identifier-transform.js'; + +/** + * Generate an `endoScript` bundle from a specified module and its + * dependencies. Optionally, also rewriting the module shimming + * identifiers introduced by `@endo/module-source` to avoid errors + * for unexpected zero-width-joiner characters. + * + * When the `bundle` options are omitted, the default options + * used are: `{ format: 'endoScript' }`. + * + * The `rewrite` operation will be skipped when: + * - The `rewrite` is `false` + * - The `rewrite` option and the `argv` options are omitted + * - The `argv` option does not include `--with-zwj-rewrite` + * + * Otherwise, when the `rewrite` option is `true` or some of its + * options are omitted, the `argv` option is checked for the + * following flags: + * - `--with-zwj-rewrite` (not recommended) + * - `--zwj-rewrite-without-validation` (not recommended) + * - `--zwj-rewrite-debug` + * - `--zwj-rewrite-verbose` + * - `--zwj-rewrite-time` + * + * @param {string} specifier - The specifier of the module. + * @param {string} outputPath - The file path where the bundle will be written. + * @param {object} [options] - The fine-grained options for the specific operations. + * @param {string} [options.scope] - The root path used for reporting sanitization. + * @param {EndoScriptBundleSourceOptions} [options.bundle] - The fine-grained options passed to `bundleSource`. + * @param {EndoScriptIdentifierTransformOptions | boolean} [options.rewrite] - Wether to explicitly opt-out (`false`), opt-in (`true`), or the fine-grained options passed to `endoScriptIdentifierTransformPlugin`. + * @param {string[] | Readonly} [options.argv] - The command-line arguments to use for determining the defaults. + * @returns {Promise} + */ +export async function generateEndoScriptBundle(specifier, outputPath, options) { + const sourcePath = fileURLToPath(import.meta.resolve(specifier)); + + let { source } = await bundleSource(sourcePath, { + format: 'endoScript', + ...options?.bundle, + }); + + if (options?.rewrite ?? options?.argv?.includes?.('--with-zwj-rewrite')) { + source = + endoScriptIdentifierTransformPlugin({ + validation: + options?.argv && + !options?.argv?.includes?.('--zwj-rewrite-without-validation'), + debugging: + options?.argv && + ((options?.argv?.includes?.('--zwj-rewrite-verbose') && 'VERBOSE') || + options?.argv?.includes?.('--zwj-rewrite-debug')), + timing: options?.argv?.includes?.('--zwj-rewrite-time'), + ...Object(options?.rewrite), + scopedRoot: + options?.scope ?? + // @ts-ignore + options?.rewrite?.scopedRoot ?? + fileURLToPath(new URL('../../../../', import.meta.url)), + }).transform(source, specifier)?.code ?? source; + } + + await writeFile(outputPath, source); +} + +/** @typedef {import('@endo/bundle-source').BundleOptions<'endoScript'>} EndoScriptBundleSourceOptions */ +/** @typedef {import('./rollup-plugin-endo-script-identifier-transform.js').EndoScriptIdentifierTransformOptions} EndoScriptIdentifierTransformOptions */ diff --git a/packages/shims/scripts/helpers/rollup-plugin-endo-script-identifier-transform.js b/packages/shims/scripts/helpers/rollup-plugin-endo-script-identifier-transform.js new file mode 100644 index 000000000..aab8b9e1f --- /dev/null +++ b/packages/shims/scripts/helpers/rollup-plugin-endo-script-identifier-transform.js @@ -0,0 +1,163 @@ +// @ts-check + +/* eslint-disable jsdoc/require-returns-type */ +/* eslint-disable spaced-comment */ +/* eslint-disable jsdoc/valid-types */ +/* eslint-disable no-plusplus */ + +/** + * Quickly removes the normalized scope prefix from a normalized pathname. + * + * @param {string} normalizedPathname - The normalized pathname. + * @param {string} [normalizedScope] - The normalized scope. + * @returns {string} The scoped path. + */ +const scopedPath = (normalizedPathname, normalizedScope = '') => + normalizedPathname.slice( + normalizedPathname.indexOf(normalizedScope) + normalizedScope.length || 0, + ); + +/** + * Rollup plugin to transform endoScript identifiers. + * + * @param {object} [options] - The plugin options. + * @param {(id: string) => boolean} [options.maybeEndoScript] - A function to determine if the script should be transformed. + * @param {string} [options.scopedRoot] - The root directory to scope the transformed script. + * @param {boolean} [options.timing] - Whether to log the transform time. + * @param {boolean | 'VERBOSE'} [options.debugging] - Whether to log the transform details. + * @param {boolean} [options.validation] - Whether to validate the transform. + * @returns The Rollup plugin. + */ +export default function endoScriptIdentifierTransformPlugin({ + maybeEndoScript = undefined, + scopedRoot = undefined, + timing = false, + debugging = false, + validation = true, +} = {}) { + const zwjIdentifierMatcher = + /(? & Record<'length'|'delta',number>>} */ ({}) + : undefined; + + let replacements = 0; + + const replacedCode = code.replace( + zwjIdentifierMatcher, + (match, prefix, { length: underscores }, identifier, index) => { + const replacement = + prefix === 'h' + ? `$${'\u034f'.repeat(underscores)}${identifier}\u034f$` + : `$\u034f$${'\u034f'.repeat(underscores)}${identifier}`; + + if (changes) { + changes[index] = { + match, + replacement, + identifier, + prefix, + length: match.length, + delta: replacement.length - match.length, + }; + } + + replacements++; + return replacement; + }, + ); + + timing && console.timeLog(tag); + + const delta = replacedCode.length - code.length; + + debugging && delta !== 0 && console.warn(`Delta: ${delta} [expected: 0]`); + + if (debugging) { + debugging === 'VERBOSE' && console.table(changes); + console.dir( + { id: scopedId ?? id, replacements, delta }, + { + depth: debugging === 'VERBOSE' ? 2 : 1, + maxStringLength: 100, + compact: true, + }, + ); + } + + if (delta !== 0) { + throw new Error( + `Mismatched lengths: ${code.length} ${delta > 0 ? '<' : '>'} ${ + replacedCode.length + } in ${scopedId ?? id}`, + ); + } + + if (changes) { + let matched = 0; + for (const match of replacedCode.matchAll(cgjIdentifierMatcher)) { + if (match[0] !== changes[match.index ?? -1]?.replacement) { + throw new Error( + `Mismatched replacement: ${match[0]} !== ${ + changes[match.index ?? -1]?.replacement + } in ${scopedId ?? id}`, + ); + } + if (match[1] !== changes[match.index ?? -1]?.identifier) { + throw new Error( + `Mismatched replacement: ${match[1]} !== ${ + changes[match.index ?? -1]?.identifier + } in ${scopedId ?? id}`, + ); + } + matched++; + } + if (matched !== replacements) { + throw new Error( + `Mismatched replacements: ${matched} !== ${replacements} in ${ + scopedId ?? id + }`, + ); + } + } + + timing && console.timeEnd(tag); + + return { + code: replacedCode, + moduleSideEffects: 'no-treeshake', + }; + }, + }; +} + +/** @typedef {Exclude[0], undefined>} EndoScriptIdentifierTransformOptions */ diff --git a/packages/shims/src/endoify.mjs b/packages/shims/src/endoify.js similarity index 54% rename from packages/shims/src/endoify.mjs rename to packages/shims/src/endoify.js index 40124f6f0..2208eeac8 100644 --- a/packages/shims/src/endoify.mjs +++ b/packages/shims/src/endoify.js @@ -1,6 +1,5 @@ -/* eslint-disable import-x/unambiguous */ -// @inline './ses.mjs'; -// @inline './eventual-send.mjs'; +import 'ses'; +import '@endo/eventual-send/shim.js'; lockdown({ consoleTaming: 'unsafe', @@ -10,5 +9,3 @@ lockdown({ domainTaming: 'unsafe', overrideTaming: 'severe', }); - -/* eslint-enable import-x/unambiguous */ diff --git a/packages/shims/src/endoify.test.ts b/packages/shims/src/endoify.test.ts index de8618c6c..e2069c343 100644 --- a/packages/shims/src/endoify.test.ts +++ b/packages/shims/src/endoify.test.ts @@ -1,4 +1,5 @@ -import { HandledPromise } from '@endo/eventual-send'; +import './endoify.js'; +import type { HandledPromiseConstructor } from '@endo/eventual-send'; import { describe, expect, it } from 'vitest'; describe('endoified', () => { @@ -10,3 +11,8 @@ describe('endoified', () => { expect(typeof HandledPromise).not.toBe('undefined'); // Due to eventual send }); }); + +declare global { + // eslint-disable-next-line no-var + var HandledPromise: HandledPromiseConstructor; +} diff --git a/packages/shims/src/vitest-environment-endoified.ts b/packages/shims/src/vitest-environment-endoified.ts deleted file mode 100644 index 1f0131bc6..000000000 --- a/packages/shims/src/vitest-environment-endoified.ts +++ /dev/null @@ -1,26 +0,0 @@ -import type { Context } from 'node:vm'; -import type { Environment } from 'vitest/environments'; - -export default { - name: 'endoified', - transformMode: 'ssr', - async setupVM() { - const vm = await import('node:vm'); - return { - getVmContext(): Context { - return vm.createContext({ - setTimeout, - clearTimeout, - }); - }, - teardown(): void { - return undefined; - }, - }; - }, - async setup() { - throw new Error( - 'endoified environment requires vitest option --pool=vmThreads or --pool=vmForks', - ); - }, -} as Environment; diff --git a/packages/shims/tsconfig.json b/packages/shims/tsconfig.json index 6f1d89de4..43b2f489c 100644 --- a/packages/shims/tsconfig.json +++ b/packages/shims/tsconfig.json @@ -1,7 +1,9 @@ { "extends": "../../tsconfig.packages.json", "compilerOptions": { - "baseUrl": "./" + "baseUrl": "./", + "allowJs": true, + "checkJs": true }, "references": [], "include": ["./src"] diff --git a/packages/shims/vitest.config.ts b/packages/shims/vitest.config.ts index 4d7480d27..6b2328fbd 100644 --- a/packages/shims/vitest.config.ts +++ b/packages/shims/vitest.config.ts @@ -12,12 +12,9 @@ const config = mergeConfig( defineConfig({ test: { pool: 'vmThreads', - environment: './vitest-environment-endoified.ts', - setupFiles: '../dist/endoify.mjs', }, }), ); -// @ts-expect-error We can and will delete this. delete config.test.coverage.thresholds; export default config;