refactor(shims): Optimize building and testing

constraints.pro

@@ -309,7 +309,6 @@ gen_enforced_field(WorkspaceCwd, 'scripts.changelog:update', CorrectChangelogUpd
 
 % All non-root packages must have the same "test" script.
 gen_enforced_field(WorkspaceCwd, 'scripts.test', 'vitest run --config vitest.config.ts') :-
-  WorkspaceCwd \= 'packages/shims',
   WorkspaceCwd \= 'packages/test-utils',
   WorkspaceCwd \= '.'.
 

packages/extension/src/background.ts

@@ -1,6 +1,6 @@
 /* eslint-disable import-x/no-unassigned-import */
 import './dev-console.js';
-import './endoify.mjs';
+import './endoify.js';
 /* eslint-enable import-x/no-unassigned-import */
 
 import type { ExtensionMessage } from './shared.js';

packages/extension/vite.config.ts

@@ -14,10 +14,7 @@ const projectRoot = './src';
  * Module specifiers that will be ignored by Rollup if imported, and therefore
  * not transformed. **Only applies to JavaScript and TypeScript files.**
  */
-const externalModules: readonly string[] = [
-  './dev-console.js',
-  './endoify.mjs',
-];
+const externalModules: readonly string[] = ['./dev-console.js', './endoify.js'];
 
 /**
  * Files that need to be statically copied to the destination directory.
@@ -28,10 +25,7 @@ const staticCopyTargets: readonly string[] = [
   'manifest.json',
   // External modules
   'dev-console.js',
-  '../../shims/dist/endoify.mjs',
-  // Dependencies of external modules
-  '../../shims/dist/eventual-send.mjs',
-  '../../../node_modules/ses/dist/ses.mjs',
+  '../../shims/dist/endoify.js',
 ];
 
 // https://vitejs.dev/config/
@@ -73,12 +67,12 @@ export default defineConfig({
  * @returns The Vite plugin.
  */
 function endoifyHtmlFilesPlugin(): Plugin {
-  const endoifyElement = '<script src="endoify.mjs" type="module"></script>';
+  const endoifyElement = '<script src="endoify.js" type="module"></script>';
 
   return {
     name: 'externalize-plugin',
     async transformIndexHtml(htmlString): Promise<string> {
-      if (htmlString.includes('endoify.mjs')) {
+      if (htmlString.includes('endoify.js')) {
         throw new Error(
           `HTML document already references endoify script:\n${htmlString}`,
         );

packages/shims/.eslintrc.cjs

@@ -5,7 +5,7 @@ module.exports = {
 
   overrides: [
     {
-      files: ['src/**/*.mjs'],
+      files: ['src/**/*.js', 'scripts/**/*.js'],
       globals: { lockdown: 'readonly' },
       rules: {
         'import-x/extensions': 'off',

packages/shims/package.json

@@ -10,7 +10,7 @@
   "sideEffects": false,
   "type": "module",
   "exports": {
-    "./endoify": "./dist/endoify.mjs",
+    "./endoify": "./dist/endoify.js",
     "./package.json": "./package.json"
   },
   "main": "./dist/index.cjs",
@@ -24,7 +24,7 @@
     "changelog:validate": "../../scripts/validate-changelog.sh @ocap/shims",
     "clean": "rimraf --glob ./dist './*.tsbuildinfo'",
     "publish:preview": "yarn npm publish --tag preview",
-    "test": "yarn build && vitest run --config vitest.config.ts --passWithNoTests",
+    "test": "vitest run --config vitest.config.ts",
     "test:clean": "yarn test --no-cache --coverage.clean",
     "test:dev": "yarn test --coverage false",
     "test:verbose": "yarn test --reporter verbose",

packages/shims/scripts/bundle.js

@@ -1,48 +1,29 @@
+// @ts-check
+
 import 'ses';
 import '@endo/lockdown/commit.js';
 
-import bundleSource from '@endo/bundle-source';
-import { createReadStream, createWriteStream } from 'fs';
-import { mkdir } from 'fs/promises';
-import path from 'path';
+import { mkdir } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { rimraf } from 'rimraf';
-import { Readable } from 'stream';
-import { fileURLToPath } from 'url';
+
+import { generateEndoScriptBundle } from './helpers/generate-endo-script-bundle.js';
 
 console.log('Bundling shims...');
 
 const rootDir = fileURLToPath(new URL('..', import.meta.url));
-const src = path.resolve(rootDir, 'src');
-const dist = path.resolve(rootDir, 'dist');
-
-await mkdir(dist, { recursive: true });
-await rimraf(`${dist}/*`, { glob: true });
-
-/**
- * Bundles the target file as endoScript and returns the content as a readable stream.
- *
- * @param {string} specifier - Import path to the file to bundle, e.g. `'@endo/eventual-send/shim.js'`.
- * @returns {Promise<Readable>} The bundled file contents as a Readable stream.
- */
-const createEndoBundleReadStream = async (specifier) => {
-  const filePath = fileURLToPath(import.meta.resolve(specifier));
-  const { source: bundle } = await bundleSource(filePath, {
-    format: 'endoScript',
-  });
-  return Readable.from(bundle);
-};
-
-const sources = {
-  ses: createReadStream(
-    path.resolve(rootDir, '../../node_modules/ses/dist/ses.mjs'),
-  ),
-  eventualSend: await createEndoBundleReadStream('@endo/eventual-send/shim.js'),
-  shim: createReadStream(path.resolve(src, 'endoify.mjs')),
-};
-
-const target = createWriteStream(path.resolve(dist, 'endoify.mjs'));
-
-sources.ses.pipe(target, { end: false });
-sources.ses.on('end', () => sources.eventualSend.pipe(target, { end: false }));
-sources.eventualSend.on('end', () => sources.shim.pipe(target, { end: true }));
-sources.shim.on('end', () => console.log('Success!'));
+const srcDir = path.resolve(rootDir, 'src');
+const distDir = path.resolve(rootDir, 'dist');
+const argv = Object.freeze([...process.argv]);
+
+await mkdir(distDir, { recursive: true });
+await rimraf(`${distDir}/*`, { glob: true });
+
+generateEndoScriptBundle(
+  path.resolve(srcDir, 'endoify.js'),
+  path.resolve(distDir, `endoify.js`),
+  { argv },
+);
+
+console.log('Success!');

packages/shims/scripts/helpers/generate-endo-script-bundle.js

@@ -0,0 +1,73 @@
+// @ts-check
+
+import bundleSource from '@endo/bundle-source';
+import { writeFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+
+import endoScriptIdentifierTransformPlugin from './rollup-plugin-endo-script-identifier-transform.js';
+
+/**
+ * Generate an `endoScript` bundle from a specified module and its
+ * dependencies. Optionally, also rewriting the module shimming
+ * identifiers introduced by `@endo/module-source` to avoid errors
+ * for unexpected zero-width-joiner characters.
+ *
+ * When the `bundle` options are omitted, the default options
+ * used are: `{ format: 'endoScript' }`.
+ *
+ * The `rewrite` operation will be skipped when:
+ *   - The `rewrite` is `false`
+ *   - The `rewrite` option  and the `argv` options are omitted
+ *   - The `argv` option does not include `--with-zwj-rewrite`
+ *
+ * Otherwise, when the `rewrite` option is `true` or some of its
+ * options are omitted, the `argv` option is checked for the
+ * following flags:
+ *   - `--with-zwj-rewrite` (not recommended)
+ *   - `--zwj-rewrite-without-validation` (not recommended)
+ *   - `--zwj-rewrite-debug`
+ *   - `--zwj-rewrite-verbose`
+ *   - `--zwj-rewrite-time`
+ *
+ * @param {string} specifier - The specifier of the module.
+ * @param {string} outputPath - The file path where the bundle will be written.
+ * @param {object} [options] - The fine-grained options for the specific operations.
+ * @param {string} [options.scope] - The root path used for reporting sanitization.
+ * @param {EndoScriptBundleSourceOptions} [options.bundle] - The fine-grained options passed to `bundleSource`.
+ * @param {EndoScriptIdentifierTransformOptions | boolean} [options.rewrite] - Wether to explicitly opt-out (`false`), opt-in (`true`), or the fine-grained options passed to `endoScriptIdentifierTransformPlugin`.
+ * @param {string[] | Readonly<string[]>} [options.argv] - The command-line arguments to use for determining the defaults.
+ * @returns {Promise<void>}
+ */
+export async function generateEndoScriptBundle(specifier, outputPath, options) {
+  const sourcePath = fileURLToPath(import.meta.resolve(specifier));
+
+  let { source } = await bundleSource(sourcePath, {
+    format: 'endoScript',
+    ...options?.bundle,
+  });
+
+  if (options?.rewrite ?? options?.argv?.includes?.('--with-zwj-rewrite')) {
+    source =
+      endoScriptIdentifierTransformPlugin({
+        validation:
+          options?.argv &&
+          !options?.argv?.includes?.('--zwj-rewrite-without-validation'),
+        debugging:
+          options?.argv &&
+          ((options?.argv?.includes?.('--zwj-rewrite-verbose') && 'VERBOSE') ||
+            options?.argv?.includes?.('--zwj-rewrite-debug')),
+        timing: options?.argv?.includes?.('--zwj-rewrite-time'),
+        ...Object(options?.rewrite),
+        scopedRoot:
+          options?.scope ??
+          // @ts-ignore
+          options?.rewrite?.scopedRoot ??
+          fileURLToPath(new URL('../../../../', import.meta.url)),
+      }).transform(source, specifier)?.code ?? source;
+  }
+
+  await writeFile(outputPath, source);
+}
+
+/** @typedef {import('@endo/bundle-source').BundleOptions<'endoScript'>} EndoScriptBundleSourceOptions */
+/** @typedef {import('./rollup-plugin-endo-script-identifier-transform.js').EndoScriptIdentifierTransformOptions} EndoScriptIdentifierTransformOptions */

packages/shims/scripts/helpers/rollup-plugin-endo-script-identifier-transform.js

@@ -0,0 +1,163 @@
+// @ts-check
+
+/* eslint-disable jsdoc/require-returns-type */
+/* eslint-disable spaced-comment */
+/* eslint-disable jsdoc/valid-types */
+/* eslint-disable no-plusplus */
+
+/**
+ * Quickly removes the normalized scope prefix from a normalized pathname.
+ *
+ * @param {string} normalizedPathname - The normalized pathname.
+ * @param {string} [normalizedScope] - The normalized scope.
+ * @returns {string} The scoped path.
+ */
+const scopedPath = (normalizedPathname, normalizedScope = '') =>
+  normalizedPathname.slice(
+    normalizedPathname.indexOf(normalizedScope) + normalizedScope.length || 0,
+  );
+
+/**
+ * Rollup plugin to transform endoScript identifiers.
+ *
+ * @param {object} [options] - The plugin options.
+ * @param {(id: string) => boolean} [options.maybeEndoScript] - A function to determine if the script should be transformed.
+ * @param {string} [options.scopedRoot] - The root directory to scope the transformed script.
+ * @param {boolean} [options.timing] - Whether to log the transform time.
+ * @param {boolean | 'VERBOSE'} [options.debugging] - Whether to log the transform details.
+ * @param {boolean} [options.validation] - Whether to validate the transform.
+ * @returns The Rollup plugin.
+ */
+export default function endoScriptIdentifierTransformPlugin({
+  maybeEndoScript = undefined,
+  scopedRoot = undefined,
+  timing = false,
+  debugging = false,
+  validation = true,
+} = {}) {
+  const zwjIdentifierMatcher =
+    /(?<!\w)\$([hc])\u200d(_{1,4})(\w+\b(?:\$*))+?(?!\w)/gu;
+  const cgjIdentifierMatcher =
+    /(?<!\w)(?=\$(?:\u034f\$)?\u034f+((?:\w+\b(?:\$*))+?))(?:\$\u034f+(?:\w+\b(?:\$*))+?\u034f\$|\$\u034f\$\u034f+(?:\w+\b(?:\$*))+?)(?!\w)/gu;
+
+  return {
+    name: 'endo-script-identifier-transform',
+    transform(code, id) {
+      if (
+        !((maybeEndoScript?.(id) ?? true) && zwjIdentifierMatcher.test(code))
+      ) {
+        debugging && console.warn(`Skipping transform: ${id}`);
+        return null;
+      }
+
+      const scopedId = scopedRoot ? scopedPath(id, scopedRoot) : undefined;
+
+      const tag = `transform ${scopedId ?? id}`;
+
+      debugging === 'VERBOSE' && console.info(tag);
+      timing && console.time(tag);
+
+      if (cgjIdentifierMatcher.test(code)) {
+        throw new Error(
+          `Endoify script contains both U+200D and U+034F identifier characters: ${
+            scopedId ?? id
+          }`,
+        );
+      }
+
+      const changes =
+        validation || debugging
+          ? /** @type {Record<number, Record<'match'|'replacement'|'identifier'|'prefix',string> & Record<'length'|'delta',number>>} */ ({})
+          : undefined;
+
+      let replacements = 0;
+
+      const replacedCode = code.replace(
+        zwjIdentifierMatcher,
+        (match, prefix, { length: underscores }, identifier, index) => {
+          const replacement =
+            prefix === 'h'
+              ? `$${'\u034f'.repeat(underscores)}${identifier}\u034f$`
+              : `$\u034f$${'\u034f'.repeat(underscores)}${identifier}`;
+
+          if (changes) {
+            changes[index] = {
+              match,
+              replacement,
+              identifier,
+              prefix,
+              length: match.length,
+              delta: replacement.length - match.length,
+            };
+          }
+
+          replacements++;
+          return replacement;
+        },
+      );
+
+      timing && console.timeLog(tag);
+
+      const delta = replacedCode.length - code.length;
+
+      debugging && delta !== 0 && console.warn(`Delta: ${delta} [expected: 0]`);
+
+      if (debugging) {
+        debugging === 'VERBOSE' && console.table(changes);
+        console.dir(
+          { id: scopedId ?? id, replacements, delta },
+          {
+            depth: debugging === 'VERBOSE' ? 2 : 1,
+            maxStringLength: 100,
+            compact: true,
+          },
+        );
+      }
+
+      if (delta !== 0) {
+        throw new Error(
+          `Mismatched lengths: ${code.length} ${delta > 0 ? '<' : '>'} ${
+            replacedCode.length
+          } in ${scopedId ?? id}`,
+        );
+      }
+
+      if (changes) {
+        let matched = 0;
+        for (const match of replacedCode.matchAll(cgjIdentifierMatcher)) {
+          if (match[0] !== changes[match.index ?? -1]?.replacement) {
+            throw new Error(
+              `Mismatched replacement: ${match[0]} !== ${
+                changes[match.index ?? -1]?.replacement
+              } in ${scopedId ?? id}`,
+            );
+          }
+          if (match[1] !== changes[match.index ?? -1]?.identifier) {
+            throw new Error(
+              `Mismatched replacement: ${match[1]} !== ${
+                changes[match.index ?? -1]?.identifier
+              } in ${scopedId ?? id}`,
+            );
+          }
+          matched++;
+        }
+        if (matched !== replacements) {
+          throw new Error(
+            `Mismatched replacements: ${matched} !== ${replacements} in ${
+              scopedId ?? id
+            }`,
+          );
+        }
+      }
+
+      timing && console.timeEnd(tag);
+
+      return {
+        code: replacedCode,
+        moduleSideEffects: 'no-treeshake',
+      };
+    },
+  };
+}
+
+/** @typedef {Exclude<Parameters<typeof endoScriptIdentifierTransformPlugin>[0], undefined>} EndoScriptIdentifierTransformOptions */