Asset relocations

package.json

@@ -14,6 +14,7 @@
   "devDependencies": {
     "@google-cloud/datastore": "^2.0.0",
     "@sentry/node": "^4.3.0",
+    "acorn": "^6.0.4",
     "apollo-server-express": "^2.2.2",
     "arg": "^2.0.0",
     "auth0": "^2.14.0",
@@ -23,6 +24,7 @@
     "copy": "^0.3.2",
     "core-js": "^2.5.7",
     "cowsay": "^1.3.1",
+    "estree-walker": "^0.5.2",
     "express": "^4.16.4",
     "firebase": "^5.5.8",
     "fontkit": "^1.7.7",
@@ -33,6 +35,7 @@
     "jest": "^23.6.0",
     "jimp": "^0.5.6",
     "koa": "^2.6.2",
+    "magic-string": "^0.25.1",
     "mailgun": "^0.5.0",
     "mariadb": "^2.0.1-beta",
     "memcached": "^2.2.2",
@@ -49,6 +52,7 @@
     "redis": "^2.8.0",
     "request": "^2.88.0",
     "resolve": "^1.8.1",
+    "rollup-pluginutils": "^2.3.3",
     "rxjs": "^6.3.3",
     "saslprep": "^1.0.2",
     "source-map-support": "^0.5.9",

scripts/build.js

@@ -15,15 +15,18 @@ async function main() {
     // to bundle it. even if we did want watching and a bigger
     // bundle, webpack (and therefore ncc) cannot currently bundle
     // chokidar, which is quite convenient
-    externals: ["chokidar"]
+    externals: ["chokidar", "./asset-relocator.js"]
   });
 
-  if (Object.keys(cliAssets).length || Object.keys(indexAssets).length) {
+  const { code: assetRelocator, assets: assetRelocatorAssets } = await ncc(__dirname + "/../src/asset-relocator");
+
+  if (Object.keys(cliAssets).length || Object.keys(indexAssets).length || Object.keys(assetRelocatorAssets).length) {
     console.error('Assets emitted by core build, these need to be written into the dist directory');
   }
 
   writeFileSync(__dirname + "/../dist/ncc/cli.js", cli);
   writeFileSync(__dirname + "/../dist/ncc/index.js", index);
+  writeFileSync(__dirname + "/../dist/ncc/asset-relocator.js", assetRelocator);
 
   // copy webpack buildin
   await copy(

src/asset-relocator.js

@@ -0,0 +1,260 @@
+const path = require('path');
+const fs = require('fs');
+const { walk } = require('estree-walker');
+const MagicString = require('magic-string');
+const { attachScopes } = require('rollup-pluginutils');
+const evaluate = require('static-eval');
+const acorn = require('acorn');
+
+// Very basic first-pass fs.readFileSync inlining
+function isReference(node, parent) {
+	if (parent.type === 'MemberExpression') return parent.computed || node === parent.object;
+
+	// disregard the `bar` in { bar: foo }
+	if (parent.type === 'Property' && node !== parent.value) return false;
+
+	// disregard the `bar` in `class Foo { bar () {...} }`
+	if (parent.type === 'MethodDefinition') return false;
+
+	// disregard the `bar` in `export { foo as bar }`
+	if (parent.type === 'ExportSpecifier' && node !== parent.local) return false;
+
+	return true;
+}
+
+const assetRegEx = /_\_dirname|_\_filename/;
+module.exports = function (code) {
+  const id = this.resourcePath;
+
+  if (id.endsWith('.json') || !code.match(assetRegEx))
+    return this.callback(null, code);
+
+  const assetNames = Object.create(null);
+  const emitAsset = (assetPath) => {
+    // JS assets to support require(assetPath) and not fs-based handling
+    // NB package.json is ambiguous here...
+    if (assetPath.endsWith('.js') || assetPath.endsWith('.mjs'))
+      return;
+
+    // console.log('Emitting ' + assetPath + ' for module ' + id);
+    const basename = path.basename(assetPath);
+    const ext = path.extname(basename);
+    let name = basename, i = 0;
+    while (assetNames[name])
+      name = basename.substr(0, basename.length - ext.length) + ++i + ext;
+
+    this.emitFile('assets/' + name, fs.readFileSync(assetPath));
+    return "__dirname + '/assets/" + JSON.stringify(name).slice(1, -1) + "'";
+  };
+
+  const magicString = new MagicString(code);
+
+  let ast, isESM;
+  try {
+    ast = acorn.parse(code, { allowReturnOutsideFunction: true });
+    isESM = false;
+  }
+  catch (e) {}
+  if (!ast) {
+    ast = acorn.parse(code, { sourceType: 'module' });
+    isESM = true;
+  }
+
+  let scope = attachScopes(ast, 'scope');
+
+  let fsId, readFileSyncId;
+  let pathId, pathImportIds = {};
+  const shadowDepths = Object.create(null);
+  shadowDepths.__filename = 0;
+  shadowDepths.__dirname = 0;
+  if (!isESM) {
+    shadowDepths.require = 0;
+  }
+  else {
+    for (const decl of ast.body) {
+      // Detects:
+      // import * as fs from 'fs';
+      // import fs from 'fs';
+      // import { readFileSync } from 'fs';
+      // import * as path from 'path';
+      // import path from 'path';
+      // import { join } from 'path';
+      if (decl.type === 'ImportDeclaration') {
+        const source = decl.source.value;
+        if (source === 'fs') {
+          for (const impt of decl.specifiers) {
+            if (impt.type === 'ImportNamespaceSpecifier' || impt.type === 'ImportDefaultSpecifier') {
+              fsId = impt.local.name;
+              shadowDepths[fsId] = 0;
+            } else if (impt.type === 'ImportSpecifier' && impt.imported.name === 'readFileSync') {
+              readFileSyncId = impt.local.name;
+              shadowDepths[readFileSyncId] = 0;
+            }
+          }
+        }
+        else if (source === 'path') {
+          for (const impt of decl.specifiers) {
+            if (impt.type === 'ImportNamespaceSpecifier' || impt.type === 'ImportDefaultSpecifier') {
+              pathId = impt.local.name;
+              shadowDepths[pathId] = 0;
+            } else if (impt.type === 'ImportSpecifier') {
+              pathImportIds[impt.local.name] = impt.imported.name;
+              shadowDepths[impt.local.name] = 0;
+            }
+          }
+        }
+      }
+    }
+  }
+  let didRelocate = false;
+
+  function computeStaticValue (expr, id) {
+    const vars = {};
+    if (shadowDepths.__filename === 0)
+      vars.__dirname = path.resolve(id, '..');
+    if (shadowDepths.__dirname === 0)
+      vars.__filename = id;
+    if (pathId) {
+      if (shadowDepths[pathId] === 0)
+        vars[pathId] = path;
+    }
+    for (const pathFn of Object.keys(pathImportIds)) {
+      if (shadowDepths[pathFn] === 0)
+        vars[pathFn] = path[pathImportIds[pathFn]];
+    }
+
+    // evaluate returns undefined for non-statically-analyzable
+    return evaluate(expr, vars);
+  }
+
+  // statically determinable leaves are tracked, and inlined when the
+  // greatest parent statically known leaf computation corresponds to an asset path
+  let staticChildNode, staticChildValue;
+
+  // detect require('asdf');
+  function isStaticRequire (node) {
+    return node &&
+        node.type === 'CallExpression' &&
+        node.callee.type === 'Identifier' &&
+        node.callee.name === 'require' &&
+        shadowDepths.require === 0 &&
+        node.arguments.length === 1 &&
+        node.arguments[0].type === 'Literal';
+  }
+
+  walk(ast, {
+    enter (node, parent) {
+      if (node.scope) {
+        scope = node.scope;
+        for (const id in node.scope.declarations) {
+          if (id in shadowDepths)
+            shadowDepths[id]++;
+        }
+      }
+
+      if (staticChildNode)
+        return this.skip();
+
+      // detect asset leaf expression triggers (if not already)
+      // __dirname and __filename only currently
+      // Can add require.resolve, import.meta.url, even path-like environment variables
+      if (node.type === 'Identifier' && isReference(node, parent)) {
+        if (!shadowDepths[node.name] &&
+            (node.name === '__dirname' || node.name === '__filename')) {
+          curStaticValue = computeStaticValue(node, id);
+          // if it computes, then we start backtracking
+          if (curStaticValue) {
+            staticChildNode = node;
+            return this.skip();
+          }
+        }
+      }
+
+      // for now we only support top-level variable declarations
+      // so "var { join } = require('path')" will only detect in the top scope.
+      // Intermediate scope handling for these requires is straightforward, but
+      // would need nested shadow depth handling of the pathIds.
+      if (parent === ast && node.type === 'VariableDeclaration') {
+        for (const decl of node.declarations) {
+          // var path = require('path')
+          if (decl.id.type === 'Identifier' &&
+              !isESM && isStaticRequire(decl.init) &&
+              decl.init.arguments[0].value === 'path') {
+            pathId = decl.id.name;
+            shadowDepths[pathId] = 0;
+          }
+          // var { join } = path | require('path');
+          else if (decl.id.type === 'ObjectPattern' && decl.init &&
+              (decl.init.type === 'Identifier' && decl.init.name === pathId && shadowDepths[pathId] === 0) ||
+              !isESM && isStaticRequire(decl.init) && decl.init.arguments[0].value === 'path') {
+            for (const prop of decl.id.properties) {
+              if (prop.type !== 'Property' ||
+                  prop.key.type !== 'Identifier' ||
+                  prop.value.type !== 'Identifier')
+                continue;
+              pathImportIds[prop.value.name] = prop.key.name;
+              shadowDepths[prop.key.name] = 0;
+            }
+          }
+          // var join = path.join | require('path').join;
+          else if (decl.id.type === 'Identifier' &&
+              decl.init &&
+              decl.init.type === 'MemberExpression' &&
+              decl.init.object.type === 'Identifier' &&
+              decl.init.object.name === pathId &&
+              shadowDepths.pathId === 0 &&
+              decl.init.computed === false &&
+              decl.init.property.type === 'Identifier') {
+            pathImportIds[decl.init.property.name] = decl.id.name;
+            shadowDepths[decl.id.name] = 0;
+          }
+        }
+      }
+    },
+    leave (node, parent) {
+      if (node.scope) {
+        scope = scope.parent;
+        for (const id in node.scope.declarations) {
+          if (id in shadowDepths)
+            shadowDepths[id]--;
+        }
+      }
+
+      // computing a static expression outward
+      // -> compute and backtrack
+      if (staticChildNode) {
+        const curStaticValue = computeStaticValue(node, id);
+        if (curStaticValue) {
+          staticChildNode = node;
+          staticChildValue = curStaticValue;
+          return;
+        }
+        // no static value -> see if we should emit the asset if it exists
+        // Currently we only handle files. In theory whole directories could also be emitted if necessary.
+        let isFile = false;
+        if (typeof staticChildValue === 'string') {
+          try {
+            isFile = fs.statSync(staticChildValue).isFile();
+          }
+          catch (e) {}
+        }
+        if (isFile) {
+          const replacement = emitAsset(path.resolve(staticChildValue));
+          if (replacement) {
+            didRelocate = true;
+            magicString.overwrite(staticChildNode.start, staticChildNode.end, replacement);
+          }
+          staticChildNode = staticChildValue = undefined;
+        }
+      }
+    }
+  });
+
+  if (!didRelocate)
+    return this.callback(null, code);
+
+  code = magicString.toString();
+  const map = magicString.generateMap();
+
+  this.callback(null, code, map);
+};

src/cli.js

@@ -46,10 +46,10 @@ switch (args._[0]) {
       const mkdirp = require("mkdirp");
       mkdirp.sync(outDir);
       fs.writeFileSync(outDir + "/index.js", code);
-      Object.keys(assets).forEach(asset => {
+      for (const asset of Object.keys(asset)) {
         mkdirp.sync(path.dirname(asset));
         fs.writeFileSync(outDir + "/" + asset, assets[asset]);
-      });
+      }
     });
   break;