From 165dfa3a3b213fc91be06ad4c765df12a09d1b2d Mon Sep 17 00:00:00 2001 From: guybedford Date: Thu, 22 Nov 2018 00:14:06 +0200 Subject: [PATCH 1/5] relocate referenced assets --- package.json | 4 + src/asset-relocator.js | 250 ++++++++++++++++++ src/cli.js | 4 +- src/index.js | 22 +- test/index.test.js | 61 ++++- .../asset.txt | 0 test/unit/asset-fs-inline-path-enc/input.js | 3 + test/unit/asset-fs-inline-path-enc/output.js | 109 ++++++++ test/unit/asset-fs-inlining/asset.txt | 1 + .../input.js | 0 test/unit/asset-fs-inlining/output.js | 102 +++++++ test/unit/fs-inlining/output.js | 11 - 12 files changed, 545 insertions(+), 22 deletions(-) create mode 100644 src/asset-relocator.js rename test/unit/{fs-inlining => asset-fs-inline-path-enc}/asset.txt (100%) create mode 100644 test/unit/asset-fs-inline-path-enc/input.js create mode 100644 test/unit/asset-fs-inline-path-enc/output.js create mode 100644 test/unit/asset-fs-inlining/asset.txt rename test/unit/{fs-inlining => asset-fs-inlining}/input.js (100%) create mode 100644 test/unit/asset-fs-inlining/output.js delete mode 100644 test/unit/fs-inlining/output.js diff --git a/package.json b/package.json index ff1f5153..371b0e66 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "devDependencies": { "@google-cloud/datastore": "^2.0.0", "@sentry/node": "^4.3.0", + "acorn": "^6.0.4", "apollo-server-express": "^2.2.2", "arg": "^2.0.0", "auth0": "^2.14.0", @@ -23,6 +24,7 @@ "copy": "^0.3.2", "core-js": "^2.5.7", "cowsay": "^1.3.1", + "estree-walker": "^0.5.2", "express": "^4.16.4", "firebase": "^5.5.8", "fontkit": "^1.7.7", @@ -33,6 +35,7 @@ "jest": "^23.6.0", "jimp": "^0.5.6", "koa": "^2.6.2", + "magic-string": "^0.25.1", "mailgun": "^0.5.0", "mariadb": "^2.0.1-beta", "memcached": "^2.2.2", @@ -49,6 +52,7 @@ "redis": "^2.8.0", "request": "^2.88.0", "resolve": "^1.8.1", + "rollup-pluginutils": "^2.3.3", "rxjs": "^6.3.3", "saslprep": "^1.0.2", "source-map-support": "^0.5.9", diff --git a/src/asset-relocator.js b/src/asset-relocator.js new file mode 100644 index 00000000..4fbe701a --- /dev/null +++ b/src/asset-relocator.js @@ -0,0 +1,250 @@ +const path = require('path'); +const fs = require('fs'); +const { walk } = require('estree-walker'); +const MagicString = require('magic-string'); +const { attachScopes } = require('rollup-pluginutils'); +const evaluate = require('static-eval'); +const acorn = require('acorn'); + +// Very basic first-pass fs.readFileSync inlining +function isReference(node, parent) { + if (parent.type === 'MemberExpression') return parent.computed || node === parent.object; + + // disregard the `bar` in { bar: foo } + if (parent.type === 'Property' && node !== parent.value) return false; + + // disregard the `bar` in `class Foo { bar () {...} }` + if (parent.type === 'MethodDefinition') return false; + + // disregard the `bar` in `export { foo as bar }` + if (parent.type === 'ExportSpecifier' && node !== parent.local) return false; + + return true; +} + +const assetRegEx = /__dirname|__filename/; +module.exports = function (code) { + if (!code.match(assetRegEx)) + return this.callback(null, code); + + const assetNames = Object.create(null); + const emitAsset = (assetPath) => { + // console.log('Emitting ' + assetPath + ' for module ' + id); + const basename = path.basename(assetPath); + let name = basename, i = 0; + while (assetNames[name]) + name = basename + ++i; + this.emitFile('assets/' + name, fs.readFileSync(assetPath)); + return "__dirname + '/assets/" + name + "'"; + }; + + const id = this.resourcePath; + + const magicString = new MagicString(code); + + let ast, isESM; + try { + ast = acorn.parse(code, { allowReturnOutsideFunction: true }); + isESM = false; + } + catch (e) {} + if (!ast) { + ast = acorn.parse(code, { sourceType: 'module' }); + isESM = true; + } + + let scope = attachScopes(ast, 'scope'); + + let fsId, readFileSyncId; + let pathId, pathImportIds = {}; + const shadowDepths = Object.create(null); + shadowDepths.__filename = 0; + shadowDepths.__dirname = 0; + if (!isESM) { + shadowDepths.require = 0; + } + else { + for (const decl of ast.body) { + // Detects: + // import * as fs from 'fs'; + // import fs from 'fs'; + // import { readFileSync } from 'fs'; + // import * as path from 'path'; + // import path from 'path'; + // import { join } from 'path'; + if (decl.type === 'ImportDeclaration') { + const source = decl.source.value; + if (source === 'fs') { + for (const impt of decl.specifiers) { + if (impt.type === 'ImportNamespaceSpecifier' || impt.type === 'ImportDefaultSpecifier') { + fsId = impt.local.name; + shadowDepths[fsId] = 0; + } else if (impt.type === 'ImportSpecifier' && impt.imported.name === 'readFileSync') { + readFileSyncId = impt.local.name; + shadowDepths[readFileSyncId] = 0; + } + } + } + else if (source === 'path') { + for (const impt of decl.specifiers) { + if (impt.type === 'ImportNamespaceSpecifier' || impt.type === 'ImportDefaultSpecifier') { + pathId = impt.local.name; + shadowDepths[pathId] = 0; + } else if (impt.type === 'ImportSpecifier') { + pathImportIds[impt.local.name] = impt.imported.name; + shadowDepths[impt.local.name] = 0; + } + } + } + } + } + } + let didRelocate = false; + + function computeStaticValue (expr, id) { + const vars = {}; + if (shadowDepths.__filename === 0) + vars.__dirname = path.resolve(id, '..'); + if (shadowDepths.__dirname === 0) + vars.__filename = id; + if (pathId) { + if (shadowDepths[pathId] === 0) + vars[pathId] = path; + } + for (const pathFn of Object.keys(pathImportIds)) { + if (shadowDepths[pathFn] === 0) + vars[pathFn] = path[pathImportIds[pathFn]]; + } + + // evaluate returns undefined for non-statically-analyzable + return evaluate(expr, vars); + } + + // statically determinable leaves are tracked, and inlined when the + // greatest parent statically known leaf computation corresponds to an asset path + let staticChildNode, staticChildValue; + + // detect require('asdf'); + function isStaticRequire (node) { + return node && + node.type === 'CallExpression' && + node.callee.type === 'Identifier' && + node.callee.name === 'require' && + shadowDepths.require === 0 && + node.arguments.length === 1 && + node.arguments[0].type === 'Literal'; + } + + walk(ast, { + enter (node, parent) { + if (node.scope) { + scope = node.scope; + for (const id in node.scope.declarations) { + if (id in shadowDepths) + shadowDepths[id]++; + } + } + + if (staticChildNode) + return this.skip(); + + // detect asset leaf expression triggers (if not already) + // __dirname and __filename only currently + // Can add require.resolve, import.meta.url, even path-like environment variables + if (node.type === 'Identifier' && isReference(node, parent)) { + if (!shadowDepths[node.name] && + (node.name === '__dirname' || node.name === '__filename')) { + curStaticValue = computeStaticValue(node, id); + // if it computes, then we start backtracking + if (curStaticValue) { + staticChildNode = node; + return this.skip(); + } + } + } + + // for now we only support top-level variable declarations + // so "var { join } = require('path')" will only detect in the top scope. + // Intermediate scope handling for these requires is straightforward, but + // would need nested shadow depth handling of the pathIds. + if (parent === ast && node.type === 'VariableDeclaration') { + for (const decl of node.declarations) { + // var path = require('path') + if (decl.id.type === 'Identifier' && + !isESM && isStaticRequire(decl.init) && + decl.init.arguments[0].value === 'path') { + pathId = decl.id.name; + shadowDepths[pathId] = 0; + } + // var { join } = path | require('path'); + else if (decl.id.type === 'ObjectPattern' && decl.init && + (decl.init.type === 'Identifier' && decl.init.name === pathId && shadowDepths[pathId] === 0) || + !isESM && isStaticRequire(decl.init) && decl.init.arguments[0].value === 'path') { + for (const prop of decl.id.properties) { + if (prop.type !== 'Property' || + prop.key.type !== 'Identifier' || + prop.value.type !== 'Identifier') + continue; + pathImportIds[prop.value.name] = prop.key.name; + shadowDepths[prop.key.name] = 0; + } + } + // var join = path.join | require('path').join; + else if (decl.id.type === 'Identifier' && + decl.init && + decl.init.type === 'MemberExpression' && + decl.init.object.type === 'Identifier' && + decl.init.object.name === pathId && + shadowDepths.pathId === 0 && + decl.init.computed === false && + decl.init.property.type === 'Identifier') { + pathImportIds[decl.init.property.name] = decl.id.name; + shadowDepths[decl.id.name] = 0; + } + } + } + }, + leave (node, parent) { + if (node.scope) { + scope = scope.parent; + for (const id in node.scope.declarations) { + if (id in shadowDepths) + shadowDepths[id]--; + } + } + + // computing a static expression outward + // -> compute and backtrack + if (staticChildNode) { + const curStaticValue = computeStaticValue(node, id); + if (curStaticValue) { + staticChildNode = node; + staticChildValue = curStaticValue; + return; + } + // no static value -> see if we should emit the asset if it exists + // Currently we only handle files. In theory whole directories could also be emitted if necessary. + let isFile = false; + if (typeof staticChildValue === 'string') { + try { + isFile = fs.statSync(staticChildValue).isFile(); + } + catch (e) {} + } + if (isFile) { + didRelocate = true; + magicString.overwrite(staticChildNode.start, staticChildNode.end, emitAsset(path.resolve(staticChildValue))); + staticChildNode = staticChildValue = undefined; + } + } + } + }); + + if (!didRelocate) + return this.callback(null, code); + + code = magicString.toString(); + const map = magicString.generateMap(); + + this.callback(null, code, map); +}; \ No newline at end of file diff --git a/src/cli.js b/src/cli.js index 1912ca9f..13409363 100755 --- a/src/cli.js +++ b/src/cli.js @@ -46,10 +46,10 @@ switch (args._[0]) { const mkdirp = require("mkdirp"); mkdirp.sync(outDir); fs.writeFileSync(outDir + "/index.js", code); - Object.keys(assets).forEach(asset => { + for (const asset of Object.keys(asset)) { mkdirp.sync(path.dirname(asset)); fs.writeFileSync(outDir + "/" + asset, assets[asset]); - }); + } }); break; diff --git a/src/index.js b/src/index.js index 84a9d413..ad619dbe 100644 --- a/src/index.js +++ b/src/index.js @@ -1,5 +1,6 @@ const resolve = require("resolve"); const fs = require("fs"); +const path = require("path"); const webpack = require("webpack"); const MemoryFS = require("memory-fs"); const WebpackParser = require('webpack/lib/Parser'); @@ -68,6 +69,12 @@ module.exports = async (entry, { externals = [], minify = true, sourceMap = fals // https://github.com/zeit/ncc/pull/29#pullrequestreview-177152175 node: false, externals: (...args) => resolveModule(...[...args, externals]), + module: { + rules: [{ + test: /\.(js|mjs)/, + use: [{ loader: path.join(__dirname, "asset-relocator.js") }] + }] + }, plugins: [ { apply(compiler) { @@ -100,21 +107,22 @@ module.exports = async (entry, { externals = [], minify = true, sourceMap = fals compiler.outputFileSystem = mfs; compiler.resolvers.normal.fileSystem = mfs; return new Promise((resolve, reject) => { - const assets = Object.create(null); - getFlatFiles(mfs.data, assets); - delete assets['/out.js']; compiler.run((err, stats) => { if (err) return reject(err); if (stats.hasErrors()) { return reject(new Error(stats.toString())); } + const assets = Object.create(null); + getFlatFiles(mfs.data, assets); + delete assets["out.js"]; + delete assets["out.js.map"]; const code = mfs.readFileSync("/out.js", "utf8"); const map = sourceMap ? mfs.readFileSync("/out.js.map", "utf8") : null; resolve({ code, map, assets - }) + }); }); }); }; @@ -125,10 +133,10 @@ function getFlatFiles (mfsData, output, curBase = '') { const item = mfsData[path]; const curPath = curBase + '/' + path; // directory - if (item[""] = true) + if (item[""] === true) getFlatFiles(item, output, curPath); // file - else - output[curPath] = mfsData[path]; + else if (!curPath.endsWith("/")) + output[curPath.substr(1)] = mfsData[path]; } } diff --git a/test/index.test.js b/test/index.test.js index 1925324e..9585a052 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -1,6 +1,9 @@ const fs = require("fs"); const sourceMapSupport = require('source-map-support'); -const ncc = require("../"); +const ncc = require("../src/index"); +const mkdirp = require("mkdirp"); +const rimraf = require("rimraf"); +const { dirname } = require("path"); const sourceMapSources = {}; sourceMapSupport.install({ @@ -15,9 +18,46 @@ sourceMapSupport.install({ } }); +for (const unitTest of fs.readdirSync(`${__dirname}/unit`)) { + it(`should generate correct output for ${unitTest}`, async () => { + const expected = fs.readFileSync(`${__dirname}/unit/${unitTest}/output.js`) + .toString().trim() + // Windows support + .replace(/\r/g, ''); + await ncc(`${__dirname}/unit/${unitTest}/input.js`, { minify: false }).then(async ({ code, assets }) => { + // very simple asset validation in unit tests + if (unitTest.startsWith('asset-')) { + expect(Object.keys(assets).length).toBe(1); + expect(assets[Object.keys(assets)[0]] instanceof Buffer); + } + const actual = code.trim() + // Windows support + .replace(/\r/g, ''); + try { + expect(actual).toBe(expected); + } + catch (e) { + // useful for updating fixtures + fs.writeFileSync(`${__dirname}/unit/${unitTest}/actual.js`, actual); + throw e; + } + }); + }); +} + // the twilio test can take a while (large codebase) jest.setTimeout(30000); +function clearTmp () { + try { + rimraf.sync(__dirname + "/tmp"); + } + catch (e) { + if (e.code !== "ENOENT") + throw e; + } +} + for (const integrationTest of fs.readdirSync(__dirname + "/integration")) { // ignore e.g.: `.json` files if (!integrationTest.endsWith(".js")) continue; @@ -25,7 +65,24 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) { const { code, map, assets } = await ncc(__dirname + "/integration/" + integrationTest, { sourceMap: true }); module.exports = null; sourceMapSources[integrationTest] = map; - eval(`${code}\n//# sourceURL=${integrationTest}`); + // integration tests will load assets relative to __dirname + clearTmp(); + for (const asset of Object.keys(assets)) { + const assetPath = __dirname + "/tmp/" + asset; + mkdirp.sync(dirname(assetPath)); + fs.writeFileSync(assetPath, assets[asset]); + } + (__dirname => { + try { + eval(`${code}\n//# sourceURL=${integrationTest}`); + } + catch (e) { + // useful for debugging + mkdirp.sync(__dirname); + fs.writeFileSync(__dirname + "/index.js", code); + throw e; + } + })(__dirname + "/tmp"); if ("function" !== typeof module.exports) { throw new Error( `Integration test "${integrationTest}" evaluation failed. It does not export a function` diff --git a/test/unit/fs-inlining/asset.txt b/test/unit/asset-fs-inline-path-enc/asset.txt similarity index 100% rename from test/unit/fs-inlining/asset.txt rename to test/unit/asset-fs-inline-path-enc/asset.txt diff --git a/test/unit/asset-fs-inline-path-enc/input.js b/test/unit/asset-fs-inline-path-enc/input.js new file mode 100644 index 00000000..90d44121 --- /dev/null +++ b/test/unit/asset-fs-inline-path-enc/input.js @@ -0,0 +1,3 @@ +const fs = require('fs'); +const { join } = require('path'); +console.log(fs.readFileSync(join(__dirname, 'asset.txt'), 'utf8')); \ No newline at end of file diff --git a/test/unit/asset-fs-inline-path-enc/output.js b/test/unit/asset-fs-inline-path-enc/output.js new file mode 100644 index 00000000..08562304 --- /dev/null +++ b/test/unit/asset-fs-inline-path-enc/output.js @@ -0,0 +1,109 @@ +module.exports = +/******/ (function(modules) { // webpackBootstrap +/******/ // The module cache +/******/ var installedModules = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ +/******/ // Check if module is in cache +/******/ if(installedModules[moduleId]) { +/******/ return installedModules[moduleId].exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = installedModules[moduleId] = { +/******/ i: moduleId, +/******/ l: false, +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__); +/******/ +/******/ // Flag the module as loaded +/******/ module.l = true; +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/******/ +/******/ // expose the modules object (__webpack_modules__) +/******/ __webpack_require__.m = modules; +/******/ +/******/ // expose the module cache +/******/ __webpack_require__.c = installedModules; +/******/ +/******/ // define getter function for harmony exports +/******/ __webpack_require__.d = function(exports, name, getter) { +/******/ if(!__webpack_require__.o(exports, name)) { +/******/ Object.defineProperty(exports, name, { enumerable: true, get: getter }); +/******/ } +/******/ }; +/******/ +/******/ // define __esModule on exports +/******/ __webpack_require__.r = function(exports) { +/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { +/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); +/******/ } +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; +/******/ +/******/ // create a fake namespace object +/******/ // mode & 1: value is a module id, require it +/******/ // mode & 2: merge all properties of value into the ns +/******/ // mode & 4: return value when already ns object +/******/ // mode & 8|1: behave like require +/******/ __webpack_require__.t = function(value, mode) { +/******/ if(mode & 1) value = __webpack_require__(value); +/******/ if(mode & 8) return value; +/******/ if((mode & 4) && typeof value === 'object' && value && value.__esModule) return value; +/******/ var ns = Object.create(null); +/******/ __webpack_require__.r(ns); +/******/ Object.defineProperty(ns, 'default', { enumerable: true, value: value }); +/******/ if(mode & 2 && typeof value != 'string') for(var key in value) __webpack_require__.d(ns, key, function(key) { return value[key]; }.bind(null, key)); +/******/ return ns; +/******/ }; +/******/ +/******/ // getDefaultExport function for compatibility with non-harmony modules +/******/ __webpack_require__.n = function(module) { +/******/ var getter = module && module.__esModule ? +/******/ function getDefault() { return module['default']; } : +/******/ function getModuleExports() { return module; }; +/******/ __webpack_require__.d(getter, 'a', getter); +/******/ return getter; +/******/ }; +/******/ +/******/ // Object.prototype.hasOwnProperty.call +/******/ __webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); }; +/******/ +/******/ // __webpack_public_path__ +/******/ __webpack_require__.p = ""; +/******/ +/******/ +/******/ // Load entry module and return exports +/******/ return __webpack_require__(__webpack_require__.s = 0); +/******/ }) +/************************************************************************/ +/******/ ([ +/* 0 */ +/***/ (function(module, exports, __webpack_require__) { + +const fs = __webpack_require__(1); +const { join } = __webpack_require__(2); +console.log(fs.readFileSync(__dirname + '/assets/asset.txt', 'utf8')); + +/***/ }), +/* 1 */ +/***/ (function(module, exports) { + +module.exports = require("fs"); + +/***/ }), +/* 2 */ +/***/ (function(module, exports) { + +module.exports = require("path"); + +/***/ }) +/******/ ]); \ No newline at end of file diff --git a/test/unit/asset-fs-inlining/asset.txt b/test/unit/asset-fs-inlining/asset.txt new file mode 100644 index 00000000..95d09f2b --- /dev/null +++ b/test/unit/asset-fs-inlining/asset.txt @@ -0,0 +1 @@ +hello world \ No newline at end of file diff --git a/test/unit/fs-inlining/input.js b/test/unit/asset-fs-inlining/input.js similarity index 100% rename from test/unit/fs-inlining/input.js rename to test/unit/asset-fs-inlining/input.js diff --git a/test/unit/asset-fs-inlining/output.js b/test/unit/asset-fs-inlining/output.js new file mode 100644 index 00000000..5b27697e --- /dev/null +++ b/test/unit/asset-fs-inlining/output.js @@ -0,0 +1,102 @@ +module.exports = +/******/ (function(modules) { // webpackBootstrap +/******/ // The module cache +/******/ var installedModules = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ +/******/ // Check if module is in cache +/******/ if(installedModules[moduleId]) { +/******/ return installedModules[moduleId].exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = installedModules[moduleId] = { +/******/ i: moduleId, +/******/ l: false, +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__); +/******/ +/******/ // Flag the module as loaded +/******/ module.l = true; +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/******/ +/******/ // expose the modules object (__webpack_modules__) +/******/ __webpack_require__.m = modules; +/******/ +/******/ // expose the module cache +/******/ __webpack_require__.c = installedModules; +/******/ +/******/ // define getter function for harmony exports +/******/ __webpack_require__.d = function(exports, name, getter) { +/******/ if(!__webpack_require__.o(exports, name)) { +/******/ Object.defineProperty(exports, name, { enumerable: true, get: getter }); +/******/ } +/******/ }; +/******/ +/******/ // define __esModule on exports +/******/ __webpack_require__.r = function(exports) { +/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { +/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); +/******/ } +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; +/******/ +/******/ // create a fake namespace object +/******/ // mode & 1: value is a module id, require it +/******/ // mode & 2: merge all properties of value into the ns +/******/ // mode & 4: return value when already ns object +/******/ // mode & 8|1: behave like require +/******/ __webpack_require__.t = function(value, mode) { +/******/ if(mode & 1) value = __webpack_require__(value); +/******/ if(mode & 8) return value; +/******/ if((mode & 4) && typeof value === 'object' && value && value.__esModule) return value; +/******/ var ns = Object.create(null); +/******/ __webpack_require__.r(ns); +/******/ Object.defineProperty(ns, 'default', { enumerable: true, value: value }); +/******/ if(mode & 2 && typeof value != 'string') for(var key in value) __webpack_require__.d(ns, key, function(key) { return value[key]; }.bind(null, key)); +/******/ return ns; +/******/ }; +/******/ +/******/ // getDefaultExport function for compatibility with non-harmony modules +/******/ __webpack_require__.n = function(module) { +/******/ var getter = module && module.__esModule ? +/******/ function getDefault() { return module['default']; } : +/******/ function getModuleExports() { return module; }; +/******/ __webpack_require__.d(getter, 'a', getter); +/******/ return getter; +/******/ }; +/******/ +/******/ // Object.prototype.hasOwnProperty.call +/******/ __webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); }; +/******/ +/******/ // __webpack_public_path__ +/******/ __webpack_require__.p = ""; +/******/ +/******/ +/******/ // Load entry module and return exports +/******/ return __webpack_require__(__webpack_require__.s = 0); +/******/ }) +/************************************************************************/ +/******/ ([ +/* 0 */ +/***/ (function(module, exports, __webpack_require__) { + +const fs = __webpack_require__(1); +console.log(fs.readFileSync(__dirname + '/assets/asset.txt')); + +/***/ }), +/* 1 */ +/***/ (function(module, exports) { + +module.exports = require("fs"); + +/***/ }) +/******/ ]); \ No newline at end of file diff --git a/test/unit/fs-inlining/output.js b/test/unit/fs-inlining/output.js deleted file mode 100644 index bb1b6e59..00000000 --- a/test/unit/fs-inlining/output.js +++ /dev/null @@ -1,11 +0,0 @@ -'use strict'; - -require('fs'); - -console.log(Buffer.from("aGVsbG8gd29ybGQ=", "base64")); - -var input = { - -}; - -module.exports = input; From e7fc1789bd3bea56e2d188b52a262f012a3e6822 Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Thu, 22 Nov 2018 23:30:41 +0200 Subject: [PATCH 2/5] switch back to self build --- test/index.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/index.test.js b/test/index.test.js index 9585a052..e02c7003 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -1,6 +1,6 @@ const fs = require("fs"); const sourceMapSupport = require('source-map-support'); -const ncc = require("../src/index"); +const ncc = require("../"); const mkdirp = require("mkdirp"); const rimraf = require("rimraf"); const { dirname } = require("path"); From 44cc77b3d9c9cb5391989d0db6a02eb03e26d251 Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Fri, 23 Nov 2018 22:41:04 +0200 Subject: [PATCH 3/5] ensure js assets are left to webpack analysis always --- src/asset-relocator.js | 25 +++++++++++++++++-------- src/index.js | 2 +- test/index.test.js | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/asset-relocator.js b/src/asset-relocator.js index 4fbe701a..099eb4d5 100644 --- a/src/asset-relocator.js +++ b/src/asset-relocator.js @@ -22,24 +22,30 @@ function isReference(node, parent) { return true; } -const assetRegEx = /__dirname|__filename/; +const assetRegEx = /_\_dirname|_\_filename/; module.exports = function (code) { - if (!code.match(assetRegEx)) + const id = this.resourcePath; + + if (id.endsWith('.json') || !code.match(assetRegEx)) return this.callback(null, code); const assetNames = Object.create(null); const emitAsset = (assetPath) => { + // JS assets to support require(assetPath) and not fs-based handling + // NB package.json is ambiguous here... + if (assetPath.endsWith('.js') || assetPath.endsWith('.mjs')) + return; + // console.log('Emitting ' + assetPath + ' for module ' + id); const basename = path.basename(assetPath); let name = basename, i = 0; while (assetNames[name]) name = basename + ++i; + this.emitFile('assets/' + name, fs.readFileSync(assetPath)); - return "__dirname + '/assets/" + name + "'"; + return "__dirname + '/assets/" + JSON.stringify(name).slice(1, -1) + "'"; }; - const id = this.resourcePath; - const magicString = new MagicString(code); let ast, isESM; @@ -232,8 +238,11 @@ module.exports = function (code) { catch (e) {} } if (isFile) { - didRelocate = true; - magicString.overwrite(staticChildNode.start, staticChildNode.end, emitAsset(path.resolve(staticChildValue))); + const replacement = emitAsset(path.resolve(staticChildValue)); + if (replacement) { + didRelocate = true; + magicString.overwrite(staticChildNode.start, staticChildNode.end, replacement); + } staticChildNode = staticChildValue = undefined; } } @@ -247,4 +256,4 @@ module.exports = function (code) { const map = magicString.generateMap(); this.callback(null, code, map); -}; \ No newline at end of file +}; diff --git a/src/index.js b/src/index.js index ad619dbe..7542a3c0 100644 --- a/src/index.js +++ b/src/index.js @@ -72,7 +72,7 @@ module.exports = async (entry, { externals = [], minify = true, sourceMap = fals module: { rules: [{ test: /\.(js|mjs)/, - use: [{ loader: path.join(__dirname, "asset-relocator.js") }] + use: [{ loader: __dirname + "/asset-relocator.js" }] }] }, plugins: [ diff --git a/test/index.test.js b/test/index.test.js index e02c7003..e909e053 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -1,6 +1,6 @@ const fs = require("fs"); const sourceMapSupport = require('source-map-support'); -const ncc = require("../"); +const ncc = require("../src/index.js"); const mkdirp = require("mkdirp"); const rimraf = require("rimraf"); const { dirname } = require("path"); From ad84388cc7d8a93fffae6e32838d938e114a67a0 Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Fri, 23 Nov 2018 23:56:02 +0200 Subject: [PATCH 4/5] fixup self build building asset-relocator separately and patching webpack require failure --- scripts/build.js | 7 +++++-- src/index.js | 12 ++++++++++++ test/index.test.js | 7 ++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/scripts/build.js b/scripts/build.js index 1c1dc982..8a468795 100644 --- a/scripts/build.js +++ b/scripts/build.js @@ -15,15 +15,18 @@ async function main() { // to bundle it. even if we did want watching and a bigger // bundle, webpack (and therefore ncc) cannot currently bundle // chokidar, which is quite convenient - externals: ["chokidar"] + externals: ["chokidar", "./asset-relocator.js"] }); - if (Object.keys(cliAssets).length || Object.keys(indexAssets).length) { + const { code: assetRelocator, assets: assetRelocatorAssets } = await ncc(__dirname + "/../src/asset-relocator"); + + if (Object.keys(cliAssets).length || Object.keys(indexAssets).length || Object.keys(assetRelocatorAssets).length) { console.error('Assets emitted by core build, these need to be written into the dist directory'); } writeFileSync(__dirname + "/../dist/ncc/cli.js", cli); writeFileSync(__dirname + "/../dist/ncc/index.js", index); + writeFileSync(__dirname + "/../dist/ncc/asset-relocator.js", assetRelocator); // copy webpack buildin await copy( diff --git a/src/index.js b/src/index.js index 7542a3c0..65fdc509 100644 --- a/src/index.js +++ b/src/index.js @@ -78,6 +78,18 @@ module.exports = async (entry, { externals = [], minify = true, sourceMap = fals plugins: [ { apply(compiler) { + // override "not found" context to try built require first + compiler.hooks.compilation.tap("ncc", compilation => { + compilation.moduleTemplates.javascript.hooks.render.tap("ncc", (moduleSourcePostModule, module, options, dependencyTemplates) => { + if (module._contextDependencies && + moduleSourcePostModule._value.match(/webpackEmptyAsyncContext|webpackEmptyContext/)) { + return moduleSourcePostModule._value.replace('var e = new Error', + `try { return require(req) }\ncatch (e) { if (e.code !== 'MODULE_NOT_FOUND') throw e }` + + `\nvar e = new Error`); + } + }); + }); + compiler.hooks.normalModuleFactory.tap("ncc", NormalModuleFactory => { function handler(parser) { parser.hooks.assign.for("require").intercept({ diff --git a/test/index.test.js b/test/index.test.js index e909e053..397c99ef 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -1,6 +1,6 @@ const fs = require("fs"); const sourceMapSupport = require('source-map-support'); -const ncc = require("../src/index.js"); +const ncc = require("../"); const mkdirp = require("mkdirp"); const rimraf = require("rimraf"); const { dirname } = require("path"); @@ -91,3 +91,8 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) { await module.exports(); }); } + +// remove me when node.js makes this the default behavior +process.on("unhandledRejection", e => { + throw e; +}); \ No newline at end of file From 990c55aa50f01eecd2944015f4c2713ef683200e Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Sat, 24 Nov 2018 15:13:46 +0200 Subject: [PATCH 5/5] fixup deduping logic --- src/asset-relocator.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/asset-relocator.js b/src/asset-relocator.js index 099eb4d5..c25e5f9e 100644 --- a/src/asset-relocator.js +++ b/src/asset-relocator.js @@ -38,9 +38,10 @@ module.exports = function (code) { // console.log('Emitting ' + assetPath + ' for module ' + id); const basename = path.basename(assetPath); + const ext = path.extname(basename); let name = basename, i = 0; while (assetNames[name]) - name = basename + ++i; + name = basename.substr(0, basename.length - ext.length) + ++i + ext; this.emitFile('assets/' + name, fs.readFileSync(assetPath)); return "__dirname + '/assets/" + JSON.stringify(name).slice(1, -1) + "'";