diff --git a/package.json b/package.json index 816491f29e25..e03357aeb8e5 100644 --- a/package.json +++ b/package.json @@ -87,6 +87,7 @@ "@bazel/jasmine": "0.35.0", "@bazel/karma": "0.35.0", "@bazel/typescript": "0.35.0", + "@types/babel__core": "7.1.3", "@types/browserslist": "^4.4.0", "@types/caniuse-lite": "^1.0.0", "@types/clean-css": "^4.2.1", diff --git a/packages/angular_devkit/build_angular/package.json b/packages/angular_devkit/build_angular/package.json index 0327d041e664..f51ae46cfd18 100644 --- a/packages/angular_devkit/build_angular/package.json +++ b/packages/angular_devkit/build_angular/package.json @@ -54,7 +54,7 @@ "stylus": "0.54.5", "stylus-loader": "3.0.2", "tree-kill": "1.2.1", - "terser": "4.1.4", + "terser": "4.3.8", "terser-webpack-plugin": "1.4.1", "webpack": "4.39.2", "webpack-dev-middleware": "3.7.0", diff --git a/packages/angular_devkit/build_angular/src/browser/action-cache.ts b/packages/angular_devkit/build_angular/src/browser/action-cache.ts new file mode 100644 index 000000000000..f04e4053b07b --- /dev/null +++ b/packages/angular_devkit/build_angular/src/browser/action-cache.ts @@ -0,0 +1,191 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import { createHash } from 'crypto'; +import * as findCacheDirectory from 'find-cache-dir'; +import * as fs from 'fs'; +import { manglingDisabled } from '../utils/mangle-options'; +import { CacheKey, ProcessBundleOptions, ProcessBundleResult } from '../utils/process-bundle'; + +const cacache = require('cacache'); +const cacheDownlevelPath = findCacheDirectory({ name: 'angular-build-dl' }); +const packageVersion = require('../../package.json').version; + +// Workaround Node.js issue prior to 10.16 with copyFile on macOS +// https://github.com/angular/angular-cli/issues/15544 & https://github.com/nodejs/node/pull/27241 +let copyFileWorkaround = false; +if (process.platform === 'darwin') { + const version = process.versions.node.split('.').map(part => Number(part)); + if (version[0] < 10 || version[0] === 11 || (version[0] === 10 && version[1] < 16)) { + copyFileWorkaround = true; + } +} + +export interface CacheEntry { + path: string; + size: number; + integrity?: string; +} + +export class BundleActionCache { + constructor(private readonly integrityAlgorithm?: string) {} + + static copyEntryContent(entry: CacheEntry | string, dest: fs.PathLike): void { + if (copyFileWorkaround) { + try { + fs.unlinkSync(dest); + } catch {} + } + + fs.copyFileSync( + typeof entry === 'string' ? entry : entry.path, + dest, + fs.constants.COPYFILE_FICLONE, + ); + if (process.platform !== 'win32') { + // The cache writes entries as readonly and when using copyFile the permissions will also be copied. + // See: https://github.com/npm/cacache/blob/073fbe1a9f789ba42d9a41de7b8429c93cf61579/lib/util/move-file.js#L36 + fs.chmodSync(dest, 0o644); + } + } + + generateBaseCacheKey(content: string): string { + // Create base cache key with elements: + // * package version - different build-angular versions cause different final outputs + // * code length/hash - ensure cached version matches the same input code + const algorithm = this.integrityAlgorithm || 'sha1'; + const codeHash = createHash(algorithm) + .update(content) + .digest('base64'); + let baseCacheKey = `${packageVersion}|${content.length}|${algorithm}-${codeHash}`; + if (manglingDisabled) { + baseCacheKey += '|MD'; + } + + return baseCacheKey; + } + + generateCacheKeys(action: ProcessBundleOptions): string[] { + const baseCacheKey = this.generateBaseCacheKey(action.code); + + // Postfix added to sourcemap cache keys when vendor sourcemaps are present + // Allows non-destructive caching of both variants + const SourceMapVendorPostfix = !!action.sourceMaps && action.vendorSourceMaps ? '|vendor' : ''; + + // Determine cache entries required based on build settings + const cacheKeys = []; + + // If optimizing and the original is not ignored, add original as required + if ((action.optimize || action.optimizeOnly) && !action.ignoreOriginal) { + cacheKeys[CacheKey.OriginalCode] = baseCacheKey + '|orig'; + + // If sourcemaps are enabled, add original sourcemap as required + if (action.sourceMaps) { + cacheKeys[CacheKey.OriginalMap] = baseCacheKey + SourceMapVendorPostfix + '|orig-map'; + } + } + // If not only optimizing, add downlevel as required + if (!action.optimizeOnly) { + cacheKeys[CacheKey.DownlevelCode] = baseCacheKey + '|dl'; + + // If sourcemaps are enabled, add downlevel sourcemap as required + if (action.sourceMaps) { + cacheKeys[CacheKey.DownlevelMap] = baseCacheKey + SourceMapVendorPostfix + '|dl-map'; + } + } + + return cacheKeys; + } + + async getCacheEntries(cacheKeys: (string | null)[]): Promise<(CacheEntry | null)[] | false> { + // Attempt to get required cache entries + const cacheEntries = []; + for (const key of cacheKeys) { + if (key) { + const entry = await cacache.get.info(cacheDownlevelPath, key); + if (!entry) { + return false; + } + cacheEntries.push({ + path: entry.path, + size: entry.size, + integrity: entry.metadata && entry.metadata.integrity, + }); + } else { + cacheEntries.push(null); + } + } + + return cacheEntries; + } + + async getCachedBundleResult(action: ProcessBundleOptions): Promise { + const entries = action.cacheKeys && await this.getCacheEntries(action.cacheKeys); + if (!entries) { + return null; + } + + const result: ProcessBundleResult = { name: action.name }; + + let cacheEntry = entries[CacheKey.OriginalCode]; + if (cacheEntry) { + result.original = { + filename: action.filename, + size: cacheEntry.size, + integrity: cacheEntry.integrity, + }; + + BundleActionCache.copyEntryContent(cacheEntry, result.original.filename); + + cacheEntry = entries[CacheKey.OriginalMap]; + if (cacheEntry) { + result.original.map = { + filename: action.filename + '.map', + size: cacheEntry.size, + }; + + BundleActionCache.copyEntryContent(cacheEntry, result.original.filename + '.map'); + } + } else if (!action.ignoreOriginal) { + // If the original wasn't processed (and therefore not cached), add info + result.original = { + filename: action.filename, + size: Buffer.byteLength(action.code, 'utf8'), + map: + action.map === undefined + ? undefined + : { + filename: action.filename + '.map', + size: Buffer.byteLength(action.map, 'utf8'), + }, + }; + } + + cacheEntry = entries[CacheKey.DownlevelCode]; + if (cacheEntry) { + result.downlevel = { + filename: action.filename.replace('es2015', 'es5'), + size: cacheEntry.size, + integrity: cacheEntry.integrity, + }; + + BundleActionCache.copyEntryContent(cacheEntry, result.downlevel.filename); + + cacheEntry = entries[CacheKey.DownlevelMap]; + if (cacheEntry) { + result.downlevel.map = { + filename: action.filename.replace('es2015', 'es5') + '.map', + size: cacheEntry.size, + }; + + BundleActionCache.copyEntryContent(cacheEntry, result.downlevel.filename + '.map'); + } + } + + return result; + } +} diff --git a/packages/angular_devkit/build_angular/src/browser/action-executor.ts b/packages/angular_devkit/build_angular/src/browser/action-executor.ts index 29f9d7a39b89..8e24c3d5d27e 100644 --- a/packages/angular_devkit/build_angular/src/browser/action-executor.ts +++ b/packages/angular_devkit/build_angular/src/browser/action-executor.ts @@ -7,47 +7,110 @@ */ import JestWorker from 'jest-worker'; import * as os from 'os'; +import * as path from 'path'; +import { ProcessBundleOptions, ProcessBundleResult } from '../utils/process-bundle'; +import { BundleActionCache } from './action-cache'; -export class ActionExecutor { - private largeWorker: JestWorker; - private smallWorker: JestWorker; +let workerFile = require.resolve('../utils/process-bundle'); +workerFile = + path.extname(workerFile) === '.ts' + ? require.resolve('../utils/process-bundle-bootstrap') + : workerFile; - private smallThreshold = 32 * 1024; +export class BundleActionExecutor { + private largeWorker?: JestWorker; + private smallWorker?: JestWorker; + private cache: BundleActionCache; + + constructor( + private workerOptions: unknown, + integrityAlgorithm?: string, + private readonly sizeThreshold = 32 * 1024, + ) { + this.cache = new BundleActionCache(integrityAlgorithm); + } + + private static executeMethod(worker: JestWorker, method: string, input: unknown): Promise { + return ((worker as unknown) as Record Promise>)[method](input); + } + + private ensureLarge(): JestWorker { + if (this.largeWorker) { + return this.largeWorker; + } - constructor(actionFile: string, private readonly actionName: string) { // larger files are processed in a separate process to limit memory usage in the main process - this.largeWorker = new JestWorker(actionFile, { - exposedMethods: [actionName], - }); + return (this.largeWorker = new JestWorker(workerFile, { + exposedMethods: ['process'], + setupArgs: [this.workerOptions], + })); + } + + private ensureSmall(): JestWorker { + if (this.smallWorker) { + return this.smallWorker; + } // small files are processed in a limited number of threads to improve speed // The limited number also prevents a large increase in memory usage for an otherwise short operation - this.smallWorker = new JestWorker(actionFile, { - exposedMethods: [actionName], + return (this.smallWorker = new JestWorker(workerFile, { + exposedMethods: ['process'], + setupArgs: [this.workerOptions], numWorkers: os.cpus().length < 2 ? 1 : 2, // Will automatically fallback to processes if not supported enableWorkerThreads: true, - }); + })); } - execute(options: Input): Promise { - if (options.size > this.smallThreshold) { - return ((this.largeWorker as unknown) as Record Promise>)[ - this.actionName - ](options); + private executeAction(method: string, action: { code: string }): Promise { + // code.length is not an exact byte count but close enough for this + if (action.code.length > this.sizeThreshold) { + return BundleActionExecutor.executeMethod(this.ensureLarge(), method, action); } else { - return ((this.smallWorker as unknown) as Record Promise>)[ - this.actionName - ](options); + return BundleActionExecutor.executeMethod(this.ensureSmall(), method, action); } } - executeAll(options: Input[]): Promise { - return Promise.all(options.map(o => this.execute(o))); + async process(action: ProcessBundleOptions) { + const cacheKeys = this.cache.generateCacheKeys(action); + action.cacheKeys = cacheKeys; + + // Try to get cached data, if it fails fallback to processing + try { + const cachedResult = await this.cache.getCachedBundleResult(action); + if (cachedResult) { + return cachedResult; + } + } catch {} + + return this.executeAction('process', action); + } + + async *processAll(actions: Iterable) { + const executions = new Map, Promise>(); + for (const action of actions) { + const execution = this.process(action); + executions.set( + execution, + execution.then(result => { + executions.delete(execution); + + return result; + }), + ); + } + + while (executions.size > 0) { + yield Promise.race(executions.values()); + } } stop() { - this.largeWorker.end(); - this.smallWorker.end(); + if (this.largeWorker) { + this.largeWorker.end(); + } + if (this.smallWorker) { + this.smallWorker.end(); + } } } diff --git a/packages/angular_devkit/build_angular/src/browser/index.ts b/packages/angular_devkit/build_angular/src/browser/index.ts index 82ef60250734..6a65263e8805 100644 --- a/packages/angular_devkit/build_angular/src/browser/index.ts +++ b/packages/angular_devkit/build_angular/src/browser/index.ts @@ -66,9 +66,7 @@ import { normalizeOptimization, normalizeSourceMaps, } from '../utils'; -import { manglingDisabled } from '../utils/mangle-options'; import { - CacheKey, ProcessBundleFile, ProcessBundleOptions, ProcessBundleResult, @@ -79,7 +77,7 @@ import { getIndexInputFile, getIndexOutputFile, } from '../utils/webpack-browser-config'; -import { ActionExecutor } from './action-executor'; +import { BundleActionExecutor } from './action-executor'; import { Schema as BrowserBuilderSchema } from './schema'; const cacache = require('cacache'); @@ -349,7 +347,8 @@ export function buildWebpackBrowser( } // If not optimizing then ES2015 polyfills do not need processing // Unlike other module scripts, it is never downleveled - if (!actionOptions.optimize && file.file.startsWith('polyfills-es2015')) { + const es2015Polyfills = file.file.startsWith('polyfills-es2015'); + if (!actionOptions.optimize && es2015Polyfills) { continue; } @@ -372,22 +371,6 @@ export function buildWebpackBrowser( filename = filename.replace('-es2015', ''); } - // ES2015 polyfills are only optimized; optimization check was performed above - if (file.file.startsWith('polyfills-es2015')) { - actions.push({ - ...actionOptions, - filename, - code, - map, - // id is always present for non-assets - // tslint:disable-next-line: no-non-null-assertion - name: file.id!, - optimizeOnly: true, - }); - - continue; - } - // Record the bundle processing action // The runtime chunk gets special processing for lazy loaded files actions.push({ @@ -400,8 +383,14 @@ export function buildWebpackBrowser( name: file.id!, runtime: file.file.startsWith('runtime'), ignoreOriginal: es5Polyfills, + optimizeOnly: es2015Polyfills, }); + // ES2015 polyfills are only optimized; optimization check was performed above + if (es2015Polyfills) { + continue; + } + // Add the newly created ES5 bundles to the index as nomodule scripts const newFilename = es5Polyfills ? file.file.replace('-es2015', '') @@ -414,206 +403,28 @@ export function buildWebpackBrowser( const processActions: typeof actions = []; let processRuntimeAction: ProcessBundleOptions | undefined; - const cacheActions: { src: string; dest: string }[] = []; const processResults: ProcessBundleResult[] = []; for (const action of actions) { - // Create base cache key with elements: - // * package version - different build-angular versions cause different final outputs - // * code length/hash - ensure cached version matches the same input code - const algorithm = action.integrityAlgorithm || 'sha1'; - const codeHash = createHash(algorithm) - .update(action.code) - .digest('base64'); - let baseCacheKey = `${packageVersion}|${action.code.length}|${algorithm}-${codeHash}`; - if (manglingDisabled) { - baseCacheKey += '|MD'; - } - - // Postfix added to sourcemap cache keys when vendor sourcemaps are present - // Allows non-destructive caching of both variants - const SourceMapVendorPostfix = - !!action.sourceMaps && action.vendorSourceMaps ? '|vendor' : ''; - - // Determine cache entries required based on build settings - const cacheKeys = []; - - // If optimizing and the original is not ignored, add original as required - if ((action.optimize || action.optimizeOnly) && !action.ignoreOriginal) { - cacheKeys[CacheKey.OriginalCode] = baseCacheKey + '|orig'; - - // If sourcemaps are enabled, add original sourcemap as required - if (action.sourceMaps) { - cacheKeys[CacheKey.OriginalMap] = - baseCacheKey + SourceMapVendorPostfix + '|orig-map'; - } - } - // If not only optimizing, add downlevel as required - if (!action.optimizeOnly) { - cacheKeys[CacheKey.DownlevelCode] = baseCacheKey + '|dl'; - - // If sourcemaps are enabled, add downlevel sourcemap as required - if (action.sourceMaps) { - cacheKeys[CacheKey.DownlevelMap] = - baseCacheKey + SourceMapVendorPostfix + '|dl-map'; - } - } - - // Attempt to get required cache entries - const cacheEntries = []; - let cached = cacheKeys.length > 0; - for (const key of cacheKeys) { - if (key) { - const entry = await cacache.get.info(cacheDownlevelPath, key); - if (!entry) { - cached = false; - break; - } - cacheEntries.push(entry); - } else { - cacheEntries.push(null); - } - } - - // If all required cached entries are present, use the cached entries - // Otherwise process the files // If SRI is enabled always process the runtime bundle // Lazy route integrity values are stored in the runtime bundle if (action.integrityAlgorithm && action.runtime) { processRuntimeAction = action; - } else if (cached) { - const result: ProcessBundleResult = { name: action.name }; - if (action.integrityAlgorithm) { - result.integrity = `${action.integrityAlgorithm}-${codeHash}`; - } - - let cacheEntry = cacheEntries[CacheKey.OriginalCode]; - if (cacheEntry) { - cacheActions.push({ - src: cacheEntry.path, - dest: action.filename, - }); - result.original = { - filename: action.filename, - size: cacheEntry.size, - integrity: cacheEntry.metadata && cacheEntry.metadata.integrity, - }; - - cacheEntry = cacheEntries[CacheKey.OriginalMap]; - if (cacheEntry) { - cacheActions.push({ - src: cacheEntry.path, - dest: action.filename + '.map', - }); - result.original.map = { - filename: action.filename + '.map', - size: cacheEntry.size, - }; - } - } else if (!action.ignoreOriginal) { - // If the original wasn't processed (and therefore not cached), add info - result.original = { - filename: action.filename, - size: Buffer.byteLength(action.code, 'utf8'), - map: - action.map === undefined - ? undefined - : { - filename: action.filename + '.map', - size: Buffer.byteLength(action.map, 'utf8'), - }, - }; - } - - cacheEntry = cacheEntries[CacheKey.DownlevelCode]; - if (cacheEntry) { - cacheActions.push({ - src: cacheEntry.path, - dest: action.filename.replace('es2015', 'es5'), - }); - result.downlevel = { - filename: action.filename.replace('es2015', 'es5'), - size: cacheEntry.size, - integrity: cacheEntry.metadata && cacheEntry.metadata.integrity, - }; - - cacheEntry = cacheEntries[CacheKey.DownlevelMap]; - if (cacheEntry) { - cacheActions.push({ - src: cacheEntry.path, - dest: action.filename.replace('es2015', 'es5') + '.map', - }); - result.downlevel.map = { - filename: action.filename.replace('es2015', 'es5') + '.map', - size: cacheEntry.size, - }; - } - } - - processResults.push(result); - } else if (action.runtime) { - processRuntimeAction = { - ...action, - cacheKeys, - cachePath: cacheDownlevelPath || undefined, - }; } else { - processActions.push({ - ...action, - cacheKeys, - cachePath: cacheDownlevelPath || undefined, - }); + processActions.push(action); } } - // Workaround Node.js issue prior to 10.16 with copyFile on macOS - // https://github.com/angular/angular-cli/issues/15544 & https://github.com/nodejs/node/pull/27241 - let copyFileWorkaround = false; - if (process.platform === 'darwin') { - const version = process.versions.node.split('.').map(part => Number(part)); - if ( - version[0] < 10 || - version[0] === 11 || - (version[0] === 10 && version[1] < 16) - ) { - copyFileWorkaround = true; - } - } - - for (const action of cacheActions) { - if (copyFileWorkaround) { - try { - fs.unlinkSync(action.dest); - } catch {} - } - - fs.copyFileSync(action.src, action.dest, fs.constants.COPYFILE_FICLONE); - if (process.platform !== 'win32') { - // The cache writes entries as readonly and when using copyFile the permissions will also be copied. - // See: https://github.com/npm/cacache/blob/073fbe1a9f789ba42d9a41de7b8429c93cf61579/lib/util/move-file.js#L36 - fs.chmodSync(action.dest, 0o644); - } - } - - if (processActions.length > 0) { - const workerFile = require.resolve('../utils/process-bundle'); - const executor = new ActionExecutor< - ProcessBundleOptions & { size: number }, - ProcessBundleResult - >( - path.extname(workerFile) !== '.ts' - ? workerFile - : require.resolve('../utils/process-bundle-bootstrap'), - 'process', - ); + const executor = new BundleActionExecutor( + { cachePath: cacheDownlevelPath }, + options.subresourceIntegrity ? 'sha384' : undefined, + ); - try { - const results = await executor.executeAll( - processActions.map(a => ({ ...a, size: a.code.length })), - ); - results.forEach(result => processResults.push(result)); - } finally { - executor.stop(); + try { + for await (const result of executor.processAll(processActions)) { + processResults.push(result); } + } finally { + executor.stop(); } // Runtime must be processed after all other files diff --git a/packages/angular_devkit/build_angular/src/utils/process-bundle.ts b/packages/angular_devkit/build_angular/src/utils/process-bundle.ts index f0e9f5c07600..606c3ab45801 100644 --- a/packages/angular_devkit/build_angular/src/utils/process-bundle.ts +++ b/packages/angular_devkit/build_angular/src/utils/process-bundle.ts @@ -8,11 +8,12 @@ import { createHash } from 'crypto'; import * as fs from 'fs'; import * as path from 'path'; -import { SourceMapConsumer, SourceMapGenerator } from 'source-map'; +import { RawSourceMap, SourceMapConsumer, SourceMapGenerator } from 'source-map'; import { minify } from 'terser'; +import { ScriptTarget, transpileModule } from 'typescript'; +import { SourceMapSource } from 'webpack-sources'; import { manglingDisabled } from './mangle-options'; -const { transformAsync } = require('@babel/core'); const cacache = require('cacache'); export interface ProcessBundleOptions { @@ -28,7 +29,6 @@ export interface ProcessBundleOptions { optimizeOnly?: boolean; ignoreOriginal?: boolean; cacheKeys?: (string | null)[]; - cachePath?: string; integrityAlgorithm?: 'sha256' | 'sha384' | 'sha512'; runtimeData?: ProcessBundleResult[]; } @@ -57,180 +57,125 @@ export const enum CacheKey { DownlevelMap = 3, } +let cachePath: string | undefined; + +export function setup(options: { cachePath: string }): void { + cachePath = options.cachePath; +} + +async function cachePut(content: string, key: string | null, integrity?: string): Promise { + if (cachePath && key) { + await cacache.put(cachePath, key, content, { + metadata: { integrity }, + }); + } +} + export async function process(options: ProcessBundleOptions): Promise { if (!options.cacheKeys) { options.cacheKeys = []; } - // If no downlevelling required than just mangle code and return - if (options.optimizeOnly) { - const result: ProcessBundleResult = { name: options.name }; - if (options.integrityAlgorithm) { - result.integrity = generateIntegrityValue(options.integrityAlgorithm, options.code); - } - - // Replace integrity hashes with updated values - // NOTE: This should eventually be a babel plugin - if (options.runtime && options.integrityAlgorithm && options.runtimeData) { - for (const data of options.runtimeData) { - if (!data.integrity || !data.original || !data.original.integrity) { - continue; - } - - options.code = options.code.replace(data.integrity, data.original.integrity); - } - } - - result.original = await mangleOriginal(options); - - return result; + const result: ProcessBundleResult = { name: options.name }; + if (options.integrityAlgorithm) { + // Store unmodified code integrity value -- used for SRI value replacement + result.integrity = generateIntegrityValue(options.integrityAlgorithm, options.code); } - // if code size is larger than 500kB, manually handle sourcemaps with newer source-map package. - // babel currently uses an older version that still supports sync calls - const codeSize = Buffer.byteLength(options.code, 'utf8'); - const mapSize = options.map ? Buffer.byteLength(options.map, 'utf8') : 0; - const manualSourceMaps = codeSize >= 500 * 1024 || mapSize >= 500 * 1024; - - // downlevel the bundle - let { code, map } = await transformAsync(options.code, { - filename: options.filename, - inputSourceMap: !manualSourceMaps && options.map !== undefined && JSON.parse(options.map), - babelrc: false, - // modules aren't needed since the bundles use webpack's custom module loading - // 'transform-typeof-symbol' generates slower code - presets: [ - ['@babel/preset-env', { modules: false, exclude: ['transform-typeof-symbol'] }], - ], - minified: options.optimize, - // `false` ensures it is disabled and prevents large file warnings - compact: options.optimize || false, - sourceMaps: options.sourceMaps, - }); - - const newFilePath = options.filename.replace('es2015', 'es5'); - - // Adjust lazy loaded scripts to point to the proper variant - // Extra spacing is intentional to align source line positions + // Runtime chunk requires specialized handling if (options.runtime) { - code = code.replace('"-es2015.', ' "-es5.'); - - // Replace integrity hashes with updated values - // NOTE: This should eventually be a babel plugin - if (options.integrityAlgorithm && options.runtimeData) { - for (const data of options.runtimeData) { - if (!data.integrity || !data.downlevel || !data.downlevel.integrity) { - continue; - } - - code = code.replace(data.integrity, data.downlevel.integrity); - } - } + return { ...result, ...(await processRuntime(options)) }; } - if (options.sourceMaps && manualSourceMaps && options.map) { - const generator = new SourceMapGenerator(); - let sourceRoot; - await SourceMapConsumer.with(options.map, null, originalConsumer => { - sourceRoot = 'sourceRoot' in originalConsumer ? originalConsumer.sourceRoot : undefined; - - return SourceMapConsumer.with(map, null, newConsumer => { - newConsumer.eachMapping(mapping => { - if (mapping.originalLine === null) { - return; - } - - const originalPosition = originalConsumer.originalPositionFor({ - line: mapping.originalLine, - column: mapping.originalColumn, - }); - - if ( - originalPosition.line === null || - originalPosition.column === null || - originalPosition.source === null - ) { - return; - } - - generator.addMapping({ - generated: { - line: mapping.generatedLine, - column: mapping.generatedColumn, - }, - name: originalPosition.name || undefined, - original: { - line: originalPosition.line, - column: originalPosition.column, - }, - source: originalPosition.source, - }); - }); - }); + const basePath = path.dirname(options.filename); + const filename = path.basename(options.filename); + const downlevelFilename = filename.replace('es2015', 'es5'); + const downlevel = !options.optimizeOnly; + + // if code size is larger than 1 MB, manually handle sourcemaps with newer source-map package. + const codeSize = Buffer.byteLength(options.code); + const mapSize = options.map ? Buffer.byteLength(options.map) : 0; + const manualSourceMaps = codeSize >= 1024 * 1024 || mapSize >= 1024 * 1024; + + const sourceCode = options.code; + const sourceMap = options.map ? JSON.parse(options.map) : undefined; + + let downlevelCode; + let downlevelMap; + if (downlevel) { + // Downlevel the bundle + const transformResult = transpileModule(sourceCode, { + fileName: downlevelFilename, + compilerOptions: { + sourceMap: !!sourceMap, + target: ScriptTarget.ES5, + }, }); - map = generator.toJSON(); - map.file = path.basename(newFilePath); - map.sourceRoot = sourceRoot; + downlevelCode = transformResult.outputText; + + if (sourceMap && transformResult.sourceMapText) { + if (manualSourceMaps) { + downlevelMap = await mergeSourcemaps(sourceMap, JSON.parse(transformResult.sourceMapText)); + } else { + // More accurate but significantly more costly + const tempSource = new SourceMapSource( + transformResult.outputText, + downlevelFilename, + JSON.parse(transformResult.sourceMapText), + sourceCode, + sourceMap, + ); + + downlevelMap = tempSource.map(); + } + } } - const result: ProcessBundleResult = { name: options.name }; - if (options.optimize) { - // Note: Investigate converting the AST instead of re-parsing - // estree -> terser is already supported; need babel -> estree/terser - - // Mangle downlevel code - const minifyOutput = minify(code, { - compress: true, - ecma: 5, - mangle: !manglingDisabled, - safari10: true, - output: { - ascii_only: true, - webkit: true, - }, - sourceMap: options.sourceMaps && { - filename: path.basename(newFilePath), - content: map, - }, - }); - - if (minifyOutput.error) { - throw minifyOutput.error; + if (downlevelCode) { + const minifyResult = terserMangle(downlevelCode, { + filename: downlevelFilename, + map: downlevelMap, + compress: true, + }); + downlevelCode = minifyResult.code; + downlevelMap = minifyResult.map; } - code = minifyOutput.code; - map = minifyOutput.map; - - // Mangle original code if (!options.ignoreOriginal) { result.original = await mangleOriginal(options); } - } else if (map) { - map = JSON.stringify(map); } - if (map) { - if (!options.hiddenSourceMaps) { - code += `\n//# sourceMappingURL=${path.basename(newFilePath)}.map`; - } + if (downlevelCode) { + const downlevelPath = path.join(basePath, downlevelFilename); - if (options.cachePath && options.cacheKeys[CacheKey.DownlevelMap]) { - await cacache.put(options.cachePath, options.cacheKeys[CacheKey.DownlevelMap], map); - } + let mapContent; + if (downlevelMap) { + if (!options.hiddenSourceMaps) { + downlevelCode += `\n//# sourceMappingURL=${downlevelFilename}.map`; + } - fs.writeFileSync(newFilePath + '.map', map); - } + mapContent = JSON.stringify(downlevelMap); + await cachePut(mapContent, options.cacheKeys[CacheKey.DownlevelMap]); + fs.writeFileSync(downlevelPath + '.map', mapContent); + } - result.downlevel = createFileEntry(newFilePath, code, map, options.integrityAlgorithm); + result.downlevel = createFileEntry( + downlevelFilename, + downlevelCode, + mapContent, + options.integrityAlgorithm, + ); - if (options.cachePath && options.cacheKeys[CacheKey.DownlevelCode]) { - await cacache.put(options.cachePath, options.cacheKeys[CacheKey.DownlevelCode], code, { - metadata: { integrity: result.downlevel.integrity }, - }); + await cachePut( + downlevelCode, + options.cacheKeys[CacheKey.DownlevelCode], + result.downlevel.integrity, + ); + fs.writeFileSync(downlevelPath, downlevelCode); } - fs.writeFileSync(newFilePath, code); // If original was not processed, add info if (!result.original && !options.ignoreOriginal) { @@ -242,74 +187,138 @@ export async function process(options: ProcessBundleOptions): Promise { + return SourceMapConsumer.with(second, null, newConsumer => { + newConsumer.eachMapping(mapping => { + if (mapping.originalLine === null) { + return; + } + const originalPosition = originalConsumer.originalPositionFor({ + line: mapping.originalLine, + column: mapping.originalColumn, + }); + if ( + originalPosition.line === null || + originalPosition.column === null || + originalPosition.source === null + ) { + return; + } + generator.addMapping({ + generated: { + line: mapping.generatedLine, + column: mapping.generatedColumn, + }, + name: originalPosition.name || undefined, + original: { + line: originalPosition.line, + column: originalPosition.column, + }, + source: originalPosition.source, + }); + }); + }); + }); + + const map = generator.toJSON(); + map.file = second.file; + map.sourceRoot = sourceRoot; + + // Put the sourceRoot back + if (sourceRoot) { + first.sourceRoot = sourceRoot; } - return result; + return map; } async function mangleOriginal(options: ProcessBundleOptions): Promise { - const resultOriginal = minify(options.code, { - compress: false, + const result = terserMangle(options.code, { + filename: path.basename(options.filename), + map: options.map ? JSON.parse(options.map) : undefined, ecma: 6, - mangle: !manglingDisabled, - safari10: true, - output: { - ascii_only: true, - webkit: true, - }, - sourceMap: options.sourceMaps && - options.map !== undefined && { - filename: path.basename(options.filename), - content: JSON.parse(options.map), - }, }); - if (resultOriginal.error) { - throw resultOriginal.error; - } - - if (resultOriginal.map) { + let mapContent; + if (result.map) { if (!options.hiddenSourceMaps) { - resultOriginal.code += `\n//# sourceMappingURL=${path.basename(options.filename)}.map`; + result.code += `\n//# sourceMappingURL=${path.basename(options.filename)}.map`; } - if (options.cachePath && options.cacheKeys && options.cacheKeys[CacheKey.OriginalMap]) { - await cacache.put( - options.cachePath, - options.cacheKeys[CacheKey.OriginalMap], - resultOriginal.map, - ); - } + mapContent = JSON.stringify(result.map); - fs.writeFileSync(options.filename + '.map', resultOriginal.map); + await cachePut( + mapContent, + (options.cacheKeys && options.cacheKeys[CacheKey.OriginalMap]) || null, + ); + fs.writeFileSync(options.filename + '.map', mapContent); } const fileResult = createFileEntry( options.filename, - // tslint:disable-next-line: no-non-null-assertion - resultOriginal.code!, - resultOriginal.map as string, + result.code, + mapContent, options.integrityAlgorithm, ); - if (options.cachePath && options.cacheKeys && options.cacheKeys[CacheKey.OriginalCode]) { - await cacache.put( - options.cachePath, - options.cacheKeys[CacheKey.OriginalCode], - resultOriginal.code, - { - metadata: { integrity: fileResult.integrity }, - }, - ); - } - - fs.writeFileSync(options.filename, resultOriginal.code); + await cachePut( + result.code, + (options.cacheKeys && options.cacheKeys[CacheKey.OriginalCode]) || null, + fileResult.integrity, + ); + fs.writeFileSync(options.filename, result.code); return fileResult; } +function terserMangle( + code: string, + options: { filename?: string; map?: RawSourceMap; compress?: boolean; ecma?: 5 | 6 } = {}, +) { + // Note: Investigate converting the AST instead of re-parsing + // estree -> terser is already supported; need babel -> estree/terser + + // Mangle downlevel code + const minifyOutput = minify(code, { + compress: options.compress || false, + ecma: options.ecma || 5, + mangle: !manglingDisabled, + safari10: true, + output: { + ascii_only: true, + webkit: true, + }, + sourceMap: + !!options.map && + ({ + filename: options.filename, + // terser uses an old version of the sourcemap typings + // tslint:disable-next-line: no-any + content: options.map as any, + asObject: true, + // typings don't include asObject option + // tslint:disable-next-line: no-any + } as any), + }); + + if (minifyOutput.error) { + throw minifyOutput.error; + } + + // tslint:disable-next-line: no-non-null-assertion + return { code: minifyOutput.code!, map: minifyOutput.map as RawSourceMap | undefined }; +} + function createFileEntry( filename: string, code: string, @@ -338,3 +347,92 @@ function generateIntegrityValue(hashAlgorithm: string, code: string) { .digest('base64') ); } + +// The webpack runtime chunk is already ES5. +// However, two variants are still needed due to lazy routing and SRI differences +// NOTE: This should eventually be a babel plugin +async function processRuntime( + options: ProcessBundleOptions, +): Promise> { + let originalCode = options.code; + let downlevelCode = options.code; + + // Replace integrity hashes with updated values + if (options.integrityAlgorithm && options.runtimeData) { + for (const data of options.runtimeData) { + if (!data.integrity) { + continue; + } + + if (data.original && data.original.integrity) { + originalCode = originalCode.replace(data.integrity, data.original.integrity); + } + if (data.downlevel && data.downlevel.integrity) { + downlevelCode = downlevelCode.replace(data.integrity, data.downlevel.integrity); + } + } + } + + // Adjust lazy loaded scripts to point to the proper variant + // Extra spacing is intentional to align source line positions + downlevelCode = downlevelCode.replace('"-es2015.', ' "-es5.'); + + const downlevelFilePath = options.filename.replace('es2015', 'es5'); + let downlevelMap; + let result; + if (options.optimize) { + const minifiyResults = terserMangle(downlevelCode, { + filename: path.basename(downlevelFilePath), + map: options.map === undefined ? undefined : JSON.parse(options.map), + }); + downlevelCode = minifiyResults.code; + downlevelMap = JSON.stringify(minifiyResults.map); + + result = { + original: await mangleOriginal({ ...options, code: originalCode }), + downlevel: createFileEntry( + downlevelFilePath, + downlevelCode, + downlevelMap, + options.integrityAlgorithm, + ), + }; + } else { + if (options.map) { + const rawMap = JSON.parse(options.map) as RawSourceMap; + rawMap.file = path.basename(downlevelFilePath); + downlevelMap = JSON.stringify(rawMap); + } + + result = { + original: createFileEntry( + options.filename, + originalCode, + options.map, + options.integrityAlgorithm, + ), + downlevel: createFileEntry( + downlevelFilePath, + downlevelCode, + downlevelMap, + options.integrityAlgorithm, + ), + }; + } + + if (downlevelMap) { + await cachePut( + downlevelMap, + (options.cacheKeys && options.cacheKeys[CacheKey.DownlevelMap]) || null, + ); + fs.writeFileSync(downlevelFilePath + '.map', downlevelMap); + downlevelCode += `\n//# sourceMappingURL=${path.basename(downlevelFilePath)}.map`; + } + await cachePut( + downlevelCode, + (options.cacheKeys && options.cacheKeys[CacheKey.DownlevelCode]) || null, + ); + fs.writeFileSync(downlevelFilePath, downlevelCode); + + return result; +} diff --git a/yarn.lock b/yarn.lock index a064ff3134c6..532aac38b068 100644 --- a/yarn.lock +++ b/yarn.lock @@ -324,6 +324,11 @@ esutils "^2.0.2" js-tokens "^4.0.0" +"@babel/parser@^7.1.0": + version "7.6.2" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.6.2.tgz#205e9c95e16ba3b8b96090677a67c9d6075b70a1" + integrity sha512-mdFqWrSPCmikBoaBYMuBulzTIKuXVPtEISFbRRVNwMWpCms/hmE2kRq0bblUHaNRKrjRlmVbx1sDHmjmRgD2Xg== + "@babel/parser@^7.1.2": version "7.1.3" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.1.3.tgz#2c92469bac2b7fbff810b67fca07bd138b48af77" @@ -772,6 +777,15 @@ lodash "^4.17.13" to-fast-properties "^2.0.0" +"@babel/types@^7.3.0": + version "7.6.1" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.6.1.tgz#53abf3308add3ac2a2884d539151c57c4b3ac648" + integrity sha512-X7gdiuaCmA0uRjCmRtYJNAVCc/q+5xSgsfKJHqMN4iNLILX39677fJE1O40arPMh0TTtS9ItH67yre6c7k6t0g== + dependencies: + esutils "^2.0.2" + lodash "^4.17.13" + to-fast-properties "^2.0.0" + "@bazel/bazel-darwin_x64@0.28.1": version "0.28.1" resolved "https://registry.yarnpkg.com/@bazel/bazel-darwin_x64/-/bazel-darwin_x64-0.28.1.tgz#415658785e1dbd6f7ab5c8f2b98c1c99c614e1d5" @@ -940,6 +954,39 @@ resolved "https://registry.yarnpkg.com/@types/anymatch/-/anymatch-1.3.1.tgz#336badc1beecb9dacc38bea2cf32adf627a8421a" integrity sha512-/+CRPXpBDpo2RK9C68N3b2cOvO0Cf5B9aPijHsoDQTHivnGSObdOF2BRQOYjojWTDy6nQvMjmqRXIxH55VjxxA== +"@types/babel__core@7.1.3": + version "7.1.3" + resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.3.tgz#e441ea7df63cd080dfcd02ab199e6d16a735fc30" + integrity sha512-8fBo0UR2CcwWxeX7WIIgJ7lXjasFxoYgRnFHUj+hRvKkpiBJbxhdAPTCY6/ZKM0uxANFVzt4yObSLuTiTnazDA== + dependencies: + "@babel/parser" "^7.1.0" + "@babel/types" "^7.0.0" + "@types/babel__generator" "*" + "@types/babel__template" "*" + "@types/babel__traverse" "*" + +"@types/babel__generator@*": + version "7.6.0" + resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.0.tgz#f1ec1c104d1bb463556ecb724018ab788d0c172a" + integrity sha512-c1mZUu4up5cp9KROs/QAw0gTeHrw/x7m52LcnvMxxOZ03DmLwPV0MlGmlgzV3cnSdjhJOZsj7E7FHeioai+egw== + dependencies: + "@babel/types" "^7.0.0" + +"@types/babel__template@*": + version "7.0.2" + resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.0.2.tgz#4ff63d6b52eddac1de7b975a5223ed32ecea9307" + integrity sha512-/K6zCpeW7Imzgab2bLkLEbz0+1JlFSrUMdw7KoIIu+IUdu51GWaBZpd3y1VXGVXzynvGa4DaIaxNZHiON3GXUg== + dependencies: + "@babel/parser" "^7.1.0" + "@babel/types" "^7.0.0" + +"@types/babel__traverse@*": + version "7.0.7" + resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.0.7.tgz#2496e9ff56196cc1429c72034e07eab6121b6f3f" + integrity sha512-CeBpmX1J8kWLcDEnI3Cl2Eo6RfbGvzUctA+CjZUhOKDFbLfcr7fc4usEqLNWetrlJd7RhAkyYe2czXop4fICpw== + dependencies: + "@babel/types" "^7.3.0" + "@types/bluebird@*": version "3.5.26" resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.26.tgz#a38c438ae84fa02431d6892edf86e46edcbca291" @@ -11131,10 +11178,10 @@ terser-webpack-plugin@1.4.1, terser-webpack-plugin@^1.4.1: webpack-sources "^1.4.0" worker-farm "^1.7.0" -terser@4.1.4: - version "4.1.4" - resolved "https://registry.yarnpkg.com/terser/-/terser-4.1.4.tgz#4478b6a08bb096a61e793fea1a4434408bab936c" - integrity sha512-+ZwXJvdSwbd60jG0Illav0F06GDJF0R4ydZ21Q3wGAFKoBGyJGo34F63vzJHgvYxc1ukOtIjvwEvl9MkjzM6Pg== +terser@4.3.8: + version "4.3.8" + resolved "https://registry.yarnpkg.com/terser/-/terser-4.3.8.tgz#707f05f3f4c1c70c840e626addfdb1c158a17136" + integrity sha512-otmIRlRVmLChAWsnSFNO0Bfk6YySuBp6G9qrHiJwlLDd4mxe2ta4sjI7TzIR+W1nBMjilzrMcPOz9pSusgx3hQ== dependencies: commander "^2.20.0" source-map "~0.6.1"