From 9912926f319ff370d2a81b35bc24cd82bb21faa6 Mon Sep 17 00:00:00 2001 From: mc-zone Date: Thu, 3 Nov 2016 22:09:26 +0800 Subject: [PATCH 1/3] Add bundle-splitting support: Support output a manifest file to record bundle's modules when build a bundle. Support build bundle with a manifest file to skip existing modules. Add tests. --- local-cli/bundle/buildBundle.js | 7 + local-cli/bundle/bundleCommandLineArgs.js | 6 + local-cli/bundle/output/bundle.js | 23 +++- packager/react-packager/src/Bundler/Bundle.js | 24 ++++ .../src/Bundler/__tests__/Bundle-test.js | 96 +++++++++++++- .../src/Bundler/__tests__/Bundler-test.js | 122 +++++++++++++++++- packager/react-packager/src/Bundler/index.js | 42 +++++- packager/react-packager/src/Server/index.js | 4 + .../react-packager/src/lib/ModuleTransport.js | 2 + 9 files changed, 314 insertions(+), 12 deletions(-) diff --git a/local-cli/bundle/buildBundle.js b/local-cli/bundle/buildBundle.js index 327a9564e088..f3d41e024145 100644 --- a/local-cli/bundle/buildBundle.js +++ b/local-cli/bundle/buildBundle.js @@ -13,6 +13,7 @@ const Server = require('../../packager/react-packager/src/Server'); const outputBundle = require('./output/bundle'); const path = require('path'); +const fs = require('fs'); const saveAssets = require('./saveAssets'); const defaultAssetExts = require('../../packager/defaults').assetExts; @@ -58,6 +59,12 @@ function buildBundle(args, config, output = outputBundle, packagerInstance) { resetCache: args.resetCache, }; + if (typeof args.manifestFile === 'string') { + options.manifestReferrence = JSON.parse( + fs.readFileSync(args.manifestFile, 'utf-8') + ); + } + packagerInstance = new Server(options); shouldClosePackager = true; } diff --git a/local-cli/bundle/bundleCommandLineArgs.js b/local-cli/bundle/bundleCommandLineArgs.js index 41a3478d534a..1e80233074b8 100644 --- a/local-cli/bundle/bundleCommandLineArgs.js +++ b/local-cli/bundle/bundleCommandLineArgs.js @@ -40,6 +40,12 @@ module.exports = [ }, { command: '--sourcemap-output [string]', description: 'File name where to store the sourcemap file for resulting bundle, ex. /tmp/groups.map', + }, { + command: '--manifest-output [string]', + description: 'File name where to store the manifest file for bundle splitting, ex. ./output/base.manifest.json', + }, { + command: '--manifest-file [path]', + description: 'Path to the manifest file if want to split bundle, ex. ./output/base.manifest.json', }, { command: '--assets-dest [string]', description: 'Directory name where to store assets referenced in the bundle', diff --git a/local-cli/bundle/output/bundle.js b/local-cli/bundle/output/bundle.js index 34d1fa71a1b0..3efc07bb3e29 100644 --- a/local-cli/bundle/output/bundle.js +++ b/local-cli/bundle/output/bundle.js @@ -31,7 +31,8 @@ function saveBundleAndMap(bundle, options, log) { bundleOutput, bundleEncoding: encoding, dev, - sourcemapOutput + sourcemapOutput, + manifestOutput } = options; log('start'); @@ -49,14 +50,28 @@ function saveBundleAndMap(bundle, options, log) { Promise.all([writeBundle, writeMetadata]) .then(() => log('Done writing bundle output')); + const writeTasks = [writeBundle]; + if (sourcemapOutput) { log('Writing sourcemap output to:', sourcemapOutput); const writeMap = writeFile(sourcemapOutput, codeWithMap.map, null); writeMap.then(() => log('Done writing sourcemap output')); - return Promise.all([writeBundle, writeMetadata, writeMap]); - } else { - return writeBundle; + writeTasks.push(writeMetadata, writeMap); + } + + if (manifestOutput) { + log('Writing manifest output to:', manifestOutput); + const manifest = createBundleManifest(bundle); + const writeManifest = writeFile(manifestOutput, manifest, null); + writeManifest.then(() => log('Done writing manifest output')); + writeTasks.push(writeManifest); } + + return Promise.all(writeTasks); +} + +function createBundleManifest(bundle) { + return JSON.stringify(bundle.getManifest(), null, 2); } exports.build = buildBundle; diff --git a/packager/react-packager/src/Bundler/Bundle.js b/packager/react-packager/src/Bundler/Bundle.js index 7bce385e2389..4e9d59f15002 100644 --- a/packager/react-packager/src/Bundler/Bundle.js +++ b/packager/react-packager/src/Bundler/Bundle.js @@ -109,6 +109,7 @@ class Bundle extends BundleBase { sourceCode: code, sourcePath: name + '.js', meta: {preloaded: true}, + isRequireCall: true, })); this._numRequireCalls += 1; } @@ -229,6 +230,29 @@ class Bundle extends BundleBase { return eTag; } + getManifest() { + const modules = this.getModules(); + const manifest = { + modules: {}, + lastId:0 + }; + modules.forEach(module => { + // Filter out polyfills and requireCalls + if (module.name && !module.isPolyfill && !module.isRequireCall ) { + manifest.modules[module.name] = { + id: module.id, + }; + } + if (typeof module.id === 'number' && typeof manifest.lastId === 'number') { + manifest.lastId = Math.max(manifest.lastId, module.id); + } else { + manifest.lastId = module.id; + } + }); + + return manifest; + } + _getSourceMapFile() { return this._sourceMapUrl ? this._sourceMapUrl.replace('.map', '.bundle') diff --git a/packager/react-packager/src/Bundler/__tests__/Bundle-test.js b/packager/react-packager/src/Bundler/__tests__/Bundle-test.js index 9aef2b35d6a8..96f1afdc7cbf 100644 --- a/packager/react-packager/src/Bundler/__tests__/Bundle-test.js +++ b/packager/react-packager/src/Bundler/__tests__/Bundle-test.js @@ -316,6 +316,84 @@ describe('Bundle', () => { }); }); + describe('manifest bundle', () => { + pit('exclude polyfills and requireCalls in manifest', () => { + const modules = [ + { + bundle: bundle, + id:0, + code: 'foo', + sourceCode: 'foo', + sourcePath: 'foo path', + name: 'path/foo.js', + }, + { + bundle: bundle, + id:'prefix-1', + code: 'bar', + sourceCode: 'bar', + sourcePath: 'bar path', + name: 'path/bar.js', + }, + { + bundle: bundle, + id:2, + code: 'foobar', + sourceCode: 'foobar', + sourcePath: 'foobar path', + name: 'path/foobar.js', + }, + { + bundle: bundle, + id:3, + code: 'uname module', + sourceCode: 'uname module', + sourcePath: '', + }, + { + bundle: bundle, + id:4, + code: 'polyfill module', + sourceCode: 'polyfill module', + sourcePath: '', + isPolyfill: true, + }, + { + bundle: bundle, + id:5, + code: ';require(0);', + sourceCode: ';require(0);', + sourcePath: '', + isRequireCall: true, + } + ]; + + return Promise.all(modules.map(module => addModule(module))).then(() => { + }).then(() => { + bundle.setMainModuleId(0); + bundle.finalize({ + runBeforeMainModule: [], + runMainModule: true, + }); + const manifest = bundle.getManifest(); + expect(manifest).toEqual({ + modules:{ + 'path/foo.js': { + id: 0, + }, + 'path/bar.js': { + id: 'prefix-1', + }, + 'path/foobar.js': { + id: 2 + } + }, + lastId:5 + }); + }); + }); + }); + describe('main module id:', function() { it('can save a main module ID', function() { const id = 'arbitrary module ID'; @@ -483,7 +561,20 @@ function resolverFor(code, map) { }; } -function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill, meta, id = ''}) { +function addModule({ + bundle, + code, + sourceCode, + sourcePath, + map, + virtual, + polyfill, + meta, + id = '', + name, + isPolyfill, + isRequireCall, +}) { return bundle.addModule( resolverFor(code, map), null, @@ -497,6 +588,9 @@ function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill meta, virtual, polyfill, + name, + isPolyfill, + isRequireCall, }), ); } diff --git a/packager/react-packager/src/Bundler/__tests__/Bundler-test.js b/packager/react-packager/src/Bundler/__tests__/Bundler-test.js index a74d9e5d484e..6309596d8a8e 100644 --- a/packager/react-packager/src/Bundler/__tests__/Bundler-test.js +++ b/packager/react-packager/src/Bundler/__tests__/Bundler-test.js @@ -28,6 +28,7 @@ jest .mock('../../lib/declareOpts'); var Bundler = require('../'); +var Bundle = require('../Bundle'); var Resolver = require('../../Resolver'); var sizeOf = require('image-size'); var fs = require('fs'); @@ -36,6 +37,7 @@ describe('Bundler', function() { function createModule({ path, + name, id, dependencies, isAsset, @@ -48,7 +50,7 @@ describe('Bundler', function() { path, resolution, getDependencies: () => Promise.resolve(dependencies), - getName: () => Promise.resolve(id), + getName: () => Promise.resolve(name ? name : id), isJSON: () => isJSON, isAsset: () => isAsset, isAsset_DEPRECATED: () => isAsset_DEPRECATED, @@ -346,4 +348,122 @@ describe('Bundler', function() { ])); }); }); + + + describe('bundle with manifest reference', () => { + var otherBundler; + var doBundle; + var getModuleIdInResolver; + var refModule = {path: '/root/ref.js', name: '/root/ref.js', dependencies: []}; + var moduleFoo = {path: '/root/foo.js', name: '/root/foo.js', dependencies: []}; + var moduleBar = {path: '/root/bar.js', name: '/root/bar.js', dependencies: []}; + var manifestReferrence = { + modules: { + [refModule.name]: { + id: 456 + } + }, + lastId: 10, + }; + + beforeEach(function() { + fs.statSync.mockImpl(function() { + return { + isDirectory: () => true + }; + }); + + Resolver.mockImpl(function() { + return { + getDependencies: ( + entryFile, + options, + transformOptions, + onProgress, + getModuleId + ) => { + getModuleIdInResolver = getModuleId; + return Promise.resolve({ + mainModuleId: 0, + dependencies: [ + createModule(refModule), + createModule(moduleFoo), + createModule(moduleBar), + createModule({path: '', name: 'aPolyfill.js', dependencies: [], isPolyfill:true}), + createModule({path: '', name: 'anotherPolyfill.js', dependencies: [], isPolyfill:true}), + ], + transformOptions, + getModuleId, + getResolvedDependencyPairs: () => [], + }) + }, + getModuleSystemDependencies: () => [], + wrapModule: options => Promise.resolve(options), + }; + }); + + Bundle.mockImpl(function() { + const _modules = []; + + return { + setRamGroups: jest.fn(), + setMainModuleId: jest.fn(), + finalize: jest.fn(), + getModules: () => { + return _modules; + }, + addModule: (resolver, resolutionResponse, module, moduleTransport) => { + return _modules.push({...moduleTransport}) - 1; + } + } + }); + + otherBundler = new Bundler({ + projectRoots: ['/root'], + assetServer: { + getAssetData: jest.fn(), + }, + manifestReferrence, + }); + + doBundle = otherBundler.bundle({ + entryFile: '/root/foo.js', + runBeforeMainModule: [], + runModule: true, + }); + }); + + it('skip dependencies that exist in the manifest reference', function() { + return doBundle.then(bundle => { + const modulesNames = bundle.getModules().map(module => module.name); + + expect(modulesNames.indexOf(refModule.name)).toBe(-1); + expect(modulesNames.indexOf(moduleFoo.name)).not.toBe(-1); + expect(modulesNames.indexOf(moduleBar.name)).not.toBe(-1); + }); + }); + + it('skip polyfills if used manifest reference', function() { + return doBundle.then(bundle => { + expect(bundle.getModules().some(module => module.name == 'somePolyfill.js')).toBeFalsy(); + expect(bundle.getModules().some(module => module.isPolyfill)).toBeFalsy(); + }); + }); + + it('get the moduleId from manifest reference that if module was already exists in the manifest', function() { + const refModuleReference = manifestReferrence.modules[refModule.name]; + return doBundle.then(bundle => { + expect(getModuleIdInResolver(refModule)).toBe(refModuleReference.id); + expect(otherBundler._getModuleId(refModule)).toBe(refModuleReference.id); + }); + }); + + it('create new moduleId should bigger than the lastId in the manifest', function() { + return doBundle.then(bundle => { + bundle.getModules().forEach(module => { + expect(module.id).toBeGreaterThan(manifestReferrence.lastId); + }); + }); + }); + }); }); diff --git a/packager/react-packager/src/Bundler/index.js b/packager/react-packager/src/Bundler/index.js index d1c2dbc6dedd..7cdeede90c51 100644 --- a/packager/react-packager/src/Bundler/index.js +++ b/packager/react-packager/src/Bundler/index.js @@ -89,6 +89,10 @@ const validateOpts = declareOpts({ type: 'boolean', default: false, }, + manifestReferrence: { + type: 'object', + required: false, + }, }); const assetPropertyBlacklist = new Set([ @@ -120,7 +124,14 @@ class Bundler { mtime, ]; - this._getModuleId = createModuleIdFactory(); + const manifest = opts.manifestReferrence || null; + this._startModuleId = manifest ? (1 + manifest.lastId) : 0; + this._extenalModules = manifest ? manifest.modules : null; + + this._getModuleId = createModuleIdFactory({ + extenalModules: this._extenalModules, + startId: this._startModuleId, + }); if (opts.transformModulePath) { const transformer = require(opts.transformModulePath); @@ -264,6 +275,11 @@ class Bundler { response.dependencies = response.dependencies.filter(module => module.path.endsWith(entryFile) ); + } else if (this._extenalModules) { + /* If used extenal reference, we don't need polyfills again */ + response.dependencies = response.dependencies.filter(module => + module.name && !module.isPolyfill() && !this._extenalModules[module.name] + ); } else { response.dependencies = moduleSystemDeps.concat(response.dependencies); } @@ -390,6 +406,12 @@ class Bundler { } return Promise.resolve(resolutionResponse).then(response => { + return Promise.all(response.dependencies.map(module => + module.getName().then(name => { + module.name = name; + }) + )).then(() => response); + }).then(response => { bundle.setRamGroups(response.transformOptions.transform.ramGroups); Activity.endEvent(findEventId); @@ -511,7 +533,10 @@ class Bundler { {dev, platform, recursive}, transformOptions, onProgress, - isolateModuleIDs ? createModuleIdFactory() : this._getModuleId, + isolateModuleIDs ? createModuleIdFactory({ + extenalModules: this._extenalModules, + startId: this._startModuleId, + }) : this._getModuleId, ); }); } @@ -591,7 +616,8 @@ class Bundler { map, meta: {dependencies, dependencyOffsets, preloaded, dependencyPairs}, sourceCode: source, - sourcePath: module.path + sourcePath: module.path, + isPolyfill: module.isPolyfill() }); }); } @@ -748,10 +774,14 @@ function verifyRootExists(root) { assert(fs.statSync(root).isDirectory(), 'Root has to be a valid directory'); } -function createModuleIdFactory() { +function createModuleIdFactory({extenalModules, startId: nextId = 0}) { const fileToIdMap = Object.create(null); - let nextId = 0; - return ({path}) => { + return ({path, name}) => { + if (extenalModules && name) { + if (name in extenalModules) { + return extenalModules[name].id; + } + } if (!(path in fileToIdMap)) { fileToIdMap[path] = nextId; nextId += 1; diff --git a/packager/react-packager/src/Server/index.js b/packager/react-packager/src/Server/index.js index d5da01dd1d18..32ebdd3a8ec0 100644 --- a/packager/react-packager/src/Server/index.js +++ b/packager/react-packager/src/Server/index.js @@ -95,6 +95,10 @@ const validateOpts = declareOpts({ type: 'boolean', default: false, }, + manifestReferrence: { + type: 'object', + required: false, + }, }); const bundleOpts = declareOpts({ diff --git a/packager/react-packager/src/lib/ModuleTransport.js b/packager/react-packager/src/lib/ModuleTransport.js index e5718e2dcae2..8ca44f68ae8b 100644 --- a/packager/react-packager/src/lib/ModuleTransport.js +++ b/packager/react-packager/src/lib/ModuleTransport.js @@ -26,6 +26,8 @@ function ModuleTransport(data) { this.virtual = data.virtual; this.meta = data.meta; this.polyfill = data.polyfill; + this.isPolyfill = data.isPolyfill; + this.isRequireCall = data.isRequireCall; this.map = data.map; Object.freeze(this); From 4a5d25fd65c0c5b3d64e7b8532c8ced12cdb7f07 Mon Sep 17 00:00:00 2001 From: mc-zone Date: Sat, 12 Nov 2016 10:51:36 +0800 Subject: [PATCH 2/3] Repair errors come from the flow check --- packager/react-packager/src/Bundler/Bundle.js | 9 ++--- packager/react-packager/src/Bundler/index.js | 33 ++++++++++++------- .../react-packager/src/lib/ModuleTransport.js | 4 +-- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/packager/react-packager/src/Bundler/Bundle.js b/packager/react-packager/src/Bundler/Bundle.js index bf8916b25b1a..07fd78e63c95 100644 --- a/packager/react-packager/src/Bundler/Bundle.js +++ b/packager/react-packager/src/Bundler/Bundle.js @@ -239,9 +239,12 @@ class Bundle extends BundleBase { getManifest() { const modules = this.getModules(); - const manifest = { + const manifest: { + modules: Object, + lastId: number, + } = { modules: {}, - lastId:0 + lastId:0, }; modules.forEach(module => { // Filter out polyfills and requireCalls @@ -252,8 +255,6 @@ class Bundle extends BundleBase { } if (typeof module.id === 'number' && typeof manifest.lastId === 'number') { manifest.lastId = Math.max(manifest.lastId, module.id); - } else { - manifest.lastId = module.id; } }); diff --git a/packager/react-packager/src/Bundler/index.js b/packager/react-packager/src/Bundler/index.js index be5780180109..ac0ab6a4fed5 100644 --- a/packager/react-packager/src/Bundler/index.js +++ b/packager/react-packager/src/Bundler/index.js @@ -147,6 +147,8 @@ class Bundler { _projectRoots: Array; _assetServer: AssetServer; _transformOptionsModule: (path: string, options: {}, bunler: Bundler) => {}; + _startModuleId: number; + _extenalModules: ?Object; constructor(options: Options) { const opts = this._opts = validateOpts(options); @@ -169,9 +171,14 @@ class Bundler { mtime, ]; - const manifest = opts.manifestReferrence || null; - this._startModuleId = manifest ? (1 + manifest.lastId) : 0; - this._extenalModules = manifest ? manifest.modules : null; + const manifest = opts.manifestReferrence; + if (manifest) { + this._startModuleId = 1 + manifest.lastId; + this._extenalModules = manifest.modules; + } else { + this._startModuleId = 0; + this._extenalModules = null; + } this._getModuleId = createModuleIdFactory({ extenalModules: this._extenalModules, @@ -335,9 +342,10 @@ class Bundler { module.path.endsWith(entryFile) ); } else if (this._extenalModules) { + const extenals = this._extenalModules; /* If used extenal reference, we don't need polyfills again */ response.dependencies = response.dependencies.filter(module => - module.name && !module.isPolyfill() && !this._extenalModules[module.name] + module.name && !module.isPolyfill() && !extenals[module.name] ); } else { response.dependencies = moduleSystemDeps.concat(response.dependencies); @@ -828,17 +836,20 @@ function verifyRootExists(root) { function createModuleIdFactory({extenalModules, startId: nextId = 0}) { const fileToIdMap = Object.create(null); - return ({path, name}) => { - if (extenalModules && name) { - if (name in extenalModules) { - return extenalModules[name].id; + return (module: { + path: string, + name?: "string" + }) => { + if (extenalModules && module.name) { + if (module.name in extenalModules) { + return extenalModules[module.name].id; } } - if (!(path in fileToIdMap)) { - fileToIdMap[path] = nextId; + if (!(module.path in fileToIdMap)) { + fileToIdMap[module.path] = nextId; nextId += 1; } - return fileToIdMap[path]; + return fileToIdMap[module.path]; }; } diff --git a/packager/react-packager/src/lib/ModuleTransport.js b/packager/react-packager/src/lib/ModuleTransport.js index 0991a5918ddd..cd7b7537d9d4 100644 --- a/packager/react-packager/src/lib/ModuleTransport.js +++ b/packager/react-packager/src/lib/ModuleTransport.js @@ -42,8 +42,8 @@ class ModuleTransport { meta?: ?Metadata, polyfill?: ?boolean, map?: ?SourceMap, - isPolyfill: ?boolean, - isRequireCall: ?boolean, + isPolyfill?: ?boolean, + isRequireCall?: ?boolean, }) { this.name = data.name; From 92b1b1c5e0e3459ccc7f8900fafbb2780e06f268 Mon Sep 17 00:00:00 2001 From: mc-zone Date: Sun, 25 Dec 2016 17:48:18 +0800 Subject: [PATCH 3/3] Repair Flow errors --- local-cli/bundle/types.flow.js | 1 + packager/react-packager/src/Bundler/index.js | 15 +++++++-------- packager/react-packager/src/node-haste/Module.js | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/local-cli/bundle/types.flow.js b/local-cli/bundle/types.flow.js index d0768fef8549..5d25f95f8c3f 100644 --- a/local-cli/bundle/types.flow.js +++ b/local-cli/bundle/types.flow.js @@ -36,6 +36,7 @@ export type OutputOptions = { dev?: boolean, platform: string, sourcemapOutput?: string, + manifestOutput?: string, }; export type RequestOptions = {| diff --git a/packager/react-packager/src/Bundler/index.js b/packager/react-packager/src/Bundler/index.js index 44243ddb28f0..4d70cc5b27e9 100644 --- a/packager/react-packager/src/Bundler/index.js +++ b/packager/react-packager/src/Bundler/index.js @@ -448,16 +448,15 @@ class Bundler { minify, isolateModuleIDs, generateSourceMaps: unbundle || generateSourceMaps, - }); - } - - return Promise.resolve(resolutionResponse).then(response => { - return Promise.all(response.dependencies.map(module => + }).then(response => Promise.all(response.dependencies.map(module => module.getName().then(name => { module.name = name; }) - )).then(() => response); - }).then(response => { + )).then(() => response)); + } + + + return Promise.resolve(resolutionResponse).then(response => { bundle.setRamGroups(response.transformOptions.transform.ramGroups); log(createActionEndEntry(transformingFilesLogEntry)); @@ -834,7 +833,7 @@ function createModuleIdFactory({extenalModules, startId: nextId = 0}) { const fileToIdMap = Object.create(null); return (module: { path: string, - name?: "string" + name: string }) => { if (extenalModules && module.name) { if (module.name in extenalModules) { diff --git a/packager/react-packager/src/node-haste/Module.js b/packager/react-packager/src/node-haste/Module.js index b4479ee07863..94c4f8769271 100644 --- a/packager/react-packager/src/node-haste/Module.js +++ b/packager/react-packager/src/node-haste/Module.js @@ -65,6 +65,7 @@ class Module { path: string; type: string; + name: string; _moduleCache: ModuleCache; _cache: Cache;