diff --git a/Makefile b/Makefile index 14fbc3e427452e..2036740f872aee 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,6 @@ endif # otherwise $(NODE_EXE) being a .PHONY target means it is always re-run. # Without the check there is a race condition between the link being deleted # and recreated which can break the addons build when running test-ci -# See comments on the build-addons target for some more info $(NODE_EXE): config.gypi out/Makefile $(MAKE) -C out BUILDTYPE=Release V=$(V) if [ ! -r $@ -o ! -L $@ ]; then ln -fs out/Release/$(NODE_EXE) $@; fi @@ -193,7 +192,6 @@ v8: test: all $(MAKE) build-addons - $(MAKE) build-addons-napi $(MAKE) cctest $(PYTHON) tools/test.py --mode=release -J \ doctool inspector known_issues message pseudo-tty parallel sequential $(CI_NATIVE_SUITES) @@ -205,98 +203,9 @@ test-parallel: all test-valgrind: all $(PYTHON) tools/test.py --mode=release --valgrind sequential parallel message -# Implicitly depends on $(NODE_EXE). We don't depend on it explicitly because -# it always triggers a rebuild due to it being a .PHONY rule. See the comment -# near the build-addons rule for more background. -test/gc/build/Release/binding.node: test/gc/binding.cc test/gc/binding.gyp - $(NODE) deps/npm/node_modules/node-gyp/bin/node-gyp rebuild \ - --python="$(PYTHON)" \ - --directory="$(shell pwd)/test/gc" \ - --nodedir="$(shell pwd)" - -# Implicitly depends on $(NODE_EXE), see the build-addons rule for rationale. -DOCBUILDSTAMP_PREREQS = tools/doc/addon-verify.js doc/api/addons.md - -ifeq ($(OSTYPE),aix) -DOCBUILDSTAMP_PREREQS := $(DOCBUILDSTAMP_PREREQS) out/$(BUILDTYPE)/node.exp -endif - -test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS) - $(RM) -r test/addons/??_*/ - $(NODE) $< - touch $@ - -ADDONS_BINDING_GYPS := \ - $(filter-out test/addons/??_*/binding.gyp, \ - $(wildcard test/addons/*/binding.gyp)) - -ADDONS_BINDING_SOURCES := \ - $(filter-out test/addons/??_*/*.cc, $(wildcard test/addons/*/*.cc)) \ - $(filter-out test/addons/??_*/*.h, $(wildcard test/addons/*/*.h)) - -# Implicitly depends on $(NODE_EXE), see the build-addons rule for rationale. -# Depends on node-gyp package.json so that build-addons is (re)executed when -# node-gyp is updated as part of an npm update. -test/addons/.buildstamp: config.gypi \ - deps/npm/node_modules/node-gyp/package.json \ - $(ADDONS_BINDING_GYPS) $(ADDONS_BINDING_SOURCES) \ - deps/uv/include/*.h deps/v8/include/*.h \ - src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \ - test/addons/.docbuildstamp -# Cannot use $(wildcard test/addons/*/) here, it's evaluated before -# embedded addons have been generated from the documentation. - @for dirname in test/addons/*/; do \ - printf "\nBuilding addon $$PWD/$$dirname\n" ; \ - env MAKEFLAGS="-j1" $(NODE) deps/npm/node_modules/node-gyp/bin/node-gyp \ - --loglevel=$(LOGLEVEL) rebuild \ - --python="$(PYTHON)" \ - --directory="$$PWD/$$dirname" \ - --nodedir="$$PWD" || exit 1 ; \ - done - touch $@ - -# .buildstamp and .docbuildstamp need $(NODE_EXE) but cannot depend on it -# directly because it calls make recursively. The parent make cannot know -# if the subprocess touched anything so it pessimistically assumes that -# .buildstamp and .docbuildstamp are out of date and need a rebuild. -# Just goes to show that recursive make really is harmful... -# TODO(bnoordhuis) Force rebuild after gyp update. -build-addons: $(NODE_EXE) test/addons/.buildstamp - -ADDONS_NAPI_BINDING_GYPS := \ - $(filter-out test/addons-napi/??_*/binding.gyp, \ - $(wildcard test/addons-napi/*/binding.gyp)) - -ADDONS_NAPI_BINDING_SOURCES := \ - $(filter-out test/addons-napi/??_*/*.cc, $(wildcard test/addons-napi/*/*.cc)) \ - $(filter-out test/addons-napi/??_*/*.h, $(wildcard test/addons-napi/*/*.h)) - -# Implicitly depends on $(NODE_EXE), see the build-addons-napi rule for rationale. -test/addons-napi/.buildstamp: config.gypi \ - deps/npm/node_modules/node-gyp/package.json \ - $(ADDONS_NAPI_BINDING_GYPS) $(ADDONS_NAPI_BINDING_SOURCES) \ - deps/uv/include/*.h deps/v8/include/*.h \ - src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \ - src/node_api.h src/node_api_types.h -# Cannot use $(wildcard test/addons-napi/*/) here, it's evaluated before -# embedded addons have been generated from the documentation. - @for dirname in test/addons-napi/*/; do \ - printf "\nBuilding addon $$PWD/$$dirname\n" ; \ - env MAKEFLAGS="-j1" $(NODE) deps/npm/node_modules/node-gyp/bin/node-gyp \ - --loglevel=$(LOGLEVEL) rebuild \ - --python="$(PYTHON)" \ - --directory="$$PWD/$$dirname" \ - --nodedir="$$PWD" || exit 1 ; \ - done - touch $@ - -# .buildstamp and .docbuildstamp need $(NODE_EXE) but cannot depend on it -# directly because it calls make recursively. The parent make cannot know -# if the subprocess touched anything so it pessimistically assumes that -# .buildstamp and .docbuildstamp are out of date and need a rebuild. -# Just goes to show that recursive make really is harmful... -# TODO(bnoordhuis) Force rebuild after gyp or node-gyp update. -build-addons-napi: $(NODE_EXE) test/addons-napi/.buildstamp +# Builds test/addons, test/addons-napi and test/gc. +build-addons: $(NODE_EXE) + ./$< tools/build-addons.js clear-stalled: # Clean up any leftover processes but don't error if found. @@ -306,17 +215,15 @@ clear-stalled: echo $${PS_OUT} | xargs kill; \ fi -test-gc: all test/gc/build/Release/binding.node +test-gc: test-build $(PYTHON) tools/test.py --mode=release gc test-gc-clean: $(RM) -r test/gc/build -test-build: | all build-addons build-addons-napi +test-build: | all build-addons -test-build-addons-napi: all build-addons-napi - -test-all: test-build test/gc/build/Release/binding.node +test-all: test-build $(PYTHON) tools/test.py --mode=debug,release test-all-valgrind: test-build @@ -327,7 +234,7 @@ CI_JS_SUITES := doctool inspector known_issues message parallel pseudo-tty seque # Build and test addons without building anything else test-ci-native: LOGLEVEL := info -test-ci-native: | test/addons/.buildstamp test/addons-napi/.buildstamp +test-ci-native: | build-addons $(PYTHON) tools/test.py $(PARALLEL_ARGS) -p tap --logfile test.tap \ --mode=release --flaky-tests=$(FLAKY_TESTS) \ $(TEST_CI_ARGS) $(CI_NATIVE_SUITES) @@ -345,7 +252,7 @@ test-ci-js: | clear-stalled fi test-ci: LOGLEVEL := info -test-ci: | clear-stalled build-addons build-addons-napi +test-ci: | clear-stalled build-addons out/Release/cctest --gtest_output=tap:cctest.tap $(PYTHON) tools/test.py $(PARALLEL_ARGS) -p tap --logfile test.tap \ --mode=release --flaky-tests=$(FLAKY_TESTS) \ @@ -397,16 +304,17 @@ test-npm: $(NODE_EXE) test-npm-publish: $(NODE_EXE) npm_package_config_publishtest=true $(NODE) deps/npm/test/run.js -test-addons-napi: test-build-addons-napi - $(PYTHON) tools/test.py --mode=release addons-napi - -test-addons: test-build test-addons-napi +test-addons: test-build $(PYTHON) tools/test.py --mode=release addons +test-addons-napi: test-build + $(PYTHON) tools/test.py --mode=release addons-napi + test-addons-clean: $(RM) -rf test/addons/??_*/ $(RM) -rf test/addons/*/build - $(RM) test/addons/.buildstamp test/addons/.docbuildstamp + $(RM) -rf test/addons/Release/ + $(RM) -rf test/addons/include/ test-timers: $(MAKE) --directory=tools faketime @@ -934,7 +842,6 @@ endif blog \ blogclean \ build-addons \ - build-addons-napi \ build-ci \ cctest \ check \ diff --git a/test/addons-napi/.gitignore b/test/addons-napi/.gitignore index bde1cf3ab9662b..ff6b007943190d 100644 --- a/test/addons-napi/.gitignore +++ b/test/addons-napi/.gitignore @@ -1,5 +1,3 @@ -.buildstamp -.docbuildstamp Makefile *.Makefile *.mk diff --git a/test/addons/.gitignore b/test/addons/.gitignore index bde1cf3ab9662b..0c54a4bd8cc2ef 100644 --- a/test/addons/.gitignore +++ b/test/addons/.gitignore @@ -1,7 +1,7 @@ -.buildstamp -.docbuildstamp Makefile *.Makefile *.mk gyp-mac-tool /*/build +/Release/ +/include/ diff --git a/test/addons/openssl-binding/binding.gyp b/test/addons/openssl-binding/binding.gyp index bafde41348ce3a..f7867c46c275da 100644 --- a/test/addons/openssl-binding/binding.gyp +++ b/test/addons/openssl-binding/binding.gyp @@ -5,7 +5,6 @@ 'conditions': [ ['node_use_openssl=="true"', { 'sources': ['binding.cc'], - 'include_dirs': ['../../../deps/openssl/openssl/include'], }] ] }, diff --git a/test/addons/zlib-binding/binding.gyp b/test/addons/zlib-binding/binding.gyp index 60a9bb82661820..751bb8325f0f58 100644 --- a/test/addons/zlib-binding/binding.gyp +++ b/test/addons/zlib-binding/binding.gyp @@ -3,7 +3,6 @@ { 'target_name': 'binding', 'sources': ['binding.cc'], - 'include_dirs': ['../../../deps/zlib'], }, ] } diff --git a/tools/build-addons.js b/tools/build-addons.js new file mode 100644 index 00000000000000..b5dfac4c5333d4 --- /dev/null +++ b/tools/build-addons.js @@ -0,0 +1,92 @@ +'use strict'; + +const fs = require('fs'); +const os = require('os'); +const { spawn, spawnSync } = require('child_process'); +const { resolve } = require('path'); + +const kTopLevelDirectory = resolve(__dirname, '..'); +const kAddonsDirectory = resolve(kTopLevelDirectory, 'test/addons'); +const kNapiAddonsDirectory = resolve(kTopLevelDirectory, 'test/addons-napi'); + +// Location where the headers are installed to. +const kIncludeDirectory = kAddonsDirectory; + +const kPython = process.env.PYTHON || 'python'; +const kNodeGyp = + resolve(kTopLevelDirectory, 'deps/npm/node_modules/node-gyp/bin/node-gyp'); + +process.chdir(kTopLevelDirectory); + +// Copy headers to `${kIncludeDirectory}/include`. install.py preserves +// timestamps so this won't cause unnecessary rebuilds. +{ + const args = [ 'tools/install.py', 'install', kIncludeDirectory, '/' ]; + const env = Object.assign({}, process.env); + env.HEADERS_ONLY = 'yes, please'; // Ask nicely. + env.LOGLEVEL = 'WARNING'; + + const options = { env, stdio: 'inherit' }; + const result = spawnSync(kPython, args, options); + if (result.status !== 0) process.exit(1); +} + +// Scrape embedded add-ons from doc/api/addons.md. +require('./doc/addon-verify.js'); + +// Regenerate build files and rebuild if necessary. +let failures = 0; +process.on('exit', () => process.exit(failures > 0)); + +const jobs = []; + +// test/gc contains a single add-on to build. +{ + const path = resolve(kTopLevelDirectory, 'test/gc'); + exec(path, 'configure', () => exec(path, 'build')); +} + +for (const directory of [kAddonsDirectory, kNapiAddonsDirectory]) { + for (const basedir of fs.readdirSync(directory)) { + const path = resolve(directory, basedir); + const gypfile = resolve(path, 'binding.gyp'); + if (!fs.existsSync(gypfile)) continue; + exec(path, 'configure', () => exec(path, 'build')); + } +} + +// FIXME(bnoordhuis) I would have liked to derive the desired level of +// parallelism from MAKEFLAGS but it's missing the actual -j flag. +for (const _ of os.cpus()) next(); // eslint-disable-line no-unused-vars + +function next() { + const job = jobs.shift(); + if (job) job(); +} + +function exec(path, command, done) { + jobs.push(() => { + if (failures > 0) return; + + const args = [kNodeGyp, + '--loglevel=silent', + '--directory=' + path, + '--nodedir=' + kIncludeDirectory, + '--python=' + kPython, + command]; + + const env = Object.assign({}, process.env); + env.MAKEFLAGS = '-j1'; + + const options = { env, stdio: 'inherit' }; + const proc = spawn(process.execPath, args, options); + + proc.on('exit', (exitCode) => { + if (exitCode !== 0) ++failures; + if (done) done(); + next(); + }); + + console.log(command, path); + }); +} diff --git a/tools/doc/addon-verify.js b/tools/doc/addon-verify.js index 86a81935892745..aee24cac315a3d 100644 --- a/tools/doc/addon-verify.js +++ b/tools/doc/addon-verify.js @@ -1,5 +1,6 @@ 'use strict'; +const assert = require('assert'); const fs = require('fs'); const path = require('path'); const marked = require('marked'); @@ -20,13 +21,8 @@ tokens.push({ type: 'heading' }); for (var i = 0; i < tokens.length; i++) { var token = tokens[i]; if (token.type === 'heading' && token.text) { - const blockName = token.text; - if (files && Object.keys(files).length !== 0) { - verifyFiles(files, - blockName, - console.log.bind(null, 'wrote'), - function(err) { if (err) throw err; }); - } + if (files && Object.keys(files).length !== 0) + verifyFiles(files, token.text); files = {}; } else if (token.type === 'code') { var match = token.text.match(/^\/\/\s+(.*\.(?:cc|h|js))[\r\n]/); @@ -36,17 +32,7 @@ for (var i = 0; i < tokens.length; i++) { } } -function once(fn) { - var once = false; - return function() { - if (once) - return; - once = true; - fn.apply(this, arguments); - }; -} - -function verifyFiles(files, blockName, onprogress, ondone) { +function verifyFiles(files, blockName) { // must have a .cc and a .js to be a valid test if (!Object.keys(files).some((name) => /\.cc$/.test(name)) || !Object.keys(files).some((name) => /\.js$/.test(name))) { @@ -91,22 +77,24 @@ ${files[name].replace('Release', "' + common.buildType + '")} }) }); - fs.mkdir(dir, function() { - // Ignore errors + try { + fs.mkdirSync(dir); + } catch (e) { + assert.strictEqual(e.code, 'EEXIST'); + } - var done = once(ondone); - var waiting = files.length; - files.forEach(function(file) { - fs.writeFile(file.path, file.content, function(err) { - if (err) - return done(err); + for (const file of files) { + try { + var content = fs.readFileSync(file.path); + } catch (e) { + assert.strictEqual(e.code, 'ENOENT'); + } - if (onprogress) - onprogress(file.path); + // Only update when file content has changed to prevent unneeded rebuilds. + if (content && content.toString() === file.content) + continue; - if (--waiting === 0) - done(); - }); - }); - }); + fs.writeFileSync(file.path, file.content); + console.log('wrote', file.path); + } } diff --git a/tools/install.py b/tools/install.py index d51ac06d7b10dd..b9b96d95b0bffc 100755 --- a/tools/install.py +++ b/tools/install.py @@ -2,13 +2,18 @@ import errno import json +import logging import os import re import shutil import sys from getmoduleversion import get_version +logging.basicConfig(format='%(message)s', + level=os.environ.get('LOGLEVEL', 'INFO')) + # set at init time +headers_only = os.environ.get('HEADERS_ONLY') node_prefix = '/usr/local' # PREFIX variable from Makefile install_path = None # base target directory (DESTDIR + PREFIX from Makefile) target_defaults = None @@ -31,7 +36,7 @@ def try_unlink(path): if e.errno != errno.ENOENT: raise def try_symlink(source_path, link_path): - print 'symlinking %s -> %s' % (source_path, link_path) + logging.info('symlinking %s -> %s', source_path, link_path) try_unlink(link_path) os.symlink(source_path, link_path) @@ -61,14 +66,15 @@ def mkpaths(path, dst): def try_copy(path, dst): source_path, target_path = mkpaths(path, dst) - print 'installing %s' % target_path + logging.info('installing %s', target_path) try_mkdir_r(os.path.dirname(target_path)) - try_unlink(target_path) # prevent ETXTBSY errors + if not headers_only: + try_unlink(target_path) # Prevent ETXTBSY errors. return shutil.copy2(source_path, target_path) def try_remove(path, dst): source_path, target_path = mkpaths(path, dst) - print 'removing %s' % target_path + logging.info('removing %s', target_path) try_unlink(target_path) try_rmdir_r(os.path.dirname(target_path)) @@ -161,6 +167,9 @@ def headers(action): if sys.platform.startswith('aix'): action(['out/Release/node.exp'], 'include/node/') + if sys.platform == 'win32': + action(['Release/node.lib'], 'Release/') + subdir_files('deps/v8/include', 'include/node/', action) if 'false' == variables.get('node_shared_libuv'): @@ -201,7 +210,7 @@ def run(args): cmd = args[1] if len(args) > 1 else 'install' - if os.environ.get('HEADERS_ONLY'): + if headers_only: if cmd == 'install': return headers(install) if cmd == 'uninstall': return headers(uninstall) else: diff --git a/vcbuild.bat b/vcbuild.bat index 86f29918875f10..935bd08ff615d1 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -28,7 +28,6 @@ set upload= set licensertf= set jslint= set cpplint= -set build_testgc_addon= set noetw= set noetw_msi_arg= set noperfctr= @@ -40,7 +39,6 @@ set enable_vtune_arg= set configure_flags= set build_addons= set dll= -set build_addons_napi= set test_node_inspect= :next-arg @@ -60,18 +58,18 @@ if /i "%1"=="nosnapshot" set nosnapshot=1&goto arg-ok if /i "%1"=="noetw" set noetw=1&goto arg-ok if /i "%1"=="noperfctr" set noperfctr=1&goto arg-ok if /i "%1"=="licensertf" set licensertf=1&goto arg-ok -if /i "%1"=="test" set test_args=%test_args% doctool known_issues message parallel sequential addons addons-napi -J&set cpplint=1&set jslint=1&set build_addons=1&set build_addons_napi=1&goto arg-ok -if /i "%1"=="test-ci" set test_args=%test_args% %test_ci_args% -p tap --logfile test.tap doctool inspector known_issues message sequential parallel addons addons-napi&set cctest_args=%cctest_args% --gtest_output=tap:cctest.tap&set build_addons=1&set build_addons_napi=1&goto arg-ok +if /i "%1"=="test" set test_args=%test_args% doctool known_issues message parallel sequential addons addons-napi -J&set cpplint=1&set jslint=1&set build_addons=1&goto arg-ok +if /i "%1"=="test-ci" set test_args=%test_args% %test_ci_args% -p tap --logfile test.tap doctool inspector known_issues message sequential parallel addons addons-napi&set cctest_args=%cctest_args% --gtest_output=tap:cctest.tap&set build_addons=1&goto arg-ok if /i "%1"=="test-addons" set test_args=%test_args% addons&set build_addons=1&goto arg-ok -if /i "%1"=="test-addons-napi" set test_args=%test_args% addons-napi&set build_addons_napi=1&goto arg-ok +if /i "%1"=="test-addons-napi" set test_args=%test_args% addons-napi&set build_addons=1&goto arg-ok if /i "%1"=="test-simple" set test_args=%test_args% sequential parallel -J&goto arg-ok if /i "%1"=="test-message" set test_args=%test_args% message&goto arg-ok -if /i "%1"=="test-gc" set test_args=%test_args% gc&set build_testgc_addon=1&goto arg-ok +if /i "%1"=="test-gc" set test_args=%test_args% gc&goto arg-ok if /i "%1"=="test-inspector" set test_args=%test_args% inspector&goto arg-ok if /i "%1"=="test-tick-processor" set test_args=%test_args% tick-processor&goto arg-ok if /i "%1"=="test-internet" set test_args=%test_args% internet&goto arg-ok if /i "%1"=="test-pummel" set test_args=%test_args% pummel&goto arg-ok -if /i "%1"=="test-all" set test_args=%test_args% sequential parallel message gc inspector internet pummel&set build_testgc_addon=1&set cpplint=1&set jslint=1&goto arg-ok +if /i "%1"=="test-all" set test_args=%test_args% sequential parallel message gc inspector internet pummel&set build_addon=1&set cpplint=1&set jslint=1&goto arg-ok if /i "%1"=="test-known-issues" set test_args=%test_args% known_issues&goto arg-ok if /i "%1"=="test-node-inspect" set test_node_inspect=1&goto arg-ok if /i "%1"=="jslint" set jslint=1&goto arg-ok @@ -303,60 +301,14 @@ scp -F %SSHCONFIG% node-v%FULLVERSION%-%target_arch%.msi %STAGINGSERVER%:nodejs/ ssh -F %SSHCONFIG% %STAGINGSERVER% "touch nodejs/%DISTTYPEDIR%/v%FULLVERSION%/node-v%FULLVERSION%-%target_arch%.msi.done nodejs/%DISTTYPEDIR%/v%FULLVERSION%/node-v%FULLVERSION%-win-%target_arch%.zip.done nodejs/%DISTTYPEDIR%/v%FULLVERSION%/node-v%FULLVERSION%-win-%target_arch%.7z.done nodejs/%DISTTYPEDIR%/v%FULLVERSION%/win-%target_arch%.done && chmod -R ug=rw-x+X,o=r+X nodejs/%DISTTYPEDIR%/v%FULLVERSION%/node-v%FULLVERSION%-%target_arch%.* nodejs/%DISTTYPEDIR%/v%FULLVERSION%/win-%target_arch%*" :run -@rem Run tests if requested. - -@rem Build test/gc add-on if required. -if "%build_testgc_addon%"=="" goto build-addons -"%config%\node" deps\npm\node_modules\node-gyp\bin\node-gyp rebuild --directory="%~dp0test\gc" --nodedir="%~dp0." -if errorlevel 1 goto build-testgc-addon-failed -goto build-addons - -:build-testgc-addon-failed -echo Failed to build test/gc add-on." -goto exit - -:build-addons -if not defined build_addons goto build-addons-napi +if not defined build_addons goto run-tests if not exist "%node_exe%" ( echo Failed to find node.exe - goto build-addons-napi + goto run-tests ) echo Building addons -:: clear -for /d %%F in (test\addons\??_*) do ( - rd /s /q %%F -) -:: generate -"%node_exe%" tools\doc\addon-verify.js +"%node_exe%" tools\build-addons.js if %errorlevel% neq 0 exit /b %errorlevel% -:: building addons -setlocal EnableDelayedExpansion -for /d %%F in (test\addons\*) do ( - "%node_exe%" deps\npm\node_modules\node-gyp\bin\node-gyp rebuild ^ - --directory="%%F" ^ - --nodedir="%cd%" - if !errorlevel! neq 0 exit /b !errorlevel! -) - -:build-addons-napi -if not defined build_addons_napi goto run-tests -if not exist "%node_exe%" ( - echo Failed to find node.exe - goto run-tests -) -echo Building addons-napi -:: clear -for /d %%F in (test\addons-napi\??_*) do ( - rd /s /q %%F -) -:: building addons-napi -for /d %%F in (test\addons-napi\*) do ( - "%node_exe%" deps\npm\node_modules\node-gyp\bin\node-gyp rebuild ^ - --directory="%%F" ^ - --nodedir="%cd%" -) -endlocal -goto run-tests :run-tests if not defined test_node_inspect goto node-tests