diff --git a/eng/azure-pipelines.yml b/eng/azure-pipelines.yml index 3ae2b131527..2cb66672430 100644 --- a/eng/azure-pipelines.yml +++ b/eng/azure-pipelines.yml @@ -40,7 +40,7 @@ stages: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 container: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-webassembly-20220531132048-00a561c + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-webassembly-20220317214646-1ad56e8 steps: - bash: | ./build.sh --ci --restore --build --configuration $(_BuildConfig) /p:TargetOS=Browser /p:TargetArchitecture=wasm $(_InternalBuildArgs) @@ -62,7 +62,7 @@ stages: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 container: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-webassembly-20220531132048-00a561c + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-webassembly-20220317214646-1ad56e8 steps: - bash: | ./build.sh --ci --restore --build --configuration $(_BuildConfig) /p:TargetOS=Browser /p:TargetArchitecture=wasm /p:WasmEnableThreads=true $(_InternalBuildArgs) diff --git a/eng/icu.proj b/eng/icu.proj index da08c33e49d..9bc24b0e59e 100644 --- a/eng/icu.proj +++ b/eng/icu.proj @@ -24,6 +24,8 @@ succeed, we need to provide a patched version of that file. Ideally, fixing upstream would be best, but until then, patch. --> + diff --git a/eng/patches/emcc.py b/eng/patches/emcc.py index 976e13a9bc2..8379d88ad74 100644 --- a/eng/patches/emcc.py +++ b/eng/patches/emcc.py @@ -151,7 +151,7 @@ ] VALID_ENVIRONMENTS = ('web', 'webview', 'worker', 'node', 'shell') -SIMD_INTEL_FEATURE_TOWER = ['-msse', '-msse2', '-msse3', '-mssse3', '-msse4.1', '-msse4.2', '-msse4', '-mavx'] +SIMD_INTEL_FEATURE_TOWER = ['-msse', '-msse2', '-msse3', '-mssse3', '-msse4.1', '-msse4.2', '-mavx'] SIMD_NEON_FLAGS = ['-mfpu=neon'] COMPILE_ONLY_FLAGS = {'--default-obj-ext'} LINK_ONLY_FLAGS = { @@ -267,6 +267,7 @@ def __init__(self): self.use_preload_plugins = False self.default_object_extension = '.o' self.valid_abspaths = [] + self.cfi = False # Specifies the line ending format to use for all generated text files. # Defaults to using the native EOL on each platform (\r\n on Windows, \n on # Linux & MacOS) @@ -305,10 +306,10 @@ def setup_environment_settings(): (settings.ENVIRONMENT_MAY_BE_NODE and settings.USE_PTHREADS) if not settings.ENVIRONMENT_MAY_BE_WORKER and settings.PROXY_TO_WORKER: - exit_with_error('If you specify --proxy-to-worker and specify a "-sENVIRONMENT=" directive, it must include "worker" as a target! (Try e.g. -sENVIRONMENT=web,worker)') + exit_with_error('If you specify --proxy-to-worker and specify a "-s ENVIRONMENT=" directive, it must include "worker" as a target! (Try e.g. -s ENVIRONMENT=web,worker)') if not settings.ENVIRONMENT_MAY_BE_WORKER and settings.SHARED_MEMORY: - exit_with_error('When building with multithreading enabled and a "-sENVIRONMENT=" directive is specified, it must include "worker" as a target! (Try e.g. -sENVIRONMENT=web,worker)') + exit_with_error('When building with multithreading enabled and a "-s ENVIRONMENT=" directive is specified, it must include "worker" as a target! (Try e.g. -s ENVIRONMENT=web,worker)') def minify_whitespace(): @@ -375,17 +376,25 @@ def apply_settings(user_settings): else: value = value.replace('\\', '\\\\') - expected_type = settings.types.get(key) + existing = getattr(settings, user_key, None) + expect_list = type(existing) == list - if filename and expected_type == list and value.strip()[0] != '[': + if filename and expect_list and value.strip()[0] != '[': # Prefer simpler one-line-per value parser value = parse_symbol_list_file(value) else: try: - value = parse_value(value, expected_type) + value = parse_value(value, expect_list) except Exception as e: exit_with_error('a problem occurred in evaluating the content after a "-s", specifically "%s=%s": %s', key, value, str(e)) + # Do some basic type checking by comparing to the existing settings. + # Sadly we can't do this generically in the SettingsManager since there are settings + # that so change types internally over time. + # We only currently worry about lists vs non-lists. + if expect_list != (type(value) == list): + exit_with_error('setting `%s` expects `%s` but got `%s`' % (user_key, type(existing), type(value))) + setattr(settings, user_key, value) if key == 'EXPORTED_FUNCTIONS': @@ -513,30 +522,27 @@ def cxx_to_c_compiler(cxx): return os.path.join(dirname, basename) -def should_run_binaryen_optimizer(): +def get_binaryen_passes(): # run the binaryen optimizer in -O2+. in -O0 we don't need it obviously, while # in -O1 we don't run it as the LLVM optimizer has been run, and it does the # great majority of the work; not running the binaryen optimizer in that case # keeps -O1 mostly-optimized while compiling quickly and without rewriting # DWARF etc. - return settings.OPT_LEVEL >= 2 - + run_binaryen_optimizer = settings.OPT_LEVEL >= 2 -def get_binaryen_passes(): passes = [] - optimizing = should_run_binaryen_optimizer() # safe heap must run before post-emscripten, so post-emscripten can apply the sbrk ptr if settings.SAFE_HEAP: passes += ['--safe-heap'] if settings.MEMORY64 == 2: passes += ['--memory64-lowering'] - if optimizing: + if run_binaryen_optimizer: passes += ['--post-emscripten'] - if optimizing: + if run_binaryen_optimizer: passes += [building.opt_level_to_str(settings.OPT_LEVEL, settings.SHRINK_LEVEL)] # when optimizing, use the fact that low memory is never used (1024 is a # hardcoded value in the binaryen pass) - if optimizing and settings.GLOBAL_BASE >= 1024: + if run_binaryen_optimizer and settings.GLOBAL_BASE >= 1024: passes += ['--low-memory-unused'] if settings.AUTODEBUG: # adding '--flatten' here may make these even more effective @@ -589,7 +595,7 @@ def check_human_readable_list(items): # the one exception is dynamic linking of a side module: the main module is ok # as it is loaded first, but the side module may be assigned memory that was # previously used. - if optimizing and not settings.SIDE_MODULE: + if run_binaryen_optimizer and not settings.SIDE_MODULE: passes += ['--zero-filled-memory'] if settings.BINARYEN_EXTRA_PASSES: @@ -793,11 +799,10 @@ def array_contains_any_of(hay, needles): if array_contains_any_of(user_args, SIMD_INTEL_FEATURE_TOWER[4:]): cflags += ['-D__SSE4_1__=1'] - # Handle both -msse4.2 and its alias -msse4. if array_contains_any_of(user_args, SIMD_INTEL_FEATURE_TOWER[5:]): cflags += ['-D__SSE4_2__=1'] - if array_contains_any_of(user_args, SIMD_INTEL_FEATURE_TOWER[7:]): + if array_contains_any_of(user_args, SIMD_INTEL_FEATURE_TOWER[6:]): cflags += ['-D__AVX__=1'] if array_contains_any_of(user_args, SIMD_NEON_FLAGS): @@ -828,9 +833,6 @@ def get_cflags(user_args): if settings.SHARED_MEMORY: cflags.append('-D__EMSCRIPTEN_SHARED_MEMORY__=1') - if settings.WASM_WORKERS: - cflags.append('-D__EMSCRIPTEN_WASM_WORKERS__=1') - if not settings.STRICT: # The preprocessor define EMSCRIPTEN is deprecated. Don't pass it to code # in strict mode. Code should use the define __EMSCRIPTEN__ instead. @@ -935,9 +937,11 @@ def in_temp(name): def dedup_list(lst): - # Since we require python 3.6, that ordering of dictionaries is guaranteed - # to be insertion order so we can use 'dict' here but not 'set'. - return list(dict.fromkeys(lst)) + rtn = [] + for item in lst: + if item not in rtn: + rtn.append(item) + return rtn def move_file(src, dst): @@ -990,8 +994,6 @@ def package_files(options, target): if not settings.ENVIRONMENT_MAY_BE_NODE: file_args.append('--no-node') if options.embed_files: - if settings.MEMORY64: - file_args += ['--wasm64'] object_file = in_temp('embedded_files.o') file_args += ['--obj-output=' + object_file] rtn.append(object_file) @@ -1038,7 +1040,7 @@ def run(args): # Strip args[0] (program name) args = args[1:] - misc_temp_files = shared.get_temp_files() + misc_temp_files = shared.configuration.get_temp_files() # Handle some global flags @@ -1422,17 +1424,11 @@ def phase_setup(options, state, newargs, user_settings): if settings.MAIN_MODULE or settings.SIDE_MODULE: settings.RELOCATABLE = 1 - # Pthreads and Wasm Workers require targeting shared Wasm memory (SAB). - if settings.USE_PTHREADS or settings.WASM_WORKERS: + if settings.USE_PTHREADS: settings.SHARED_MEMORY = 1 - if settings.USE_PTHREADS and '-pthread' not in newargs: + if settings.SHARED_MEMORY and '-pthread' not in newargs: newargs += ['-pthread'] - elif settings.SHARED_MEMORY: - if '-matomics' not in newargs: - newargs += ['-matomics'] - if '-mbulk-memory' not in newargs: - newargs += ['-mbulk-memory'] if 'DISABLE_EXCEPTION_CATCHING' in user_settings and 'EXCEPTION_CATCHING_ALLOWED' in user_settings: # If we get here then the user specified both DISABLE_EXCEPTION_CATCHING and EXCEPTION_CATCHING_ALLOWED @@ -1449,11 +1445,6 @@ def phase_setup(options, state, newargs, user_settings): if settings.DISABLE_EXCEPTION_THROWING and not settings.DISABLE_EXCEPTION_CATCHING: exit_with_error("DISABLE_EXCEPTION_THROWING was set (probably from -fno-exceptions) but is not compatible with enabling exception catching (DISABLE_EXCEPTION_CATCHING=0). If you don't want exceptions, set DISABLE_EXCEPTION_CATCHING to 1; if you do want exceptions, don't link with -fno-exceptions") - if settings.MEMORY64: - default_setting(user_settings, 'SUPPORT_LONGJMP', 0) - if settings.SUPPORT_LONGJMP: - exit_with_error('MEMORY64 is not compatible with SUPPORT_LONGJMP') - # SUPPORT_LONGJMP=1 means the default SjLj handling mechanism, currently # 'emscripten' if settings.SUPPORT_LONGJMP == 1: @@ -1476,95 +1467,6 @@ def phase_setup(options, state, newargs, user_settings): return (newargs, input_files) -def setup_pthreads(target): - if settings.RELOCATABLE: - # phtreads + dyanmic linking has certain limitations - if settings.SIDE_MODULE: - diagnostics.warning('experimental', '-sSIDE_MODULE + pthreads is experimental') - elif settings.MAIN_MODULE: - diagnostics.warning('experimental', '-sMAIN_MODULE + pthreads is experimental') - elif settings.LINKABLE: - diagnostics.warning('experimental', '-sLINKABLE + pthreads is experimental') - if settings.ALLOW_MEMORY_GROWTH: - diagnostics.warning('pthreads-mem-growth', 'USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code slowly, see https://github.com/WebAssembly/design/issues/1271') - - # Functions needs to be exported from the module since they are used in worker.js - settings.REQUIRED_EXPORTS += [ - 'emscripten_dispatch_to_thread_', - '_emscripten_thread_free_data', - 'emscripten_main_browser_thread_id', - 'emscripten_main_thread_process_queued_calls', - 'emscripten_run_in_main_runtime_thread_js', - 'emscripten_stack_set_limits', - ] - - if settings.MAIN_MODULE: - settings.REQUIRED_EXPORTS += ['_emscripten_thread_sync_code', '__dl_seterr'] - - settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += [ - '$exitOnMainThread', - ] - # Some symbols are required by worker.js. - # Because emitDCEGraph only considers the main js file, and not worker.js - # we have explicitly mark these symbols as user-exported so that they will - # kept alive through DCE. - # TODO: Find a less hacky way to do this, perhaps by also scanning worker.js - # for roots. - worker_imports = [ - '__emscripten_thread_init', - '__emscripten_thread_exit', - '__emscripten_thread_crashed', - '__emscripten_tls_init', - '_pthread_self', - 'executeNotifiedProxyingQueue', - ] - settings.EXPORTED_FUNCTIONS += worker_imports - building.user_requested_exports.update(worker_imports) - - # set location of worker.js - settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + '.worker.js' - - # memalign is used to ensure allocated thread stacks are aligned. - settings.REQUIRED_EXPORTS += ['emscripten_builtin_memalign'] - - if settings.MINIMAL_RUNTIME: - building.user_requested_exports.add('exit') - - if settings.PROXY_TO_PTHREAD: - settings.REQUIRED_EXPORTS += ['emscripten_proxy_main'] - - # All proxying async backends will need this. - if settings.WASMFS: - settings.REQUIRED_EXPORTS += ['emscripten_proxy_finish'] - - # pthread stack setup and other necessary utilities - def include_and_export(name): - settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$' + name] - settings.EXPORTED_FUNCTIONS += [name] - - include_and_export('establishStackSpace') - include_and_export('invokeEntryPoint') - include_and_export('PThread') - if not settings.MINIMAL_RUNTIME: - # keepRuntimeAlive does not apply to MINIMAL_RUNTIME. - settings.EXPORTED_RUNTIME_METHODS += ['keepRuntimeAlive', 'ExitStatus', 'wasmMemory'] - - if settings.MODULARIZE: - if not settings.EXPORT_ES6 and settings.EXPORT_NAME == 'Module': - exit_with_error('pthreads + MODULARIZE currently require you to set -sEXPORT_NAME=Something (see settings.js) to Something != Module, so that the .worker.js file can work') - - # MODULARIZE+USE_PTHREADS mode requires extra exports out to Module so that worker.js - # can access them: - - # general threading variables: - settings.EXPORTED_RUNTIME_METHODS += ['PThread'] - - # To keep code size to minimum, MINIMAL_RUNTIME does not utilize the global ExitStatus - # object, only regular runtime has it. - if not settings.MINIMAL_RUNTIME: - settings.EXPORTED_RUNTIME_METHODS += ['ExitStatus'] - - @ToolchainProfiler.profile_block('linker_setup') def phase_linker_setup(options, state, newargs, user_settings): autoconf = os.environ.get('EMMAKEN_JUST_CONFIGURE') or 'conftest.c' in state.orig_args @@ -1637,8 +1539,8 @@ def phase_linker_setup(options, state, newargs, user_settings): diagnostics.warning('deprecated', 'EXTRA_EXPORTED_RUNTIME_METHODS is deprecated, please use EXPORTED_RUNTIME_METHODS instead') settings.EXPORTED_RUNTIME_METHODS += settings.EXTRA_EXPORTED_RUNTIME_METHODS - # If no output format was specified we try to deduce the format based on - # the output filename extension + # If no output format was sepecific we try to imply the format based on + # the output filename extension. if not options.oformat and (options.relocatable or (options.shared and not settings.SIDE_MODULE)): # Until we have a better story for actually producing runtime shared libraries # we support a compatibility mode where shared libraries are actually just @@ -1736,7 +1638,7 @@ def phase_linker_setup(options, state, newargs, user_settings): # Note the exports the user requested building.user_requested_exports.update(settings.EXPORTED_FUNCTIONS) - # -sASSERTIONS implies basic stack overflow checks, and ASSERTIONS=2 + # -s ASSERTIONS=1 implies basic stack overflow checks, and ASSERTIONS=2 # implies full stack overflow checks. if settings.ASSERTIONS: # However, we don't set this default in PURE_WASI, or when we are linking without standard @@ -1806,17 +1708,14 @@ def phase_linker_setup(options, state, newargs, user_settings): if settings.MINIMAL_RUNTIME_STREAMING_WASM_COMPILATION and settings.MINIMAL_RUNTIME_STREAMING_WASM_INSTANTIATION: exit_with_error('MINIMAL_RUNTIME_STREAMING_WASM_COMPILATION and MINIMAL_RUNTIME_STREAMING_WASM_INSTANTIATION are mutually exclusive!') - if options.emrun: - if settings.MINIMAL_RUNTIME: - exit_with_error('--emrun is not compatible with MINIMAL_RUNTIME') - if options.oformat != OFormat.HTML: - exit_with_error('--emrun is only compatible with html output') + if options.emrun and settings.MINIMAL_RUNTIME: + exit_with_error('--emrun is not compatible with MINIMAL_RUNTIME') if options.use_closure_compiler: - settings.USE_CLOSURE_COMPILER = 1 + settings.USE_CLOSURE_COMPILER = options.use_closure_compiler if settings.CLOSURE_WARNINGS not in ['quiet', 'warn', 'error']: - exit_with_error('Invalid option -sCLOSURE_WARNINGS=%s specified! Allowed values are "quiet", "warn" or "error".' % settings.CLOSURE_WARNINGS) + exit_with_error('Invalid option -s CLOSURE_WARNINGS=%s specified! Allowed values are "quiet", "warn" or "error".' % settings.CLOSURE_WARNINGS) # Include dynCall() function by default in DYNCALLS builds in classic runtime; in MINIMAL_RUNTIME, must add this explicitly. if settings.DYNCALLS and not settings.MINIMAL_RUNTIME: @@ -1825,10 +1724,6 @@ def phase_linker_setup(options, state, newargs, user_settings): if not settings.BOOTSTRAPPING_STRUCT_INFO: # Include the internal library function since they are used by runtime functions. settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$getWasmTableEntry', '$setWasmTableEntry'] - if settings.SAFE_HEAP: - settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$getValue_safe', '$setValue_safe'] - if not settings.MINIMAL_RUNTIME: - settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$getValue', '$setValue'] if settings.MAIN_MODULE: assert not settings.SIDE_MODULE @@ -1860,7 +1755,7 @@ def phase_linker_setup(options, state, newargs, user_settings): if settings.USE_PTHREADS: settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += [ - '$registerTLSInit', + '$registerTlsInit', ] if settings.RELOCATABLE: @@ -1952,7 +1847,7 @@ def phase_linker_setup(options, state, newargs, user_settings): if settings.MODULARIZE: if settings.PROXY_TO_WORKER: - exit_with_error('-sMODULARIZE is not compatible with --proxy-to-worker (if you want to run in a worker with -sMODULARIZE, you likely want to do the worker side setup manually)') + exit_with_error('-s MODULARIZE=1 is not compatible with --proxy-to-worker (if you want to run in a worker with -s MODULARIZE=1, you likely want to do the worker side setup manually)') # in MINIMAL_RUNTIME we may not need to emit the Promise code, as the # HTML output creates a singleton instance, and it does so without the # Promise. However, in Pthreads mode the Promise is used for worker @@ -2014,7 +1909,7 @@ def phase_linker_setup(options, state, newargs, user_settings): exit_with_error('MIN_WEBGL_VERSION must be smaller or equal to MAX_WEBGL_VERSION!') if not settings.GL_SUPPORT_SIMPLE_ENABLE_EXTENSIONS and settings.GL_SUPPORT_AUTOMATIC_ENABLE_EXTENSIONS: - exit_with_error('-sGL_SUPPORT_SIMPLE_ENABLE_EXTENSIONS=0 only makes sense with -sGL_SUPPORT_AUTOMATIC_ENABLE_EXTENSIONS=0!') + exit_with_error('-s GL_SUPPORT_SIMPLE_ENABLE_EXTENSIONS=0 only makes sense with -s GL_SUPPORT_AUTOMATIC_ENABLE_EXTENSIONS=0!') if settings.WASMFS: state.forced_stdlibs.append('libwasmfs') @@ -2022,12 +1917,6 @@ def phase_linker_setup(options, state, newargs, user_settings): settings.SYSCALLS_REQUIRE_FILESYSTEM = 0 settings.JS_LIBRARIES.append((0, 'library_wasmfs.js')) settings.REQUIRED_EXPORTS += ['_wasmfs_read_file'] - if settings.MAIN_MODULE: - # Dynamic library support uses JS API internals, so include it all - # TODO: rewriting more of the dynamic linking support code into wasm could - # avoid this. also, after we remove the old FS, we could write a - # more specific API for wasmfs/dynamic linking integration perhaps - settings.FORCE_FILESYSTEM = 1 if settings.FORCE_FILESYSTEM: # Add exports for the JS API. Like the old JS FS, WasmFS by default # includes just what JS parts it actually needs, and FORCE_FILESYSTEM is @@ -2039,7 +1928,6 @@ def phase_linker_setup(options, state, newargs, user_settings): '_wasmfs_chdir', '_wasmfs_symlink', '_wasmfs_chmod', - '_wasmfs_identify', ] # Explicitly drop linking in a malloc implementation if program is not using any dynamic allocation calls. @@ -2102,11 +1990,12 @@ def phase_linker_setup(options, state, newargs, user_settings): ] if not settings.STANDALONE_WASM and (settings.EXIT_RUNTIME or settings.ASSERTIONS): - # to flush streams on FS exit, we need to be able to call fflush - # we only include it if the runtime is exitable, or when ASSERTIONS + # We use __stdio_exit to shut down musl's stdio subsystems and flush + # streams on exit. + # We only include it if the runtime is exitable, or when ASSERTIONS # (ASSERTIONS will check that streams do not need to be flushed, # helping people see when they should have enabled EXIT_RUNTIME) - settings.EXPORT_IF_DEFINED += ['fflush'] + settings.EXPORT_IF_DEFINED += ['__stdio_exit'] if settings.SUPPORT_ERRNO: # so setErrNo JS library function can report errno back to C @@ -2128,29 +2017,54 @@ def phase_linker_setup(options, state, newargs, user_settings): default_setting(user_settings, 'ABORTING_MALLOC', 0) if settings.USE_PTHREADS: - setup_pthreads(target) + if settings.ALLOW_MEMORY_GROWTH: + diagnostics.warning('pthreads-mem-growth', 'USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code slowly, see https://github.com/WebAssembly/design/issues/1271') settings.JS_LIBRARIES.append((0, 'library_pthread.js')) + # Functions needs to be exported from the module since they are used in worker.js + settings.REQUIRED_EXPORTS += [ + 'emscripten_dispatch_to_thread_', + '_emscripten_thread_free_data', + '_emscripten_allow_main_runtime_queued_calls', + 'emscripten_main_browser_thread_id', + 'emscripten_main_thread_process_queued_calls', + 'emscripten_run_in_main_runtime_thread_js', + 'emscripten_stack_set_limits', + 'emscripten_sync_run_in_main_thread_2', + 'emscripten_sync_run_in_main_thread_4', + ] + + if settings.MAIN_MODULE: + settings.REQUIRED_EXPORTS += ['_emscripten_thread_sync_code', '__dl_seterr'] + + settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += [ + '$exitOnMainThread', + ] + # Some symbols are required by worker.js. + # Because emitDCEGraph only considers the main js file, and not worker.js + # we have explicitly mark these symbols as user-exported so that they will + # kept alive through DCE. + # TODO: Find a less hacky way to do this, perhaps by also scanning worker.js + # for roots. + worker_imports = [ + '__emscripten_thread_init', + '__emscripten_thread_exit', + '__emscripten_thread_crashed', + '_emscripten_tls_init', + '_emscripten_current_thread_process_queued_calls', + '_pthread_self', + ] + settings.EXPORTED_FUNCTIONS += worker_imports + building.user_requested_exports.update(worker_imports) + + # set location of worker.js + settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + '.worker.js' else: - if settings.PROXY_TO_PTHREAD: - exit_with_error('-sPROXY_TO_PTHREAD requires -sUSE_PTHREADS to work!') settings.JS_LIBRARIES.append((0, 'library_pthread_stub.js')) # TODO: Move this into the library JS file once it becomes possible. # See https://github.com/emscripten-core/emscripten/pull/15982 if settings.INCLUDE_FULL_LIBRARY and not settings.DISABLE_EXCEPTION_CATCHING: - settings.EXPORTED_FUNCTIONS += ['___get_exception_message', '_free'] - - if settings.WASM_WORKERS: - # TODO: After #15982 is resolved, these dependencies can be declared in library_wasm_worker.js - # instead of having to record them here. - wasm_worker_imports = ['_emscripten_wasm_worker_initialize'] - settings.EXPORTED_FUNCTIONS += wasm_worker_imports - building.user_requested_exports.update(wasm_worker_imports) - settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['_wasm_worker_initializeRuntime'] - # set location of Wasm Worker bootstrap JS file - if settings.WASM_WORKERS == 1: - settings.WASM_WORKER_FILE = unsuffixed(os.path.basename(target)) + '.ww.js' - settings.JS_LIBRARIES.append((0, shared.path_from_root('src', 'library_wasm_worker.js'))) + settings.EXPORTED_FUNCTIONS += ['_emscripten_format_exception', '_free'] if settings.FORCE_FILESYSTEM and not settings.MINIMAL_RUNTIME: # when the filesystem is forced, we export by default methods that filesystem usage @@ -2161,20 +2075,63 @@ def phase_linker_setup(options, state, newargs, user_settings): 'FS_createPath', 'FS_createDataFile', 'FS_createPreloadedFile', + 'FS_createLazyFile', + 'FS_createDevice', 'FS_unlink' ] - if not settings.WASMFS: - # The old FS has some functionality that WasmFS lacks. - settings.EXPORTED_RUNTIME_METHODS += [ - 'FS_createLazyFile', - 'FS_createDevice' - ] settings.EXPORTED_RUNTIME_METHODS += [ 'addRunDependency', 'removeRunDependency', ] + if settings.USE_PTHREADS: + # memalign is used to ensure allocated thread stacks are aligned. + settings.REQUIRED_EXPORTS += ['emscripten_builtin_memalign'] + + if settings.MINIMAL_RUNTIME: + building.user_requested_exports.add('exit') + + if settings.PROXY_TO_PTHREAD: + settings.REQUIRED_EXPORTS += ['emscripten_proxy_main'] + + # pthread stack setup and other necessary utilities + def include_and_export(name): + settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$' + name] + settings.EXPORTED_FUNCTIONS += [name] + + include_and_export('establishStackSpace') + include_and_export('invokeEntryPoint') + if not settings.MINIMAL_RUNTIME: + # keepRuntimeAlive does not apply to MINIMAL_RUNTIME. + settings.EXPORTED_RUNTIME_METHODS += ['keepRuntimeAlive'] + + if settings.MODULARIZE: + if not settings.EXPORT_ES6 and settings.EXPORT_NAME == 'Module': + exit_with_error('pthreads + MODULARIZE currently require you to set -s EXPORT_NAME=Something (see settings.js) to Something != Module, so that the .worker.js file can work') + + # MODULARIZE+USE_PTHREADS mode requires extra exports out to Module so that worker.js + # can access them: + + # general threading variables: + settings.EXPORTED_RUNTIME_METHODS += ['PThread'] + + # To keep code size to minimum, MINIMAL_RUNTIME does not utilize the global ExitStatus + # object, only regular runtime has it. + if not settings.MINIMAL_RUNTIME: + settings.EXPORTED_RUNTIME_METHODS += ['ExitStatus'] + + if settings.RELOCATABLE: + # phtreads + dyanmic linking has certain limitations + if settings.SIDE_MODULE: + diagnostics.warning('experimental', '-s SIDE_MODULE + pthreads is experimental') + elif settings.MAIN_MODULE: + diagnostics.warning('experimental', '-s MAIN_MODULE + pthreads is experimental') + elif settings.LINKABLE: + diagnostics.warning('experimental', '-s LINKABLE + pthreads is experimental') + elif settings.PROXY_TO_PTHREAD: + exit_with_error('-s PROXY_TO_PTHREAD=1 requires -s USE_PTHREADS to work!') + def check_memory_setting(setting): if settings[setting] % webassembly.WASM_PAGE_SIZE != 0: exit_with_error(f'{setting} must be a multiple of WebAssembly page size (64KiB), was {settings[setting]}') @@ -2242,7 +2199,7 @@ def check_memory_setting(setting): if settings.MODULARIZE and not (settings.EXPORT_ES6 and not settings.SINGLE_FILE) and \ settings.EXPORT_NAME == 'Module' and options.oformat == OFormat.HTML and \ (options.shell_path == utils.path_from_root('src/shell.html') or options.shell_path == utils.path_from_root('src/shell_minimal.html')): - exit_with_error(f'Due to collision in variable name "Module", the shell file "{options.shell_path}" is not compatible with build options "-sMODULARIZE -sEXPORT_NAME=Module". Either provide your own shell file, change the name of the export to something else to avoid the name collision. (see https://github.com/emscripten-core/emscripten/issues/7950 for details)') + exit_with_error(f'Due to collision in variable name "Module", the shell file "{options.shell_path}" is not compatible with build options "-s MODULARIZE=1 -s EXPORT_NAME=Module". Either provide your own shell file, change the name of the export to something else to avoid the name collision. (see https://github.com/emscripten-core/emscripten/issues/7950 for details)') if settings.STANDALONE_WASM: if settings.MINIMAL_RUNTIME: @@ -2255,17 +2212,6 @@ def check_memory_setting(setting): if settings.SHARED_MEMORY or settings.RELOCATABLE or settings.ASYNCIFY_LAZY_LOAD_CODE or settings.WASM2JS: settings.IMPORTED_MEMORY = 1 - # Any "pointers" passed to JS will now be i64's, in both modes. - # Also turn off minifying, which clashes with instrumented functions in preamble.js - if settings.MEMORY64: - if settings.RELOCATABLE: - exit_with_error('MEMORY64 is not compatible with dynamic linking') - if not settings.DISABLE_EXCEPTION_CATCHING: - exit_with_error('MEMORY64 is not compatible with DISABLE_EXCEPTION_CATCHING=0') - settings.WASM_BIGINT = 1 - settings.MINIFY_WASM_IMPORTS_AND_EXPORTS = 0 - settings.MINIFY_WASM_IMPORTED_MODULES = 0 - if settings.WASM_BIGINT: settings.LEGALIZE_JS_FFI = 0 @@ -2512,13 +2458,8 @@ def get_full_import_name(name): settings.ASYNCIFY_IMPORTS = [get_full_import_name(i) for i in settings.ASYNCIFY_IMPORTS] - if settings.WASM2JS: - if settings.GENERATE_SOURCE_MAP: - exit_with_error('wasm2js does not support source maps yet (debug in wasm for now)') - if settings.WASM_BIGINT: - exit_with_error('wasm2js does not support WASM_BIGINT') - if settings.MEMORY64: - exit_with_error('wasm2js does not support MEMORY64') + if settings.WASM2JS and settings.GENERATE_SOURCE_MAP: + exit_with_error('wasm2js does not support source maps yet (debug in wasm for now)') if settings.NODE_CODE_CACHING: if settings.WASM_ASYNC_COMPILATION: @@ -2537,6 +2478,14 @@ def get_full_import_name(name): 'emscripten_stack_get_base', 'emscripten_stack_get_end'] + # Any "pointers" passed to JS will now be i64's, in both modes. + # Also turn off minifying, which clashes with instrumented functions in preamble.js + if settings.MEMORY64: + if user_settings.get('WASM_BIGINT') == '0': + exit_with_error('MEMORY64 is not compatible with WASM_BIGINT=0') + settings.WASM_BIGINT = 1 + settings.MINIFY_WASM_IMPORTS_AND_EXPORTS = 0 + # check if we can address the 2GB mark and higher: either if we start at # 2GB, or if we allow growth to either any amount or to 2GB or more. if settings.INITIAL_MEMORY > 2 * 1024 * 1024 * 1024 or \ @@ -2554,19 +2503,6 @@ def get_full_import_name(name): if settings.WASMFS: settings.LINK_AS_CXX = True - # Some settings make no sense when not linking as C++ - if not settings.LINK_AS_CXX: - cxx_only_settings = [ - 'DEMANGLE_SUPPORT', - 'EXCEPTION_DEBUG', - 'DISABLE_EXCEPTION_CATCHING', - 'EXCEPTION_CATCHING_ALLOWED', - 'DISABLE_EXCEPTION_THROWING', - ] - for setting in cxx_only_settings: - if setting in user_settings: - diagnostics.warning('linkflags', 'setting `%s` is not meaningful unless linking as C++', setting) - # Export tag objects which are likely needed by the native code, but which are # currently not reported in the metadata of wasm-emscripten-finalize if settings.RELOCATABLE: @@ -2575,9 +2511,6 @@ def get_full_import_name(name): if settings.SUPPORT_LONGJMP == 'wasm': settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.append('__c_longjmp') - if settings.EXCEPTION_HANDLING: - settings.REQUIRED_EXPORTS += ['__trap'] - return target, wasm_target @@ -2739,9 +2672,9 @@ def compile_source_file(i, input_file): @ToolchainProfiler.profile_block('calculate system libraries') def phase_calculate_system_libraries(state, linker_arguments, linker_inputs, newargs): extra_files_to_link = [] - # Link in ports and system libraries, if necessary + # link in ports and system libraries, if necessary if not settings.SIDE_MODULE: - # Ports are always linked into the main module, never the side module. + # Ports are always linked into the main module, never the size module. extra_files_to_link += ports.get_libs(settings) all_linker_inputs = [f for _, f in sorted(linker_inputs)] + extra_files_to_link extra_files_to_link += system_libs.calculate(all_linker_inputs, newargs, forced=state.forced_stdlibs) @@ -2872,8 +2805,8 @@ def phase_final_emitting(options, state, target, wasm_target, memfile): # src = re.sub(r'\n+[ \n]*\n+', '\n', src) # write_file(final_js, src) - target_dir = os.path.dirname(os.path.abspath(target)) if settings.USE_PTHREADS: + target_dir = os.path.dirname(os.path.abspath(target)) worker_output = os.path.join(target_dir, settings.PTHREAD_WORKER_FILE) contents = shared.read_and_preprocess(utils.path_from_root('src/worker.js'), expand_macros=True) write_file(worker_output, contents) @@ -2883,17 +2816,6 @@ def phase_final_emitting(options, state, target, wasm_target, memfile): minified_worker = building.acorn_optimizer(worker_output, ['minifyWhitespace'], return_output=True) write_file(worker_output, minified_worker) - # Deploy the Wasm Worker bootstrap file as an output file (*.ww.js) - if settings.WASM_WORKERS == 1: - worker_output = os.path.join(target_dir, settings.WASM_WORKER_FILE) - with open(worker_output, 'w') as f: - f.write(shared.read_and_preprocess(shared.path_from_root('src', 'wasm_worker.js'), expand_macros=True)) - - # Minify the wasm_worker.js file in optimized builds - if (settings.OPT_LEVEL >= 1 or settings.SHRINK_LEVEL >= 1) and not settings.DEBUG_LEVEL: - minified_worker = building.acorn_optimizer(worker_output, ['minifyWhitespace'], return_output=True) - open(worker_output, 'w').write(minified_worker) - # track files that will need native eols generated_text_files_with_native_eols = [] @@ -2902,13 +2824,13 @@ def phase_final_emitting(options, state, target, wasm_target, memfile): module_export_name_substitution() - # Run a final optimization pass to clean up items that were not possible to optimize by Closure, or unoptimalities that were left behind + # Run a final regex pass to clean up items that were not possible to optimize by Closure, or unoptimalities that were left behind # by processing steps that occurred after Closure. - if settings.MINIMAL_RUNTIME == 2 and settings.USE_CLOSURE_COMPILER and settings.DEBUG_LEVEL == 0: - shared.run_js_tool(utils.path_from_root('tools/unsafe_optimizations.js'), [final_js, '-o', final_js], cwd=utils.path_from_root('.')) - # Finally, rerun Closure compile with simple optimizations. It will be able to further minify the code. (n.b. it would not be safe - # to run in advanced mode) - final_js = building.closure_compiler(final_js, pretty=False, advanced=False, extra_closure_args=options.closure_args) + if settings.MINIMAL_RUNTIME == 2 and settings.USE_CLOSURE_COMPILER and settings.DEBUG_LEVEL == 0 and not settings.SINGLE_FILE: + # Process .js runtime file. Note that we need to handle the license text + # here, so that it will not confuse the hacky script. + js_manipulation.handle_license(final_js) + shared.run_process([shared.PYTHON, utils.path_from_root('tools/hacky_postprocess_around_closure_limitations.py'), final_js]) # Unmangle previously mangled `import.meta` references in both main code and libraries. # See also: `preprocess` in parseTools.js. @@ -3045,7 +2967,7 @@ def consume_arg_file(): options.requested_level = 2 settings.SHRINK_LEVEL = 2 settings_changes.append('INLINING_LIMIT=1') - settings.OPT_LEVEL = validate_arg_level(options.requested_level, 3, 'invalid optimization level: ' + arg, clamp=True) + settings.OPT_LEVEL = validate_arg_level(options.requested_level, 3, 'Invalid optimization level: ' + arg, clamp=True) elif check_arg('--js-opts'): logger.warning('--js-opts ignored when using llvm backend') consume_arg() @@ -3094,14 +3016,14 @@ def consume_arg_file(): requested_level = strip_prefix(arg, '-g') or '3' if is_int(requested_level): # the -gX value is the debug level (-g1, -g2, etc.) - settings.DEBUG_LEVEL = validate_arg_level(requested_level, 4, 'invalid debug level: ' + arg) + settings.DEBUG_LEVEL = validate_arg_level(requested_level, 4, 'Invalid debug level: ' + arg) # if we don't need to preserve LLVM debug info, do not keep this flag # for clang if settings.DEBUG_LEVEL < 3: - newargs[i] = '-g0' + newargs[i] = '' else: - # for 3+, report -g3 to clang as -g4 etc. are not accepted - newargs[i] = '-g3' + # for 3+, report -g to clang as -g4 etc. are not accepted + newargs[i] = '-g' if settings.DEBUG_LEVEL == 4: settings.GENERATE_SOURCE_MAP = 1 diagnostics.warning('deprecated', 'please replace -g4 with -gsource-map') @@ -3231,8 +3153,8 @@ def consume_arg_file(): options.default_object_extension = consume_arg() if not options.default_object_extension.startswith('.'): options.default_object_extension = '.' + options.default_object_extension - elif arg.startswith('-fsanitize=cfi'): - exit_with_error('emscripten does not currently support -fsanitize=cfi') + elif arg == '-fsanitize=cfi': + options.cfi = True elif check_arg('--output_eol'): style = consume_arg() if style.lower() == 'windows': @@ -3258,7 +3180,7 @@ def consume_arg_file(): else: value = '1' if key in settings.keys(): - exit_with_error(f'{arg}: cannot change built-in settings values with a -jsD directive. Pass -s{key}={value} instead!') + exit_with_error(f'{arg}: cannot change built-in settings values with a -jsD directive. Pass -s {key}={value} instead!') user_js_defines += [(key, value)] newargs[i] = '' elif check_flag('-shared'): @@ -3337,11 +3259,6 @@ def phase_binaryen(target, options, wasm_target): # the only reason we need intermediate debug info, we can stop keeping it if settings.ASYNCIFY: intermediate_debug_info -= 1 - # currently binaryen's DWARF support will limit some optimizations; warn on - # that. see https://github.com/emscripten-core/emscripten/issues/15269 - dwarf_info = settings.DEBUG_LEVEL >= 3 - if dwarf_info: - diagnostics.warning('limited-postlink-optimizations', 'running limited binaryen optimizations because DWARF info requested (or indirectly required)') building.run_wasm_opt(wasm_target, wasm_target, args=passes, @@ -3436,7 +3353,7 @@ def preprocess_wasm2js_script(): symbols_file=symbols_file, symbols_file_js=symbols_file_js) - shared.get_temp_files().note(wasm2js) + shared.configuration.get_temp_files().note(wasm2js) if settings.WASM == 2: safe_copy(wasm2js, wasm2js_template) @@ -3459,7 +3376,7 @@ def preprocess_wasm2js_script(): save_intermediate_with_wasm('symbolmap', wasm_target) if settings.DEBUG_LEVEL >= 3 and settings.SEPARATE_DWARF and os.path.exists(wasm_target): - building.emit_debug_on_side(wasm_target) + building.emit_debug_on_side(wasm_target, settings.SEPARATE_DWARF) if settings.WASM2C: wasm2c.do_wasm2c(wasm_target) @@ -3558,7 +3475,7 @@ def modularize(): exports["%(EXPORT_NAME)s"] = %(EXPORT_NAME)s; ''' % {'EXPORT_NAME': settings.EXPORT_NAME}) - shared.get_temp_files().note(final_js) + shared.configuration.get_temp_files().note(final_js) save_intermediate('modularized') @@ -3575,10 +3492,10 @@ def module_export_name_substitution(): src = re.sub(r'{\s*[\'"]?__EMSCRIPTEN_PRIVATE_MODULE_EXPORT_NAME_SUBSTITUTION__[\'"]?:\s*1\s*}', replacement, src) # For Node.js and other shell environments, create an unminified Module object so that # loading external .asm.js file that assigns to Module['asm'] works even when Closure is used. - if settings.MINIMAL_RUNTIME and not settings.MODULARIZE and (shared.target_environment_may_be('node') or shared.target_environment_may_be('shell')): + if settings.MINIMAL_RUNTIME and (shared.target_environment_may_be('node') or shared.target_environment_may_be('shell')): src = 'if(typeof Module==="undefined"){var Module={};}\n' + src write_file(final_js, src) - shared.get_temp_files().note(final_js) + shared.configuration.get_temp_files().note(final_js) save_intermediate('module_export_name_substitution') @@ -3920,7 +3837,7 @@ def parse_symbol_list_file(contents): return [v.strip() for v in values] -def parse_value(text, expected_type): +def parse_value(text, expect_list): # Note that using response files can introduce whitespace, if the file # has a newline at the end. For that reason, we rstrip() in relevant # places here. @@ -3975,7 +3892,7 @@ def parse_string_list(text): return [] return parse_string_list_members(text) - if expected_type == list or (text and text[0] == '['): + if expect_list or (text and text[0] == '['): # if json parsing fails, we fall back to our own parser, which can handle a few # simpler syntaxes try: @@ -3983,12 +3900,6 @@ def parse_string_list(text): except ValueError: return parse_string_list(text) - if expected_type == float: - try: - return float(text) - except ValueError: - pass - try: if text.startswith('0x'): base = 16 @@ -4003,13 +3914,13 @@ def validate_arg_level(level_string, max_level, err_msg, clamp=False): try: level = int(level_string) except ValueError: - exit_with_error(err_msg) + raise Exception(err_msg) if clamp: if level > max_level: logger.warning("optimization level '-O" + level_string + "' is not supported; using '-O" + str(max_level) + "' instead") level = max_level if not 0 <= level <= max_level: - exit_with_error(err_msg) + raise Exception(err_msg) return level diff --git a/eng/patches/emcmake.py b/eng/patches/emcmake.py new file mode 100644 index 00000000000..b16f565c1c8 --- /dev/null +++ b/eng/patches/emcmake.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright 2016 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +import sys +from tools import shared +from tools import config +from tools import utils +from subprocess import CalledProcessError + + +# +# Main run() function +# +def run(): + if len(sys.argv) < 2 or sys.argv[1] in ('--version', '--help'): + print('''\ +emcmake is a helper for cmake, setting various environment +variables so that emcc etc. are used. Typical usage: + + emcmake cmake [FLAGS] +''', file=sys.stderr) + return 1 + + args = sys.argv[1:] + + def has_substr(args, substr): + return any(substr in s for s in args) + + # Append the Emscripten toolchain file if the user didn't specify one. + if not has_substr(args, '-DCMAKE_TOOLCHAIN_FILE'): + args.append('-DCMAKE_TOOLCHAIN_FILE=' + utils.path_from_root('cmake/Modules/Platform/Emscripten.cmake')) + + if not has_substr(args, '-DCMAKE_CROSSCOMPILING_EMULATOR'): + node_js = config.NODE_JS[0] + # In order to allow cmake to run code built with pthreads we need to pass some extra flags to node. + # Note that we also need --experimental-wasm-bulk-memory which is true by default and hence not added here + # See https://github.com/emscripten-core/emscripten/issues/15522 + args.append(f'-DCMAKE_CROSSCOMPILING_EMULATOR={node_js};--experimental-wasm-threads') + + # On Windows specify MinGW Makefiles or ninja if we have them and no other + # toolchain was specified, to keep CMake from pulling in a native Visual + # Studio, or Unix Makefiles. + if utils.WINDOWS and not any(arg.startswith('-G') for arg in args): + if utils.which('mingw32-make'): + args += ['-G', 'MinGW Makefiles'] + elif utils.which('ninja'): + args += ['-G', 'Ninja'] + else: + print('emcmake: no compatible cmake generator found; Please install ninja or mingw32-make, or specify a generator explicitly using -G', file=sys.stderr) + return 1 + + print('configure: ' + shared.shlex_join(args), file=sys.stderr) + try: + shared.check_call(args) + return 0 + except CalledProcessError as e: + return e.returncode + + +if __name__ == '__main__': + sys.exit(run())