diff --git a/.circleci/config.yml b/.circleci/config.yml index e21abe9f61..71292f78d2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -201,9 +201,9 @@ jobs: - checkout - run: brew install grep - run: - name: install bazel + name: install bazelisk command: | - brew install bazel + brew install bazelisk - run: test/test_bazel_mac.sh test-bazel-windows: diff --git a/bazel/BUILD b/bazel/BUILD index dfec803d24..87fe18133b 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -35,23 +35,34 @@ config_setting( filegroup(name = "empty") alias( - name = "binaries", + name = "compiler_files", actual = select({ - ":linux": "@emscripten_bin_linux//:all", - ":macos": "@emscripten_bin_mac//:all", - ":macos_arm64": "@emscripten_bin_mac_arm64//:all", - ":windows": "@emscripten_bin_win//:all", + ":linux": "@emscripten_bin_linux//:compiler_files", + ":macos": "@emscripten_bin_mac//:compiler_files", + ":macos_arm64": "@emscripten_bin_mac_arm64//:compiler_files", + ":windows": "@emscripten_bin_win//:compiler_files", "//conditions:default": ":empty", }), ) alias( - name = "node_modules", + name = "linker_files", actual = select({ - ":linux": "@emscripten_npm_linux//:node_modules", - ":macos": "@emscripten_npm_mac//:node_modules", - ":macos_arm64": "@emscripten_npm_mac//:node_modules", - ":windows": "@emscripten_npm_win//:node_modules", + ":linux": "@emscripten_bin_linux//:linker_files", + ":macos": "@emscripten_bin_mac//:linker_files", + ":macos_arm64": "@emscripten_bin_mac_arm64//:linker_files", + ":windows": "@emscripten_bin_win//:linker_files", + "//conditions:default": ":empty", + }), +) + +alias( + name = "ar_files", + actual = select({ + ":linux": "@emscripten_bin_linux//:ar_files", + ":macos": "@emscripten_bin_mac//:ar_files", + ":macos_arm64": "@emscripten_bin_mac_arm64//:ar_files", + ":windows": "@emscripten_bin_win//:ar_files", "//conditions:default": ":empty", }), ) diff --git a/bazel/emscripten_deps.bzl b/bazel/emscripten_deps.bzl index 95801ba758..01baa01351 100644 --- a/bazel/emscripten_deps.bzl +++ b/bazel/emscripten_deps.bzl @@ -1,10 +1,57 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@build_bazel_rules_nodejs//:index.bzl", "npm_install", "node_repositories") +load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories", "npm_install") load(":revisions.bzl", "EMSCRIPTEN_TAGS") def _parse_version(v): return [int(u) for u in v.split(".")] +BUILD_FILE_CONTENT_TEMPLATE = """ +package(default_visibility = ['//visibility:public']) + +filegroup( + name = "includes", + srcs = glob([ + "emscripten/cache/sysroot/include/c++/v1/**", + "emscripten/cache/sysroot/include/compat/**", + "emscripten/cache/sysroot/include/**", + "lib/clang/15.0.0/include/**", + ]), +) + +filegroup( + name = "compiler_files", + srcs = [ + "emscripten/emcc.py", + "bin/clang{bin_extension}", + "bin/clang++{bin_extension}", + ":includes", + ], +) + +filegroup( + name = "linker_files", + srcs = [ + "emscripten/emcc.py", + "bin/clang{bin_extension}", + "bin/llc{bin_extension}", + "bin/llvm-ar{bin_extension}", + "bin/llvm-nm{bin_extension}", + "bin/llvm-objcopy{bin_extension}", + "bin/wasm-emscripten-finalize{bin_extension}", + "bin/wasm-ld{bin_extension}", + "bin/wasm-opt{bin_extension}", + ] + glob(["emscripten/node_modules/**"]), +) + +filegroup( + name = "ar_files", + srcs = [ + "emscripten/emar.py", + "bin/llvm-ar{bin_extension}", + ], +) +""" + def emscripten_deps(emscripten_version = "latest"): version = emscripten_version @@ -36,7 +83,7 @@ def emscripten_deps(emscripten_version = "latest"): strip_prefix = "install", url = emscripten_url.format("linux", revision.hash, "", "tbz2"), sha256 = revision.sha_linux, - build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", + build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension = ""), type = "tar.bz2", ) @@ -46,7 +93,7 @@ def emscripten_deps(emscripten_version = "latest"): strip_prefix = "install", url = emscripten_url.format("mac", revision.hash, "", "tbz2"), sha256 = revision.sha_mac, - build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", + build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension = ""), type = "tar.bz2", ) @@ -56,7 +103,7 @@ def emscripten_deps(emscripten_version = "latest"): strip_prefix = "install", url = emscripten_url.format("mac", revision.hash, "-arm64", "tbz2"), sha256 = revision.sha_mac_arm64, - build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", + build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension = ""), type = "tar.bz2", ) @@ -66,7 +113,7 @@ def emscripten_deps(emscripten_version = "latest"): strip_prefix = "install", url = emscripten_url.format("win", revision.hash, "", "zip"), sha256 = revision.sha_win, - build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", + build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension = ".exe"), type = "zip", ) diff --git a/bazel/emscripten_toolchain/BUILD.bazel b/bazel/emscripten_toolchain/BUILD.bazel index 4c85acfce2..d101c67061 100644 --- a/bazel/emscripten_toolchain/BUILD.bazel +++ b/bazel/emscripten_toolchain/BUILD.bazel @@ -8,45 +8,43 @@ package(default_visibility = ["//visibility:public"]) node_files = "@nodejs_host//:node_files" if existing_rule("@nodejs_host//:node_files") else "@nodejs//:node_files" filegroup( - name = "common-script-includes", + name = "common_files", srcs = [ - "emar.sh", - "emar.bat", - "emcc.sh", - "emcc.bat", "emscripten_config", "env.sh", "env.bat", - "@emsdk//:binaries", - "@emsdk//:node_modules", node_files, ], ) filegroup( - name = "compile-emscripten", - srcs = [":common-script-includes"], + name = "compiler_files", + srcs = [ + "emcc.sh", + "emcc.bat", + "@emsdk//:compiler_files", + ":common_files", + ], ) filegroup( - name = "link-emscripten", + name = "linker_files", srcs = [ "emcc_link.sh", "emcc_link.bat", "link_wrapper.py", - ":common-script-includes", - "@emsdk//:binaries", - node_files, + "@emsdk//:linker_files", + ":common_files", ], ) filegroup( - name = "every-file", + name = "ar_files", srcs = [ - ":compile-emscripten", - ":link-emscripten", - "@emsdk//:binaries", - node_files, + "emar.sh", + "emar.bat", + "@emsdk//:ar_files", + ":common_files", ], ) @@ -59,7 +57,7 @@ emscripten_cc_toolchain_config_rule( name = "wasm", cpu = "wasm", em_config = "emscripten_config", - emscripten_binaries = "@emsdk//:binaries", + emscripten_binaries = "@emsdk//:compiler_files", script_extension = select({ "@bazel_tools//src/conditions:host_windows": "bat", "//conditions:default": "sh", @@ -68,12 +66,12 @@ emscripten_cc_toolchain_config_rule( cc_toolchain( name = "cc-compiler-wasm", - all_files = ":every-file", - ar_files = ":common-script-includes", + all_files = ":empty", + ar_files = ":ar_files", as_files = ":empty", - compiler_files = ":compile-emscripten", + compiler_files = ":compiler_files", dwp_files = ":empty", - linker_files = ":link-emscripten", + linker_files = ":linker_files", objcopy_files = ":empty", strip_files = ":empty", toolchain_config = "wasm", diff --git a/bazel/emscripten_toolchain/emscripten.BUILD b/bazel/emscripten_toolchain/emscripten.BUILD deleted file mode 100644 index 6f11852874..0000000000 --- a/bazel/emscripten_toolchain/emscripten.BUILD +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ['//visibility:public']) - -filegroup( - name = "all", - srcs = glob(["**"]), -) diff --git a/bazel/emscripten_toolchain/emscripten_config b/bazel/emscripten_toolchain/emscripten_config index 86a864b1ee..85a98e34b0 100644 --- a/bazel/emscripten_toolchain/emscripten_config +++ b/bazel/emscripten_toolchain/emscripten_config @@ -3,8 +3,8 @@ import platform ROOT_DIR = os.environ["ROOT_DIR"] EMSCRIPTEN_ROOT = os.environ["EMSCRIPTEN"] -BINARYEN_ROOT = ROOT_DIR + "/" + os.environ["EM_BIN_PATH"] -LLVM_ROOT = BINARYEN_ROOT + "/bin" +BINARYEN_ROOT = os.path.join(ROOT_DIR, os.environ["EM_BIN_PATH"]) +LLVM_ROOT = os.path.join(BINARYEN_ROOT, "bin") FROZEN_CACHE = True system = platform.system() @@ -12,3 +12,12 @@ system = platform.system() machine = "arm64" if platform.machine() == "arm64" else "amd64" nodejs_binary = "bin/nodejs/node.exe" if(system =="Windows") else "bin/node" NODE_JS = ROOT_DIR + "/external/nodejs_{}_{}/{}".format(system.lower(), machine, nodejs_binary) + + +# This works around an issue with Bazel RBE where the symlinks in node_modules/.bin +# are uploaded as the linked files, which means the cli.js cannot load its +# dependencies from the expected locations. +# See https://github.com/emscripten-core/emscripten/pull/16640 for more +if system != "Windows": + CLOSURE_COMPILER = [NODE_JS, os.path.join(EMSCRIPTEN_ROOT, "node_modules", + "google-closure-compiler", "cli.js")] diff --git a/bazel/emscripten_toolchain/wasm_binary.py b/bazel/emscripten_toolchain/wasm_binary.py index 8c67c7cc8f..d7d6142376 100644 --- a/bazel/emscripten_toolchain/wasm_binary.py +++ b/bazel/emscripten_toolchain/wasm_binary.py @@ -3,16 +3,9 @@ This script will take a tar archive containing the output of the emscripten toolchain. This file contains any output files produced by a wasm_cc_binary or a cc_binary built with --config=wasm. The files are extracted into the given -output path. +output paths. -The name of archive is expected to be of the format `foo` or `foo.XXX` and -the contents are expected to be foo.js and foo.wasm. - -Several optional files may also be in the archive, including but not limited to -foo.js.mem, pthread-main.js, and foo.wasm.map. - -If the file is not a tar archive, the passed file will simply be copied to its -destination. +The contents of the archive are expected to match the given outputs extnames. This script and its accompanying Bazel rule should allow you to extract a WebAssembly binary into a larger web application. @@ -29,39 +22,34 @@ def ensure(f): pass -def check(f): - if not os.path.exists(f): - raise Exception('Expected file in archive: %s' % f) - - def main(): parser = argparse.ArgumentParser() parser.add_argument('--archive', help='The archive to extract from.') - parser.add_argument('--output_path', help='The path to extract into.') + parser.add_argument('--outputs', help='Comma separated list of files that should be extracted from the archive. Only the extname has to match a file in the archive.') + parser.add_argument('--allow_empty_outputs', help='If an output listed in --outputs does not exist, create it anyways.', action='store_true') args = parser.parse_args() args.archive = os.path.normpath(args.archive) + args.outputs = args.outputs.split(",") - basename = os.path.basename(args.archive) - stem = basename.split('.')[0] - - # Extract all files from the tarball. tar = tarfile.open(args.archive) - tar.extractall(args.output_path) - - # At least one of these two files should exist at this point. - ensure(os.path.join(args.output_path, stem + '.js')) - ensure(os.path.join(args.output_path, stem + '.wasm')) - # And can optionally contain these extra files. - ensure(os.path.join(args.output_path, stem + '.wasm.map')) - ensure(os.path.join(args.output_path, stem + '.worker.js')) - ensure(os.path.join(args.output_path, stem + '.js.mem')) - ensure(os.path.join(args.output_path, stem + '.data')) - ensure(os.path.join(args.output_path, stem + '.fetch.js')) - ensure(os.path.join(args.output_path, stem + '.js.symbols')) - ensure(os.path.join(args.output_path, stem + '.wasm.debug.wasm')) - ensure(os.path.join(args.output_path, stem + '.html')) + for member in tar.getmembers(): + extname = '.' + member.name.split('.', 1)[1] + for idx, output in enumerate(args.outputs): + if output.endswith(extname): + member_file = tar.extractfile(member) + with open(output, "wb") as output_file: + output_file.write(member_file.read()) + args.outputs.pop(idx) + break + + for output in args.outputs: + extname = '.' + output.split('.', 1)[1] + if args.allow_empty_outputs: + ensure(output) + else: + print("[ERROR] Archive does not contain file with extname: %s" % extname) if __name__ == '__main__': diff --git a/bazel/emscripten_toolchain/wasm_cc_binary.bzl b/bazel/emscripten_toolchain/wasm_cc_binary.bzl index dbdb2384ff..9cf6328aff 100644 --- a/bazel/emscripten_toolchain/wasm_cc_binary.bzl +++ b/bazel/emscripten_toolchain/wasm_cc_binary.bzl @@ -54,11 +54,81 @@ _wasm_transition = transition( ], ) -def _wasm_binary_impl(ctx): +_ALLOW_OUTPUT_EXTNAMES = [ + ".js", + ".wasm", + ".wasm.map", + ".worker.js", + ".js.mem", + ".data", + ".fetch.js", + ".js.symbols", + ".wasm.debug.wasm", + ".html", +] + +_WASM_BINARY_COMMON_ATTRS = { + "backend": attr.string( + default = "_default", + values = ["_default", "emscripten", "llvm"], + ), + "cc_target": attr.label( + cfg = _wasm_transition, + mandatory = True, + ), + "exit_runtime": attr.bool( + default = False, + ), + "threads": attr.string( + default = "_default", + values = ["_default", "emscripten", "off"], + ), + "simd": attr.bool( + default = False, + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + "_wasm_binary_extractor": attr.label( + executable = True, + allow_files = True, + cfg = "exec", + default = Label("@emsdk//emscripten_toolchain:wasm_binary"), + ), +} + +def _wasm_cc_binary_impl(ctx): args = ctx.actions.args() - args.add("--output_path", ctx.outputs.loader.dirname) + cc_target = ctx.attr.cc_target[0] + + for output in ctx.outputs.outputs: + valid_extname = False + for allowed_extname in _ALLOW_OUTPUT_EXTNAMES: + if output.path.endswith(allowed_extname): + valid_extname = True + break + if not valid_extname: + fail("Invalid output '{}'. Allowed extnames: {}".format(output.basename, ", ".join(_ALLOW_OUTPUT_EXTNAMES))) + args.add_all("--archive", ctx.files.cc_target) + args.add_joined("--outputs", ctx.outputs.outputs, join_with = ",") + ctx.actions.run( + inputs = ctx.files.cc_target, + outputs = ctx.outputs.outputs, + arguments = [args], + executable = ctx.executable._wasm_binary_extractor, + ) + + return DefaultInfo( + files = depset(ctx.outputs.outputs), + # This is needed since rules like web_test usually have a data + # dependency on this target. + data_runfiles = ctx.runfiles(transitive_files = depset(ctx.outputs.outputs)), + ) + +def _wasm_cc_binary_legacy_impl(ctx): + cc_target = ctx.attr.cc_target[0] outputs = [ ctx.outputs.loader, ctx.outputs.wasm, @@ -72,6 +142,11 @@ def _wasm_binary_impl(ctx): ctx.outputs.html, ] + args = ctx.actions.args() + args.add("--allow_empty_outputs") + args.add_all("--archive", ctx.files.cc_target) + args.add_joined("--outputs", outputs, join_with = ",") + ctx.actions.run( inputs = ctx.files.cc_target, outputs = outputs, @@ -87,7 +162,19 @@ def _wasm_binary_impl(ctx): data_runfiles = ctx.runfiles(transitive_files = depset(outputs)), ) -def _wasm_binary_outputs(name, cc_target): +_wasm_cc_binary = rule( + name = "wasm_cc_binary", + implementation = _wasm_cc_binary_impl, + attrs = dict( + _WASM_BINARY_COMMON_ATTRS, + outputs = attr.output_list( + allow_empty = False, + mandatory = True, + ), + ), +) + +def _wasm_binary_legacy_outputs(name, cc_target): basename = cc_target.name basename = basename.split(".")[0] outputs = { @@ -105,6 +192,13 @@ def _wasm_binary_outputs(name, cc_target): return outputs +_wasm_cc_binary_legacy = rule( + name = "wasm_cc_binary", + implementation = _wasm_cc_binary_legacy_impl, + attrs = _WASM_BINARY_COMMON_ATTRS, + outputs = _wasm_binary_legacy_outputs, +) + # Wraps a C++ Blaze target, extracting the appropriate files. # # This rule will transition to the emscripten toolchain in order @@ -113,36 +207,10 @@ def _wasm_binary_outputs(name, cc_target): # Args: # name: The name of the rule. # cc_target: The cc_binary or cc_library to extract files from. -wasm_cc_binary = rule( - implementation = _wasm_binary_impl, - attrs = { - "backend": attr.string( - default = "_default", - values = ["_default", "emscripten", "llvm"], - ), - "cc_target": attr.label( - cfg = _wasm_transition, - mandatory = True, - ), - "exit_runtime": attr.bool( - default = False, - ), - "threads": attr.string( - default = "_default", - values = ["_default", "emscripten", "off"], - ), - "simd": attr.bool( - default = False, - ), - "_allowlist_function_transition": attr.label( - default = "@bazel_tools//tools/allowlists/function_transition_allowlist", - ), - "_wasm_binary_extractor": attr.label( - executable = True, - allow_files = True, - cfg = "exec", - default = Label("@emsdk//emscripten_toolchain:wasm_binary"), - ), - }, - outputs = _wasm_binary_outputs, -) +def wasm_cc_binary(outputs = None, **kwargs): + # for backwards compatibility if no outputs are set the deprecated + # implementation is used. + if not outputs: + _wasm_cc_binary_legacy(**kwargs) + else: + _wasm_cc_binary(outputs = outputs, **kwargs) diff --git a/bazel/revisions.bzl b/bazel/revisions.bzl index 1a4efda442..f3505bbedd 100644 --- a/bazel/revisions.bzl +++ b/bazel/revisions.bzl @@ -2,6 +2,27 @@ # DO NOT MODIFY EMSCRIPTEN_TAGS = { + "3.1.12": struct( + hash = "a8c3b314d61e2bb98581d522f858132b2fc21488", + sha_linux = "ac8ae46b2fe2fbef07077cdeefc8288d2a73e3189958f32b36f2d17d868275d0", + sha_mac = "c33afddd7c8f7a5293cb427ef26eb65f51fa3121d0577568824174227aa37ef3", + sha_mac_arm64 = "253feff779385d2499764cd988175446e21db8cbb9952746e96969c2a763924c", + sha_win = "04015fb6a1b4ad4d7c16587a7eeaabf19c5b35097f3e28efa029c0c67547067c", + ), + "3.1.11": struct( + hash = "8c3a799341c01148692c52fda73bbba5e89c5727", + sha_linux = "ba52cfd784362530866c9d554ddc62cfa3f0690f44007c0b3b36e189bb579d5e", + sha_mac = "c46548425e0bf4acd3c4275aff6a463c90ff1faf283ae7f5237d8c17bf84d779", + sha_mac_arm64 = "c5ae40c468955ed02b86c54061278d2b4075b1230612bae5910f836aa9c200b3", + sha_win = "74481a1998236fd9d296f367584934d5ab8bbf174446ceb647f714031671de98", + ), + "3.1.10": struct( + hash = "8bd05c7221b4ce34d4bedec40b672d94e681a765", + sha_linux = "f5a937383b5c9fa15071a31d679a2ddd5c03bc8952cbbd5bfbf7c0a86c2dae5a", + sha_mac = "e73491f2787cbda75e718c3947916b57259164eddd9b2db16b9c876d3deb16a9", + sha_mac_arm64 = "d7485ce3b13f183484af5163d7bec79ece9a1fdc5845f8152e36270e6f90cfd9", + sha_win = "dd75061405bc902ecd983bd3e4cfd6931a866e1c9de602c4458280cbeb271720", + ), "3.1.9": struct( hash = "edabe25af34554d19c046078f853999b074259ca", sha_linux = "89fa75c981e47ad19942b8236d2604b2666dfd516a08626aaa1bfb0d657c87bf", diff --git a/bazel/test_external/BUILD b/bazel/test_external/BUILD index 4e69238cf6..e3a8e9e645 100644 --- a/bazel/test_external/BUILD +++ b/bazel/test_external/BUILD @@ -8,6 +8,10 @@ cc_binary( wasm_cc_binary( name = "hello-world-wasm", cc_target = ":hello-world", + outputs = [ + "hello-world.js", + "hello-world.wasm", + ], ) BASE_LINKOPTS = [ diff --git a/emscripten-releases-tags.json b/emscripten-releases-tags.json index b9dc4122cc..e5f0140562 100644 --- a/emscripten-releases-tags.json +++ b/emscripten-releases-tags.json @@ -1,6 +1,6 @@ { "aliases": { - "latest": "3.1.9", + "latest": "3.1.12", "latest-sdk": "latest", "latest-64bit": "latest", "sdk-latest-64bit": "latest", @@ -9,6 +9,12 @@ "latest-releases-upstream": "latest" }, "releases": { + "3.1.12": "a8c3b314d61e2bb98581d522f858132b2fc21488", + "3.1.12-asserts": "0ec1936aa3cb809d96abfebcc5356cd0cb15f6b8", + "3.1.11": "8c3a799341c01148692c52fda73bbba5e89c5727", + "3.1.11-asserts": "4d27b98351e021e9b7d2681a84cbab5a0ddc7a88", + "3.1.10": "8bd05c7221b4ce34d4bedec40b672d94e681a765", + "3.1.10-asserts": "d9b20effb2d660936fb5be525744e941fd900bc6", "3.1.9": "edabe25af34554d19c046078f853999b074259ca", "3.1.9-asserts": "c751721b1dfa47c03ede0f0da89be453c79b34ef", "3.1.8": "8c9e0a76ebed2c5e88a718d43e8b62452def3771", diff --git a/eng/Versions.props b/eng/Versions.props index ba47d0f75c..94f677d474 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -5,7 +5,7 @@ preview 6 7.0.100 - 3.1.7 + 3.1.12 7.0.0-beta.22276.1