bazel: Use hermetic toolchain for LLVM

.bazelrc

@@ -45,12 +45,6 @@ build --incompatible_merge_fixed_and_default_shell_env
 # A workaround for slow ICU download.
 build --http_timeout_scaling=6.0
 
-# Pass CC, CXX and LLVM_CONFIG variables from the environment.
-# We assume they have stable values, so this won't cause action cache misses.
-build --action_env=CC --host_action_env=CC
-build --action_env=CXX --host_action_env=CXX
-build --action_env=LLVM_CONFIG --host_action_env=LLVM_CONFIG
-
 # Allow stamped caches to bust when local filesystem changes.
 # Requires setting `BAZEL_VOLATILE_DIRTY` in the env.
 build --action_env=BAZEL_VOLATILE_DIRTY --host_action_env=BAZEL_VOLATILE_DIRTY
@@ -99,13 +93,14 @@ build:linux --cxxopt=-fsized-deallocation --host_cxxopt=-fsized-deallocation
 build:linux --conlyopt=-fexceptions
 build:linux --fission=dbg,opt
 build:linux --features=per_object_debug_info
-build:linux --action_env=BAZEL_LINKOPTS=-lm:-fuse-ld=gold
 
 # macOS
 build:macos --action_env=PATH=/opt/homebrew/bin:/opt/local/bin:/usr/local/bin:/usr/bin:/bin
 build:macos --host_action_env=PATH=/opt/homebrew/bin:/opt/local/bin:/usr/local/bin:/usr/bin:/bin
 build:macos --define tcmalloc=disabled
 build:macos --cxxopt=-Wno-nullability-completeness
+build:macos --@toolchains_llvm//toolchain/config:compiler-rt=false
+build:macos --@toolchains_llvm//toolchain/config:libunwind=false
 
 
 #############################################################################
@@ -114,15 +109,14 @@ build:macos --cxxopt=-Wno-nullability-completeness
 
 # Common flags for Clang (shared between all clang variants)
 common:clang-common --linkopt=-fuse-ld=lld
-common:clang-common --action_env=BAZEL_COMPILER=clang
-common:clang-common --action_env=LDFLAGS="-fuse-ld=lld"
-common:clang-common --action_env=CC=clang --host_action_env=CC=clang
-common:clang-common --action_env=CXX=clang++ --host_action_env=CXX=clang++
+common:clang-common --@toolchains_llvm//toolchain/config:compiler-rt=false
+common:clang-common --@toolchains_llvm//toolchain/config:libunwind=false
 
 # Clang with libc++ (default)
 common:clang --config=clang-common
 common:clang --config=libc++
 common:clang --host_platform=@clang_platform
+common:clang --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
 
 # Clang installed to non-standard location (ie not /opt/llvm/)
 common:clang-local --config=clang-common
@@ -147,6 +141,8 @@ build:gcc --cxxopt=-Wno-dangling-reference
 build:gcc --cxxopt=-Wno-nonnull-compare
 build:gcc --linkopt=-fuse-ld=gold --host_linkopt=-fuse-ld=gold
 build:gcc --host_platform=@envoy//bazel/rbe/toolchains:rbe_linux_gcc_platform
+build:gcc --linkopt=-fuse-ld=gold --host_linkopt=-fuse-ld=gold
+build:gcc --action_env=BAZEL_LINKOPTS=-lm:-fuse-ld=gold
 
 # libc++ - default for clang
 common:libc++ --action_env=CXXFLAGS=-stdlib=libc++

.github/config.yml

@@ -1,19 +1,19 @@
 agent-ubuntu: ubuntu-24.04
 build-image:
   # Authoritative configuration for build image/s
-  repo: docker.io/envoyproxy/envoy-build-ubuntu
+  repo: docker.io/envoyproxy/envoy-build
   repo-gcr: gcr.io/envoy-ci/envoy-build
-  # Until toolchains land, these all just point to the main image or the mobile one
-  sha: faefd182b53aa2148382b3aaf4a8253f0981b77cb6e6104c7bdda6aa3e3ea2ca
-  sha-ci: faefd182b53aa2148382b3aaf4a8253f0981b77cb6e6104c7bdda6aa3e3ea2ca
-  sha-devtools: faefd182b53aa2148382b3aaf4a8253f0981b77cb6e6104c7bdda6aa3e3ea2ca
-  sha-docker: faefd182b53aa2148382b3aaf4a8253f0981b77cb6e6104c7bdda6aa3e3ea2ca
-  sha-gcc: faefd182b53aa2148382b3aaf4a8253f0981b77cb6e6104c7bdda6aa3e3ea2ca
-  sha-mobile: 224feb2b7d513bf7f2ac0c7c529db1cb46398952f31b7d46e51211642d0b2ba0
-  sha-worker: faefd182b53aa2148382b3aaf4a8253f0981b77cb6e6104c7bdda6aa3e3ea2ca
+  # default ci caching (ci)
+  sha: 5fcc9d3e10f1a0e628250b44b4c39bde1bdfc6cb8fe6075838a732c2ba04ef42
+  sha-ci: 5fcc9d3e10f1a0e628250b44b4c39bde1bdfc6cb8fe6075838a732c2ba04ef42
+  sha-devtools: fa69c24f030ce114371de36b2b74ede3370ca70463c7bbcc35e82db900bf2566
+  sha-docker: 82a823b14ac45da97942d24a649ff5b9db35ebb640999c449d391f4b88711e83
+  sha-gcc: c8a21f4efd097f063edd94c6473f33fd8b2346d4b2e3105b8d029287ba919cb5
+  sha-mobile: 04ae4e9abd445f2cfd4cbad24dfcdef85144cd150e5f268ce6f5aade92fdd4d4
+  sha-worker: 3d28abae22d41631f14cf3264694dd294fe9f9a924fcc35b202d27afb59db7ad
   # TODO: remove this dupe (currently used by ci request handler)
-  mobile-sha: 224feb2b7d513bf7f2ac0c7c529db1cb46398952f31b7d46e51211642d0b2ba0
-  tag: abd41d4081d77bacbb246c9b952d49f816a9ee2f
+  mobile-sha: 04ae4e9abd445f2cfd4cbad24dfcdef85144cd150e5f268ce6f5aade92fdd4d4
+  tag: e0b4993c78551c1638ac00cf21d36313fe35d81d
 
 config:
   envoy:
@@ -401,7 +401,7 @@ tables:
     title: Build image
     table-title: Container image/s (as used in this CI run)
     filter: |
-      "https://hub.docker.com/r/envoyproxy/envoy-build-ubuntu/tags?page=1&name=" as $dockerLink
+      "https://hub.docker.com/r/envoyproxy/envoy-build/tags?page=1&name=" as $dockerLink
       | .request["build-image"]
       | del(.changed)
       | with_entries(
@@ -413,7 +413,7 @@ tables:
     title: Build image (current)
     table-title: Current or previous container image
     filter: |
-      "https://hub.docker.com/r/envoyproxy/envoy-build-ubuntu/tags?page=1&name=" as $dockerLink
+      "https://hub.docker.com/r/envoyproxy/envoy-build/tags?page=1&name=" as $dockerLink
       | if .request["build-image"].changed then
           .request["build-image-current"]
           | with_entries(

.github/workflows/codeql-daily.yml

@@ -74,10 +74,10 @@ jobs:
 
     - name: Build
       run: |
-       bazel/setup_clang.sh "$(realpath bin/clang18.1.8)"
        bazelisk shutdown
        bazel build \
            -c fastbuild \
+           --repo_env=BAZEL_LLVM_PATH="$(realpath bin/clang18.1.8)" \
            --spawn_strategy=local \
            --discard_analysis_cache \
            --nouse_action_cache \

.github/workflows/codeql-push.yml

@@ -110,16 +110,15 @@ jobs:
 
     - name: Build
       run: |
-        bazel/setup_clang.sh "$(realpath bin/clang18.1.8)"
-        PATH=$(bin/clang18.1.8/bin/llvm-config --bindir):$local_clang:$PATH
         bazel shutdown
         bazel build \
             -c fastbuild \
+            --repo_env=BAZEL_LLVM_PATH="$(realpath bin/clang18.1.8)" \
             --spawn_strategy=local \
             --discard_analysis_cache \
             --nouse_action_cache \
             --features="-layering_check" \
-            --config=clang-local \
+            --config=clang \
             --config=ci \
             ${BUILD_TARGETS:-${MINIMAL_BUILD_TARGET}}
         echo -e "Built targets...\n${BUILD_TARGETS:-${MINIMAL_BUILD_TARGET}}"

.github/workflows/envoy-dependency.yml

@@ -187,15 +187,14 @@ jobs:
       id: build-images
       uses: envoyproxy/toolshed/gh-actions/docker/shas@actions-v0.3.34
       with:
-        # Until toolchains land, these all just point to the main image or the mobile one
         images: |
-           sha: docker.io/envoyproxy/envoy-build-ubuntu:${{ steps.build-tools.outputs.tag }}
-           sha-ci: docker.io/envoyproxy/envoy-build-ubuntu:${{ steps.build-tools.outputs.tag }}
-           sha-devtools: docker.io/envoyproxy/envoy-build-ubuntu:${{ steps.build-tools.outputs.tag }}
-           sha-docker: docker.io/envoyproxy/envoy-build-ubuntu:${{ steps.build-tools.outputs.tag }}
-           sha-gcc: docker.io/envoyproxy/envoy-build-ubuntu:${{ steps.build-tools.outputs.tag }}
-           sha-mobile: docker.io/envoyproxy/envoy-build-ubuntu:mobile-${{ steps.build-tools.outputs.tag }}
-           sha-worker: docker.io/envoyproxy/envoy-build-ubuntu:${{ steps.build-tools.outputs.tag }}
+           sha: docker.io/envoyproxy/envoy-build:devtools-${{ steps.build-tools.outputs.tag }}
+           sha-ci: docker.io/envoyproxy/envoy-build:ci-${{ steps.build-tools.outputs.tag }}
+           sha-devtools: docker.io/envoyproxy/envoy-build:devtools-${{ steps.build-tools.outputs.tag }}
+           sha-docker: docker.io/envoyproxy/envoy-build:docker-${{ steps.build-tools.outputs.tag }}
+           sha-gcc: docker.io/envoyproxy/envoy-build:gcc-${{ steps.build-tools.outputs.tag }}
+           sha-mobile: docker.io/envoyproxy/envoy-build:mobile-${{ steps.build-tools.outputs.tag }}
+           sha-worker: docker.io/envoyproxy/envoy-build:worker-${{ steps.build-tools.outputs.tag }}
 
     - run: |
         SHA_REPLACE=(

api/bazel/repository_locations.bzl

@@ -176,6 +176,8 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         urls = ["https://github.com/envoyproxy/toolshed/archive/bazel-v{version}.tar.gz"],
         use_category = ["build", "controlplane", "dataplane_core"],
         implied_untracked_deps = [
+            "sysroot_linux_amd64",
+            "sysroot_linux_arm64",
             "tsan_libs",
             "msan_libs",
         ],

api/tools/data/tap2pcap_h2_ipv4.txt

@@ -1,6 +1,6 @@
-    1   0.000000    127.0.0.1 → 127.0.0.1    HTTP2 157 Magic, SETTINGS[0], WINDOW_UPDATE[0], HEADERS[1]: GET /
-    2   0.013713    127.0.0.1 → 127.0.0.1    HTTP2 91 SETTINGS[0], SETTINGS[0], WINDOW_UPDATE[0]
-    3   0.013821    127.0.0.1 → 127.0.0.1    HTTP2 63 SETTINGS[0]
-    4   0.128649    127.0.0.1 → 127.0.0.1    HTTP2 5586 HEADERS[1]: 200 OK
-    5   0.130007    127.0.0.1 → 127.0.0.1    HTTP2 7573 DATA[1]
-    6   0.131045    127.0.0.1 → 127.0.0.1    HTTP2 3152 DATA[1], DATA[1] (text/html)
+    1 0.000000000    127.0.0.1 → 127.0.0.1    HTTP2 157 Magic, SETTINGS[0], WINDOW_UPDATE[0], HEADERS[1]: GET /
+    2 0.013713000    127.0.0.1 → 127.0.0.1    HTTP2 91 SETTINGS[0], SETTINGS[0], WINDOW_UPDATE[0]
+    3 0.013821000    127.0.0.1 → 127.0.0.1    HTTP2 63 SETTINGS[0]
+    4 0.128649000    127.0.0.1 → 127.0.0.1    HTTP2 5586 HEADERS[1]: 200 OK
+    5 0.130007000    127.0.0.1 → 127.0.0.1    HTTP2 7573 DATA[1]
+    6 0.131045000    127.0.0.1 → 127.0.0.1    HTTP2 3152 DATA[1], DATA[1] (text/html)

api/tools/tap2pcap.py

@@ -73,7 +73,7 @@ def tap2pcap(tap_path, pcap_path):
         pass
 
     text2pcap_args = [
-        'text2pcap', '-D', '-t', '%Y-%m-%d %H:%M:%S.', '-6' if ipv6 else '-4',
+        'text2pcap', '-D', '-t', '%Y-%m-%d %H:%M:%S.%f', '-6' if ipv6 else '-4',
         '%s,%s' % (remote_address, local_address), '-T',
         '%d,%d' % (remote_port, local_port), '-', pcap_path
     ]

api/tools/tap2pcap_test.py

@@ -37,6 +37,6 @@
         expected_output = f.read()
     if actual_output != expected_output:
         print('Mismatch')
-        print('Expected: %s' % expected_output)
-        print('Actual: %s' % actual_output)
+        print('Expected:\n %s' % expected_output)
+        print('Actual:\n %s' % actual_output)
         sys.exit(1)

bazel/dependency_imports_extra.bzl

@@ -1,5 +1,7 @@
 load("@dynamic_modules_rust_sdk_crate_index//:defs.bzl", "crate_repositories")
+load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains")
 
 # Dependencies that rely on a first stage of envoy_dependency_imports() in dependency_imports.bzl.
 def envoy_dependency_imports_extra():
     crate_repositories()
+    llvm_register_toolchains()

bazel/envoy_test.bzl

@@ -182,7 +182,10 @@ def envoy_cc_test(
     cc_test(
         name = name,
         srcs = srcs,
-        data = data,
+        data = data + select({
+            "%s//bazel:asan_build" % repository: ["@llvm_toolchain_llvm//:symbolizer"],
+            "//conditions:default": [],
+        }),
         copts = envoy_copts(repository, test = True) + copts + envoy_pch_copts(repository, "//test:test_pch"),
         additional_linker_inputs = envoy_exported_symbols_input(),
         linkopts = _envoy_test_linkopts() + linkopts,
@@ -201,7 +204,10 @@ def envoy_cc_test(
         shard_count = shard_count,
         size = size,
         flaky = flaky,
-        env = env,
+        env = env | select({
+            "%s//bazel:asan_build" % repository: {"ASAN_SYMBOLIZER_PATH": "$(location @llvm_toolchain_llvm//:symbolizer)"},
+            "//conditions:default": {},
+        }),
         exec_properties = exec_properties,
     )
 

bazel/external/boringssl_fips.BUILD

@@ -50,15 +50,28 @@ genrule(
         "@fips_ninja//:configure.py",
     ],
     outs = ["ninja"],
-    cmd = select(ninja_build_command()),
+    cmd = select(ninja_build_command(
+        SUPPORTED_ARCHES,
+        STDLIBS,
+    )),
     toolchains = [
         "@rules_python//python:current_py_toolchain",
         "@bazel_tools//tools/cpp:current_cc_toolchain",
     ],
     tools = [
         "@bazel_tools//tools/cpp:current_cc_toolchain",
         "@rules_python//python:current_py_toolchain",
-    ],
+    ] + select({
+        "@platforms//cpu:x86_64": [
+            "@sysroot_linux_amd64//:WORKSPACE",
+            "@sysroot_linux_amd64//:sysroot",
+        ],
+        "@platforms//cpu:aarch64": [
+            "@sysroot_linux_arm64//:WORKSPACE",
+            "@sysroot_linux_arm64//:sysroot",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 genrule(
@@ -92,12 +105,16 @@ genrule(
             "@fips_cmake_linux_x86_64//:bin/cmake",
             "@fips_go_linux_amd64//:all",
             "@fips_go_linux_amd64//:bin/go",
+            "@sysroot_linux_amd64//:WORKSPACE",
+            "@sysroot_linux_amd64//:sysroot",
         ],
         "@platforms//cpu:aarch64": [
             "@fips_cmake_linux_aarch64//:all",
             "@fips_cmake_linux_aarch64//:bin/cmake",
             "@fips_go_linux_arm64//:all",
             "@fips_go_linux_arm64//:bin/go",
+            "@sysroot_linux_arm64//:WORKSPACE",
+            "@sysroot_linux_arm64//:sysroot",
         ],
         "//conditions:default": [],
     }),

bazel/external/fips_build.bzl

@@ -5,15 +5,19 @@ BUILD_COMMAND = """
 set -eo pipefail
 
 # c++
+SYSROOT="$$(realpath $$(dirname "$(location %s)"))"
 export CC="$$(realpath $(CC))"
 # bazel doesnt expose CXX so we have to construct it (or use foreign_cc)
 if [[ "%s" == "libc++" ]]; then
-    export CXXFLAGS="-stdlib=libc++"
-    export LDFLAGS="-fuse-ld=lld -stdlib=libc++ -lc++ -lc++abi -lm -pthread"
+    export CXXFLAGS="-stdlib=libc++ --sysroot=$${SYSROOT}"
+    export LDFLAGS="-fuse-ld=lld -stdlib=libc++ -lc++ -lc++abi -lm -pthread --sysroot=$${SYSROOT}"
 else
-    export CXXFLAGS=""
-    export LDFLAGS="-fuse-ld=lld -lstdc++ -lm -pthread"
+    export CXXFLAGS="--sysroot=$${SYSROOT}"
+    export LDFLAGS="-fuse-ld=lld -lstdc++ -lm -pthread --sysroot=$${SYSROOT}"
 fi
+export CGO_CFLAGS="--sysroot=$${SYSROOT}"
+export CGO_CXXFLAGS="$${CXXFLAGS}"
+export CGO_LDFLAGS="$${LDFLAGS}"
 
 # ninja
 NINJA_BINDIR=$$(realpath $$(dirname $(location :ninja_bin)))
@@ -58,13 +62,15 @@ OUT_FILE=$$(realpath $@)
 PYTHON_BIN=$$(realpath $(PYTHON3))
 export CC="$$(realpath $(CC))"
 export CXX="$$(realpath $(CC))"
+export AR="$$(realpath $(AR))"
+SYSROOT="$$(realpath $$(dirname "$(location %s)"))"
 # bazel doesnt expose CXX so we have to construct it (or use foreign_cc)
 if [[ "%s" == "libc++" ]]; then
-    export CXXFLAGS="-stdlib=libc++"
-    export LDFLAGS="-fuse-ld=lld -stdlib=libc++ -lc++ -lc++abi -lm -pthread"
+    export CXXFLAGS="-stdlib=libc++ --sysroot=$${SYSROOT}"
+    export LDFLAGS="-fuse-ld=lld -stdlib=libc++ -lc++ -lc++abi -lm -pthread --sysroot=$${SYSROOT}"
 else
-    export CXXFLAGS=""
-    export LDFLAGS="-fuse-ld=lld -lstdc++ -lm -pthread"
+    export CXXFLAGS="--sysroot=$${SYSROOT}"
+    export LDFLAGS="-fuse-ld=lld -lstdc++ -lm -pthread --sysroot=$${SYSROOT}"
 fi
 cd $$SRC_DIR
 OUTPUT=$$(mktemp)
@@ -76,20 +82,21 @@ fi
 cp ninja $$OUT_FILE
 """
 
-def _create_boringssl_fips_build_config(lib, arch, arch_alias):
+def _create_build_config(prefix, lib, arch, arch_alias):
     """Create the config_setting_group combination."""
     conditions = ["@platforms//cpu:%s" % arch]
     if lib == "libc++":
         conditions += ["@envoy//bazel:libc++_enabled"]
     selects.config_setting_group(
-        name = "%s_%s" % (arch, lib),
+        name = "%s_%s_%s" % (prefix, arch, lib),
         match_all = conditions,
     )
 
 def _create_boringssl_fips_build_command(lib, arch, arch_alias):
     """Create the command."""
-    _create_boringssl_fips_build_config(lib, arch, arch_alias)
+    _create_build_config("boringssl", lib, arch, arch_alias)
     return BUILD_COMMAND % (
+        "@sysroot_linux_%s//:WORKSPACE" % arch_alias,
         lib,
         "@fips_cmake_linux_%s" % arch,
         "@fips_go_linux_%s" % arch_alias,
@@ -98,7 +105,7 @@ def _create_boringssl_fips_build_command(lib, arch, arch_alias):
 def boringssl_fips_build_command(arches, libs):
     """Create conditional commands from the cartesian product of possible arches/stdlib."""
     return {
-        ":%s_%s" % (arch, lib): _create_boringssl_fips_build_command(
+        ":boringssl_%s_%s" % (arch, lib): _create_boringssl_fips_build_command(
             lib,
             arch,
             arch_alias,
@@ -107,9 +114,22 @@ def boringssl_fips_build_command(arches, libs):
         for lib in libs
     }
 
-def ninja_build_command():
+def _create_ninja_build_command(lib, arch, arch_alias):
+    """Create the command."""
+    _create_build_config("ninja", lib, arch, arch_alias)
+    return NINJA_BUILD_COMMAND % (
+        "@sysroot_linux_%s//:WORKSPACE" % arch_alias,
+        lib,
+    )
+
+def ninja_build_command(arches, libs):
     """Create the ninja command conditioned to correct stdlib."""
     return {
-        "@envoy//bazel:libc++_enabled": NINJA_BUILD_COMMAND % "libc++",
-        "//conditions:default": NINJA_BUILD_COMMAND % "libstdc++",
+        ":ninja_%s_%s" % (arch, lib): _create_ninja_build_command(
+            lib,
+            arch,
+            arch_alias,
+        )
+        for arch, arch_alias in arches.items()
+        for lib in libs
     }