From 5af3a8bfde952e1d74a666b25b0dd84e26df79e3 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 7 Nov 2025 16:12:49 -0500 Subject: [PATCH 01/50] update offload build steps --- src/bootstrap/src/core/build_steps/llvm.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index db2a76c4a2dff..c9a95e8e83ee3 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -461,7 +461,13 @@ impl Step for Llvm { //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. enabled_llvm_runtimes.push("openmp"); - enabled_llvm_projects.push("compiler-rt"); + enabled_llvm_runtimes.push("compiler-rt"); + + let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; + cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); + + cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); + cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); } if !enabled_llvm_projects.is_empty() { From 3873e6e40d7435e7826318b3039036d5405fca3e Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 7 Nov 2025 16:23:57 -0500 Subject: [PATCH 02/50] I surely picked the wrong runner again, right? --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index cb57478761994..ba62750643d4c 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -90,6 +90,7 @@ ENV RUST_CONFIGURE_ARGS \ --set target.x86_64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \ --set llvm.thin-lto=true \ --set llvm.ninja=false \ + --set llvm.offload=true \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From d0b06b9f6090a3d791eb1d574fb07527d9c99b27 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 7 Nov 2025 18:04:49 -0500 Subject: [PATCH 03/50] try fix --- src/bootstrap/src/core/build_steps/llvm.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index c9a95e8e83ee3..2722e9128fc10 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -463,11 +463,15 @@ impl Step for Llvm { enabled_llvm_runtimes.push("openmp"); enabled_llvm_runtimes.push("compiler-rt"); - let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; - cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); + // -DRUNTIMES_CMAKE_ARGS="-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx; + cfg.define("RUNTIMES_CMAKE_ARGS", "-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx"); + //cfg.define("LLVM_TARGETS_TO_BUILD", "host;AMDGPU;NVPTX"); - cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); - cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); + //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; + //cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); + + //cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); + //cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); } if !enabled_llvm_projects.is_empty() { From bdd4e16054c956aec31676953055edb99896fbd8 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 7 Nov 2025 21:53:46 -0500 Subject: [PATCH 04/50] try fix --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index ba62750643d4c..6e93d29f03605 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -91,6 +91,7 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.thin-lto=true \ --set llvm.ninja=false \ --set llvm.offload=true \ + --set llvm.clang=true \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From b639b4100b64483adfdb59afbd197145c50defcf Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 14 Nov 2025 18:45:51 -0500 Subject: [PATCH 05/50] try fix --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 6e93d29f03605..2a80940ad1056 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -92,6 +92,7 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.ninja=false \ --set llvm.offload=true \ --set llvm.clang=true \ + --set llvm.lld=true \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From 3f220c9bef426309eb3ce2b74ceab92f83f9e2ef Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sat, 15 Nov 2025 17:50:27 -0500 Subject: [PATCH 06/50] try fix --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 2a80940ad1056..8800c2fbe7053 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -92,7 +92,7 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.ninja=false \ --set llvm.offload=true \ --set llvm.clang=true \ - --set llvm.lld=true \ + --set rust.lld=true \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From 9850a0dd95f0f3da0f2d375cbd536d219b4335c4 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sat, 15 Nov 2025 21:03:12 -0500 Subject: [PATCH 07/50] try fix --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 8800c2fbe7053..355c387550445 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -93,6 +93,8 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.offload=true \ --set llvm.clang=true \ --set rust.lld=true \ + --set llvm.cxxflags = "-fuse-ld=lld" \ + --set llvm.cflags = "-fuse-ld=lld" \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From d659eab484378af9554231ba0fed2b9ba6845a56 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sun, 16 Nov 2025 03:20:09 -0500 Subject: [PATCH 08/50] try fix --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 355c387550445..d02b0eaf260bd 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -93,8 +93,8 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.offload=true \ --set llvm.clang=true \ --set rust.lld=true \ - --set llvm.cxxflags = "-fuse-ld=lld" \ - --set llvm.cflags = "-fuse-ld=lld" \ + --set llvm.cxxflags="-fuse-ld=lld" \ + --set llvm.cflags="-fuse-ld=lld" \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From be8c4ff0c31bf91766a14d2e687b753e15a4d332 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sun, 16 Nov 2025 09:33:38 -0500 Subject: [PATCH 09/50] try fix --- src/bootstrap/src/core/build_steps/llvm.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 2722e9128fc10..ba37c109b2f1f 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -457,6 +457,7 @@ impl Step for Llvm { // This is an experimental flag, which likely builds more than necessary. // We will optimize it when we get closer to releasing it on nightly. if builder.config.llvm_offload { + enabled_llvm_projects.push("lld"); enabled_llvm_runtimes.push("offload"); //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. From e0e17ba7ea741de93c1eb17a17fddd305d2a962a Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Sun, 16 Nov 2025 19:23:38 -0500 Subject: [PATCH 10/50] try fix --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index d02b0eaf260bd..ba62750643d4c 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -91,10 +91,6 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.thin-lto=true \ --set llvm.ninja=false \ --set llvm.offload=true \ - --set llvm.clang=true \ - --set rust.lld=true \ - --set llvm.cxxflags="-fuse-ld=lld" \ - --set llvm.cflags="-fuse-ld=lld" \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From 09a98b43cb9bff9eebbc1ddeef18ed2b3a77c99e Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 24 Nov 2025 14:28:05 -0500 Subject: [PATCH 11/50] disable compiler-rt --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index ba37c109b2f1f..23c624c3d8a37 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -462,7 +462,7 @@ impl Step for Llvm { //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. enabled_llvm_runtimes.push("openmp"); - enabled_llvm_runtimes.push("compiler-rt"); + //enabled_llvm_runtimes.push("compiler-rt"); // -DRUNTIMES_CMAKE_ARGS="-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx; cfg.define("RUNTIMES_CMAKE_ARGS", "-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx"); From 62de85c99904958c94794421cbb678bc5187039e Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 24 Nov 2025 14:31:14 -0500 Subject: [PATCH 12/50] disable lld project --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 23c624c3d8a37..3e01821ac71dc 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -457,7 +457,7 @@ impl Step for Llvm { // This is an experimental flag, which likely builds more than necessary. // We will optimize it when we get closer to releasing it on nightly. if builder.config.llvm_offload { - enabled_llvm_projects.push("lld"); + //enabled_llvm_projects.push("lld"); enabled_llvm_runtimes.push("offload"); //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. From a38c37c420835d85c91255de71e31cf932350551 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 24 Nov 2025 14:51:37 -0500 Subject: [PATCH 13/50] disable test building --- src/bootstrap/src/core/build_steps/llvm.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 3e01821ac71dc..90aef477a7cb8 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -471,6 +471,7 @@ impl Step for Llvm { //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; //cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); + cfg.define("-DLLVM_BUILD_TESTS", "OFF"); //cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); //cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); } From e47d1ed03362bc5b9406261b78498f5fbb1dd77f Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 24 Nov 2025 15:37:10 -0500 Subject: [PATCH 14/50] try again to disable tests --- src/bootstrap/src/core/build_steps/llvm.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 90aef477a7cb8..4fd8ec3846be7 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -471,7 +471,8 @@ impl Step for Llvm { //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; //cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); - cfg.define("-DLLVM_BUILD_TESTS", "OFF"); + cfg.define("LLVM_INCLUDE_TESTS", "OFF"); + cfg.define("LLVM_BUILD_TESTS", "OFF"); //cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); //cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); } From 4b8cc802e2d298e6c3cbd6c49e4f866c6bdeb09e Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 24 Nov 2025 16:32:15 -0500 Subject: [PATCH 15/50] temporarily test if we can at least build offload without openmp --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 4fd8ec3846be7..2581730392ec8 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -461,7 +461,7 @@ impl Step for Llvm { enabled_llvm_runtimes.push("offload"); //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. - enabled_llvm_runtimes.push("openmp"); + //enabled_llvm_runtimes.push("openmp"); //enabled_llvm_runtimes.push("compiler-rt"); // -DRUNTIMES_CMAKE_ARGS="-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx; From 16f406bb746b6dfd15807d9d846ba8c5ba254788 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 24 Nov 2025 16:52:23 -0500 Subject: [PATCH 16/50] disable RUNTIMES_CMAKE_ARGS --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 2581730392ec8..b819bbe4a1969 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -465,7 +465,7 @@ impl Step for Llvm { //enabled_llvm_runtimes.push("compiler-rt"); // -DRUNTIMES_CMAKE_ARGS="-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx; - cfg.define("RUNTIMES_CMAKE_ARGS", "-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx"); + //cfg.define("RUNTIMES_CMAKE_ARGS", "-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx"); //cfg.define("LLVM_TARGETS_TO_BUILD", "host;AMDGPU;NVPTX"); //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; From b25e4aa80dbbb7671ebb4a773c41e05861399f37 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Thu, 27 Nov 2025 12:54:01 -0800 Subject: [PATCH 17/50] working standalone openmp build --- src/bootstrap/src/core/build_steps/compile.rs | 5 + src/bootstrap/src/core/build_steps/llvm.rs | 129 +++++++++++++++++- 2 files changed, 131 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index ed154133dec2e..71b33797008d9 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1431,6 +1431,11 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect cargo.env("LLVM_OFFLOAD", "1"); } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); + if builder.config.llvm_enzyme { + builder.ensure(llvm::Offload { target }); + cargo.env("LLVM_OFFLOAD", "1"); + } + cargo.env("LLVM_CONFIG", &host_llvm_config); // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index b819bbe4a1969..4a71ecbab17fb 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -458,7 +458,7 @@ impl Step for Llvm { // We will optimize it when we get closer to releasing it on nightly. if builder.config.llvm_offload { //enabled_llvm_projects.push("lld"); - enabled_llvm_runtimes.push("offload"); + //enabled_llvm_runtimes.push("offload"); //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. //enabled_llvm_runtimes.push("openmp"); @@ -471,8 +471,8 @@ impl Step for Llvm { //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; //cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); - cfg.define("LLVM_INCLUDE_TESTS", "OFF"); - cfg.define("LLVM_BUILD_TESTS", "OFF"); + //cfg.define("LLVM_INCLUDE_TESTS", "OFF"); + //cfg.define("LLVM_BUILD_TESTS", "OFF"); //cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); //cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); } @@ -909,6 +909,129 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option { .or_else(|| env::var_os(var_base)) } +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Offload { + pub target: TargetSelection, +} + +impl Step for Offload { + type Output = PathBuf; + const IS_HOST: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/openmp") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Offload { target: run.target }); + } + + /// Compile OpenMP offload runtimes for `target`. + fn run(self, builder: &Builder<'_>) -> PathBuf { + if builder.config.dry_run() { + return PathBuf::from("/"); + } + let target = self.target; + + let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target }); + + let out_dir = builder.llvm_out(target); + static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); + //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { + // generate_smart_stamp_hash( + // builder, + // &builder.config.src.join("src/llvm-project/openmp"), + // builder.offload_info.sha().unwrap_or_default(), + // ) + //}); + //let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").add_stamp(smart_stamp_hash); + + trace!("checking build stamp to see if we need to rebuild offload/openmp artifacts"); + //if stamp.is_up_to_date() { + // trace!(?out_dir, "offload/openmp build artifacts are up to date"); + // if stamp.stamp().is_empty() { + // builder.info( + // "Could not determine the Offload submodule commit hash. \ + // Assuming that an Offload rebuild is not necessary.", + // ); + // builder.info(&format!( + // "To force Offload to rebuild, remove the file `{}`", + // stamp.path().display() + // )); + // } + // return out_dir; + //} + + trace!(?target, "(re)building openmp artifacts"); + builder.info(&format!("Building OpenMP for {target}")); + //t!(stamp.remove()); + let _time = helpers::timeit(builder); + t!(fs::create_dir_all(&out_dir)); + + builder.config.update_submodule(Path::new("src").join("llvm-project").to_str().unwrap()); + let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/runtimes/")); + configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]); + + // Re-use the same flags as llvm to control the level of debug information + // generated by Enzyme. + // FIXME(ZuseZ4): Find a nicer way to use Enzyme Debug builds. + let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { + (false, _) => "Debug", + (true, false) => "Release", + (true, true) => "RelWithDebInfo", + }; + trace!(?profile); + + cfg.out_dir(&out_dir) + .profile(profile) + .env("LLVM_CONFIG_REAL", &host_llvm_config) + .define("LLVM_ENABLE_ASSERTIONS", "ON") + .define("TARGET_TRIPLE", &target.triple) + .define("LLVM_ENABLE_RUNTIMES", "openmp") + .define("CMAKE_C_COMPILER", builder.cc(target)) + .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) + .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) + .define("OPENMP_STANDALONE_BUILD", "ON") + .define( + "LLVM_ROOT", + "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", + ) + .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); + ///tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm + //$> cmake ../runtimes \ # Point to the runtimes build + // -G Ninja \ + // -DLLVM_ENABLE_RUNTIMES=openmp \ + // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ + // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ + // -DLLVM_DEFAULT_TARGET_TRIPLE=$TARGET_TRIPLE \ + // -DCMAKE_BUILD_TYPE=Release + cfg.build(); + + //t!(stamp.write()); + out_dir + + //$> cd llvm-project # The llvm-project checkout + //$> mkdir build # A different build directory for the build tools + //$> cd build + //$> TARGET_TRIPLE= + //$> TARGET_C_COMPILER= + //$> TARGET_CXX_COMPILER= + //$> ninja install + // + //You can do the same thing for the offload project. + // + //$> TARGET_C_COMPILER= + //$> TARGET_CXX_COMPILER= + //$> cmake ../runtimes \ # Point to the runtimes build + // -G Ninja \ + // -DLLVM_ENABLE_RUNTIMES=openmp \ + // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ + // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ + // -DCMAKE_BUILD_TYPE=Release + //$> ninja install + } +} + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Enzyme { pub target: TargetSelection, From c146ee232c607c401849d1b35deccb04d7fae844 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Thu, 27 Nov 2025 13:33:43 -0800 Subject: [PATCH 18/50] working standalone offload build, fails runtime testing --- src/bootstrap/src/core/build_steps/compile.rs | 1 + src/bootstrap/src/core/build_steps/llvm.rs | 123 ++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 71b33797008d9..5dafa68739b0f 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1432,6 +1432,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); if builder.config.llvm_enzyme { + builder.ensure(llvm::Openmp { target }); builder.ensure(llvm::Offload { target }); cargo.env("LLVM_OFFLOAD", "1"); } diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 4a71ecbab17fb..d79512ae62092 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -909,6 +909,129 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option { .or_else(|| env::var_os(var_base)) } +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Openmp { + pub target: TargetSelection, +} + +impl Step for Openmp { + type Output = PathBuf; + const IS_HOST: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/openmp") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Openmp { target: run.target }); + } + + /// Compile OpenMP offload runtimes for `target`. + fn run(self, builder: &Builder<'_>) -> PathBuf { + if builder.config.dry_run() { + return PathBuf::from("/"); + } + let target = self.target; + + let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target }); + + let out_dir = builder.llvm_out(target); + static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); + //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { + // generate_smart_stamp_hash( + // builder, + // &builder.config.src.join("src/llvm-project/openmp"), + // builder.offload_info.sha().unwrap_or_default(), + // ) + //}); + //let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").add_stamp(smart_stamp_hash); + + trace!("checking build stamp to see if we need to rebuild offload/openmp artifacts"); + //if stamp.is_up_to_date() { + // trace!(?out_dir, "offload/openmp build artifacts are up to date"); + // if stamp.stamp().is_empty() { + // builder.info( + // "Could not determine the Offload submodule commit hash. \ + // Assuming that an Offload rebuild is not necessary.", + // ); + // builder.info(&format!( + // "To force Offload to rebuild, remove the file `{}`", + // stamp.path().display() + // )); + // } + // return out_dir; + //} + + trace!(?target, "(re)building openmp artifacts"); + builder.info(&format!("Building OpenMP for {target}")); + //t!(stamp.remove()); + let _time = helpers::timeit(builder); + t!(fs::create_dir_all(&out_dir)); + + builder.config.update_submodule(Path::new("src").join("llvm-project").to_str().unwrap()); + let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/runtimes/")); + configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]); + + // Re-use the same flags as llvm to control the level of debug information + // generated by Enzyme. + // FIXME(ZuseZ4): Find a nicer way to use Enzyme Debug builds. + let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { + (false, _) => "Debug", + (true, false) => "Release", + (true, true) => "RelWithDebInfo", + }; + trace!(?profile); + + cfg.out_dir(&out_dir) + .profile(profile) + .env("LLVM_CONFIG_REAL", &host_llvm_config) + .define("LLVM_ENABLE_ASSERTIONS", "ON") + .define("TARGET_TRIPLE", &target.triple) + .define("LLVM_ENABLE_RUNTIMES", "offload") + .define("CMAKE_C_COMPILER", builder.cc(target)) + .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) + .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) + .define("OFFLOAD_STANDALONE_BUILD", "ON") + .define( + "LLVM_ROOT", + "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", + ) + .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); + ///tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm + //$> cmake ../runtimes \ # Point to the runtimes build + // -G Ninja \ + // -DLLVM_ENABLE_RUNTIMES=openmp \ + // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ + // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ + // -DLLVM_DEFAULT_TARGET_TRIPLE=$TARGET_TRIPLE \ + // -DCMAKE_BUILD_TYPE=Release + cfg.build(); + + //t!(stamp.write()); + out_dir + + //$> cd llvm-project # The llvm-project checkout + //$> mkdir build # A different build directory for the build tools + //$> cd build + //$> TARGET_TRIPLE= + //$> TARGET_C_COMPILER= + //$> TARGET_CXX_COMPILER= + //$> ninja install + // + //You can do the same thing for the offload project. + // + //$> TARGET_C_COMPILER= + //$> TARGET_CXX_COMPILER= + //$> cmake ../runtimes \ # Point to the runtimes build + // -G Ninja \ + // -DLLVM_ENABLE_RUNTIMES=openmp \ + // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ + // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ + // -DCMAKE_BUILD_TYPE=Release + //$> ninja install + } +} + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Offload { pub target: TargetSelection, From c47ba3e76c3e1e2058be2723fc3bf416e6f292ca Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Thu, 27 Nov 2025 14:18:58 -0800 Subject: [PATCH 19/50] wip --- src/bootstrap/src/core/build_steps/compile.rs | 2 +- src/bootstrap/src/core/build_steps/llvm.rs | 32 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 5dafa68739b0f..89da8dddadc9a 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1431,7 +1431,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect cargo.env("LLVM_OFFLOAD", "1"); } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - if builder.config.llvm_enzyme { + if builder.config.llvm_offload { builder.ensure(llvm::Openmp { target }); builder.ensure(llvm::Offload { target }); cargo.env("LLVM_OFFLOAD", "1"); diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index d79512ae62092..3d0639bdd2b85 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -988,8 +988,16 @@ impl Step for Openmp { .define("LLVM_ENABLE_ASSERTIONS", "ON") .define("TARGET_TRIPLE", &target.triple) .define("LLVM_ENABLE_RUNTIMES", "offload") - .define("CMAKE_C_COMPILER", builder.cc(target)) - .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) + //.define("CMAKE_C_COMPILER", builder.cc(target)) + //.define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) + .define( + "CMAKE_C_COMPILER", + "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", + ) + .define( + "CMAKE_CXX_COMPILER", + "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", + ) .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) .define("OFFLOAD_STANDALONE_BUILD", "ON") .define( @@ -1042,7 +1050,7 @@ impl Step for Offload { const IS_HOST: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/openmp") + run.path("src/llvm-project/offload") } fn make_run(run: RunConfig<'_>) { @@ -1063,7 +1071,7 @@ impl Step for Offload { //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { // generate_smart_stamp_hash( // builder, - // &builder.config.src.join("src/llvm-project/openmp"), + // &builder.config.src.join("src/llvm-project/offload"), // builder.offload_info.sha().unwrap_or_default(), // ) //}); @@ -1085,8 +1093,8 @@ impl Step for Offload { // return out_dir; //} - trace!(?target, "(re)building openmp artifacts"); - builder.info(&format!("Building OpenMP for {target}")); + trace!(?target, "(re)building offload artifacts"); + builder.info(&format!("Building Offload for {target}")); //t!(stamp.remove()); let _time = helpers::timeit(builder); t!(fs::create_dir_all(&out_dir)); @@ -1111,8 +1119,16 @@ impl Step for Offload { .define("LLVM_ENABLE_ASSERTIONS", "ON") .define("TARGET_TRIPLE", &target.triple) .define("LLVM_ENABLE_RUNTIMES", "openmp") - .define("CMAKE_C_COMPILER", builder.cc(target)) - .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) + .define( + "CMAKE_C_COMPILER", + "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", + ) + .define( + "CMAKE_CXX_COMPILER", + "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", + ) + //.define("CMAKE_C_COMPILER", builder.cc(target)) + //.define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) .define("OPENMP_STANDALONE_BUILD", "ON") .define( From ba2db9055288a363746d8b3eec6df153ab4f4d0f Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 1 Dec 2025 19:14:35 -0800 Subject: [PATCH 20/50] try to not hardcode paths --- src/bootstrap/src/core/build_steps/compile.rs | 3 -- src/bootstrap/src/core/build_steps/llvm.rs | 47 ++++++++++++++----- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 89da8dddadc9a..42798c317736e 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1427,9 +1427,6 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect if builder.config.llvm_enzyme { cargo.env("LLVM_ENZYME", "1"); } - if builder.config.llvm_offload { - cargo.env("LLVM_OFFLOAD", "1"); - } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); if builder.config.llvm_offload { builder.ensure(llvm::Openmp { target }); diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 3d0639bdd2b85..366a977c8875c 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -990,22 +990,43 @@ impl Step for Openmp { .define("LLVM_ENABLE_RUNTIMES", "offload") //.define("CMAKE_C_COMPILER", builder.cc(target)) //.define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) - .define( - "CMAKE_C_COMPILER", - "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", - ) - .define( - "CMAKE_CXX_COMPILER", - "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", - ) + //.define( + // "CMAKE_C_COMPILER", + // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", + //) + //.define( + // "CMAKE_CXX_COMPILER", + // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", + //) .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) .define("OFFLOAD_STANDALONE_BUILD", "ON") - .define( - "LLVM_ROOT", - "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", - ) + .define("LLVM_ROOT", builder.llvm_out(target)) + //.define( + // "LLVM_ROOT", + // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", + //) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); - ///tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm + + if builder.config.llvm_clang { + // This is likely the case locally. If we just build clang in the previous step, we + // should use it. We wouldn't even need a standalone build here, but it's easier to + // unify this with the case below, which needs standalone builds. + // We likely wouldn't need to specify these here, they should be found by default. + cfg.define("CMAKE_C_COMPILER", builder.cc(target)) + .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()); + } else { + // This is the (more complicated) case which we have e.g. in CI. We first build a + // standalone llvm/llvm-project including clang. Then we use this clang to build the + // llvm submodule of rustc in src/llvm-project. In this case we do *NOT* build clang + // again in the llvm submodule to save compile times, this is a hard requirement. + // If we build the offload and openmp runtimes in-tree, they assume that clang was + // already build in-tree, which is not the case here. We therefore must use standalone + // builds of these two runtimes. We now have to specify clang(++) manually to match the + // one we build in llvm/llvm-project + cfg.define("CMAKE_C_COMPILER", builder.cc(target)) + .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()); + } + // /tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm //$> cmake ../runtimes \ # Point to the runtimes build // -G Ninja \ // -DLLVM_ENABLE_RUNTIMES=openmp \ From f64157639110dd0fe33c4c6f51dae75f7bccdc5f Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 1 Dec 2025 19:46:15 -0800 Subject: [PATCH 21/50] unify both ol and omp runtime builds into one step --- src/bootstrap/src/core/build_steps/compile.rs | 3 +- src/bootstrap/src/core/build_steps/llvm.rs | 205 ++---------------- 2 files changed, 14 insertions(+), 194 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 42798c317736e..b6b02ebaa482d 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1429,8 +1429,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); if builder.config.llvm_offload { - builder.ensure(llvm::Openmp { target }); - builder.ensure(llvm::Offload { target }); + builder.ensure(llvm::OmpOffload { target }); cargo.env("LLVM_OFFLOAD", "1"); } diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 366a977c8875c..cdac8771fc8e1 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -910,23 +910,24 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option { } #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Openmp { +pub struct OmpOffload { pub target: TargetSelection, } -impl Step for Openmp { +impl Step for OmpOffload { type Output = PathBuf; const IS_HOST: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/openmp") + run.path("src/llvm-project/offload") } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Openmp { target: run.target }); + run.builder.ensure(OmpOffload { target: run.target }); } /// Compile OpenMP offload runtimes for `target`. + #[allow(unused)] fn run(self, builder: &Builder<'_>) -> PathBuf { if builder.config.dry_run() { return PathBuf::from("/"); @@ -940,7 +941,7 @@ impl Step for Openmp { //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { // generate_smart_stamp_hash( // builder, - // &builder.config.src.join("src/llvm-project/openmp"), + // &builder.config.src.join("src/llvm-project/offload"), // builder.offload_info.sha().unwrap_or_default(), // ) //}); @@ -962,8 +963,8 @@ impl Step for Openmp { // return out_dir; //} - trace!(?target, "(re)building openmp artifacts"); - builder.info(&format!("Building OpenMP for {target}")); + trace!(?target, "(re)building offload artifacts"); + builder.info(&format!("Building Omp/Offload for {target}")); //t!(stamp.remove()); let _time = helpers::timeit(builder); t!(fs::create_dir_all(&out_dir)); @@ -986,10 +987,7 @@ impl Step for Openmp { .profile(profile) .env("LLVM_CONFIG_REAL", &host_llvm_config) .define("LLVM_ENABLE_ASSERTIONS", "ON") - .define("TARGET_TRIPLE", &target.triple) - .define("LLVM_ENABLE_RUNTIMES", "offload") - //.define("CMAKE_C_COMPILER", builder.cc(target)) - //.define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) + .define("LLVM_ENABLE_RUNTIMES", "openmp;offload") //.define( // "CMAKE_C_COMPILER", // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", @@ -998,197 +996,20 @@ impl Step for Openmp { // "CMAKE_CXX_COMPILER", // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", //) + .define("CMAKE_C_COMPILER", builder.cc(target)) + .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) - .define("OFFLOAD_STANDALONE_BUILD", "ON") - .define("LLVM_ROOT", builder.llvm_out(target)) + .define("OPENMP_STANDALONE_BUILD", "ON") + .define("LLVM_ROOT", builder.llvm_out(target).join("build")) //.define( // "LLVM_ROOT", // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", //) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); - - if builder.config.llvm_clang { - // This is likely the case locally. If we just build clang in the previous step, we - // should use it. We wouldn't even need a standalone build here, but it's easier to - // unify this with the case below, which needs standalone builds. - // We likely wouldn't need to specify these here, they should be found by default. - cfg.define("CMAKE_C_COMPILER", builder.cc(target)) - .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()); - } else { - // This is the (more complicated) case which we have e.g. in CI. We first build a - // standalone llvm/llvm-project including clang. Then we use this clang to build the - // llvm submodule of rustc in src/llvm-project. In this case we do *NOT* build clang - // again in the llvm submodule to save compile times, this is a hard requirement. - // If we build the offload and openmp runtimes in-tree, they assume that clang was - // already build in-tree, which is not the case here. We therefore must use standalone - // builds of these two runtimes. We now have to specify clang(++) manually to match the - // one we build in llvm/llvm-project - cfg.define("CMAKE_C_COMPILER", builder.cc(target)) - .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()); - } - // /tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm - //$> cmake ../runtimes \ # Point to the runtimes build - // -G Ninja \ - // -DLLVM_ENABLE_RUNTIMES=openmp \ - // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ - // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ - // -DLLVM_DEFAULT_TARGET_TRIPLE=$TARGET_TRIPLE \ - // -DCMAKE_BUILD_TYPE=Release cfg.build(); //t!(stamp.write()); out_dir - - //$> cd llvm-project # The llvm-project checkout - //$> mkdir build # A different build directory for the build tools - //$> cd build - //$> TARGET_TRIPLE= - //$> TARGET_C_COMPILER= - //$> TARGET_CXX_COMPILER= - //$> ninja install - // - //You can do the same thing for the offload project. - // - //$> TARGET_C_COMPILER= - //$> TARGET_CXX_COMPILER= - //$> cmake ../runtimes \ # Point to the runtimes build - // -G Ninja \ - // -DLLVM_ENABLE_RUNTIMES=openmp \ - // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ - // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ - // -DCMAKE_BUILD_TYPE=Release - //$> ninja install - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Offload { - pub target: TargetSelection, -} - -impl Step for Offload { - type Output = PathBuf; - const IS_HOST: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/offload") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Offload { target: run.target }); - } - - /// Compile OpenMP offload runtimes for `target`. - fn run(self, builder: &Builder<'_>) -> PathBuf { - if builder.config.dry_run() { - return PathBuf::from("/"); - } - let target = self.target; - - let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target }); - - let out_dir = builder.llvm_out(target); - static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); - //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { - // generate_smart_stamp_hash( - // builder, - // &builder.config.src.join("src/llvm-project/offload"), - // builder.offload_info.sha().unwrap_or_default(), - // ) - //}); - //let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").add_stamp(smart_stamp_hash); - - trace!("checking build stamp to see if we need to rebuild offload/openmp artifacts"); - //if stamp.is_up_to_date() { - // trace!(?out_dir, "offload/openmp build artifacts are up to date"); - // if stamp.stamp().is_empty() { - // builder.info( - // "Could not determine the Offload submodule commit hash. \ - // Assuming that an Offload rebuild is not necessary.", - // ); - // builder.info(&format!( - // "To force Offload to rebuild, remove the file `{}`", - // stamp.path().display() - // )); - // } - // return out_dir; - //} - - trace!(?target, "(re)building offload artifacts"); - builder.info(&format!("Building Offload for {target}")); - //t!(stamp.remove()); - let _time = helpers::timeit(builder); - t!(fs::create_dir_all(&out_dir)); - - builder.config.update_submodule(Path::new("src").join("llvm-project").to_str().unwrap()); - let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/runtimes/")); - configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]); - - // Re-use the same flags as llvm to control the level of debug information - // generated by Enzyme. - // FIXME(ZuseZ4): Find a nicer way to use Enzyme Debug builds. - let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { - (false, _) => "Debug", - (true, false) => "Release", - (true, true) => "RelWithDebInfo", - }; - trace!(?profile); - - cfg.out_dir(&out_dir) - .profile(profile) - .env("LLVM_CONFIG_REAL", &host_llvm_config) - .define("LLVM_ENABLE_ASSERTIONS", "ON") - .define("TARGET_TRIPLE", &target.triple) - .define("LLVM_ENABLE_RUNTIMES", "openmp") - .define( - "CMAKE_C_COMPILER", - "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", - ) - .define( - "CMAKE_CXX_COMPILER", - "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", - ) - //.define("CMAKE_C_COMPILER", builder.cc(target)) - //.define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) - .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) - .define("OPENMP_STANDALONE_BUILD", "ON") - .define( - "LLVM_ROOT", - "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", - ) - .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); - ///tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/lib/cmake/llvm - //$> cmake ../runtimes \ # Point to the runtimes build - // -G Ninja \ - // -DLLVM_ENABLE_RUNTIMES=openmp \ - // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ - // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ - // -DLLVM_DEFAULT_TARGET_TRIPLE=$TARGET_TRIPLE \ - // -DCMAKE_BUILD_TYPE=Release - cfg.build(); - - //t!(stamp.write()); - out_dir - - //$> cd llvm-project # The llvm-project checkout - //$> mkdir build # A different build directory for the build tools - //$> cd build - //$> TARGET_TRIPLE= - //$> TARGET_C_COMPILER= - //$> TARGET_CXX_COMPILER= - //$> ninja install - // - //You can do the same thing for the offload project. - // - //$> TARGET_C_COMPILER= - //$> TARGET_CXX_COMPILER= - //$> cmake ../runtimes \ # Point to the runtimes build - // -G Ninja \ - // -DLLVM_ENABLE_RUNTIMES=openmp \ - // -DCMAKE_C_COMPILER=$TARGET_C_COMPILER \ - // -DCMAKE_CXX_COMPILER=$TARGET_CXX_COMPILER \ - // -DCMAKE_BUILD_TYPE=Release - //$> ninja install } } From 875885a96abec7c4955865ad4a85fd95de1a2e25 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 1 Dec 2025 20:14:51 -0800 Subject: [PATCH 22/50] disable default triple, should likely bei either amdgcn or nvptx and we don't know which --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index cdac8771fc8e1..938e50bc527a1 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -998,7 +998,7 @@ impl Step for OmpOffload { //) .define("CMAKE_C_COMPILER", builder.cc(target)) .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) - .define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) + //.define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) .define("OPENMP_STANDALONE_BUILD", "ON") .define("LLVM_ROOT", builder.llvm_out(target).join("build")) //.define( From fd129ad9d0daeb8bde2862b6f3019eb3fa9e9caf Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 16:55:19 -0800 Subject: [PATCH 23/50] allow specifying cc/cxx for llvm_offload subproject build --- src/bootstrap/src/core/build_steps/llvm.rs | 68 +++++++++++++--------- src/bootstrap/src/core/config/config.rs | 10 ++++ src/bootstrap/src/core/config/toml/llvm.rs | 4 ++ src/bootstrap/src/lib.rs | 5 ++ 4 files changed, 58 insertions(+), 29 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 938e50bc527a1..3d97674d36bb6 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -938,34 +938,34 @@ impl Step for OmpOffload { let out_dir = builder.llvm_out(target); static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); - //let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { - // generate_smart_stamp_hash( - // builder, - // &builder.config.src.join("src/llvm-project/offload"), - // builder.offload_info.sha().unwrap_or_default(), - // ) - //}); - //let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").add_stamp(smart_stamp_hash); + let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { + generate_smart_stamp_hash( + builder, + &builder.config.src.join("src/llvm-project/offload"), + builder.offload_info.sha().unwrap_or_default(), + ) + }); + let stamp = BuildStamp::new(&out_dir).with_prefix("offload").add_stamp(smart_stamp_hash); trace!("checking build stamp to see if we need to rebuild offload/openmp artifacts"); - //if stamp.is_up_to_date() { - // trace!(?out_dir, "offload/openmp build artifacts are up to date"); - // if stamp.stamp().is_empty() { - // builder.info( - // "Could not determine the Offload submodule commit hash. \ - // Assuming that an Offload rebuild is not necessary.", - // ); - // builder.info(&format!( - // "To force Offload to rebuild, remove the file `{}`", - // stamp.path().display() - // )); - // } - // return out_dir; - //} - - trace!(?target, "(re)building offload artifacts"); - builder.info(&format!("Building Omp/Offload for {target}")); - //t!(stamp.remove()); + if stamp.is_up_to_date() { + trace!(?out_dir, "offload/openmp build artifacts are up to date"); + if stamp.stamp().is_empty() { + builder.info( + "Could not determine the Offload submodule commit hash. \ + Assuming that an Offload rebuild is not necessary.", + ); + builder.info(&format!( + "To force Offload/OpenMP to rebuild, remove the file `{}`", + stamp.path().display() + )); + } + return out_dir; + } + + trace!(?target, "(re)building offload/openmp artifacts"); + builder.info(&format!("Building OpenMP/Offload for {target}")); + t!(stamp.remove()); let _time = helpers::timeit(builder); t!(fs::create_dir_all(&out_dir)); @@ -983,6 +983,17 @@ impl Step for OmpOffload { }; trace!(?profile); + let cc = if let Some(p) = &builder.build.config.llvm_offload_cc { + p.clone() + } else { + builder.cc(target) + }; + let cxx = if let Some(p) = &builder.build.config.llvm_offload_cxx { + p.clone() + } else { + builder.cxx(target).unwrap() + }; + cfg.out_dir(&out_dir) .profile(profile) .env("LLVM_CONFIG_REAL", &host_llvm_config) @@ -996,9 +1007,8 @@ impl Step for OmpOffload { // "CMAKE_CXX_COMPILER", // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", //) - .define("CMAKE_C_COMPILER", builder.cc(target)) - .define("CMAKE_CXX_COMPILER", builder.cxx(target).unwrap()) - //.define("LLVM_DEFAULT_TARGET_TRIPLE", &target.triple) + .define("CMAKE_C_COMPILER", cc) + .define("CMAKE_CXX_COMPILER", cxx) .define("OPENMP_STANDALONE_BUILD", "ON") .define("LLVM_ROOT", builder.llvm_out(target).join("build")) //.define( diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index a81b8ccb9140a..14a0489eac184 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -169,6 +169,8 @@ pub struct Config { pub llvm_link_jobs: Option, pub llvm_version_suffix: Option, pub llvm_use_linker: Option, + pub llvm_offload_cc: Option, + pub llvm_offload_cxx: Option, pub llvm_allow_old_toolchain: bool, pub llvm_polly: bool, pub llvm_clang: bool, @@ -291,6 +293,7 @@ pub struct Config { pub miri_info: channel::GitInfo, pub rustfmt_info: channel::GitInfo, pub enzyme_info: channel::GitInfo, + pub offload_info: channel::GitInfo, pub in_tree_llvm_info: channel::GitInfo, pub in_tree_gcc_info: channel::GitInfo, @@ -602,6 +605,8 @@ impl Config { ldflags: llvm_ldflags, use_libcxx: llvm_use_libcxx, use_linker: llvm_use_linker, + offload_cc: llvm_offload_cc, + offload_cxx: llvm_offload_cxx, allow_old_toolchain: llvm_allow_old_toolchain, offload: llvm_offload, polly: llvm_polly, @@ -1261,6 +1266,8 @@ impl Config { let in_tree_gcc_info = git_info(&exec_ctx, false, &src.join("src/gcc")); let in_tree_llvm_info = git_info(&exec_ctx, false, &src.join("src/llvm-project")); let enzyme_info = git_info(&exec_ctx, omit_git_hash, &src.join("src/tools/enzyme")); + let offload_info = + git_info(&exec_ctx, omit_git_hash, &src.join("src/llvm-project/offload")); let miri_info = git_info(&exec_ctx, omit_git_hash, &src.join("src/tools/miri")); let rust_analyzer_info = git_info(&exec_ctx, omit_git_hash, &src.join("src/tools/rust-analyzer")); @@ -1324,6 +1331,7 @@ impl Config { ehcont_guard: rust_ehcont_guard.unwrap_or(false), enable_bolt_settings: flags_enable_bolt_settings, enzyme_info, + offload_info, exec_ctx, explicit_stage_from_cli: flags_stage.is_some(), explicit_stage_from_config, @@ -1365,6 +1373,8 @@ impl Config { llvm_enable_warnings: llvm_enable_warnings.unwrap_or(false), llvm_enzyme: llvm_enzyme.unwrap_or(false), llvm_experimental_targets, + llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), + llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), llvm_from_ci, llvm_ldflags, llvm_libunwind_default: rust_llvm_libunwind diff --git a/src/bootstrap/src/core/config/toml/llvm.rs b/src/bootstrap/src/core/config/toml/llvm.rs index 9523f80214849..30eaa67fc4725 100644 --- a/src/bootstrap/src/core/config/toml/llvm.rs +++ b/src/bootstrap/src/core/config/toml/llvm.rs @@ -32,6 +32,8 @@ define_config! { ldflags: Option = "ldflags", use_libcxx: Option = "use-libcxx", use_linker: Option = "use-linker", + offload_cc: Option = "offload-cc", + offload_cxx: Option = "offload-cxx", allow_old_toolchain: Option = "allow-old-toolchain", offload: Option = "offload", polly: Option = "polly", @@ -110,6 +112,8 @@ pub fn check_incompatible_options_for_ci_llvm( ldflags, use_libcxx, use_linker, + offload_cc, + offload_cxx, allow_old_toolchain, offload, polly, diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index adae9a1b8b081..6a7f3c5e52216 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -212,12 +212,15 @@ pub struct Build { src: PathBuf, out: PathBuf, bootstrap_out: PathBuf, + //offload_cc: PathBuf, + //offload_cxx: PathBuf, cargo_info: GitInfo, rust_analyzer_info: GitInfo, clippy_info: GitInfo, miri_info: GitInfo, rustfmt_info: GitInfo, enzyme_info: GitInfo, + offload_info: GitInfo, in_tree_llvm_info: GitInfo, in_tree_gcc_info: GitInfo, local_rebuild: bool, @@ -476,6 +479,7 @@ impl Build { let miri_info = config.miri_info.clone(); let rustfmt_info = config.rustfmt_info.clone(); let enzyme_info = config.enzyme_info.clone(); + let offload_info = config.offload_info.clone(); let in_tree_llvm_info = config.in_tree_llvm_info.clone(); let in_tree_gcc_info = config.in_tree_gcc_info.clone(); @@ -568,6 +572,7 @@ impl Build { miri_info, rustfmt_info, enzyme_info, + offload_info, in_tree_llvm_info, in_tree_gcc_info, cc: HashMap::new(), From 4d5911fb47d308c8d59ac6bc639fa06756b5acfc Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 16:59:19 -0800 Subject: [PATCH 24/50] cleanup --- src/bootstrap/src/core/build_steps/llvm.rs | 23 ---------------------- 1 file changed, 23 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 3d97674d36bb6..d2fbc60d9b70c 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -454,29 +454,6 @@ impl Step for Llvm { enabled_llvm_runtimes.push("compiler-rt"); } - // This is an experimental flag, which likely builds more than necessary. - // We will optimize it when we get closer to releasing it on nightly. - if builder.config.llvm_offload { - //enabled_llvm_projects.push("lld"); - //enabled_llvm_runtimes.push("offload"); - //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. - //Remove this line once they achieved it. - //enabled_llvm_runtimes.push("openmp"); - //enabled_llvm_runtimes.push("compiler-rt"); - - // -DRUNTIMES_CMAKE_ARGS="-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx; - //cfg.define("RUNTIMES_CMAKE_ARGS", "-DLIBOMPTARGET_PLUGINS_TO_BUILD=amdgpu;nvptx"); - //cfg.define("LLVM_TARGETS_TO_BUILD", "host;AMDGPU;NVPTX"); - - //let runtime_targets = vec!["default", "amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; - //cfg.define("LLVM_RUNTIME_TARGETS", runtime_targets.join(";")); - - //cfg.define("LLVM_INCLUDE_TESTS", "OFF"); - //cfg.define("LLVM_BUILD_TESTS", "OFF"); - //cfg.define("RUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES", "openmp"); - //cfg.define("RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES", "openmp"); - } - if !enabled_llvm_projects.is_empty() { enabled_llvm_projects.sort(); enabled_llvm_projects.dedup(); From 056721671315ec66f9888106d6e88688ff26de75 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 19:50:09 -0800 Subject: [PATCH 25/50] worked locally with setting LLVM_ROOT and cc/cxx --- src/bootstrap/src/core/build_steps/llvm.rs | 13 +++++++++++-- src/bootstrap/src/core/config/config.rs | 3 +++ src/bootstrap/src/core/config/toml/llvm.rs | 2 ++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index d2fbc60d9b70c..51263e885dbf4 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -970,12 +970,20 @@ impl Step for OmpOffload { } else { builder.cxx(target).unwrap() }; + let root = if let Some(p) = &builder.build.config.llvm_root_offload { + p.clone() + } else { + builder.llvm_out(target).join("build") + }; cfg.out_dir(&out_dir) .profile(profile) .env("LLVM_CONFIG_REAL", &host_llvm_config) .define("LLVM_ENABLE_ASSERTIONS", "ON") .define("LLVM_ENABLE_RUNTIMES", "openmp;offload") + .define("LLVM_INCLUDE_TESTS", "OFF") + .define("LLVM_BUILD_TESTS", "OFF") + .define("OFFLOAD_INCLUDE_TESTS", "OFF") //.define( // "CMAKE_C_COMPILER", // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", @@ -986,8 +994,9 @@ impl Step for OmpOffload { //) .define("CMAKE_C_COMPILER", cc) .define("CMAKE_CXX_COMPILER", cxx) + .define("Clang_DIR", "/tmp/drehwald1/prog/llvm/lib/cmake/clang") .define("OPENMP_STANDALONE_BUILD", "ON") - .define("LLVM_ROOT", builder.llvm_out(target).join("build")) + .define("LLVM_ROOT", root) //.define( // "LLVM_ROOT", // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", @@ -995,7 +1004,7 @@ impl Step for OmpOffload { .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); cfg.build(); - //t!(stamp.write()); + t!(stamp.write()); out_dir } } diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 14a0489eac184..6b95855ac222c 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -171,6 +171,7 @@ pub struct Config { pub llvm_use_linker: Option, pub llvm_offload_cc: Option, pub llvm_offload_cxx: Option, + pub llvm_root_offload: Option, pub llvm_allow_old_toolchain: bool, pub llvm_polly: bool, pub llvm_clang: bool, @@ -607,6 +608,7 @@ impl Config { use_linker: llvm_use_linker, offload_cc: llvm_offload_cc, offload_cxx: llvm_offload_cxx, + root_offload: llvm_root_offload, allow_old_toolchain: llvm_allow_old_toolchain, offload: llvm_offload, polly: llvm_polly, @@ -1375,6 +1377,7 @@ impl Config { llvm_experimental_targets, llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), + llvm_root_offload: llvm_root_offload.map(PathBuf::from), llvm_from_ci, llvm_ldflags, llvm_libunwind_default: rust_llvm_libunwind diff --git a/src/bootstrap/src/core/config/toml/llvm.rs b/src/bootstrap/src/core/config/toml/llvm.rs index 30eaa67fc4725..43252ceea08a4 100644 --- a/src/bootstrap/src/core/config/toml/llvm.rs +++ b/src/bootstrap/src/core/config/toml/llvm.rs @@ -34,6 +34,7 @@ define_config! { use_linker: Option = "use-linker", offload_cc: Option = "offload-cc", offload_cxx: Option = "offload-cxx", + root_offload: Option = "root-offload", allow_old_toolchain: Option = "allow-old-toolchain", offload: Option = "offload", polly: Option = "polly", @@ -114,6 +115,7 @@ pub fn check_incompatible_options_for_ci_llvm( use_linker, offload_cc, offload_cxx, + root_offload, allow_old_toolchain, offload, polly, From 4a577cf5229b4f979f7effd90bb6f8b3832b1cdc Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 20:21:01 -0800 Subject: [PATCH 26/50] new try --- src/bootstrap/src/core/build_steps/llvm.rs | 46 ++++++++++------------ src/bootstrap/src/core/config/config.rs | 3 ++ src/bootstrap/src/core/config/toml/llvm.rs | 1 + 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 51263e885dbf4..4f001ae3589b5 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -960,21 +960,28 @@ impl Step for OmpOffload { }; trace!(?profile); - let cc = if let Some(p) = &builder.build.config.llvm_offload_cc { - p.clone() - } else { - builder.cc(target) - }; - let cxx = if let Some(p) = &builder.build.config.llvm_offload_cxx { - p.clone() - } else { - builder.cxx(target).unwrap() - }; + //let cc = if let Some(p) = &builder.build.config.llvm_offload_cc { + // //p.clone() + // builder.cc(target) + //} else { + // builder.cc(target) + //}; + //let cxx = if let Some(p) = &builder.build.config.llvm_offload_cxx { + // //p.clone() + // builder.cxx(target).unwrap() + //} else { + // builder.cxx(target).unwrap() + //}; let root = if let Some(p) = &builder.build.config.llvm_root_offload { p.clone() } else { builder.llvm_out(target).join("build") }; + let clang_dir = if !builder.config.llvm_clang { + &builder.build.config.llvm_clang_dir.unwrap() + } else { + None + }; cfg.out_dir(&out_dir) .profile(profile) @@ -982,26 +989,15 @@ impl Step for OmpOffload { .define("LLVM_ENABLE_ASSERTIONS", "ON") .define("LLVM_ENABLE_RUNTIMES", "openmp;offload") .define("LLVM_INCLUDE_TESTS", "OFF") - .define("LLVM_BUILD_TESTS", "OFF") .define("OFFLOAD_INCLUDE_TESTS", "OFF") - //.define( - // "CMAKE_C_COMPILER", - // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang", - //) - //.define( - // "CMAKE_CXX_COMPILER", - // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++", - //) - .define("CMAKE_C_COMPILER", cc) - .define("CMAKE_CXX_COMPILER", cxx) + //.define("CMAKE_C_COMPILER", cc) + //.define("CMAKE_CXX_COMPILER", cxx) .define("Clang_DIR", "/tmp/drehwald1/prog/llvm/lib/cmake/clang") .define("OPENMP_STANDALONE_BUILD", "ON") .define("LLVM_ROOT", root) - //.define( - // "LLVM_ROOT", - // "/tmp/drehwald1/prog/rust/build/x86_64-unknown-linux-gnu/llvm/build/", - //) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); + if let Some(p) = Clang_DIR { + cfg.define("CLANG_DIR", p); cfg.build(); t!(stamp.write()); diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 6b95855ac222c..a6f36118a795f 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -172,6 +172,7 @@ pub struct Config { pub llvm_offload_cc: Option, pub llvm_offload_cxx: Option, pub llvm_root_offload: Option, + pub llvm_clang_dir: Option, pub llvm_allow_old_toolchain: bool, pub llvm_polly: bool, pub llvm_clang: bool, @@ -609,6 +610,7 @@ impl Config { offload_cc: llvm_offload_cc, offload_cxx: llvm_offload_cxx, root_offload: llvm_root_offload, + clang_dir: llvm_clang_dir, allow_old_toolchain: llvm_allow_old_toolchain, offload: llvm_offload, polly: llvm_polly, @@ -1378,6 +1380,7 @@ impl Config { llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), llvm_root_offload: llvm_root_offload.map(PathBuf::from), + llvm_clang_dir: llvm_clang_dir.map(PathBuf::from), llvm_from_ci, llvm_ldflags, llvm_libunwind_default: rust_llvm_libunwind diff --git a/src/bootstrap/src/core/config/toml/llvm.rs b/src/bootstrap/src/core/config/toml/llvm.rs index 43252ceea08a4..17cd9c7b33bdc 100644 --- a/src/bootstrap/src/core/config/toml/llvm.rs +++ b/src/bootstrap/src/core/config/toml/llvm.rs @@ -35,6 +35,7 @@ define_config! { offload_cc: Option = "offload-cc", offload_cxx: Option = "offload-cxx", root_offload: Option = "root-offload", + clang_dir: Option = "offload-clang-dir", allow_old_toolchain: Option = "allow-old-toolchain", offload: Option = "offload", polly: Option = "polly", From a4f7e936b1b711b878411f2ec07e4dfcef830a04 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 20:22:10 -0800 Subject: [PATCH 27/50] new try --- src/bootstrap/src/core/build_steps/llvm.rs | 8 ++++++-- src/bootstrap/src/core/config/config.rs | 4 ++-- src/bootstrap/src/core/config/toml/llvm.rs | 7 ++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 4f001ae3589b5..2fc1bfb811195 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -978,8 +978,11 @@ impl Step for OmpOffload { builder.llvm_out(target).join("build") }; let clang_dir = if !builder.config.llvm_clang { - &builder.build.config.llvm_clang_dir.unwrap() + // We must have an external clang to use. + assert!(&builder.build.config.llvm_clang_dir.is_some()); + builder.build.config.llvm_clang_dir.clone() } else { + // No need to specify it, since we use the in-tree clang None }; @@ -996,8 +999,9 @@ impl Step for OmpOffload { .define("OPENMP_STANDALONE_BUILD", "ON") .define("LLVM_ROOT", root) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); - if let Some(p) = Clang_DIR { + if let Some(p) = clang_dir { cfg.define("CLANG_DIR", p); + } cfg.build(); t!(stamp.write()); diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index a6f36118a795f..b6fdfd0a1ab25 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -1335,7 +1335,6 @@ impl Config { ehcont_guard: rust_ehcont_guard.unwrap_or(false), enable_bolt_settings: flags_enable_bolt_settings, enzyme_info, - offload_info, exec_ctx, explicit_stage_from_cli: flags_stage.is_some(), explicit_stage_from_config, @@ -1373,6 +1372,7 @@ impl Config { llvm_cflags, llvm_clang: llvm_clang.unwrap_or(false), llvm_clang_cl, + llvm_clang_dir: llvm_clang_dir.map(PathBuf::from), llvm_cxxflags, llvm_enable_warnings: llvm_enable_warnings.unwrap_or(false), llvm_enzyme: llvm_enzyme.unwrap_or(false), @@ -1380,7 +1380,6 @@ impl Config { llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), llvm_root_offload: llvm_root_offload.map(PathBuf::from), - llvm_clang_dir: llvm_clang_dir.map(PathBuf::from), llvm_from_ci, llvm_ldflags, llvm_libunwind_default: rust_llvm_libunwind @@ -1417,6 +1416,7 @@ impl Config { musl_root: rust_musl_root.map(PathBuf::from), ninja_in_file: llvm_ninja.unwrap_or(true), nodejs: build_nodejs.map(PathBuf::from), + offload_info, omit_git_hash, on_fail: flags_on_fail, optimized_compiler_builtins, diff --git a/src/bootstrap/src/core/config/toml/llvm.rs b/src/bootstrap/src/core/config/toml/llvm.rs index 17cd9c7b33bdc..a83df8d865c71 100644 --- a/src/bootstrap/src/core/config/toml/llvm.rs +++ b/src/bootstrap/src/core/config/toml/llvm.rs @@ -114,9 +114,10 @@ pub fn check_incompatible_options_for_ci_llvm( ldflags, use_libcxx, use_linker, - offload_cc, - offload_cxx, - root_offload, + offload_cc: _, + offload_cxx: _, + root_offload: _, + clang_dir: _, allow_old_toolchain, offload, polly, From fea02cbbb3239aeeccd344768973b9a4da73838e Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 20:42:19 -0800 Subject: [PATCH 28/50] new try --- src/bootstrap/src/core/build_steps/llvm.rs | 3 +-- src/bootstrap/src/core/config/config.rs | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 2fc1bfb811195..c9c30380a2207 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -995,12 +995,11 @@ impl Step for OmpOffload { .define("OFFLOAD_INCLUDE_TESTS", "OFF") //.define("CMAKE_C_COMPILER", cc) //.define("CMAKE_CXX_COMPILER", cxx) - .define("Clang_DIR", "/tmp/drehwald1/prog/llvm/lib/cmake/clang") .define("OPENMP_STANDALONE_BUILD", "ON") .define("LLVM_ROOT", root) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); if let Some(p) = clang_dir { - cfg.define("CLANG_DIR", p); + cfg.define("Clang_DIR", p); } cfg.build(); diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index b6fdfd0a1ab25..9b60b8475ef08 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -1377,8 +1377,6 @@ impl Config { llvm_enable_warnings: llvm_enable_warnings.unwrap_or(false), llvm_enzyme: llvm_enzyme.unwrap_or(false), llvm_experimental_targets, - llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), - llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), llvm_root_offload: llvm_root_offload.map(PathBuf::from), llvm_from_ci, llvm_ldflags, @@ -1394,6 +1392,8 @@ impl Config { .or((!llvm_from_ci && llvm_thin_lto.unwrap_or(false)).then_some(true)), ), llvm_offload: llvm_offload.unwrap_or(false), + llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), + llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), llvm_optimize: llvm_optimize.unwrap_or(true), llvm_plugins: llvm_plugin.unwrap_or(false), llvm_polly: llvm_polly.unwrap_or(false), From b363c21908bc07902f9c8c8321a69653901a3ee2 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Wed, 10 Dec 2025 20:47:50 -0800 Subject: [PATCH 29/50] try to guess the Clang_DIR to ClangConfig.cmake --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index ba62750643d4c..0a8ecb4d75941 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -91,6 +91,7 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.thin-lto=true \ --set llvm.ninja=false \ --set llvm.offload=true \ + --set llvm.offload-clang-dir="/rustroot/lib/cmake/clang" \ --set llvm.libzstd=true \ --set rust.jemalloc \ --set rust.bootstrap-override-lld=true \ From a91c0e110d5db4de29955a68ca37c54f4e7d9088 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Thu, 11 Dec 2025 02:09:19 -0800 Subject: [PATCH 30/50] fix new arg parsing --- bootstrap.example.toml | 3 +++ src/bootstrap/configure.py | 1 + 2 files changed, 4 insertions(+) diff --git a/bootstrap.example.toml b/bootstrap.example.toml index 63bf50722ca09..e233fd329b89b 100644 --- a/bootstrap.example.toml +++ b/bootstrap.example.toml @@ -106,6 +106,9 @@ # Whether to build LLVM with support for it's gpu offload runtime. #llvm.offload = false +# absolute path to ClangConfig.cmake +#llvm.offload-clang-dir = "" + # When true, link libstdc++ statically into the rustc_llvm. # This is useful if you don't want to use the dynamic version of that # library provided by LLVM. diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index e3895d4e0a189..f8f2fb2278ff9 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -120,6 +120,7 @@ def v(*args): o("llvm-assertions", "llvm.assertions", "build LLVM with assertions") o("llvm-enzyme", "llvm.enzyme", "build LLVM with enzyme") o("llvm-offload", "llvm.offload", "build LLVM with gpu offload support") +o("llvm-offload-clang-dir", "llvm.offload-clang-dir", "pass the absolute directory of ClangConfig.cmake") o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface") o("debug-assertions", "rust.debug-assertions", "build with debugging assertions") o( From 399eeff85caf47071446709c9731f3cb0355e91b Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Thu, 11 Dec 2025 08:16:27 -0800 Subject: [PATCH 31/50] explicitely add llvm Include dir to overwrite out-of-tree clang's default --- src/bootstrap/src/core/build_steps/llvm.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index c9c30380a2207..d65e8525a3188 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -778,6 +778,10 @@ fn configure_cmake( if builder.config.llvm_clang_cl.is_some() { cflags.push(format!(" --target={target}")); } + // Manuel + let base = builder.llvm_out(target).join("include"); + let inc_dir = base.display(); + cflags.push(format!(" -I {inc_dir}")); cfg.define("CMAKE_C_FLAGS", cflags); let mut cxxflags: OsString = builder .cc_handled_clags(target, CLang::Cxx) @@ -791,6 +795,7 @@ fn configure_cmake( .collect::>() .join(" ") .into(); + cxxflags.push(format!(" -I {inc_dir}")); if let Some(ref s) = builder.config.llvm_cxxflags { cxxflags.push(" "); cxxflags.push(s); From e63bcddfb34debe2d819cbd9e1f85a4d51f2ae14 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Thu, 11 Dec 2025 10:07:48 -0800 Subject: [PATCH 32/50] try not building omp/offload if pgo is enabled --- src/bootstrap/src/core/build_steps/compile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index b6b02ebaa482d..0f226c325bde0 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1428,7 +1428,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect cargo.env("LLVM_ENZYME", "1"); } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - if builder.config.llvm_offload { + if builder.config.llvm_offload && !builder.config.llvm_profile_generate { builder.ensure(llvm::OmpOffload { target }); cargo.env("LLVM_OFFLOAD", "1"); } From 351fdec55fec32ed1dff0a3a0d90a3c637d3b052 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 08:14:51 -0800 Subject: [PATCH 33/50] use out subdir to prevent artifact deletions --- src/bootstrap/src/core/build_steps/llvm.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index d65e8525a3188..5645c496bd465 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -912,19 +912,23 @@ impl Step for OmpOffload { #[allow(unused)] fn run(self, builder: &Builder<'_>) -> PathBuf { if builder.config.dry_run() { - return PathBuf::from("/"); + return builder.config.tempdir().join("llvm-offload-dry-run"); } let target = self.target; let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target }); - let out_dir = builder.llvm_out(target); + let out_dir = builder.llvm_out(target).join("offload-outdir"); + if std::fs::exists(&out_dir).is_ok_and(|x| x == false) { + std::fs::DirBuilder::new().create(&out_dir).unwrap(); + dbg!("Created out subdir!"); + } static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { generate_smart_stamp_hash( builder, &builder.config.src.join("src/llvm-project/offload"), - builder.offload_info.sha().unwrap_or_default(), + builder.in_tree_llvm_info.sha().unwrap_or_default(), ) }); let stamp = BuildStamp::new(&out_dir).with_prefix("offload").add_stamp(smart_stamp_hash); @@ -951,7 +955,7 @@ impl Step for OmpOffload { let _time = helpers::timeit(builder); t!(fs::create_dir_all(&out_dir)); - builder.config.update_submodule(Path::new("src").join("llvm-project").to_str().unwrap()); + builder.config.update_submodule("src/llvm-project"); let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/runtimes/")); configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]); From 3c8092deca90ec24d4cfd122c0dd242099150f81 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 08:46:28 -0800 Subject: [PATCH 34/50] remove outdated offload_info --- src/bootstrap/src/lib.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 6a7f3c5e52216..adae9a1b8b081 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -212,15 +212,12 @@ pub struct Build { src: PathBuf, out: PathBuf, bootstrap_out: PathBuf, - //offload_cc: PathBuf, - //offload_cxx: PathBuf, cargo_info: GitInfo, rust_analyzer_info: GitInfo, clippy_info: GitInfo, miri_info: GitInfo, rustfmt_info: GitInfo, enzyme_info: GitInfo, - offload_info: GitInfo, in_tree_llvm_info: GitInfo, in_tree_gcc_info: GitInfo, local_rebuild: bool, @@ -479,7 +476,6 @@ impl Build { let miri_info = config.miri_info.clone(); let rustfmt_info = config.rustfmt_info.clone(); let enzyme_info = config.enzyme_info.clone(); - let offload_info = config.offload_info.clone(); let in_tree_llvm_info = config.in_tree_llvm_info.clone(); let in_tree_gcc_info = config.in_tree_gcc_info.clone(); @@ -572,7 +568,6 @@ impl Build { miri_info, rustfmt_info, enzyme_info, - offload_info, in_tree_llvm_info, in_tree_gcc_info, cc: HashMap::new(), From 66af9ecbe0f3010cced318d46bc4acbbe29beb44 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 10:54:40 -0800 Subject: [PATCH 35/50] cleanup --- src/bootstrap/configure.py | 6 ++- src/bootstrap/src/core/build_steps/llvm.rs | 46 +++++++++++----------- src/bootstrap/src/core/config/config.rs | 9 ----- src/bootstrap/src/core/config/toml/llvm.rs | 6 --- 4 files changed, 29 insertions(+), 38 deletions(-) diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index f8f2fb2278ff9..206e274aa53d7 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -120,7 +120,11 @@ def v(*args): o("llvm-assertions", "llvm.assertions", "build LLVM with assertions") o("llvm-enzyme", "llvm.enzyme", "build LLVM with enzyme") o("llvm-offload", "llvm.offload", "build LLVM with gpu offload support") -o("llvm-offload-clang-dir", "llvm.offload-clang-dir", "pass the absolute directory of ClangConfig.cmake") +o( + "llvm-offload-clang-dir", + "llvm.offload-clang-dir", + "pass the absolute directory of ClangConfig.cmake", +) o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface") o("debug-assertions", "rust.debug-assertions", "build with debugging assertions") o( diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 5645c496bd465..337318db57050 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -891,6 +891,17 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option { .or_else(|| env::var_os(var_base)) } +// FIXME(offload): In an ideal world, we would just enable the offload runtime in our previous LLVM +// build step. For now, we still depend on the openmp runtime since we use some of it's API, so we +// build both. However, when building those runtimes as part of the LLVM step, then LLVM's cmake +// implicitely assumes that Clang has also been build and will try to use it. In the Rust CI, we +// don't always build clang (due to compile times), but instead use a slightly older external clang. +// LLVM tries to remove this build dependency of offload/openmp on Clang for LLVM-22, so in the +// future we might be able to integrate this step into the LLVM step. For now, we instead introduce +// a Clang_DIR bootstrap option, which allows us tell CMake to use an external clang for these two +// runtimes. This external clang will try to use it's own (older) include dirs when building our +// in-tree LLVM submodule, which will cause build failures. To prevent those, we now also +// explicitely set our include dirs. #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct OmpOffload { pub target: TargetSelection, @@ -918,11 +929,17 @@ impl Step for OmpOffload { let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target }); + // Running cmake twice in the same folder is known to cause issues, like deleting existing + // binaries. We therefore write our offload artifacts into it's own subfolder. We use a + // subfolder, so that all the logic that processes our build artifacts (hopefully) also + // automatically manages our artifacts in the subfolder. let out_dir = builder.llvm_out(target).join("offload-outdir"); if std::fs::exists(&out_dir).is_ok_and(|x| x == false) { std::fs::DirBuilder::new().create(&out_dir).unwrap(); dbg!("Created out subdir!"); } + + // Offload/OpenMP are just subfolders of LLVM, so we can use the LLVM sha. static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { generate_smart_stamp_hash( @@ -960,8 +977,7 @@ impl Step for OmpOffload { configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]); // Re-use the same flags as llvm to control the level of debug information - // generated by Enzyme. - // FIXME(ZuseZ4): Find a nicer way to use Enzyme Debug builds. + // generated for offload. let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { (false, _) => "Debug", (true, false) => "Release", @@ -969,23 +985,8 @@ impl Step for OmpOffload { }; trace!(?profile); - //let cc = if let Some(p) = &builder.build.config.llvm_offload_cc { - // //p.clone() - // builder.cc(target) - //} else { - // builder.cc(target) - //}; - //let cxx = if let Some(p) = &builder.build.config.llvm_offload_cxx { - // //p.clone() - // builder.cxx(target).unwrap() - //} else { - // builder.cxx(target).unwrap() - //}; - let root = if let Some(p) = &builder.build.config.llvm_root_offload { - p.clone() - } else { - builder.llvm_out(target).join("build") - }; + // OpenMP/Offload builds currently (LLVM-21) still depend on Clang, although there are + // intentions to loosen this requirement for LLVM-22. If we were to let clang_dir = if !builder.config.llvm_clang { // We must have an external clang to use. assert!(&builder.build.config.llvm_clang_dir.is_some()); @@ -995,6 +996,9 @@ impl Step for OmpOffload { None }; + // FIXME(offload): Once we move from OMP to Offload (Ol) APIs, we should drop the openmp + // runtime to simplify our build. We should also re-evaluate the LLVM_Root and try to get + // rid of the Clang_DIR, once we upgrade to LLVM-22. cfg.out_dir(&out_dir) .profile(profile) .env("LLVM_CONFIG_REAL", &host_llvm_config) @@ -1002,10 +1006,8 @@ impl Step for OmpOffload { .define("LLVM_ENABLE_RUNTIMES", "openmp;offload") .define("LLVM_INCLUDE_TESTS", "OFF") .define("OFFLOAD_INCLUDE_TESTS", "OFF") - //.define("CMAKE_C_COMPILER", cc) - //.define("CMAKE_CXX_COMPILER", cxx) .define("OPENMP_STANDALONE_BUILD", "ON") - .define("LLVM_ROOT", root) + .define("LLVM_ROOT", builder.llvm_out(target).join("build")) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); if let Some(p) = clang_dir { cfg.define("Clang_DIR", p); diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 9b60b8475ef08..d6496297a6b32 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -169,9 +169,6 @@ pub struct Config { pub llvm_link_jobs: Option, pub llvm_version_suffix: Option, pub llvm_use_linker: Option, - pub llvm_offload_cc: Option, - pub llvm_offload_cxx: Option, - pub llvm_root_offload: Option, pub llvm_clang_dir: Option, pub llvm_allow_old_toolchain: bool, pub llvm_polly: bool, @@ -607,9 +604,6 @@ impl Config { ldflags: llvm_ldflags, use_libcxx: llvm_use_libcxx, use_linker: llvm_use_linker, - offload_cc: llvm_offload_cc, - offload_cxx: llvm_offload_cxx, - root_offload: llvm_root_offload, clang_dir: llvm_clang_dir, allow_old_toolchain: llvm_allow_old_toolchain, offload: llvm_offload, @@ -1377,7 +1371,6 @@ impl Config { llvm_enable_warnings: llvm_enable_warnings.unwrap_or(false), llvm_enzyme: llvm_enzyme.unwrap_or(false), llvm_experimental_targets, - llvm_root_offload: llvm_root_offload.map(PathBuf::from), llvm_from_ci, llvm_ldflags, llvm_libunwind_default: rust_llvm_libunwind @@ -1392,8 +1385,6 @@ impl Config { .or((!llvm_from_ci && llvm_thin_lto.unwrap_or(false)).then_some(true)), ), llvm_offload: llvm_offload.unwrap_or(false), - llvm_offload_cc: llvm_offload_cc.map(PathBuf::from), - llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from), llvm_optimize: llvm_optimize.unwrap_or(true), llvm_plugins: llvm_plugin.unwrap_or(false), llvm_polly: llvm_polly.unwrap_or(false), diff --git a/src/bootstrap/src/core/config/toml/llvm.rs b/src/bootstrap/src/core/config/toml/llvm.rs index a83df8d865c71..8f42c0027aa5f 100644 --- a/src/bootstrap/src/core/config/toml/llvm.rs +++ b/src/bootstrap/src/core/config/toml/llvm.rs @@ -32,9 +32,6 @@ define_config! { ldflags: Option = "ldflags", use_libcxx: Option = "use-libcxx", use_linker: Option = "use-linker", - offload_cc: Option = "offload-cc", - offload_cxx: Option = "offload-cxx", - root_offload: Option = "root-offload", clang_dir: Option = "offload-clang-dir", allow_old_toolchain: Option = "allow-old-toolchain", offload: Option = "offload", @@ -114,9 +111,6 @@ pub fn check_incompatible_options_for_ci_llvm( ldflags, use_libcxx, use_linker, - offload_cc: _, - offload_cxx: _, - root_offload: _, clang_dir: _, allow_old_toolchain, offload, From 3172aee69ec5c198a546d836f98d18f996ce24bd Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 11:06:07 -0800 Subject: [PATCH 36/50] cleanup --- src/bootstrap/configure.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index 206e274aa53d7..fda8f004eff42 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -121,9 +121,9 @@ def v(*args): o("llvm-enzyme", "llvm.enzyme", "build LLVM with enzyme") o("llvm-offload", "llvm.offload", "build LLVM with gpu offload support") o( - "llvm-offload-clang-dir", - "llvm.offload-clang-dir", - "pass the absolute directory of ClangConfig.cmake", + "llvm-offload-clang-dir", + "llvm.offload-clang-dir", + "pass the absolute directory of ClangConfig.cmake", ) o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface") o("debug-assertions", "rust.debug-assertions", "build with debugging assertions") From e910bf2948536b5c1759fe0d0448a02ed53abbe8 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 11:16:04 -0800 Subject: [PATCH 37/50] cleanup --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 337318db57050..2ef1ea9071fa5 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -894,7 +894,7 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option { // FIXME(offload): In an ideal world, we would just enable the offload runtime in our previous LLVM // build step. For now, we still depend on the openmp runtime since we use some of it's API, so we // build both. However, when building those runtimes as part of the LLVM step, then LLVM's cmake -// implicitely assumes that Clang has also been build and will try to use it. In the Rust CI, we +// implicitly assumes that Clang has also been build and will try to use it. In the Rust CI, we // don't always build clang (due to compile times), but instead use a slightly older external clang. // LLVM tries to remove this build dependency of offload/openmp on Clang for LLVM-22, so in the // future we might be able to integrate this step into the LLVM step. For now, we instead introduce From c8e683349d5636325e19df40d1f0bdfe0f542ba6 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 12:49:35 -0800 Subject: [PATCH 38/50] typo --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 2ef1ea9071fa5..2c0f147847ce3 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -901,7 +901,7 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option { // a Clang_DIR bootstrap option, which allows us tell CMake to use an external clang for these two // runtimes. This external clang will try to use it's own (older) include dirs when building our // in-tree LLVM submodule, which will cause build failures. To prevent those, we now also -// explicitely set our include dirs. +// explicitly set our include dirs. #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct OmpOffload { pub target: TargetSelection, From 6c7edd3fb51901e36eefb54cc6eec2e8011467fa Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 12:59:02 -0800 Subject: [PATCH 39/50] clean up Include dir handling --- src/bootstrap/src/core/build_steps/llvm.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 2c0f147847ce3..395ba14735fea 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -768,6 +768,17 @@ fn configure_cmake( .collect::>() .join(" ") .into(); + + // If we use an external clang as opposed to building our own llvm_clang, than that clang will + // come with it's own set of default include directories, which are based on a potentially older + // LLVM. This can cause issues, so we overwrite it to include headers based on our + // `src/llvm-project` submodule instead. + // FIXME(offload): With LLVM-22 we hopefully won't need an external clang anymore. + let base = builder.llvm_out(target).join("include"); + let inc_dir = base.display(); + if builder.config.llvm_offload && !builder.config.llvm_clang { + cflags.push(format!(" -I {inc_dir}")); + } if let Some(ref s) = builder.config.llvm_cflags { cflags.push(" "); cflags.push(s); @@ -778,11 +789,8 @@ fn configure_cmake( if builder.config.llvm_clang_cl.is_some() { cflags.push(format!(" --target={target}")); } - // Manuel - let base = builder.llvm_out(target).join("include"); - let inc_dir = base.display(); - cflags.push(format!(" -I {inc_dir}")); cfg.define("CMAKE_C_FLAGS", cflags); + let mut cxxflags: OsString = builder .cc_handled_clags(target, CLang::Cxx) .into_iter() @@ -795,7 +803,9 @@ fn configure_cmake( .collect::>() .join(" ") .into(); - cxxflags.push(format!(" -I {inc_dir}")); + if builder.config.llvm_offload && !builder.config.llvm_clang { + cxxflags.push(format!(" -I {inc_dir}")); + } if let Some(ref s) = builder.config.llvm_cxxflags { cxxflags.push(" "); cxxflags.push(s); From ee649dd09257c3c77e7f0ef5011797f0f96bdf2e Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 13:03:58 -0800 Subject: [PATCH 40/50] Update ChangeInfo to account for new offload-clang-dir flag --- src/bootstrap/src/utils/change_tracker.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index d7990c2316d15..1077077cb97a2 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -606,4 +606,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Info, summary: "New option `gcc.libgccjit-libs-dir` to specify which libgccjit.so to use per target.", }, + ChangeInfo { + change_id: 148671, + severity: ChangeSeverity::Info, + summary: "New option `llvm.offload-clang-dir` to allow building an in-tree llvm offload and openmp runtime with an external clang.", + }, ]; From 141ff187c76479a4f22fce34f1f742844a36cb9d Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 13:05:56 -0800 Subject: [PATCH 41/50] remove checking for llvm_profile_generate, since it likely wasn't the culprit --- src/bootstrap/src/core/build_steps/compile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 0f226c325bde0..b6b02ebaa482d 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1428,7 +1428,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect cargo.env("LLVM_ENZYME", "1"); } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - if builder.config.llvm_offload && !builder.config.llvm_profile_generate { + if builder.config.llvm_offload { builder.ensure(llvm::OmpOffload { target }); cargo.env("LLVM_OFFLOAD", "1"); } From ecf9534212c27ddade90e598ee96c24540cca2a4 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Fri, 12 Dec 2025 13:15:42 -0800 Subject: [PATCH 42/50] cleanup --- src/bootstrap/src/core/build_steps/llvm.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 395ba14735fea..36eaf025b4eaf 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -946,7 +946,6 @@ impl Step for OmpOffload { let out_dir = builder.llvm_out(target).join("offload-outdir"); if std::fs::exists(&out_dir).is_ok_and(|x| x == false) { std::fs::DirBuilder::new().create(&out_dir).unwrap(); - dbg!("Created out subdir!"); } // Offload/OpenMP are just subfolders of LLVM, so we can use the LLVM sha. From df47148bfe15a3d88da8d85e0272d8385da6cf9c Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 15 Dec 2025 04:05:08 -0800 Subject: [PATCH 43/50] new changes, not passed ci yet --- src/bootstrap/src/core/build_steps/compile.rs | 52 +++++++++++++++++++ src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index b6b02ebaa482d..341d5f3e9620e 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -2295,6 +2295,58 @@ impl Step for Assemble { } } + if builder.config.llvm_offload && !builder.config.dry_run() { + debug!("`llvm_offload` requested"); + let offload_install = builder.ensure(llvm::OmpOffload { target: build_compiler.host }); + if let Some(_llvm_config) = builder.llvm_config(builder.config.host_target) { + let src_dir = offload_install.join("lib"); + let libdir = builder.sysroot_target_libdir(build_compiler, build_compiler.host); + let target_libdir = + builder.sysroot_target_libdir(target_compiler, target_compiler.host); + let lib_ext = std::env::consts::DLL_EXTENSION; + + let libenzyme = format!("libLLVMOffload"); + let src_lib = src_dir.join(&libenzyme).with_extension(lib_ext); + let dst_lib = libdir.join(&libenzyme).with_extension(lib_ext); + let target_dst_lib = target_libdir.join(&libenzyme).with_extension(lib_ext); + builder.resolve_symlink_and_copy(&src_lib, &dst_lib); + builder.resolve_symlink_and_copy(&src_lib, &target_dst_lib); + + // FIXME(offload): With LLVM-22, we should be able to drop everything below here. + let omp = format!("libomp"); + let src_omp = src_dir.join(&omp).with_extension(lib_ext); + let dst_omp_lib = libdir.join(&omp).with_extension(lib_ext); + let target_omp_dst_lib = target_libdir.join(&omp).with_extension(lib_ext); + builder.resolve_symlink_and_copy(&src_omp, &dst_omp_lib); + builder.resolve_symlink_and_copy(&src_omp, &target_omp_dst_lib); + + let tgt = format!("libomptarget"); + let src_tgt = src_dir.join(&tgt).with_extension(lib_ext); + let dst_tgt_lib = libdir.join(&tgt).with_extension(lib_ext); + let target_tgt_dst_lib = target_libdir.join(&tgt).with_extension(lib_ext); + builder.resolve_symlink_and_copy(&src_tgt, &dst_tgt_lib); + builder.resolve_symlink_and_copy(&src_tgt, &target_tgt_dst_lib); + + // The last one is slightly more tricky, since we have the same file twice, in two + // subfolders for amdgcn and nvptx64. We'll likely find two more in the future, once + // Intel and Spir-V support lands in offload. + let gpu_tgts = ["amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; + let device = format!("libompdevice.a"); + for tgt in gpu_tgts { + let dst_tgt_dir = libdir.join(&tgt); + let target_tgt_dst_dir = target_libdir.join(&tgt); + t!(fs::create_dir_all(&dst_tgt_dir)); + t!(fs::create_dir_all(&target_tgt_dst_dir)); + let dst_tgt_lib = dst_tgt_dir.join(&device); + let target_tgt_dst_lib = target_tgt_dst_dir.join(&device); + builder.resolve_symlink_and_copy(&src_tgt, &dst_tgt_lib); + builder.resolve_symlink_and_copy(&src_tgt, &target_tgt_dst_lib); + // FIXME(offload): copy the files within the directories as well, but figure out + // the naming scheme before. + } + } + } + // Build the libraries for this compiler to link to (i.e., the libraries // it uses at runtime). debug!( diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 36eaf025b4eaf..2ed38b41cfe6b 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -944,7 +944,7 @@ impl Step for OmpOffload { // subfolder, so that all the logic that processes our build artifacts (hopefully) also // automatically manages our artifacts in the subfolder. let out_dir = builder.llvm_out(target).join("offload-outdir"); - if std::fs::exists(&out_dir).is_ok_and(|x| x == false) { + if std::fs::exists(&out_dir).is_ok_and(|x| !x) { std::fs::DirBuilder::new().create(&out_dir).unwrap(); } From 8738dd6065b4f6dc74dd10d327bbe211d93e3da8 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 15 Dec 2025 04:17:35 -0800 Subject: [PATCH 44/50] try lld for runtime builds --- src/bootstrap/src/core/build_steps/llvm.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 2ed38b41cfe6b..185d2cc26bd21 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -1018,6 +1018,8 @@ impl Step for OmpOffload { .define("OPENMP_STANDALONE_BUILD", "ON") .define("LLVM_ROOT", builder.llvm_out(target).join("build")) .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm")); + + cfg.define("LLVM_USE_LINKER", "lld"); if let Some(p) = clang_dir { cfg.define("Clang_DIR", p); } From 1e09a3c5decc80ded96088af1a17ddf157c9bd05 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 15 Dec 2025 06:24:27 -0800 Subject: [PATCH 45/50] disable offload (again) during profiling --- src/bootstrap/src/core/build_steps/compile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 341d5f3e9620e..7e856a3978d16 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1428,7 +1428,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect cargo.env("LLVM_ENZYME", "1"); } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - if builder.config.llvm_offload { + if builder.config.llvm_offload && !builder.config.llvm_profile_generate { builder.ensure(llvm::OmpOffload { target }); cargo.env("LLVM_OFFLOAD", "1"); } From 0537fb8073ab66f3ef1ffa2306db7ca214576d73 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 15 Dec 2025 07:18:15 -0800 Subject: [PATCH 46/50] last try --- src/bootstrap/src/core/build_steps/compile.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 7e856a3978d16..1785aa444d439 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1428,7 +1428,10 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect cargo.env("LLVM_ENZYME", "1"); } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - if builder.config.llvm_offload && !builder.config.llvm_profile_generate { + if builder.config.llvm_offload + && !builder.config.llvm_profile_generate + && !builder.config.llvm_profile_use + { builder.ensure(llvm::OmpOffload { target }); cargo.env("LLVM_OFFLOAD", "1"); } From 139aeed9e1ef807581ff5217368bfc75fc20bedf Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 15 Dec 2025 07:45:42 -0800 Subject: [PATCH 47/50] last try --- src/bootstrap/src/core/build_steps/compile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 1785aa444d439..adc850a9e2079 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1430,7 +1430,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); if builder.config.llvm_offload && !builder.config.llvm_profile_generate - && !builder.config.llvm_profile_use + && builder.config.llvm_profile_use.is_none() { builder.ensure(llvm::OmpOffload { target }); cargo.env("LLVM_OFFLOAD", "1"); From e9ef6c02932d4484b79f7dfc7ca4d33ecf90e451 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 15 Dec 2025 09:18:12 -0800 Subject: [PATCH 48/50] last try --- src/bootstrap/src/core/build_steps/compile.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index adc850a9e2079..b8cc0e6a15d6a 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -2298,7 +2298,11 @@ impl Step for Assemble { } } - if builder.config.llvm_offload && !builder.config.dry_run() { + if builder.config.llvm_offload + && !builder.config.dry_run() + && !builder.config.llvm_profile_generate + && builder.config.llvm_profile_use.is_none() + { debug!("`llvm_offload` requested"); let offload_install = builder.ensure(llvm::OmpOffload { target: build_compiler.host }); if let Some(_llvm_config) = builder.llvm_config(builder.config.host_target) { From e1dbd9c726df2f27c545d843528a86fc0194701c Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Mon, 15 Dec 2025 21:50:54 +0100 Subject: [PATCH 49/50] move offload checks into the step, still copy files --- src/bootstrap/src/core/build_steps/compile.rs | 11 ++--------- src/bootstrap/src/core/build_steps/llvm.rs | 11 ++++++++++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index b8cc0e6a15d6a..341d5f3e9620e 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1428,10 +1428,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect cargo.env("LLVM_ENZYME", "1"); } let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - if builder.config.llvm_offload - && !builder.config.llvm_profile_generate - && builder.config.llvm_profile_use.is_none() - { + if builder.config.llvm_offload { builder.ensure(llvm::OmpOffload { target }); cargo.env("LLVM_OFFLOAD", "1"); } @@ -2298,11 +2295,7 @@ impl Step for Assemble { } } - if builder.config.llvm_offload - && !builder.config.dry_run() - && !builder.config.llvm_profile_generate - && builder.config.llvm_profile_use.is_none() - { + if builder.config.llvm_offload && !builder.config.dry_run() { debug!("`llvm_offload` requested"); let offload_install = builder.ensure(llvm::OmpOffload { target: build_compiler.host }); if let Some(_llvm_config) = builder.llvm_config(builder.config.host_target) { diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 185d2cc26bd21..abe12c3da52c4 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -935,9 +935,10 @@ impl Step for OmpOffload { if builder.config.dry_run() { return builder.config.tempdir().join("llvm-offload-dry-run"); } + let target = self.target; - let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target }); + let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target }); // Running cmake twice in the same folder is known to cause issues, like deleting existing // binaries. We therefore write our offload artifacts into it's own subfolder. We use a @@ -948,6 +949,14 @@ impl Step for OmpOffload { std::fs::DirBuilder::new().create(&out_dir).unwrap(); } + if !builder.config.llvm_offload + || builder.config.dry_run() + || builder.config.llvm_profile_generate + || builder.config.llvm_profile_use.is_some() + { + return out_dir; + } + // Offload/OpenMP are just subfolders of LLVM, so we can use the LLVM sha. static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { From e82edfe8a8ebcfc484914c70582fafb4a2a80429 Mon Sep 17 00:00:00 2001 From: Manuel Drehwald Date: Tue, 16 Dec 2025 07:26:06 -0800 Subject: [PATCH 50/50] only copy ol files if they exist, to avoid complex logic --- src/bootstrap/src/core/build_steps/compile.rs | 66 ++++++++++--------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 341d5f3e9620e..3f8668124d7bb 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -2309,40 +2309,42 @@ impl Step for Assemble { let src_lib = src_dir.join(&libenzyme).with_extension(lib_ext); let dst_lib = libdir.join(&libenzyme).with_extension(lib_ext); let target_dst_lib = target_libdir.join(&libenzyme).with_extension(lib_ext); - builder.resolve_symlink_and_copy(&src_lib, &dst_lib); - builder.resolve_symlink_and_copy(&src_lib, &target_dst_lib); - - // FIXME(offload): With LLVM-22, we should be able to drop everything below here. - let omp = format!("libomp"); - let src_omp = src_dir.join(&omp).with_extension(lib_ext); - let dst_omp_lib = libdir.join(&omp).with_extension(lib_ext); - let target_omp_dst_lib = target_libdir.join(&omp).with_extension(lib_ext); - builder.resolve_symlink_and_copy(&src_omp, &dst_omp_lib); - builder.resolve_symlink_and_copy(&src_omp, &target_omp_dst_lib); - - let tgt = format!("libomptarget"); - let src_tgt = src_dir.join(&tgt).with_extension(lib_ext); - let dst_tgt_lib = libdir.join(&tgt).with_extension(lib_ext); - let target_tgt_dst_lib = target_libdir.join(&tgt).with_extension(lib_ext); - builder.resolve_symlink_and_copy(&src_tgt, &dst_tgt_lib); - builder.resolve_symlink_and_copy(&src_tgt, &target_tgt_dst_lib); - - // The last one is slightly more tricky, since we have the same file twice, in two - // subfolders for amdgcn and nvptx64. We'll likely find two more in the future, once - // Intel and Spir-V support lands in offload. - let gpu_tgts = ["amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; - let device = format!("libompdevice.a"); - for tgt in gpu_tgts { - let dst_tgt_dir = libdir.join(&tgt); - let target_tgt_dst_dir = target_libdir.join(&tgt); - t!(fs::create_dir_all(&dst_tgt_dir)); - t!(fs::create_dir_all(&target_tgt_dst_dir)); - let dst_tgt_lib = dst_tgt_dir.join(&device); - let target_tgt_dst_lib = target_tgt_dst_dir.join(&device); + if src_lib.exists() { + builder.resolve_symlink_and_copy(&src_lib, &dst_lib); + builder.resolve_symlink_and_copy(&src_lib, &target_dst_lib); + + // FIXME(offload): With LLVM-22, we should be able to drop everything below here. + let omp = format!("libomp"); + let src_omp = src_dir.join(&omp).with_extension(lib_ext); + let dst_omp_lib = libdir.join(&omp).with_extension(lib_ext); + let target_omp_dst_lib = target_libdir.join(&omp).with_extension(lib_ext); + builder.resolve_symlink_and_copy(&src_omp, &dst_omp_lib); + builder.resolve_symlink_and_copy(&src_omp, &target_omp_dst_lib); + + let tgt = format!("libomptarget"); + let src_tgt = src_dir.join(&tgt).with_extension(lib_ext); + let dst_tgt_lib = libdir.join(&tgt).with_extension(lib_ext); + let target_tgt_dst_lib = target_libdir.join(&tgt).with_extension(lib_ext); builder.resolve_symlink_and_copy(&src_tgt, &dst_tgt_lib); builder.resolve_symlink_and_copy(&src_tgt, &target_tgt_dst_lib); - // FIXME(offload): copy the files within the directories as well, but figure out - // the naming scheme before. + + // The last one is slightly more tricky, since we have the same file twice, in two + // subfolders for amdgcn and nvptx64. We'll likely find two more in the future, once + // Intel and Spir-V support lands in offload. + let gpu_tgts = ["amdgcn-amd-amdhsa", "nvptx64-nvidia-cuda"]; + let device = format!("libompdevice.a"); + for tgt in gpu_tgts { + let dst_tgt_dir = libdir.join(&tgt); + let target_tgt_dst_dir = target_libdir.join(&tgt); + t!(fs::create_dir_all(&dst_tgt_dir)); + t!(fs::create_dir_all(&target_tgt_dst_dir)); + let dst_tgt_lib = dst_tgt_dir.join(&device); + let target_tgt_dst_lib = target_tgt_dst_dir.join(&device); + builder.resolve_symlink_and_copy(&src_tgt, &dst_tgt_lib); + builder.resolve_symlink_and_copy(&src_tgt, &target_tgt_dst_lib); + // FIXME(offload): copy the files within the directories as well, but figure out + // the naming scheme before. + } } } }