diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 29458c7db0d9..d92876fab8a4 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,5 +1,165 @@
 ##
-# Lists of source files. Keep ALL lists sorted in alphabetical order.
+# Set up additional backend options for Halide
+##
+option(TARGET_OPENCL "Include OpenCL-C target" ON)
+option(TARGET_OPENGL "Include OpenGL/GLSL target" ON)
+option(TARGET_METAL "Include Metal target" ON)
+option(TARGET_D3D12COMPUTE "Include Direct3D 12 Compute target" ON)
+
+##
+# List of optional targets
+##
+
+set(OPTIONAL_TARGETS
+    AARCH64
+    AMDGPU
+    ARM
+    HEXAGON
+    MIPS
+    NVPTX
+    POWERPC
+    RISCV
+    WEBASSEMBLY
+    X86
+    OPENCL
+    OPENGL
+    METAL
+    D3D12COMPUTE
+    )
+
+##
+# Lists of target-specific source files.
+# Keep ALL lists sorted in alphabetical order.
+##
+
+##
+# Header files
+##
+
+# The externally-visible header files that go into making Halide.h.
+# Don't include anything here that includes llvm headers.
+
+set(AARCH64_HEADER_FILES
+    )
+
+set(AMDGPU_HEADER_FILES
+    )
+
+set(ARM_HEADER_FILES
+    CodeGen_ARM.h
+    )
+
+set(HEXAGON_HEADER_FILES
+    HexagonOffload.h
+    HexagonOptimize.h
+    )
+
+set(MIPS_HEADER_FILES
+    CodeGen_MIPS.h
+    )
+
+set(NVPTX_HEADER_FILES
+    CodeGen_PTX_Dev.h
+    )
+
+set(POWERPC_HEADER_FILES
+    CodeGen_PowerPC.h
+    )
+
+set(RISCV_HEADER_FILES
+    CodeGen_RISCV.h
+    )
+
+set(WEBASSEMBLY_HEADER_FILES
+    CodeGen_WebAssembly.h
+    )
+
+set(X86_HEADER_FILES
+    CodeGen_X86.h
+    )
+
+set(OPENCL_HEADER_FILES
+    CodeGen_OpenCL_Dev.h
+    )
+
+set(OPENGL_HEADER_FILES
+    CodeGen_OpenGL_Dev.h
+    CodeGen_OpenGLCompute_Dev.h
+    InjectOpenGLIntrinsics.h
+    )
+
+set(METAL_HEADER_FILES
+    CodeGen_Metal_Dev.h
+    )
+
+set(D3D12COMPUTE_HEADER_FILES
+    CodeGen_D3D12Compute_Dev.h
+    )
+
+##
+# Source files
+##
+set(AARCH64_SOURCE_FILES
+    )
+
+set(AMDGPU_SOURCE_FILES
+    )
+
+set(ARM_SOURCE_FILES
+    CodeGen_ARM.cpp
+    )
+
+set(HEXAGON_SOURCE_FILES
+    CodeGen_Hexagon.cpp
+    HexagonOffload.cpp
+    HexagonOptimize.cpp
+    )
+
+set(MIPS_SOURCE_FILES
+    CodeGen_MIPS.cpp
+    )
+
+set(NVPTX_SOURCE_FILES
+    CodeGen_PTX_Dev.cpp
+    )
+
+set(POWERPC_SOURCE_FILES
+    CodeGen_PowerPC.cpp
+    )
+
+set(RISCV_SOURCE_FILES
+    CodeGen_RISCV.cpp
+    )
+
+set(WEBASSEMBLY_SOURCE_FILES
+    CodeGen_WebAssembly.cpp
+    )
+
+set(X86_SOURCE_FILES
+    CodeGen_X86.cpp
+    )
+
+set(OPENCL_SOURCE_FILES
+    CodeGen_OpenCL_Dev.cpp
+    )
+
+set(OPENGL_SOURCE_FILES
+    CodeGen_OpenGL_Dev.cpp
+    CodeGen_OpenGLCompute_Dev.cpp
+    InjectOpenGLIntrinsics.cpp
+    )
+
+set(METAL_SOURCE_FILES
+    CodeGen_Metal_Dev.cpp
+    )
+
+set(D3D12COMPUTE_SOURCE_FILES
+    CodeGen_D3D12Compute_Dev.cpp
+    )
+
+##
+# Lists of non-optional source files.
+# Keep ALL lists sorted in alphabetical order.
 ##
 
 # The externally-visible header files that go into making Halide.h.
@@ -23,25 +183,13 @@ set(HEADER_FILES
     Buffer.h
     CanonicalizeGPUVars.h
     Closure.h
-    CodeGen_ARM.h
     CodeGen_C.h
-    CodeGen_D3D12Compute_Dev.h
     CodeGen_GPU_Dev.h
     CodeGen_GPU_Host.h
     CodeGen_Internal.h
     CodeGen_LLVM.h
-    CodeGen_Metal_Dev.h
-    CodeGen_MIPS.h
-    CodeGen_OpenCL_Dev.h
-    CodeGen_OpenGL_Dev.h
-    CodeGen_OpenGLCompute_Dev.h
     CodeGen_Posix.h
-    CodeGen_PowerPC.h
-    CodeGen_PTX_Dev.h
     CodeGen_PyTorch.h
-    CodeGen_RISCV.h
-    CodeGen_WebAssembly.h
-    CodeGen_X86.h
     CompilerLogger.h
     ConciseCasts.h
     CPlusPlusMangle.h
@@ -76,12 +224,9 @@ set(HEADER_FILES
     FuseGPUThreadLoops.h
     FuzzFloatStores.h
     Generator.h
-    HexagonOffload.h
-    HexagonOptimize.h
     ImageParam.h
     InferArguments.h
     InjectHostDevBufferCopies.h
-    InjectOpenGLIntrinsics.h
     Inline.h
     InlineReductions.h
     IntegerDivisionTable.h
@@ -188,26 +333,13 @@ set(SOURCE_FILES
     Buffer.cpp
     CanonicalizeGPUVars.cpp
     Closure.cpp
-    CodeGen_ARM.cpp
     CodeGen_C.cpp
-    CodeGen_D3D12Compute_Dev.cpp
     CodeGen_GPU_Dev.cpp
     CodeGen_GPU_Host.cpp
-    CodeGen_Hexagon.cpp
     CodeGen_Internal.cpp
     CodeGen_LLVM.cpp
-    CodeGen_Metal_Dev.cpp
-    CodeGen_MIPS.cpp
-    CodeGen_OpenCL_Dev.cpp
-    CodeGen_OpenGL_Dev.cpp
-    CodeGen_OpenGLCompute_Dev.cpp
     CodeGen_Posix.cpp
-    CodeGen_PowerPC.cpp
-    CodeGen_PTX_Dev.cpp
     CodeGen_PyTorch.cpp
-    CodeGen_RISCV.cpp
-    CodeGen_WebAssembly.cpp
-    CodeGen_X86.cpp
     CompilerLogger.cpp
     CPlusPlusMangle.cpp
     CSE.cpp
@@ -236,12 +368,9 @@ set(SOURCE_FILES
     FuseGPUThreadLoops.cpp
     FuzzFloatStores.cpp
     Generator.cpp
-    HexagonOffload.cpp
-    HexagonOptimize.cpp
     ImageParam.cpp
     InferArguments.cpp
     InjectHostDevBufferCopies.cpp
-    InjectOpenGLIntrinsics.cpp
     Inline.cpp
     InlineReductions.cpp
     IntegerDivisionTable.cpp
@@ -341,6 +470,28 @@ set(SOURCE_FILES
     WrapCalls.cpp
     )
 
+##
+# Create optional targets
+##
+
+foreach(OPTIONAL_TARGET ${OPTIONAL_TARGETS})
+    if(TARGET_${OPTIONAL_TARGET})
+        add_library(Halide_${OPTIONAL_TARGET} INTERFACE)
+        target_sources(Halide_${OPTIONAL_TARGET} INTERFACE
+            ${${OPTIONAL_TARGET}_SOURCE_FILES}
+            )
+        target_compile_definitions(Halide_${OPTIONAL_TARGET} INTERFACE
+            WITH_${OPTIONAL_TARGET})
+
+        list(APPEND HEADER_FILES ${${OPTIONAL_TARGET}_HEADER_FILES})
+    endif()
+endforeach()
+
+# Special case definition for D3D12COMPUTE for compatibility
+if (TARGET_D3D12COMPUTE)
+    target_compile_definitions(Halide_D3D12COMPUTE INTERFACE WITH_D3D12)
+endif ()
+
 ##
 # Build and import the runtime.
 ##
@@ -373,6 +524,16 @@ add_library(Halide
             $<TARGET_OBJECTS:Halide_initmod>)
 add_library(Halide::Halide ALIAS Halide)
 
+##
+# Link optional targets to the main library
+##
+
+foreach(OPTIONAL_TARGET ${OPTIONAL_TARGETS})
+    target_link_libraries(Halide PRIVATE
+        $<BUILD_INTERFACE:$<TARGET_NAME_IF_EXISTS:Halide_${OPTIONAL_TARGET}>>
+        )
+endforeach()
+
 target_link_libraries(Halide PRIVATE Halide::LLVM)
 target_link_libraries(Halide PUBLIC Halide::LanguageOptions)
 target_compile_definitions(Halide
@@ -445,30 +606,6 @@ target_compile_definitions(Halide
                            $<$<CXX_COMPILER_ID:MSVC>:_SCL_SECURE_NO_WARNINGS>
                            )
 
-##
-# Set up additional backend options for Halide
-##
-
-option(TARGET_OPENCL "Include OpenCL-C target" ON)
-if (TARGET_OPENCL)
-    target_compile_definitions(Halide PRIVATE WITH_OPENCL)
-endif ()
-
-option(TARGET_OPENGL "Include OpenGL/GLSL target" ON)
-if (TARGET_OPENGL)
-    target_compile_definitions(Halide PRIVATE WITH_OPENGL)
-endif ()
-
-option(TARGET_METAL "Include Metal target" ON)
-if (TARGET_METAL)
-    target_compile_definitions(Halide PRIVATE WITH_METAL)
-endif ()
-
-option(TARGET_D3D12COMPUTE "Include Direct3D 12 Compute target" ON)
-if (TARGET_D3D12COMPUTE)
-    target_compile_definitions(Halide PRIVATE WITH_D3D12)
-endif ()
-
 ##
 # Add autoschedulers to the build.
 ##
diff --git a/src/CodeGen_GPU_Host.cpp b/src/CodeGen_GPU_Host.cpp
index 4cb09da4712d..fefff95609fb 100644
--- a/src/CodeGen_GPU_Host.cpp
+++ b/src/CodeGen_GPU_Host.cpp
@@ -1,13 +1,23 @@
 #include <sstream>
 
-#include "CodeGen_D3D12Compute_Dev.h"
+#ifdef WITH_D3D12
+    #include "CodeGen_D3D12Compute_Dev.h"
+#endif
 #include "CodeGen_GPU_Host.h"
 #include "CodeGen_Internal.h"
-#include "CodeGen_Metal_Dev.h"
-#include "CodeGen_OpenCL_Dev.h"
-#include "CodeGen_OpenGLCompute_Dev.h"
-#include "CodeGen_OpenGL_Dev.h"
-#include "CodeGen_PTX_Dev.h"
+#ifdef WITH_METAL
+    #include "CodeGen_Metal_Dev.h"
+#endif
+#ifdef WITH_OPENCL
+    #include "CodeGen_OpenCL_Dev.h"
+#endif
+#ifdef WITH_OPENGL
+    #include "CodeGen_OpenGLCompute_Dev.h"
+    #include "CodeGen_OpenGL_Dev.h"
+#endif
+#ifdef WITH_NVPTX
+    #include "CodeGen_PTX_Dev.h"
+#endif
 #include "Debug.h"
 #include "DeviceArgument.h"
 #include "ExprUsesVar.h"
@@ -102,6 +112,7 @@ CodeGen_GPU_Host<CodeGen_CPU>::CodeGen_GPU_Host(Target target)
     // OpenCL, CUDA, OpenGLCompute, and OpenGL last.
     // The code is in reverse order to allow later tests to override
     // earlier ones.
+#ifdef WITH_OPENGL
     if (target.has_feature(Target::OpenGL)) {
         debug(1) << "Constructing OpenGL device codegen\n";
         cgdev[DeviceAPI::GLSL] = new CodeGen_OpenGL_Dev(target);
@@ -110,22 +121,29 @@ CodeGen_GPU_Host<CodeGen_CPU>::CodeGen_GPU_Host(Target target)
         debug(1) << "Constructing OpenGL Compute device codegen\n";
         cgdev[DeviceAPI::OpenGLCompute] = new CodeGen_OpenGLCompute_Dev(target);
     }
+#endif
     if (target.has_feature(Target::CUDA)) {
         debug(1) << "Constructing CUDA device codegen\n";
         cgdev[DeviceAPI::CUDA] = new CodeGen_PTX_Dev(target);
     }
+#ifdef WITH_OPENCL
     if (target.has_feature(Target::OpenCL)) {
         debug(1) << "Constructing OpenCL device codegen\n";
         cgdev[DeviceAPI::OpenCL] = new CodeGen_OpenCL_Dev(target);
     }
+#endif
+#ifdef WITH_METAL
     if (target.has_feature(Target::Metal)) {
         debug(1) << "Constructing Metal device codegen\n";
         cgdev[DeviceAPI::Metal] = new CodeGen_Metal_Dev(target);
     }
+#endif // WITH_METAL
+#ifdef WITH_D3D12
     if (target.has_feature(Target::D3D12Compute)) {
         debug(1) << "Constructing Direct3D 12 Compute device codegen\n";
         cgdev[DeviceAPI::D3D12Compute] = new CodeGen_D3D12Compute_Dev(target);
     }
+#endif // WITH_D3D12
 
     if (cgdev.empty()) {
         internal_error << "Requested unknown GPU target: " << target.to_string() << "\n";
diff --git a/src/CodeGen_LLVM.cpp b/src/CodeGen_LLVM.cpp
index 2fc6f7ab338b..4fde223b09d4 100644
--- a/src/CodeGen_LLVM.cpp
+++ b/src/CodeGen_LLVM.cpp
@@ -5,16 +5,30 @@
 
 #include "CPlusPlusMangle.h"
 #include "CSE.h"
-#include "CodeGen_ARM.h"
+#if defined(WITH_ARM) || defined(WITH_AARCH64)
+    #include "CodeGen_ARM.h"
+#endif
 #include "CodeGen_GPU_Host.h"
-#include "CodeGen_Hexagon.h"
+#ifdef WITH_HEXAGON
+    #include "CodeGen_Hexagon.h"
+#endif
 #include "CodeGen_Internal.h"
 #include "CodeGen_LLVM.h"
-#include "CodeGen_MIPS.h"
-#include "CodeGen_PowerPC.h"
-#include "CodeGen_RISCV.h"
-#include "CodeGen_WebAssembly.h"
-#include "CodeGen_X86.h"
+#ifdef WITH_MIPS
+    #include "CodeGen_MIPS.h"
+#endif
+#ifdef WITH_POWERPC
+    #include "CodeGen_PowerPC.h"
+#endif
+#ifdef WITH_RISCV
+    #include "CodeGen_RISCV.h"
+#endif
+#ifdef WITH_WEBASSEMBLY
+    #include "CodeGen_WebAssembly.h"
+#endif
+#ifdef WITH_X86
+    #include "CodeGen_X86.h"
+#endif
 #include "CompilerLogger.h"
 #include "Debug.h"
 #include "Deinterleave.h"
@@ -352,20 +366,34 @@ CodeGen_LLVM *CodeGen_LLVM::new_for_target(const Target &target,
                    << target.to_string() << "\n";
         return nullptr;
 
+#ifdef WITH_X86
     } else if (target.arch == Target::X86) {
         return make_codegen<CodeGen_X86>(target, context);
+#endif
+#ifdef WITH_ARM
     } else if (target.arch == Target::ARM) {
         return make_codegen<CodeGen_ARM>(target, context);
+#endif
+#ifdef WITH_MIPS
     } else if (target.arch == Target::MIPS) {
         return make_codegen<CodeGen_MIPS>(target, context);
+#endif
+#ifdef WITH_POWERPC
     } else if (target.arch == Target::POWERPC) {
         return make_codegen<CodeGen_PowerPC>(target, context);
+#endif
+#ifdef WITH_HEXAGON
     } else if (target.arch == Target::Hexagon) {
         return make_codegen<CodeGen_Hexagon>(target, context);
+#endif
+#ifdef WITH_WEBASSEMBLY
     } else if (target.arch == Target::WebAssembly) {
         return make_codegen<CodeGen_WebAssembly>(target, context);
+#endif
+#ifdef WITH_RISCV
     } else if (target.arch == Target::RISCV) {
         return make_codegen<CodeGen_RISCV>(target, context);
+#endif
     }
 
     user_error << "Unknown target architecture: "
diff --git a/src/Lower.cpp b/src/Lower.cpp
index a25d18de30b3..e0d3aef3de0f 100644
--- a/src/Lower.cpp
+++ b/src/Lower.cpp
@@ -448,6 +448,7 @@ Module lower(const vector<Function> &output_funcs,
     debug(1) << "Lowering after final simplification:\n"
              << s << "\n\n";
 
+#ifdef WITH_HEXAGON
     if (t.arch != Target::Hexagon && t.has_feature(Target::HVX)) {
         debug(1) << "Splitting off Hexagon offload...\n";
         s = inject_hexagon_rpc(s, t, result_module);
@@ -456,6 +457,7 @@ Module lower(const vector<Function> &output_funcs,
     } else {
         debug(1) << "Skipping Hexagon offload...\n";
     }
+#endif //WITH_HEXAGON
 
     if (!custom_passes.empty()) {
         for (size_t i = 0; i < custom_passes.size(); i++) {
diff --git a/src/Module.cpp b/src/Module.cpp
index 74a3714e4abd..5dc25c3a7108 100644
--- a/src/Module.cpp
+++ b/src/Module.cpp
@@ -472,12 +472,14 @@ Module link_modules(const std::string &name, const std::vector<Module> &modules)
 }
 
 Buffer<uint8_t> Module::compile_to_buffer() const {
+#ifdef WITH_HEXAGON
     // TODO: This Hexagon specific code should be removed as soon as possible.
     // This may involve adding more general support for post-processing and
     // a way of specifying to use it.
     if (target().arch == Target::Hexagon) {
         return compile_module_to_hexagon_shared_object(*this);
     }
+#endif // WITH_HEXAGON
 
     llvm::LLVMContext context;
     std::unique_ptr<llvm::Module> llvm_module(compile_module_to_llvm_module(*this, context));