diff --git a/Makefile b/Makefile
index edd0cdc7078d..8bb3f80d4e38 100644
--- a/Makefile
+++ b/Makefile
@@ -271,6 +271,9 @@ TEST_CXX_FLAGS += -DLLVM_VERSION=$(LLVM_VERSION_TIMES_10)
 # In the tests, default to exporting no symbols that aren't explicitly exported
 TEST_CXX_FLAGS += -fvisibility=hidden -fvisibility-inlines-hidden
 
+# In the tests, enable the debug() and internal_assert() macros
+TEST_CXX_FLAGS += -DHALIDE_KEEP_MACROS
+
 # gcc 4.8 fires a bogus warning on old versions of png.h
 ifneq (,$(findstring g++,$(CXX_VERSION)))
 ifneq (,$(findstring 4.8,$(CXX_VERSION)))
diff --git a/cmake/HalideTestHelpers.cmake b/cmake/HalideTestHelpers.cmake
index 1bbe0bcb17e5..3e5c64f47f28 100644
--- a/cmake/HalideTestHelpers.cmake
+++ b/cmake/HalideTestHelpers.cmake
@@ -18,6 +18,9 @@ if (NOT TARGET Halide::Test)
     # Obviously link to libHalide, but also grant all tests access to the threads library.
     target_link_libraries(Halide_test INTERFACE Halide::Halide Threads::Threads)
 
+    # Make internal_assert, debug, etc. available to tests
+    target_compile_definitions(Halide_test INTERFACE HALIDE_KEEP_MACROS)
+
     # Everyone gets to see the common headers
     target_include_directories(Halide_test
                                INTERFACE
diff --git a/src/AddImageChecks.cpp b/src/AddImageChecks.cpp
index 6a1f66ae332b..537f7e8827dc 100644
--- a/src/AddImageChecks.cpp
+++ b/src/AddImageChecks.cpp
@@ -359,15 +359,17 @@ Stmt add_image_checks_inner(Stmt s,
         }
 
         // Check that the region passed in (after applying constraints) is within the region used
-        if (debug::debug_level() >= 3) {
-            debug(3) << "In image " << name << " region touched is:\n";
+        debug(3) << [&] {
+            std::stringstream ss;
+            ss << "In image " << name << " region touched is:\n";
             for (int j = 0; j < dimensions; j++) {
-                debug(3) << "  " << j << ": " << (touched.empty() ? Expr() : touched[j].min)
-                         << " .. "
-                         << (touched.empty() ? Expr() : touched[j].max)
-                         << "\n";
+                ss << "  " << j << ": " << (touched.empty() ? Expr() : touched[j].min)
+                   << " .. "
+                   << (touched.empty() ? Expr() : touched[j].max)
+                   << "\n";
             }
-        }
+            return ss.str();
+        }();
 
         for (int j = 0; j < dimensions; j++) {
             string dim = std::to_string(j);
diff --git a/src/CodeGen_LLVM.cpp b/src/CodeGen_LLVM.cpp
index bed6af28df36..c3c88a333a19 100644
--- a/src/CodeGen_LLVM.cpp
+++ b/src/CodeGen_LLVM.cpp
@@ -1094,9 +1094,10 @@ void CodeGen_LLVM::optimize_module() {
 
     auto time_start = std::chrono::high_resolution_clock::now();
 
-    if (debug::debug_level() >= 3) {
+    debug(3) << [&] {
         module->print(dbgs(), nullptr, false, true);
-    }
+        return "";
+    }();
 
     std::unique_ptr<TargetMachine> tm = make_target_machine(*module);
 
@@ -1239,9 +1240,10 @@ void CodeGen_LLVM::optimize_module() {
     }
 
     debug(3) << "After LLVM optimizations:\n";
-    if (debug::debug_level() >= 2) {
+    debug(2) << [&] {
         module->print(dbgs(), nullptr, false, true);
-    }
+        return "";
+    }();
 
     auto *logger = get_compiler_logger();
     if (logger) {
@@ -1264,23 +1266,15 @@ void CodeGen_LLVM::sym_pop(const string &name) {
 
 llvm::Value *CodeGen_LLVM::sym_get(const string &name, bool must_succeed) const {
     // look in the symbol table
-    llvm::Value *const *v = symbol_table.find(name);
-    if (!v) {
-        if (must_succeed) {
-            std::ostringstream err;
-            err << "Symbol not found: " << name << "\n";
-
-            if (debug::debug_level() > 0) {
-                err << "The following names are in scope:\n"
-                    << symbol_table << "\n";
-            }
-
-            internal_error << err.str();
-        } else {
-            return nullptr;
-        }
+    if (const auto *v = symbol_table.find(name)) {
+        return *v;
     }
-    return *v;
+    if (must_succeed) {
+        debug(1) << "The following names are in scope:\n"
+                 << symbol_table;
+        internal_error << "Symbol not found: " << name;
+    }
+    return nullptr;
 }
 
 bool CodeGen_LLVM::sym_exists(const string &name) const {
@@ -1318,12 +1312,14 @@ Value *CodeGen_LLVM::codegen(const Expr &e) {
                        e.type().is_handle() ||
                        value->getType()->isVoidTy() ||
                        value->getType() == llvm_type_of(e.type());
-    if (!types_match && debug::debug_level() > 0) {
-        debug(1) << "Unexpected LLVM type for generated expression. Expected (llvm_type_of(e.type())): ";
-        llvm_type_of(e.type())->print(dbgs(), true);
-        debug(1) << " got (value->getType()): ";
-        value->print(dbgs(), true);
-        debug(1) << "\n";
+    if (!types_match) {
+        debug(1) << [&] {
+            std::cerr << "Unexpected LLVM type for generated expression. Expected (llvm_type_of(e.type())): ";
+            llvm_type_of(e.type())->print(dbgs(), true);
+            std::cerr << " got (value->getType()): ";
+            value->print(dbgs(), true);
+            return "\n";
+        }();
     }
     internal_assert(types_match)
         << "Codegen of Expr " << e
diff --git a/src/CodeGen_PTX_Dev.cpp b/src/CodeGen_PTX_Dev.cpp
index 0fce9ef93fad..14e5234d2038 100644
--- a/src/CodeGen_PTX_Dev.cpp
+++ b/src/CodeGen_PTX_Dev.cpp
@@ -730,10 +730,10 @@ vector<char> CodeGen_PTX_Dev::compile_to_src() {
     module_pass_manager.run(*module);
 
     // Codegen pipeline completed.
-    if (debug::debug_level() >= 2) {
+    debug(2) << [&] {
         dump();
-    }
-    debug(2) << "Done with CodeGen_PTX_Dev::compile_to_src";
+        return "Done with CodeGen_PTX_Dev::compile_to_src";
+    }();
 
     debug(1) << "PTX kernel:\n"
              << outstr.c_str() << "\n";
@@ -741,9 +741,8 @@ vector<char> CodeGen_PTX_Dev::compile_to_src() {
     vector<char> buffer(outstr.begin(), outstr.end());
 
     // Dump the SASS too if the cuda SDK is in the path
-    if (debug::debug_level() >= 2) {
-        debug(2) << "Compiling PTX to SASS. Will fail if CUDA SDK is not installed (and in the path).\n";
-
+    debug(2) << "Compiling PTX to SASS. Will fail if CUDA SDK is not installed (and in the path).\n";
+    debug(2) << [&] {
         TemporaryFile ptx(get_current_kernel_name(), ".ptx");
         TemporaryFile sass(get_current_kernel_name(), ".sass");
 
@@ -772,7 +771,8 @@ vector<char> CodeGen_PTX_Dev::compile_to_src() {
             f.read(buffer.data(), sz);
         }
         */
-    }
+        return "";
+    }();
 
     // Null-terminate the ptx source
     buffer.push_back(0);
diff --git a/src/CodeGen_Vulkan_Dev.cpp b/src/CodeGen_Vulkan_Dev.cpp
index f2a80bb696ff..0f378b29182a 100644
--- a/src/CodeGen_Vulkan_Dev.cpp
+++ b/src/CodeGen_Vulkan_Dev.cpp
@@ -2866,9 +2866,10 @@ void CodeGen_Vulkan_Dev::add_kernel(Stmt stmt,
     emitter.encode_spirv_module(spirv_module);
 
     // Dump the SPIR-V if debug is enabled
-    if (debug::debug_level() >= 2) {
+    debug(2) << [&] {
         emitter.dump_spirv_module();
-    }
+        return "";
+    }();
 
     // Copy the SPIR-V module into the Kernel Module table
     KernelModule kernel_module;
@@ -3059,4 +3060,4 @@ std::unique_ptr<CodeGen_GPU_Dev> new_CodeGen_Vulkan_Dev(const Target &target) {
 }  // namespace Internal
 }  // namespace Halide
 
-#endif  // WITH_SPIRV
\ No newline at end of file
+#endif  // WITH_SPIRV
diff --git a/src/Debug.cpp b/src/Debug.cpp
index 22d26e83cc31..40f0c4c73097 100644
--- a/src/Debug.cpp
+++ b/src/Debug.cpp
@@ -1,16 +1,117 @@
 #include "Debug.h"
+#include "Error.h"
 #include "Util.h"
 
-namespace Halide {
-namespace Internal {
+#include <algorithm>
+#include <climits>
+#include <optional>
 
-int debug::debug_level() {
-    static int cached_debug_level = ([]() -> int {
-        std::string lvl = get_env_variable("HL_DEBUG_CODEGEN");
-        return !lvl.empty() ? atoi(lvl.c_str()) : 0;
-    })();
-    return cached_debug_level;
+namespace Halide::Internal {
+
+namespace {
+
+std::string read_until(const char *&str, const char *delims) {
+    const char *start = str;
+    for (; *str; ++str) {
+        for (const char *ch = delims; *ch; ++ch) {
+            if (*str == *ch) {
+                return {start, str};
+            }
+        }
+    }
+    return {start, str};
+}
+
+bool parse_int(const std::string &number, int &value) {
+    const char *start = number.c_str();
+    char *end;
+    value = static_cast<int>(strtol(start, &end, 10));
+    return start < end && *end == '\0';
+}
+
+class DebugRule {
+    int verbosity = 0;
+    std::string file_suffix = "";
+    int line_low = -1;
+    int line_high = INT_MAX;
+    std::string function_suffix = "";
+    enum Complexity { VerbosityOnly,
+                      NeedsMatching } complexity = VerbosityOnly;
+
+public:
+    static std::optional<DebugRule> parse(const std::string &spec) {
+        DebugRule rule;
+        const char *ptr = spec.c_str();
+
+        if (!parse_int(read_until(ptr, ",@"), rule.verbosity)) {
+            return std::nullopt;
+        }
+
+        if (*ptr == '\0') {
+            return rule;
+        }
+
+        if (*ptr == ',') {
+            rule.file_suffix = read_until(++ptr, ":@");
+            if (*ptr == ':') {
+                if (!parse_int(read_until(++ptr, "-@"), rule.line_low)) {
+                    return std::nullopt;
+                }
+                rule.line_high = rule.line_low;
+                if (*ptr == '-') {
+                    if (!parse_int(read_until(++ptr, "@"), rule.line_high)) {
+                        return std::nullopt;
+                    }
+                }
+            }
+        }
+
+        if (*ptr == '@') {
+            rule.function_suffix = std::string{ptr + 1};
+        }
+
+        rule.complexity = NeedsMatching;
+        return rule;
+    }
+
+    bool accepts(const int verbosity, const char *file, const char *function,
+                 const int line) const {
+        switch (complexity) {
+        case VerbosityOnly:
+            return verbosity <= this->verbosity;
+        case NeedsMatching:
+            return verbosity <= this->verbosity &&
+                   ends_with(file, file_suffix) &&
+                   ends_with(function, function_suffix) &&
+                   line_low <= line && line <= line_high;
+        }
+        return false;
+    }
+};
+
+std::vector<DebugRule> parse_rules(const std::string &env) {
+    std::vector<DebugRule> rules;
+    for (const std::string &spec : split_string(env, ";")) {
+        if (auto rule = DebugRule::parse(spec)) {
+            rules.push_back(*rule);
+        } else if (!spec.empty()) {
+            user_warning
+                << "Ignoring malformed HL_DEBUG_CODEGEN entry: [" << spec << "]\n"
+                << "The expected format is:\n    "
+                << "verbosity[,filename[:line_low[-line_high]]][@func]";
+        }
+    }
+    return rules;
+}
+
+}  // namespace
+
+bool debug_is_active_impl(const int verbosity, const char *file, const char *function,
+                          const int line) {
+    static const std::vector<DebugRule> rules = parse_rules(get_env_variable("HL_DEBUG_CODEGEN"));
+    return std::any_of(rules.begin(), rules.end(), [&](const auto &rule) {
+        return rule.accepts(verbosity, file, function, line);
+    });
 }
 
-}  // namespace Internal
-}  // namespace Halide
+}  // namespace Halide::Internal
diff --git a/src/Debug.h b/src/Debug.h
index 432ba07dc115..4e2e88bf6fbd 100644
--- a/src/Debug.h
+++ b/src/Debug.h
@@ -32,12 +32,15 @@ std::ostream &operator<<(std::ostream &stream, const Stmt &);
 struct LoweredFunc;
 std::ostream &operator<<(std::ostream &, const LoweredFunc &);
 
-/** For optional debugging during codegen, use the debug class as
+bool debug_is_active_impl(int verbosity, const char *file, const char *function, int line);
+#define debug_is_active(n) (::Halide::Internal::debug_is_active_impl((n), __FILE__, __FUNCTION__, __LINE__))
+
+/** For optional debugging during codegen, use the debug macro as
  * follows:
  *
- \code
- debug(verbosity) << "The expression is " << expr << "\n";
- \endcode
+ * \code
+ * debug(verbosity) << "The expression is " << expr << "\n";
+ * \endcode
  *
  * verbosity of 0 always prints, 1 should print after every major
  * stage, 2 should be used for more detail, and 3 should be used for
@@ -45,25 +48,9 @@ std::ostream &operator<<(std::ostream &, const LoweredFunc &);
  * is determined by the value of the environment variable
  * HL_DEBUG_CODEGEN
  */
-
-class debug {
-    const bool logging;
-
-public:
-    debug(int verbosity)
-        : logging(verbosity <= debug_level()) {
-    }
-
-    template<typename T>
-    debug &operator<<(T &&x) {
-        if (logging) {
-            std::cerr << std::forward<T>(x);
-        }
-        return *this;
-    }
-
-    static int debug_level();
-};
+#define debug(n)                                     \
+    /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
+    (!debug_is_active((n))) ? (void)0 : ::Halide::Internal::Voidifier() & std::cerr
 
 /** Allow easily printing the contents of containers, or std::vector-like containers,
  *  in debug output. Used like so:
diff --git a/src/Derivative.cpp b/src/Derivative.cpp
index 05d3168c95d3..a7e9ade253fe 100644
--- a/src/Derivative.cpp
+++ b/src/Derivative.cpp
@@ -191,9 +191,9 @@ void ReverseAccumulationVisitor::propagate_adjoints(
         realization_order({output.function()}, env).first;
     vector<Func> funcs;
     funcs.reserve(order.size());
-    // Internal::debug(0) << "Sorted Func list:\n";
+    // debug(0) << "Sorted Func list:\n";
     // for (const auto &func_name : order) {
-    //     Internal::debug(0) << "  . " << func_name << "\n";
+    //     debug(0) << "  . " << func_name << "\n";
     // }
     for (const auto &func_name : order) {
         funcs.emplace_back(env[func_name]);
@@ -1919,7 +1919,7 @@ void ReverseAccumulationVisitor::propagate_halide_function_call(
 Func Derivative::operator()(const Func &func, int update_id) const {
     auto it = adjoints.find(FuncKey{func.name(), update_id});
     if (it == adjoints.end()) {
-        Internal::debug(1) << "Could not find Func " << func.name() << "\n";
+        debug(1) << "Could not find Func " << func.name() << "\n";
         return Func();
     }
     return it->second;
@@ -1928,7 +1928,7 @@ Func Derivative::operator()(const Func &func, int update_id) const {
 Func Derivative::operator()(const Buffer<> &buffer) const {
     auto it = adjoints.find(FuncKey{buffer.name(), -1});
     if (it == adjoints.end()) {
-        Internal::debug(1) << "Could not find Buffer " << buffer.name() << "\n";
+        debug(1) << "Could not find Buffer " << buffer.name() << "\n";
         return Func();
     }
     return it->second;
@@ -1937,7 +1937,7 @@ Func Derivative::operator()(const Buffer<> &buffer) const {
 Func Derivative::operator()(const Param<> &param) const {
     auto it = adjoints.find(FuncKey{param.name(), -1});
     if (it == adjoints.end()) {
-        Internal::debug(1) << "Could not find Param " << param.name() << "\n";
+        debug(1) << "Could not find Param " << param.name() << "\n";
         return Func();
     }
     return it->second;
@@ -1946,7 +1946,7 @@ Func Derivative::operator()(const Param<> &param) const {
 Func Derivative::operator()(const std::string &name) const {
     auto it = adjoints.find(FuncKey{name, -1});
     if (it == adjoints.end()) {
-        Internal::debug(1) << "Could not find name: " << name << "\n";
+        debug(1) << "Could not find name: " << name << "\n";
         return Func();
     }
     return it->second;
diff --git a/src/DeviceArgument.cpp b/src/DeviceArgument.cpp
index 104538611a65..e020fceb5a17 100644
--- a/src/DeviceArgument.cpp
+++ b/src/DeviceArgument.cpp
@@ -6,9 +6,7 @@ namespace Halide {
 namespace Internal {
 
 std::vector<DeviceArgument> HostClosure::arguments() {
-    if (debug::debug_level() >= 2) {
-        debug(2) << *this;
-    }
+    debug(2) << *this;
 
     std::vector<DeviceArgument> res;
     for (const auto &v : vars) {
diff --git a/src/Error.h b/src/Error.h
index 7644fe8312cd..7c1fa7412356 100644
--- a/src/Error.h
+++ b/src/Error.h
@@ -125,9 +125,8 @@ template<typename T>
 struct ReportBase {
     std::ostringstream msg;
 
-    ReportBase(const char *file, int line, const char *condition_string, const char *prefix) {
-        // Only mention where inside libHalide the error tripped if we have debug level > 0
-        if (debug::debug_level() > 0) {
+    ReportBase(const char *file, const char *function, int line, const char *condition_string, const char *prefix) {
+        if (debug_is_active_impl(1, file, function, line)) {
             msg << prefix << " at " << file << ":" << line << ' ';
             if (condition_string) {
                 msg << "Condition failed: " << condition_string << ' ';
@@ -159,8 +158,8 @@ template<typename Exception>
 struct ErrorReport : ReportBase<ErrorReport<Exception>> {
     using Base = ReportBase<ErrorReport>;
 
-    ErrorReport(const char *file, int line, const char *condition_string)
-        : Base(file, line, condition_string, Exception::error_name) {
+    ErrorReport(const char *file, const char *function, int line, const char *condition_string)
+        : Base(file, function, line, condition_string, Exception::error_name) {
         this->msg << "Error: ";
     }
 
@@ -178,8 +177,8 @@ struct ErrorReport : ReportBase<ErrorReport<Exception>> {
 };
 
 struct WarningReport : ReportBase<WarningReport> {
-    WarningReport(const char *file, int line, const char *condition_string)
-        : ReportBase(file, line, condition_string, "Warning") {
+    WarningReport(const char *file, const char *function, int line, const char *condition_string)
+        : ReportBase(file, function, line, condition_string, "Warning") {
         this->msg << "Warning: ";
     }
 
@@ -219,12 +218,12 @@ struct Voidifier {
  */
 #define _halide_internal_assertion(condition, type)  \
     /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
-    (condition) ? (void)0 : ::Halide::Internal::Voidifier() & ::Halide::Internal::ErrorReport<type>(__FILE__, __LINE__, #condition).ref()
+    (condition) ? (void)0 : ::Halide::Internal::Voidifier() & ::Halide::Internal::ErrorReport<type>(__FILE__, __FUNCTION__, __LINE__, #condition).ref()
 
-#define internal_error Halide::Internal::ErrorReport<Halide::InternalError>(__FILE__, __LINE__, nullptr)
-#define user_error Halide::Internal::ErrorReport<Halide::CompileError>(__FILE__, __LINE__, nullptr)
-#define user_warning Halide::Internal::WarningReport(__FILE__, __LINE__, nullptr)
-#define halide_runtime_error Halide::Internal::ErrorReport<Halide::RuntimeError>(__FILE__, __LINE__, nullptr)
+#define internal_error Halide::Internal::ErrorReport<Halide::InternalError>(__FILE__, __FUNCTION__, __LINE__, nullptr)
+#define user_error Halide::Internal::ErrorReport<Halide::CompileError>(__FILE__, __FUNCTION__, __LINE__, nullptr)
+#define user_warning Halide::Internal::WarningReport(__FILE__, __FUNCTION__, __LINE__, nullptr)
+#define halide_runtime_error Halide::Internal::ErrorReport<Halide::RuntimeError>(__FILE__, __FUNCTION__, __LINE__, nullptr)
 
 #define internal_assert(c) _halide_internal_assertion(c, Halide::InternalError)
 #define user_assert(c) _halide_internal_assertion(c, Halide::CompileError)
diff --git a/src/Func.cpp b/src/Func.cpp
index a9aef3ca8d82..da714f7f7b28 100644
--- a/src/Func.cpp
+++ b/src/Func.cpp
@@ -271,7 +271,7 @@ std::pair<int, int> Func::add_implicit_vars(vector<Var> &args) const {
         // It's important to use func.dimensions() here, *not* this->dimensions(),
         // since the latter can return the Func's required dimensions rather than its actual dimensions.
         while ((int)args.size() < func.dimensions()) {
-            Internal::debug(2) << "Adding implicit var " << i << " to call to " << name() << "\n";
+            debug(2) << "Adding implicit var " << i << " to call to " << name() << "\n";
             iter = args.insert(iter, Var::implicit(i++));
             iter++;
             count++;
@@ -304,7 +304,7 @@ std::pair<int, int> Func::add_implicit_vars(vector<Expr> &args) const {
         // It's important to use func.dimensions() here, *not* this->dimensions(),
         // since the latter can return the Func's required dimensions rather than its actual dimensions.
         while ((int)args.size() < func.dimensions()) {
-            Internal::debug(2) << "Adding implicit var " << i << " to call to " << name() << "\n";
+            debug(2) << "Adding implicit var " << i << " to call to " << name() << "\n";
             iter = args.insert(iter, Var::implicit(i++));
             iter++;
             count++;
@@ -3082,7 +3082,7 @@ vector<Expr> FuncRef::args_with_implicit_vars(const vector<Expr> &exprs) const {
             internal_assert(implicit_count == 0)
                 << "Pure definition can't possibly already have implicit variables defined\n";
 
-            Internal::debug(2) << "Adding " << count.count << " implicit vars to LHS of " << func.name() << "\n";
+            debug(2) << "Adding " << count.count << " implicit vars to LHS of " << func.name() << "\n";
 
             vector<Expr>::iterator iter = result.begin() + implicit_placeholder_pos;
             for (int i = 0; i < count.count; i++) {
diff --git a/src/HexagonOffload.cpp b/src/HexagonOffload.cpp
index 2d7d5e74acf7..72549deb6ff2 100644
--- a/src/HexagonOffload.cpp
+++ b/src/HexagonOffload.cpp
@@ -1026,14 +1026,15 @@ Buffer<uint8_t> compile_module_to_hexagon_shared_object(const Module &device_cod
     compile_llvm_module_to_object(*llvm_module, object_stream);
 
     int min_debug_level = device_code.name() == runtime_module_name ? 3 : 2;
-    if (debug::debug_level() >= min_debug_level) {
-        debug(0) << "Hexagon device code assembly: "
-                 << "\n";
+    debug(min_debug_level) << [&] {
+        std::stringstream ss;
+        ss << "Hexagon device code assembly: \n";
         llvm::SmallString<4096> assembly;
         llvm::raw_svector_ostream assembly_stream(assembly);
         compile_llvm_module_to_assembly(*llvm_module, assembly_stream);
-        debug(0) << assembly.c_str() << "\n";
-    }
+        ss << assembly.c_str() << "\n";
+        return ss.str();
+    }();
 
     auto obj = Elf::Object::parse_object(object.data(), object.size());
     internal_assert(obj);
diff --git a/src/LLVM_Output.cpp b/src/LLVM_Output.cpp
index e4b94a5234b0..13bfacdb3093 100644
--- a/src/LLVM_Output.cpp
+++ b/src/LLVM_Output.cpp
@@ -328,7 +328,7 @@ namespace {
 // llvm::CloneModule has issues with debug info. As a workaround,
 // serialize it to bitcode in memory, and then parse the bitcode back in.
 std::unique_ptr<llvm::Module> clone_module(const llvm::Module &module_in) {
-    Internal::debug(2) << "Cloning module " << module_in.getName().str() << "\n";
+    debug(2) << "Cloning module " << module_in.getName().str() << "\n";
 
     // Write the module to a buffer.
     llvm::SmallVector<char, 16> clone_buffer;
@@ -351,11 +351,11 @@ std::unique_ptr<llvm::Module> clone_module(const llvm::Module &module_in) {
 
 void emit_file(const llvm::Module &module_in, Internal::LLVMOStream &out,
                llvm::CodeGenFileType file_type) {
-    Internal::debug(1) << "emit_file.Compiling to native code...\n";
+    debug(1) << "emit_file.Compiling to native code...\n";
 #if LLVM_VERSION >= 210
-    Internal::debug(2) << "Target triple: " << module_in.getTargetTriple().str() << "\n";
+    debug(2) << "Target triple: " << module_in.getTargetTriple().str() << "\n";
 #else
-    Internal::debug(2) << "Target triple: " << module_in.getTargetTriple() << "\n";
+    debug(2) << "Target triple: " << module_in.getTargetTriple() << "\n";
 #endif
 
     auto time_start = std::chrono::high_resolution_clock::now();
@@ -394,7 +394,7 @@ void emit_file(const llvm::Module &module_in, Internal::LLVMOStream &out,
     pass_manager.add(llvm::createAlwaysInlinerLegacyPass());
 
     if (target_machine->isPositionIndependent()) {
-        Internal::debug(1) << "Target machine is Position Independent!\n";
+        debug(1) << "Target machine is Position Independent!\n";
     }
 
     // Override default to generate verbose assembly.
diff --git a/src/LowerParallelTasks.cpp b/src/LowerParallelTasks.cpp
index 70f47885528c..d6ed27ca0905 100644
--- a/src/LowerParallelTasks.cpp
+++ b/src/LowerParallelTasks.cpp
@@ -427,12 +427,14 @@ Stmt lower_parallel_tasks(const Stmt &s, std::vector<LoweredFunc> &closure_imple
     Stmt result = lowering_mutator.mutate(s);
 
     // Main body will be dumped as part of standard lowering debugging, but closures will not be.
-    if (debug::debug_level() >= 2) {
+    debug(2) << [&] {
+        std::stringstream ss;
         for (const auto &lf : lowering_mutator.closure_implementations) {
-            debug(2) << "lower_parallel_tasks generated closure lowered function " << lf.name << ":\n"
-                     << lf.body << "\n\n";
+            ss << "lower_parallel_tasks generated closure lowered function " << lf.name << ":\n"
+               << lf.body << "\n\n";
         }
-    }
+        return ss.str();
+    }();
 
     // Append to the end rather than replacing the list entirely.
     closure_implementations.insert(closure_implementations.end(),
diff --git a/src/Module.cpp b/src/Module.cpp
index 5bece0d7ebdd..7b6dbd0cc487 100644
--- a/src/Module.cpp
+++ b/src/Module.cpp
@@ -499,14 +499,15 @@ Buffer<uint8_t> Module::compile_to_buffer() const {
     llvm::raw_svector_ostream object_stream(object);
     compile_llvm_module_to_object(*llvm_module, object_stream);
 
-    if (debug::debug_level() >= 2) {
-        debug(2) << "Submodule assembly for " << name() << ": "
-                 << "\n";
+    debug(2) << [&] {
+        std::stringstream ss;
+        ss << "Submodule assembly for " << name() << ":\n";
         llvm::SmallString<4096> assembly;
         llvm::raw_svector_ostream assembly_stream(assembly);
         compile_llvm_module_to_assembly(*llvm_module, assembly_stream);
-        debug(2) << assembly.c_str() << "\n";
-    }
+        ss << assembly.c_str() << "\n";
+        return ss.str();
+    }();
 
     Buffer<uint8_t> result(object.size(), name());
     memcpy(result.data(), reinterpret_cast<uint8_t *>(&object[0]), object.size());
diff --git a/src/ParallelRVar.cpp b/src/ParallelRVar.cpp
index a79b9eac56ba..15ff78c0ed72 100644
--- a/src/ParallelRVar.cpp
+++ b/src/ParallelRVar.cpp
@@ -146,7 +146,7 @@ bool can_parallelize_rvar(const string &v,
 
     // Add the definition's predicate if there is any
     if (pred.defined() || !is_const_one(pred)) {
-        Expr this_pred = pred;
+        const Expr &this_pred = pred;
         Expr other_pred = renamer.mutate(pred);
         debug(3) << "......this thread predicate: " << this_pred << "\n";
         debug(3) << "......other thread predicate: " << other_pred << "\n";
diff --git a/src/Pipeline.cpp b/src/Pipeline.cpp
index 9109c0dd4ac1..397ce9ba324c 100644
--- a/src/Pipeline.cpp
+++ b/src/Pipeline.cpp
@@ -701,11 +701,13 @@ Callable Pipeline::compile_to_callable(const std::vector<Argument> &args_in, con
         for (const LoweredFunc &f : module.functions()) {
             f.body.accept(&find_externs);
         }
-        if (debug::debug_level() >= 1) {
+        debug(1) << [&] {
+            std::stringstream ss;
             for (const auto &p : jit_externs) {
-                debug(1) << "Found extern: " << p.first << "\n";
+                ss << "Found extern: " << p.first << "\n";
             }
-        }
+            return ss.str();
+        }();
 
         wasm_module = WasmModule::compile(module, args,
                                           module.name(), jit_externs, externs_jit_module);
@@ -1140,10 +1142,10 @@ void Pipeline::infer_input_bounds(JITUserContext *context,
             tb.orig = tb.query;
         }
 
-        Internal::debug(2) << "Calling jitted function\n";
+        debug(2) << "Calling jitted function\n";
         int exit_status = call_jit_code(args);
         jit_context.finalize(exit_status);
-        Internal::debug(2) << "Back from jitted function\n";
+        debug(2) << "Back from jitted function\n";
         bool changed = false;
 
         // Check if there were any changes
diff --git a/src/ScheduleFunctions.cpp b/src/ScheduleFunctions.cpp
index 02f5553f748c..72fe1193466c 100644
--- a/src/ScheduleFunctions.cpp
+++ b/src/ScheduleFunctions.cpp
@@ -32,6 +32,18 @@ using std::vector;
 
 namespace {
 
+std::ostream &operator<<(std::ostream &out, const std::vector<Function> &v) {
+    out << "{ ";
+    for (size_t i = 0; i < v.size(); ++i) {
+        out << v[i].name();
+        if (i != v.size() - 1) {
+            out << ", ";
+        }
+    }
+    out << " }";
+    return out;
+}
+
 // A structure representing a containing LetStmt, IfThenElse, or For
 // loop. Used in build_provide_loop_nest below. Both If and IfInner represent
 // IfThenElse stmts, however, IfInner should not be reordered to outside of
@@ -2560,20 +2572,6 @@ bool group_should_be_inlined(const vector<Function> &funcs) {
 
 }  // namespace
 
-// We want this to have internal linkage, but putting it in an anonymous
-// namespace doesn't work due to two-phase lookup peculiarities.
-static std::ostream &operator<<(std::ostream &out, const std::vector<Function> &v) {  // NOLINT
-    out << "{ ";
-    for (size_t i = 0; i < v.size(); ++i) {
-        out << v[i].name();
-        if (i != v.size() - 1) {
-            out << ", ";
-        }
-    }
-    out << " }";
-    return out;
-}
-
 Stmt schedule_functions(const vector<Function> &outputs,
                         const vector<vector<string>> &fused_groups,
                         const map<string, Function> &env,
diff --git a/src/Simplify.cpp b/src/Simplify.cpp
index 494ca5665f91..bf5518cc3ec8 100644
--- a/src/Simplify.cpp
+++ b/src/Simplify.cpp
@@ -432,7 +432,7 @@ bool can_prove(Expr e, const Scope<Interval> &bounds) {
 
     // Take a closer look at all failed proof attempts to hunt for
     // simplifier weaknesses
-    const bool check_failed_proofs = debug::debug_level() > 0 || get_compiler_logger() != nullptr;
+    const bool check_failed_proofs = debug_is_active(1) || get_compiler_logger() != nullptr;
     if (check_failed_proofs && !is_const(e)) {
         struct RenameVariables : public IRMutator {
             using IRMutator::visit;
@@ -487,8 +487,8 @@ bool can_prove(Expr e, const Scope<Interval> &bounds) {
             get_compiler_logger()->record_failed_to_prove(e, orig);
         }
 
-        debug(1) << "Failed to prove, but could not find a counter-example:\n " << e << "\n";
-        debug(1) << "Original expression:\n"
+        debug(1) << "Failed to prove, but could not find a counter-example:\n " << e << "\n"
+                 << "Original expression:\n"
                  << orig << "\n";
         return false;
     }
diff --git a/src/SimplifyCorrelatedDifferences.cpp b/src/SimplifyCorrelatedDifferences.cpp
index ba9c2d772fff..3afe5d84dcce 100644
--- a/src/SimplifyCorrelatedDifferences.cpp
+++ b/src/SimplifyCorrelatedDifferences.cpp
@@ -227,7 +227,7 @@ class SimplifyCorrelatedDifferences : public IRMutator {
             e = PartiallyCancelDifferences().mutate(e);
             e = simplify(e);
 
-            const bool check_non_monotonic = debug::debug_level() > 0 || get_compiler_logger() != nullptr;
+            const bool check_non_monotonic = debug_is_active(1) || get_compiler_logger() != nullptr;
             if (check_non_monotonic &&
                 is_monotonic(e, loop_var) == Monotonic::Unknown) {
                 // Might be a missed simplification opportunity. Log to help improve the simplifier.
diff --git a/src/Target.cpp b/src/Target.cpp
index 1fc467c624b3..c5d47bcdf43b 100644
--- a/src/Target.cpp
+++ b/src/Target.cpp
@@ -823,7 +823,7 @@ bool merge_string(Target &t, const std::string &target) {
     vector<string> tokens;
     size_t first_dash;
     while ((first_dash = rest.find('-')) != string::npos) {
-        // Internal::debug(0) << first_dash << ", " << rest << "\n";
+        // debug(0) << first_dash << ", " << rest << "\n";
         tokens.push_back(rest.substr(0, first_dash));
         rest = rest.substr(first_dash + 1);
     }
@@ -1644,16 +1644,16 @@ bool Target::get_runtime_compatible_target(const Target &other, Target &result)
     }
 
     if (arch != other.arch || bits != other.bits || os != other.os) {
-        Internal::debug(1) << "runtime targets must agree on platform (arch-bits-os)\n"
-                           << "  this:  " << *this << "\n"
-                           << "  other: " << other << "\n";
+        debug(1) << "runtime targets must agree on platform (arch-bits-os)\n"
+                 << "  this:  " << *this << "\n"
+                 << "  other: " << other << "\n";
         return false;
     }
 
     if ((features & matching_mask) != (other.features & matching_mask)) {
-        Internal::debug(1) << "runtime targets must agree on SoftFloatABI, Debug, TSAN, ASAN, MSAN, HVX, HexagonDma, SanitizerCoverage\n"
-                           << "  this:  " << *this << "\n"
-                           << "  other: " << other << "\n";
+        debug(1) << "runtime targets must agree on SoftFloatABI, Debug, TSAN, ASAN, MSAN, HVX, HexagonDma, SanitizerCoverage\n"
+                 << "  this:  " << *this << "\n"
+                 << "  other: " << other << "\n";
         return false;
     }
 
diff --git a/src/autoschedulers/anderson2021/AutoSchedule.cpp b/src/autoschedulers/anderson2021/AutoSchedule.cpp
index 3e1a4c2ede9a..35207bf640cf 100644
--- a/src/autoschedulers/anderson2021/AutoSchedule.cpp
+++ b/src/autoschedulers/anderson2021/AutoSchedule.cpp
@@ -42,15 +42,7 @@
 
 #endif
 */
-#include <algorithm>
-#include <chrono>
-#include <fstream>
-#include <iostream>
-#include <queue>
-#include <set>
-#include <sstream>
-#include <unordered_map>
-#include <unordered_set>
+#include "HalidePlugin.h"
 
 #include "ASLog.h"
 #include "AutoSchedule.h"
@@ -59,8 +51,6 @@
 #include "Errors.h"
 #include "Featurization.h"
 #include "FunctionDAG.h"
-#include "Halide.h"
-#include "HalidePlugin.h"
 #include "LoopNest.h"
 #include "LoopNestParser.h"
 #include "NetworkSize.h"
@@ -68,6 +58,16 @@
 #include "PerfectHashMap.h"
 #include "State.h"
 
+#include <algorithm>
+#include <chrono>
+#include <fstream>
+#include <iostream>
+#include <queue>
+#include <set>
+#include <sstream>
+#include <unordered_map>
+#include <unordered_set>
+
 #ifdef _WIN32
 #include <io.h>
 #define _isatty isatty;
diff --git a/src/autoschedulers/common/Errors.h b/src/autoschedulers/common/Errors.h
index 286758765578..d10c4537e1ff 100644
--- a/src/autoschedulers/common/Errors.h
+++ b/src/autoschedulers/common/Errors.h
@@ -3,9 +3,9 @@
 
 #include "Halide.h"
 
-#define internal_error Halide::Internal::ErrorReport<Halide::InternalError>(__FILE__, __LINE__, nullptr)
-#define user_error Halide::Internal::ErrorReport<Halide::CompileError>(__FILE__, __LINE__, nullptr)
-#define user_warning Halide::Internal::WarningReport(__FILE__, __LINE__, nullptr)
+#define internal_error Halide::Internal::ErrorReport<Halide::InternalError>(__FILE__, __FUNCTION__, __LINE__, nullptr)
+#define user_error Halide::Internal::ErrorReport<Halide::CompileError>(__FILE__, __FUNCTION__, __LINE__, nullptr)
+#define user_warning Halide::Internal::WarningReport(__FILE__, __FUNCTION__, __LINE__, nullptr)
 
 #define user_assert(c) _halide_internal_assertion(c, Halide::CompileError)
 #define internal_assert(c) _halide_internal_assertion(c, Halide::InternalError)
diff --git a/src/autoschedulers/common/HalidePlugin.h b/src/autoschedulers/common/HalidePlugin.h
index c5ffb1e16ec9..fc97c33cafa8 100644
--- a/src/autoschedulers/common/HalidePlugin.h
+++ b/src/autoschedulers/common/HalidePlugin.h
@@ -1,6 +1,9 @@
 #ifndef HALIDE_HALIDEPLUGIN_H
 #define HALIDE_HALIDEPLUGIN_H
 
+#define HALIDE_KEEP_MACROS
+#include "Halide.h"
+
 #include "Errors.h"
 
 #define REGISTER_AUTOSCHEDULER(NAME)                                  \
diff --git a/src/autoschedulers/li2018/GradientAutoscheduler.cpp b/src/autoschedulers/li2018/GradientAutoscheduler.cpp
index 76949b4b49fc..5ab2d6d96245 100644
--- a/src/autoschedulers/li2018/GradientAutoscheduler.cpp
+++ b/src/autoschedulers/li2018/GradientAutoscheduler.cpp
@@ -1,6 +1,6 @@
-#include "Errors.h"
-#include "Halide.h"
 #include "HalidePlugin.h"
+
+#include "Errors.h"
 #include "ParamParser.h"
 
 namespace Halide {
diff --git a/src/autoschedulers/mullapudi2016/AutoSchedule.cpp b/src/autoschedulers/mullapudi2016/AutoSchedule.cpp
index 60a30266043d..0b12ee67be27 100644
--- a/src/autoschedulers/mullapudi2016/AutoSchedule.cpp
+++ b/src/autoschedulers/mullapudi2016/AutoSchedule.cpp
@@ -1,14 +1,13 @@
 #include "HalidePlugin.h"
 
+#include "ParamParser.h"
+
 #include <algorithm>
 #include <map>
 #include <regex>
 #include <set>
 #include <utility>
 
-#include "Halide.h"
-#include "ParamParser.h"
-
 namespace Halide {
 namespace Internal {
 namespace Autoscheduler {
@@ -1430,9 +1429,10 @@ map<string, Expr> Partitioner::evaluate_reuse(const FStage &stg,
 
     for (int d = 0; d < (int)dims.size() - 1; d++) {
         Expr total_reuse = make_zero(Int(64));
-        if (debug::debug_level() >= 3) {
+        debug(3) << [&] {
             disp_regions(reuse_regions[d]);
-        }
+            return "";
+        }();
         for (const auto &reg : reuse_regions[d]) {
             Expr size = box_size(reg.second);
             if (!size.defined()) {
@@ -1780,9 +1780,10 @@ void Partitioner::group(Partitioner::Level level) {
         }
 
         Cost post_merge = get_pipeline_cost();
-        if (debug::debug_level() >= 3) {
+        debug(3) << [&] {
             disp_pipeline_costs();
-        }
+            return "";
+        }();
     }
 }
 
@@ -3300,9 +3301,10 @@ string generate_schedules(const vector<Function> &outputs, const Target &target,
     // Compute the expression costs for each function in the pipeline.
     debug(2) << "Initializing region costs...\n";
     RegionCosts costs(env, order);
-    if (debug::debug_level() >= 3) {
+    debug(3) << [&] {
         costs.disp_func_costs();
-    }
+        return "";
+    }();
 
     debug(2) << "Initializing dependence analysis...\n";
     DependenceAnalysis dep_analysis(env, order, func_val_bounds);
@@ -3363,31 +3365,35 @@ string generate_schedules(const vector<Function> &outputs, const Target &target,
 
     // Display the current pipeline graph.
     // TODO: Output the graph in dot format.
-    if (debug::debug_level() >= 3) {
+    debug(3) << [&] {
         part.disp_pipeline_graph();
         part.disp_pipeline_bounds();
-    }
+        return "";
+    }();
 
     debug(2) << "Partitioner initializing groups...\n";
     part.initialize_groups();
-    if (debug::debug_level() >= 3) {
+    debug(3) << [&] {
         part.disp_pipeline_costs();
-    }
+        return "";
+    }();
 
     debug(2) << "Partitioner computing inline group...\n";
     part.group(Partitioner::Level::Inline);
-    if (debug::debug_level() >= 3) {
+    debug(3) << [&] {
         part.disp_grouping();
-    }
+        return "";
+    }();
 
     debug(2) << "Partitioner computing fast-mem group...\n";
     part.grouping_cache.clear();
     part.group(Partitioner::Level::FastMem);
-    if (debug::debug_level() >= 3) {
+    debug(3) << [&] {
         part.disp_pipeline_costs();
         part.disp_grouping();
         part.disp_pipeline_graph();
-    }
+        return "";
+    }();
 
     debug(2) << "Initializing AutoSchedule...\n";
     AutoSchedule sched(env, top_order);
diff --git a/test/correctness/custom_error_reporter.cpp b/test/correctness/custom_error_reporter.cpp
index 405c25b9198b..f7432b1c3bdc 100644
--- a/test/correctness/custom_error_reporter.cpp
+++ b/test/correctness/custom_error_reporter.cpp
@@ -62,7 +62,7 @@ int main(int argc, char **argv) {
     MyCustomErrorReporter reporter;
     set_custom_compile_time_error_reporter(&reporter);
 
-    Halide::Internal::WarningReport("", 0, nullptr) << "Here is a warning.";
+    Halide::Internal::WarningReport("", "", 0, nullptr) << "Here is a warning.";
 
     // This call should not return.
     _halide_user_assert(argc == 0) << should_be_evaluated();
diff --git a/test/correctness/simplify.cpp b/test/correctness/simplify.cpp
index 264471294380..0566b40f4382 100644
--- a/test/correctness/simplify.cpp
+++ b/test/correctness/simplify.cpp
@@ -3,8 +3,6 @@
 using namespace Halide;
 using namespace Halide::Internal;
 
-#define internal_assert _halide_user_assert
-
 // Helper to wrap an expression in a statement using the expression
 // that won't be simplified away.
 Stmt not_no_op(Expr x) {
diff --git a/tools/build_halide_h.cpp b/tools/build_halide_h.cpp
index aa14a0a5266b..45e0bf415430 100644
--- a/tools/build_halide_h.cpp
+++ b/tools/build_halide_h.cpp
@@ -97,12 +97,16 @@ int main(int argc, char **argv) {
     fprintf(stdout, "\n");
     fprintf(stdout,
             "// Clean up macros used inside Halide headers\n"
+            "#ifndef HALIDE_KEEP_MACROS\n"
             "#undef user_assert\n"
             "#undef user_error\n"
             "#undef user_warning\n"
             "#undef internal_error\n"
             "#undef internal_assert\n"
             "#undef halide_runtime_error\n"
+            "#undef debug\n"
+            "#undef debug_is_active\n"
+            "#endif\n"
             "#endif  // HALIDE_H\n");
 
     return 0;