Skip to content

fuzzing: implement limited fuzzing #25342

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andrewrk merged 5 commits into from
Sep 26, 2025
Merged

fuzzing: implement limited fuzzing #25342

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0feacc2
fuzzing: implement limited fuzzing
kristoff-it Sep 24, 2025
9bb0b43
implement review suggestions
kristoff-it Sep 25, 2025
98253bc
Compilation: --debug-rt always Debug
andrewrk Sep 26, 2025
2da8ec9
fuzzing: fix off-by-one in limit count
andrewrk Sep 26, 2025
52a13f6
web ui: fix not sending initial context sometimes
andrewrk Sep 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 88 additions & 6 deletions lib/compiler/build_runner.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ pub fn main() !void {
var steps_menu = false;
var output_tmp_nonce: ?[16]u8 = null;
var watch = false;
var fuzz = false;
var fuzz: ?std.Build.Fuzz.Mode = null;
var debounce_interval_ms: u16 = 50;
var webui_listen: ?std.net.Address = null;

Expand Down Expand Up @@ -274,10 +274,44 @@ pub fn main() !void {
webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
}
} else if (mem.eql(u8, arg, "--fuzz")) {
fuzz = true;
fuzz = .{ .forever = undefined };
if (webui_listen == null) {
webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
}
} else if (mem.startsWith(u8, arg, "--fuzz=")) {
const value = arg["--fuzz=".len..];
if (value.len == 0) fatal("missing argument to --fuzz", .{});

const unit: u8 = value[value.len - 1];
const digits = switch (unit) {
'0'...'9' => value,
'K', 'M', 'G' => value[0 .. value.len - 1],
else => fatal(
"invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
.{},
),
};

const amount = std.fmt.parseInt(u64, digits, 10) catch {
fatal(
"invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
.{},
);
};

const normalized_amount = std.math.mul(u64, amount, switch (unit) {
else => unreachable,
'0'...'9' => 1,
'K' => 1000,
'M' => 1_000_000,
'G' => 1_000_000_000,
}) catch fatal("fuzzing limit amount overflows u64", .{});

fuzz = .{
.limit = .{
.amount = normalized_amount,
},
};
} else if (mem.eql(u8, arg, "-fincremental")) {
graph.incremental = true;
} else if (mem.eql(u8, arg, "-fno-incremental")) {
Expand Down Expand Up @@ -476,6 +510,7 @@ pub fn main() !void {
targets.items,
main_progress_node,
&run,
fuzz,
) catch |err| switch (err) {
error.UncleanExit => {
assert(!run.watch and run.web_server == null);
Expand All @@ -485,7 +520,12 @@ pub fn main() !void {
};

if (run.web_server) |*web_server| {
web_server.finishBuild(.{ .fuzz = fuzz });
if (fuzz) |mode| if (mode != .forever) fatal(
"error: limited fuzzing is not implemented yet for --webui",
.{},
);

web_server.finishBuild(.{ .fuzz = fuzz != null });
}

if (!watch and run.web_server == null) {
Expand Down Expand Up @@ -651,6 +691,7 @@ fn runStepNames(
step_names: []const []const u8,
parent_prog_node: std.Progress.Node,
run: *Run,
fuzz: ?std.Build.Fuzz.Mode,
) !void {
const gpa = run.gpa;
const step_stack = &run.step_stack;
Expand All @@ -676,6 +717,7 @@ fn runStepNames(
});
}
}

assert(run.memory_blocked_steps.items.len == 0);

var test_skip_count: usize = 0;
Expand Down Expand Up @@ -724,6 +766,45 @@ fn runStepNames(
}
}

const ttyconf = run.ttyconf;

if (fuzz) |mode| blk: {
switch (builtin.os.tag) {
// Current implementation depends on two things that need to be ported to Windows:
// * Memory-mapping to share data between the fuzzer and build runner.
// * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving
// many addresses to source locations).
.windows => fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}),
else => {},
}
if (@bitSizeOf(usize) != 64) {
// Current implementation depends on posix.mmap()'s second parameter, `length: usize`,
// being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case
// on 32-bit platforms.
// Affects or affected by issues #5185, #22523, and #22464.
fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
}

switch (mode) {
.forever => break :blk,
.limit => {},
}

assert(mode == .limit);
var f = std.Build.Fuzz.init(
gpa,
thread_pool,
step_stack.keys(),
parent_prog_node,
ttyconf,
mode,
) catch |err| fatal("failed to start fuzzer: {s}", .{@errorName(err)});
defer f.deinit();

f.start();
f.waitAndPrintReport();
Comment on lines +804 to +805
Copy link
Member

@andrewrk andrewrk Sep 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fine for this branch, but noting for any contributors watching this PR: a future enhancement will be to make fuzzing limited mode be part of the main build graph pipeline. In other words, limited mode fuzzing should run at the same time as the rest of the build.

}

// A proper command line application defaults to silently succeeding.
// The user may request verbose mode if they have a different preference.
const failures_only = switch (run.summary) {
Expand All @@ -737,8 +818,6 @@ fn runStepNames(
std.Progress.setStatus(.failure);
}

const ttyconf = run.ttyconf;

if (run.summary != .none) {
const w = std.debug.lockStderrWriter(&stdio_buffer_allocation);
defer std.debug.unlockStderrWriter();
Expand Down Expand Up @@ -1366,7 +1445,10 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
\\ --watch Continuously rebuild when source files are modified
\\ --debounce <ms> Delay before rebuilding after changed file detected
\\ --webui[=ip] Enable the web interface on the given IP address
\\ --fuzz Continuously search for unit test failures (implies '--webui')
\\ --fuzz[=limit] Continuously search for unit test failures with an optional
\\ limit to the max number of iterations. The argument supports
\\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies
\\ '--webui' when no limit is specified.
\\ --time-report Force full rebuild and provide detailed information on
\\ compilation time of Zig source code (implies '--webui')
\\ -fincremental Enable incremental compilation
Expand Down
36 changes: 27 additions & 9 deletions lib/compiler/test_runner.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
const builtin = @import("builtin");

const std = @import("std");
const fatal = std.process.fatal;
const testing = std.testing;
const assert = std.debug.assert;
const fuzz_abi = std.Build.abi.fuzz;
Expand Down Expand Up @@ -55,12 +56,13 @@ pub fn main() void {
}
}

fba.reset();
if (builtin.fuzz) {
const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument");
fuzz_abi.fuzzer_init(.fromSlice(cache_dir));
}

fba.reset();

if (listen) {
return mainServer() catch @panic("internal test runner failure");
} else {
Expand All @@ -79,8 +81,13 @@ fn mainServer() !void {
});

if (builtin.fuzz) {
const coverage_id = fuzz_abi.fuzzer_coverage_id();
try server.serveU64Message(.coverage_id, coverage_id);
const coverage = fuzz_abi.fuzzer_coverage();
try server.serveCoverageIdMessage(
coverage.id,
coverage.runs,
coverage.unique,
coverage.seen,
);
}

while (true) {
Expand Down Expand Up @@ -158,26 +165,33 @@ fn mainServer() !void {
if (!builtin.fuzz) unreachable;

const index = try server.receiveBody_u32();
const mode: fuzz_abi.LimitKind = @enumFromInt(try server.receiveBody_u8());
const amount_or_instance = try server.receiveBody_u64();

const test_fn = builtin.test_functions[index];
const entry_addr = @intFromPtr(test_fn.func);

try server.serveU64Message(.fuzz_start_addr, entry_addr);
defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
is_fuzz_test = false;
fuzz_test_index = index;
fuzz_mode = mode;
fuzz_amount_or_instance = amount_or_instance;

test_fn.func() catch |err| switch (err) {
error.SkipZigTest => return,
else => {
if (@errorReturnTrace()) |trace| {
std.debug.dumpStackTrace(trace.*);
}
std.debug.print("failed with error.{s}\n", .{@errorName(err)});
std.debug.print("failed with error.{t}\n", .{err});
std.process.exit(1);
},
};
if (!is_fuzz_test) @panic("missed call to std.testing.fuzz");
if (log_err_count != 0) @panic("error logs detected");
assert(mode != .forever);
std.process.exit(0);
},

else => {
Expand Down Expand Up @@ -240,11 +254,11 @@ fn mainTerminal() void {
else => {
fail_count += 1;
if (have_tty) {
std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{
i + 1, test_fn_list.len, test_fn.name, @errorName(err),
std.debug.print("{d}/{d} {s}...FAIL ({t})\n", .{
i + 1, test_fn_list.len, test_fn.name, err,
});
} else {
std.debug.print("FAIL ({s})\n", .{@errorName(err)});
std.debug.print("FAIL ({t})\n", .{err});
}
if (@errorReturnTrace()) |trace| {
std.debug.dumpStackTrace(trace.*);
Expand Down Expand Up @@ -343,6 +357,8 @@ pub fn mainSimple() anyerror!void {

var is_fuzz_test: bool = undefined;
var fuzz_test_index: u32 = undefined;
var fuzz_mode: fuzz_abi.LimitKind = undefined;
var fuzz_amount_or_instance: u64 = undefined;

pub fn fuzz(
context: anytype,
Expand Down Expand Up @@ -383,7 +399,7 @@ pub fn fuzz(
else => {
std.debug.lockStdErr();
if (@errorReturnTrace()) |trace| std.debug.dumpStackTrace(trace.*);
std.debug.print("failed with error.{s}\n", .{@errorName(err)});
std.debug.print("failed with error.{t}\n", .{err});
std.process.exit(1);
},
};
Expand All @@ -401,9 +417,11 @@ pub fn fuzz(

global.ctx = context;
fuzz_abi.fuzzer_init_test(&global.test_one, .fromSlice(builtin.test_functions[fuzz_test_index].name));

for (options.corpus) |elem|
fuzz_abi.fuzzer_new_input(.fromSlice(elem));
fuzz_abi.fuzzer_main();

fuzz_abi.fuzzer_main(fuzz_mode, fuzz_amount_or_instance);
return;
}

Expand Down
Loading