Skip to content

feat(fuzz): add semantic property invariants to all fuzz targets + global stdout cap #159

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
thieman wants to merge 2 commits into from
Closed

feat(fuzz): add semantic property invariants to all fuzz targets + global stdout cap #159

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2cba7c7
feat(interp): add global 10 MiB stdout cap to Runner.Run
thieman Mar 26, 2026
48322a2
feat(fuzz): add semantic property invariants to all fuzz targets
thieman Mar 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 82 additions & 2 deletions builtins/tests/cat/cat_fuzz_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,14 +85,34 @@ func FuzzCat(f *testing.F) {
if t.Context().Err() != nil {
return
}
// Invariant 3: exit code validity.
if code != 0 && code != 1 {
t.Errorf("unexpected exit code %d", code)
}
// Invariant 1: output bounded.
if len(stdout) > 10*1024*1024 {
t.Errorf("output exceeds 10 MiB: %d bytes", len(stdout))
}

// cat must output exactly the file contents
if code == 0 && stdout != string(input) {
t.Errorf("cat output differs from input: got %d bytes, want %d bytes", len(stdout), len(input))
}

// Invariant 4: no panic — reaching this line proves no panic escaped Run().

// Invariant 2: determinism.
ctx2, cancel2 := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel2()
stdout2, _, code2 := cmdRunCtx(ctx2, t, "cat input.txt", dir)
cancel2()
if t.Context().Err() != nil {
return
}
if stdout != stdout2 || code != code2 {
t.Errorf("determinism violation on cat: outputs differ on identical input\nrun1: exit=%d, len=%d\nrun2: exit=%d, len=%d",
code, len(stdout), code2, len(stdout2))
}
})
}

Expand Down Expand Up @@ -137,14 +157,34 @@ func FuzzCatNumberLines(f *testing.F) {

ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel() // safety net if t.Fatal fires before explicit cancel
_, _, code := cmdRunCtx(ctx, t, "cat -n input.txt", dir)
stdout, _, code := cmdRunCtx(ctx, t, "cat -n input.txt", dir)
cancel()
if t.Context().Err() != nil {
return
}
// Invariant 3: exit code validity.
if code != 0 && code != 1 {
t.Errorf("cat -n unexpected exit code %d", code)
}
// Invariant 1: output bounded.
if len(stdout) > 10*1024*1024 {
t.Errorf("cat -n output exceeds 10 MiB: %d bytes", len(stdout))
}

// Invariant 4: no panic — reaching this line proves no panic escaped Run().

// Invariant 2: determinism.
ctx2, cancel2 := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel2()
stdout2, _, code2 := cmdRunCtx(ctx2, t, "cat -n input.txt", dir)
cancel2()
if t.Context().Err() != nil {
return
}
if stdout != stdout2 || code != code2 {
t.Errorf("determinism violation on cat -n: outputs differ on identical input\nrun1: exit=%d, len=%d\nrun2: exit=%d, len=%d",
code, len(stdout), code2, len(stdout2))
}
})
}

Expand Down Expand Up @@ -205,14 +245,34 @@ func FuzzCatDisplayFlags(f *testing.F) {

ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel() // safety net if t.Fatal fires before explicit cancel
_, _, code := cmdRunCtx(ctx, t, "cat"+flags+" input.bin", dir)
stdout, _, code := cmdRunCtx(ctx, t, "cat"+flags+" input.bin", dir)
cancel()
if t.Context().Err() != nil {
return
}
// Invariant 3: exit code validity.
if code != 0 && code != 1 {
t.Errorf("cat%s unexpected exit code %d", flags, code)
}
// Invariant 1: output bounded.
if len(stdout) > 10*1024*1024 {
t.Errorf("cat%s output exceeds 10 MiB: %d bytes", flags, len(stdout))
}

// Invariant 4: no panic — reaching this line proves no panic escaped Run().

// Invariant 2: determinism.
ctx2, cancel2 := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel2()
stdout2, _, code2 := cmdRunCtx(ctx2, t, "cat"+flags+" input.bin", dir)
cancel2()
if t.Context().Err() != nil {
return
}
if stdout != stdout2 || code != code2 {
t.Errorf("determinism violation on cat%s: outputs differ on identical input\nrun1: exit=%d, len=%d\nrun2: exit=%d, len=%d",
flags, code, len(stdout), code2, len(stdout2))
}
})
}

Expand Down Expand Up @@ -254,12 +314,32 @@ func FuzzCatStdin(f *testing.F) {
if t.Context().Err() != nil {
return
}
// Invariant 3: exit code validity.
if code != 0 && code != 1 {
t.Errorf("cat stdin unexpected exit code %d", code)
}
// Invariant 1: output bounded.
if len(stdout) > 10*1024*1024 {
t.Errorf("cat stdin output exceeds 10 MiB: %d bytes", len(stdout))
}

if code == 0 && stdout != string(input) {
t.Errorf("cat stdin output differs from input: got %d bytes, want %d bytes", len(stdout), len(input))
}

// Invariant 4: no panic — reaching this line proves no panic escaped Run().

// Invariant 2: determinism.
ctx2, cancel2 := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel2()
stdout2, _, code2 := cmdRunCtx(ctx2, t, "cat < stdin.txt", dir)
cancel2()
if t.Context().Err() != nil {
return
}
if stdout != stdout2 || code != code2 {
t.Errorf("determinism violation on cat stdin: outputs differ on identical input\nrun1: exit=%d, len=%d\nrun2: exit=%d, len=%d",
code, len(stdout), code2, len(stdout2))
}
})
}
113 changes: 108 additions & 5 deletions builtins/tests/cut/cut_fuzz_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,14 +102,35 @@ func FuzzCutFields(f *testing.F) {

ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel() // safety net if t.Fatal fires before explicit cancel
_, _, code := cmdRunCtxFuzz(ctx, t, fmt.Sprintf("cut -f %s input.txt", fieldSpec), dir)
script := fmt.Sprintf("cut -f %s input.txt", fieldSpec)
stdout, _, code := cmdRunCtxFuzz(ctx, t, script, dir)
cancel()
if t.Context().Err() != nil {
return
}
// Invariant 3: exit code validity.
if code != 0 && code != 1 {
t.Errorf("cut -f %s unexpected exit code %d", fieldSpec, code)
}
// Invariant 1: output bounded.
if len(stdout) > 10*1024*1024 {
t.Errorf("cut -f %s output exceeds 10 MiB: %d bytes", fieldSpec, len(stdout))
}

// Invariant 4: no panic — reaching this line proves no panic escaped Run().

// Invariant 2: determinism.
ctx2, cancel2 := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel2()
stdout2, _, code2 := cmdRunCtxFuzz(ctx2, t, script, dir)
cancel2()
if t.Context().Err() != nil {
return
}
if stdout != stdout2 || code != code2 {
t.Errorf("determinism violation on cut -f %s: outputs differ on identical input\nrun1: exit=%d, len=%d\nrun2: exit=%d, len=%d",
fieldSpec, code, len(stdout), code2, len(stdout2))
}
})
}

Expand Down Expand Up @@ -176,14 +197,35 @@ func FuzzCutBytes(f *testing.F) {

ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel() // safety net if t.Fatal fires before explicit cancel
_, _, code := cmdRunCtxFuzz(ctx, t, fmt.Sprintf("cut -b %s input.txt", byteSpec), dir)
script := fmt.Sprintf("cut -b %s input.txt", byteSpec)
stdout, _, code := cmdRunCtxFuzz(ctx, t, script, dir)
cancel()
if t.Context().Err() != nil {
return
}
// Invariant 3: exit code validity.
if code != 0 && code != 1 {
t.Errorf("cut -b %s unexpected exit code %d", byteSpec, code)
}
// Invariant 1: output bounded.
if len(stdout) > 10*1024*1024 {
t.Errorf("cut -b %s output exceeds 10 MiB: %d bytes", byteSpec, len(stdout))
}

// Invariant 4: no panic — reaching this line proves no panic escaped Run().

// Invariant 2: determinism.
ctx2, cancel2 := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel2()
stdout2, _, code2 := cmdRunCtxFuzz(ctx2, t, script, dir)
cancel2()
if t.Context().Err() != nil {
return
}
if stdout != stdout2 || code != code2 {
t.Errorf("determinism violation on cut -b %s: outputs differ on identical input\nrun1: exit=%d, len=%d\nrun2: exit=%d, len=%d",
byteSpec, code, len(stdout), code2, len(stdout2))
}
})
}

Expand Down Expand Up @@ -249,14 +291,34 @@ func FuzzCutDelimiter(f *testing.F) {
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel() // safety net if t.Fatal fires before explicit cancel
script := fmt.Sprintf("cut -d '%s' -f %s input.txt", delim, fieldSpec)
_, _, code := cmdRunCtxFuzz(ctx, t, script, dir)
stdout, _, code := cmdRunCtxFuzz(ctx, t, script, dir)
cancel()
if t.Context().Err() != nil {
return
}
// Invariant 3: exit code validity.
if code != 0 && code != 1 {
t.Errorf("cut -d '%s' -f %s unexpected exit code %d", delim, fieldSpec, code)
}
// Invariant 1: output bounded.
if len(stdout) > 10*1024*1024 {
t.Errorf("cut -d '%s' -f %s output exceeds 10 MiB: %d bytes", delim, fieldSpec, len(stdout))
}

// Invariant 4: no panic — reaching this line proves no panic escaped Run().

// Invariant 2: determinism.
ctx2, cancel2 := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel2()
stdout2, _, code2 := cmdRunCtxFuzz(ctx2, t, script, dir)
cancel2()
if t.Context().Err() != nil {
return
}
if stdout != stdout2 || code != code2 {
t.Errorf("determinism violation on cut -d '%s' -f %s: outputs differ on identical input\nrun1: exit=%d, len=%d\nrun2: exit=%d, len=%d",
delim, fieldSpec, code, len(stdout), code2, len(stdout2))
}
})
}

Expand Down Expand Up @@ -313,14 +375,35 @@ func FuzzCutComplement(f *testing.F) {

ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel() // safety net if t.Fatal fires before explicit cancel
_, _, code := cmdRunCtxFuzz(ctx, t, fmt.Sprintf("cut --complement -b %s input.txt", byteSpec), dir)
script := fmt.Sprintf("cut --complement -b %s input.txt", byteSpec)
stdout, _, code := cmdRunCtxFuzz(ctx, t, script, dir)
cancel()
if t.Context().Err() != nil {
return
}
// Invariant 3: exit code validity.
if code != 0 && code != 1 {
t.Errorf("cut --complement -b %s unexpected exit code %d", byteSpec, code)
}
// Invariant 1: output bounded.
if len(stdout) > 10*1024*1024 {
t.Errorf("cut --complement -b %s output exceeds 10 MiB: %d bytes", byteSpec, len(stdout))
}

// Invariant 4: no panic — reaching this line proves no panic escaped Run().

// Invariant 2: determinism.
ctx2, cancel2 := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel2()
stdout2, _, code2 := cmdRunCtxFuzz(ctx2, t, script, dir)
cancel2()
if t.Context().Err() != nil {
return
}
if stdout != stdout2 || code != code2 {
t.Errorf("determinism violation on cut --complement -b %s: outputs differ on identical input\nrun1: exit=%d, len=%d\nrun2: exit=%d, len=%d",
byteSpec, code, len(stdout), code2, len(stdout2))
}
})
}

Expand Down Expand Up @@ -360,13 +443,33 @@ func FuzzCutStdin(f *testing.F) {

ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel() // safety net if t.Fatal fires before explicit cancel
_, _, code := cmdRunCtxFuzz(ctx, t, "cut -f 1 < stdin.txt", dir)
stdout, _, code := cmdRunCtxFuzz(ctx, t, "cut -f 1 < stdin.txt", dir)
cancel()
if t.Context().Err() != nil {
return
}
// Invariant 3: exit code validity.
if code != 0 && code != 1 {
t.Errorf("cut stdin unexpected exit code %d", code)
}
// Invariant 1: output bounded.
if len(stdout) > 10*1024*1024 {
t.Errorf("cut stdin output exceeds 10 MiB: %d bytes", len(stdout))
}

// Invariant 4: no panic — reaching this line proves no panic escaped Run().

// Invariant 2: determinism.
ctx2, cancel2 := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel2()
stdout2, _, code2 := cmdRunCtxFuzz(ctx2, t, "cut -f 1 < stdin.txt", dir)
cancel2()
if t.Context().Err() != nil {
return
}
if stdout != stdout2 || code != code2 {
t.Errorf("determinism violation on cut stdin: outputs differ on identical input\nrun1: exit=%d, len=%d\nrun2: exit=%d, len=%d",
code, len(stdout), code2, len(stdout2))
}
})
}
Loading
Loading