From b5deb35b2fd56261bc7feef673648f4a49a48990 Mon Sep 17 00:00:00 2001 From: Josh Deprez Date: Wed, 3 Dec 2025 12:15:12 +1100 Subject: [PATCH] redactor add: Add optional filtering like env vars --- clicommand/redactor_add.go | 39 ++++++++++--- clicommand/redactor_add_test.go | 100 ++++++++++++++++++++++++-------- 2 files changed, 107 insertions(+), 32 deletions(-) diff --git a/clicommand/redactor_add.go b/clicommand/redactor_add.go index 3ed11aefaa..595ad27035 100644 --- a/clicommand/redactor_add.go +++ b/clicommand/redactor_add.go @@ -11,6 +11,8 @@ import ( "slices" "strings" + "github.com/buildkite/agent/v3/env" + "github.com/buildkite/agent/v3/internal/redact" "github.com/buildkite/agent/v3/jobapi" "github.com/buildkite/agent/v3/logger" "github.com/urfave/cli" @@ -38,8 +40,10 @@ type RedactorAddConfig struct { GlobalConfig APIConfig - File string `cli:"arg:0"` - Format string `cli:"format"` + File string `cli:"arg:0"` + Format string `cli:"format"` + ApplyVarsFilter bool `cli:"apply-vars-filter"` + RedactedVars []string `cli:"redacted-vars"` } var RedactorAddCommand = cli.Command{ @@ -86,6 +90,12 @@ JSON does not allow duplicate keys. If you repeat the same key ("key"), the JSON EnvVar: "BUILDKITE_AGENT_REDACT_ADD_FORMAT", Value: FormatStringNone, }, + cli.BoolFlag{ + Name: "apply-vars-filter", + Usage: fmt.Sprintf("When the input is in 'json' format, filters the secrets to redact using the same rules used to detect secrets from environment variables: secrets must be at least %d characters long, and names must match the patterns defined by --redacted-vars or $BUILDKITE_REDACTED_VARS.", redact.LengthMin), + EnvVar: "BUILDKITE_AGENT_REDACT_VARS_FILTER", + }, + RedactedVars, }), Action: func(c *cli.Context) error { ctx := context.Background() @@ -144,14 +154,29 @@ func ParseSecrets( ) ([]string, error) { switch cfg.Format { case FormatStringJSON: - secrets := &map[string]string{} + secrets := map[string]string{} if err := json.NewDecoder(secretsReader).Decode(&secrets); err != nil { return nil, fmt.Errorf("failed to parse as string valued JSON: %w", err) } - parsedSecrets := make([]string, 0, len(*secrets)) - for _, secret := range *secrets { - parsedSecrets = append(parsedSecrets, secret) + var parsedSecrets []string + if cfg.ApplyVarsFilter { + matched, short, err := redact.Vars(cfg.RedactedVars, env.FromMap(secrets).DumpPairs()) + if err != nil { + return nil, fmt.Errorf("couldn't match object keys against redacted-vars: %w", err) + } + if len(short) > 0 { + l.Warn("Some object keys had values below minimum length (%d bytes) and will not be redacted: %s", redact.LengthMin, strings.Join(short, ", ")) + } + parsedSecrets = make([]string, 0, len(matched)) + for _, m := range matched { + parsedSecrets = append(parsedSecrets, m.Value) + } + } else { + parsedSecrets = make([]string, 0, len(secrets)) + for _, secret := range secrets { + parsedSecrets = append(parsedSecrets, secret) + } } return parsedSecrets, nil @@ -165,7 +190,7 @@ func ParseSecrets( return []string{strings.TrimSpace(string(readSecret))}, nil default: - return nil, fmt.Errorf("%s: %w", cfg.Format, errUnknownFormat) + return nil, fmt.Errorf("%w %q", errUnknownFormat, cfg.Format) } } diff --git a/clicommand/redactor_add_test.go b/clicommand/redactor_add_test.go index b1dda5664b..b578fbe411 100644 --- a/clicommand/redactor_add_test.go +++ b/clicommand/redactor_add_test.go @@ -1,58 +1,108 @@ package clicommand_test import ( + "encoding/json" + "errors" "slices" "strings" "testing" "github.com/buildkite/agent/v3/clicommand" "github.com/buildkite/agent/v3/logger" - "gotest.tools/v3/assert" + "github.com/google/go-cmp/cmp" ) func TestParseSecrets(t *testing.T) { t.Parallel() for _, tc := range []struct { - name string - inputData string - formatString string - expectedSecrets []string - errorTextContains string + name string + inputData string + formatString string + applyVarsFilter bool + wantSecrets []string }{ { - name: "json", - inputData: `{"hello": "world", "password": "hunter2"}`, - formatString: clicommand.FormatStringJSON, - expectedSecrets: []string{"world", "hunter2"}, + name: "json", + inputData: `{"hello": "world", "password": "hunter2"}`, + formatString: clicommand.FormatStringJSON, + wantSecrets: []string{"world", "hunter2"}, }, { - name: "plaintext", - inputData: "hunter2\n", - formatString: clicommand.FormatStringNone, - expectedSecrets: []string{"hunter2"}, + name: "plaintext", + inputData: "hunter2\n", + formatString: clicommand.FormatStringNone, + wantSecrets: []string{"hunter2"}, }, + { - name: "invalid_json", - inputData: `{"hello": 1, "password": "hunter2"}`, - formatString: clicommand.FormatStringJSON, - errorTextContains: "failed to parse as string valued JSON", + name: "vars filter", + inputData: `{"HELLO": "1", "MY_PASSWORD": "hunter2"}`, + applyVarsFilter: true, + formatString: clicommand.FormatStringJSON, + wantSecrets: []string{"hunter2"}, }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() input := strings.NewReader(tc.inputData) - secrets, err := clicommand.ParseSecrets(logger.Discard, clicommand.RedactorAddConfig{Format: tc.formatString}, input) - if tc.errorTextContains != "" { - assert.ErrorContains(t, err, tc.errorTextContains) - return + secrets, err := clicommand.ParseSecrets( + logger.Discard, + clicommand.RedactorAddConfig{ + Format: tc.formatString, + ApplyVarsFilter: tc.applyVarsFilter, + RedactedVars: *clicommand.RedactedVars.Value, + }, + input, + ) + if err != nil { + t.Errorf("clicommand.ParseSecrets(logger, cfg, %q) error = %v", input, err) } - assert.NilError(t, err) slices.Sort(secrets) - slices.Sort(tc.expectedSecrets) - assert.DeepEqual(t, secrets, tc.expectedSecrets) + slices.Sort(tc.wantSecrets) + if diff := cmp.Diff(secrets, tc.wantSecrets); diff != "" { + t.Errorf("clicommand.ParseSecrets(logger, cfg, %q) secrets diff (-got +want):\n%s", input, diff) + } + }) + } +} + +func TestParseSecrets_JSONErrors(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + inputData string + wantError any + }{ + { + name: "type mismatch", + inputData: `{"hello": 1, "password": "hunter2"}`, + wantError: new(*json.UnmarshalTypeError), + }, + { + name: "syntax error", + inputData: `}}{"hello": , "pas: "hun'}`, + wantError: new(*json.SyntaxError), + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + input := strings.NewReader(tc.inputData) + + _, err := clicommand.ParseSecrets( + logger.Discard, + clicommand.RedactorAddConfig{ + Format: clicommand.FormatStringJSON, + }, + input, + ) + if !errors.As(err, tc.wantError) { + t.Errorf("clicommand.ParseSecrets(logger, cfg, %q) error = %v, want error wrapping %T", input, err, tc.wantError) + } }) } }