diff --git a/api/client.go b/api/client.go index 11a8515365..ced8b714f9 100644 --- a/api/client.go +++ b/api/client.go @@ -4,6 +4,7 @@ package api import ( "bytes" + "compress/gzip" "context" "crypto/tls" "encoding/json" @@ -49,6 +50,9 @@ type Config struct { // If true timings for each request will be logged TraceHTTP bool + // If true, request bodies will be gzip compressed + GzipAPIRequests bool + // The http client used, leave nil for the default HTTPClient *http.Client @@ -224,10 +228,22 @@ func (c *Client) newRequest( u := joinURLPath(c.conf.Endpoint, urlStr) buf := new(bytes.Buffer) + if body != nil { - err := json.NewEncoder(buf).Encode(body) - if err != nil { - return nil, err + if c.conf.GzipAPIRequests { + gzipWriter := gzip.NewWriter(buf) + err := json.NewEncoder(gzipWriter).Encode(body) + if err != nil { + return nil, err + } + + if err := gzipWriter.Close(); err != nil { + return nil, fmt.Errorf("closing gzip writer: %w", err) + } + } else { + if err := json.NewEncoder(buf).Encode(body); err != nil { + return nil, err + } } } @@ -260,6 +276,9 @@ func (c *Client) newRequest( if body != nil { req.Header.Add("Content-Type", "application/json") + if c.conf.GzipAPIRequests { + req.Header.Add("Content-Encoding", "gzip") + } } return req, nil @@ -309,7 +328,6 @@ func newResponse(r *http.Response) *Response { // interface, the raw response body will be written to v, without attempting to // first decode it. func (c *Client) doRequest(req *http.Request, v any) (*Response, error) { - resp, err := agenthttp.Do(c.logger, c.client, req, agenthttp.WithDebugHTTP(c.conf.DebugHTTP), agenthttp.WithTraceHTTP(c.conf.TraceHTTP), diff --git a/api/client_test.go b/api/client_test.go index fb45a8490d..404e81290d 100644 --- a/api/client_test.go +++ b/api/client_test.go @@ -1,9 +1,12 @@ package api_test import ( + "bytes" + "compress/gzip" "context" "crypto/tls" "fmt" + "io" "net/http" "net/http/httptest" "strings" @@ -78,6 +81,100 @@ func TestRegisteringAndConnectingClient(t *testing.T) { } } +func TestCompressionBehavior(t *testing.T) { + tests := []struct { + name string + gzipAPIRequests bool + expectCompressed bool + }{ + { + name: "compression disabled by default", + gzipAPIRequests: false, + expectCompressed: false, + }, + { + name: "compression enabled when requested", + gzipAPIRequests: true, + expectCompressed: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var requestBody []byte + var isCompressed bool + + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + // Read the request body + body, err := io.ReadAll(req.Body) + if err != nil { + http.Error(rw, err.Error(), http.StatusInternalServerError) + return + } + requestBody = body + + // Check if the request is compressed + isCompressed = req.Header.Get("Content-Encoding") == "gzip" + + // If compressed, try to decompress to verify it's valid gzip + if isCompressed { + gzReader, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + http.Error(rw, "Invalid gzip: "+err.Error(), http.StatusBadRequest) + return + } + defer gzReader.Close() + + _, err = io.ReadAll(gzReader) + if err != nil { + http.Error(rw, "Failed to decompress: "+err.Error(), http.StatusBadRequest) + return + } + } + + rw.WriteHeader(http.StatusOK) + fmt.Fprint(rw, `{}`) + })) + defer server.Close() + + ctx := context.Background() + client := api.NewClient(logger.Discard, api.Config{ + Endpoint: server.URL, + Token: "test-token", + GzipAPIRequests: tt.gzipAPIRequests, + }) + + // Make a request that will have a body (pipeline upload) + testPipeline := map[string]interface{}{ + "steps": []interface{}{ + map[string]interface{}{ + "command": "echo hello", + }, + }, + } + + _, err := client.UploadPipeline(ctx, "test-job-id", &api.PipelineChange{ + UUID: "test-uuid", + Pipeline: testPipeline, + }) + + if err != nil { + t.Fatalf("UploadPipeline failed: %v", err) + } + + // Verify compression behavior matches expectation + if isCompressed != tt.expectCompressed { + t.Errorf("Expected compressed=%v, got compressed=%v", tt.expectCompressed, isCompressed) + } + + // Verify we received a non-empty body + if len(requestBody) == 0 { + t.Error("Expected non-empty request body") + } + }) + } +} + func authToken(req *http.Request) string { return strings.TrimPrefix(req.Header.Get("Authorization"), "Token ") } diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go index a90c511c4e..1e6e8816b8 100644 --- a/clicommand/agent_start.go +++ b/clicommand/agent_start.go @@ -185,11 +185,12 @@ type AgentStartConfig struct { NoMultipartArtifactUpload bool `cli:"no-multipart-artifact-upload"` // API config - DebugHTTP bool `cli:"debug-http"` - TraceHTTP bool `cli:"trace-http"` - Token string `cli:"token" validate:"required"` - Endpoint string `cli:"endpoint" validate:"required"` - NoHTTP2 bool `cli:"no-http2"` + DebugHTTP bool `cli:"debug-http"` + TraceHTTP bool `cli:"trace-http"` + GzipAPIRequests bool `cli:"gzip-api-requests"` + Token string `cli:"token" validate:"required"` + Endpoint string `cli:"endpoint" validate:"required"` + NoHTTP2 bool `cli:"no-http2"` // Deprecated NoSSHFingerprintVerification bool `cli:"no-automatic-ssh-fingerprint-verification" deprecated-and-renamed-to:"NoSSHKeyscan"` @@ -742,6 +743,7 @@ var AgentStartCommand = cli.Command{ NoHTTP2Flag, DebugHTTPFlag, TraceHTTPFlag, + GzipAPIRequestsFlag, // Other shared flags RedactedVars, diff --git a/clicommand/global.go b/clicommand/global.go index 52667c439b..73441274ab 100644 --- a/clicommand/global.go +++ b/clicommand/global.go @@ -128,6 +128,12 @@ var ( EnvVar: "BUILDKITE_AGENT_EXPERIMENT", } + GzipAPIRequestsFlag = cli.BoolFlag{ + Name: "gzip-api-requests", + Usage: "Enable gzip compression for API request bodies", + EnvVar: "BUILDKITE_GZIP_API_REQUESTS", + } + RedactedVars = cli.StringSliceFlag{ Name: "redacted-vars", Usage: "Pattern of environment variable names containing sensitive values", @@ -172,6 +178,7 @@ type APIConfig struct { TraceHTTP bool `cli:"trace-http"` Endpoint string `cli:"endpoint" validate:"required"` NoHTTP2 bool `cli:"no-http2"` + GzipAPIRequests bool `cli:"gzip-api-requests"` } func globalFlags() []cli.Flag { @@ -191,6 +198,7 @@ func apiFlags() []cli.Flag { NoHTTP2Flag, DebugHTTPFlag, TraceHTTPFlag, + GzipAPIRequestsFlag, } } @@ -355,6 +363,11 @@ func loadAPIClientConfig(cfg any, tokenField string) api.Config { conf.DisableHTTP2 = noHTTP2.(bool) } + gzipAPIRequests, err := reflections.GetField(cfg, "GzipAPIRequests") + if err == nil { + conf.GzipAPIRequests = gzipAPIRequests.(bool) + } + return conf }