Gzip agent api request bodies before sending them off

api/client.go

@@ -4,6 +4,7 @@ package api
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/tls"
 	"encoding/json"
@@ -49,6 +50,9 @@ type Config struct {
 	// If true timings for each request will be logged
 	TraceHTTP bool
 
+	// If true, request bodies will be gzip compressed
+	GzipAPIRequests bool
+
 	// The http client used, leave nil for the default
 	HTTPClient *http.Client
 
@@ -224,10 +228,22 @@ func (c *Client) newRequest(
 	u := joinURLPath(c.conf.Endpoint, urlStr)
 
 	buf := new(bytes.Buffer)
+
 	if body != nil {
-		err := json.NewEncoder(buf).Encode(body)
-		if err != nil {
-			return nil, err
+		if c.conf.GzipAPIRequests {
+			gzipWriter := gzip.NewWriter(buf)
+			err := json.NewEncoder(gzipWriter).Encode(body)
+			if err != nil {
+				return nil, err
+			}
+
+			if err := gzipWriter.Close(); err != nil {
+				return nil, fmt.Errorf("closing gzip writer: %w", err)
+			}
+		} else {
+			if err := json.NewEncoder(buf).Encode(body); err != nil {
+				return nil, err
+			}
 		}
 	}
 
@@ -260,6 +276,9 @@ func (c *Client) newRequest(
 
 	if body != nil {
 		req.Header.Add("Content-Type", "application/json")
+		if c.conf.GzipAPIRequests {
+			req.Header.Add("Content-Encoding", "gzip")
+		}
 	}
 
 	return req, nil
@@ -309,7 +328,6 @@ func newResponse(r *http.Response) *Response {
 // interface, the raw response body will be written to v, without attempting to
 // first decode it.
 func (c *Client) doRequest(req *http.Request, v any) (*Response, error) {
-
 	resp, err := agenthttp.Do(c.logger, c.client, req,
 		agenthttp.WithDebugHTTP(c.conf.DebugHTTP),
 		agenthttp.WithTraceHTTP(c.conf.TraceHTTP),

api/client_test.go

@@ -1,9 +1,12 @@
 package api_test
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/tls"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -78,6 +81,100 @@ func TestRegisteringAndConnectingClient(t *testing.T) {
 	}
 }
 
+func TestCompressionBehavior(t *testing.T) {
+	tests := []struct {
+		name             string
+		gzipAPIRequests  bool
+		expectCompressed bool
+	}{
+		{
+			name:             "compression disabled by default",
+			gzipAPIRequests:  false,
+			expectCompressed: false,
+		},
+		{
+			name:             "compression enabled when requested",
+			gzipAPIRequests:  true,
+			expectCompressed: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var requestBody []byte
+			var isCompressed bool
+
+			server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+				// Read the request body
+				body, err := io.ReadAll(req.Body)
+				if err != nil {
+					http.Error(rw, err.Error(), http.StatusInternalServerError)
+					return
+				}
+				requestBody = body
+
+				// Check if the request is compressed
+				isCompressed = req.Header.Get("Content-Encoding") == "gzip"
+
+				// If compressed, try to decompress to verify it's valid gzip
+				if isCompressed {
+					gzReader, err := gzip.NewReader(bytes.NewReader(body))
+					if err != nil {
+						http.Error(rw, "Invalid gzip: "+err.Error(), http.StatusBadRequest)
+						return
+					}
+					defer gzReader.Close()
+
+					_, err = io.ReadAll(gzReader)
+					if err != nil {
+						http.Error(rw, "Failed to decompress: "+err.Error(), http.StatusBadRequest)
+						return
+					}
+				}
+
+				rw.WriteHeader(http.StatusOK)
+				fmt.Fprint(rw, `{}`)
+			}))
+			defer server.Close()
+
+			ctx := context.Background()
+			client := api.NewClient(logger.Discard, api.Config{
+				Endpoint:        server.URL,
+				Token:           "test-token",
+				GzipAPIRequests: tt.gzipAPIRequests,
+			})
+
+			// Make a request that will have a body (pipeline upload)
+			testPipeline := map[string]interface{}{
+				"steps": []interface{}{
+					map[string]interface{}{
+						"command": "echo hello",
+					},
+				},
+			}
+
+			_, err := client.UploadPipeline(ctx, "test-job-id", &api.PipelineChange{
+				UUID:     "test-uuid",
+				Pipeline: testPipeline,
+			})
+
+			if err != nil {
+				t.Fatalf("UploadPipeline failed: %v", err)
+			}
+
+			// Verify compression behavior matches expectation
+			if isCompressed != tt.expectCompressed {
+				t.Errorf("Expected compressed=%v, got compressed=%v", tt.expectCompressed, isCompressed)
+			}
+
+			// Verify we received a non-empty body
+			if len(requestBody) == 0 {
+				t.Error("Expected non-empty request body")
+			}
+		})
+	}
+}
+
 func authToken(req *http.Request) string {
 	return strings.TrimPrefix(req.Header.Get("Authorization"), "Token ")
 }

clicommand/agent_start.go

@@ -185,11 +185,12 @@ type AgentStartConfig struct {
 	NoMultipartArtifactUpload bool   `cli:"no-multipart-artifact-upload"`
 
 	// API config
-	DebugHTTP bool   `cli:"debug-http"`
-	TraceHTTP bool   `cli:"trace-http"`
-	Token     string `cli:"token" validate:"required"`
-	Endpoint  string `cli:"endpoint" validate:"required"`
-	NoHTTP2   bool   `cli:"no-http2"`
+	DebugHTTP       bool   `cli:"debug-http"`
+	TraceHTTP       bool   `cli:"trace-http"`
+	GzipAPIRequests bool   `cli:"gzip-api-requests"`
+	Token           string `cli:"token" validate:"required"`
+	Endpoint        string `cli:"endpoint" validate:"required"`
+	NoHTTP2         bool   `cli:"no-http2"`
 
 	// Deprecated
 	NoSSHFingerprintVerification bool     `cli:"no-automatic-ssh-fingerprint-verification" deprecated-and-renamed-to:"NoSSHKeyscan"`
@@ -742,6 +743,7 @@ var AgentStartCommand = cli.Command{
 		NoHTTP2Flag,
 		DebugHTTPFlag,
 		TraceHTTPFlag,
+		GzipAPIRequestsFlag,
 
 		// Other shared flags
 		RedactedVars,

clicommand/global.go

@@ -128,6 +128,12 @@ var (
 		EnvVar: "BUILDKITE_AGENT_EXPERIMENT",
 	}
 
+	GzipAPIRequestsFlag = cli.BoolFlag{
+		Name:   "gzip-api-requests",
+		Usage:  "Enable gzip compression for API request bodies",
+		EnvVar: "BUILDKITE_GZIP_API_REQUESTS",
+	}
+
 	RedactedVars = cli.StringSliceFlag{
 		Name:   "redacted-vars",
 		Usage:  "Pattern of environment variable names containing sensitive values",
@@ -172,6 +178,7 @@ type APIConfig struct {
 	TraceHTTP        bool   `cli:"trace-http"`
 	Endpoint         string `cli:"endpoint" validate:"required"`
 	NoHTTP2          bool   `cli:"no-http2"`
+	GzipAPIRequests  bool   `cli:"gzip-api-requests"`
 }
 
 func globalFlags() []cli.Flag {
@@ -191,6 +198,7 @@ func apiFlags() []cli.Flag {
 		NoHTTP2Flag,
 		DebugHTTPFlag,
 		TraceHTTPFlag,
+		GzipAPIRequestsFlag,
 	}
 }
 
@@ -355,6 +363,11 @@ func loadAPIClientConfig(cfg any, tokenField string) api.Config {
 		conf.DisableHTTP2 = noHTTP2.(bool)
 	}
 
+	gzipAPIRequests, err := reflections.GetField(cfg, "GzipAPIRequests")
+	if err == nil {
+		conf.GzipAPIRequests = gzipAPIRequests.(bool)
+	}
+
 	return conf
 }