From f218ddb03e66ed03101f899be6f0638e11d51ef0 Mon Sep 17 00:00:00 2001
From: Goutham Veeramachaneni <gouthamve@gmail.com>
Date: Tue, 11 Jun 2019 01:46:42 +0530
Subject: [PATCH] Make sure we send a 400 for large responses.

While this is not super ideal, we will still catch 99% of the cases
through this.

Fixes: https://github.com/cortexproject/cortex/issues/1450

Signed-off-by: Goutham Veeramachaneni <gouthamve@gmail.com>
---
 pkg/querier/frontend/worker.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/pkg/querier/frontend/worker.go b/pkg/querier/frontend/worker.go
index cf9326f9fa9..da1cdb551b0 100644
--- a/pkg/querier/frontend/worker.go
+++ b/pkg/querier/frontend/worker.go
@@ -3,6 +3,7 @@ package frontend
 import (
 	"context"
 	"flag"
+	"fmt"
 	"net/http"
 	"sync"
 	"time"
@@ -202,6 +203,17 @@ func (w *worker) process(ctx context.Context, c Frontend_ProcessClient) error {
 			}
 		}
 
+		if len(response.Body) >= w.cfg.GRPCClientConfig.MaxSendMsgSize {
+			errMsg := fmt.Sprintf("the response is larger than the max (%d vs %d)", len(response.Body), w.cfg.GRPCClientConfig.MaxSendMsgSize)
+
+			// This makes sure the request is not retried, else a 500 is sent and we retry the large query again.
+			response = &httpgrpc.HTTPResponse{
+				Code: http.StatusRequestEntityTooLarge,
+				Body: []byte(errMsg),
+			}
+			level.Error(w.log).Log("msg", "error processing query", "err", errMsg)
+		}
+
 		if err := c.Send(&ProcessResponse{
 			HttpResponse: response,
 		}); err != nil {