After parents are exhausted, serve stale content from child

proxy/http/HttpTransact.cc

@@ -3758,7 +3758,9 @@ HttpTransact::handle_response_from_parent(State *s)
     return CallOSDNSLookup(s);
     break;
   case HOST_NONE:
-    handle_parent_died(s);
+    // Check if content can be served from cache
+    s->current.request_to = PARENT_PROXY;
+    handle_server_connection_not_open(s);
     break;
   default:
     // This handles:
@@ -4047,7 +4049,17 @@ HttpTransact::handle_server_connection_not_open(State *s)
     TxnDebug("http_trans", "[hscno] serving stale doc to client");
     build_response_from_cache(s, HTTP_WARNING_CODE_REVALIDATION_FAILED);
   } else {
-    handle_server_died(s);
+    switch (s->current.request_to) {
+    case PARENT_PROXY:
+      handle_parent_died(s);
+      break;
+    case ORIGIN_SERVER:
+      handle_server_died(s);
+      break;
+    default:
+      ink_assert(!("s->current.request_to is not P.P. or O.S. - hmmm."));
+      break;
+    }
     s->next_action = SM_ACTION_SEND_ERROR_CACHE_NOOP;
   }
 

tests/gold_tests/pluginTest/test_hooks/body_buffer.test.py

@@ -99,10 +99,10 @@ def setupTS(self):
         })
 
         self._ts.Streams.stderr = Testers.ContainsExpression(
-            rf"request_buffer_plugin gets the request body with length\[{self.content_length_size}\]",
+            fr"request_buffer_plugin gets the request body with length\[{self.content_length_size}\]",
             "Verify that the plugin parsed the content-length request body data.")
         self._ts.Streams.stderr += Testers.ContainsExpression(
-            rf"request_buffer_plugin gets the request body with length\[{self.encoded_chunked_size}\]",
+            fr"request_buffer_plugin gets the request body with length\[{self.encoded_chunked_size}\]",
             "Verify that the plugin parsed the chunked request body.")
 
     def run(self):

tests/gold_tests/proxy_protocol/gold/proxy_serve_stale.gold

@@ -0,0 +1,28 @@
+``
+> GET / HTTP/1.1
+> Host: ``
+> User-Agent: curl/``
+> Accept: */*
+``
+< HTTP/1.1 200 OK
+< Server: ATS/10.0.0
+< Accept-Ranges: bytes
+< Content-Length: 6
+< Cache-Control: public, max-age=5
+< Age: ``
+< Date: ``
+< Connection: keep-alive
+< Warning: ``
+``
+> GET / HTTP/1.1
+> Host: ``
+> User-Agent: curl/``
+> Accept: */*
+``
+< HTTP/1.1 502 Next Hop Connection Failed
+< Date: ``
+< Connection: keep-alive
+< Server: ATS/10.0.0
+< Cache-Control: ``
+< Content-Length: ``
+``

tests/gold_tests/proxy_protocol/proxy_serve_stale.test.py

@@ -0,0 +1,70 @@
+'''
+Test child proxy serving stale content when parents are exhausted
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+Test.ContinueOnFail = True
+# Set up hierarchical caching processes
+ts_child = Test.MakeATSProcess("ts_child")
+ts_parent = "unknown.com:8080"
+server = Test.MakeOriginServer("server")
+
+Test.testName = "STALE"
+
+# Request from client
+request_header = {"headers":
+                  "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n",
+                  "timestamp": "1469733493.993",
+                  "body": ""}
+# Expected response from the origin server
+response_header = {"headers":
+                   "HTTP/1.1 200 OK\r\nConnection: close\r\nCache-Control: max-age=5,public\r\n\r\n",
+                   "timestamp": "1469733493.993",
+                   "body": "CACHED"}
+
+# add request/response
+server.addResponse("sessionlog.log", request_header, response_header)
+
+# Config child proxy to route to parent proxy
+ts_child.Disk.records_config.update({
+    'proxy.config.diags.debug.enabled': 1,
+    'proxy.config.diags.debug.tags': 'http|remap|parent|host',
+    'proxy.config.http.cache.max_stale_age': 90,
+})
+ts_child.Disk.parent_config.AddLine(
+    f'dest_domain=. parent="{ts_parent}" round_robin=consistent_hash go_direct=false'
+)
+ts_child.Disk.remap_config.AddLine(
+    f'map http://localhost:{ts_child.Variables.port} http://localhost:{server.Variables.Port}'
+)
+
+stale_output = "HTTP/1.1 200 OK\nServer: ATS/10.0.0\nAccept-Ranges: bytes\nContent-Length: 6\nCache-Control: public, max-age=5\n\nCACHED"
+
+curl_request = (
+    f'curl -X PUSH -d "{stale_output}" "http://localhost:{ts_child.Variables.port}";'
+    f'sleep 10; curl -s -v http://localhost:{ts_child.Variables.port};'  # hit-stale, serve stale with warning
+    f'sleep 90; curl -v http://localhost:{ts_child.Variables.port}'  # max_stale_age expires, can't reach parent
+)
+
+# Test case for when parent server is down but child proxy can serve cache object
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = curl_request
+tr.Processes.Default.ReturnCode = 0
+tr.Processes.Default.StartBefore(server)
+tr.Processes.Default.StartBefore(ts_child)
+tr.Processes.Default.Streams.stderr = "gold/proxy_serve_stale.gold"
+tr.StillRunningAfter = ts_child