Adding back set_connect_fail for generic I/O error

proxy/http/HttpSM.cc

@@ -1919,9 +1919,6 @@ HttpSM::state_http_server_open(int event, void *data)
 
       server_entry->write_vio = server_txn->do_io_write(this, nbytes, server_txn->get_remote_reader());
 
-      // Pre-emptively set a server connect failure that will be cleared once a WRITE_READY is received from origin or
-      // bytes are received back
-      t_state.set_connect_fail(EIO);
     } else { // in the case of an intercept plugin don't to the connect timeout change
       SMDebug("http", "not setting handler for TCP handshake");
       handle_http_server_open();
@@ -7626,6 +7623,9 @@ HttpSM::set_next_state()
   }
 
   case HttpTransact::SM_ACTION_ORIGIN_SERVER_OPEN: {
+    // Pre-emptively set a server connect failure that will be cleared once a WRITE_READY is received from origin or
+    // bytes are received back
+    t_state.set_connect_fail(EIO);
     HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::state_http_server_open);
 
     // We need to close the previous attempt

tests/gold_tests/dns/dns_host_down.test.py

@@ -0,0 +1,102 @@
+'''
+Verify ATS handles down origin servers with domain cached correctly.
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from ports import get_port
+import os
+
+Test.Summary = '''
+Verify ATS handles down origin servers with cached domain correctly.
+'''
+
+
+class DownCachedOriginServerTest:
+    replay_file = "replay/server_down.replay.yaml"
+
+    def __init__(self):
+        """Initialize the Test processes for the test runs."""
+        self._server = Test.MakeVerifierServerProcess("server", DownCachedOriginServerTest.replay_file)
+        self._configure_trafficserver()
+
+    def _configure_trafficserver(self):
+        """Configure Traffic Server."""
+        self._ts = Test.MakeATSProcess("ts", enable_cache=False)
+
+        self._ts.Disk.remap_config.AddLine(
+            f"map / http://resolve.this.com:{self._server.Variables.http_port}/"
+        )
+
+        self._ts.Disk.records_config.update({
+            'proxy.config.diags.debug.enabled': 1,
+            'proxy.config.diags.debug.tags': 'hostdb|dns|http|socket',
+            'proxy.config.http.connect_attempts_max_retries': 0,
+            'proxy.config.http.connect_attempts_rr_retries': 0,
+            'proxy.config.hostdb.fail.timeout': 10,
+            'proxy.config.dns.resolv_conf': 'NULL',
+            'proxy.config.hostdb.ttl_mode': 1,
+            'proxy.config.hostdb.timeout': 2,
+            'proxy.config.hostdb.lookup_timeout': 2,
+            'proxy.config.http.transaction_no_activity_timeout_in': 2,
+            'proxy.config.http.connect_attempts_timeout': 2,
+            'proxy.config.hostdb.host_file.interval': 1,
+            'proxy.config.hostdb.host_file.path': os.path.join(Test.TestDirectory, "hosts_file"),
+        })
+
+    # Even when the origin server is down, SM will return a hit-fresh domain from HostDB.
+    # After request has failed, SM should mark the IP as down
+    def _test_host_mark_down(self):
+        tr = Test.AddTestRun()
+
+        tr.Processes.Default.StartBefore(self._server)
+        tr.Processes.Default.StartBefore(self._ts)
+
+        tr.AddVerifierClientProcess(
+            "client-1",
+            DownCachedOriginServerTest.replay_file,
+            http_ports=[self._ts.Variables.port],
+            other_args='--keys 1')
+
+    # After host has been marked down from previous test, HostDB should not return
+    # the host as available and DNS lookup should fail.
+    def _test_host_unreachable(self):
+        tr = Test.AddTestRun()
+
+        tr.AddVerifierClientProcess(
+            "client-2",
+            DownCachedOriginServerTest.replay_file,
+            http_ports=[self._ts.Variables.port],
+            other_args='--keys 2')
+
+    # Verify error log marking host down exists
+    def _test_error_log(self):
+        tr = Test.AddTestRun()
+        tr.Processes.Default.Command = (
+            os.path.join(Test.Variables.AtsTestToolsDir, 'condwait') + ' 60 1 -f ' +
+            os.path.join(self._ts.Variables.LOGDIR, 'error.log')
+        )
+
+        self._ts.Disk.error_log.Content = Testers.ContainsExpression("/dns/mark/down' marking down", "host should be marked down")
+        self._ts.Disk.error_log.Content = Testers.ContainsExpression(
+            "DNS Error: no valid server http://resolve.this.com", "DNS lookup should fail")
+
+    def run(self):
+        self._test_host_mark_down()
+        self._test_host_unreachable()
+        self._test_error_log()
+
+
+DownCachedOriginServerTest().run()

tests/gold_tests/dns/hosts_file

@@ -0,0 +1,17 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+0.0.0.1 resolve.this.com

tests/gold_tests/dns/replay/server_down.replay.yaml

@@ -0,0 +1,61 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+meta:
+  version: "1.0"
+
+sessions:
+- transactions:
+  - client-request:
+      # Delay to allow hostdb to sync external host file
+      delay: 2s
+      method: "GET"
+      version: "1.1"
+      url: /dns/mark/down
+      headers:
+        fields:
+        - [ Host, example.com ]
+        - [ X-Request, request ]
+        - [ uuid, 1 ]
+
+    # Shouldn't be reached since server IP is unreachable
+    server-response:
+      status: 200
+
+    # Returns 502 since server connection is unreachable
+    proxy-response:
+      status: 502
+
+  - client-request:
+      # Delay to allow hostdb to sync external host file
+      delay: 2s
+      method: "GET"
+      version: "1.1"
+      url: /dns/unreachable
+      headers:
+        fields:
+        - [ Host, example.com ]
+        - [ X-Request, request ]
+        - [ uuid, 2 ]
+
+    # Shouldn't be reached since server IP is unreachable
+    server-response:
+      status: 200
+
+    # Previous request marked host down so DNS lookup returns unsuccessful
+    # 500 response indicates no valid server
+    proxy-response:
+      status: 500