Add hooks for outbound TLS start and close.

doc/developer-guide/api/functions/TSNetInvokingGet.en.rst

@@ -0,0 +1,37 @@
+.. Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed
+   with this work for additional information regarding copyright
+   ownership.  The ASF licenses this file to you under the Apache
+   License, Version 2.0 (the "License"); you may not use this file
+   except in compliance with the License.  You may obtain a copy of
+   the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+   implied.  See the License for the specific language governing
+   permissions and limitations under the License.
+
+.. include:: ../../../common.defs
+
+.. default-domain:: c
+
+TSNetInvokingContGet
+********************
+
+Synopsis
+========
+
+`#include <ts/ts.h>`
+
+.. function:: TSCont TSNetInvokingContGet(TSVConn conn)
+
+.. function:: TSHttpTxn TSNetInvokingTxnGet(TSVConn conn)
+
+Description
+===========
+
+The TSNetInvokingContGet and TSNetInvokingTxnGet returns the continuation or transaction that
+started the connection request associated with the conn parameter.

doc/developer-guide/api/types/TSHttpHookID.en.rst

@@ -72,8 +72,12 @@ Enumeration Members
 
 .. c:macro:: TSHttpHookID TS_VCONN_START_HOOK
 
+.. c:macro:: TSHttpHookID TS_VCONN_OUTBOUND_START_HOOK
+
 .. c:macro:: TSHttpHookID TS_VCONN_CLOSE_HOOK
 
+.. c:macro:: TSHttpHookID TS_VCONN_OUTBOUND_CLOSE_HOOK
+
 .. c:macro:: TSHttpHookID TS_SSL_SNI_HOOK
 
 .. c:macro:: TSHttpHookID TS_SSL_CERT_HOOK

doc/developer-guide/plugins/hooks-and-transactions/ssl-hooks.en.rst

@@ -58,7 +58,7 @@ callback or from another piece of code.
 TS_VCONN_CLOSE_HOOK
 ------------------------
 
-This hook is invoked after the SSL handshake is done and when the IO is closing. The TSVConnArgs should be cleaned up here.
+This hook is invoked after the SSL handshake is done and when the IO is closing. The TSVConnArgs should be cleaned up here. A callback at this point must reenable.
 
 TS_SSL_SERVERNAME_HOOK
 ----------------------
@@ -107,11 +107,27 @@ Processing will continue regardless of whether the hook callback executes
 :c:func:`TSSslVConnReenable()` since the openssl implementation does not allow
 for pausing processing during the certificate verify callback.
 
+TS_VCONN_OUTBOUND_START_HOOK
+----------------------------
+
+This hook is invoked after ATS has connected to the upstream server and before the SSL handshake has started.  This gives the plugin the option of 
+overriding the default SSL connection options on the SSL object.
+
+In theory this hook could apply and be useful for non-SSL connections as well, but at this point this hook is only called in the SSL sequence.
+
+The TLS handshake processing will not proceed until :c:func:`TSSslVConnReenable()` is called either from within the hook
+callback or from another piece of code.
+
+TS_VCONN_OUTBOUND_CLOSE_HOOK
+-----------------------------
+
+This hook is invoked after the SSL handshake is done and right before the outbound connection closes.  A callback at this point must reenable.
+
 TLS Hook State Diagram
 ----------------------
 
 .. graphviz::
-   :alt: TLS Hook State Diagram
+   :alt: TLS Inbound Hook State Diagram
 
    digraph tls_hook_state_diagram{
      HANDSHAKE_HOOKS_PRE -> TS_VCONN_START_HOOK;
@@ -144,4 +160,14 @@ TLS Hook State Diagram
      HANDSHAKE_HOOKS_DONE [shape=box];
    }
 
+.. graphviz::
+   :alt: TLS Outbound Hook State Diagram
+
+   digraph tls_hook_state_diagram{
+     HANDSHAKE_HOOKS_OUTBOUND_PRE -> HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE;
+     HANDSHAKE_HOOKS_PRE_INVOKE -> TSSslVConnReenable;
+     TSSslVConnReenable -> HANDSHAKE_HOOKS_OUTBOUND_PRE;
+     HANDSHAKE_HOOKS_OUTBOUND_PRE -> HANDSHAKE_HOOKS_DONE;
+     HANDSHAKE_HOOKS_DONE -> HANDSHAKE_HOOKS_OUTBOUND_CLOSE;
+   }
 

include/ts/apidefs.h.in

@@ -293,8 +293,10 @@ typedef enum {
   TS_SSL_VERIFY_SERVER_HOOK = TS_SSL_SERVER_VERIFY_HOOK,
   TS_SSL_VERIFY_CLIENT_HOOK,
   TS_SSL_SESSION_HOOK,
-  TS_SSL_LAST_HOOK = TS_SSL_SESSION_HOOK,
-  TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK = 24,
+  TS_VCONN_OUTBOUND_START_HOOK,
+  TS_VCONN_OUTBOUND_CLOSE_HOOK,
+  TS_SSL_LAST_HOOK = TS_VCONN_OUTBOUND_CLOSE_HOOK,
+  TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK,
   TS_HTTP_LAST_HOOK
 } TSHttpHookID;
 
@@ -462,9 +464,10 @@ typedef enum {
   TS_EVENT_INTERNAL_60202                       = 60202,
   TS_EVENT_SSL_CERT                             = 60203,
   TS_EVENT_SSL_SERVERNAME                       = 60204,
-  TS_EVENT_SSL_SERVER_VERIFY_HOOK               = 60205,
   TS_EVENT_SSL_VERIFY_SERVER                    = 60205,
-  TS_EVENT_SSL_VERIFY_CLIENT                    = 60206
+  TS_EVENT_SSL_VERIFY_CLIENT                    = 60206,
+  TS_EVENT_VCONN_OUTBOUND_START                 = 60207,
+  TS_EVENT_VCONN_OUTBOUND_CLOSE                 = 60208
 } TSEvent;
 #define TS_EVENT_HTTP_READ_REQUEST_PRE_REMAP TS_EVENT_HTTP_PRE_REMAP /* backwards compat */
 

include/ts/ts.h

@@ -1742,6 +1742,16 @@ tsapi TSAction TSNetConnect(
   struct sockaddr const *to /**< Address to which to connect. */
 );
 
+/**
+ * Retrieves the continuation associated with creating the TSVConn
+ */
+tsapi TSCont TSNetInvokingContGet(TSVConn conn);
+
+/**
+ * Retrieves the transaction associated with creating the TSVConn
+ */
+tsapi TSHttpTxn TSNetInvokingTxnGet(TSVConn conn);
+
 tsapi TSAction TSNetAccept(TSCont contp, int port, int domain, int accept_threads);
 
 /**

iocore/net/P_SSLNetVConnection.h

@@ -261,6 +261,15 @@ class SSLNetVConnection : public UnixNetVConnection
       }
       break;
 
+    case HANDSHAKE_HOOKS_OUTBOUND_PRE:
+    case HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE:
+      if (eventId == TS_EVENT_VCONN_OUTBOUND_START) {
+        if (curHook) {
+          retval = true;
+        }
+      }
+      break;
+
     case HANDSHAKE_HOOKS_DONE:
       retval = true;
       break;
@@ -343,6 +352,8 @@ class SSLNetVConnection : public UnixNetVConnection
     HANDSHAKE_HOOKS_CERT_INVOKE,
     HANDSHAKE_HOOKS_CLIENT_CERT,
     HANDSHAKE_HOOKS_CLIENT_CERT_INVOKE,
+    HANDSHAKE_HOOKS_OUTBOUND_PRE,
+    HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE,
     HANDSHAKE_HOOKS_DONE
   } sslHandshakeHookState = HANDSHAKE_HOOKS_PRE;
 

iocore/net/SSLNetVConnection.cc

@@ -878,35 +878,42 @@ SSLNetVConnection::SSLNetVConnection() {}
 void
 SSLNetVConnection::do_io_close(int lerrno)
 {
-  if (this->ssl != nullptr && sslHandShakeComplete) {
-    callHooks(TS_EVENT_VCONN_CLOSE);
-    int shutdown_mode = SSL_get_shutdown(ssl);
-    Debug("ssl-shutdown", "previous shutdown state 0x%x", shutdown_mode);
-    int new_shutdown_mode = shutdown_mode | SSL_RECEIVED_SHUTDOWN;
-
-    if (new_shutdown_mode != shutdown_mode) {
-      // We do not need to sit around and wait for the client's close-notify if
-      // they have not already sent it.  We will still be standards compliant
-      Debug("ssl-shutdown", "new SSL_set_shutdown 0x%x", new_shutdown_mode);
-      SSL_set_shutdown(ssl, new_shutdown_mode);
+  if (this->ssl != nullptr) {
+    if (get_context() == NET_VCONNECTION_OUT) {
+      callHooks(TS_EVENT_VCONN_OUTBOUND_CLOSE);
+    } else {
+      callHooks(TS_EVENT_VCONN_CLOSE);
     }
 
-    // If the peer has already sent a FIN, don't bother with the shutdown
-    // They will just send us a RST for our troubles
-    // This test is not foolproof.  The client's fin could be on the wire
-    // at the same time we send the close-notify.  If so, the client will likely
-    // send RST anyway
-    char c;
-    ssize_t x = recv(this->con.fd, &c, 1, MSG_PEEK);
-    // x < 0 means error.  x == 0 means fin sent
-    bool do_shutdown = (x > 0);
-    if (x < 0) {
-      do_shutdown = (errno == EAGAIN || errno == EWOULDBLOCK);
-    }
-    if (do_shutdown) {
-      // Send the close-notify
-      int ret = SSL_shutdown(ssl);
-      Debug("ssl-shutdown", "SSL_shutdown %s", (ret) ? "success" : "failed");
+    if (sslHandShakeComplete) {
+      int shutdown_mode = SSL_get_shutdown(ssl);
+      Debug("ssl-shutdown", "previous shutdown state 0x%x", shutdown_mode);
+      int new_shutdown_mode = shutdown_mode | SSL_RECEIVED_SHUTDOWN;
+
+      if (new_shutdown_mode != shutdown_mode) {
+        // We do not need to sit around and wait for the client's close-notify if
+        // they have not already sent it.  We will still be standards compliant
+        Debug("ssl-shutdown", "new SSL_set_shutdown 0x%x", new_shutdown_mode);
+        SSL_set_shutdown(ssl, new_shutdown_mode);
+      }
+
+      // If the peer has already sent a FIN, don't bother with the shutdown
+      // They will just send us a RST for our troubles
+      // This test is not foolproof.  The client's fin could be on the wire
+      // at the same time we send the close-notify.  If so, the client will likely
+      // send RST anyway
+      char c;
+      ssize_t x = recv(this->con.fd, &c, 1, MSG_PEEK);
+      // x < 0 means error.  x == 0 means fin sent
+      bool do_shutdown = (x > 0);
+      if (x < 0) {
+        do_shutdown = (errno == EAGAIN || errno == EWOULDBLOCK);
+      }
+      if (do_shutdown) {
+        // Send the close-notify
+        int ret = SSL_shutdown(ssl);
+        Debug("ssl-shutdown", "SSL_shutdown %s", (ret) ? "success" : "failed");
+      }
     }
   }
   // Go on and do the unix socket cleanups
@@ -1366,6 +1373,33 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err)
 
   ink_assert(SSLNetVCAccess(ssl) == this);
 
+  // Initialize properly for a client connection
+  if (sslHandshakeHookState == HANDSHAKE_HOOKS_PRE) {
+    sslHandshakeHookState = HANDSHAKE_HOOKS_OUTBOUND_PRE;
+  }
+
+  // Do outbound hook processing here
+  // Continue on if we are in the invoked state.  The hook has not yet reenabled
+  if (sslHandshakeHookState == HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE) {
+    return SSL_WAIT_FOR_HOOK;
+  }
+
+  // Go do the preaccept hooks
+  if (sslHandshakeHookState == HANDSHAKE_HOOKS_OUTBOUND_PRE) {
+    if (!curHook) {
+      Debug("ssl", "Initialize outbound connect curHook from NULL");
+      curHook = ssl_hooks->get(TS_VCONN_OUTBOUND_START_INTERNAL_HOOK);
+    } else {
+      curHook = curHook->next();
+    }
+    // If no more hooks, carry on
+    if (nullptr != curHook) {
+      sslHandshakeHookState = HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE;
+      ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_OUTBOUND_START, this);
+      return SSL_WAIT_FOR_HOOK;
+    }
+  }
+
   ssl_error = SSLConnect(ssl);
   switch (ssl_error) {
   case SSL_ERROR_NONE:
@@ -1521,6 +1555,9 @@ SSLNetVConnection::reenable(NetHandler *nh)
   case HANDSHAKE_HOOKS_PRE_INVOKE:
     sslHandshakeHookState = HANDSHAKE_HOOKS_PRE;
     break;
+  case HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE:
+    sslHandshakeHookState = HANDSHAKE_HOOKS_OUTBOUND_PRE;
+    break;
   case HANDSHAKE_HOOKS_CERT_INVOKE:
     sslHandshakeHookState = HANDSHAKE_HOOKS_CERT;
     break;
@@ -1553,6 +1590,16 @@ SSLNetVConnection::reenable(NetHandler *nh)
       Debug("ssl", "Reenable preaccept");
       sslHandshakeHookState = HANDSHAKE_HOOKS_PRE_INVOKE;
       ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_START, this);
+    } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_OUTBOUND_PRE) {
+      Debug("ssl", "Reenable outbound connect");
+      sslHandshakeHookState = HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE;
+      ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_OUTBOUND_START, this);
+    } else if (sslHandshakeHookState == HANDSHAKE_HOOKS_DONE) {
+      if (this->get_context() == NET_VCONNECTION_OUT) {
+        ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_OUTBOUND_CLOSE, this);
+      } else {
+        ContWrapper::wrap(nh->mutex.get(), curHook->m_cont, TS_EVENT_VCONN_CLOSE, this);
+      }
     }
     return;
   } else {
@@ -1569,6 +1616,13 @@ SSLNetVConnection::reenable(NetHandler *nh)
     case HANDSHAKE_HOOKS_CERT_INVOKE:
       sslHandshakeHookState = HANDSHAKE_HOOKS_CLIENT_CERT;
       break;
+    case HANDSHAKE_HOOKS_OUTBOUND_PRE:
+    case HANDSHAKE_HOOKS_OUTBOUND_PRE_INVOKE:
+      this->write.triggered = true;
+      this->write.enabled   = true;
+      this->writeReschedule(nh);
+      sslHandshakeHookState = HANDSHAKE_HOOKS_DONE;
+      break;
     case HANDSHAKE_HOOKS_CLIENT_CERT:
     case HANDSHAKE_HOOKS_CLIENT_CERT_INVOKE:
       sslHandshakeHookState = HANDSHAKE_HOOKS_DONE;
@@ -1600,7 +1654,7 @@ SSLNetVConnection::callHooks(TSEvent eventId)
 {
   // Only dealing with the SNI/CERT hook so far.
   ink_assert(eventId == TS_EVENT_SSL_CERT || eventId == TS_EVENT_SSL_SERVERNAME || eventId == TS_EVENT_SSL_VERIFY_SERVER ||
-             eventId == TS_EVENT_SSL_VERIFY_CLIENT || eventId == TS_EVENT_VCONN_CLOSE);
+             eventId == TS_EVENT_SSL_VERIFY_CLIENT || eventId == TS_EVENT_VCONN_CLOSE || eventId == TS_EVENT_VCONN_OUTBOUND_CLOSE);
   Debug("ssl", "callHooks sslHandshakeHookState=%d", this->sslHandshakeHookState);
 
   // Move state if it is appropriate
@@ -1663,8 +1717,21 @@ SSLNetVConnection::callHooks(TSEvent eventId)
     }
   // fallthrough
   case HANDSHAKE_HOOKS_DONE:
+  case HANDSHAKE_HOOKS_OUTBOUND_PRE:
     if (eventId == TS_EVENT_VCONN_CLOSE) {
-      curHook = ssl_hooks->get(TS_VCONN_CLOSE_INTERNAL_HOOK);
+      sslHandshakeHookState = HANDSHAKE_HOOKS_DONE;
+      if (curHook == nullptr) {
+        curHook = ssl_hooks->get(TS_VCONN_CLOSE_INTERNAL_HOOK);
+      } else {
+        curHook = curHook->next();
+      }
+    } else if (eventId == TS_EVENT_VCONN_OUTBOUND_CLOSE) {
+      sslHandshakeHookState = HANDSHAKE_HOOKS_DONE;
+      if (curHook == nullptr) {
+        curHook = ssl_hooks->get(TS_VCONN_OUTBOUND_CLOSE_INTERNAL_HOOK);
+      } else {
+        curHook = curHook->next();
+      }
     }
     break;
   default:
@@ -1675,13 +1742,15 @@ SSLNetVConnection::callHooks(TSEvent eventId)
 
   Debug("ssl", "callHooks iterated to curHook=%p", curHook);
 
+  bool reenabled = true;
+
   this->serverName = const_cast<char *>(SSL_get_servername(this->ssl, TLSEXT_NAMETYPE_host_name));
   if (this->serverName) {
     auto *hs = TunnelMap.find(this->serverName);
     if (hs != nullptr) {
       this->SNIMapping = true;
       this->attributes = HttpProxyPort::TRANSPORT_BLIND_TUNNEL;
-      return EVENT_DONE;
+      return reenabled;
     }
   }
 
@@ -1692,10 +1761,9 @@ SSLNetVConnection::callHooks(TSEvent eventId)
     // we get out of this callback, and then will shuffle
     // over the buffered handshake packets to the O.S.
     // sslHandShakeComplete = 1;
-    return EVENT_DONE;
+    return reenabled;
   }
 
-  bool reenabled = true;
   if (curHook != nullptr) {
     curHook->invoke(eventId, this);
     reenabled =

proxy/InkAPIInternal.h

@@ -282,6 +282,8 @@ typedef enum {
   TS_SSL_VERIFY_SERVER_INTERNAL_HOOK,
   TS_SSL_VERIFY_CLIENT_INTERNAL_HOOK,
   TS_SSL_SESSION_INTERNAL_HOOK,
+  TS_VCONN_OUTBOUND_START_INTERNAL_HOOK,
+  TS_VCONN_OUTBOUND_CLOSE_INTERNAL_HOOK,
   TS_SSL_INTERNAL_LAST_HOOK
 } TSSslHookInternalID;
 

proxy/http/HttpDebugNames.cc

@@ -470,6 +470,10 @@ HttpDebugNames::get_api_hook_name(TSHttpHookID t)
     return "TS_SSL_VERIFY_CLIENT_HOOK";
   case TS_SSL_SESSION_HOOK:
     return "TS_SSL_SESSION_HOOK";
+  case TS_VCONN_OUTBOUND_START_HOOK:
+    return "TS_VCONN_OUTBOUND_START_HOOK";
+  case TS_VCONN_OUTBOUND_CLOSE_HOOK:
+    return "TS_VCONN_OUTBOUND_CLOSE_HOOK";
   }
 
   return "unknown hook";