Feature request #4715: make ssh connect timeout configurable

CHANGELOG.rst

@@ -6,7 +6,9 @@ in development
 
 Added
 ~~~~~
-
+* Add support for a configurable connect timeout for SSH connections as requested in #4715
+  by adding the new configuration parameter ``ssh_connect_timeout`` to the ``ssh_runner``
+  group in st2.conf.
 * Add support for blacklisting / whitelisting hosts to the HTTP runner by adding new
   ``url_hosts_blacklist`` and ``url_hosts_whitelist`` runner attribute. (new feature)
   #4757

st2common/st2common/config.py

@@ -421,7 +421,10 @@ def register_opts(ignore_errors=False):
             help='Use the .ssh/config file. Useful to override ports etc.'),
         cfg.StrOpt(
             'ssh_config_file_path', default='~/.ssh/config',
-            help='Path to the ssh config file.')
+            help='Path to the ssh config file.'),
+        cfg.IntOpt(
+            'ssh_connect_timeout', default=60,
+            help='Max time in seconds to establish the SSH connection.')
     ]
 
     do_register_opts(ssh_runner_opts, group='ssh_runner')

st2common/st2common/runners/paramiko_ssh.py

@@ -49,7 +49,7 @@ class SSHCommandTimeoutError(Exception):
     Exception which is raised when an SSH command times out.
     """
 
-    def __init__(self, cmd, timeout, stdout=None, stderr=None):
+    def __init__(self, cmd, timeout, ssh_connect_timeout, stdout=None, stderr=None):
         """
         :param stdout: Stdout which was consumed until the timeout occured.
         :type stdout: ``str``
@@ -59,14 +59,16 @@ def __init__(self, cmd, timeout, stdout=None, stderr=None):
         """
         self.cmd = cmd
         self.timeout = timeout
+        self.ssh_connect_timeout = ssh_connect_timeout
         self.stdout = stdout
         self.stderr = stderr
-        self.message = 'Command didn\'t finish in %s seconds' % (timeout)
+        self.message = 'Command didn\'t finish in %s seconds or the SSH connection did not succeed in %s seconds' % \
+                        (timeout, ssh_connect_timeout)
         super(SSHCommandTimeoutError, self).__init__(self.message)
 
     def __repr__(self):
-        return ('<SSHCommandTimeoutError: cmd="%s",timeout=%s)>' %
-                (self.cmd, self.timeout))
+        return ('<SSHCommandTimeoutError: cmd="%s", timeout=%s, ssh_connect_timeout=%s)>' %
+                (self.cmd, self.timeout, self.ssh_connect_timeout))
 
     def __str__(self):
         return self.message
@@ -83,9 +85,6 @@ class ParamikoSSHClient(object):
     # How long to sleep while waiting for command to finish to prevent busy waiting
     SLEEP_DELAY = 0.2
 
-    # Connect socket timeout
-    CONNECT_TIMEOUT = 60
-
     def __init__(self, hostname, port=DEFAULT_SSH_PORT, username=None, password=None,
                  bastion_host=None, key_files=None, key_material=None, timeout=None,
                  passphrase=None, handle_stdout_line_func=None, handle_stderr_line_func=None):
@@ -105,17 +104,23 @@ def __init__(self, hostname, port=DEFAULT_SSH_PORT, username=None, password=None
         self.username = username
         self.password = password
         self.key_files = key_files
-        self.timeout = timeout or ParamikoSSHClient.CONNECT_TIMEOUT
+        self.timeout = timeout
         self.key_material = key_material
         self.bastion_host = bastion_host
         self.passphrase = passphrase
+        self.ssh_connect_timeout = cfg.CONF.ssh_runner.ssh_connect_timeout
         self._handle_stdout_line_func = handle_stdout_line_func
         self._handle_stderr_line_func = handle_stderr_line_func
 
         self.ssh_config_file = os.path.expanduser(
             cfg.CONF.ssh_runner.ssh_config_file_path or
             '~/.ssh/config'
         )
+
+        if self.timeout and self.ssh_connect_timeout > self.timeout - 2:
+            # the connect timeout should not be greater than the action timeout
+            self.ssh_connect_timeout = self.timeout - 2
+
         self.logger = logging.getLogger(__name__)
 
         self.client = None
@@ -415,8 +420,9 @@ def run(self, cmd, timeout=None, quote=False, call_line_handler_func=False):
 
                 stdout = sanitize_output(stdout.getvalue(), uses_pty=uses_pty)
                 stderr = sanitize_output(stderr.getvalue(), uses_pty=uses_pty)
-                raise SSHCommandTimeoutError(cmd=cmd, timeout=timeout, stdout=stdout,
-                                             stderr=stderr)
+                raise SSHCommandTimeoutError(cmd=cmd, timeout=timeout, 
+                                             ssh_connect_timeout=self.ssh_connect_timeout,
+                                             stdout=stdout, stderr=stderr)
 
             stdout_data = self._consume_stdout(chan=chan,
                                                call_line_handler_func=call_line_handler_func)
@@ -633,7 +639,7 @@ def _connect(self, host, socket=None):
         conninfo = {'hostname': host,
                     'allow_agent': False,
                     'look_for_keys': False,
-                    'timeout': self.timeout}
+                    'timeout': self.ssh_connect_timeout}
 
         ssh_config_file_info = {}
         if cfg.CONF.ssh_runner.use_ssh_config:
@@ -702,7 +708,7 @@ def _connect(self, host, socket=None):
             conninfo['look_for_keys'] = True
 
         extra = {'_hostname': host, '_port': self.port,
-                 '_username': self.username, '_timeout': self.timeout}
+                 '_username': self.username, '_timeout': self.ssh_connect_timeout}
         self.logger.debug('Connecting to server', extra=extra)
 
         self.socket = socket or ssh_config_file_info.get('sock', None)