Skip to content

pycloudlib: increase a variety of retry frequencies #115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
OddBloke merged 4 commits into from
Feb 16, 2021
Merged

pycloudlib: increase a variety of retry frequencies #115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8933f9f
instance: only retry for 10 minutes in _ssh_connect
OddBloke Feb 5, 2021
5d1b1cc
instance: retry more frequently in _ssh_connect
OddBloke Feb 5, 2021
5d927b2
instance: retry more frequently in _wait_for_execute
OddBloke Feb 5, 2021
bd7cdc0
lxd/instance: retry more frequently in .ip
OddBloke Feb 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 13 additions & 10 deletions pycloudlib/instance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,10 @@ def _ssh_connect(self):
except SSHException:
pass

retries = 60
start = time.time()
end = start + 600
last_exception = None
while retries:
while True:
try:
client.connect(
username=self.username,
Expand All @@ -328,13 +329,15 @@ def _ssh_connect(self):
except (ConnectionRefusedError, AuthenticationException,
BadHostKeyException, ConnectionResetError, SSHException,
OSError) as e:
self._log.info(
"%s\nRetrying ssh connection %d more time(s) to %s@%s:%s",
e, retries, self.username, self.ip, self.port
)
last_exception = e
retries -= 1
time.sleep(10)
if time.time() > end:
break
self._log.info(
"%s\nRetrying SSH connection to %s@%s:%s (%ds left)",
last_exception, self.username, self.ip, self.port,
end - time.time()
)
time.sleep(1)

self._log.error('Failed ssh connection to %s@%s:%s after 10 minutes',
self.username, self.ip, self.port)
Expand Down Expand Up @@ -372,15 +375,15 @@ def _wait_for_execute(self):
test_instance_command = "whoami"
result = self.execute(test_instance_command)
if result.failed:
retries = 10
retries = 100
while retries:
result = self.execute(test_instance_command)

if result.ok:
break

retries -= 1
time.sleep(10)
time.sleep(1)

if result.failed:
raise OSError(
Expand Down
4 changes: 2 additions & 2 deletions pycloudlib/lxd/instance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def ip(self):
IP address assigned to instance.

"""
retries = 5
retries = 150

while retries != 0:
command = 'lxc list {} -c 4 --format csv'.format(self.name)
Expand All @@ -83,7 +83,7 @@ def ip(self):
break

retries -= 1
time.sleep(20)
time.sleep(1)

ip_address = result.split()[0]
return ip_address
Expand Down
4 changes: 2 additions & 2 deletions pycloudlib/tests/test_instance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,14 @@ def test_wait_execute_failure(
expected_msg = "{}\n{}".format(
"Instance can't be reached", "Failed to execute whoami command"
)
expected_call_args = [mock.call("whoami")] * 11
expected_call_args = [mock.call("whoami")] * 101

with pytest.raises(OSError) as excinfo:
instance.wait()

assert expected_msg == str(excinfo.value)
assert expected_call_args == m_execute.call_args_list
assert m_sleep.call_count == 10
assert m_sleep.call_count == 100


class TestWaitForCloudinit:
Expand Down