Skip to content

Fix agent stuck when using output container #135

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
GBrawl merged 1 commit into from
Aug 21, 2020
Merged

Fix agent stuck when using output container #135

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 25 additions & 22 deletions stackl/agent/agent/kubernetes/handlers/base_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,10 @@
from typing import Dict, List

import stackl_client
from kubernetes import client, config
from kubernetes.client.rest import ApiException

from agent.kubernetes.outputs.output import Output
from agent.kubernetes.secrets.conjur_secret_handler import ConjurSecretHandler
from kubernetes import client, config
from kubernetes.client.rest import ApiException


def create_job_object(name: str,
Expand Down Expand Up @@ -271,17 +270,16 @@ def __init__(self, invoc):
self.stackl_namespace = os.environ['STACKL_NAMESPACE']
self.service_account = os.environ['SERVICE_ACCOUNT']

def wait_for_job(self, job_name: str, namespace: str):
ready = False
api_response = None
while not ready:
def wait_for_job(self, job_pod_name: str, namespace: str):
while True:
sleep(5)
api_response = self._api_instance.read_namespaced_job(
job_name, namespace)
logging.debug(f"Api response: {api_response}")
if api_response.status.failed or api_response.status.succeeded:
ready = True
return api_response
api_response = self._api_instance_core.read_namespaced_pod_status(job_pod_name, namespace)
print(api_response)
for cs in api_response.status.container_statuses:
if cs.name == "jobcontainer" and cs.state.terminated is not None:
if cs.state.terminated.reason == "Error":
return False
return True

def handle(self):
logging.info(f"Invocation: {self._invoc}")
Expand All @@ -295,7 +293,7 @@ def handle(self):
"stackl.io/stack-instance": self._invoc.stack_instance,
"stackl.io/service": self._invoc.service,
"stackl.io/functional-requirement":
self._invoc.functional_requirement
self._invoc.functional_requirement
}
body, cms = create_job_object(name=name,
container_image=container_image,
Expand All @@ -309,7 +307,7 @@ def handle(self):
service_account=self.service_account,
output=self._output,
labels=labels)
print(body)

try:
for cm in cms:
self._api_instance_core.create_namespaced_config_map(
Expand All @@ -321,9 +319,15 @@ def handle(self):
f"Exception when calling BatchV1Api->create_namespaced_job: {e}\n"
)
logging.debug("job created")
api_response = self.wait_for_job(body.metadata.name,
self.stackl_namespace)
if api_response.status.succeeded == 1:
sleep(5)
job_pods = self._api_instance_core.list_namespaced_pod(
self.stackl_namespace,
label_selector=f"job-name={created_job.metadata.name}")
job_succeeded = self.wait_for_job(job_pods.items[0].metadata.name,
self.stackl_namespace)

if job_succeeded:
print("job succeeded")
try:
for cm in cms:
self._api_instance_core.delete_namespaced_config_map(
Expand All @@ -338,11 +342,10 @@ def handle(self):
)
return 0, ""
else:
job_pods = self._api_instance_core.list_namespaced_pod(
self.stackl_namespace,
label_selector=f"job-name={created_job.metadata.name}")
print("job failed")
logs = self._api_instance_core.read_namespaced_pod_log(
job_pods.items[0].metadata.name, self.stackl_namespace)
job_pods.items[0].metadata.name, self.stackl_namespace, container="jobcontainer")
print(logs)
return 1, logs

@property
Expand Down