From 10a5651fca37f6d052b5ffe414adc0b7abd0c89f Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 20 Aug 2020 13:37:03 +0200 Subject: [PATCH 1/2] WIP ansible outputs test --- .../kubernetes/handlers/ansible_handler.py | 26 +++------- .../kubernetes/outputs/ansible_output.py | 11 ++++ stackl/cli/scripts/convert_json_from_spec.py | 52 +++++++++++++------ 3 files changed, 53 insertions(+), 36 deletions(-) diff --git a/stackl/agent/agent/kubernetes/handlers/ansible_handler.py b/stackl/agent/agent/kubernetes/handlers/ansible_handler.py index 7ef1c19e..dd15f775 100644 --- a/stackl/agent/agent/kubernetes/handlers/ansible_handler.py +++ b/stackl/agent/agent/kubernetes/handlers/ansible_handler.py @@ -273,24 +273,16 @@ def parse(self, inventory, loader, path, cache): """ playbook_include_role = """ +- hosts: "{{ pattern }}" + tasks: + - include_role: + name: "{{ ansible_role }}" - hosts: localhost connection: local gather_facts: no tasks: - - include_role: - name: "{{ ansible_role }}" - - set_fact: - output_dict: {} - set_fact: - output_dict: "{{ output_dict | combine(vars) }}" - - set_fact: - output_dict: "{{ output_dict | combine({'environment': environment}) }}" - - set_fact: - output_dict: "{{ output_dict | combine({'group_names': group_names}) }}" - - set_fact: - output_dict: "{{ output_dict | combine({'groups': groups}) }}" - - set_fact: - output_dict: "{{ output_dict | combine({'hostvars': hostvars}) }}" + output_dict: "{{ hostvars }}" - copy: content: "{{ output_dict | to_nice_json }}" dest: "{{ outputs_path | default('/tmp/outputs.json') }}" @@ -421,12 +413,8 @@ def create_command_args(self) -> List[str]: pattern = self._service + "_" + str(self.index) self._command_args[ 0] += f' && ansible {pattern} -m include_role -v -i /opt/ansible/playbooks/inventory/stackl.yml -a name={self._functional_requirement}' - # self._command_args[ - # 0] += f' && ansible-playbook /opt/ansible/playbooks/stackl/playbook-role.yml -v ' - # self._command_args[ - # 0] += f'-i /opt/ansible/playbooks/inventory/stackl.yml ' - # self._command_args[ - # 0] += f'-e ansible_role={self._functional_requirement} ' + self._command_args[ + 0] += f' && ansible-playbook /opt/ansible/playbooks/stackl/playbook-role.yml -e ansible_role={self._functional_requirement} -i /opt/ansible/playbooks/inventory/stackl.yml -e pattern={pattern}' if self._output: self._command_args[ 0] += f'-e outputs_path={self._output.output_file} ' diff --git a/stackl/agent/agent/kubernetes/outputs/ansible_output.py b/stackl/agent/agent/kubernetes/outputs/ansible_output.py index 2702a074..fddf24da 100644 --- a/stackl/agent/agent/kubernetes/outputs/ansible_output.py +++ b/stackl/agent/agent/kubernetes/outputs/ansible_output.py @@ -11,3 +11,14 @@ def __init__(self, functional_requirement, stackl_instance_name): "type": "empty_dir", "mount_path": "/mnt/ansible/output/" }) + + + @property + def stackl_cli_command_args(self): + return f'\ + echo "Waiting for automation output to appear" &&\ + while [[ ! -s "{self.output_file}" ]]; do sleep 2; done;\ + cat {self.output_file} && \ + convert_json_from_spec -f ansible --doc {self.output_file} --spec {self._spec_mount["mount_path"]}/spec.json --output {self.output_file} && \ + stackl connect {self.stackl_host} && \ + stackl update instance {self.stackl_instance_name} -p "$(cat {self.output_file})" -d' \ No newline at end of file diff --git a/stackl/cli/scripts/convert_json_from_spec.py b/stackl/cli/scripts/convert_json_from_spec.py index 8cb70a4f..c89e8b3b 100644 --- a/stackl/cli/scripts/convert_json_from_spec.py +++ b/stackl/cli/scripts/convert_json_from_spec.py @@ -1,24 +1,23 @@ #!/opt/venv/bin/python -from abc import ABC, abstractmethod -from glom import glom -import json import argparse +import json import sys +from abc import ABC, abstractmethod +from collections import defaultdict + +from glom import glom class Converter(ABC): @property - @abstractmethod def input_doc(self): pass @property - @abstractmethod def spec_doc(self): pass @property - @abstractmethod def output_path(self): pass @@ -26,20 +25,34 @@ def output_path(self): def convert(self): pass + def tree(self): + return defaultdict(self.tree) + class JsonConverter(Converter): - @property - def spec_doc(self): - pass + def __init__(self, + spec_doc_file: str = 'spec.json', + json_doc_file: str = 'inputs.json', + outputs_file: str = 'outputs.json'): + self.spec_doc_file = spec_doc_file + self.json_doc_file = json_doc_file + self.outputs_file = outputs_file + self.read_files() - @property - def output_path(self): - pass + def read_files(self): + with open(self.json_doc_file) as f: + self.json_doc = json.load(f) + with open(self.spec_doc_file) as f: + self.json_spec = json.load(f) + + def convert(self): + result = {} + for field_name, field_spec in self.json_spec.items(): + result[field_name] = glom(self.json_doc, field_spec) + return result - @property - def input_doc(self): - pass +class AnsibleConverter(Converter): def __init__(self, spec_doc_file: str = 'spec.json', json_doc_file: str = 'inputs.json', @@ -56,9 +69,10 @@ def read_files(self): self.json_spec = json.load(f) def convert(self): - result = {} + result = self.tree() for field_name, field_spec in self.json_spec.items(): - result[field_name] = glom(self.json_doc, field_spec) + for host_ip, values in self.json_doc.items(): + result[field_name][host_ip] = values[field_spec] return result @@ -109,6 +123,10 @@ def main(): converter = JsonConverter(json_doc_file=args.doc, spec_doc_file=args.spec, outputs_file=args.outputs_file) + if doc_format == 'ansible': + converter = AnsibleConverter(json_doc_file=args.doc, + spec_doc_file=args.spec, + outputs_file=args.outputs_file) result = converter.convert() with open(args.outputs_file, 'w') as f: json.dump(result, f) From 087ed51820877fc878a5dab7b57e3f14f88c128f Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 20 Aug 2020 18:01:58 +0200 Subject: [PATCH 2/2] Fix ansible outputs --- .../kubernetes/handlers/ansible_handler.py | 19 +++---- .../agent/agent/kubernetes/outputs/output.py | 4 +- stackl/cli/commands/connect.py | 8 ++- stackl/cli/context.py | 54 ++++++++++--------- stackl/cli/scripts/convert_json_from_spec.py | 2 +- stackl/cli/setup.py | 2 +- 6 files changed, 47 insertions(+), 42 deletions(-) diff --git a/stackl/agent/agent/kubernetes/handlers/ansible_handler.py b/stackl/agent/agent/kubernetes/handlers/ansible_handler.py index dd15f775..7cc81c5b 100644 --- a/stackl/agent/agent/kubernetes/handlers/ansible_handler.py +++ b/stackl/agent/agent/kubernetes/handlers/ansible_handler.py @@ -274,6 +274,7 @@ def parse(self, inventory, loader, path, cache): playbook_include_role = """ - hosts: "{{ pattern }}" + gather_facts: no tasks: - include_role: name: "{{ ansible_role }}" @@ -309,7 +310,6 @@ def __init__(self, invoc): super().__init__(invoc) self._secret_handler = get_secret_handler(invoc, self._stack_instance, "yaml") - # If any outputs are defined in the functional requirement set in base_handler if self._functional_requirement_obj.outputs: self._output = AnsibleOutput(self._functional_requirement_obj, self._invoc.stack_instance) @@ -385,9 +385,8 @@ def __init__(self, invoc): "playbook-role.yml": playbook_include_role } }] - if self._output: - self._volumes.append(self._output.volume_mount) - self._volumes.append(self._output.spec_mount) + # If any outputs are defined in the functional requirement set in base_handler + self._init_containers = [] self._command = ["/bin/sh", "-c"] self._command_args = [ @@ -409,15 +408,17 @@ def create_command_args(self) -> List[str]: if "ansible_playbook_path" in self.provisioning_parameters: self._command_args[ 0] += f' && ansible-playbook {self.provisioning_parameters["ansible_playbook_path"]} -v -i /opt/ansible/playbooks/inventory/stackl.yml' + elif self._output: + pattern = self._service + "_" + str(self.index) + self._command_args[ + 0] += f' && ansible-playbook /opt/ansible/playbooks/stackl/playbook-role.yml -e ansible_role={self._functional_requirement} -i /opt/ansible/playbooks/inventory/stackl.yml -e pattern={pattern} ' + self._command_args[ + 0] += f'-e outputs_path={self._output.output_file} ' else: pattern = self._service + "_" + str(self.index) self._command_args[ 0] += f' && ansible {pattern} -m include_role -v -i /opt/ansible/playbooks/inventory/stackl.yml -a name={self._functional_requirement}' - self._command_args[ - 0] += f' && ansible-playbook /opt/ansible/playbooks/stackl/playbook-role.yml -e ansible_role={self._functional_requirement} -i /opt/ansible/playbooks/inventory/stackl.yml -e pattern={pattern}' - if self._output: - self._command_args[ - 0] += f'-e outputs_path={self._output.output_file} ' + return self._command_args @property diff --git a/stackl/agent/agent/kubernetes/outputs/output.py b/stackl/agent/agent/kubernetes/outputs/output.py index b3afdc83..1fe9ce67 100644 --- a/stackl/agent/agent/kubernetes/outputs/output.py +++ b/stackl/agent/agent/kubernetes/outputs/output.py @@ -10,10 +10,10 @@ def __init__(self, functional_requirement, stackl_instance_name: str): self.stack_instance = None self.output_file = '' self.stackl_host = f'http://{os.environ["STACKL_HOST"]}' - self.stackl_cli_image = 'stacklio/stackl-cli:v0.2.0' + self.stackl_cli_image = 'stacklio/stackl-cli:v0.2.1dev' self.stackl_cli_command = ['/bin/bash', '-c'] self._functional_requirement = functional_requirement - self._env_list = None + self._env_list = {} self.stackl_instance_name = stackl_instance_name self._spec_mount = { "name": "outputs-spec", diff --git a/stackl/cli/commands/connect.py b/stackl/cli/commands/connect.py index f184fabc..700bbf6c 100644 --- a/stackl/cli/commands/connect.py +++ b/stackl/cli/commands/connect.py @@ -2,6 +2,7 @@ from pathlib import Path import click +from context import get_config_path @click.command() @@ -12,11 +13,8 @@ def connect(host): homedir = os.getcwd() if not os.path.exists(homedir + os.sep + '.stackl'): os.makedirs(homedir + os.sep + '.stackl') - with open(config_path, 'w+') as stackl_config: + with open(get_config_path(), 'w+') as stackl_config: stackl_config.write(host) -if len(str(Path.home())) == 0: - config_path = os.getcwd() + os.sep + '.stackl' + os.sep + 'config' -else: - config_path = str(Path.home()) + os.sep + '.stackl' + os.sep + 'config' + diff --git a/stackl/cli/context.py b/stackl/cli/context.py index 945d0664..754053d3 100644 --- a/stackl/cli/context.py +++ b/stackl/cli/context.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import click import stackl_client @@ -6,34 +7,39 @@ class StacklContext(object): def __init__(self): - host = None try: - with open(config_path, 'r+') as stackl_config: + with open(get_config_path(), 'r+') as stackl_config: host = stackl_config.read() + configuration = stackl_client.Configuration() + configuration.host = host + self.api_client = stackl_client.ApiClient(configuration=configuration) + self.infrastructure_base_api = stackl_client.InfrastructureBaseApi( + api_client=self.api_client) + self.functional_requirements_api = stackl_client.FunctionalRequirementsApi( + api_client=self.api_client) + self.services_api = stackl_client.ServicesApi( + api_client=self.api_client) + self.sat_api = stackl_client.StackApplicationTemplatesApi( + api_client=self.api_client) + self.sit_api = stackl_client.StackInfrastructureTemplatesApi( + api_client=self.api_client) + self.stack_instances_api = stackl_client.StackInstancesApi( + api_client=self.api_client) + self.policy_templates_api = stackl_client.PolicyTemplatesApi( + api_client=self.api_client) + self.snapshot_api = stackl_client.SnapshotsApi( + api_client=self.api_client) except FileNotFoundError: - click.echo("Config file not found, run `stackl connect` first") - exit(1) - configuration = stackl_client.Configuration() - configuration.host = host - self.api_client = stackl_client.ApiClient(configuration=configuration) - self.infrastructure_base_api = stackl_client.InfrastructureBaseApi( - api_client=self.api_client) - self.functional_requirements_api = stackl_client.FunctionalRequirementsApi( - api_client=self.api_client) - self.services_api = stackl_client.ServicesApi( - api_client=self.api_client) - self.sat_api = stackl_client.StackApplicationTemplatesApi( - api_client=self.api_client) - self.sit_api = stackl_client.StackInfrastructureTemplatesApi( - api_client=self.api_client) - self.stack_instances_api = stackl_client.StackInstancesApi( - api_client=self.api_client) - self.policy_templates_api = stackl_client.PolicyTemplatesApi( - api_client=self.api_client) - self.snapshot_api = stackl_client.SnapshotsApi( - api_client=self.api_client) + click.echo( + "Config file not found, run `stackl connect` first, ignore this if you are running stackl connect") pass_stackl_context = click.make_pass_decorator(StacklContext, ensure=True) -config_path = os.path.expanduser('~') + os.sep + '.stackl' + os.sep + 'config' + +def get_config_path(): + if len(str(Path.home())) == 0: + config_path = os.getcwd() + os.sep + '.stackl' + os.sep + 'config' + else: + config_path = str(Path.home()) + os.sep + '.stackl' + os.sep + 'config' + return config_path diff --git a/stackl/cli/scripts/convert_json_from_spec.py b/stackl/cli/scripts/convert_json_from_spec.py index c89e8b3b..5d48a635 100644 --- a/stackl/cli/scripts/convert_json_from_spec.py +++ b/stackl/cli/scripts/convert_json_from_spec.py @@ -123,7 +123,7 @@ def main(): converter = JsonConverter(json_doc_file=args.doc, spec_doc_file=args.spec, outputs_file=args.outputs_file) - if doc_format == 'ansible': + elif doc_format == 'ansible': converter = AnsibleConverter(json_doc_file=args.doc, spec_doc_file=args.spec, outputs_file=args.outputs_file) diff --git a/stackl/cli/setup.py b/stackl/cli/setup.py index 3a593c0d..de5fa3ed 100644 --- a/stackl/cli/setup.py +++ b/stackl/cli/setup.py @@ -8,7 +8,7 @@ py_modules=['stackl', 'commands', 'context'], packages=find_packages(), install_requires=[ - 'stackl-client==0.2.1dev', 'pyYAML==5.3', 'Click==7.0', + 'stackl-client==0.2.0', 'pyYAML==5.3', 'Click==7.0', 'mergedeep==1.3.0', 'tabulate==0.8.6', 'glom==19.10.0' ], entry_points='''