diff --git a/getdeck/__main__.py b/getdeck/__main__.py index 2f3d7d4..8005941 100755 --- a/getdeck/__main__.py +++ b/getdeck/__main__.py @@ -2,6 +2,7 @@ import argparse import logging import os +import traceback os.environ["PYOXIDIZER"] = "1" @@ -161,7 +162,9 @@ def main(): parser.print_help() exit(0) except Exception as e: - logger.fatal(f"There was an error running Deck: {e}") + if args.debug: + traceback.print_exc() + logger.fatal(f"There was an error running deck: {e}") exit(1) diff --git a/getdeck/api/get.py b/getdeck/api/get.py index 26eb457..3027318 100644 --- a/getdeck/api/get.py +++ b/getdeck/api/get.py @@ -1,4 +1,5 @@ import logging +import shutil from typing import Callable from getdeck.api import stopwatch, remove @@ -28,8 +29,9 @@ def run_deck( # noqa: C901 if progress_callback: progress_callback(0) - deckfile = read_deckfile_from_location(deckfile_location, config) - + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( + deckfile_location, config + ) if progress_callback: progress_callback(5) # @@ -52,7 +54,9 @@ def run_deck( # noqa: C901 # 2. generate the Deck's workload # try: - generated_deck = prepare_k8s_workload_for_deck(config, deckfile, deck_name) + generated_deck = prepare_k8s_workload_for_deck( + config, deckfile, deck_name, working_dir_path + ) except Exception as e: if cluster_created: # remove this just created cluster as it probably is in an inconsistent state from the beginning @@ -120,6 +124,8 @@ def run_deck( # noqa: C901 if notes := deckfile.get_deck(deck_name).notes: logger.info(notes) + if is_temp_dir: + shutil.rmtree(working_dir_path) if wait: _wait_ready(config, generated_deck, timeout) return True diff --git a/getdeck/api/hosts.py b/getdeck/api/hosts.py index 1c87c5e..d29b978 100644 --- a/getdeck/api/hosts.py +++ b/getdeck/api/hosts.py @@ -1,4 +1,5 @@ import logging +import shutil import socket from python_hosts import Hosts, HostsEntry @@ -17,35 +18,43 @@ def run_hosts( deck_name: str = None, config=default_configuration, ) -> bool: - deckfile = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( + deckfile_location, config + ) deck = deckfile.get_deck(deck_name) deck_hosts = deck.hosts hosts = Hosts() - if host_action == "list": - - logger.info("Ingress hosts:") - for host in deck_hosts: - logger.info(f"{host}") - - elif host_action == "remove": - logger.debug("Removing hosts from hosts file...") - for host in deck_hosts: - hosts.remove_all_matching(name=host) - hosts.write() - logger.info("Hosts have been removed from hosts file...") - - elif host_action == "write": - - logger.info("Writing hosts to hosts file...") - new_entry = HostsEntry(entry_type="ipv4", address="127.0.0.1", names=deck_hosts) - hosts.add([new_entry]) - hosts.write() - logger.info("Hosts should resolve to '127.0.0.1' now.") - + if deck_hosts: + if host_action == "list": + + logger.info("Ingress hosts:") + for host in deck_hosts: + logger.info(f"{host}") + + elif host_action == "remove": + logger.debug("Removing hosts from hosts file...") + for host in deck_hosts: + hosts.remove_all_matching(name=host) + hosts.write() + logger.info("Hosts have been removed from hosts file...") + + elif host_action == "write": + + logger.info("Writing hosts to hosts file...") + new_entry = HostsEntry( + entry_type="ipv4", address="127.0.0.1", names=deck_hosts + ) + hosts.add([new_entry]) + hosts.write() + logger.info("Hosts should resolve to '127.0.0.1' now.") + + else: + logger.error(f"Unknown host action '{host_action}'") else: - logger.error(f"Unknown host action '{host_action}'") - + logger.info("No hosts specified in Deckfile") + if is_temp_dir: + shutil.rmtree(working_dir_path) return True diff --git a/getdeck/api/list.py b/getdeck/api/list.py index 0b89f57..7211a39 100644 --- a/getdeck/api/list.py +++ b/getdeck/api/list.py @@ -1,4 +1,5 @@ import logging +import shutil from typing import List from getdeck.api.utils import stopwatch @@ -11,7 +12,11 @@ def get_available_decks(deckfile_location: str, config=default_configuration) -> List: from getdeck.utils import read_deckfile_from_location - deckfile = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( + deckfile_location, config + ) available_decks = deckfile.get_decks() logger.debug(available_decks) + if is_temp_dir: + shutil.rmtree(working_dir_path) return available_decks diff --git a/getdeck/api/remove.py b/getdeck/api/remove.py index a16c91e..cef0cb7 100644 --- a/getdeck/api/remove.py +++ b/getdeck/api/remove.py @@ -1,4 +1,5 @@ import logging +import shutil from typing import Callable from getdeck.configuration import default_configuration @@ -16,12 +17,16 @@ def remove_cluster( ) -> bool: from getdeck.utils import read_deckfile_from_location, ensure_cluster - deckfile = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( + deckfile_location, config + ) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) if k8s_provider.exists(): k8s_provider.delete() else: logger.info("Cluster does not exist") + if is_temp_dir: + shutil.rmtree(working_dir_path) return True @@ -40,7 +45,9 @@ def remove_deck( from getdeck.k8s import k8s_delete_object from getdeck.sources.utils import prepare_k8s_workload_for_deck - deckfile = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( + deckfile_location, config + ) if progress_callback: progress_callback(10) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) @@ -49,7 +56,9 @@ def remove_deck( config.kubeconfig = k8s_provider.get_kubeconfig() if k8s_provider.exists(): - generated_deck = prepare_k8s_workload_for_deck(config, deckfile, deck_name) + generated_deck = prepare_k8s_workload_for_deck( + config, deckfile, deck_name, working_dir_path + ) logger.info(f"Removing Deck {generated_deck.name}") if progress_callback: progress_callback(30) @@ -73,4 +82,7 @@ def remove_deck( logger.info(f"All workloads from Deck {generated_deck.name} removed") else: logger.info("Cluster does not exist") + + if is_temp_dir: + shutil.rmtree(working_dir_path) return True diff --git a/getdeck/api/stop.py b/getdeck/api/stop.py index a011854..b4c839a 100644 --- a/getdeck/api/stop.py +++ b/getdeck/api/stop.py @@ -1,4 +1,5 @@ import logging +import shutil from typing import Callable from getdeck.api import stopwatch @@ -16,7 +17,11 @@ def stop_cluster( ) -> bool: from getdeck.utils import read_deckfile_from_location, ensure_cluster - deckfile = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( + deckfile_location, config + ) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) logger.info("Stopping cluster") + if is_temp_dir: + shutil.rmtree(working_dir_path) k8s_provider.stop() diff --git a/getdeck/sources/fetcher.py b/getdeck/sources/fetcher.py index 0859257..b8831a0 100644 --- a/getdeck/sources/fetcher.py +++ b/getdeck/sources/fetcher.py @@ -25,11 +25,13 @@ def __init__( source: Union[DeckfileFileSource, DeckfileKustomizeSource, DeckfileHelmSource], config: ClientConfiguration, namespace: str, + working_dir: str = None, ): self.path = path self.source = source self.config = config self.namespace = namespace + self.working_dir = working_dir @property def not_supported_message(self): diff --git a/getdeck/sources/file.py b/getdeck/sources/file.py index 604c828..2dfdfff 100644 --- a/getdeck/sources/file.py +++ b/getdeck/sources/file.py @@ -1,6 +1,5 @@ import logging import os -from pathlib import PurePath import tempfile from typing import List @@ -55,7 +54,9 @@ def _parse_source_directory(self, ref: str) -> List[K8sSourceFile]: k8s_workload_files = self._parse_source_files(refs=refs) return k8s_workload_files - def _parse_source(self, ref: str) -> List[K8sSourceFile]: + def _parse_source(self, ref: str, working_dir: str = None) -> List[K8sSourceFile]: + if working_dir: + ref = os.path.join(working_dir, ref.removeprefix("./")) if os.path.isdir(ref): k8s_workload_files = self._parse_source_directory( ref=ref, @@ -96,9 +97,13 @@ def fetch_https(self, **kwargs): def fetch_local(self, **kwargs): try: logger.debug(f"Reading file {self.source.ref}") - ref = str(PurePath(os.path.join(self.path, self.source.ref))) - - k8s_workload_files = self._parse_source(ref=ref) + if not os.path.isabs(self.source.ref): + fpath = os.path.join( + self.working_dir, self.source.ref.removeprefix("./") + ) + k8s_workload_files = self._parse_source(ref=fpath) + else: + k8s_workload_files = self._parse_source(ref=self.source.ref) return k8s_workload_files except Exception as e: logger.error(f"Error loading file from http {e}") diff --git a/getdeck/sources/helm.py b/getdeck/sources/helm.py index d56616d..d1208f8 100644 --- a/getdeck/sources/helm.py +++ b/getdeck/sources/helm.py @@ -58,7 +58,7 @@ def k8s_api_version(self) -> str: return f"{data['major']}.{data['minor']}" def _helm_prep(self) -> List[str]: - if self.type == "git": + if self.type in ["git", "local"]: return self._helm_dep_up() else: # http(s) return self._helm_repo_add() @@ -67,6 +67,8 @@ def _helm_repo_add(self) -> List[str]: return ["helm", "repo", "add", "this", self.source.ref] def _helm_dep_up(self) -> List[str]: + if self.type == "local": + return ["helm", "dep", "up", self.source.ref] return ["helm", "dep", "up", self.source.path] def _helm_with_plugins(self) -> List[str]: @@ -88,6 +90,29 @@ def _helm_template(self) -> List[str]: ["--values", os.path.join(self.source.path, _valuefile)] ) return temp + elif self.type == "local": + temp = [ + "template", + f"{self.source.releaseName}", + f"/sources/{self.source.ref.removeprefix('./')}", + "--include-crds", + "--namespace", + self.namespace, + ] + if self.source.valueFiles: + for _valuefile in self.source.valueFiles: + temp.extend( + [ + "--values", + os.path.join( + "/sources", + self.source.ref.removeprefix("./"), + self.source.path or "", + _valuefile.removeprefix("./"), + ), + ] + ) + return temp else: # http(s) return [ "template", diff --git a/getdeck/sources/tooler.py b/getdeck/sources/tooler.py index 5c4edb5..ea2bd59 100644 --- a/getdeck/sources/tooler.py +++ b/getdeck/sources/tooler.py @@ -1,6 +1,7 @@ import io import logging import os +import shutil import subprocess import sys import tempfile @@ -121,7 +122,25 @@ def fetch_content(self, **kwargs) -> List[K8sSourceFile]: raise NotImplementedError def fetch_local(self, **kwargs): - raise NotImplementedError + cmd = self.build_command() + try: + if not os.path.isabs(self.source.ref): + path = os.path.join( + self.working_dir, self.source.ref.removeprefix("./") + ) + self._parse_source(ref=path, working_dir=self.working_dir) + dst = os.path.join( + self.tmp_source.name, self.source.ref.removeprefix("./") + ) + shutil.copytree(path, dst, dirs_exist_ok=True) + else: + self._parse_source(ref=self.source.ref) + dst = os.path.join(self.tmp_source.name, self.source.ref) + shutil.copytree(self.source.ref, dst, dirs_exist_ok=True) + self.run_tooler(cmd) + return self.collect_workload_files() + finally: + self.cleanup() def fetch_remote(self, git=False): cmd = self.build_command() diff --git a/getdeck/sources/utils.py b/getdeck/sources/utils.py index ec5cbff..c612ab9 100644 --- a/getdeck/sources/utils.py +++ b/getdeck/sources/utils.py @@ -11,7 +11,10 @@ def prepare_k8s_workload_for_deck( - config: ClientConfiguration, deckfile: Deckfile, deck_name: str + config: ClientConfiguration, + deckfile: Deckfile, + deck_name: str, + working_dir: str = None, ) -> GeneratedDeck: deck = deckfile.get_deck(deck_name) logger.debug(deck) @@ -46,7 +49,7 @@ def prepare_k8s_workload_for_deck( namespace = deck.namespace or "default" fetcher_context.strategy = strategy( - deckfile.file_path, source, config, namespace + deckfile.file_path, source, config, namespace, working_dir ) # fetch source files diff --git a/getdeck/utils.py b/getdeck/utils.py index 273345f..ca189f5 100644 --- a/getdeck/utils.py +++ b/getdeck/utils.py @@ -1,8 +1,10 @@ import logging import os +import shutil import subprocess import tempfile from time import sleep +from typing import Optional, Tuple import requests from git import Repo, GitError @@ -33,8 +35,11 @@ def sniff_protocol(ref: str): return None -def read_deckfile_from_location(location: str, config: ClientConfiguration) -> Deckfile: +def read_deckfile_from_location( + location: str, config: ClientConfiguration +) -> Tuple[Deckfile, Optional[str], bool]: protocol = sniff_protocol(location) + logger.info(f"Reading Deckfile from: {location}") if location == ".": # load default file from this location return config.deckfile_selector.get( @@ -46,20 +51,19 @@ def read_deckfile_from_location(location: str, config: ClientConfiguration) -> D else: ref = location rev = "HEAD" - tmp_dir = tempfile.TemporaryDirectory() + tmp_dir = tempfile.mkdtemp() try: - repo = Repo.clone_from(ref, tmp_dir.name) + repo = Repo.clone_from(ref, tmp_dir) repo.git.checkout(rev) deckfile = config.deckfile_selector.get( - os.path.join(tmp_dir.name, configuration.DECKFILE_FILE) + os.path.join(tmp_dir, configuration.DECKFILE_FILE) ) - tmp_dir.cleanup() - return deckfile + return deckfile, tmp_dir, True except GitError as e: - tmp_dir.cleanup() + shutil.rmtree(tmp_dir) raise RuntimeError(f"Cannot checkout {rev} from {ref}: {e}") except Exception as e: - tmp_dir.cleanup() + shutil.rmtree(tmp_dir) raise e elif protocol in ["http", "https"]: download = tempfile.NamedTemporaryFile(delete=False) @@ -74,7 +78,7 @@ def read_deckfile_from_location(location: str, config: ClientConfiguration) -> D download.close() deckfile = config.deckfile_selector.get(download.name) os.remove(download.name) - return deckfile + return deckfile, None, False except Exception as e: download.close() os.remove(download.name) @@ -85,7 +89,11 @@ def read_deckfile_from_location(location: str, config: ClientConfiguration) -> D # this is probably a file system location if os.path.isfile(location): logger.debug("Is file location") - return config.deckfile_selector.get(location) + return ( + config.deckfile_selector.get(location), + os.path.dirname(location), + False, + ) else: raise RuntimeError(f"Cannot identify {location} as Deckfile") else: diff --git a/setup.cfg b/setup.cfg index 0b192c6..2b1154d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,7 @@ doctests = True enable-extensions = G strictness = long max-line-length = 120 -max-complexity = 25 +max-complexity = 27 exclude = .git,__pycache__,.venv,venv,.eggs,*.egg,testing,stowaway,carrier,cargo ignore = D100, D104, D106, D401, X100, W503, W504, RST303, RST304, DAR103, DAR203