From c9604b499921370604fc8db1e0ab15e90334fd40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Fr=C3=B6lich?= Date: Thu, 18 Aug 2022 11:50:21 +0200 Subject: [PATCH 1/9] feat: add local helm chart support --- getdeck/sources/helm.py | 27 ++++++++++++++++++++++++++- getdeck/sources/tooler.py | 11 ++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/getdeck/sources/helm.py b/getdeck/sources/helm.py index 5a8f80e..f378582 100644 --- a/getdeck/sources/helm.py +++ b/getdeck/sources/helm.py @@ -56,7 +56,7 @@ def k8s_api_version(self) -> str: return f"{data['major']}.{data['minor']}" def _helm_prep(self) -> List[str]: - if self.type == "git": + if self.type in ["git", "local"]: return self._helm_dep_up() else: # http(s) return self._helm_repo_add() @@ -65,6 +65,8 @@ def _helm_repo_add(self) -> List[str]: return ["helm", "repo", "add", "this", self.source.ref] def _helm_dep_up(self) -> List[str]: + if self.type == "local": + return ["helm", "dep", "up", self.source.ref] return ["helm", "dep", "up", self.source.path] def _helm_with_plugins(self) -> List[str]: @@ -86,6 +88,29 @@ def _helm_template(self) -> List[str]: ["--values", os.path.join(self.source.path, _valuefile)] ) return temp + if self.type == "local": + temp = [ + "template", + f"{self.source.releaseName}", + f"/sources/{self.source.ref}", + "--include-crds", + "--namespace", + self.namespace, + ] + if self.source.valueFiles: + for _valuefile in self.source.valueFiles: + temp.extend( + [ + "--values", + os.path.join( + "/sources", + self.source.ref, + self.source.path, + _valuefile, + ), + ] + ) + return temp else: # http(s) return [ "template", diff --git a/getdeck/sources/tooler.py b/getdeck/sources/tooler.py index 5c4edb5..be8e4b2 100644 --- a/getdeck/sources/tooler.py +++ b/getdeck/sources/tooler.py @@ -1,6 +1,7 @@ import io import logging import os +import shutil import subprocess import sys import tempfile @@ -121,7 +122,15 @@ def fetch_content(self, **kwargs) -> List[K8sSourceFile]: raise NotImplementedError def fetch_local(self, **kwargs): - raise NotImplementedError + cmd = self.build_command() + try: + self._parse_source(ref=self.source.ref) + dst = os.path.join(self.tmp_source.name, self.source.ref) + shutil.copytree(self.source.ref, dst, dirs_exist_ok=True) + self.run_tooler(cmd) + return self.collect_workload_files() + finally: + self.cleanup() def fetch_remote(self, git=False): cmd = self.build_command() From a1300ce3502fa7855d4d27a159ab1f5700a8a682 Mon Sep 17 00:00:00 2001 From: Michael Schilonka Date: Fri, 19 Aug 2022 11:05:55 +0200 Subject: [PATCH 2/9] fix: WIP --- getdeck/__main__.py | 3 +++ getdeck/api/get.py | 5 ++--- getdeck/api/hosts.py | 2 +- getdeck/api/list.py | 2 +- getdeck/api/remove.py | 4 ++-- getdeck/api/stop.py | 2 +- getdeck/sources/file.py | 13 +++++++++++-- getdeck/sources/helm.py | 10 +++++----- getdeck/sources/tooler.py | 15 ++++++++++++--- getdeck/sources/utils.py | 13 +++++++------ getdeck/utils.py | 21 +++++++++++---------- 11 files changed, 56 insertions(+), 34 deletions(-) diff --git a/getdeck/__main__.py b/getdeck/__main__.py index 6c3e9a2..70587b1 100755 --- a/getdeck/__main__.py +++ b/getdeck/__main__.py @@ -2,6 +2,7 @@ import argparse import logging import os +import traceback from getdeck.api import stop_cluster @@ -155,6 +156,8 @@ def main(): parser.print_help() exit(0) except Exception as e: + if args.debug: + traceback.print_exc() logger.fatal(f"There was an error running Deck: {e}") exit(1) diff --git a/getdeck/api/get.py b/getdeck/api/get.py index eb40a34..6b96727 100644 --- a/getdeck/api/get.py +++ b/getdeck/api/get.py @@ -28,8 +28,7 @@ def run_deck( if progress_callback: progress_callback(0) - deckfile = read_deckfile_from_location(deckfile_location, config) - + deckfile, working_dir_path = read_deckfile_from_location(deckfile_location, config) if progress_callback: progress_callback(5) # @@ -51,7 +50,7 @@ def run_deck( # 2. generate the Deck's workload # try: - generated_deck = prepare_k8s_workload_for_deck(config, deckfile, deck_name) + generated_deck = prepare_k8s_workload_for_deck(config, deckfile, deck_name, working_dir_path) except Exception as e: if cluster_created: # remove this just created cluster as it probably is in an inconsistent state from the beginning diff --git a/getdeck/api/hosts.py b/getdeck/api/hosts.py index 1c87c5e..982ddc9 100644 --- a/getdeck/api/hosts.py +++ b/getdeck/api/hosts.py @@ -17,7 +17,7 @@ def run_hosts( deck_name: str = None, config=default_configuration, ) -> bool: - deckfile = read_deckfile_from_location(deckfile_location, config) + deckfile, _ = read_deckfile_from_location(deckfile_location, config) deck = deckfile.get_deck(deck_name) deck_hosts = deck.hosts diff --git a/getdeck/api/list.py b/getdeck/api/list.py index 0b89f57..c2facec 100644 --- a/getdeck/api/list.py +++ b/getdeck/api/list.py @@ -11,7 +11,7 @@ def get_available_decks(deckfile_location: str, config=default_configuration) -> List: from getdeck.utils import read_deckfile_from_location - deckfile = read_deckfile_from_location(deckfile_location, config) + deckfile, _ = read_deckfile_from_location(deckfile_location, config) available_decks = deckfile.get_decks() logger.debug(available_decks) return available_decks diff --git a/getdeck/api/remove.py b/getdeck/api/remove.py index a16c91e..ad5b034 100644 --- a/getdeck/api/remove.py +++ b/getdeck/api/remove.py @@ -16,7 +16,7 @@ def remove_cluster( ) -> bool: from getdeck.utils import read_deckfile_from_location, ensure_cluster - deckfile = read_deckfile_from_location(deckfile_location, config) + deckfile, _ = read_deckfile_from_location(deckfile_location, config) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) if k8s_provider.exists(): k8s_provider.delete() @@ -40,7 +40,7 @@ def remove_deck( from getdeck.k8s import k8s_delete_object from getdeck.sources.utils import prepare_k8s_workload_for_deck - deckfile = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path = read_deckfile_from_location(deckfile_location, config) if progress_callback: progress_callback(10) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) diff --git a/getdeck/api/stop.py b/getdeck/api/stop.py index a011854..77d32c8 100644 --- a/getdeck/api/stop.py +++ b/getdeck/api/stop.py @@ -16,7 +16,7 @@ def stop_cluster( ) -> bool: from getdeck.utils import read_deckfile_from_location, ensure_cluster - deckfile = read_deckfile_from_location(deckfile_location, config) + deckfile, _ = read_deckfile_from_location(deckfile_location, config) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) logger.info("Stopping cluster") k8s_provider.stop() diff --git a/getdeck/sources/file.py b/getdeck/sources/file.py index bf2af9a..94540bb 100644 --- a/getdeck/sources/file.py +++ b/getdeck/sources/file.py @@ -30,10 +30,12 @@ def __init__( source: Union[DeckfileFileSource, DeckfileKustomizeSource, DeckfileHelmSource], config: ClientConfiguration, namespace: str, + working_dir: str ): self.source = source self.config = config self.namespace = namespace + self.working_dir = working_dir @property def not_supported_message(self): @@ -77,6 +79,7 @@ def not_supported_message(self): @staticmethod def _parse_source_file(ref: str) -> List[K8sSourceFile]: + logger.debug(ref) with open(ref, "r") as input_file: docs = yaml.load_all(input_file.read(), Loader=yaml.FullLoader) @@ -113,7 +116,9 @@ def _parse_source_directory(ref: str) -> List[K8sSourceFile]: return k8s_workload_files @staticmethod - def _parse_source(ref: str) -> List[K8sSourceFile]: + def _parse_source(ref: str, working_dir: str = None) -> List[K8sSourceFile]: + if working_dir: + ref = os.path.join(working_dir, ref.removeprefix("./")) if os.path.isdir(ref): k8s_workload_files = FileFetcher._parse_source_directory(ref=ref) else: @@ -147,7 +152,11 @@ def fetch_https(self, **kwargs): def fetch_local(self, **kwargs): try: logger.debug(f"Reading file {self.source.ref}") - k8s_workload_files = self._parse_source(ref=self.source.ref) + if not os.path.isabs(self.source.ref): + fpath = os.path.join(self.working_dir, self.source.ref.removeprefix("./")) + k8s_workload_files = self._parse_source(ref=fpath) + else: + k8s_workload_files = self._parse_source(ref=self.source.ref) return k8s_workload_files except Exception as e: logger.error(f"Error loading file from http {e}") diff --git a/getdeck/sources/helm.py b/getdeck/sources/helm.py index f378582..3873e30 100644 --- a/getdeck/sources/helm.py +++ b/getdeck/sources/helm.py @@ -88,11 +88,11 @@ def _helm_template(self) -> List[str]: ["--values", os.path.join(self.source.path, _valuefile)] ) return temp - if self.type == "local": + elif self.type == "local": temp = [ "template", f"{self.source.releaseName}", - f"/sources/{self.source.ref}", + f"/sources/{self.source.ref.removeprefix('./')}", "--include-crds", "--namespace", self.namespace, @@ -104,9 +104,9 @@ def _helm_template(self) -> List[str]: "--values", os.path.join( "/sources", - self.source.ref, - self.source.path, - _valuefile, + self.source.ref.removeprefix('./'), + self.source.path or "", + _valuefile.removeprefix('./'), ), ] ) diff --git a/getdeck/sources/tooler.py b/getdeck/sources/tooler.py index be8e4b2..f5f0856 100644 --- a/getdeck/sources/tooler.py +++ b/getdeck/sources/tooler.py @@ -124,9 +124,18 @@ def fetch_content(self, **kwargs) -> List[K8sSourceFile]: def fetch_local(self, **kwargs): cmd = self.build_command() try: - self._parse_source(ref=self.source.ref) - dst = os.path.join(self.tmp_source.name, self.source.ref) - shutil.copytree(self.source.ref, dst, dirs_exist_ok=True) + if not os.path.isabs(self.source.ref): + path = os.path.join(self.working_dir, self.source.ref.removeprefix("./")) + self._parse_source(ref=path, working_dir=self.working_dir) + dst = os.path.join(self.tmp_source.name, self.source.ref.removeprefix("./")) + logger.debug("COPYING FROM: - TO:") + logger.debug(path) + logger.debug(dst) + shutil.copytree(path, dst, dirs_exist_ok=True) + else: + self._parse_source(ref=self.source.ref) + dst = os.path.join(self.tmp_source.name, self.source.ref) + shutil.copytree(self.source.ref, dst, dirs_exist_ok=True) self.run_tooler(cmd) return self.collect_workload_files() finally: diff --git a/getdeck/sources/utils.py b/getdeck/sources/utils.py index 8a2bf4a..977bf00 100644 --- a/getdeck/sources/utils.py +++ b/getdeck/sources/utils.py @@ -26,28 +26,29 @@ def fetch_deck_source( DeckfileKustomizeSource, ], namespace: str = "default", + working_dir: str = None, ) -> List[K8sSourceFile]: logger.info( "Processing source " f"{source.__class__.__name__}: {'no ref' if not source.ref else source.ref}" ) if isinstance(source, DeckfileHelmSource): - fetcher = HelmFetcher(source, config, namespace) + fetcher = HelmFetcher(source, config, namespace, working_dir) elif isinstance(source, DeckfileFileSource): - fetcher = FileFetcher(source, config, namespace) + fetcher = FileFetcher(source, config, namespace, working_dir) elif isinstance(source, DeckfileKustomizeSource): - fetcher = KustomizeFetcher(source, config, namespace) + fetcher = KustomizeFetcher(source, config, namespace, working_dir) else: logger.info( "Skipping source " f"{source.__class__.__name__}: {'no ref' if not source.ref else source.ref}" ) - fetcher = Fetcher(source, config, namespace) + fetcher = Fetcher(source, config, namespace, working_dir) return fetcher.fetch() def prepare_k8s_workload_for_deck( - config: ClientConfiguration, deckfile: Deckfile, deck_name: str + config: ClientConfiguration, deckfile: Deckfile, deck_name: str, working_dir: str = None ) -> GeneratedDeck: deck = deckfile.get_deck(deck_name) logger.debug(deck) @@ -56,5 +57,5 @@ def prepare_k8s_workload_for_deck( generated_deck = GeneratedDeck(name=deck.name, namespace=namespace, files=[]) logger.info(f"Processing {len(deck.sources)} source(s)") for source in deck.sources: - generated_deck.files.extend(fetch_deck_source(config, source, namespace)) + generated_deck.files.extend(fetch_deck_source(config, source, namespace, working_dir)) return generated_deck diff --git a/getdeck/utils.py b/getdeck/utils.py index f67b0d4..5103242 100644 --- a/getdeck/utils.py +++ b/getdeck/utils.py @@ -1,8 +1,10 @@ import logging import os +import shutil import subprocess import tempfile from time import sleep +from typing import Optional, Tuple import requests from git import Repo, GitError @@ -32,7 +34,7 @@ def sniff_protocol(ref: str): return None -def read_deckfile_from_location(location: str, config: ClientConfiguration) -> Deckfile: +def read_deckfile_from_location(location: str, config: ClientConfiguration) -> Tuple[Deckfile, Optional[str]]: protocol = sniff_protocol(location) if location == ".": # load default file from this location @@ -45,20 +47,19 @@ def read_deckfile_from_location(location: str, config: ClientConfiguration) -> D else: ref = location rev = "HEAD" - tmp_dir = tempfile.TemporaryDirectory() + tmp_dir = tempfile.mkdtemp() try: - repo = Repo.clone_from(ref, tmp_dir.name) + repo = Repo.clone_from(ref, tmp_dir) repo.git.checkout(rev) deckfile = config.deckfile_selector.get( - os.path.join(tmp_dir.name, configuration.DECKFILE_FILE) + os.path.join(tmp_dir, configuration.DECKFILE_FILE) ) - tmp_dir.cleanup() - return deckfile + return deckfile, tmp_dir except GitError as e: - tmp_dir.cleanup() + shutil.rmtree(tmp_dir) raise RuntimeError(f"Cannot checkout {rev} from {ref}: {e}") except Exception as e: - tmp_dir.cleanup() + shutil.rmtree(tmp_dir) raise e elif protocol in ["http", "https"]: download = tempfile.NamedTemporaryFile(delete=False) @@ -73,7 +74,7 @@ def read_deckfile_from_location(location: str, config: ClientConfiguration) -> D download.close() deckfile = config.deckfile_selector.get(download.name) os.remove(download.name) - return deckfile + return deckfile, None except Exception as e: download.close() os.remove(download.name) @@ -84,7 +85,7 @@ def read_deckfile_from_location(location: str, config: ClientConfiguration) -> D # this is probably a file system location if os.path.isfile(location): logger.debug("Is file location") - return config.deckfile_selector.get(location) + return config.deckfile_selector.get(location), os.path.dirname(location) else: raise RuntimeError(f"Cannot identify {location} as Deckfile") else: From 20323029f84684ef4313761bb83c99f5bd0d441e Mon Sep 17 00:00:00 2001 From: Michael Schilonka Date: Fri, 19 Aug 2022 11:12:10 +0200 Subject: [PATCH 3/9] fix: WIP code styling --- getdeck/api/get.py | 4 +++- getdeck/sources/file.py | 6 ++++-- getdeck/sources/helm.py | 4 ++-- getdeck/sources/tooler.py | 8 ++++++-- getdeck/sources/utils.py | 9 +++++++-- getdeck/utils.py | 4 +++- 6 files changed, 25 insertions(+), 10 deletions(-) diff --git a/getdeck/api/get.py b/getdeck/api/get.py index 6b96727..70b8647 100644 --- a/getdeck/api/get.py +++ b/getdeck/api/get.py @@ -50,7 +50,9 @@ def run_deck( # 2. generate the Deck's workload # try: - generated_deck = prepare_k8s_workload_for_deck(config, deckfile, deck_name, working_dir_path) + generated_deck = prepare_k8s_workload_for_deck( + config, deckfile, deck_name, working_dir_path + ) except Exception as e: if cluster_created: # remove this just created cluster as it probably is in an inconsistent state from the beginning diff --git a/getdeck/sources/file.py b/getdeck/sources/file.py index 94540bb..5a501af 100644 --- a/getdeck/sources/file.py +++ b/getdeck/sources/file.py @@ -30,7 +30,7 @@ def __init__( source: Union[DeckfileFileSource, DeckfileKustomizeSource, DeckfileHelmSource], config: ClientConfiguration, namespace: str, - working_dir: str + working_dir: str, ): self.source = source self.config = config @@ -153,7 +153,9 @@ def fetch_local(self, **kwargs): try: logger.debug(f"Reading file {self.source.ref}") if not os.path.isabs(self.source.ref): - fpath = os.path.join(self.working_dir, self.source.ref.removeprefix("./")) + fpath = os.path.join( + self.working_dir, self.source.ref.removeprefix("./") + ) k8s_workload_files = self._parse_source(ref=fpath) else: k8s_workload_files = self._parse_source(ref=self.source.ref) diff --git a/getdeck/sources/helm.py b/getdeck/sources/helm.py index 3873e30..d81195e 100644 --- a/getdeck/sources/helm.py +++ b/getdeck/sources/helm.py @@ -104,9 +104,9 @@ def _helm_template(self) -> List[str]: "--values", os.path.join( "/sources", - self.source.ref.removeprefix('./'), + self.source.ref.removeprefix("./"), self.source.path or "", - _valuefile.removeprefix('./'), + _valuefile.removeprefix("./"), ), ] ) diff --git a/getdeck/sources/tooler.py b/getdeck/sources/tooler.py index f5f0856..43469e3 100644 --- a/getdeck/sources/tooler.py +++ b/getdeck/sources/tooler.py @@ -125,9 +125,13 @@ def fetch_local(self, **kwargs): cmd = self.build_command() try: if not os.path.isabs(self.source.ref): - path = os.path.join(self.working_dir, self.source.ref.removeprefix("./")) + path = os.path.join( + self.working_dir, self.source.ref.removeprefix("./") + ) self._parse_source(ref=path, working_dir=self.working_dir) - dst = os.path.join(self.tmp_source.name, self.source.ref.removeprefix("./")) + dst = os.path.join( + self.tmp_source.name, self.source.ref.removeprefix("./") + ) logger.debug("COPYING FROM: - TO:") logger.debug(path) logger.debug(dst) diff --git a/getdeck/sources/utils.py b/getdeck/sources/utils.py index 977bf00..c1a7f81 100644 --- a/getdeck/sources/utils.py +++ b/getdeck/sources/utils.py @@ -48,7 +48,10 @@ def fetch_deck_source( def prepare_k8s_workload_for_deck( - config: ClientConfiguration, deckfile: Deckfile, deck_name: str, working_dir: str = None + config: ClientConfiguration, + deckfile: Deckfile, + deck_name: str, + working_dir: str = None, ) -> GeneratedDeck: deck = deckfile.get_deck(deck_name) logger.debug(deck) @@ -57,5 +60,7 @@ def prepare_k8s_workload_for_deck( generated_deck = GeneratedDeck(name=deck.name, namespace=namespace, files=[]) logger.info(f"Processing {len(deck.sources)} source(s)") for source in deck.sources: - generated_deck.files.extend(fetch_deck_source(config, source, namespace, working_dir)) + generated_deck.files.extend( + fetch_deck_source(config, source, namespace, working_dir) + ) return generated_deck diff --git a/getdeck/utils.py b/getdeck/utils.py index 5103242..5a8b25d 100644 --- a/getdeck/utils.py +++ b/getdeck/utils.py @@ -34,7 +34,9 @@ def sniff_protocol(ref: str): return None -def read_deckfile_from_location(location: str, config: ClientConfiguration) -> Tuple[Deckfile, Optional[str]]: +def read_deckfile_from_location( + location: str, config: ClientConfiguration +) -> Tuple[Deckfile, Optional[str]]: protocol = sniff_protocol(location) if location == ".": # load default file from this location From b309dd6dc188f476a606317c366f03ef369971b9 Mon Sep 17 00:00:00 2001 From: Michael Schilonka Date: Fri, 19 Aug 2022 12:03:55 +0200 Subject: [PATCH 4/9] chore: remove logger, tested --- getdeck/sources/file.py | 1 - getdeck/sources/tooler.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/getdeck/sources/file.py b/getdeck/sources/file.py index 5a501af..0484a42 100644 --- a/getdeck/sources/file.py +++ b/getdeck/sources/file.py @@ -79,7 +79,6 @@ def not_supported_message(self): @staticmethod def _parse_source_file(ref: str) -> List[K8sSourceFile]: - logger.debug(ref) with open(ref, "r") as input_file: docs = yaml.load_all(input_file.read(), Loader=yaml.FullLoader) diff --git a/getdeck/sources/tooler.py b/getdeck/sources/tooler.py index 43469e3..ea2bd59 100644 --- a/getdeck/sources/tooler.py +++ b/getdeck/sources/tooler.py @@ -132,9 +132,6 @@ def fetch_local(self, **kwargs): dst = os.path.join( self.tmp_source.name, self.source.ref.removeprefix("./") ) - logger.debug("COPYING FROM: - TO:") - logger.debug(path) - logger.debug(dst) shutil.copytree(path, dst, dirs_exist_ok=True) else: self._parse_source(ref=self.source.ref) From a767b1742eb1ceae18f57accbbd10d8a41a4dd34 Mon Sep 17 00:00:00 2001 From: Michael Schilonka Date: Fri, 19 Aug 2022 13:38:59 +0200 Subject: [PATCH 5/9] fix: remove deck, missing argument --- getdeck/api/remove.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getdeck/api/remove.py b/getdeck/api/remove.py index ad5b034..115d2ee 100644 --- a/getdeck/api/remove.py +++ b/getdeck/api/remove.py @@ -49,7 +49,7 @@ def remove_deck( config.kubeconfig = k8s_provider.get_kubeconfig() if k8s_provider.exists(): - generated_deck = prepare_k8s_workload_for_deck(config, deckfile, deck_name) + generated_deck = prepare_k8s_workload_for_deck(config, deckfile, deck_name, working_dir_path) logger.info(f"Removing Deck {generated_deck.name}") if progress_callback: progress_callback(30) From 27cafa0c90e52754e8034f47ba08abcc4dbd5576 Mon Sep 17 00:00:00 2001 From: Michael Schilonka Date: Fri, 19 Aug 2022 14:21:52 +0200 Subject: [PATCH 6/9] fix: add remove temp dir once action is done --- getdeck/api/get.py | 5 +++- getdeck/api/hosts.py | 45 ++++++++++++++++++-------------- getdeck/api/list.py | 5 +++- getdeck/api/remove.py | 10 +++++-- getdeck/api/stop.py | 5 +++- getdeck/provider/k3d/__init__.py | 2 +- getdeck/utils.py | 9 ++++--- 7 files changed, 52 insertions(+), 29 deletions(-) diff --git a/getdeck/api/get.py b/getdeck/api/get.py index 70b8647..5c9d50a 100644 --- a/getdeck/api/get.py +++ b/getdeck/api/get.py @@ -1,4 +1,5 @@ import logging +import shutil from typing import Callable from getdeck.api import stopwatch, remove @@ -28,7 +29,7 @@ def run_deck( if progress_callback: progress_callback(0) - deckfile, working_dir_path = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(deckfile_location, config) if progress_callback: progress_callback(5) # @@ -109,6 +110,8 @@ def run_deck( if notes := deckfile.get_deck(deck_name).notes: logger.info(notes) + if is_temp_dir: + shutil.rmtree(working_dir_path) if wait: _wait_ready(config, generated_deck, timeout) return True diff --git a/getdeck/api/hosts.py b/getdeck/api/hosts.py index 982ddc9..1181d8b 100644 --- a/getdeck/api/hosts.py +++ b/getdeck/api/hosts.py @@ -1,4 +1,5 @@ import logging +import shutil import socket from python_hosts import Hosts, HostsEntry @@ -17,35 +18,41 @@ def run_hosts( deck_name: str = None, config=default_configuration, ) -> bool: - deckfile, _ = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(deckfile_location, config) deck = deckfile.get_deck(deck_name) deck_hosts = deck.hosts hosts = Hosts() - if host_action == "list": + if deck_hosts: + if host_action == "list": - logger.info("Ingress hosts:") - for host in deck_hosts: - logger.info(f"{host}") - elif host_action == "remove": - logger.debug("Removing hosts from hosts file...") - for host in deck_hosts: - hosts.remove_all_matching(name=host) - hosts.write() - logger.info("Hosts have been removed from hosts file...") + logger.info("Ingress hosts:") + for host in deck_hosts: + logger.info(f"{host}") - elif host_action == "write": - logger.info("Writing hosts to hosts file...") - new_entry = HostsEntry(entry_type="ipv4", address="127.0.0.1", names=deck_hosts) - hosts.add([new_entry]) - hosts.write() - logger.info("Hosts should resolve to '127.0.0.1' now.") + elif host_action == "remove": + logger.debug("Removing hosts from hosts file...") + for host in deck_hosts: + hosts.remove_all_matching(name=host) + hosts.write() + logger.info("Hosts have been removed from hosts file...") - else: - logger.error(f"Unknown host action '{host_action}'") + elif host_action == "write": + + logger.info("Writing hosts to hosts file...") + new_entry = HostsEntry(entry_type="ipv4", address="127.0.0.1", names=deck_hosts) + hosts.add([new_entry]) + hosts.write() + logger.info("Hosts should resolve to '127.0.0.1' now.") + else: + logger.error(f"Unknown host action '{host_action}'") + else: + logger.info("No hosts specified in Deckfile") + if is_temp_dir: + shutil.rmtree(working_dir_path) return True diff --git a/getdeck/api/list.py b/getdeck/api/list.py index c2facec..6ea135d 100644 --- a/getdeck/api/list.py +++ b/getdeck/api/list.py @@ -1,4 +1,5 @@ import logging +import shutil from typing import List from getdeck.api.utils import stopwatch @@ -11,7 +12,9 @@ def get_available_decks(deckfile_location: str, config=default_configuration) -> List: from getdeck.utils import read_deckfile_from_location - deckfile, _ = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(deckfile_location, config) available_decks = deckfile.get_decks() logger.debug(available_decks) + if is_temp_dir: + shutil.rmtree(working_dir_path) return available_decks diff --git a/getdeck/api/remove.py b/getdeck/api/remove.py index 115d2ee..9a9db02 100644 --- a/getdeck/api/remove.py +++ b/getdeck/api/remove.py @@ -1,4 +1,5 @@ import logging +import shutil from typing import Callable from getdeck.configuration import default_configuration @@ -16,12 +17,14 @@ def remove_cluster( ) -> bool: from getdeck.utils import read_deckfile_from_location, ensure_cluster - deckfile, _ = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(deckfile_location, config) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) if k8s_provider.exists(): k8s_provider.delete() else: logger.info("Cluster does not exist") + if is_temp_dir: + shutil.rmtree(working_dir_path) return True @@ -40,7 +43,7 @@ def remove_deck( from getdeck.k8s import k8s_delete_object from getdeck.sources.utils import prepare_k8s_workload_for_deck - deckfile, working_dir_path = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(deckfile_location, config) if progress_callback: progress_callback(10) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) @@ -73,4 +76,7 @@ def remove_deck( logger.info(f"All workloads from Deck {generated_deck.name} removed") else: logger.info("Cluster does not exist") + + if is_temp_dir: + shutil.rmtree(working_dir_path) return True diff --git a/getdeck/api/stop.py b/getdeck/api/stop.py index 77d32c8..a775eac 100644 --- a/getdeck/api/stop.py +++ b/getdeck/api/stop.py @@ -1,4 +1,5 @@ import logging +import shutil from typing import Callable from getdeck.api import stopwatch @@ -16,7 +17,9 @@ def stop_cluster( ) -> bool: from getdeck.utils import read_deckfile_from_location, ensure_cluster - deckfile, _ = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(deckfile_location, config) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) logger.info("Stopping cluster") + if is_temp_dir: + shutil.rmtree(working_dir_path) k8s_provider.stop() diff --git a/getdeck/provider/k3d/__init__.py b/getdeck/provider/k3d/__init__.py index 8e8b6c4..f73ee3b 100644 --- a/getdeck/provider/k3d/__init__.py +++ b/getdeck/provider/k3d/__init__.py @@ -78,7 +78,7 @@ def stop(self): return True def delete(self): - logger.info(f"Deleting the k3d cluster with name {self.k3d_cluster_name}") + logger.info(f"Deleting the k3d cluster with name '{self.k3d_cluster_name}'") arguments = ["cluster", "delete", self.k3d_cluster_name] self._execute(arguments) return True diff --git a/getdeck/utils.py b/getdeck/utils.py index 5a8b25d..4c4a805 100644 --- a/getdeck/utils.py +++ b/getdeck/utils.py @@ -36,8 +36,9 @@ def sniff_protocol(ref: str): def read_deckfile_from_location( location: str, config: ClientConfiguration -) -> Tuple[Deckfile, Optional[str]]: +) -> Tuple[Deckfile, Optional[str], bool]: protocol = sniff_protocol(location) + logger.info(f"Reading Deckfile from: {location}") if location == ".": # load default file from this location return config.deckfile_selector.get( @@ -56,7 +57,7 @@ def read_deckfile_from_location( deckfile = config.deckfile_selector.get( os.path.join(tmp_dir, configuration.DECKFILE_FILE) ) - return deckfile, tmp_dir + return deckfile, tmp_dir, True except GitError as e: shutil.rmtree(tmp_dir) raise RuntimeError(f"Cannot checkout {rev} from {ref}: {e}") @@ -76,7 +77,7 @@ def read_deckfile_from_location( download.close() deckfile = config.deckfile_selector.get(download.name) os.remove(download.name) - return deckfile, None + return deckfile, None, False except Exception as e: download.close() os.remove(download.name) @@ -87,7 +88,7 @@ def read_deckfile_from_location( # this is probably a file system location if os.path.isfile(location): logger.debug("Is file location") - return config.deckfile_selector.get(location), os.path.dirname(location) + return config.deckfile_selector.get(location), os.path.dirname(location), False else: raise RuntimeError(f"Cannot identify {location} as Deckfile") else: From 26de55831f0fad7ccd552476e169c72da474c5f4 Mon Sep 17 00:00:00 2001 From: Michael Schilonka Date: Fri, 19 Aug 2022 14:48:23 +0200 Subject: [PATCH 7/9] fix: code styling --- getdeck/api/get.py | 4 +++- getdeck/api/hosts.py | 16 +++++++++------- getdeck/api/list.py | 4 +++- getdeck/api/remove.py | 12 +++++++++--- getdeck/api/stop.py | 4 +++- getdeck/utils.py | 6 +++++- setup.cfg | 2 +- 7 files changed, 33 insertions(+), 15 deletions(-) diff --git a/getdeck/api/get.py b/getdeck/api/get.py index 5c9d50a..13c3954 100644 --- a/getdeck/api/get.py +++ b/getdeck/api/get.py @@ -29,7 +29,9 @@ def run_deck( if progress_callback: progress_callback(0) - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( + deckfile_location, config + ) if progress_callback: progress_callback(5) # diff --git a/getdeck/api/hosts.py b/getdeck/api/hosts.py index 1181d8b..d29b978 100644 --- a/getdeck/api/hosts.py +++ b/getdeck/api/hosts.py @@ -18,7 +18,9 @@ def run_hosts( deck_name: str = None, config=default_configuration, ) -> bool: - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( + deckfile_location, config + ) deck = deckfile.get_deck(deck_name) deck_hosts = deck.hosts @@ -26,11 +28,9 @@ def run_hosts( if deck_hosts: if host_action == "list": - - logger.info("Ingress hosts:") - for host in deck_hosts: - logger.info(f"{host}") - + logger.info("Ingress hosts:") + for host in deck_hosts: + logger.info(f"{host}") elif host_action == "remove": logger.debug("Removing hosts from hosts file...") @@ -42,7 +42,9 @@ def run_hosts( elif host_action == "write": logger.info("Writing hosts to hosts file...") - new_entry = HostsEntry(entry_type="ipv4", address="127.0.0.1", names=deck_hosts) + new_entry = HostsEntry( + entry_type="ipv4", address="127.0.0.1", names=deck_hosts + ) hosts.add([new_entry]) hosts.write() logger.info("Hosts should resolve to '127.0.0.1' now.") diff --git a/getdeck/api/list.py b/getdeck/api/list.py index 6ea135d..7211a39 100644 --- a/getdeck/api/list.py +++ b/getdeck/api/list.py @@ -12,7 +12,9 @@ def get_available_decks(deckfile_location: str, config=default_configuration) -> List: from getdeck.utils import read_deckfile_from_location - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( + deckfile_location, config + ) available_decks = deckfile.get_decks() logger.debug(available_decks) if is_temp_dir: diff --git a/getdeck/api/remove.py b/getdeck/api/remove.py index 9a9db02..cef0cb7 100644 --- a/getdeck/api/remove.py +++ b/getdeck/api/remove.py @@ -17,7 +17,9 @@ def remove_cluster( ) -> bool: from getdeck.utils import read_deckfile_from_location, ensure_cluster - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( + deckfile_location, config + ) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) if k8s_provider.exists(): k8s_provider.delete() @@ -43,7 +45,9 @@ def remove_deck( from getdeck.k8s import k8s_delete_object from getdeck.sources.utils import prepare_k8s_workload_for_deck - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( + deckfile_location, config + ) if progress_callback: progress_callback(10) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) @@ -52,7 +56,9 @@ def remove_deck( config.kubeconfig = k8s_provider.get_kubeconfig() if k8s_provider.exists(): - generated_deck = prepare_k8s_workload_for_deck(config, deckfile, deck_name, working_dir_path) + generated_deck = prepare_k8s_workload_for_deck( + config, deckfile, deck_name, working_dir_path + ) logger.info(f"Removing Deck {generated_deck.name}") if progress_callback: progress_callback(30) diff --git a/getdeck/api/stop.py b/getdeck/api/stop.py index a775eac..b4c839a 100644 --- a/getdeck/api/stop.py +++ b/getdeck/api/stop.py @@ -17,7 +17,9 @@ def stop_cluster( ) -> bool: from getdeck.utils import read_deckfile_from_location, ensure_cluster - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(deckfile_location, config) + deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( + deckfile_location, config + ) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) logger.info("Stopping cluster") if is_temp_dir: diff --git a/getdeck/utils.py b/getdeck/utils.py index 4c4a805..fad2cc0 100644 --- a/getdeck/utils.py +++ b/getdeck/utils.py @@ -88,7 +88,11 @@ def read_deckfile_from_location( # this is probably a file system location if os.path.isfile(location): logger.debug("Is file location") - return config.deckfile_selector.get(location), os.path.dirname(location), False + return ( + config.deckfile_selector.get(location), + os.path.dirname(location), + False, + ) else: raise RuntimeError(f"Cannot identify {location} as Deckfile") else: diff --git a/setup.cfg b/setup.cfg index 0b192c6..2b1154d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,7 @@ doctests = True enable-extensions = G strictness = long max-line-length = 120 -max-complexity = 25 +max-complexity = 27 exclude = .git,__pycache__,.venv,venv,.eggs,*.egg,testing,stowaway,carrier,cargo ignore = D100, D104, D106, D401, X100, W503, W504, RST303, RST304, DAR103, DAR203 From d6941464da7b08a12ac42fffcb3e30678f6b1b27 Mon Sep 17 00:00:00 2001 From: Michael Schilonka Date: Fri, 19 Aug 2022 18:17:42 +0200 Subject: [PATCH 8/9] chore: merge main into PR --- getdeck/__main__.py | 2 +- getdeck/sources/fetcher.py | 2 ++ getdeck/sources/utils.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/getdeck/__main__.py b/getdeck/__main__.py index 7945396..8005941 100755 --- a/getdeck/__main__.py +++ b/getdeck/__main__.py @@ -164,7 +164,7 @@ def main(): except Exception as e: if args.debug: traceback.print_exc() - logger.fatal(f"There was an error running Deck: {e}") + logger.fatal(f"There was an error running deck: {e}") exit(1) diff --git a/getdeck/sources/fetcher.py b/getdeck/sources/fetcher.py index 0859257..9cab31f 100644 --- a/getdeck/sources/fetcher.py +++ b/getdeck/sources/fetcher.py @@ -25,11 +25,13 @@ def __init__( source: Union[DeckfileFileSource, DeckfileKustomizeSource, DeckfileHelmSource], config: ClientConfiguration, namespace: str, + working_dir: str = None ): self.path = path self.source = source self.config = config self.namespace = namespace + self.working_dir = working_dir @property def not_supported_message(self): diff --git a/getdeck/sources/utils.py b/getdeck/sources/utils.py index fddf2c7..c612ab9 100644 --- a/getdeck/sources/utils.py +++ b/getdeck/sources/utils.py @@ -49,7 +49,7 @@ def prepare_k8s_workload_for_deck( namespace = deck.namespace or "default" fetcher_context.strategy = strategy( - deckfile.file_path, source, config, namespace + deckfile.file_path, source, config, namespace, working_dir ) # fetch source files From da91248497d4b5ab654d9a63dfd846635f9a6f5a Mon Sep 17 00:00:00 2001 From: Michael Schilonka Date: Fri, 19 Aug 2022 18:18:43 +0200 Subject: [PATCH 9/9] fix: code styling --- getdeck/sources/fetcher.py | 2 +- getdeck/sources/file.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/getdeck/sources/fetcher.py b/getdeck/sources/fetcher.py index 9cab31f..b8831a0 100644 --- a/getdeck/sources/fetcher.py +++ b/getdeck/sources/fetcher.py @@ -25,7 +25,7 @@ def __init__( source: Union[DeckfileFileSource, DeckfileKustomizeSource, DeckfileHelmSource], config: ClientConfiguration, namespace: str, - working_dir: str = None + working_dir: str = None, ): self.path = path self.source = source diff --git a/getdeck/sources/file.py b/getdeck/sources/file.py index 3de079f..2dfdfff 100644 --- a/getdeck/sources/file.py +++ b/getdeck/sources/file.py @@ -1,6 +1,5 @@ import logging import os -from pathlib import PurePath import tempfile from typing import List