From 1a2165f4fcc379a21c918efd2d33e471db4dbccc Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Tue, 18 Oct 2022 16:24:45 +0200 Subject: [PATCH 1/7] refactor: rename DeckFileData to DeckfileAux --- getdeck/deckfile/fetch/deck_fetcher.py | 16 ++++++++-------- getdeck/utils.py | 4 ++-- test/src/test_deck_fetch_behavior.py | 14 +++++++------- test/src/test_deck_fetcher.py | 4 ++-- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/getdeck/deckfile/fetch/deck_fetcher.py b/getdeck/deckfile/fetch/deck_fetcher.py index cd03a65..b4c7a6e 100644 --- a/getdeck/deckfile/fetch/deck_fetcher.py +++ b/getdeck/deckfile/fetch/deck_fetcher.py @@ -19,7 +19,7 @@ class FetchError(Exception): pass -class DeckFileData(BaseModel): +class DeckfileAux(BaseModel): argument_location: str cwd: str = os.getcwd() path: str = None @@ -30,16 +30,16 @@ class DeckFileData(BaseModel): class DeckFetchBehavior(ABC): @abstractmethod - def fetch(self, data: DeckFileData) -> DeckFileData: + def fetch(self, data: DeckfileAux) -> DeckfileAux: pass @abstractmethod - def clean_up(self, data: DeckFileData): + def clean_up(self, data: DeckfileAux): pass class Git(DeckFetchBehavior): - def fetch(self, data: DeckFileData) -> DeckFileData: + def fetch(self, data: DeckfileAux) -> DeckfileAux: location = data.argument_location if "#" in location: @@ -65,12 +65,12 @@ def fetch(self, data: DeckFileData) -> DeckFileData: return data - def clean_up(self, data: DeckFileData): + def clean_up(self, data: DeckfileAux): shutil.rmtree(data.working_dir_path) class Http(DeckFetchBehavior): - def fetch(self, data: DeckFileData) -> DeckFileData: + def fetch(self, data: DeckfileAux) -> DeckfileAux: location = data.argument_location download = tempfile.NamedTemporaryFile(delete=False) @@ -95,7 +95,7 @@ def fetch(self, data: DeckFileData) -> DeckFileData: return data - def clean_up(self, data: DeckFileData): + def clean_up(self, data: DeckfileAux): os.remove(os.path.join(data.path, data.name)) @@ -111,7 +111,7 @@ def fetch_behavior(self) -> DeckFetchBehavior: def fetch_behavior(self, fetch_behavior: DeckFetchBehavior) -> None: self._fetch_behavior = fetch_behavior - def fetch(self, data: DeckFileData) -> DeckFileData: + def fetch(self, data: DeckfileAux) -> DeckfileAux: data = self._fetch_behavior.fetch(data=data) return data diff --git a/getdeck/utils.py b/getdeck/utils.py index 82b6fc6..ffeca94 100644 --- a/getdeck/utils.py +++ b/getdeck/utils.py @@ -9,7 +9,7 @@ from getdeck.configuration import ClientConfiguration from getdeck.deckfile.fetch.deck_fetcher import ( DeckFetcher, - DeckFileData, + DeckfileAux, select_fetch_behavior, ) from getdeck.deckfile.fetch.utils import get_path_and_name @@ -44,7 +44,7 @@ def read_deckfile_from_location( logger.info(f"Reading Deckfile from: {location}") # fetch - data = DeckFileData(argument_location=location) + data = DeckfileAux(argument_location=location) fetch_behavior = select_fetch_behavior(location=location) if fetch_behavior: deck_fetcher = DeckFetcher(fetch_behavior=fetch_behavior) diff --git a/test/src/test_deck_fetch_behavior.py b/test/src/test_deck_fetch_behavior.py index 7cc5782..8a65cac 100644 --- a/test/src/test_deck_fetch_behavior.py +++ b/test/src/test_deck_fetch_behavior.py @@ -1,12 +1,12 @@ import os from unittest import TestCase from getdeck import configuration -from getdeck.deckfile.fetch.deck_fetcher import DeckFileData, FetchError, Git, Http +from getdeck.deckfile.fetch.deck_fetcher import DeckfileAux, FetchError, Git, Http class GitTest(TestCase): def test_git(self): - data = DeckFileData(argument_location="git@github.com:Getdeck/getdeck.git") + data = DeckfileAux(argument_location="git@github.com:Getdeck/getdeck.git") fetch_behavior = Git() data = fetch_behavior.fetch(data=data) @@ -19,7 +19,7 @@ def test_git(self): self.assertFalse(os.path.isdir(data.working_dir_path)) def test_branch(self): - data = DeckFileData(argument_location="git@github.com:Getdeck/getdeck.git#main") + data = DeckfileAux(argument_location="git@github.com:Getdeck/getdeck.git#main") fetch_behavior = Git() data = fetch_behavior.fetch(data=data) @@ -32,7 +32,7 @@ def test_branch(self): self.assertFalse(os.path.isdir(data.working_dir_path)) def test_branch_invalid(self): - data = DeckFileData( + data = DeckfileAux( argument_location="git@github.com:Getdeck/getdeck.git#invalid" ) @@ -41,7 +41,7 @@ def test_branch_invalid(self): data = fetch_behavior.fetch(data=data) def test_https(self): - data = DeckFileData(argument_location="https://github.com/Getdeck/getdeck.git") + data = DeckfileAux(argument_location="https://github.com/Getdeck/getdeck.git") fetch_behavior = Git() data = fetch_behavior.fetch(data=data) @@ -56,7 +56,7 @@ def test_https(self): class HttpTest(TestCase): def test_default(self): - data = DeckFileData( + data = DeckfileAux( argument_location="https://raw.githubusercontent.com/Getdeck/getdeck/main/test/deckfile/deck.empty.yaml" ) @@ -71,7 +71,7 @@ def test_default(self): self.assertFalse(os.path.isfile(os.path.join(data.path, data.name))) def test_url_invalid(self): - data = DeckFileData( + data = DeckfileAux( argument_location="https://raw.githubusercontent.com/Getdeck/getdeck/invalid/deck.yaml" ) diff --git a/test/src/test_deck_fetcher.py b/test/src/test_deck_fetcher.py index 2cd1989..59c4e43 100644 --- a/test/src/test_deck_fetcher.py +++ b/test/src/test_deck_fetcher.py @@ -1,7 +1,7 @@ from unittest import TestCase from getdeck.deckfile.fetch.deck_fetcher import ( DeckFetcher, - DeckFileData, + DeckfileAux, Git, Http, select_fetch_behavior, @@ -40,7 +40,7 @@ class DeckFetcherTest(TestCase): def test_default(self): location = "git@github.com:Getdeck/getdeck.git" - data = DeckFileData(argument_location=location) + data = DeckfileAux(argument_location=location) fetch_behavior = select_fetch_behavior(location=location) deck_fetcher = DeckFetcher(fetch_behavior=fetch_behavior) data = deck_fetcher.fetch(data=data) From 2aca25b3ad83969f54c6e661a211e9e643796cdf Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Thu, 20 Oct 2022 18:22:16 +0200 Subject: [PATCH 2/7] WIP: refactoring --- getdeck/api/get.py | 26 ++-- getdeck/api/hosts.py | 6 +- getdeck/api/list.py | 6 +- getdeck/api/remove.py | 12 +- getdeck/api/stop.py | 6 +- getdeck/deckfile/file.py | 27 ++-- getdeck/{deckfile => }/fetch/deck_fetcher.py | 60 +++----- getdeck/fetch/fetch.py | 91 ++++++++++++ getdeck/fetch/source_fetcher.py | 138 +++++++++++++++++++ getdeck/fetch/types.py | 80 +++++++++++ getdeck/{deckfile => }/fetch/utils.py | 0 getdeck/sources/fetcher.py | 85 ------------ getdeck/sources/file.py | 108 +++------------ getdeck/sources/generator.py | 69 ++++++++++ getdeck/sources/helm.py | 13 +- getdeck/sources/inline.py | 22 +++ getdeck/sources/kustomize.py | 7 +- getdeck/sources/selector.py | 35 +++-- getdeck/sources/tooler.py | 63 +++------ getdeck/sources/utils.py | 57 +++++--- getdeck/utils.py | 62 --------- test/sources/deck.file.yaml | 3 +- test/src/test_deck_fetch_behavior.py | 2 +- test/src/test_deck_fetch_utils.py | 2 +- test/src/test_deck_fetcher.py | 16 +-- test/src/test_deckfile.py | 9 +- 26 files changed, 591 insertions(+), 414 deletions(-) rename getdeck/{deckfile => }/fetch/deck_fetcher.py (63%) create mode 100644 getdeck/fetch/fetch.py create mode 100644 getdeck/fetch/source_fetcher.py create mode 100644 getdeck/fetch/types.py rename getdeck/{deckfile => }/fetch/utils.py (100%) delete mode 100644 getdeck/sources/fetcher.py create mode 100644 getdeck/sources/generator.py create mode 100644 getdeck/sources/inline.py diff --git a/getdeck/api/get.py b/getdeck/api/get.py index f9ecba3..5012401 100644 --- a/getdeck/api/get.py +++ b/getdeck/api/get.py @@ -1,6 +1,4 @@ import logging -import os -import shutil from typing import Callable from getdeck.api import stopwatch, remove @@ -23,22 +21,23 @@ def run_deck( # noqa: C901 progress_callback: Callable = None, ) -> bool: from getdeck.sources.utils import prepare_k8s_workload_for_deck - from getdeck.utils import read_deckfile_from_location, ensure_cluster + from getdeck.utils import ensure_cluster from getdeck.k8s import k8s_create_or_patch, get_ingress_rules + from getdeck.fetch.fetch import fetch_data cluster_created = False if progress_callback: progress_callback(0) - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( - deckfile_location, config - ) + data_aux = fetch_data(deckfile_location, deck_name=deck_name) if progress_callback: progress_callback(5) # # 1. set up a local K8s cluster # - k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=True) + k8s_provider = ensure_cluster( + data_aux.deckfile, config, ignore_cluster, do_install=True + ) if progress_callback: progress_callback(10) # 1.b check or set up local cluster @@ -62,9 +61,7 @@ def run_deck( # noqa: C901 # 2. generate the Deck's workload # try: - generated_deck = prepare_k8s_workload_for_deck( - config, deckfile, deck_name, working_dir_path - ) + generated_deck = prepare_k8s_workload_for_deck(config, data_aux, deck_name) except Exception as e: if cluster_created: # remove this just created cluster as it probably is in an inconsistent state from the beginning @@ -124,17 +121,16 @@ def run_deck( # noqa: C901 if ingress_rules: for host, path in ingress_rules: logger.info(f"Ingress: {host} -> {path}") - handle_hosts_resolution(deckfile_location, deckfile, deck_name) + handle_hosts_resolution(deckfile_location, data_aux.deckfile, deck_name) logger.info(f"Published ports are: {k8s_provider.get_ports()}") - if notes := deckfile.get_deck(deck_name).notes: + if notes := data_aux.deckfile.get_deck(deck_name).notes: logger.info(notes) - # TODO: refactor/remove? - if is_temp_dir and os.path.isdir(working_dir_path): - shutil.rmtree(working_dir_path) + del data_aux if wait: _wait_ready(config, generated_deck, timeout) + return True diff --git a/getdeck/api/hosts.py b/getdeck/api/hosts.py index ad28261..e03fa79 100644 --- a/getdeck/api/hosts.py +++ b/getdeck/api/hosts.py @@ -7,7 +7,7 @@ from getdeck.api import stopwatch from getdeck.configuration import default_configuration -from getdeck.utils import read_deckfile_from_location +from getdeck.fetch.fetch import fetch_data logger = logging.getLogger("deck") @@ -19,8 +19,8 @@ def run_hosts( deck_name: str = None, config=default_configuration, ) -> bool: - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( - deckfile_location, config + deckfile, working_dir_path, is_temp_dir = fetch_data( + deckfile_location, deck_name=deck_name ) deck = deckfile.get_deck(deck_name) deck_hosts = deck.hosts diff --git a/getdeck/api/list.py b/getdeck/api/list.py index cb682dd..d78aeca 100644 --- a/getdeck/api/list.py +++ b/getdeck/api/list.py @@ -11,11 +11,9 @@ @stopwatch def get_available_decks(deckfile_location: str, config=default_configuration) -> List: - from getdeck.utils import read_deckfile_from_location + from getdeck.utils import fetch_data - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( - deckfile_location, config - ) + deckfile, working_dir_path, is_temp_dir = fetch_data(deckfile_location) available_decks = deckfile.get_decks() logger.debug(available_decks) diff --git a/getdeck/api/remove.py b/getdeck/api/remove.py index cd52f26..b1bca89 100644 --- a/getdeck/api/remove.py +++ b/getdeck/api/remove.py @@ -16,11 +16,9 @@ def remove_cluster( config=default_configuration, ignore_cluster: bool = False, ) -> bool: - from getdeck.utils import read_deckfile_from_location, ensure_cluster + from getdeck.utils import fetch_data, ensure_cluster - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( - deckfile_location, config - ) + deckfile, working_dir_path, is_temp_dir = fetch_data(deckfile_location) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) if k8s_provider.exists(): k8s_provider.delete() @@ -45,12 +43,12 @@ def remove_deck( if progress_callback: progress_callback(0) - from getdeck.utils import read_deckfile_from_location, ensure_cluster + from getdeck.utils import fetch_data, ensure_cluster from getdeck.k8s import k8s_delete_object from getdeck.sources.utils import prepare_k8s_workload_for_deck - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( - deckfile_location, config + deckfile, working_dir_path, is_temp_dir = fetch_data( + deckfile_location, deck_name=deck_name ) if progress_callback: progress_callback(10) diff --git a/getdeck/api/stop.py b/getdeck/api/stop.py index 69b55fc..9c741c2 100644 --- a/getdeck/api/stop.py +++ b/getdeck/api/stop.py @@ -16,11 +16,9 @@ def stop_cluster( config=default_configuration, progress_callback: Callable = None, ) -> bool: - from getdeck.utils import read_deckfile_from_location, ensure_cluster + from getdeck.utils import fetch_data, ensure_cluster - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location( - deckfile_location, config - ) + deckfile, working_dir_path, is_temp_dir = fetch_data(deckfile_location) k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) logger.info("Stopping cluster") diff --git a/getdeck/deckfile/file.py b/getdeck/deckfile/file.py index 43f9c33..c5e64dd 100644 --- a/getdeck/deckfile/file.py +++ b/getdeck/deckfile/file.py @@ -49,10 +49,14 @@ class DeckfileHelmSource(BaseModel): helmPlugins: List[str] = None +class DeckfileInlineSource(BaseModel): + type: str = "inline" + content: Dict = None + + class DeckfileFileSource(BaseModel): type: str = "file" ref: str = None - content: Dict = None targetRevision: str = "" path: str = "" @@ -79,9 +83,10 @@ class DeckfileDeck(BaseModel): hosts: List[str] = [] sources: List[ Union[ - DeckfileHelmSource, - DeckfileDirectorySource, + DeckfileInlineSource, DeckfileFileSource, + DeckfileDirectorySource, + DeckfileHelmSource, DeckfileKustomizeSource, ] ] @@ -93,14 +98,14 @@ def __init__(self, *args, **data): if tsources: try: for source in tsources: - if source["type"].lower() == "helm": - self.sources.append(DeckfileHelmSource(**source)) - elif source["type"].lower() == "file": - self.sources.append(DeckfileFileSource(**source)) - elif source["type"].lower() == "directory": - self.sources.append(DeckfileDirectorySource(**source)) - elif source["type"].lower() == "kustomize": - self.sources.append(DeckfileKustomizeSource(**source)) + source_class = { + "inline": DeckfileInlineSource, + "file": DeckfileFileSource, + "directory": DeckfileDirectorySource, + "kustomize": DeckfileKustomizeSource, + "helm": DeckfileHelmSource, + }.get(source["type"].lower()) + self.sources.append(source_class(**source)) except KeyError: raise DeckfileError( f"A source from Deck {data.get('name')} did not specify the 'type' argument." diff --git a/getdeck/deckfile/fetch/deck_fetcher.py b/getdeck/fetch/deck_fetcher.py similarity index 63% rename from getdeck/deckfile/fetch/deck_fetcher.py rename to getdeck/fetch/deck_fetcher.py index b4c7a6e..ad9b341 100644 --- a/getdeck/deckfile/fetch/deck_fetcher.py +++ b/getdeck/fetch/deck_fetcher.py @@ -2,15 +2,13 @@ import os from typing import Optional -from pydantic import BaseModel import logging -import shutil import tempfile import requests from git import Repo, GitError -from getdeck import configuration +from getdeck.fetch.types import DeckfileAux, TemporaryData logger = logging.getLogger("deck") @@ -19,24 +17,11 @@ class FetchError(Exception): pass -class DeckfileAux(BaseModel): - argument_location: str - cwd: str = os.getcwd() - path: str = None - name: str = configuration.DECKFILE_FILE - working_dir_path: str = None - is_temp_dir: bool = False - - class DeckFetchBehavior(ABC): @abstractmethod def fetch(self, data: DeckfileAux) -> DeckfileAux: pass - @abstractmethod - def clean_up(self, data: DeckfileAux): - pass - class Git(DeckFetchBehavior): def fetch(self, data: DeckfileAux) -> DeckfileAux: @@ -48,34 +33,32 @@ def fetch(self, data: DeckfileAux) -> DeckfileAux: ref = location rev = "HEAD" - tmp_dir = tempfile.mkdtemp() - data.path = tmp_dir - data.working_dir_path = tmp_dir - data.is_temp_dir = True + temporary_folder = tempfile.mkdtemp() + data.path = temporary_folder + data.working_dir_path = temporary_folder + data.temporary_data = TemporaryData(data=temporary_folder, is_folder=True) try: - repo = Repo.clone_from(ref, tmp_dir) + repo = Repo.clone_from(ref, temporary_folder) repo.git.checkout(rev) except GitError as e: - self.clean_up(data=data) + del data raise FetchError(f"Cannot checkout {rev} from {ref}: {e}") except Exception as e: - self.clean_up(data=data) + del data # noqa: F821 raise e - return data - - def clean_up(self, data: DeckfileAux): - shutil.rmtree(data.working_dir_path) + return data # noqa: F821 class Http(DeckFetchBehavior): def fetch(self, data: DeckfileAux) -> DeckfileAux: location = data.argument_location - download = tempfile.NamedTemporaryFile(delete=False) - data.path = os.path.dirname(download.name) - data.name = os.path.basename(download.name) + temporary_file = tempfile.NamedTemporaryFile(delete=False) + data.path = os.path.dirname(temporary_file.name) + data.name = os.path.basename(temporary_file.name) + data.temporary_data = TemporaryData(data=temporary_file.name, is_file=True) try: logger.debug(f"Requesting {location}") @@ -83,20 +66,17 @@ def fetch(self, data: DeckfileAux) -> DeckfileAux: res.raise_for_status() for chunk in res.iter_content(chunk_size=4096): if chunk: - download.write(chunk) - download.flush() - download.close() + temporary_file.write(chunk) + temporary_file.flush() + temporary_file.close() except Exception as e: - download.close() - self.clean_up(data=data) + temporary_file.close() + del data raise FetchError( f"Cannot download Deckfile from http(s) location {location}: {e}" ) - return data - - def clean_up(self, data: DeckfileAux): - os.remove(os.path.join(data.path, data.name)) + return data # noqa: F821 class DeckFetcher: @@ -116,7 +96,7 @@ def fetch(self, data: DeckfileAux) -> DeckfileAux: return data -def select_fetch_behavior(location: str) -> Optional[DeckFetchBehavior]: +def select_deck_fetch_behavior(location: str) -> Optional[DeckFetchBehavior]: if "#" in location: location, _ = location.split("#") diff --git a/getdeck/fetch/fetch.py b/getdeck/fetch/fetch.py new file mode 100644 index 0000000..55dfba0 --- /dev/null +++ b/getdeck/fetch/fetch.py @@ -0,0 +1,91 @@ +import logging +import os +from typing import List + + +from getdeck.fetch.deck_fetcher import ( + DeckFetcher, + DeckfileAux, + select_deck_fetch_behavior, +) +from getdeck.fetch.types import DataAux +from getdeck.fetch.utils import get_path_and_name +from getdeck.deckfile.file import DeckfileDeck + +from getdeck.deckfile.selector import deckfile_selector +from getdeck.fetch.source_fetcher import ( + SourceAux, + SourceFetcher, + select_source_fetch_behavior, +) + + +logger = logging.getLogger("deck") + + +def fetch_sources(deck: DeckfileDeck) -> List[SourceAux]: + source_fetcher = SourceFetcher(fetch_behavior=None) + + source_auxs = [] + for source in deck.sources: + ref = getattr(source, "ref", None) + logger.info( + "Fetching source " f"{source.__class__.__name__}: {ref or 'no ref'}" + ) + + data = SourceAux(location=ref, source=source) + fetch_behavior = select_source_fetch_behavior(source=source) + source_fetcher.fetch_behavior = fetch_behavior + if not fetch_behavior: + continue + + try: + source_dict = source.dict() + data = source_fetcher.fetch(data=data, **source_dict) + except Exception as e: + logger.debug(str(e)) + del data, source_auxs[:] + raise e + + source_auxs.append(data) # noqa: F821 + + return source_auxs + + +def fetch_data(location: str, deck_name: str = None, *args, **kwargs) -> DataAux: + """ + delete returned DataAux to clean up temporary resources + """ + + logger.info(f"Reading Deckfile from: {location}") + data_aux = DataAux() + + # fetch + deckfile_aux = DeckfileAux(argument_location=location) + fetch_behavior = select_deck_fetch_behavior(location=location) + if fetch_behavior: + deck_fetcher = DeckFetcher(fetch_behavior=fetch_behavior) + deckfile_aux = deck_fetcher.fetch(data=deckfile_aux) + else: + # local path and name + path, name = get_path_and_name(location=location) + deckfile_aux.path = path + deckfile_aux.name = name + deckfile_aux.working_dir_path = os.path.dirname(location) + + data_aux.deckfile_aux = deckfile_aux + + # validate + file = os.path.join(deckfile_aux.path, deckfile_aux.name) + if not os.path.isfile(file): + del data_aux + raise RuntimeError(f"Cannot identify {location} as Deckfile") + + # parse + fetch sources + deckfile = deckfile_selector.get(file) + data_aux.deckfile = deckfile + deck = deckfile.get_deck(deck_name) + source_auxs = fetch_sources(deck=deck) + data_aux.source_auxs = source_auxs + + return data_aux diff --git a/getdeck/fetch/source_fetcher.py b/getdeck/fetch/source_fetcher.py new file mode 100644 index 0000000..8dcb47e --- /dev/null +++ b/getdeck/fetch/source_fetcher.py @@ -0,0 +1,138 @@ +from abc import ABC, abstractmethod +import os +from typing import Optional + +import logging +import tempfile + +import requests +from git import Repo, GitError + +from getdeck.fetch.types import SourceAux, TemporaryData + + +logger = logging.getLogger("deck") + + +class FetchError(Exception): + pass + + +class SourceFetchBehavior(ABC): + @abstractmethod + def fetch(self, data: SourceAux) -> SourceAux: + pass + + +class Git(SourceFetchBehavior): + def fetch(self, data: SourceAux, *args, **kwargs) -> SourceAux: + location = data.location + + if "#" in location: + ref, rev = location.split("#") + else: + ref = location + rev = "HEAD" + + rev = kwargs.get("targetRevision", rev) + path = kwargs.get("path", "") + + temporary_folder = tempfile.mkdtemp() + path = os.path.join(temporary_folder, path) + data.path = os.path.dirname(path) + data.name = os.path.basename(path) + data.temporary_data = TemporaryData(data=temporary_folder, is_folder=True) + + try: + repo = Repo.clone_from(ref, temporary_folder) + repo.git.checkout(rev) + except GitError as e: + raise FetchError(f"Cannot checkout {rev} from {ref}: {e}") + except Exception as e: + raise e + + return data + + +class Http(SourceFetchBehavior): + def fetch(self, data: SourceAux, *args, **kwargs) -> SourceAux: + temporary_file = tempfile.NamedTemporaryFile(delete=False) + data.path = os.path.dirname(temporary_file.name) + data.name = os.path.basename(temporary_file.name) + data.temporary_data = TemporaryData(data=temporary_file.name, is_file=True) + + try: + logger.debug(f"Requesting {data.location}") + with requests.get(data.location, stream=True, timeout=10.0) as res: + res.raise_for_status() + for chunk in res.iter_content(chunk_size=4096): + if chunk: + temporary_file.write(chunk) + temporary_file.flush() + temporary_file.close() + except Exception as e: + temporary_file.close() + raise FetchError( + f"Cannot download Source from http(s) location {data.location}: {e}" + ) + + return data + + +class Local(SourceFetchBehavior): + def fetch(self, data: SourceAux, *args, **kwargs) -> SourceAux: + data.path = os.path.dirname(data.location) + data.name = os.path.basename(data.location) + return data + + +class Content(SourceFetchBehavior): + def fetch(self, data: SourceAux, *args, **kwargs) -> SourceAux: + return data # TODO: remove Content class? + + +class SourceFetcher: + def __init__(self, fetch_behavior: SourceFetchBehavior) -> None: + self._fetch_behavior = fetch_behavior + + @property + def fetch_behavior(self) -> SourceFetchBehavior: + return self._fetch_behavior + + @fetch_behavior.setter + def fetch_behavior(self, fetch_behavior: SourceFetchBehavior) -> None: + self._fetch_behavior = fetch_behavior + + def fetch(self, data: SourceAux, *args, **kwargs) -> SourceAux: + data = self._fetch_behavior.fetch(data, *args, **kwargs) + return data + + +def select_source_fetch_behavior(source) -> Optional[SourceFetchBehavior]: + try: + _ = source.content + has_content = True + except Exception: + has_content = False + + ref = getattr(source, "ref", None) + if not ref: + if has_content: + return Content() + return None + + if "#" in ref: + ref, _ = ref.split("#") + + ref_lo = ref.lower() + + if ref_lo.startswith("git") or ref_lo.endswith(".git"): + return Git() + + if ref_lo.startswith("https") or ref_lo.startswith("http"): + return Http() + + if ref_lo[0] in "./~": + return Local() + + return None diff --git a/getdeck/fetch/types.py b/getdeck/fetch/types.py new file mode 100644 index 0000000..c8a5ad6 --- /dev/null +++ b/getdeck/fetch/types.py @@ -0,0 +1,80 @@ +from typing import Any, List, Optional, Union + +from pydantic import BaseModel +import os + +import shutil + + +from getdeck import configuration +from getdeck.deckfile.file import ( + DeckfileDirectorySource, + DeckfileFileSource, + DeckfileHelmSource, + DeckfileInlineSource, + DeckfileKustomizeSource, +) + + +class TemporaryData(BaseModel): + data: str + is_file: bool = False + is_folder: bool = False + + +class SourceAux(BaseModel): + location: str = None + path: str = None + name: str = None + temporary_data: Optional[TemporaryData] = None + + source: Union[ + DeckfileInlineSource, + DeckfileFileSource, + DeckfileDirectorySource, + DeckfileHelmSource, + DeckfileKustomizeSource, + ] = None + + def __del__(self): + if not self.temporary_data: + return + + if self.temporary_data.is_file: + os.remove(self.temporary_data.data) + return + + if self.temporary_data.is_folder: + shutil.rmtree(self.temporary_data.data) + return + + +class DeckfileAux(BaseModel): + argument_location: str # TODO: rename + cwd: str = os.getcwd() + path: str = None + name: str = configuration.DECKFILE_FILE + working_dir_path: str = None + temporary_data: Optional[TemporaryData] = None + + def __del__(self): + if not self.temporary_data: + return + + if self.temporary_data.is_file: + os.remove(self.temporary_data.data) + return + + if self.temporary_data.is_folder: + shutil.rmtree(self.temporary_data.data) + return + + +class DataAux(BaseModel): + deckfile: Any = None # TODO: typing? + deckfile_aux: DeckfileAux = None + source_auxs: List[SourceAux] = [] + + def __del__(self): + del self.deckfile_aux + del self.source_auxs[:] diff --git a/getdeck/deckfile/fetch/utils.py b/getdeck/fetch/utils.py similarity index 100% rename from getdeck/deckfile/fetch/utils.py rename to getdeck/fetch/utils.py diff --git a/getdeck/sources/fetcher.py b/getdeck/sources/fetcher.py deleted file mode 100644 index b8831a0..0000000 --- a/getdeck/sources/fetcher.py +++ /dev/null @@ -1,85 +0,0 @@ -import logging -from operator import methodcaller -from typing import List, Union - -from getdeck.configuration import ClientConfiguration -from getdeck.deckfile.file import ( - DeckfileFileSource, - DeckfileKustomizeSource, - DeckfileHelmSource, -) -from getdeck.sources.types import K8sSourceFile -from getdeck.utils import sniff_protocol - -logger = logging.getLogger("deck") - - -class FetcherError(Exception): - pass - - -class Fetcher: - def __init__( - self, - path: str, - source: Union[DeckfileFileSource, DeckfileKustomizeSource, DeckfileHelmSource], - config: ClientConfiguration, - namespace: str, - working_dir: str = None, - ): - self.path = path - self.source = source - self.config = config - self.namespace = namespace - self.working_dir = working_dir - - @property - def not_supported_message(self): - return "Could not fetch source" - - def fetch(self, **kwargs) -> List[K8sSourceFile]: - handler = methodcaller(f"fetch_{self.type}", **kwargs) - try: - return handler(self) - except NotImplementedError: - logger.warning(self.not_supported_message) - return [] - - @property - def type(self) -> str: - if getattr(self.source, "content", None) is not None: - return "content" - protocol = sniff_protocol(self.source.ref) - return protocol - - def fetch_git(self, **kwargs): - raise NotImplementedError - - def fetch_http(self, **kwargs): - raise NotImplementedError - - def fetch_https(self, **kwargs): - raise NotImplementedError - - def fetch_local(self, **kwargs): - raise NotImplementedError - - def fetch_content(self, **kwargs): - raise NotImplementedError - - -class FetcherContext: - def __init__(self, strategy: Fetcher) -> None: - self._strategy = strategy - - @property - def strategy(self) -> Fetcher: - return self._strategy - - @strategy.setter - def strategy(self, strategy: Fetcher) -> None: - self._strategy = strategy - - def fetch_source_files(self) -> List[K8sSourceFile]: - source_files = self._strategy.fetch() - return source_files diff --git a/getdeck/sources/file.py b/getdeck/sources/file.py index 2dfdfff..3cd3a0a 100644 --- a/getdeck/sources/file.py +++ b/getdeck/sources/file.py @@ -1,23 +1,17 @@ import logging import os -import tempfile from typing import List -import requests import yaml +from getdeck.fetch.types import DeckfileAux, SourceAux -from getdeck.sources.fetcher import Fetcher, FetcherError +from getdeck.sources.generator import RenderBehavior, RenderError from getdeck.sources.types import K8sSourceFile -from git import Repo logger = logging.getLogger("deck") -class FileFetcher(Fetcher): - @property - def not_supported_message(self): - return f"Protocol {self.type} not supported for {type(self.source).__name__}" - +class File(RenderBehavior): def _parse_source_file(self, ref: str) -> List[K8sSourceFile]: with open(ref, "r") as input_file: docs = yaml.load_all(input_file.read(), Loader=yaml.FullLoader) @@ -30,20 +24,11 @@ def _parse_source_file(self, ref: str) -> List[K8sSourceFile]: ) return k8s_workload_files - def _parse_source_files(self, refs: List[str]) -> List[K8sSourceFile]: - k8s_workload_files = [] - for ref in refs: - workloads = self._parse_source_file(ref=ref) - k8s_workload_files += workloads - return k8s_workload_files - def _parse_source_directory(self, ref: str) -> List[K8sSourceFile]: refs = [] if not os.path.isdir(ref): - raise FetcherError( - f"The provided path does not point to a directory: {ref}" - ) + raise RenderError(f"The provided path does not point to a directory: {ref}") extensions = (".yaml", ".yml") for file in os.listdir(ref): @@ -51,83 +36,32 @@ def _parse_source_directory(self, ref: str) -> List[K8sSourceFile]: refs.append(os.path.join(ref, file)) # parse workloads - k8s_workload_files = self._parse_source_files(refs=refs) + k8s_workload_files = [] + for ref in refs: + workloads = self._parse_source_file(ref=ref) + k8s_workload_files += workloads + return k8s_workload_files - def _parse_source(self, ref: str, working_dir: str = None) -> List[K8sSourceFile]: - if working_dir: - ref = os.path.join(working_dir, ref.removeprefix("./")) + def _parse_source(self, ref: str) -> List[K8sSourceFile]: if os.path.isdir(ref): - k8s_workload_files = self._parse_source_directory( - ref=ref, - ) + k8s_workload_files = self._parse_source_directory(ref=ref) else: k8s_workload_files = self._parse_source_file(ref=ref) return k8s_workload_files - def fetch_content(self, **kwargs) -> List[K8sSourceFile]: - source_file = K8sSourceFile( - name="Deckfile", content=self.source.content, namespace=self.namespace - ) - return [source_file] - - def fetch_http(self, **kwargs) -> List[K8sSourceFile]: - k8s_workload_files = [] + def render(self, deckfile_aux: DeckfileAux, source_aux: SourceAux, **kwargs): try: - logger.debug(f"Requesting file {self.source.ref}") - with requests.get(self.source.ref, timeout=10.0) as res: - res.raise_for_status() - docs = yaml.load_all(res.content, Loader=yaml.FullLoader) - - for doc in docs: - if doc: - k8s_workload_files.append( - K8sSourceFile( - name=self.source.ref, content=doc, namespace=self.namespace - ) - ) - return k8s_workload_files - except Exception as e: - logger.error(f"Error loading file from http {e}") - raise e - - def fetch_https(self, **kwargs): - return self.fetch_http(**kwargs) - - def fetch_local(self, **kwargs): - try: - logger.debug(f"Reading file {self.source.ref}") - if not os.path.isabs(self.source.ref): - fpath = os.path.join( - self.working_dir, self.source.ref.removeprefix("./") + source_file = os.path.join(source_aux.path, source_aux.name or "") + logger.debug(f"Render file {source_file}") + if not os.path.isabs(source_file): + source_file = os.path.join( + deckfile_aux.path, source_file.removeprefix("./") ) - k8s_workload_files = self._parse_source(ref=fpath) - else: - k8s_workload_files = self._parse_source(ref=self.source.ref) - return k8s_workload_files - except Exception as e: - logger.error(f"Error loading file from http {e}") - raise e - def fetch_git(self, **kwargs) -> List[K8sSourceFile]: - try: - with tempfile.TemporaryDirectory() as tmp_source: - logger.debug(f"Cloning from {self.source.ref} to {tmp_source}") - - source_path = "" - if self.source.path: - source_path = self.source.path - - # clone & checkout repository - repo = Repo.clone_from(self.source.ref, tmp_source) - if self.source.targetRevision: - repo.git.checkout(self.source.targetRevision) - - # file / directory - tmp_source_path = os.path.join(tmp_source, source_path) - k8s_workload_files = self._parse_source(ref=tmp_source_path) - - return k8s_workload_files + k8s_workload_files = self._parse_source(ref=source_file) except Exception as e: - logger.error(f"Error loading file(s) from git repository {e}") + logger.error(f"Error processing file: {e}") raise e + + return k8s_workload_files diff --git a/getdeck/sources/generator.py b/getdeck/sources/generator.py new file mode 100644 index 0000000..0326230 --- /dev/null +++ b/getdeck/sources/generator.py @@ -0,0 +1,69 @@ +from abc import ABC, abstractmethod +import logging +from typing import List, Union + +from getdeck.configuration import ClientConfiguration +from getdeck.deckfile.file import ( + DeckfileDirectorySource, + DeckfileFileSource, + DeckfileInlineSource, + DeckfileKustomizeSource, + DeckfileHelmSource, +) +from getdeck.fetch.types import DeckfileAux, SourceAux +from getdeck.sources.types import K8sSourceFile + +logger = logging.getLogger("deck") + + +class RenderError(Exception): + pass + + +class RenderBehavior(ABC): + def __init__( + self, + path: str, + source: Union[ + DeckfileInlineSource, + DeckfileFileSource, + DeckfileDirectorySource, + DeckfileHelmSource, + DeckfileKustomizeSource, + ], + config: ClientConfiguration, + namespace: str, + ): + self.path = path + self.source = source + self.config = config + self.namespace = namespace + + @property + def not_supported_message(self): + return "Could not render source" + + @abstractmethod + def render(self, **kwargs) -> List[K8sSourceFile]: + raise NotImplementedError + + +class ResourceGenerator: + def __init__(self, render_behavior: RenderBehavior) -> None: + self._render_behavior = render_behavior + + @property + def render_behavior(self) -> RenderBehavior: + return self._render_behavior + + @render_behavior.setter + def render_behavior(self, render_behavior: RenderBehavior) -> None: + self._render_behavior = render_behavior + + def render( + self, deckfile_aux: DeckfileAux, source_aux: SourceAux + ) -> List[K8sSourceFile]: + source_files = self._render_behavior.render( + deckfile_aux=deckfile_aux, source_aux=source_aux + ) + return source_files diff --git a/getdeck/sources/helm.py b/getdeck/sources/helm.py index f1aeb64..2d6a9ad 100644 --- a/getdeck/sources/helm.py +++ b/getdeck/sources/helm.py @@ -6,17 +6,26 @@ import yaml -from getdeck.sources.tooler import ToolerFetcher +from getdeck.sources.tooler import Tooler from getdeck.sources.types import K8sSourceFile logger = logging.getLogger("deck") -class HelmFetcher(ToolerFetcher): +class Helm(Tooler): @property def not_supported_message(self): return "This helm source is currently not supported" + @property + def type(self) -> str: + from getdeck.sources.utils import sniff_protocol + + if self.source.ref is None: + raise RuntimeError("`source.ref` not specified") + protocol = sniff_protocol(self.source.ref) + return protocol + def build_command(self) -> List[str]: helm_cmd = self._helm_prep() helm_cmd.append("&&") diff --git a/getdeck/sources/inline.py b/getdeck/sources/inline.py new file mode 100644 index 0000000..1eb9ca7 --- /dev/null +++ b/getdeck/sources/inline.py @@ -0,0 +1,22 @@ +import logging + +from getdeck.fetch.types import DeckfileAux, SourceAux + +from getdeck.sources.generator import RenderBehavior +from getdeck.sources.types import K8sSourceFile + +logger = logging.getLogger("deck") + + +class Inline(RenderBehavior): + def render(self, deckfile_aux: DeckfileAux, source_aux: SourceAux, **kwargs): + try: + source_file = K8sSourceFile( + name="Deckfile", + content=source_aux.source.content, + namespace=self.namespace, + ) + return [source_file] + except Exception as e: + logger.error(f"Error processing file: {e}") + raise e diff --git a/getdeck/sources/kustomize.py b/getdeck/sources/kustomize.py index 7250427..eecf089 100644 --- a/getdeck/sources/kustomize.py +++ b/getdeck/sources/kustomize.py @@ -3,18 +3,19 @@ import yaml -from getdeck.sources.tooler import ToolerFetcher +from getdeck.sources.tooler import Tooler from getdeck.sources.types import K8sSourceFile -from getdeck.utils import sniff_protocol logger = logging.getLogger("deck") -class KustomizeFetcher(ToolerFetcher): +class Kustomize(Tooler): FILENAME = "manifest.yaml" @property def type(self) -> str: + from getdeck.sources.utils import sniff_protocol + if self.source.ref is None: raise RuntimeError("`source.ref` not specified") protocol = sniff_protocol(self.source.ref) diff --git a/getdeck/sources/selector.py b/getdeck/sources/selector.py index 262d606..a8a7a4f 100644 --- a/getdeck/sources/selector.py +++ b/getdeck/sources/selector.py @@ -1,21 +1,32 @@ from typing import Union, Optional from getdeck.deckfile.file import ( + DeckfileDirectorySource, DeckfileFileSource, DeckfileHelmSource, + DeckfileInlineSource, DeckfileKustomizeSource, ) -from getdeck.sources.fetcher import Fetcher -from getdeck.sources.file import FileFetcher -from getdeck.sources.helm import HelmFetcher -from getdeck.sources.kustomize import KustomizeFetcher +from getdeck.sources.generator import RenderBehavior +from getdeck.sources.file import File +from getdeck.sources.helm import Helm +from getdeck.sources.inline import Inline +from getdeck.sources.kustomize import Kustomize -def select_fetcher_strategy( - source: Union[DeckfileFileSource, DeckfileHelmSource, DeckfileKustomizeSource] -) -> Optional[Fetcher]: - fetcher_strategy = { - DeckfileFileSource: FileFetcher, - DeckfileHelmSource: HelmFetcher, - DeckfileKustomizeSource: KustomizeFetcher, +def select_render_behavior( + source: Union[ + DeckfileInlineSource, + DeckfileFileSource, + DeckfileDirectorySource, + DeckfileHelmSource, + DeckfileKustomizeSource, + ], +) -> Optional[RenderBehavior]: + render_behavior = { + DeckfileInlineSource: Inline, + DeckfileFileSource: File, + DeckfileDirectorySource: File, + DeckfileHelmSource: Helm, + DeckfileKustomizeSource: Kustomize, }.get(type(source), None) - return fetcher_strategy + return render_behavior diff --git a/getdeck/sources/tooler.py b/getdeck/sources/tooler.py index ea2bd59..2226c9a 100644 --- a/getdeck/sources/tooler.py +++ b/getdeck/sources/tooler.py @@ -1,3 +1,4 @@ +from abc import abstractmethod import io import logging import os @@ -8,11 +9,10 @@ from functools import cached_property from typing import List, Union -from git import Repo - from getdeck.configuration import ClientConfiguration +from getdeck.fetch.types import DeckfileAux, SourceAux from getdeck.sources import tooler -from getdeck.sources.file import FileFetcher +from getdeck.sources.generator import RenderBehavior from getdeck.sources.types import K8sSourceFile logger = logging.getLogger("deck") @@ -114,56 +114,33 @@ def build_user_container(config: ClientConfiguration): ) -class ToolerFetcher(FileFetcher): +class Tooler(RenderBehavior): SOURCES = "/sources" OUTPUT = "/output" - def fetch_content(self, **kwargs) -> List[K8sSourceFile]: - raise NotImplementedError - - def fetch_local(self, **kwargs): + def render(self, deckfile_aux: DeckfileAux, source_aux: SourceAux, **kwargs): cmd = self.build_command() try: - if not os.path.isabs(self.source.ref): - path = os.path.join( - self.working_dir, self.source.ref.removeprefix("./") - ) - self._parse_source(ref=path, working_dir=self.working_dir) - dst = os.path.join( - self.tmp_source.name, self.source.ref.removeprefix("./") + source_path = os.path.join( + source_aux.path, source_aux.name or "" + ) # TODO: check if required + logger.debug(f"Render {source_path}") + if not os.path.isabs(source_path): + source_path = os.path.join( + deckfile_aux.path, source_path.removeprefix("./") ) - shutil.copytree(path, dst, dirs_exist_ok=True) + + if os.path.isdir(source_path): + shutil.copytree(source_path, self.tmp_source.name, dirs_exist_ok=True) else: - self._parse_source(ref=self.source.ref) - dst = os.path.join(self.tmp_source.name, self.source.ref) - shutil.copytree(self.source.ref, dst, dirs_exist_ok=True) - self.run_tooler(cmd) - return self.collect_workload_files() - finally: - self.cleanup() + shutil.copy(source_path, self.tmp_source.name) - def fetch_remote(self, git=False): - cmd = self.build_command() - try: - if git: - self._checkout_git() self.run_tooler(cmd) - return self.collect_workload_files() + source_files = self.collect_workload_files() + return source_files finally: self.cleanup() - def fetch_http(self, **kwargs) -> List[K8sSourceFile]: - return self.fetch_remote(git=False) - - def fetch_git(self, **kwargs) -> List[K8sSourceFile]: - return self.fetch_remote(git=True) - - def _checkout_git(self): - logger.debug(f"Cloning from {self.source.ref} to {self.tmp_source.name}") - repo = Repo.clone_from(self.source.ref, self.tmp_source.name) - if self.source.targetRevision: - repo.git.checkout(self.source.targetRevision) - @cached_property def tmp_output(self): return tempfile.TemporaryDirectory() @@ -176,6 +153,7 @@ def cleanup(self): self.tmp_output.cleanup() self.tmp_source.cleanup() + @abstractmethod def build_command(self) -> List[str]: raise NotImplementedError @@ -189,5 +167,6 @@ def run_tooler(self, cmd): ], ) - def collect_workload_files(self): + @abstractmethod + def collect_workload_files(self) -> List[K8sSourceFile]: raise NotImplementedError diff --git a/getdeck/sources/utils.py b/getdeck/sources/utils.py index c612ab9..d5e6fce 100644 --- a/getdeck/sources/utils.py +++ b/getdeck/sources/utils.py @@ -1,59 +1,74 @@ import logging from getdeck.configuration import ClientConfiguration -from getdeck.deckfile.file import Deckfile -from getdeck.sources.fetcher import FetcherContext +from getdeck.fetch.types import DataAux +from getdeck.sources.generator import ResourceGenerator -from getdeck.sources.selector import select_fetcher_strategy +from getdeck.sources.selector import select_render_behavior from getdeck.sources.types import GeneratedDeck logger = logging.getLogger("deck") +def sniff_protocol(ref: str): + if "#" in ref: + ref, rev = ref.split("#") + ref_lo = ref.lower() + if ref_lo.startswith("git") or ref_lo.endswith(".git"): + return "git" + if ref_lo.startswith("https"): + return "https" + if ref_lo.startswith("http"): + return "http" + if ref_lo[0] in "./~": + return "local" + return None + + def prepare_k8s_workload_for_deck( config: ClientConfiguration, - deckfile: Deckfile, + data_aux: DataAux, deck_name: str, - working_dir: str = None, ) -> GeneratedDeck: - deck = deckfile.get_deck(deck_name) + deck = data_aux.deckfile.get_deck(deck_name) logger.debug(deck) - # fetch all sources namespace = deck.namespace or "default" generated_deck = GeneratedDeck(name=deck.name, namespace=namespace, files=[]) logger.info(f"Processing {len(deck.sources)} source(s)") - fetcher_context = FetcherContext(strategy=None) + resource_generator = ResourceGenerator(render_behavior=None) - for source in deck.sources: + for source_aux in data_aux.source_auxs: + ref = getattr(source_aux.source, "ref", None) logger.info( "Processing source " - f"{source.__class__.__name__}: {'no ref' if not source.ref else source.ref}" + f"{source_aux.source.__class__.__name__}: {ref or 'no ref'}" ) # update current fetcher strategy - strategy = select_fetcher_strategy(source=source) - if not strategy: + render_behavior = select_render_behavior(source=source_aux.source) + if not render_behavior: # TODO: check earlier after/during parsing? logger.info( "Skipping source " - f"{source.__class__.__name__}: {'no ref' if not source.ref else source.ref}" + f"{source_aux.source.__class__.__name__}: {ref or 'no ref'}" ) continue # if a source, such as Helm, specifies another namespace - logger.debug(source) - if hasattr(source, "namespace"): - namespace = source.namespace or deck.namespace or "default" + logger.debug(source_aux.source) + if hasattr(source_aux.source, "namespace"): + namespace = source_aux.source.namespace or deck.namespace or "default" else: namespace = deck.namespace or "default" - fetcher_context.strategy = strategy( - deckfile.file_path, source, config, namespace, working_dir + # render source files + resource_generator.render_behavior = render_behavior( + data_aux.deckfile.file_path, source_aux.source, config, namespace + ) + source_files = resource_generator.render( + deckfile_aux=data_aux.deckfile_aux, source_aux=source_aux ) - - # fetch source files - source_files = fetcher_context.fetch_source_files() generated_deck.files.extend(source_files) return generated_deck diff --git a/getdeck/utils.py b/getdeck/utils.py index ffeca94..b467afb 100644 --- a/getdeck/utils.py +++ b/getdeck/utils.py @@ -2,81 +2,19 @@ import os import subprocess from time import sleep -from typing import Optional, Tuple from semantic_version import Version from getdeck.configuration import ClientConfiguration -from getdeck.deckfile.fetch.deck_fetcher import ( - DeckFetcher, - DeckfileAux, - select_fetch_behavior, -) -from getdeck.deckfile.fetch.utils import get_path_and_name from getdeck.deckfile.file import Deckfile from getdeck.provider.abstract import AbstractProvider from getdeck.provider.errors import NotSupportedError from getdeck.provider.types import ProviderType -from getdeck.deckfile.selector import deckfile_selector logger = logging.getLogger("deck") -def sniff_protocol(ref: str): - if "#" in ref: - ref, rev = ref.split("#") - ref_lo = ref.lower() - if ref_lo.startswith("git") or ref_lo.endswith(".git"): - return "git" - if ref_lo.startswith("https"): - return "https" - if ref_lo.startswith("http"): - return "http" - if ref_lo[0] in "./~": - return "local" - return None - - -def read_deckfile_from_location( - location: str, *args, **kwargs -) -> Tuple[Deckfile, Optional[str], bool]: - logger.info(f"Reading Deckfile from: {location}") - - # fetch - data = DeckfileAux(argument_location=location) - fetch_behavior = select_fetch_behavior(location=location) - if fetch_behavior: - deck_fetcher = DeckFetcher(fetch_behavior=fetch_behavior) - data = deck_fetcher.fetch(data=data) - else: - # local path and name - path, name = get_path_and_name(location=location) - data.path = path - data.name = name - data.working_dir_path = os.path.dirname(location) - - # validate (error flag used to raise exception after clean up) - error = False - file = os.path.join(data.path, data.name) - if not os.path.isfile(file): - error = True - - # parse - if not error: - deckfile = deckfile_selector.get(file) - - # clean up - if fetch_behavior: - fetch_behavior.clean_up(data=data) - - # error - if error: - raise RuntimeError(f"Cannot identify {location} as Deckfile") - - return deckfile, data.working_dir_path, data.is_temp_dir - - def ensure_cluster( deckfile: Deckfile, config: ClientConfiguration, diff --git a/test/sources/deck.file.yaml b/test/sources/deck.file.yaml index 6129e58..787be86 100644 --- a/test/sources/deck.file.yaml +++ b/test/sources/deck.file.yaml @@ -61,8 +61,7 @@ decks: targetRevision: main path: test/beiboot/hello.yaml - - type: file - ref: content + - type: inline content: { "kind": "Namespace", diff --git a/test/src/test_deck_fetch_behavior.py b/test/src/test_deck_fetch_behavior.py index 8a65cac..9089336 100644 --- a/test/src/test_deck_fetch_behavior.py +++ b/test/src/test_deck_fetch_behavior.py @@ -1,7 +1,7 @@ import os from unittest import TestCase from getdeck import configuration -from getdeck.deckfile.fetch.deck_fetcher import DeckfileAux, FetchError, Git, Http +from getdeck.fetch.deck_fetcher import DeckfileAux, FetchError, Git, Http class GitTest(TestCase): diff --git a/test/src/test_deck_fetch_utils.py b/test/src/test_deck_fetch_utils.py index 535d96a..4f032e9 100644 --- a/test/src/test_deck_fetch_utils.py +++ b/test/src/test_deck_fetch_utils.py @@ -1,7 +1,7 @@ import os from unittest import TestCase from getdeck import configuration -from getdeck.deckfile.fetch.utils import get_path_and_name +from getdeck.fetch.utils import get_path_and_name class GetPathAndNameTest(TestCase): diff --git a/test/src/test_deck_fetcher.py b/test/src/test_deck_fetcher.py index 59c4e43..803e492 100644 --- a/test/src/test_deck_fetcher.py +++ b/test/src/test_deck_fetcher.py @@ -1,38 +1,38 @@ from unittest import TestCase -from getdeck.deckfile.fetch.deck_fetcher import ( +from getdeck.fetch.deck_fetcher import ( DeckFetcher, DeckfileAux, Git, Http, - select_fetch_behavior, + select_deck_fetch_behavior, ) class SelectFetchBehaviorTest(TestCase): def test_git(self): - fetch_behavior = select_fetch_behavior( + fetch_behavior = select_deck_fetch_behavior( location="git@github.com:Getdeck/getdeck.git" ) self.assertIsInstance(fetch_behavior, Git) def test_git_https(self): - fetch_behavior = select_fetch_behavior( + fetch_behavior = select_deck_fetch_behavior( location="https://github.com/Getdeck/getdeck.git" ) self.assertIsInstance(fetch_behavior, Git) def test_https(self): - fetch_behavior = select_fetch_behavior( + fetch_behavior = select_deck_fetch_behavior( location="https://raw.githubusercontent.com/Getdeck/getdeck/main/test/deckfile/deck.empty.yaml" ) self.assertIsInstance(fetch_behavior, Http) def test_local_dot(self): - fetch_behavior = select_fetch_behavior(location=".") + fetch_behavior = select_deck_fetch_behavior(location=".") self.assertIsNone(fetch_behavior) def test_local_path(self): - fetch_behavior = select_fetch_behavior(location="./test/deck.yaml") + fetch_behavior = select_deck_fetch_behavior(location="./test/deck.yaml") self.assertIsNone(fetch_behavior) @@ -41,7 +41,7 @@ def test_default(self): location = "git@github.com:Getdeck/getdeck.git" data = DeckfileAux(argument_location=location) - fetch_behavior = select_fetch_behavior(location=location) + fetch_behavior = select_deck_fetch_behavior(location=location) deck_fetcher = DeckFetcher(fetch_behavior=fetch_behavior) data = deck_fetcher.fetch(data=data) fetch_behavior.clean_up(data=data) diff --git a/test/src/test_deckfile.py b/test/src/test_deckfile.py index e448e69..64a2ed8 100644 --- a/test/src/test_deckfile.py +++ b/test/src/test_deckfile.py @@ -1,11 +1,12 @@ from unittest import TestCase -from getdeck.utils import read_deckfile_from_location + +from getdeck.fetch.fetch import fetch_data class DeckFileLocationTest(TestCase): def test_local(self): location = "./test/deckfile/deck.empty.yaml" - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(location) + deckfile, working_dir_path, is_temp_dir = fetch_data(location) self.assertIsNotNone(deckfile) self.assertEqual(working_dir_path, "./test/deckfile") self.assertFalse(is_temp_dir) @@ -13,11 +14,11 @@ def test_local(self): def test_git_with_no_deckfile(self): location = "git@github.com:Getdeck/getdeck.git" with self.assertRaises(RuntimeError): - _ = read_deckfile_from_location(location) + _ = fetch_data(location) def test_https(self): location = "https://raw.githubusercontent.com/Getdeck/getdeck/main/test/deckfile/deck.empty.yaml" - deckfile, working_dir_path, is_temp_dir = read_deckfile_from_location(location) + deckfile, working_dir_path, is_temp_dir = fetch_data(location) self.assertIsNotNone(deckfile) self.assertIsNone(working_dir_path) self.assertFalse(is_temp_dir) From 23eede17e83e06643cd6546c1aa17d78e50ecb9b Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Fri, 21 Oct 2022 12:29:10 +0200 Subject: [PATCH 3/7] refactor: fetch sources + render sources - fetch sources before cluster creation - decouples source generation from fetching --- getdeck/api/hosts.py | 15 +--- getdeck/api/list.py | 17 ++-- getdeck/api/remove.py | 34 ++++---- getdeck/api/stop.py | 18 ++--- getdeck/deckfile/file.py | 3 +- getdeck/deckfile/selector.py | 3 +- getdeck/fetch/deck_fetcher.py | 5 +- getdeck/fetch/fetch.py | 15 ++-- getdeck/fetch/source_fetcher.py | 14 ---- getdeck/fetch/types.py | 3 +- getdeck/sources/utils.py | 2 +- test/deckfile/deck.empty.yaml | 3 +- test/src/fetch/test_deck_fetch_behavior.py | 80 +++++++++++++++++++ test/src/{ => fetch}/test_deck_fetcher.py | 8 +- .../{test_deckfile.py => fetch/test_fetch.py} | 18 ++--- test/src/fetch/test_source_fetcher.py | 67 ++++++++++++++++ .../test_utils.py} | 0 test/src/test_deck_fetch_behavior.py | 80 ------------------- 18 files changed, 206 insertions(+), 179 deletions(-) create mode 100644 test/src/fetch/test_deck_fetch_behavior.py rename test/src/{ => fetch}/test_deck_fetcher.py (88%) rename test/src/{test_deckfile.py => fetch/test_fetch.py} (51%) create mode 100644 test/src/fetch/test_source_fetcher.py rename test/src/{test_deck_fetch_utils.py => fetch/test_utils.py} (100%) delete mode 100644 test/src/test_deck_fetch_behavior.py diff --git a/getdeck/api/hosts.py b/getdeck/api/hosts.py index e03fa79..f9fef01 100644 --- a/getdeck/api/hosts.py +++ b/getdeck/api/hosts.py @@ -1,12 +1,9 @@ import logging -import os -import shutil import socket from python_hosts import Hosts, HostsEntry from getdeck.api import stopwatch -from getdeck.configuration import default_configuration from getdeck.fetch.fetch import fetch_data logger = logging.getLogger("deck") @@ -17,12 +14,9 @@ def run_hosts( deckfile_location: str, host_action: str, deck_name: str = None, - config=default_configuration, ) -> bool: - deckfile, working_dir_path, is_temp_dir = fetch_data( - deckfile_location, deck_name=deck_name - ) - deck = deckfile.get_deck(deck_name) + data_aux = fetch_data(deckfile_location, deck_name=deck_name) + deck = data_aux.deckfile.get_deck(deck_name) deck_hosts = deck.hosts hosts = Hosts() @@ -55,10 +49,7 @@ def run_hosts( else: logger.info("No hosts specified in Deckfile") - # TODO: refactor/remove? - if is_temp_dir and os.path.isdir(working_dir_path): - shutil.rmtree(working_dir_path) - + del data_aux return True diff --git a/getdeck/api/list.py b/getdeck/api/list.py index d78aeca..7ccde84 100644 --- a/getdeck/api/list.py +++ b/getdeck/api/list.py @@ -1,24 +1,17 @@ import logging -import os -import shutil from typing import List from getdeck.api.utils import stopwatch -from getdeck.configuration import default_configuration +from getdeck.fetch.fetch import fetch_data logger = logging.getLogger("deck") @stopwatch -def get_available_decks(deckfile_location: str, config=default_configuration) -> List: - from getdeck.utils import fetch_data +def get_available_decks(deckfile_location: str) -> List: + data_aux = fetch_data(deckfile_location) + available_decks = data_aux.deckfile.get_decks() + del data_aux - deckfile, working_dir_path, is_temp_dir = fetch_data(deckfile_location) - available_decks = deckfile.get_decks() logger.debug(available_decks) - - # TODO: refactor/remove? - if is_temp_dir and os.path.isdir(working_dir_path): - shutil.rmtree(working_dir_path) - return available_decks diff --git a/getdeck/api/remove.py b/getdeck/api/remove.py index b1bca89..7f4f93f 100644 --- a/getdeck/api/remove.py +++ b/getdeck/api/remove.py @@ -1,10 +1,9 @@ import logging -import os -import shutil from typing import Callable from getdeck.configuration import default_configuration from getdeck.api import stopwatch +from getdeck.fetch.fetch import fetch_data logger = logging.getLogger("deck") @@ -16,19 +15,18 @@ def remove_cluster( config=default_configuration, ignore_cluster: bool = False, ) -> bool: - from getdeck.utils import fetch_data, ensure_cluster + from getdeck.utils import ensure_cluster - deckfile, working_dir_path, is_temp_dir = fetch_data(deckfile_location) - k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) + data_aux = fetch_data(deckfile_location) + k8s_provider = ensure_cluster( + data_aux.deckfile, config, ignore_cluster, do_install=False + ) if k8s_provider.exists(): k8s_provider.delete() else: logger.info("Cluster does not exist") - # TODO: refactor/remove? - if is_temp_dir and os.path.isdir(working_dir_path): - shutil.rmtree(working_dir_path) - + del data_aux return True @@ -43,23 +41,24 @@ def remove_deck( if progress_callback: progress_callback(0) - from getdeck.utils import fetch_data, ensure_cluster + from getdeck.utils import ensure_cluster from getdeck.k8s import k8s_delete_object from getdeck.sources.utils import prepare_k8s_workload_for_deck - deckfile, working_dir_path, is_temp_dir = fetch_data( - deckfile_location, deck_name=deck_name - ) + data_aux = fetch_data(deckfile_location, deck_name=deck_name) + if progress_callback: progress_callback(10) - k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) + k8s_provider = ensure_cluster( + data_aux.deckfile, config, ignore_cluster, do_install=False + ) if progress_callback: progress_callback(20) config.kubeconfig = k8s_provider.get_kubeconfig() if k8s_provider.exists(): generated_deck = prepare_k8s_workload_for_deck( - config, deckfile, deck_name, working_dir_path + config, data_aux.deckfile, deck_name ) logger.info(f"Removing Deck {generated_deck.name}") if progress_callback: @@ -85,8 +84,5 @@ def remove_deck( else: logger.info("Cluster does not exist") - # TODO: refactor/remove? - if is_temp_dir and os.path.isdir(working_dir_path): - shutil.rmtree(working_dir_path) - + del data_aux return True diff --git a/getdeck/api/stop.py b/getdeck/api/stop.py index 9c741c2..175675b 100644 --- a/getdeck/api/stop.py +++ b/getdeck/api/stop.py @@ -1,10 +1,8 @@ import logging -import os -import shutil -from typing import Callable from getdeck.api import stopwatch from getdeck.configuration import default_configuration +from getdeck.fetch.fetch import fetch_data logger = logging.getLogger("deck") @@ -14,16 +12,14 @@ def stop_cluster( deckfile_location: str, ignore_cluster: bool = False, config=default_configuration, - progress_callback: Callable = None, ) -> bool: - from getdeck.utils import fetch_data, ensure_cluster + from getdeck.utils import ensure_cluster - deckfile, working_dir_path, is_temp_dir = fetch_data(deckfile_location) - k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False) + data_aux = fetch_data(deckfile_location) + k8s_provider = ensure_cluster( + data_aux.deckfile, config, ignore_cluster, do_install=False + ) logger.info("Stopping cluster") - # TODO: refactor/remove? - if is_temp_dir and os.path.isdir(working_dir_path): - shutil.rmtree(working_dir_path) - + del data_aux k8s_provider.stop() diff --git a/getdeck/deckfile/file.py b/getdeck/deckfile/file.py index c5e64dd..41daff5 100644 --- a/getdeck/deckfile/file.py +++ b/getdeck/deckfile/file.py @@ -4,7 +4,6 @@ from pydantic import BaseModel -from getdeck.configuration import ClientConfiguration from getdeck.deckfile.errors import DeckfileError from getdeck.provider.abstract import AbstractProvider @@ -17,7 +16,7 @@ class DeckfileCluster(BaseModel): name: str nativeConfig: dict = None - def get_provider(self, config: ClientConfiguration) -> AbstractProvider: + def get_provider(self, config) -> AbstractProvider: from getdeck.provider.factory import cluster_factory from getdeck.provider.types import ProviderType diff --git a/getdeck/deckfile/selector.py b/getdeck/deckfile/selector.py index ec00851..f7df849 100644 --- a/getdeck/deckfile/selector.py +++ b/getdeck/deckfile/selector.py @@ -11,7 +11,6 @@ DeckfileVersionError, DeckfileError, ) -from getdeck.deckfile.file import Deckfile from getdeck.deckfile.deckfile_1 import Deckfile_1_0 logger = logging.getLogger("deck") @@ -21,7 +20,7 @@ class DeckfileSelector: def __init__(self, options: dict): self.options = options - def get(self, path_deckfile: str = None) -> Union[Deckfile, Deckfile_1_0, None]: + def get(self, path_deckfile: str = None) -> Union[Deckfile_1_0, None]: # default file path if not path_deckfile: path_deckfile = os.path.join(os.getcwd(), configuration.DECKFILE_FILE) diff --git a/getdeck/fetch/deck_fetcher.py b/getdeck/fetch/deck_fetcher.py index ad9b341..759eb50 100644 --- a/getdeck/fetch/deck_fetcher.py +++ b/getdeck/fetch/deck_fetcher.py @@ -25,7 +25,7 @@ def fetch(self, data: DeckfileAux) -> DeckfileAux: class Git(DeckFetchBehavior): def fetch(self, data: DeckfileAux) -> DeckfileAux: - location = data.argument_location + location = data.location if "#" in location: ref, rev = location.split("#") @@ -35,7 +35,6 @@ def fetch(self, data: DeckfileAux) -> DeckfileAux: temporary_folder = tempfile.mkdtemp() data.path = temporary_folder - data.working_dir_path = temporary_folder data.temporary_data = TemporaryData(data=temporary_folder, is_folder=True) try: @@ -53,7 +52,7 @@ def fetch(self, data: DeckfileAux) -> DeckfileAux: class Http(DeckFetchBehavior): def fetch(self, data: DeckfileAux) -> DeckfileAux: - location = data.argument_location + location = data.location temporary_file = tempfile.NamedTemporaryFile(delete=False) data.path = os.path.dirname(temporary_file.name) diff --git a/getdeck/fetch/fetch.py b/getdeck/fetch/fetch.py index 55dfba0..01aafb4 100644 --- a/getdeck/fetch/fetch.py +++ b/getdeck/fetch/fetch.py @@ -33,21 +33,25 @@ def fetch_sources(deck: DeckfileDeck) -> List[SourceAux]: "Fetching source " f"{source.__class__.__name__}: {ref or 'no ref'}" ) - data = SourceAux(location=ref, source=source) + source_aux = SourceAux(location=ref) + # assigning source during SourceAux initialization changes DeckfileHelmSource to DeckfileInlineSource.. ??? + source_aux.source = source + fetch_behavior = select_source_fetch_behavior(source=source) source_fetcher.fetch_behavior = fetch_behavior if not fetch_behavior: + source_auxs.append(source_aux) continue try: source_dict = source.dict() - data = source_fetcher.fetch(data=data, **source_dict) + source_aux = source_fetcher.fetch(data=source_aux, **source_dict) except Exception as e: logger.debug(str(e)) - del data, source_auxs[:] + del source_aux, source_auxs[:] raise e - source_auxs.append(data) # noqa: F821 + source_auxs.append(source_aux) # noqa: F821 return source_auxs @@ -61,7 +65,7 @@ def fetch_data(location: str, deck_name: str = None, *args, **kwargs) -> DataAux data_aux = DataAux() # fetch - deckfile_aux = DeckfileAux(argument_location=location) + deckfile_aux = DeckfileAux(location=location) fetch_behavior = select_deck_fetch_behavior(location=location) if fetch_behavior: deck_fetcher = DeckFetcher(fetch_behavior=fetch_behavior) @@ -71,7 +75,6 @@ def fetch_data(location: str, deck_name: str = None, *args, **kwargs) -> DataAux path, name = get_path_and_name(location=location) deckfile_aux.path = path deckfile_aux.name = name - deckfile_aux.working_dir_path = os.path.dirname(location) data_aux.deckfile_aux = deckfile_aux diff --git a/getdeck/fetch/source_fetcher.py b/getdeck/fetch/source_fetcher.py index 8dcb47e..3e275f0 100644 --- a/getdeck/fetch/source_fetcher.py +++ b/getdeck/fetch/source_fetcher.py @@ -7,7 +7,6 @@ import requests from git import Repo, GitError - from getdeck.fetch.types import SourceAux, TemporaryData @@ -86,11 +85,6 @@ def fetch(self, data: SourceAux, *args, **kwargs) -> SourceAux: return data -class Content(SourceFetchBehavior): - def fetch(self, data: SourceAux, *args, **kwargs) -> SourceAux: - return data # TODO: remove Content class? - - class SourceFetcher: def __init__(self, fetch_behavior: SourceFetchBehavior) -> None: self._fetch_behavior = fetch_behavior @@ -109,16 +103,8 @@ def fetch(self, data: SourceAux, *args, **kwargs) -> SourceAux: def select_source_fetch_behavior(source) -> Optional[SourceFetchBehavior]: - try: - _ = source.content - has_content = True - except Exception: - has_content = False - ref = getattr(source, "ref", None) if not ref: - if has_content: - return Content() return None if "#" in ref: diff --git a/getdeck/fetch/types.py b/getdeck/fetch/types.py index c8a5ad6..ff2237b 100644 --- a/getdeck/fetch/types.py +++ b/getdeck/fetch/types.py @@ -50,11 +50,10 @@ def __del__(self): class DeckfileAux(BaseModel): - argument_location: str # TODO: rename + location: str cwd: str = os.getcwd() path: str = None name: str = configuration.DECKFILE_FILE - working_dir_path: str = None temporary_data: Optional[TemporaryData] = None def __del__(self): diff --git a/getdeck/sources/utils.py b/getdeck/sources/utils.py index d5e6fce..8fd7344 100644 --- a/getdeck/sources/utils.py +++ b/getdeck/sources/utils.py @@ -46,7 +46,7 @@ def prepare_k8s_workload_for_deck( f"{source_aux.source.__class__.__name__}: {ref or 'no ref'}" ) - # update current fetcher strategy + # update current render behavior render_behavior = select_render_behavior(source=source_aux.source) if not render_behavior: # TODO: check earlier after/during parsing? logger.info( diff --git a/test/deckfile/deck.empty.yaml b/test/deckfile/deck.empty.yaml index 43c98de..ce9ea58 100644 --- a/test/deckfile/deck.empty.yaml +++ b/test/deckfile/deck.empty.yaml @@ -31,8 +31,7 @@ decks: - name: empty namespace: default sources: - - type: file - ref: content + - type: inline content: { "kind": "Namespace", diff --git a/test/src/fetch/test_deck_fetch_behavior.py b/test/src/fetch/test_deck_fetch_behavior.py new file mode 100644 index 0000000..13a01bb --- /dev/null +++ b/test/src/fetch/test_deck_fetch_behavior.py @@ -0,0 +1,80 @@ +import os +from unittest import TestCase +from getdeck import configuration +from getdeck.fetch.deck_fetcher import DeckfileAux, FetchError, Git, Http + + +class GitTest(TestCase): + def test_git(self): + deckfile_aux = DeckfileAux(location="git@github.com:Getdeck/getdeck.git") + + fetch_behavior = Git() + deckfile_aux = fetch_behavior.fetch(data=deckfile_aux) + + self.assertTrue(os.path.isdir(deckfile_aux.path)) + self.assertEqual(deckfile_aux.name, configuration.DECKFILE_FILE) + + path = deckfile_aux.path + del deckfile_aux + self.assertFalse(os.path.isdir(path)) + + def test_branch(self): + deckfile_aux = DeckfileAux(location="git@github.com:Getdeck/getdeck.git#main") + + fetch_behavior = Git() + deckfile_aux = fetch_behavior.fetch(data=deckfile_aux) + + self.assertTrue(os.path.isdir(deckfile_aux.path)) + self.assertEqual(deckfile_aux.name, configuration.DECKFILE_FILE) + + path = deckfile_aux.path + del deckfile_aux + self.assertFalse(os.path.isdir(path)) + + def test_branch_invalid(self): + deckfile_aux = DeckfileAux( + location="git@github.com:Getdeck/getdeck.git#invalid" + ) + + fetch_behavior = Git() + with self.assertRaises(FetchError): + _ = fetch_behavior.fetch(data=deckfile_aux) + + def test_https(self): + deckfile_aux = DeckfileAux(location="https://github.com/Getdeck/getdeck.git") + + fetch_behavior = Git() + deckfile_aux = fetch_behavior.fetch(data=deckfile_aux) + + self.assertTrue(os.path.isdir(deckfile_aux.path)) + self.assertEqual(deckfile_aux.name, configuration.DECKFILE_FILE) + + path = deckfile_aux.path + del deckfile_aux + self.assertFalse(os.path.isdir(path)) + + +class HttpTest(TestCase): + def test_default(self): + deckfile_aux = DeckfileAux( + location="https://raw.githubusercontent.com/Getdeck/getdeck/main/test/deckfile/deck.empty.yaml" + ) + + fetch_behavior = Http() + deckfile_aux = fetch_behavior.fetch(data=deckfile_aux) + + self.assertTrue(os.path.isdir(deckfile_aux.path)) + self.assertIsNotNone(deckfile_aux.name) + + file = os.path.join(deckfile_aux.path, deckfile_aux.name) + del deckfile_aux + self.assertFalse(os.path.isfile(file)) + + def test_url_invalid(self): + deckfile_aux = DeckfileAux( + location="https://raw.githubusercontent.com/Getdeck/getdeck/invalid/deck.yaml" + ) + + fetch_behavior = Http() + with self.assertRaises(FetchError): + _ = fetch_behavior.fetch(data=deckfile_aux) diff --git a/test/src/test_deck_fetcher.py b/test/src/fetch/test_deck_fetcher.py similarity index 88% rename from test/src/test_deck_fetcher.py rename to test/src/fetch/test_deck_fetcher.py index 803e492..b9f4dd8 100644 --- a/test/src/test_deck_fetcher.py +++ b/test/src/fetch/test_deck_fetcher.py @@ -8,7 +8,7 @@ ) -class SelectFetchBehaviorTest(TestCase): +class SelectDeckFetchBehaviorTest(TestCase): def test_git(self): fetch_behavior = select_deck_fetch_behavior( location="git@github.com:Getdeck/getdeck.git" @@ -40,8 +40,8 @@ class DeckFetcherTest(TestCase): def test_default(self): location = "git@github.com:Getdeck/getdeck.git" - data = DeckfileAux(argument_location=location) + deckfile_aux = DeckfileAux(location=location) fetch_behavior = select_deck_fetch_behavior(location=location) deck_fetcher = DeckFetcher(fetch_behavior=fetch_behavior) - data = deck_fetcher.fetch(data=data) - fetch_behavior.clean_up(data=data) + deckfile_aux = deck_fetcher.fetch(data=deckfile_aux) + del deckfile_aux diff --git a/test/src/test_deckfile.py b/test/src/fetch/test_fetch.py similarity index 51% rename from test/src/test_deckfile.py rename to test/src/fetch/test_fetch.py index 64a2ed8..dff1de1 100644 --- a/test/src/test_deckfile.py +++ b/test/src/fetch/test_fetch.py @@ -3,13 +3,13 @@ from getdeck.fetch.fetch import fetch_data -class DeckFileLocationTest(TestCase): +class FetchDataTest(TestCase): def test_local(self): location = "./test/deckfile/deck.empty.yaml" - deckfile, working_dir_path, is_temp_dir = fetch_data(location) - self.assertIsNotNone(deckfile) - self.assertEqual(working_dir_path, "./test/deckfile") - self.assertFalse(is_temp_dir) + data_aux = fetch_data(location) + self.assertIsNotNone(data_aux.deckfile) + self.assertIsNotNone(data_aux.deckfile_aux) + self.assertEqual(len(data_aux.source_auxs), 1) def test_git_with_no_deckfile(self): location = "git@github.com:Getdeck/getdeck.git" @@ -18,7 +18,7 @@ def test_git_with_no_deckfile(self): def test_https(self): location = "https://raw.githubusercontent.com/Getdeck/getdeck/main/test/deckfile/deck.empty.yaml" - deckfile, working_dir_path, is_temp_dir = fetch_data(location) - self.assertIsNotNone(deckfile) - self.assertIsNone(working_dir_path) - self.assertFalse(is_temp_dir) + data_aux = fetch_data(location) + self.assertIsNotNone(data_aux.deckfile) + self.assertIsNotNone(data_aux.deckfile_aux) + self.assertEqual(len(data_aux.source_auxs), 1) diff --git a/test/src/fetch/test_source_fetcher.py b/test/src/fetch/test_source_fetcher.py new file mode 100644 index 0000000..ec9a639 --- /dev/null +++ b/test/src/fetch/test_source_fetcher.py @@ -0,0 +1,67 @@ +from unittest import TestCase +from getdeck.deckfile.file import ( + DeckfileFileSource, + DeckfileHelmSource, + DeckfileInlineSource, +) +from getdeck.fetch.source_fetcher import ( + Git, + Http, + Local, + SourceFetcher, + select_source_fetch_behavior, +) +from getdeck.fetch.types import SourceAux + + +class SelectSourceFetchBehaviorTest(TestCase): + def test_git_file_source(self): + source = DeckfileFileSource(ref="git@github.com:Getdeck/getdeck.git") + fetch_behavior = select_source_fetch_behavior(source=source) + self.assertIsInstance(fetch_behavior, Git) + + def test_git_file_source_https(self): + source = DeckfileFileSource(ref="https://github.com/Getdeck/getdeck.git") + fetch_behavior = select_source_fetch_behavior(source=source) + self.assertIsInstance(fetch_behavior, Git) + + def test_git_helm_source(self): + source = DeckfileHelmSource( + ref="git@github.com:Getdeck/getdeck.git", releaseName="test" + ) + fetch_behavior = select_source_fetch_behavior(source=source) + self.assertIsInstance(fetch_behavior, Git) + + def test_http_file_source(self): + source = DeckfileFileSource( + ref="https://raw.githubusercontent.com/Getdeck/getdeck/main/test/sources/resources/hello.yaml" + ) + fetch_behavior = select_source_fetch_behavior(source=source) + self.assertIsInstance(fetch_behavior, Http) + + def test_local_file_source_dot(self): + source = DeckfileFileSource(ref=".") + fetch_behavior = select_source_fetch_behavior(source=source) + self.assertIsInstance(fetch_behavior, Local) + + def test_local_file_source_path(self): + source = DeckfileFileSource(ref="./test/hello.yaml") + fetch_behavior = select_source_fetch_behavior(source=source) + self.assertIsInstance(fetch_behavior, Local) + + def test_none_inline_source(self): + source = DeckfileInlineSource(content={}) + fetch_behavior = select_source_fetch_behavior(source=source) + self.assertIsNone(fetch_behavior) + + +class SourceFetcherTest(TestCase): + def test_inline(self): + location = "https://raw.githubusercontent.com/Getdeck/getdeck/main/test/sources/resources/hello.yaml" + source = DeckfileFileSource(ref=location) + source_aux = SourceAux(location=location) + source_aux.source = source + + fetch_behavior = select_source_fetch_behavior(source=source) + deck_fetcher = SourceFetcher(fetch_behavior=fetch_behavior) + source_aux = deck_fetcher.fetch(data=source_aux) diff --git a/test/src/test_deck_fetch_utils.py b/test/src/fetch/test_utils.py similarity index 100% rename from test/src/test_deck_fetch_utils.py rename to test/src/fetch/test_utils.py diff --git a/test/src/test_deck_fetch_behavior.py b/test/src/test_deck_fetch_behavior.py deleted file mode 100644 index 9089336..0000000 --- a/test/src/test_deck_fetch_behavior.py +++ /dev/null @@ -1,80 +0,0 @@ -import os -from unittest import TestCase -from getdeck import configuration -from getdeck.fetch.deck_fetcher import DeckfileAux, FetchError, Git, Http - - -class GitTest(TestCase): - def test_git(self): - data = DeckfileAux(argument_location="git@github.com:Getdeck/getdeck.git") - - fetch_behavior = Git() - data = fetch_behavior.fetch(data=data) - - self.assertTrue(os.path.isdir(data.path)) - self.assertIsNotNone(data.working_dir_path) - self.assertEqual(data.name, configuration.DECKFILE_FILE) - - fetch_behavior.clean_up(data=data) - self.assertFalse(os.path.isdir(data.working_dir_path)) - - def test_branch(self): - data = DeckfileAux(argument_location="git@github.com:Getdeck/getdeck.git#main") - - fetch_behavior = Git() - data = fetch_behavior.fetch(data=data) - - self.assertTrue(os.path.isdir(data.path)) - self.assertIsNotNone(data.working_dir_path) - self.assertEqual(data.name, configuration.DECKFILE_FILE) - - fetch_behavior.clean_up(data=data) - self.assertFalse(os.path.isdir(data.working_dir_path)) - - def test_branch_invalid(self): - data = DeckfileAux( - argument_location="git@github.com:Getdeck/getdeck.git#invalid" - ) - - fetch_behavior = Git() - with self.assertRaises(FetchError): - data = fetch_behavior.fetch(data=data) - - def test_https(self): - data = DeckfileAux(argument_location="https://github.com/Getdeck/getdeck.git") - - fetch_behavior = Git() - data = fetch_behavior.fetch(data=data) - - self.assertTrue(os.path.isdir(data.path)) - self.assertIsNotNone(data.working_dir_path) - self.assertEqual(data.name, configuration.DECKFILE_FILE) - - fetch_behavior.clean_up(data=data) - self.assertFalse(os.path.isdir(data.working_dir_path)) - - -class HttpTest(TestCase): - def test_default(self): - data = DeckfileAux( - argument_location="https://raw.githubusercontent.com/Getdeck/getdeck/main/test/deckfile/deck.empty.yaml" - ) - - fetch_behavior = Http() - data = fetch_behavior.fetch(data=data) - - self.assertTrue(os.path.isdir(data.path)) - self.assertIsNone(data.working_dir_path) - self.assertIsNotNone(data.name) - - fetch_behavior.clean_up(data=data) - self.assertFalse(os.path.isfile(os.path.join(data.path, data.name))) - - def test_url_invalid(self): - data = DeckfileAux( - argument_location="https://raw.githubusercontent.com/Getdeck/getdeck/invalid/deck.yaml" - ) - - fetch_behavior = Http() - with self.assertRaises(FetchError): - _ = fetch_behavior.fetch(data=data) From 38dd07c7f81049f9c79f9bf286106ef2a1d1c9b3 Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Fri, 21 Oct 2022 13:25:55 +0200 Subject: [PATCH 4/7] fix: add type inline deprecated warning --- getdeck/deckfile/file.py | 11 +++++++- test/deckfile/deck.deprecated.yaml | 40 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 test/deckfile/deck.deprecated.yaml diff --git a/getdeck/deckfile/file.py b/getdeck/deckfile/file.py index 41daff5..710e2ba 100644 --- a/getdeck/deckfile/file.py +++ b/getdeck/deckfile/file.py @@ -97,13 +97,22 @@ def __init__(self, *args, **data): if tsources: try: for source in tsources: + # inline deprecated warning + type = source["type"].lower() + if type == "file" and source.get("content", None): + logger.warning( + "'type: file' is deprecated for inline sources, " + "use 'type: inline' instead", + ) + type = "inline" + source_class = { "inline": DeckfileInlineSource, "file": DeckfileFileSource, "directory": DeckfileDirectorySource, "kustomize": DeckfileKustomizeSource, "helm": DeckfileHelmSource, - }.get(source["type"].lower()) + }.get(type) self.sources.append(source_class(**source)) except KeyError: raise DeckfileError( diff --git a/test/deckfile/deck.deprecated.yaml b/test/deckfile/deck.deprecated.yaml new file mode 100644 index 0000000..56bc039 --- /dev/null +++ b/test/deckfile/deck.deprecated.yaml @@ -0,0 +1,40 @@ +version: "1" + +cluster: + provider: k3d + minVersion: 4.0.0 + name: test-sources-file + nativeConfig: + apiVersion: k3d.io/v1alpha4 + kind: Simple + servers: 1 + agents: 1 + image: rancher/k3s:v1.22.9-k3s1 + options: + k3s: + extraArgs: + - arg: --disable=traefik + nodeFilters: + - server:* + ports: + - port: 61346:80 + nodeFilters: + - loadbalancer + - port: 8443:443 + nodeFilters: + - loadbalancer + - port: 31820:31820/UDP + nodeFilters: + - agent:0 + +decks: + - name: hello + namespace: default + sources: + - type: file + content: + { + "kind": "Namespace", + "apiVersion": "v1", + "metadata": { "name": "content", "labels": { "name": "content" } }, + } From 3267f1704c4e9b9acbb0329ffa289e3d46dfc959 Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Fri, 21 Oct 2022 14:39:16 +0200 Subject: [PATCH 5/7] fix: helm --- getdeck/fetch/source_fetcher.py | 4 +++- getdeck/sources/tooler.py | 27 +++++++++++++++------------ test/src/fetch/test_source_fetcher.py | 9 +++++++++ 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/getdeck/fetch/source_fetcher.py b/getdeck/fetch/source_fetcher.py index 3e275f0..d70124e 100644 --- a/getdeck/fetch/source_fetcher.py +++ b/getdeck/fetch/source_fetcher.py @@ -115,7 +115,9 @@ def select_source_fetch_behavior(source) -> Optional[SourceFetchBehavior]: if ref_lo.startswith("git") or ref_lo.endswith(".git"): return Git() - if ref_lo.startswith("https") or ref_lo.startswith("http"): + if ( + ref_lo.startswith("https") or ref_lo.startswith("http") + ) and source.type != "helm": return Http() if ref_lo[0] in "./~": diff --git a/getdeck/sources/tooler.py b/getdeck/sources/tooler.py index 2226c9a..1f26aea 100644 --- a/getdeck/sources/tooler.py +++ b/getdeck/sources/tooler.py @@ -121,19 +121,22 @@ class Tooler(RenderBehavior): def render(self, deckfile_aux: DeckfileAux, source_aux: SourceAux, **kwargs): cmd = self.build_command() try: - source_path = os.path.join( - source_aux.path, source_aux.name or "" - ) # TODO: check if required - logger.debug(f"Render {source_path}") - if not os.path.isabs(source_path): + if source_aux.path: source_path = os.path.join( - deckfile_aux.path, source_path.removeprefix("./") - ) - - if os.path.isdir(source_path): - shutil.copytree(source_path, self.tmp_source.name, dirs_exist_ok=True) - else: - shutil.copy(source_path, self.tmp_source.name) + source_aux.path, source_aux.name or "" + ) # TODO: check if required + logger.debug(f"Render {source_path}") + if not os.path.isabs(source_path): + source_path = os.path.join( + deckfile_aux.path, source_path.removeprefix("./") + ) + + if os.path.isdir(source_path): + shutil.copytree( + source_path, self.tmp_source.name, dirs_exist_ok=True + ) + else: + shutil.copy(source_path, self.tmp_source.name) self.run_tooler(cmd) source_files = self.collect_workload_files() diff --git a/test/src/fetch/test_source_fetcher.py b/test/src/fetch/test_source_fetcher.py index ec9a639..7ef6df0 100644 --- a/test/src/fetch/test_source_fetcher.py +++ b/test/src/fetch/test_source_fetcher.py @@ -54,6 +54,15 @@ def test_none_inline_source(self): fetch_behavior = select_source_fetch_behavior(source=source) self.assertIsNone(fetch_behavior) + def test_none_helm_source(self): + source = DeckfileHelmSource( + ref="https://kubernetes.github.io/dashboard/", + chart="kubernetes-dashboard", + releaseName="dashboard", + ) + fetch_behavior = select_source_fetch_behavior(source=source) + self.assertIsNone(fetch_behavior) + class SourceFetcherTest(TestCase): def test_inline(self): From d22a9fd48ff460cd989c663046046a20e8df6695 Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Fri, 21 Oct 2022 14:43:55 +0200 Subject: [PATCH 6/7] fix: add __init__ --- getdeck/fetch/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 getdeck/fetch/__init__.py diff --git a/getdeck/fetch/__init__.py b/getdeck/fetch/__init__.py new file mode 100644 index 0000000..e69de29 From a74e59de6f9ec0b064db5989a2581196af3ad2a3 Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Mon, 24 Oct 2022 10:55:08 +0200 Subject: [PATCH 7/7] fix: remove code smells + fix stop/remove --- getdeck/api/remove.py | 6 +- getdeck/deckfile/deckfile_1.py | 9 ++- getdeck/deckfile/file.py | 8 +-- getdeck/fetch/fetch.py | 55 +++++++++------ getdeck/fetch/types.py | 37 +++++----- getdeck/fetch/utils.py | 4 +- getdeck/sources/tooler.py | 4 +- getdeck/sources/utils.py | 2 +- test/deckfile/deck.helm.yaml | 78 ++++++++++++++++++++++ test/src/fetch/test_deck_fetch_behavior.py | 4 +- test/src/fetch/test_utils.py | 23 ++++--- 11 files changed, 160 insertions(+), 70 deletions(-) create mode 100644 test/deckfile/deck.helm.yaml diff --git a/getdeck/api/remove.py b/getdeck/api/remove.py index 7f4f93f..394c375 100644 --- a/getdeck/api/remove.py +++ b/getdeck/api/remove.py @@ -17,7 +17,7 @@ def remove_cluster( ) -> bool: from getdeck.utils import ensure_cluster - data_aux = fetch_data(deckfile_location) + data_aux = fetch_data(deckfile_location, fetch_sources=False) k8s_provider = ensure_cluster( data_aux.deckfile, config, ignore_cluster, do_install=False ) @@ -57,9 +57,7 @@ def remove_deck( config.kubeconfig = k8s_provider.get_kubeconfig() if k8s_provider.exists(): - generated_deck = prepare_k8s_workload_for_deck( - config, data_aux.deckfile, deck_name - ) + generated_deck = prepare_k8s_workload_for_deck(config, data_aux, deck_name) logger.info(f"Removing Deck {generated_deck.name}") if progress_callback: progress_callback(30) diff --git a/getdeck/deckfile/deckfile_1.py b/getdeck/deckfile/deckfile_1.py index cdaf916..38f5d0e 100644 --- a/getdeck/deckfile/deckfile_1.py +++ b/getdeck/deckfile/deckfile_1.py @@ -19,11 +19,10 @@ class Deckfile_1_0(Deckfile, BaseModel): decks: List[DeckfileDeck] def get_deck(self, name: str = None) -> DeckfileDeck: - if name is None and len(self.decks) > 1: - raise ValueError( - "Name of Deck is missing and there are multiple Decks available" - ) - elif name is None and len(self.decks) == 1: + if name is None and len(self.decks) >= 1: + for deck in self.decks: + if deck.name.lower() == "default": + return deck return self.decks[0] else: for deck in self.decks: diff --git a/getdeck/deckfile/file.py b/getdeck/deckfile/file.py index 710e2ba..288b4ae 100644 --- a/getdeck/deckfile/file.py +++ b/getdeck/deckfile/file.py @@ -98,13 +98,13 @@ def __init__(self, *args, **data): try: for source in tsources: # inline deprecated warning - type = source["type"].lower() - if type == "file" and source.get("content", None): + source_type = source["type"].lower() + if source_type == "file" and source.get("content", None): logger.warning( "'type: file' is deprecated for inline sources, " "use 'type: inline' instead", ) - type = "inline" + source_type = "inline" source_class = { "inline": DeckfileInlineSource, @@ -112,7 +112,7 @@ def __init__(self, *args, **data): "directory": DeckfileDirectorySource, "kustomize": DeckfileKustomizeSource, "helm": DeckfileHelmSource, - }.get(type) + }.get(source_type) self.sources.append(source_class(**source)) except KeyError: raise DeckfileError( diff --git a/getdeck/fetch/fetch.py b/getdeck/fetch/fetch.py index 01aafb4..02946bb 100644 --- a/getdeck/fetch/fetch.py +++ b/getdeck/fetch/fetch.py @@ -23,7 +23,27 @@ logger = logging.getLogger("deck") -def fetch_sources(deck: DeckfileDeck) -> List[SourceAux]: +class FetchError(Exception): + pass + + +def _fetch_deck(data_aux: DataAux, location: str) -> DataAux: + deckfile_aux = DeckfileAux(location=location) + fetch_behavior = select_deck_fetch_behavior(location=location) + if fetch_behavior: + deck_fetcher = DeckFetcher(fetch_behavior=fetch_behavior) + deckfile_aux = deck_fetcher.fetch(data=deckfile_aux) + else: + # local path and name + path, name = get_path_and_name(location=location) + deckfile_aux.path = path + deckfile_aux.name = name + + data_aux.deckfile_aux = deckfile_aux + return data_aux + + +def _fetch_sources(deck: DeckfileDeck) -> List[SourceAux]: source_fetcher = SourceFetcher(fetch_behavior=None) source_auxs = [] @@ -49,46 +69,37 @@ def fetch_sources(deck: DeckfileDeck) -> List[SourceAux]: except Exception as e: logger.debug(str(e)) del source_aux, source_auxs[:] - raise e + raise FetchError(f"Source fetching error: {str(e)}") source_auxs.append(source_aux) # noqa: F821 return source_auxs -def fetch_data(location: str, deck_name: str = None, *args, **kwargs) -> DataAux: +def fetch_data( + location: str, deck_name: str = None, fetch_sources: bool = True +) -> DataAux: """ delete returned DataAux to clean up temporary resources """ logger.info(f"Reading Deckfile from: {location}") data_aux = DataAux() - - # fetch - deckfile_aux = DeckfileAux(location=location) - fetch_behavior = select_deck_fetch_behavior(location=location) - if fetch_behavior: - deck_fetcher = DeckFetcher(fetch_behavior=fetch_behavior) - deckfile_aux = deck_fetcher.fetch(data=deckfile_aux) - else: - # local path and name - path, name = get_path_and_name(location=location) - deckfile_aux.path = path - deckfile_aux.name = name - - data_aux.deckfile_aux = deckfile_aux + data_aux = _fetch_deck(data_aux=data_aux, location=location) # validate - file = os.path.join(deckfile_aux.path, deckfile_aux.name) + file = os.path.join(data_aux.deckfile_aux.path, data_aux.deckfile_aux.name) if not os.path.isfile(file): del data_aux raise RuntimeError(f"Cannot identify {location} as Deckfile") - # parse + fetch sources deckfile = deckfile_selector.get(file) data_aux.deckfile = deckfile - deck = deckfile.get_deck(deck_name) - source_auxs = fetch_sources(deck=deck) - data_aux.source_auxs = source_auxs + + # parse + fetch sources + if fetch_sources: + deck = deckfile.get_deck(deck_name) + source_auxs = _fetch_sources(deck=deck) + data_aux.source_auxs = source_auxs return data_aux diff --git a/getdeck/fetch/types.py b/getdeck/fetch/types.py index ff2237b..532004b 100644 --- a/getdeck/fetch/types.py +++ b/getdeck/fetch/types.py @@ -23,17 +23,19 @@ class TemporaryData(BaseModel): class SourceAux(BaseModel): - location: str = None - path: str = None - name: str = None + location: Optional[str] = None + path: Optional[str] = None + name: Optional[str] = None temporary_data: Optional[TemporaryData] = None - source: Union[ - DeckfileInlineSource, - DeckfileFileSource, - DeckfileDirectorySource, - DeckfileHelmSource, - DeckfileKustomizeSource, + source: Optional[ + Union[ + DeckfileInlineSource, + DeckfileFileSource, + DeckfileDirectorySource, + DeckfileHelmSource, + DeckfileKustomizeSource, + ] ] = None def __del__(self): @@ -46,13 +48,12 @@ def __del__(self): if self.temporary_data.is_folder: shutil.rmtree(self.temporary_data.data) - return class DeckfileAux(BaseModel): location: str cwd: str = os.getcwd() - path: str = None + path: Optional[str] = None name: str = configuration.DECKFILE_FILE temporary_data: Optional[TemporaryData] = None @@ -66,14 +67,16 @@ def __del__(self): if self.temporary_data.is_folder: shutil.rmtree(self.temporary_data.data) - return class DataAux(BaseModel): - deckfile: Any = None # TODO: typing? - deckfile_aux: DeckfileAux = None - source_auxs: List[SourceAux] = [] + deckfile: Any = None + deckfile_aux: Optional[DeckfileAux] = None + source_auxs: List[SourceAux] = None def __del__(self): - del self.deckfile_aux - del self.source_auxs[:] + if self.deckfile_aux: + del self.deckfile_aux + + if self.source_auxs: + del self.source_auxs[:] diff --git a/getdeck/fetch/utils.py b/getdeck/fetch/utils.py index 80272a7..86c84c8 100644 --- a/getdeck/fetch/utils.py +++ b/getdeck/fetch/utils.py @@ -1,10 +1,10 @@ import os -from typing import Tuple +from typing import Optional, Tuple from getdeck import configuration -def get_path_and_name(location: str) -> Tuple[str, str]: +def get_path_and_name(location: Optional[str]) -> Tuple[str, str]: # None if location is None: location = "" diff --git a/getdeck/sources/tooler.py b/getdeck/sources/tooler.py index 1f26aea..2f1d10a 100644 --- a/getdeck/sources/tooler.py +++ b/getdeck/sources/tooler.py @@ -122,9 +122,7 @@ def render(self, deckfile_aux: DeckfileAux, source_aux: SourceAux, **kwargs): cmd = self.build_command() try: if source_aux.path: - source_path = os.path.join( - source_aux.path, source_aux.name or "" - ) # TODO: check if required + source_path = os.path.join(source_aux.path, source_aux.name or "") logger.debug(f"Render {source_path}") if not os.path.isabs(source_path): source_path = os.path.join( diff --git a/getdeck/sources/utils.py b/getdeck/sources/utils.py index 8fd7344..793880e 100644 --- a/getdeck/sources/utils.py +++ b/getdeck/sources/utils.py @@ -48,7 +48,7 @@ def prepare_k8s_workload_for_deck( # update current render behavior render_behavior = select_render_behavior(source=source_aux.source) - if not render_behavior: # TODO: check earlier after/during parsing? + if not render_behavior: logger.info( "Skipping source " f"{source_aux.source.__class__.__name__}: {ref or 'no ref'}" diff --git a/test/deckfile/deck.helm.yaml b/test/deckfile/deck.helm.yaml new file mode 100644 index 0000000..db6c30d --- /dev/null +++ b/test/deckfile/deck.helm.yaml @@ -0,0 +1,78 @@ +version: "1" + +cluster: + provider: k3d + minVersion: 4.0.0 + name: test-helm-file + nativeConfig: + apiVersion: k3d.io/v1alpha4 + kind: Simple + servers: 1 + agents: 1 + image: rancher/k3s:v1.22.9-k3s1 + options: + k3s: + extraArgs: + - arg: --disable=traefik + nodeFilters: + - server:* + ports: + - port: 61348:80 + nodeFilters: + - loadbalancer + - port: 8443:443 + nodeFilters: + - loadbalancer + - port: 31820:31820/UDP + nodeFilters: + - agent:0 + +decks: + - name: dashboard + namespace: default + sources: + - type: helm + ref: https://kubernetes.github.io/ingress-nginx + chart: ingress-nginx + releaseName: ingress-nginx + namespace: ingress-nginx + helmArgs: + - --create-namespace + parameters: + - name: controller.admissionWebhooks.enabled + value: false + - name: controller.ingressClassResource.default + value: true + + - type: helm + ref: https://kubernetes.github.io/dashboard/ + chart: kubernetes-dashboard + releaseName: dashboard + parameters: + - name: ingress.enabled + value: true + - name: ingress.hosts + value: "{dashboard.127.0.0.1.nip.io}" + - name: protocolHttp + value: true + - name: service.externalPort + value: 61348 + - name: serviceAccount.create + value: true + - name: serviceAccount.name + value: kubernetes-dashboard + + - type: inline + content: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: kubernetes-dashboard + namespace: kubernetes-dashboard + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: kubernetes-dashboard diff --git a/test/src/fetch/test_deck_fetch_behavior.py b/test/src/fetch/test_deck_fetch_behavior.py index 13a01bb..7a5dcba 100644 --- a/test/src/fetch/test_deck_fetch_behavior.py +++ b/test/src/fetch/test_deck_fetch_behavior.py @@ -66,9 +66,9 @@ def test_default(self): self.assertTrue(os.path.isdir(deckfile_aux.path)) self.assertIsNotNone(deckfile_aux.name) - file = os.path.join(deckfile_aux.path, deckfile_aux.name) + location = os.path.join(deckfile_aux.path, deckfile_aux.name) del deckfile_aux - self.assertFalse(os.path.isfile(file)) + self.assertFalse(os.path.isfile(location)) def test_url_invalid(self): deckfile_aux = DeckfileAux( diff --git a/test/src/fetch/test_utils.py b/test/src/fetch/test_utils.py index 4f032e9..b90c3f4 100644 --- a/test/src/fetch/test_utils.py +++ b/test/src/fetch/test_utils.py @@ -4,6 +4,9 @@ from getdeck.fetch.utils import get_path_and_name +DECK_YAML = "deck.yaml" + + class GetPathAndNameTest(TestCase): def test_name_default(self): location = configuration.DECKFILE_FILE @@ -20,11 +23,11 @@ def test_name_custom(self): self.assertEqual(name, "deck.empty.yaml") def test_name_yaml(self): - location = "deck.yaml" + location = DECK_YAML path, name = get_path_and_name(location=location) self.assertEqual(path, os.getcwd()) - self.assertEqual(name, "deck.yaml") + self.assertEqual(name, DECK_YAML) def test_name_yml(self): location = "deck.yml" @@ -38,53 +41,53 @@ def test_path_relative(self): path, name = get_path_and_name(location=location) self.assertEqual(path, os.getcwd()) - self.assertEqual(name, "deck.yaml") + self.assertEqual(name, DECK_YAML) def test_path_relative_subfolder(self): location = "./path/deck.yaml" path, name = get_path_and_name(location=location) self.assertEqual(path, os.path.join(os.getcwd(), "path")) - self.assertEqual(name, "deck.yaml") + self.assertEqual(name, DECK_YAML) def test_path_relative_parentfolder(self): location = "../deck.yaml" path, name = get_path_and_name(location=location) self.assertEqual(path, os.path.dirname(os.getcwd())) - self.assertEqual(name, "deck.yaml") + self.assertEqual(name, DECK_YAML) def test_path_user(self): location = "~/deck.yaml" path, name = get_path_and_name(location=location) self.assertEqual(path, os.path.expanduser("~")) - self.assertEqual(name, "deck.yaml") + self.assertEqual(name, DECK_YAML) def test_path_user_subfolder(self): location = "~/path/deck.yaml" path, name = get_path_and_name(location=location) self.assertEqual(path, os.path.join(os.path.expanduser("~"), "path")) - self.assertEqual(name, "deck.yaml") + self.assertEqual(name, DECK_YAML) def test_dot(self): location = "." path, name = get_path_and_name(location=location) self.assertEqual(path, os.getcwd()) - self.assertEqual(name, "deck.yaml") + self.assertEqual(name, DECK_YAML) def test_empty(self): location = "" path, name = get_path_and_name(location=location) self.assertEqual(path, os.getcwd()) - self.assertEqual(name, "deck.yaml") + self.assertEqual(name, DECK_YAML) def test_none(self): location = None path, name = get_path_and_name(location=location) self.assertEqual(path, os.getcwd()) - self.assertEqual(name, "deck.yaml") + self.assertEqual(name, DECK_YAML)