From af62b7812123abd34534c2ce628ac7776eb41d9b Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Wed, 2 Nov 2022 11:45:14 +0100 Subject: [PATCH 1/3] fix: copy data into tooler - helm source with path option failed to copy data correctly - targetRevision defaulted to "" which caused a git checkout error --- getdeck/deckfile/file.py | 10 +++++----- getdeck/fetch/source_fetcher.py | 16 +++++++++++----- getdeck/sources/tooler.py | 32 +++++++++++++++++--------------- test/sources/deck.file.yaml | 6 +++++- test/src/fetch/test_fetch.py | 2 +- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/getdeck/deckfile/file.py b/getdeck/deckfile/file.py index 288b4ae..fb1abd0 100644 --- a/getdeck/deckfile/file.py +++ b/getdeck/deckfile/file.py @@ -1,6 +1,6 @@ import logging from abc import ABC, abstractmethod -from typing import List, Dict, Union +from typing import List, Dict, Optional, Union from pydantic import BaseModel @@ -37,7 +37,7 @@ def get_provider(self, config) -> AbstractProvider: class DeckfileHelmSource(BaseModel): type: str = "helm" ref: str - targetRevision: str = "" + targetRevision: Optional[str] = None namespace: str = "" path: str = None chart: str = None # this is set when pulling directly from a Helm repo @@ -56,21 +56,21 @@ class DeckfileInlineSource(BaseModel): class DeckfileFileSource(BaseModel): type: str = "file" ref: str = None - targetRevision: str = "" + targetRevision: Optional[str] = None path: str = "" class DeckfileKustomizeSource(BaseModel): type: str = "kustomize" ref: str - targetRevision: str = "" + targetRevision: Optional[str] = None path: str = "" class DeckfileDirectorySource(BaseModel): type: str = "directory" ref: str - targetRevision: str = "" + targetRevision: Optional[str] = None path: str recursive: bool diff --git a/getdeck/fetch/source_fetcher.py b/getdeck/fetch/source_fetcher.py index d70124e..a53a2e8 100644 --- a/getdeck/fetch/source_fetcher.py +++ b/getdeck/fetch/source_fetcher.py @@ -31,25 +31,31 @@ def fetch(self, data: SourceAux, *args, **kwargs) -> SourceAux: ref, rev = location.split("#") else: ref = location - rev = "HEAD" + rev = None rev = kwargs.get("targetRevision", rev) path = kwargs.get("path", "") temporary_folder = tempfile.mkdtemp() - path = os.path.join(temporary_folder, path) - data.path = os.path.dirname(path) - data.name = os.path.basename(path) data.temporary_data = TemporaryData(data=temporary_folder, is_folder=True) try: repo = Repo.clone_from(ref, temporary_folder) - repo.git.checkout(rev) + if rev: + repo.git.checkout(rev) except GitError as e: raise FetchError(f"Cannot checkout {rev} from {ref}: {e}") except Exception as e: raise e + temporary_path = os.path.join(temporary_folder, path) + if os.path.isdir(temporary_path): + data.path = temporary_path + data.name = None + else: + data.name = os.path.basename(temporary_path) + data.path = os.path.dirname(temporary_path) + return data diff --git a/getdeck/sources/tooler.py b/getdeck/sources/tooler.py index f3778dd..e20b75c 100644 --- a/getdeck/sources/tooler.py +++ b/getdeck/sources/tooler.py @@ -124,21 +124,23 @@ class Tooler(RenderBehavior): def render(self, deckfile_aux: DeckfileAux, source_aux: SourceAux, **kwargs): cmd = self.build_command() try: - if source_aux.path: - source_path = os.path.join(source_aux.path, source_aux.name or "") - logger.debug(f"Render {source_path}") - if not os.path.isabs(source_path): - source_path = os.path.join( - deckfile_aux.path, source_path.removeprefix("./") - ) - - if os.path.isdir(source_path): - shutil.copytree( - source_path, self.tmp_source.name, dirs_exist_ok=True - ) - else: - shutil.copy(source_path, self.tmp_source.name) - + if source_aux.temporary_data: + source_path = source_aux.temporary_data.data + else: + source_path = source_aux.path + + if not os.path.isabs(source_path): + source_path = os.path.join( + deckfile_aux.path, source_path.removeprefix("./") + ) + + # copy data + if os.path.isdir(source_path): + shutil.copytree(source_path, self.tmp_source.name, dirs_exist_ok=True) + else: + shutil.copy(source_path, self.tmp_source.name) + + logger.debug(f"Render: {source_path}") self.run_tooler(cmd) source_files = self.collect_workload_files() return source_files diff --git a/test/sources/deck.file.yaml b/test/sources/deck.file.yaml index 411ccb9..5509951 100644 --- a/test/sources/deck.file.yaml +++ b/test/sources/deck.file.yaml @@ -43,7 +43,11 @@ decks: - type: file ref: https://raw.githubusercontent.com/Getdeck/getdeck/main/test/beiboot/hello.yaml + - type: file + ref: git@github.com:Getdeck/getdeck.git + path: test/resources/test/hello.yaml + - type: file ref: git@github.com:Getdeck/getdeck.git targetRevision: main - path: test/beiboot/hello.yaml + path: test/resources/test/hello.yaml diff --git a/test/src/fetch/test_fetch.py b/test/src/fetch/test_fetch.py index 4e33f6b..a8a7df4 100644 --- a/test/src/fetch/test_fetch.py +++ b/test/src/fetch/test_fetch.py @@ -23,7 +23,7 @@ def test_local_file(self): data_aux = fetch_data(location) self.assertIsNotNone(data_aux.deckfile) self.assertIsNotNone(data_aux.deckfile_aux) - self.assertEqual(len(data_aux.source_auxs), 5) + self.assertEqual(len(data_aux.source_auxs), 6) def test_local_helm(self): location = "./test/sources/deck.helm.yaml" From 06488544aae9bd98497c5b92ae06d87efb1d32a5 Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Wed, 2 Nov 2022 16:42:10 +0100 Subject: [PATCH 2/3] fix: source_path none --- getdeck/sources/tooler.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/getdeck/sources/tooler.py b/getdeck/sources/tooler.py index e20b75c..506a843 100644 --- a/getdeck/sources/tooler.py +++ b/getdeck/sources/tooler.py @@ -129,16 +129,19 @@ def render(self, deckfile_aux: DeckfileAux, source_aux: SourceAux, **kwargs): else: source_path = source_aux.path - if not os.path.isabs(source_path): - source_path = os.path.join( - deckfile_aux.path, source_path.removeprefix("./") - ) - - # copy data - if os.path.isdir(source_path): - shutil.copytree(source_path, self.tmp_source.name, dirs_exist_ok=True) - else: - shutil.copy(source_path, self.tmp_source.name) + if source_path: + if not os.path.isabs(source_path): + source_path = os.path.join( + deckfile_aux.path, source_path.removeprefix("./") + ) + + # copy data + if os.path.isdir(source_path): + shutil.copytree( + source_path, self.tmp_source.name, dirs_exist_ok=True + ) + else: + shutil.copy(source_path, self.tmp_source.name) logger.debug(f"Render: {source_path}") self.run_tooler(cmd) From 9323eff59aa66590917212f759950243dcdc4406 Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Wed, 2 Nov 2022 17:06:04 +0100 Subject: [PATCH 3/3] fix: print debug location --- getdeck/sources/tooler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getdeck/sources/tooler.py b/getdeck/sources/tooler.py index 506a843..91d92f9 100644 --- a/getdeck/sources/tooler.py +++ b/getdeck/sources/tooler.py @@ -143,7 +143,7 @@ def render(self, deckfile_aux: DeckfileAux, source_aux: SourceAux, **kwargs): else: shutil.copy(source_path, self.tmp_source.name) - logger.debug(f"Render: {source_path}") + logger.debug(f"Render: {source_aux.location}") self.run_tooler(cmd) source_files = self.collect_workload_files() return source_files