refactor: strategy pattern for deckfile fetching

.github/workflows/python-test-gefyra.yaml

@@ -10,7 +10,7 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python

.github/workflows/python-test.yaml

@@ -9,6 +9,12 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: ssh agent
+        uses: webfactory/ssh-agent@v0.5.4
+        with:
+          ssh-private-key: |
+            ${{ secrets.SSH_KEY_OF_GETDECK }}
+
       - name: Setup Python
         uses: actions/setup-python@v2
         with:

getdeck/__main__.py

@@ -134,15 +134,19 @@ def main():
             logger.info(names)
         elif args.action == "get":
             if args.wait:
-                run_deck(
-                    args.Deckfile,
-                    args.name,
-                    ignore_cluster=args.no_cluster,
-                    wait=True,
-                    timeout=int(args.timeout),
-                )
+                wait = True
+                timeout = int(args.timeout)
             else:
-                run_deck(args.Deckfile, args.name, ignore_cluster=args.no_cluster)
+                wait = False
+                timeout = None
+
+            run_deck(
+                args.Deckfile,
+                args.name,
+                ignore_cluster=args.no_cluster,
+                wait=wait,
+                timeout=timeout,
+            )
         elif args.action == "remove":
             if args.cluster:
                 remove_cluster(args.Deckfile, ignore_cluster=args.no_cluster)

getdeck/api/get.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import shutil
 from typing import Callable
 
@@ -128,8 +129,10 @@ def run_deck(  # noqa: C901
     if notes := deckfile.get_deck(deck_name).notes:
         logger.info(notes)
 
-    if is_temp_dir:
+    # TODO: refactor/remove?
+    if is_temp_dir and os.path.isdir(working_dir_path):
         shutil.rmtree(working_dir_path)
+
     if wait:
         _wait_ready(config, generated_deck, timeout)
     return True

getdeck/api/hosts.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import shutil
 import socket
 
@@ -53,8 +54,11 @@ def run_hosts(
             logger.error(f"Unknown host action '{host_action}'")
     else:
         logger.info("No hosts specified in Deckfile")
-    if is_temp_dir:
+
+    # TODO: refactor/remove?
+    if is_temp_dir and os.path.isdir(working_dir_path):
         shutil.rmtree(working_dir_path)
+
     return True
 
 

getdeck/api/list.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import shutil
 from typing import List
 
@@ -17,6 +18,9 @@ def get_available_decks(deckfile_location: str, config=default_configuration) ->
     )
     available_decks = deckfile.get_decks()
     logger.debug(available_decks)
-    if is_temp_dir:
+
+    # TODO: refactor/remove?
+    if is_temp_dir and os.path.isdir(working_dir_path):
         shutil.rmtree(working_dir_path)
+
     return available_decks

getdeck/api/remove.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import shutil
 from typing import Callable
 
@@ -25,8 +26,11 @@ def remove_cluster(
         k8s_provider.delete()
     else:
         logger.info("Cluster does not exist")
-    if is_temp_dir:
+
+    # TODO: refactor/remove?
+    if is_temp_dir and os.path.isdir(working_dir_path):
         shutil.rmtree(working_dir_path)
+
     return True
 
 
@@ -83,6 +87,8 @@ def remove_deck(
     else:
         logger.info("Cluster does not exist")
 
-    if is_temp_dir:
+    # TODO: refactor/remove?
+    if is_temp_dir and os.path.isdir(working_dir_path):
         shutil.rmtree(working_dir_path)
+
     return True

getdeck/api/stop.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import shutil
 from typing import Callable
 
@@ -22,6 +23,9 @@ def stop_cluster(
     )
     k8s_provider = ensure_cluster(deckfile, config, ignore_cluster, do_install=False)
     logger.info("Stopping cluster")
-    if is_temp_dir:
+
+    # TODO: refactor/remove?
+    if is_temp_dir and os.path.isdir(working_dir_path):
         shutil.rmtree(working_dir_path)
+
     k8s_provider.stop()

getdeck/deckfile/fetch/deck_fetcher.py

@@ -0,0 +1,129 @@
+from abc import ABC, abstractmethod
+import os
+from pydantic import BaseModel
+import logging
+import shutil
+import tempfile
+
+import requests
+from git import Repo, GitError
+
+from getdeck import configuration
+
+logger = logging.getLogger("deck")
+
+
+class FetchError(Exception):
+    pass
+
+
+class DeckFileData(BaseModel):
+    argument_location: str
+    cwd: str = os.getcwd()
+    path: str = None
+    name: str = configuration.DECKFILE_FILE
+    working_dir_path: str = None
+    is_temp_dir: bool = False
+
+
+class DeckFetchBehavior(ABC):
+    @abstractmethod
+    def fetch(self, data: DeckFileData) -> DeckFileData:
+        pass
+
+    @abstractmethod
+    def clean_up(self, data: DeckFileData):
+        pass
+
+
+class Git(DeckFetchBehavior):
+    def fetch(self, data: DeckFileData) -> DeckFileData:
+        location = data.argument_location
+
+        if "#" in location:
+            ref, rev = location.split("#")
+        else:
+            ref = location
+            rev = "HEAD"
+
+        tmp_dir = tempfile.mkdtemp()
+        data.path = tmp_dir
+        data.working_dir_path = tmp_dir
+        data.is_temp_dir = True
+
+        try:
+            repo = Repo.clone_from(ref, tmp_dir)
+            repo.git.checkout(rev)
+        except GitError as e:
+            self.clean_up(data=data)
+            raise FetchError(f"Cannot checkout {rev} from {ref}: {e}")
+        except Exception as e:
+            self.clean_up(data=data)
+            raise e
+
+        return data
+
+    def clean_up(self, data: DeckFileData):
+        shutil.rmtree(data.working_dir_path)
+
+
+class Http(DeckFetchBehavior):
+    def fetch(self, data: DeckFileData) -> DeckFileData:
+        location = data.argument_location
+
+        download = tempfile.NamedTemporaryFile(delete=False)
+        data.path = os.path.dirname(download.name)
+        data.name = os.path.basename(download.name)
+
+        try:
+            logger.debug(f"Requesting {location}")
+            with requests.get(location, stream=True, timeout=10.0) as res:
+                res.raise_for_status()
+                for chunk in res.iter_content(chunk_size=4096):
+                    if chunk:
+                        download.write(chunk)
+                download.flush()
+            download.close()
+        except Exception as e:
+            download.close()
+            self.clean_up(data=data)
+            raise FetchError(
+                f"Cannot download Deckfile from http(s) location {location}: {e}"
+            )
+
+        return data
+
+    def clean_up(self, data: DeckFileData):
+        os.remove(os.path.join(data.path, data.name))
+
+
+class DeckFetcher:
+    def __init__(self, fetch_behavior: DeckFetchBehavior) -> None:
+        self._fetch_behavior = fetch_behavior
+
+    @property
+    def fetch_behavior(self) -> DeckFetchBehavior:
+        return self._fetch_behavior
+
+    @fetch_behavior.setter
+    def fetch_behavior(self, fetch_behavior: DeckFetchBehavior) -> None:
+        self._fetch_behavior = fetch_behavior
+
+    def fetch(self, data: DeckFileData) -> DeckFileData:
+        data = self._fetch_behavior.fetch(data=data)
+        return data
+
+
+def select_fetch_behavior(location: str) -> DeckFetchBehavior | None:
+    if "#" in location:
+        location, _ = location.split("#")
+
+    location_lo = location.lower()
+
+    if location_lo.startswith("git") or location_lo.endswith(".git"):
+        return Git()
+
+    if location_lo.startswith("https") or location_lo.startswith("http"):
+        return Http()
+
+    return None

getdeck/deckfile/fetch/utils.py

@@ -0,0 +1,33 @@
+import os
+from typing import Tuple
+
+from getdeck import configuration
+
+
+def get_path_and_name(location: str) -> Tuple[str, str]:
+    # None
+    if location is None:
+        location = ""
+
+    # "~"
+    if location.startswith("~"):
+        location = os.path.expanduser(location)
+
+    # ".", ""
+    if location in [".", ""]:
+        for extension in [".yaml", ".yml"]:
+            location_default = os.path.join(
+                os.getcwd(),
+                os.path.splitext(configuration.DECKFILE_FILE)[0] + extension,
+            )
+            if os.path.isfile(location_default):
+                location = location_default
+                break
+        else:
+            location = os.path.join(os.getcwd(), configuration.DECKFILE_FILE)
+
+    name = os.path.basename(location)
+    dirname = os.path.dirname(location)
+    path = os.path.abspath(dirname)
+
+    return path, name