diff --git a/.github/workflows/python-tester.yaml b/.github/workflows/python-tester.yaml index b7fecb3..138e69a 100644 --- a/.github/workflows/python-tester.yaml +++ b/.github/workflows/python-tester.yaml @@ -22,21 +22,21 @@ jobs: - name: Poetry installation run: | poetry install - # Run the 'Try it yourself' - - uses: Getdeck/getdeck-action@main - name: Create Infrastructure from Deckfile - with: - deck-file-path: https://github.com/gefyrahq/gefyra-demos.git - timeout: "180" - - name: Remove cluster using getdeck - run: | - poetry run coverage run -a -m getdeck remove deck.gefyra.test.yaml --name oauth2-demo + - name: Install k3d + shell: bash + run: "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash" - name: Create a cluster using getdeck run: | - poetry run coverage run -a -m getdeck get deck.gefyra.test.yaml --name oauth2-demo - - name: Stop cluster + poetry run coverage run -a -m getdeck get --name oauth2-demo --wait --timeout 180 https://github.com/gefyrahq/gefyra-demos.git + - name: Stop the cluster + run: | + poetry run coverage run -a -m getdeck stop https://github.com/gefyrahq/gefyra-demos.git + - name: Start the cluster again + run: | + poetry run coverage run -a -m getdeck get --name oauth2-demo https://github.com/gefyrahq/gefyra-demos.git + - name: Remove the cluster run: | - poetry run coverage run -a -m getdeck stop deck.gefyra.test.yaml + poetry run coverage run -a -m getdeck remove --cluster https://github.com/gefyrahq/gefyra-demos.git - name: Show coverage report run: | poetry run coverage report diff --git a/getdeck/deckfile/deckfile_1.py b/getdeck/deckfile/deckfile_1.py index 5e72195..cdaf916 100644 --- a/getdeck/deckfile/deckfile_1.py +++ b/getdeck/deckfile/deckfile_1.py @@ -9,6 +9,11 @@ class Deckfile_1_0(Deckfile, BaseModel): + # meta + file_path: str + file_name: str + + # content version: Optional[str] cluster: DeckfileCluster = None decks: List[DeckfileDeck] diff --git a/getdeck/deckfile/selector.py b/getdeck/deckfile/selector.py index 5ce67da..ec00851 100644 --- a/getdeck/deckfile/selector.py +++ b/getdeck/deckfile/selector.py @@ -52,7 +52,11 @@ def get(self, path_deckfile: str = None) -> Union[Deckfile, Deckfile_1_0, None]: else: raise DeckfileVersionError("Version in Deckfile is missing") logger.debug("The raw Deckfile data: " + str(data)) - return deckfile_class(**data) + + file_path = os.path.dirname(path_deckfile) + file_name = os.path.basename(path_deckfile) + deckfile = deckfile_class(file_path=file_path, file_name=file_name, **data) + return deckfile deckfile_selector = DeckfileSelector( diff --git a/getdeck/sources/fetcher.py b/getdeck/sources/fetcher.py new file mode 100644 index 0000000..0859257 --- /dev/null +++ b/getdeck/sources/fetcher.py @@ -0,0 +1,83 @@ +import logging +from operator import methodcaller +from typing import List, Union + +from getdeck.configuration import ClientConfiguration +from getdeck.deckfile.file import ( + DeckfileFileSource, + DeckfileKustomizeSource, + DeckfileHelmSource, +) +from getdeck.sources.types import K8sSourceFile +from getdeck.utils import sniff_protocol + +logger = logging.getLogger("deck") + + +class FetcherError(Exception): + pass + + +class Fetcher: + def __init__( + self, + path: str, + source: Union[DeckfileFileSource, DeckfileKustomizeSource, DeckfileHelmSource], + config: ClientConfiguration, + namespace: str, + ): + self.path = path + self.source = source + self.config = config + self.namespace = namespace + + @property + def not_supported_message(self): + return "Could not fetch source" + + def fetch(self, **kwargs) -> List[K8sSourceFile]: + handler = methodcaller(f"fetch_{self.type}", **kwargs) + try: + return handler(self) + except NotImplementedError: + logger.warning(self.not_supported_message) + return [] + + @property + def type(self) -> str: + if getattr(self.source, "content", None) is not None: + return "content" + protocol = sniff_protocol(self.source.ref) + return protocol + + def fetch_git(self, **kwargs): + raise NotImplementedError + + def fetch_http(self, **kwargs): + raise NotImplementedError + + def fetch_https(self, **kwargs): + raise NotImplementedError + + def fetch_local(self, **kwargs): + raise NotImplementedError + + def fetch_content(self, **kwargs): + raise NotImplementedError + + +class FetcherContext: + def __init__(self, strategy: Fetcher) -> None: + self._strategy = strategy + + @property + def strategy(self) -> Fetcher: + return self._strategy + + @strategy.setter + def strategy(self, strategy: Fetcher) -> None: + self._strategy = strategy + + def fetch_source_files(self) -> List[K8sSourceFile]: + source_files = self._strategy.fetch() + return source_files diff --git a/getdeck/sources/file.py b/getdeck/sources/file.py index bf2af9a..604c828 100644 --- a/getdeck/sources/file.py +++ b/getdeck/sources/file.py @@ -1,101 +1,44 @@ import logging -from operator import methodcaller import os +from pathlib import PurePath import tempfile -from typing import List, Union +from typing import List import requests import yaml -from getdeck.configuration import ClientConfiguration -from getdeck.deckfile.file import ( - DeckfileFileSource, - DeckfileKustomizeSource, - DeckfileHelmSource, -) +from getdeck.sources.fetcher import Fetcher, FetcherError from getdeck.sources.types import K8sSourceFile -from getdeck.utils import sniff_protocol from git import Repo logger = logging.getLogger("deck") -class FetcherError(Exception): - pass - - -class Fetcher: - def __init__( - self, - source: Union[DeckfileFileSource, DeckfileKustomizeSource, DeckfileHelmSource], - config: ClientConfiguration, - namespace: str, - ): - self.source = source - self.config = config - self.namespace = namespace - - @property - def not_supported_message(self): - return "Could not fetch source" - - def fetch(self, **kwargs) -> List[K8sSourceFile]: - handler = methodcaller(f"fetch_{self.type}", **kwargs) - try: - return handler(self) - except NotImplementedError: - logger.warning(self.not_supported_message) - return [] - - @property - def type(self) -> str: - if getattr(self.source, "content", None) is not None: - return "content" - protocol = sniff_protocol(self.source.ref) - return protocol - - def fetch_git(self, **kwargs): - raise NotImplementedError - - def fetch_http(self, **kwargs): - raise NotImplementedError - - def fetch_https(self, **kwargs): - raise NotImplementedError - - def fetch_local(self, **kwargs): - raise NotImplementedError - - def fetch_content(self, **kwargs): - raise NotImplementedError - - class FileFetcher(Fetcher): @property def not_supported_message(self): return f"Protocol {self.type} not supported for {type(self.source).__name__}" - @staticmethod - def _parse_source_file(ref: str) -> List[K8sSourceFile]: + def _parse_source_file(self, ref: str) -> List[K8sSourceFile]: with open(ref, "r") as input_file: docs = yaml.load_all(input_file.read(), Loader=yaml.FullLoader) k8s_workload_files = [] for doc in docs: if doc: - k8s_workload_files.append(K8sSourceFile(name=ref, content=doc)) + k8s_workload_files.append( + K8sSourceFile(name=ref, content=doc, namespace=self.namespace) + ) return k8s_workload_files - @staticmethod - def _parse_source_files(refs: List[str]) -> List[K8sSourceFile]: + def _parse_source_files(self, refs: List[str]) -> List[K8sSourceFile]: k8s_workload_files = [] for ref in refs: - workloads = FileFetcher._parse_source_file(ref=ref) + workloads = self._parse_source_file(ref=ref) k8s_workload_files += workloads return k8s_workload_files - @staticmethod - def _parse_source_directory(ref: str) -> List[K8sSourceFile]: + def _parse_source_directory(self, ref: str) -> List[K8sSourceFile]: refs = [] if not os.path.isdir(ref): @@ -109,19 +52,23 @@ def _parse_source_directory(ref: str) -> List[K8sSourceFile]: refs.append(os.path.join(ref, file)) # parse workloads - k8s_workload_files = FileFetcher._parse_source_files(refs=refs) + k8s_workload_files = self._parse_source_files(refs=refs) return k8s_workload_files - @staticmethod - def _parse_source(ref: str) -> List[K8sSourceFile]: + def _parse_source(self, ref: str) -> List[K8sSourceFile]: if os.path.isdir(ref): - k8s_workload_files = FileFetcher._parse_source_directory(ref=ref) + k8s_workload_files = self._parse_source_directory( + ref=ref, + ) else: - k8s_workload_files = FileFetcher._parse_source_file(ref=ref) + k8s_workload_files = self._parse_source_file(ref=ref) return k8s_workload_files def fetch_content(self, **kwargs) -> List[K8sSourceFile]: - return [K8sSourceFile(name="Deckfile", content=self.source.content)] + source_file = K8sSourceFile( + name="Deckfile", content=self.source.content, namespace=self.namespace + ) + return [source_file] def fetch_http(self, **kwargs) -> List[K8sSourceFile]: k8s_workload_files = [] @@ -134,7 +81,9 @@ def fetch_http(self, **kwargs) -> List[K8sSourceFile]: for doc in docs: if doc: k8s_workload_files.append( - K8sSourceFile(name=self.source.ref, content=doc) + K8sSourceFile( + name=self.source.ref, content=doc, namespace=self.namespace + ) ) return k8s_workload_files except Exception as e: @@ -147,7 +96,9 @@ def fetch_https(self, **kwargs): def fetch_local(self, **kwargs): try: logger.debug(f"Reading file {self.source.ref}") - k8s_workload_files = self._parse_source(ref=self.source.ref) + ref = str(PurePath(os.path.join(self.path, self.source.ref))) + + k8s_workload_files = self._parse_source(ref=ref) return k8s_workload_files except Exception as e: logger.error(f"Error loading file from http {e}") diff --git a/getdeck/sources/selector.py b/getdeck/sources/selector.py new file mode 100644 index 0000000..9a3f07c --- /dev/null +++ b/getdeck/sources/selector.py @@ -0,0 +1,21 @@ +from typing import Union +from getdeck.deckfile.file import ( + DeckfileFileSource, + DeckfileHelmSource, + DeckfileKustomizeSource, +) +from getdeck.sources.fetcher import Fetcher +from getdeck.sources.file import FileFetcher +from getdeck.sources.helm import HelmFetcher +from getdeck.sources.kustomize import KustomizeFetcher + + +def select_fetcher_strategy( + source: Union[DeckfileFileSource, DeckfileHelmSource, DeckfileKustomizeSource] +) -> Fetcher | None: + fetcher_strategy = { + DeckfileFileSource: FileFetcher, + DeckfileHelmSource: HelmFetcher, + DeckfileKustomizeSource: KustomizeFetcher, + }.get(type(source), None) + return fetcher_strategy diff --git a/getdeck/sources/utils.py b/getdeck/sources/utils.py index 9724f78..13d2d1e 100644 --- a/getdeck/sources/utils.py +++ b/getdeck/sources/utils.py @@ -1,49 +1,13 @@ import logging -from typing import Union, List from getdeck.configuration import ClientConfiguration -from getdeck.deckfile.file import ( - DeckfileHelmSource, - DeckfileDirectorySource, - Deckfile, - DeckfileFileSource, - DeckfileKustomizeSource, -) -from getdeck.sources.file import FileFetcher, Fetcher -from getdeck.sources.helm import HelmFetcher -from getdeck.sources.kustomize import KustomizeFetcher -from getdeck.sources.types import GeneratedDeck, K8sSourceFile - -logger = logging.getLogger("deck") +from getdeck.deckfile.file import Deckfile +from getdeck.sources.fetcher import FetcherContext +from getdeck.sources.selector import select_fetcher_strategy +from getdeck.sources.types import GeneratedDeck -def fetch_deck_source( - config: ClientConfiguration, - source: Union[ - DeckfileFileSource, - DeckfileHelmSource, - DeckfileDirectorySource, - DeckfileKustomizeSource, - ], - namespace: str = "default", -) -> List[K8sSourceFile]: - logger.info( - "Processing source " - f"{source.__class__.__name__}: {'no ref' if not source.ref else source.ref}" - ) - if isinstance(source, DeckfileHelmSource): - fetcher = HelmFetcher(source, config, namespace) - elif isinstance(source, DeckfileFileSource): - fetcher = FileFetcher(source, config, namespace) - elif isinstance(source, DeckfileKustomizeSource): - fetcher = KustomizeFetcher(source, config, namespace) - else: - logger.info( - "Skipping source " - f"{source.__class__.__name__}: {'no ref' if not source.ref else source.ref}" - ) - fetcher = Fetcher(source, config, namespace) - return fetcher.fetch() +logger = logging.getLogger("deck") def prepare_k8s_workload_for_deck( @@ -51,16 +15,42 @@ def prepare_k8s_workload_for_deck( ) -> GeneratedDeck: deck = deckfile.get_deck(deck_name) logger.debug(deck) + # fetch all sources namespace = deck.namespace or "default" generated_deck = GeneratedDeck(name=deck.name, namespace=namespace, files=[]) + logger.info(f"Processing {len(deck.sources)} source(s)") + fetcher_context = FetcherContext(strategy=None) + for source in deck.sources: + logger.info( + "Processing source " + f"{source.__class__.__name__}: {'no ref' if not source.ref else source.ref}" + ) + + # update current fetcher strategy + strategy = select_fetcher_strategy(source=source) + if not strategy: + logger.info( + "Skipping source " + f"{source.__class__.__name__}: {'no ref' if not source.ref else source.ref}" + ) + continue + + fetcher_context.strategy = strategy( + deckfile.file_path, source, config, namespace + ) + # if a source, such as Helm, specifies another namespace logger.debug(source) if hasattr(source, "namespace"): namespace = source.namespace or deck.namespace or "default" else: namespace = deck.namespace or "default" - generated_deck.files.extend(fetch_deck_source(config, source, namespace)) + + # fetch source files + source_files = fetcher_context.fetch_source_files() + generated_deck.files.extend(source_files) + return generated_deck diff --git a/test/beiboot/deck.test.yaml b/test/beiboot/deck.test.yaml index 399f1d9..0915a4d 100644 --- a/test/beiboot/deck.test.yaml +++ b/test/beiboot/deck.test.yaml @@ -2,15 +2,15 @@ version: "1" cluster: provider: beiboot - minVersion: 0.3.0 + minVersion: 0.6.0 name: beiboot-cluster nativeConfig: - context: + context: ports: - port: 8080:80 decks: - - name: dashboard + - name: hello namespace: default sources: - type: helm diff --git a/test/beiboot/hello.yaml b/test/beiboot/hello.yaml index 6034cd3..c4ddc78 100644 --- a/test/beiboot/hello.yaml +++ b/test/beiboot/hello.yaml @@ -16,7 +16,7 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: hello-nginxdemo + name: hello-nginx-demo spec: selector: matchLabels: diff --git a/test/resources/file/hello.yaml b/test/resources/file/hello.yaml new file mode 100644 index 0000000..c4ddc78 --- /dev/null +++ b/test/resources/file/hello.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Service +metadata: + name: hello-nginx + labels: + run: hello-nginx +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: hello-nginx + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-nginx-demo +spec: + selector: + matchLabels: + app: hello-nginx + replicas: 1 + template: + metadata: + labels: + app: hello-nginx + spec: + containers: + - name: hello-nginx + image: nginxdemos/hello + ports: + - containerPort: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: hello-ingress +spec: + ingressClassName: nginx + rules: + - host: hello.127.0.0.1.nip.io + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: hello-nginx + port: + number: 80 diff --git a/test/resources/helm/.gitkeep b/test/resources/helm/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/test/resources/kustomize/.gitkeep b/test/resources/kustomize/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/test/sources/deck.file.yaml b/test/sources/deck.file.yaml new file mode 100644 index 0000000..6129e58 --- /dev/null +++ b/test/sources/deck.file.yaml @@ -0,0 +1,71 @@ +version: "1" + +cluster: + provider: k3d + minVersion: 4.0.0 + name: test-sources-file + nativeConfig: + apiVersion: k3d.io/v1alpha4 + kind: Simple + servers: 1 + agents: 1 + image: rancher/k3s:v1.22.9-k3s1 + options: + k3s: + extraArgs: + - arg: --disable=traefik + nodeFilters: + - server:* + ports: + - port: 61346:80 + nodeFilters: + - loadbalancer + - port: 8443:443 + nodeFilters: + - loadbalancer + - port: 31820:31820/UDP + nodeFilters: + - agent:0 + +decks: + - name: hello + namespace: default + sources: + - type: helm + ref: https://kubernetes.github.io/ingress-nginx + chart: ingress-nginx + releaseName: ingress-nginx + namespace: ingress-nginx + helmArgs: + - --create-namespace + parameters: + - name: controller.admissionWebhooks.enabled + value: false + - name: controller.ingressClassResource.default + value: true + + - type: file + ref: ./hello.yaml + + - type: file + ref: ./resources/hello.yaml + + - type: file + ref: ../resources/file/hello.yaml + + - type: file + ref: https://raw.githubusercontent.com/Getdeck/getdeck/main/test/beiboot/hello.yaml + + - type: file + ref: git@github.com:Getdeck/getdeck.git + targetRevision: main + path: test/beiboot/hello.yaml + + - type: file + ref: content + content: + { + "kind": "Namespace", + "apiVersion": "v1", + "metadata": { "name": "content", "labels": { "name": "content" } }, + } diff --git a/test/sources/hello.yaml b/test/sources/hello.yaml new file mode 100644 index 0000000..c4ddc78 --- /dev/null +++ b/test/sources/hello.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Service +metadata: + name: hello-nginx + labels: + run: hello-nginx +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: hello-nginx + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-nginx-demo +spec: + selector: + matchLabels: + app: hello-nginx + replicas: 1 + template: + metadata: + labels: + app: hello-nginx + spec: + containers: + - name: hello-nginx + image: nginxdemos/hello + ports: + - containerPort: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: hello-ingress +spec: + ingressClassName: nginx + rules: + - host: hello.127.0.0.1.nip.io + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: hello-nginx + port: + number: 80 diff --git a/test/sources/resources/hello.yaml b/test/sources/resources/hello.yaml new file mode 100644 index 0000000..c4ddc78 --- /dev/null +++ b/test/sources/resources/hello.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Service +metadata: + name: hello-nginx + labels: + run: hello-nginx +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: hello-nginx + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-nginx-demo +spec: + selector: + matchLabels: + app: hello-nginx + replicas: 1 + template: + metadata: + labels: + app: hello-nginx + spec: + containers: + - name: hello-nginx + image: nginxdemos/hello + ports: + - containerPort: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: hello-ingress +spec: + ingressClassName: nginx + rules: + - host: hello.127.0.0.1.nip.io + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + service: + name: hello-nginx + port: + number: 80