From c6d9e51086a20114b30d5ae825dee048ed4fe808 Mon Sep 17 00:00:00 2001 From: DQ Date: Thu, 1 May 2025 21:35:25 +1000 Subject: [PATCH] doc: Add documentation for Operation Cache Controller and related components --- doc/arch/0-overview.md | 230 +++++++++++++++++++++++++ doc/arch/1-client.md | 44 +++++ doc/arch/2-requirement-controller.md | 51 ++++++ doc/arch/3-operation-controller.md | 78 +++++++++ doc/arch/4-appdeployment-controller.md | 130 ++++++++++++++ doc/arch/5-cache-controller.md | 99 +++++++++++ 6 files changed, 632 insertions(+) create mode 100644 doc/arch/0-overview.md create mode 100644 doc/arch/1-client.md create mode 100644 doc/arch/2-requirement-controller.md create mode 100644 doc/arch/3-operation-controller.md create mode 100644 doc/arch/4-appdeployment-controller.md create mode 100644 doc/arch/5-cache-controller.md diff --git a/doc/arch/0-overview.md b/doc/arch/0-overview.md new file mode 100644 index 0000000..3b40be1 --- /dev/null +++ b/doc/arch/0-overview.md @@ -0,0 +1,230 @@ +# Overview + +The Operation Cache controller is responsible for the following tasks: + +- Provisioning the resources for the user. +- Pre-Provsion the resources for future use(Cache). +- Managing the lifecycle of the resources and cache + +## The Interaction between User and Operation Cache Controller + +``` mermaid + +--- +title: "Operation Cache Controller Context Diagram" +--- +graph TD + +usr["🧑‍💻 User"] + +registry["📦 Image Registry +[Service]"] + +requirement["📄 Requirement Spec"] +deployctl["🦾 deployctl +[Binary]"] + +deploy-controller["⚙️ Operation Cache Controller Manager +[K8S Operator]"] + +deploy["🖥️ Resources +[Resource]"] + +deploy-cache["🖥️ Cached Resources +[Resource]"] + +usr -- "Acquire resource" --> deployctl +usr -- "Push provisioner image to" --> registry + +deployctl <-- "Create a spec of requested resources" --> requirement +deploy-controller <-- "reconcile" --> requirement +deployctl -- "Return a OperationID" --> usr + +deploy-controller -- "Provision/Teardown" ---> deploy +deploy-controller -- "Manage lifecycle of" ---> deploy-cache + +classDef focusSystem fill:#1168bd,stroke:#0b4884,color:#ffffff +classDef supportingSystem fill:#666,stroke:#0b4884,color:#ffffff +classDef user fill:#08427b,stroke:#052e56,color:#ffffff + +class usr user +class deploy-controller,deployctl,requirement focusSystem +class registry supportingSystem +``` + +## Interactions between Operation Cache Controller Components + +The detailed interaction of Operation Cache Controller components is as follows: + +- The Operation Cache controller comsumes Requirement CRD and return the Operation ID which is used to indicate the resources are provisioned. +- The Operation Reconciler will create the Cache CRD and Operation CRD based on spec in Operation CRD +- The Cache CRD is used to store the pre-provisioned resources. It will create Operation CRs to provision the resources. +- The AppDeployment CRD is used to create the Provision and Teardown Job that doing the actual provisioning and teardown of the resources. + +```mermaid +--- +title: "Operation Cache Controller Interaction" +--- + +graph TD +requirement-crd["📜 Requirement Spec"] +subgraph deploy-controller["Operation Cache Controller"] + operation-crd["📜 Operation Spec"] + app-deployment-crd["📜 AppDeployment Spec"] + cache-crd["📜 cache Spec"] + + requirement-rc["♻️ Requirement Reconciler + [K8S Controller]"] + operation-rc["♻️ Operation Reconciler + [K8S Controller]"] + cache-rc["♻️ Cache Reconciler + [K8S Controller]"] + app-deployment-rc["♻️ App Deployment Reconciler + [K8S Controller]"] + + pj["🧑‍🔧 Provison Job + [K8S Job]"] + + tj["🧑‍🔧 Teardown Job + [K8S Job]"] +end + +Resource["🖥️ Resources Provisioned +[Resource]"] + +requirement-crd <-. "Reconcile" .-> requirement-rc + +requirement-rc -- "Create" --> cache-crd +requirement-rc -- "Create" --> operation-crd + +cache-crd <-. "Reconcile" .-> cache-rc +cache-rc -- "Create" --> operation-crd + +operation-rc <-. "Reconcile" .-> operation-crd +operation-rc -- "Create/Delete" --> app-deployment-crd + +app-deployment-rc <-. "Reconcile" .-> app-deployment-crd +app-deployment-rc -- "Create" --> pj +app-deployment-rc -- "Create" --> tj + +pj -- "Provision" --> Resource +tj -- "Teardown" --> Resource + +classDef focusSystem fill:#1168bd,stroke:#0b4884,color:#ffffff +classDef supportingSystem fill:#666,stroke:#0b4884,color:#ffffff +classDef user fill:#08427b,stroke:#052e56,color:#ffffff +classDef boundary fill:none,stroke:#666,stroke-dasharray:5,color:#ffffff + +class deploy-controller boundary +class requirement-crd user +class operation-rc,app-deployment-rc,cache-rc,requirement-rc focusSystem +class cache-crd,operation-crd,app-deployment-crd,pj,tj supportingSystem +``` + +## The spec of CRDs that Operation Cache controller uses + +### Requirement + +```yaml +apiVersion: apps.devinfra.aks.goms.io/v1alpha1 +kind: Requirement +metadata: + name: my-requirement +spec: + applications: + - name: my-app-requirement-1 + image: my-image + arguments: ["arg1", "arg2"] + env: + - name: MY_ENV + value: my-value + - name: my-app-requirement-2 + image: my-image-2 + arguments: + - arg1 + - arg2 + env: + - name: MY_ENV + value: my-value-2 + dependencies: + - my-app-requirement-1 + cachable: false +``` + +### Operation + +```yaml +apiVersion: apps.devinfra.aks.goms.io/v1alpha1 +kind: Operation +metadata: + name: my-deploy-1 +spec: + applications: + - name: my-app-1 + spec: + image: my-image + arguments: ["arg1", "arg2"] + env: + - name: MY_ENV + value: my-value + - name: my-app-2 + spec: + image: my-image-2 + arguments: + - arg1 + - arg2 + env: + - name: MY_ENV + value: my-value-2 + dependencies: + - my-app-1 +``` + +### AppDeployment + +```yaml +kind: AppDeployment +metadata: + name: my-operation-id-my-app-2 +spec: + opID: my-operation-id + spec: + image: my-image + arguments: ["arg1", "arg2"] + env: + - name: MY_ENV + value: my-value + dependencies: + - my-app-1 +``` + +### Cache + +```yaml +kind: Cache +metadata: + name: my-cache-a31acb88 +spec: + applications: + - name: my-app-1 + spec: + image: my-image + arguments: ["arg1", "arg2"] + env: + - name: MY_ENV + value: my-value + - name: my-app-2 + spec: + image: my-image-2 + arguments: + - arg1 + - arg2 + env: + - name: MY_ENV + value: my-value-2 + dependencies: + - my-app-1 + CacheID: a31acb88138b21b131b331231131287 + CacheDuration: 2h + AutoCount: true +``` diff --git a/doc/arch/1-client.md b/doc/arch/1-client.md new file mode 100644 index 0000000..3e487c8 --- /dev/null +++ b/doc/arch/1-client.md @@ -0,0 +1,44 @@ +# deployctl + +deployctl is a command-line tool that interacts with the Operation Cache Controller Manager to create and manage resources in a Kubernetes cluster. + +## Create + +```shell +deployctl create -f +``` + +## Delete + +```shell +deployctl delete -o +``` + +## Update + +```shell +deployctl update -o -f +``` + +## Sequence Diagram interact with Operation Cache Controller Manager + +### Without cache + +``` mermaid +--- +title: "Operation Cache Controller Manager Sequence Diagram" +--- + +sequenceDiagram + autonumber + actor user + participant ctl as deployctl + participant k8s + + user ->>+ ctl: deployctl create -f + ctl ->>+ k8s: Create Requirement CR + k8s -->>- ctl: Requirement CR created successfully + ctl ->>+ k8s: check if the Requirement is ready + k8s -->>- ctl: Requirement is ready + ctl -->>- user: return DeployID +``` diff --git a/doc/arch/2-requirement-controller.md b/doc/arch/2-requirement-controller.md new file mode 100644 index 0000000..f69c997 --- /dev/null +++ b/doc/arch/2-requirement-controller.md @@ -0,0 +1,51 @@ +# Requirement Controller + +## Cache Hit + +``` mermaid +--- +title: "Operation Cache Controller Manager Sequence Diagram" +--- + +sequenceDiagram + autonumber + actor user + participant ctl as deployctl + participant k8s + + user ->>+ ctl: deployctl create -f + ctl ->>+ k8s: Check if cache CR for exist + k8s -->>- ctl: Cache CR exist + ctl ->>+ k8s: Check if available cache items exist + k8s -->>- ctl: Available cache items exist + ctl ->>+ k8s: select one cached deployment get deployid and set annotation to acquired + k8s -->>- ctl: Annotation for cached deployment is set successfully + ctl -->>- user: return DeployID +``` + +## Cache Miss + +``` mermaid + +--- +title: "Operation Cache Controller Manager Sequence Diagram" +--- + +sequenceDiagram + autonumber + actor user + participant ctl as deployctl + participant k8s + + user ->>+ ctl: deployctl create -f + ctl ->>+ k8s: Check if cache CR for exist + k8s -->>- ctl: Cache CR does not exist + ctl ->>+ k8s: Create Cache CR + k8s -->>- ctl: Cache CR created successfully + ctl ->>+ k8s: create Deployment CR + k8s -->>- ctl: Deployment CR created successfully + ctl ->>+ k8s: check if the Deployment is ready + k8s -->>- ctl: Deployment is ready + ctl -->>- user: return DeployID + +``` diff --git a/doc/arch/3-operation-controller.md b/doc/arch/3-operation-controller.md new file mode 100644 index 0000000..7a0e599 --- /dev/null +++ b/doc/arch/3-operation-controller.md @@ -0,0 +1,78 @@ +# Operation Controller + +## Reconcile Sequence Diagram + +``` mermaid +--- +title: "Operation Reconcile Sequence Diagram" +--- + +sequenceDiagram + participant user as User + participant k8s + participant oc as Operation Controller + participant adc as AppDeployment Controller + + user ->>+ k8s: Create Operation CR + k8s -->>- user: Operation CR created successfully + user ->>+ k8s: check if the Operation is ready + + oc ->> k8s: list & watch the Operation CR + k8s -->> oc: New Operation CR Added + oc ->>+ oc: reconciling the Operation + oc ->> oc: generate DeployID for the Operation + oc ->> k8s: Create all child AppDeployment CR + k8s -->> oc: All AppDeployment CR created successfully + adc ->> k8s: list & watch the AppDeployment CR + k8s -->> adc: New AppDeployment CR Added + adc ->>+ adc: reconciling the AppDeployment + adc ->> k8s: update the AppDeployment CR status to ready + k8s -->> adc: AppDeployment CR status updated successfully + adc -->>- adc: AppDeployment reconciled successfully + + oc -->>- oc: Operation reconciled successfully + oc ->> k8s: update the Operation CR status to ready + k8s -->> oc: Operation CR status updated successfully + k8s -->>- user: Operation is ready + +``` + +## Finalize Sequence Diagram + +``` mermaid + +--- +title: "Operation Finalize Sequence Diagram" +--- + +sequenceDiagram + participant user as User + participant k8s + participant oc as Operation Controller + participant adc as AppDeployment Controller + + alt User deleted + user ->>+ k8s: Delete Operation CR + k8s -->>- user: Operation CR deleted successfully + oc ->> k8s: list & watch the Operation CR + k8s -->> oc: New Operation CR Deleted + else Expired + oc ->> k8s: list & watch the Operation CR + k8s -->> oc: Operation CR Expired + end + oc ->>+ oc: Finalizing the Operation + oc ->> k8s: cascade delete all child AppDeployment CR + adc ->> k8s: list & watch the AppDeployment CR + k8s -->> adc: New AppDeployment CR Deleted + adc ->>+ adc: Finalizing the AppDeployment + adc -->> adc: AppDeployment finalized successfully + adc ->> k8s: Clean AppDeployment CR finalizer annotation + k8s -->> adc: AppDeployment finalizer annotation cleaned successfully + adc -->>- adc: AppDeployment deleted successfully + + k8s -->> oc: All AppDeployment CR deleted successfully + oc ->> k8s: Clean Operation finalizer annotation + k8s -->> oc: Operation finalizer annotation cleaned successfully + oc -->>- oc: Operation deleted successfully + +``` diff --git a/doc/arch/4-appdeployment-controller.md b/doc/arch/4-appdeployment-controller.md new file mode 100644 index 0000000..83e6f5f --- /dev/null +++ b/doc/arch/4-appdeployment-controller.md @@ -0,0 +1,130 @@ +# App Deployment + +## Provision Sequence Diagram + +``` mermaid + +sequenceDiagram + title: App Deployment Provision Sequence Diagram + participant user as User + participant k8s + participant adc as AppDeployment Controller + participant job as K8S Job Controller + participant reg as Image Registry + + user ->>+ k8s: Create AppDeployment CR + k8s -->>- user: AppDeployment CR created successfully + user ->+ k8s: check if the AppDeployment is ready + + adc ->> k8s: list & watch the AppDeployment CR + k8s -->> adc: New AppDeployment CR Added + + adc ->>+ adc: reconciling the AppDeployment + + adc ->> k8s: list all dependencies of the AppDeployment + k8s -->> adc: dependencies listed successfully + + adc ->> adc: check if all dependencies are ready + adc ->> k8s: create Provision Job CR + k8s -->> adc: Create Job CR successfully + adc ->> k8s: check if the Job CR is ready + job ->> k8s: list & watch the Job CR + k8s -->> job: New Job CR Added + job ->>+ job: reconciling the job + job ->>+ reg: pull the provisioner image + reg -->>- job: provisioner image pulled successfully + job ->> job: running the provisioning job + job ->> k8s: update the Job CR status to completed + k8s -->> job: Job CR status updated successfully + job -->>- job: Job reconciled successfully + k8s -->> adc: Job CR is ready + adc ->> k8s: update the AppDeployment CR status to ready + adc -->>- adc: AppDeployment reconciled successfully + k8s -->>- user: AppDeployment is ready + +``` + +## Teardown Sequence Diagram + +``` mermaid + +sequenceDiagram + title: "App Deployment Teardown Sequence Diagram" + participant user as User + participant k8s + participant adc as AppDeployment Controller + participant job as K8S Job Controller + participant reg as Image Registry + + user ->>+ k8s: Delete AppDeployment CR + k8s -->>- user: AppDeployment CR deleted successfully + user ->+ k8s: check if the AppDeployment is deleted + + adc ->> k8s: list & watch the AppDeployment CR + k8s -->> adc: AppDeployment CR deleted + + adc ->>+ adc: reconciling the AppDeployment + + adc ->> k8s: create Teardown Job CR + k8s -->> adc: Create Job CR successfully + adc ->> k8s: check if the Job CR is ready + job ->> k8s: list & watch the Job CR + k8s -->> job: New Job CR Added + job ->>+ job: reconciling the job + job ->>+ reg: pull the provisioner image + reg -->>- job: provisioner image pulled successfully + job ->> job: running the teardown job + job ->> k8s: update the Job CR status to completed + k8s -->> job: Job CR status updated successfully + job -->>- job: Job reconciled successfully + k8s -->> adc: Job CR is ready + adc ->> k8s: update the AppDeployment CR status to deleted + adc -->>- adc: AppDeployment reconciled successfully + k8s -->>- user: AppDeployment is deleted +``` + +## Spec for Job CR + +### Provsion Job + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: provision-envid-deployment-1 + namespace: +spec: + template: + spec: + containers: + - name: + image: + env: + - name: + value: + command: "create" + restartPolicy: Never + backoffLimit: 4 +``` + +### Teardown Job + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: teardown-envid-deployment-1 + namespace: +spec: + template: + spec: + containers: + - name: + image: + env: + - name: + value: + command: "delete" + restartPolicy: Never + backoffLimit: 4 +``` diff --git a/doc/arch/5-cache-controller.md b/doc/arch/5-cache-controller.md new file mode 100644 index 0000000..ad43112 --- /dev/null +++ b/doc/arch/5-cache-controller.md @@ -0,0 +1,99 @@ +# Cache Controller + +The cache controller is responsible for managing the cache resources. It watches the cache resources and updates the cache status based on the cache resource status. The cache controller is responsible for creating, updating, and deleting the cache resources. + +## Cache Controller Reconcile Sequence Diagram + +::: mermaid +sequenceDiagram + title: Create Cache Controller Sequence Diagram + participant user as User + participant k8s + participant cc as Cache Controller + participant oc as Operation Controller + participant adc as AppDeployment Controller + participant kcs as Keepalive Count Service + + user ->>+ k8s: Create Cache CR + k8s -->>- user: Cache CR created successfully + cc ->>+ k8s: List & Watch Cache CR + k8s -->> cc: New Cache CR Added + cc ->>+ cc: Reconciling the Cache + cc ->>+ k8s: Check if the Cache is ready + cc ->> kcs: Get the Keepalive Count of current Cache + kcs -->> cc: Keepalive Count is 3 + cc ->> k8s: list current available cached operations + k8s -->> cc: List of cached operations + alt Available Cache Count < Keepalive Count + cc ->>+ k8s: create (Keepalive Count - Available Cache Count) cached Operation CRs + k8s -->> cc: Cached Operation CRs created successfully + oc ->>+ k8s: list & watch the Operation CR + k8s -->> oc: New Operation CR Added + oc ->>+ oc: Reconciling the Operation + oc ->> k8s: Create AppDeployment CRs + adc ->>+ k8s: list & watch the AppDeployment CR + k8s -->> adc: New AppDeployment CR Added + adc ->>+ adc: Reconciling the AppDeployment + adc ->>+ k8s: Check if all AppDeployments are ready + k8s -->> adc: All AppDeployments are ready + adc -->>- adc: AppDeployment reconciled successfully + oc -->>- oc: Operation reconciled successfully + else Available Cache Count > Keepalive Count + cc ->>+ k8s: delete the (Available Cache Count - Keepalive Count) cached Operation CRs + oc ->>+ k8s: list & watch the Operation CR + k8s -->> oc: Operation CR Deleted + oc ->>+ oc: Finalizing the Operation + oc ->> k8s: cascade delete AppDeployment CRs + adc ->>+ k8s: list & watch the AppDeployment CR + k8s -->> adc: AppDeployment CR Deleted + adc ->>+ adc: Finalizing the AppDeployment + adc ->> k8s: Clean AppDeployment finalizer annotation + k8s -->> adc: AppDeployment finalizer annotation cleaned successfully + adc -->>- adc: AppDeployment finalized successfully + oc ->> k8s: Clean Operation finalizer annotation + k8s -->> oc: Operation finalizer annotation cleaned successfully + oc -->>- oc: Operation finalized successfully + end + cc -->>- cc: Cache reconciled successfully +::: + +## Cache Controller Finalize Sequence Diagram + +::: mermaid +sequenceDiagram + title: Delete Cache Controller Sequence Diagram + participant user as User + participant k8s + participant cc as Cache Controller + participant oc as Operation Controller + participant adc as AppDeployment Controller + + user ->>+ k8s: Delete Cache CR + cc ->>+ k8s: List & Watch Cache CR + k8s -->>- cc: New Deleted Cache CR + cc ->>+ cc: Finalizing the Cache + cc ->> k8s: list current child Operation CRs + k8s -->> cc: Return list of cached Operation CRs + opt if operation was acquired + cc ->>+ k8s: remove the ownership of the Operation CRs + k8s -->>- cc: Operation CRs ownership deleted successfully + end + cc ->>+ k8s: cascade delete the cached child Operation CRs + oc ->>+ k8s: list & watch the Operation CR + k8s -->>- oc: New Operation CR Deleted + oc ->>+ oc: Finalizing the Operation + oc ->> k8s: cascade delete AppDeployment CRs + adc ->>+ k8s: list & watch the AppDeployment CR + k8s -->> adc: AppDeployment CR Deleted + adc ->>+ adc: Finalizing the AppDeployment + adc ->> k8s: Clean AppDeployment finalizer annotation + k8s -->> adc: AppDeployment finalizer annotation cleaned successfully + adc -->>- adc: AppDeployment finalized successfully + oc ->> k8s: Clean Operation finalizer annotation + k8s -->> oc: Operation finalizer annotation cleaned successfully + oc -->>- oc: Operation finalized successfully + cc -->>- cc: Cache finalized successfully + k8s -->>- cc: Cached Operation CRs deleted successfully + + k8s -->>- user: Cache CR deleted successfully +:::