diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3a87272..f519f24 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,8 +55,9 @@ jobs: uses: ietf-tools/semver-action@v1.5.1 with: token: ${{ github.token }} + patchAll: true # fall back to dev because we want to have a valid semver - branch: ${{ fromJSON('{"main":"dev"}')[github.ref_name] || github.ref_name }} + branch: ${{ fromJSON('{"main":"main"}')[github.ref_name] || github.ref_name }} noVersionBumpBehavior: current - name: Set OPERATOR_VERSION @@ -198,7 +199,7 @@ jobs: with: token: ${{ github.token }} # calculate the changelog from the last tag to the current dev state - fromTag: ${{ fromJSON('{"main":"dev"}')[github.ref_name] || github.ref_name }} + fromTag: ${{ fromJSON('{"main":"main"}')[github.ref_name] || github.ref_name }} toTag: ${{ env.CURRENT }} # Create a new release on GitHub with the semantic OPERATOR_VERSION number - name: Create Release diff --git a/Makefile b/Makefile index 9645173..85e0ccf 100644 --- a/Makefile +++ b/Makefile @@ -86,11 +86,13 @@ help: ## Display this help. ##@ Development .PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. +manifests: controller-gen enum-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(ENUM_GEN) --marshal --ptr -f api/**/enums.go $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases .PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. +generate: controller-gen enum-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(ENUM_GEN) --marshal --ptr -f api/**/enums.go $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." .PHONY: fmt @@ -173,6 +175,8 @@ $(LOCALBIN): ## Tool Binaries KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENUM_GEN ?= $(LOCALBIN)/go-enum +ENUM_GEN_VERSION ?= latest ENVTEST ?= $(LOCALBIN)/setup-envtest ## Tool Versions @@ -190,6 +194,11 @@ controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessar $(CONTROLLER_GEN): $(LOCALBIN) test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) +.PHONY: enum-gen +enum-gen: $(ENUM_GEN) ## Download enum-gen locally if necessary. +$(ENUM_GEN): $(LOCALBIN) + test -s $(LOCALBIN)/go-enum || GOBIN=$(LOCALBIN) go install github.com/abice/go-enum@$(ENUM_GEN_VERSION) + .PHONY: envtest envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) diff --git a/README.md b/README.md index 6199560..7b5c97f 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,25 @@ ExecDAT is a tool to execute data analysis tasks on Kubernetes. It is designed t ## Getting Started -You’ll need a Kubernetes cluster to run against. See k3d for a quick way to get a local cluster up and running. -**Note:** Your controller will automatically use the current context in your kubeconfig file (i.e. whatever cluster `kubectl cluster-info` shows). +### Prerequisites + +* operator-sdk +* container engine (docker, podman, ...) +* kubernetes cluster (minikube, k3d, ...) + +### Test out the operator + +```shell +make install +make run +``` + +### Run using OLM + +```shell +operator-sdk olm install +operator-sdk run bundle-upgrade ghcr.io/austriandatalab/execdat-operator-bundle:v0.2.0 +``` ### Running on the cluster @@ -47,10 +64,6 @@ UnDeploy the controller to the cluster: make undeploy ``` -## Contributing - -// TODO(user): Add detailed information on how you would like others to contribute to this project - ### How it works This project aims to follow the Kubernetes [Operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) @@ -58,22 +71,6 @@ This project aims to follow the Kubernetes [Operator pattern](https://kubernetes It uses [Controllers](https://kubernetes.io/docs/concepts/architecture/controller/) which provides a reconcile function responsible for synchronizing resources untile the desired state is reached on the cluster -### Test It Out - -1. Install the CRDs into the cluster: - -```sh -make install -``` - -2. Run your controller (this will run in the foreground, so switch to a new terminal if you want to leave it running): - -```sh -make run -``` - -**NOTE:** You can also run this in one step by running: `make install run` - ### Modifying the API definitions If you are editing the API definitions, generate the manifests such as CRs or CRDs using: diff --git a/api/v1alpha1/build_types.go b/api/v1alpha1/build_types.go index 2da2688..e0bb950 100644 --- a/api/v1alpha1/build_types.go +++ b/api/v1alpha1/build_types.go @@ -36,8 +36,7 @@ type BuildSpec struct { // BuildStatus defines the observed state of Build type BuildStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file + CurrentPhase *CurrentPhase `json:"currentPhase"` //TODO: add fields of status and dataurls to kubectl get outputs } //+kubebuilder:object:root=true diff --git a/api/v1alpha1/builder.go b/api/v1alpha1/builder.go index 5b06719..bfafbd7 100644 --- a/api/v1alpha1/builder.go +++ b/api/v1alpha1/builder.go @@ -5,14 +5,15 @@ import ( "k8s.io/utils/pointer" ) -type PodSpecData struct { +type BuildPodSpecData struct { INIT_SH string Dockerfile string ImageName string + ImageTag string } // SetPodSpec sets the pod spec for the build -func (build *Build) SetPodSpec(podSpec *kcore.PodSpec, podSpecData PodSpecData) error { +func (build *Build) SetPodSpec(podSpec *kcore.PodSpec, buildPodSpecData BuildPodSpecData) error { podSpec.RestartPolicy = kcore.RestartPolicyNever @@ -27,22 +28,23 @@ func (build *Build) SetPodSpec(podSpec *kcore.PodSpec, podSpecData PodSpecData) } podSpec.Containers = []kcore.Container{ { - Name: "buildah", - Image: "quay.io/buildah/stable", - Command: []string{"/bin/bash", "-c", "--"}, - Args: []string{"trap : TERM INT; echo \"$INIT_SH\" | bash"}, + Name: "buildah", + Image: "quay.io/buildah/stable", + ImagePullPolicy: kcore.PullIfNotPresent, + Command: []string{"/bin/bash", "-c", "--"}, + Args: []string{"trap : TERM INT; echo \"$INIT_SH\" | bash"}, Env: []kcore.EnvVar{ - {Name: "INIT_SH", Value: podSpecData.INIT_SH}, - {Name: "DOCKERFILE", Value: podSpecData.Dockerfile}, + {Name: "INIT_SH", Value: buildPodSpecData.INIT_SH}, + {Name: "DOCKERFILE", Value: buildPodSpecData.Dockerfile}, {Name: "BASE_IMAGE", Value: build.Spec.BaseImage}, - {Name: "IMAGE_NAME", Value: podSpecData.ImageName}, - // {Name: "IMAGE_TAG", Value: build.Spec.ImageTag}, + {Name: "IMAGE_NAME", Value: buildPodSpecData.ImageName}, + {Name: "IMAGE_TAG", Value: buildPodSpecData.ImageTag}, // {Name: "IMAGE_REGISTRY", Value: build.Spec.ImageRegistry}, // {Name: "IMAGE_REGISTRY_USER", Value: build.Spec.ImageRegistryUser}, // {Name: "IMAGE_REGISTRY_PASSWORD", Value: build.Spec.ImageRegistryPassword}, // {Name: "IMAGE_REGISTRY_INSECURE", Value: build.Spec.ImageRegistryInsecure}, // {Name: "IMAGE_REGISTRY_VERIFY_TLS", Value: build.Spec.ImageRegistryVerifyTLS}, - {Name: "ENTRYPOINT", Value: build.Spec.SourceCode.Entrypoint}, + {Name: "ENTRYPOINT", Value: build.Spec.SourceCode.EntryPoint}, {Name: "GIT_REPO", Value: build.Spec.SourceCode.URL}, {Name: "GIT_BRANCH", Value: build.Spec.SourceCode.Branch}, {Name: "BUILD_CMD", Value: build.Spec.SourceCode.BuildCMD}, diff --git a/api/v1alpha1/enums.go b/api/v1alpha1/enums.go new file mode 100644 index 0000000..05e31c6 --- /dev/null +++ b/api/v1alpha1/enums.go @@ -0,0 +1,5 @@ +package v1alpha1 + +// swagger:enum CurrentPhase +// ENUM(pending, building, running, buildComplete, runCompleted, failed) +type CurrentPhase string diff --git a/api/v1alpha1/enums_enum.go b/api/v1alpha1/enums_enum.go new file mode 100644 index 0000000..76ff49b --- /dev/null +++ b/api/v1alpha1/enums_enum.go @@ -0,0 +1,77 @@ +// Code generated by go-enum DO NOT EDIT. +// Version: +// Revision: +// Build Date: +// Built By: + +package v1alpha1 + +import ( + "errors" + "fmt" +) + +const ( + // CurrentPhasePending is a CurrentPhase of type pending. + CurrentPhasePending CurrentPhase = "pending" + // CurrentPhaseBuilding is a CurrentPhase of type building. + CurrentPhaseBuilding CurrentPhase = "building" + // CurrentPhaseRunning is a CurrentPhase of type running. + CurrentPhaseRunning CurrentPhase = "running" + // CurrentPhaseBuildComplete is a CurrentPhase of type buildComplete. + CurrentPhaseBuildComplete CurrentPhase = "buildComplete" + // CurrentPhaseRunCompleted is a CurrentPhase of type runCompleted. + CurrentPhaseRunCompleted CurrentPhase = "runCompleted" + // CurrentPhaseFailed is a CurrentPhase of type failed. + CurrentPhaseFailed CurrentPhase = "failed" +) + +var ErrInvalidCurrentPhase = errors.New("not a valid CurrentPhase") + +// String implements the Stringer interface. +func (x CurrentPhase) String() string { + return string(x) +} + +// IsValid provides a quick way to determine if the typed value is +// part of the allowed enumerated values +func (x CurrentPhase) IsValid() bool { + _, err := ParseCurrentPhase(string(x)) + return err == nil +} + +var _CurrentPhaseValue = map[string]CurrentPhase{ + "pending": CurrentPhasePending, + "building": CurrentPhaseBuilding, + "running": CurrentPhaseRunning, + "buildComplete": CurrentPhaseBuildComplete, + "runCompleted": CurrentPhaseRunCompleted, + "failed": CurrentPhaseFailed, +} + +// ParseCurrentPhase attempts to convert a string to a CurrentPhase. +func ParseCurrentPhase(name string) (CurrentPhase, error) { + if x, ok := _CurrentPhaseValue[name]; ok { + return x, nil + } + return CurrentPhase(""), fmt.Errorf("%s is %w", name, ErrInvalidCurrentPhase) +} + +func (x CurrentPhase) Ptr() *CurrentPhase { + return &x +} + +// MarshalText implements the text marshaller method. +func (x CurrentPhase) MarshalText() ([]byte, error) { + return []byte(string(x)), nil +} + +// UnmarshalText implements the text unmarshaller method. +func (x *CurrentPhase) UnmarshalText(text []byte) error { + tmp, err := ParseCurrentPhase(string(text)) + if err != nil { + return err + } + *x = tmp + return nil +} diff --git a/api/v1alpha1/generic_types.go b/api/v1alpha1/generic_types.go index be5b83b..32259c9 100644 --- a/api/v1alpha1/generic_types.go +++ b/api/v1alpha1/generic_types.go @@ -24,17 +24,17 @@ type SourceCodeSpec struct { Dependencies DependenciesSpec `json:"dependencies,omitempty"` DependencyCMD string `json:"dependencycmd,omitempty"` BuildCMD string `json:"buildcmd,omitempty"` - Entrypoint string `json:"entrypoint"` + EntryPoint string `json:"entrypoint"` } type InputDataSpec struct { - URL string `json:"url" description:"URL of the data repo of input data"` - Type string `json:"type" description:"Type of the input data source, e.g. s3, git, http, https, etc."` DataPath string `json:"datapath" description:"Path to the data directory with input data, has to be a unix path."` + URL string `json:"url" description:"URL of the data repo of input data"` + Type string `json:"type,omitempty" description:"Type of the input data source, e.g. s3, git, http, https, etc."` TransformCMD string `json:"transformcmd,omitempty" description:"Command to transform the input data"` } type OutputDataSpec struct { - URL string `json:"url" description:"URL of the data repo of output data"` DataPath string `json:"datapath" description:"Path to the data directory with output data, has to be a unix path."` + URL string `json:"url,omitempty" description:"URL of the data repo of output data"` } diff --git a/api/v1alpha1/run_types.go b/api/v1alpha1/run_types.go index 4c26641..20ffad9 100644 --- a/api/v1alpha1/run_types.go +++ b/api/v1alpha1/run_types.go @@ -29,19 +29,17 @@ type RunSpec struct { // Important: Run "make" to regenerate code after modifying this file // Foo is an example field of Run. Edit run_types.go to remove/update - Build BuildSpec `json:"build"` - OutputData OutputDataSpec `json:"outputdata"` - //+optional - InputData InputDataSpec `json:"inputdata,omitempty"` - Description string `json:"description,omitempty"` + Build BuildSpec `json:"build"` + OutputData OutputDataSpec `json:"outputdata,omitempty"` + InputData InputDataSpec `json:"inputdata,omitempty"` + Description string `json:"description,omitempty"` } // RunStatus defines the observed state of Run type RunStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster // Important: Run "make" to regenerate code after modifying this file - CurrentPhase string `json:"currentPhase"` //TODO: add fields of status and dataurls to kubectl get outputs - Test string `json:"test"` + CurrentPhase *CurrentPhase `json:"currentPhase"` //TODO: add fields of status and dataurls to kubectl get outputs } //+kubebuilder:object:root=true diff --git a/api/v1alpha1/runner.go b/api/v1alpha1/runner.go new file mode 100644 index 0000000..91e0b60 --- /dev/null +++ b/api/v1alpha1/runner.go @@ -0,0 +1,68 @@ +package v1alpha1 + +import ( + kcore "k8s.io/api/core/v1" + "k8s.io/utils/pointer" +) + +type RunPodSpecData struct { + INIT_SH string + ImageName string + ImageTag string + InputDataPath string + OutputDataPath string +} + +// SetPodSpec sets the pod spec for the run +func (run *Run) SetPodSpec(podSpec *kcore.PodSpec, runPodSpecData RunPodSpecData) error { + + podSpec.RestartPolicy = kcore.RestartPolicyNever + + podSpec.Volumes = []kcore.Volume{ + { + Name: "input", + VolumeSource: kcore.VolumeSource{ + EmptyDir: &kcore.EmptyDirVolumeSource{}, + }, + }, + { + Name: "output", + VolumeSource: kcore.VolumeSource{ + EmptyDir: &kcore.EmptyDirVolumeSource{}, + }, + }, + } + + podSpec.Containers = []kcore.Container{ + { + Name: "buildah", + Image: "harbor.caas-0013.dev.austrianopencloudcommunity.org/execdev/" + runPodSpecData.ImageName + ":" + runPodSpecData.ImageTag, + ImagePullPolicy: kcore.PullIfNotPresent, + Command: []string{"/bin/bash", "-c", "--"}, + Args: []string{"trap : TERM INT; echo \"$INIT_SH\" | bash"}, + TTY: true, + Stdin: true, + Env: []kcore.EnvVar{ + {Name: "INIT_SH", Value: runPodSpecData.INIT_SH}, + {Name: "MINIO_ENDPOINT", Value: "http://minio.single-minio.svc.cluster.local:9000"}, + {Name: "MINIO_ACCESS_KEY", Value: "cache-user-1"}, + {Name: "MINIO_SECRET_KEY", Value: "CACHE_USER_PASS_XYZ_123"}, + {Name: "BUCKET_NAME", Value: "cache-bucket-1"}, + {Name: "HOME", Value: "/tmp"}, + }, + SecurityContext: &kcore.SecurityContext{ + RunAsUser: pointer.Int64(1000), + RunAsGroup: pointer.Int64(1000), + }, + VolumeMounts: []kcore.VolumeMount{ + {Name: "input", MountPath: runPodSpecData.InputDataPath}, + {Name: "output", MountPath: runPodSpecData.OutputDataPath}, + }, + }, + } + podSpec.SecurityContext = &kcore.PodSecurityContext{ + RunAsUser: pointer.Int64(1000), + RunAsGroup: pointer.Int64(1000), + } + return nil +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 0fd71e2..5b37027 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -31,7 +31,7 @@ func (in *Build) DeepCopyInto(out *Build) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Build. @@ -84,6 +84,21 @@ func (in *BuildList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BuildPodSpecData) DeepCopyInto(out *BuildPodSpecData) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BuildPodSpecData. +func (in *BuildPodSpecData) DeepCopy() *BuildPodSpecData { + if in == nil { + return nil + } + out := new(BuildPodSpecData) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BuildSpec) DeepCopyInto(out *BuildSpec) { *out = *in @@ -103,6 +118,11 @@ func (in *BuildSpec) DeepCopy() *BuildSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BuildStatus) DeepCopyInto(out *BuildStatus) { *out = *in + if in.CurrentPhase != nil { + in, out := &in.CurrentPhase, &out.CurrentPhase + *out = new(CurrentPhase) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BuildStatus. @@ -220,28 +240,13 @@ func (in *OutputDataSpec) DeepCopy() *OutputDataSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PodSpecData) DeepCopyInto(out *PodSpecData) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodSpecData. -func (in *PodSpecData) DeepCopy() *PodSpecData { - if in == nil { - return nil - } - out := new(PodSpecData) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Run) DeepCopyInto(out *Run) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Run. @@ -294,6 +299,21 @@ func (in *RunList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RunPodSpecData) DeepCopyInto(out *RunPodSpecData) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunPodSpecData. +func (in *RunPodSpecData) DeepCopy() *RunPodSpecData { + if in == nil { + return nil + } + out := new(RunPodSpecData) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RunSpec) DeepCopyInto(out *RunSpec) { *out = *in @@ -315,6 +335,11 @@ func (in *RunSpec) DeepCopy() *RunSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RunStatus) DeepCopyInto(out *RunStatus) { *out = *in + if in.CurrentPhase != nil { + in, out := &in.CurrentPhase, &out.CurrentPhase + *out = new(CurrentPhase) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunStatus. diff --git a/config/crd/bases/task.execd.at_builds.yaml b/config/crd/bases/task.execd.at_builds.yaml index ed589e0..f9d30dc 100644 --- a/config/crd/bases/task.execd.at_builds.yaml +++ b/config/crd/bases/task.execd.at_builds.yaml @@ -175,6 +175,13 @@ spec: type: object status: description: BuildStatus defines the observed state of Build + properties: + currentPhase: + description: swagger:enum CurrentPhase ENUM(pending, building, running, + buildComplete, runCompleted, failed) + type: string + required: + - currentPhase type: object type: object served: true diff --git a/config/crd/bases/task.execd.at_runs.yaml b/config/crd/bases/task.execd.at_runs.yaml index eddaf32..7233714 100644 --- a/config/crd/bases/task.execd.at_runs.yaml +++ b/config/crd/bases/task.execd.at_runs.yaml @@ -191,7 +191,6 @@ spec: type: string required: - datapath - - type - url type: object outputdata: @@ -202,11 +201,9 @@ spec: type: string required: - datapath - - url type: object required: - build - - outputdata type: object status: description: RunStatus defines the observed state of Run diff --git a/config/samples/task_v1alpha1_run.yaml b/config/samples/task_v1alpha1_run.yaml index c74383d..36c1eb6 100644 --- a/config/samples/task_v1alpha1_run.yaml +++ b/config/samples/task_v1alpha1_run.yaml @@ -9,4 +9,20 @@ metadata: app.kubernetes.io/created-by: execdat-operator name: run-sample spec: - # TODO(user): Add fields here + description: "Test hunde demo 2" + build: + baseimage: "ubuntu:jammy" + description: "default image for demos" + sourcecode: + url: "https://github.com/AustrianDataLAB/execDAT" + branch: "feature/example" + dependencies: + os: + - name: curl + version: latest + entrypoint: examples/entrypoint.sh + outputdata: + datapath: /output + inputdata: + url: https://www.wien.gv.at/finanzen/ogd/hunde-wien.csv + datapath: /input diff --git a/controllers/build_controller.go b/controllers/build_controller.go index 6aa5754..25e1930 100644 --- a/controllers/build_controller.go +++ b/controllers/build_controller.go @@ -18,6 +18,9 @@ package controllers import ( "context" + "fmt" + "strings" + "time" //kapps "k8s.io/api/apps/v1" @@ -64,17 +67,16 @@ func (r *BuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{}, client.IgnoreNotFound(err) } + if build.Status.CurrentPhase != nil && *build.Status.CurrentPhase == taskv1alpha1.CurrentPhaseBuildComplete { + log.V(1).Info("Build already completed", "build", req.NamespacedName) + return ctrl.Result{}, nil + } + var resourceName string = build.Name var resourceNamespace string = build.Namespace - scriptTemplates := []string{"./templates/init.sh.tmpl"} - templateData := lib.InitTemplateData{ - BaseImage: build.Spec.BaseImage, - // GitRepo: build.Spec.SourceCode.URL, - // GitBranch: build.Spec.SourceCode.Branch, - // BuildCmd: build.Spec.SourceCode.BuildCMD, - } - init_sh, err := lib.CreateTemplate(scriptTemplates, templateData) + scriptTemplates := []string{"./templates/init_build.sh.tmpl"} + init_sh, err := lib.CreateTemplate(scriptTemplates, build.Spec) if err != nil { return ctrl.Result{}, err } @@ -85,19 +87,20 @@ func (r *BuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{}, err } - newPodSpecData := taskv1alpha1.PodSpecData{ + newBuildPodSpecData := taskv1alpha1.BuildPodSpecData{ INIT_SH: init_sh, Dockerfile: dockerfile, - ImageName: build.ObjectMeta.Name, + ImageName: build.ObjectMeta.Labels["runRef"], + ImageTag: strings.Split(build.ObjectMeta.Name, build.ObjectMeta.GenerateName)[1], } podSpec := &kcore.PodSpec{} - if err := build.SetPodSpec(podSpec, newPodSpecData); err != nil { + if err := build.SetPodSpec(podSpec, newBuildPodSpecData); err != nil { return ctrl.Result{}, err } job := &kbatch.Job{} - job.GenerateName = resourceName + "-" + job.Name = resourceName job.ObjectMeta.Namespace = resourceNamespace job.Spec.TTLSecondsAfterFinished = pointer.Int32(60) job.Spec.Template.Spec = *podSpec @@ -110,9 +113,33 @@ func (r *BuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl err = r.Get(ctx, types.NamespacedName{Name: resourceName, Namespace: resourceNamespace}, foundJob) if err != nil && errors.IsNotFound(err) { log.V(1).Info("Creating Job", "job", job.Name) + //TODO create the Job in a separate namespace err = r.Create(ctx, job) } else if err == nil { + log.V(1).Info("Job already created", "job", job.Name) + if build.Status.CurrentPhase == nil { + build.Status.CurrentPhase = taskv1alpha1.CurrentPhaseBuilding.Ptr() + if err := r.Status().Update(ctx, build); err != nil { + return ctrl.Result{RequeueAfter: time.Second * 10}, fmt.Errorf("failed to update build status: %w", err) + } + } + + // Check if the Job has completed + jobComplete, err := CheckJobCompletion(job, r.Client) + if err != nil { + log.Error(err, "Failed to check Job completion") + return ctrl.Result{}, err + } + if jobComplete { + log.V(1).Info("Job has completed", "job", job.Name) + build.Status.CurrentPhase = taskv1alpha1.CurrentPhaseBuildComplete.Ptr() + if err := r.Status().Update(ctx, build); err != nil { + return ctrl.Result{RequeueAfter: time.Second * 10}, fmt.Errorf("failed to update build status: %w", err) + } + return ctrl.Result{}, nil + } + return ctrl.Result{RequeueAfter: time.Second * 10}, nil } return ctrl.Result{}, err @@ -122,5 +149,6 @@ func (r *BuildReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl func (r *BuildReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&taskv1alpha1.Build{}). + Owns(&kbatch.Job{}). Complete(r) } diff --git a/controllers/controller.go b/controllers/controller.go new file mode 100644 index 0000000..a92b992 --- /dev/null +++ b/controllers/controller.go @@ -0,0 +1,27 @@ +package controllers + +import ( + "context" + "fmt" + + kbatch "k8s.io/api/batch/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// CheckJobCompletion checks if a Job has completed +func CheckJobCompletion(job *kbatch.Job, r client.Client) (bool, error) { + ctx := context.Background() + + err := r.Get(ctx, client.ObjectKeyFromObject(job), job) + if err != nil { + return false, fmt.Errorf("failed to get job: %w", err) + } + + if job.Status.CompletionTime != nil { + // Job has completed + return true, nil + } + + // Job is still running, wait and check again + return false, nil +} diff --git a/controllers/run_controller.go b/controllers/run_controller.go index 30460a5..a5263c9 100644 --- a/controllers/run_controller.go +++ b/controllers/run_controller.go @@ -18,13 +18,23 @@ package controllers import ( "context" + "fmt" + "strings" + "time" + kbatch "k8s.io/api/batch/v1" + kcore "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + pointer "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" taskv1alpha1 "github.com/AustrianDataLAB/execDAT-operator/api/v1alpha1" + lib "github.com/AustrianDataLAB/execDAT-operator/lib" ) // RunReconciler reconciles a Run object @@ -47,9 +57,143 @@ type RunReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) + logger := log.FromContext(ctx) + log := logger.WithValues("taskv1alpha1.Run", req.NamespacedName) - // TODO(user): your logic here + run := &taskv1alpha1.Run{} + if err := r.Get(ctx, req.NamespacedName, run); err != nil { + log.V(1).Info("unable to fetch run", "run", req.NamespacedName) + // we'll ignore not-found errors, since they can't be fixed by an immediate + // requeue (we'll need to wait for a new notification), and we can get them + // on deleted requests. + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + log.V(1).Info("reconciling run", "run", run) + var resourceName string = run.Name + var resourceNamespace string = run.Namespace + + build := &taskv1alpha1.Build{} + build.GenerateName = resourceName + "-" + build.ObjectMeta.Namespace = resourceNamespace + // Define labels to filter Jobs + labels := map[string]string{ + "runRef": run.Name, + "runGeneration": fmt.Sprint(run.ObjectMeta.Generation), + } + build.ObjectMeta.Labels = labels + build.Spec = run.Spec.Build + + if err := controllerutil.SetControllerReference(run, build, r.Scheme); err != nil { + return ctrl.Result{}, err + } + + foundBuildList := &taskv1alpha1.BuildList{} + err := r.List(ctx, + foundBuildList, + client.InNamespace(resourceNamespace), + client.MatchingLabels(labels), + ) + if err == nil && len(foundBuildList.Items) <= 0 { + log.V(1).Info("Creating Build", "build", build.Name) + if err := r.Create(ctx, build); err != nil { + log.Error(err, "Failed to create new Build", "build", build.Name) + return ctrl.Result{}, err + } + run.Status.CurrentPhase = taskv1alpha1.CurrentPhaseBuilding.Ptr() + if err := r.Status().Update(ctx, run); err != nil { + log.Error(err, "Failed to update Run status") + return ctrl.Result{}, err + } + } else if len(foundBuildList.Items) > 0 { + + log.V(1).Info("Build already created", "build", build.Name) + + currentBuild := foundBuildList.Items[0] + + if currentBuild.Status.CurrentPhase == nil { + return ctrl.Result{}, nil + } + + switch *currentBuild.Status.CurrentPhase { + case taskv1alpha1.CurrentPhaseBuildComplete: + log.V(1).Info("Build completed", "run", build.Name) + //TODO: create RUN + scriptTemplates := []string{"./templates/init_run.sh.tmpl"} + init_sh, err := lib.CreateTemplate(scriptTemplates, run.Spec) + if err != nil { + return ctrl.Result{}, err + } + + newRunPodSpecData := taskv1alpha1.RunPodSpecData{ + INIT_SH: init_sh, + ImageName: currentBuild.ObjectMeta.Labels["runRef"], + ImageTag: strings.Split(currentBuild.ObjectMeta.Name, currentBuild.ObjectMeta.GenerateName)[1], + InputDataPath: run.Spec.InputData.DataPath, + OutputDataPath: run.Spec.OutputData.DataPath, + } + + podSpec := &kcore.PodSpec{} + if err := run.SetPodSpec(podSpec, newRunPodSpecData); err != nil { + return ctrl.Result{}, err + } + + job := &kbatch.Job{} + job.Name = resourceName + job.ObjectMeta.Namespace = resourceNamespace + job.Spec.TTLSecondsAfterFinished = pointer.Int32(60) + job.Spec.Template.Spec = *podSpec + + if err := controllerutil.SetControllerReference(run, job, r.Scheme); err != nil { + return ctrl.Result{}, err + } + + foundJob := &kbatch.Job{} + err = r.Get(ctx, types.NamespacedName{Name: resourceName, Namespace: resourceNamespace}, foundJob) + if err != nil && errors.IsNotFound(err) && *run.Status.CurrentPhase != taskv1alpha1.CurrentPhaseRunCompleted { + + log.V(1).Info("Creating Run Job", "job", job.Name) + //TODO create the Job in a separate namespace + if err := r.Create(ctx, job); err != nil { + log.Error(err, "Failed to create new Run Job", "job", job.Name) + return ctrl.Result{}, err + } + + run.Status.CurrentPhase = taskv1alpha1.CurrentPhaseRunning.Ptr() + if err := r.Status().Update(ctx, run); err != nil { + log.Error(err, "Failed to update Run status") + return ctrl.Result{}, err + } + + } else if err == nil { + log.V(1).Info("Run Job already created", "job", job.Name) + + // Check if the Job has completed + jobComplete, err := CheckJobCompletion(job, r.Client) + if err != nil { + log.Error(err, "Failed to check Job completion") + return ctrl.Result{}, err + } + if jobComplete { + log.V(1).Info("Job has completed", "job", job.Name) + run.Status.CurrentPhase = taskv1alpha1.CurrentPhaseRunCompleted.Ptr() + if err := r.Status().Update(ctx, run); err != nil { + return ctrl.Result{RequeueAfter: time.Second * 10}, fmt.Errorf("failed to update run status: %w", err) + } + return ctrl.Result{}, nil + } + return ctrl.Result{RequeueAfter: time.Second * 10}, nil + } + + case taskv1alpha1.CurrentPhaseFailed: + log.V(1).Info("Build failed", "build", build.Name) + run.Status.CurrentPhase = taskv1alpha1.CurrentPhaseFailed.Ptr() + if err := r.Status().Update(ctx, run); err != nil { + log.Error(err, "Failed to update Run status") + return ctrl.Result{}, err + } + } + } return ctrl.Result{}, nil } @@ -58,5 +202,7 @@ func (r *RunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R func (r *RunReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&taskv1alpha1.Run{}). + Owns(&taskv1alpha1.Build{}). + Owns(&kbatch.Job{}). Complete(r) } diff --git a/go.mod b/go.mod index 68076fc..452986a 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,13 @@ require ( ) require ( + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.2.0 // indirect + github.com/Masterminds/sprig/v3 v3.2.3 // indirect + github.com/abice/go-enum v0.5.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect @@ -27,29 +32,47 @@ require ( github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect + github.com/huandu/xstrings v1.3.3 // indirect + github.com/imdario/mergo v0.3.13 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/labstack/gommon v0.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/goveralls v0.0.11 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.15.1 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shopspring/decimal v1.2.0 // indirect + github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.5.0 // indirect + github.com/stretchr/testify v1.8.2 // indirect + github.com/urfave/cli/v2 v2.25.1 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect + golang.org/x/crypto v0.3.0 // indirect + golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect golang.org/x/sys v0.8.0 // indirect @@ -57,6 +80,7 @@ require ( golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.1 // indirect + golang.org/x/tools/cmd/cover v0.1.0-deprecated // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.30.0 // indirect diff --git a/go.sum b/go.sum index 0a8001c..94f0cdb 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,13 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/abice/go-enum v0.5.6 h1:Ury51IQXUppbIl56MqRU/++A8SSeLG4plePphPjxW1s= +github.com/abice/go-enum v0.5.6/go.mod h1:X2GpCT8VkCXLkVm48hebWx3cVgFJ8zM5nY5iUrJZO1Q= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -11,6 +19,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -25,6 +35,7 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -44,6 +55,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -71,16 +84,24 @@ github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -91,10 +112,26 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= +github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/goveralls v0.0.11 h1:eJXea6R6IFlL1QMKNMzDvvHv/hwGrnvyig4N+0+XiMM= +github.com/mattn/goveralls v0.0.11/go.mod h1:gU8SyhNswsJKchEV93xRQxX6X3Ei4PJdQk/6ZHvrvRk= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -102,6 +139,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss= github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= @@ -121,12 +159,20 @@ github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -135,9 +181,18 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/urfave/cli/v2 v2.25.1 h1:zw8dSP7ghX0Gmm8vugrs6q9Ku0wzweqPyshy+syu9Gw= +github.com/urfave/cli/v2 v2.25.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -149,6 +204,9 @@ go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -157,7 +215,9 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -167,7 +227,10 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -180,6 +243,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -188,15 +252,27 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= @@ -209,9 +285,13 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools/cmd/cover v0.1.0-deprecated h1:Rwy+mWYz6loAF+LnG1jHG/JWMHRMMC2/1XX3Ejkx9lA= +golang.org/x/tools/cmd/cover v0.1.0-deprecated/go.mod h1:hMDiIvlpN1NoVgmjLjUJE9tMHyxHjFX7RuQ+rW12mSA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -250,10 +330,13 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/lib/template.go b/lib/template.go index fc03a72..534bd9c 100644 --- a/lib/template.go +++ b/lib/template.go @@ -7,11 +7,7 @@ import ( taskv1alpha1 "github.com/AustrianDataLAB/execDAT-operator/api/v1alpha1" ) -type InitTemplateData struct { - BaseImage string -} - -func CreateTemplate[D InitTemplateData | taskv1alpha1.BuildSpec](templatePaths []string, data D) (string, error) { +func CreateTemplate[D taskv1alpha1.BuildSpec | taskv1alpha1.RunSpec](templatePaths []string, data D) (string, error) { tmpl, err := template.ParseFiles(templatePaths...) tmpl = template.Must(tmpl, err) if err != nil { diff --git a/templates/Dockerfile.tmpl b/templates/Dockerfile.tmpl index ca9c08c..3c22f9a 100644 --- a/templates/Dockerfile.tmpl +++ b/templates/Dockerfile.tmpl @@ -1,15 +1,27 @@ +FROM docker://minio/mc:RELEASE.2023-06-23T18-12-07Z as mc FROM {{ .BaseImage }} as base -ENV DEBIAN_FRONTEND=noninteractive +COPY --from=mc /usr/bin/mc /usr/bin/mc +RUN chmod +x /usr/bin/mc +ENV PATH="/usr/bin:${PATH}" +WORKDIR /execdat -RUN apt update -y && apt upgrade -y +ENV DEBIAN_FRONTEND=noninteractive -{{ range .SourceCode.Dependencies.OS }} -RUN apt install -y {{ .Name -}} -{{ end }} +RUN apt update -y && apt install -y \ + git \ + curl \ + wget \ +{{ range .SourceCode.Dependencies.OS -}} + {{ .Name }} \ +{{ end -}} + && rm -rf /var/lib/apt/lists/* {{ range .SourceCode.Dependencies.Pip }} RUN pip3 install {{ .Name -}} {{ end }} -ENTRYPOINT ["{{ .SourceCode.Entrypoint }}"] +ARG CACHE_DATE=2023-01-01 +RUN git clone --depth 1 -b {{ .SourceCode.Branch }} --single-branch {{ .SourceCode.URL }} . + +ENTRYPOINT ["{{ .SourceCode.EntryPoint }}"] diff --git a/templates/init.sh.tmpl b/templates/init.sh.tmpl deleted file mode 100644 index 71718cd..0000000 --- a/templates/init.sh.tmpl +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -e - -cd $HOME - -echo "TODO CLONE SOURCE" - -echo "Building with buildah upstream image" -echo "Building from baseimage $BASE_IMAGE" -echo "Using registry $REGISTRY_BASE" -echo "Using image name $IMAGE_NAME" -echo "$DOCKERFILE" > Dockerfile -cat Dockerfile -buildah bud -t $IMAGE_NAME -f Dockerfile . -buildah push --creds $REGISTRY_USER:$REGISTRY_KEY $IMAGE_NAME docker://$REGISTRY_BASE/$IMAGE_NAME diff --git a/templates/init_build.sh.tmpl b/templates/init_build.sh.tmpl new file mode 100644 index 0000000..66b6f78 --- /dev/null +++ b/templates/init_build.sh.tmpl @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +cd $HOME + +echo "TODO CLONE SOURCE" + +echo "Building with buildah upstream image" +echo "Building from baseimage $BASE_IMAGE" +echo "Using registry $REGISTRY_BASE" +echo "Using image name $IMAGE_NAME" +echo "$DOCKERFILE" > Dockerfile +cat Dockerfile +buildah login -u $REGISTRY_USER -p $REGISTRY_KEY $REGISTRY_BASE +buildah bud --build-arg CACHE_DATE="$(date)" --layers --retry=1 --retry-delay=1s --cache-to $REGISTRY_BASE/$IMAGE_NAME/cache --cache-from $REGISTRY_BASE/$IMAGE_NAME/cache -t $IMAGE_NAME:$IMAGE_TAG -f Dockerfile . +buildah tag $IMAGE_NAME:$IMAGE_TAG $REGISTRY_BASE/$IMAGE_NAME:$IMAGE_TAG +buildah push --all $REGISTRY_BASE/$IMAGE_NAME:$IMAGE_TAG docker://$REGISTRY_BASE/$IMAGE_NAME:$IMAGE_TAG diff --git a/templates/init_run.sh.tmpl b/templates/init_run.sh.tmpl new file mode 100644 index 0000000..34ff53b --- /dev/null +++ b/templates/init_run.sh.tmpl @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +wget -P {{ .InputData.DataPath }} {{ .InputData.URL }} + +{{ .Build.SourceCode.EntryPoint }} + +echo "Pushing results to Minio bucket" + +# Upload the results to the Minio bucket +mc alias set minio $MINIO_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY +mc cp {{ .OutputData.DataPath }} minio/$BUCKET_NAME/ --recursive + +echo "Run process completed"