From 5832296ef56574bb61dd3ecfeeda7caa352489f7 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 3 Feb 2022 23:47:42 +0100 Subject: [PATCH 1/9] Document Bucket API v1beta2 spec Signed-off-by: Hidde Beydals --- api/v1beta2/artifact_types.go | 29 +- api/v1beta2/bucket_types.go | 77 +- api/v1beta2/condition_types.go | 36 +- api/v1beta2/source.go | 19 +- .../source.toolkit.fluxcd.io_buckets.yaml | 71 +- ...rce.toolkit.fluxcd.io_gitrepositories.yaml | 49 +- .../source.toolkit.fluxcd.io_helmcharts.yaml | 22 +- ...ce.toolkit.fluxcd.io_helmrepositories.yaml | 22 +- controllers/bucket_controller.go | 89 +- controllers/bucket_controller_test.go | 2 +- docs/api/source.md | 95 ++- docs/spec/README.md | 1 + docs/spec/v1beta2/README.md | 20 + docs/spec/v1beta2/buckets.md | 780 ++++++++++++++++++ 14 files changed, 1083 insertions(+), 229 deletions(-) create mode 100644 docs/spec/v1beta2/README.md create mode 100644 docs/spec/v1beta2/buckets.md diff --git a/api/v1beta2/artifact_types.go b/api/v1beta2/artifact_types.go index 64829b6ba..4bd1bfede 100644 --- a/api/v1beta2/artifact_types.go +++ b/api/v1beta2/artifact_types.go @@ -23,32 +23,31 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// Artifact represents the output of a Source synchronisation. +// Artifact represents the output of a Source reconciliation. type Artifact struct { - // Path is the relative file path of this Artifact. - // It can be used to locate the Artifact file in the root of the Artifact - // storage on the local file system of the controller managing the Source. + // Path is the relative file path of the Artifact. It can be used to locate + // the file in the root of the Artifact storage on the local file system of + // the controller managing the Source. // +required Path string `json:"path"` - // URL is the HTTP address of this artifact. - // It is used by the consumers of the artifacts to fetch and use the - // artifacts. It is expected to be resolvable from within the cluster. + // URL is the HTTP address of the Artifact as exposed by the controller + // managing the Source. It can be used to retrieve the Artifact for + // consumption, e.g. by another controller applying the Artifact contents. // +required URL string `json:"url"` - // Revision is a human readable identifier traceable in the origin source - // system. It can be a Git commit SHA, Git tag, a Helm index timestamp, a Helm - // chart version, etc. + // Revision is a human-readable identifier traceable in the origin source + // system. It can be a Git commit SHA, Git tag, a Helm chart version, etc. // +optional Revision string `json:"revision"` - // Checksum is the SHA256 checksum of the artifact. + // Checksum is the SHA256 checksum of the Artifact file. // +optional Checksum string `json:"checksum"` - // LastUpdateTime is the timestamp corresponding to the last update of this - // artifact. + // LastUpdateTime is the timestamp corresponding to the last update of the + // Artifact. // +required LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` @@ -67,14 +66,14 @@ func (in *Artifact) HasRevision(revision string) bool { } // ArtifactDir returns the artifact dir path in the form of -// //. +// '//'. func ArtifactDir(kind, namespace, name string) string { kind = strings.ToLower(kind) return path.Join(kind, namespace, name) } // ArtifactPath returns the artifact path in the form of -// ///. +// '//name>/'. func ArtifactPath(kind, namespace, name, filename string) string { return path.Join(ArtifactDir(kind, namespace, name), filename) } diff --git a/api/v1beta2/bucket_types.go b/api/v1beta2/bucket_types.go index 3cccef13a..2ea66e465 100644 --- a/api/v1beta2/bucket_types.go +++ b/api/v1beta2/bucket_types.go @@ -31,46 +31,57 @@ const ( ) const ( + // GenericBucketProvider for any S3 API compatible storage Bucket. GenericBucketProvider string = "generic" - AmazonBucketProvider string = "aws" - GoogleBucketProvider string = "gcp" - AzureBucketProvider string = "azure" + // AmazonBucketProvider for an AWS S3 object storage Bucket. + // Provides support for retrieving credentials from the AWS EC2 service. + AmazonBucketProvider string = "aws" + // GoogleBucketProvider for a Google Cloud Storage Bucket. + // Provides support for authentication using a workload identity. + GoogleBucketProvider string = "gcp" + // AzureBucketProvider for an Azure Blob Storage Bucket. + // Provides support for authentication using a Service Principal, + // Managed Identity or Shared Key. + AzureBucketProvider string = "azure" ) -// BucketSpec defines the desired state of an S3 compatible bucket +// BucketSpec specifies the required configuration to produce an Artifact for +// an object storage bucket. type BucketSpec struct { - // The S3 compatible storage provider name, default ('generic'). + // Provider of the object storage bucket. + // Defaults to 'generic', which expects an S3 (API) compatible object + // storage. // +kubebuilder:validation:Enum=generic;aws;gcp;azure // +kubebuilder:default:=generic // +optional Provider string `json:"provider,omitempty"` - // The bucket name. + // BucketName is the name of the object storage bucket. // +required BucketName string `json:"bucketName"` - // The bucket endpoint address. + // Endpoint is the object storage address the BucketName is located at. // +required Endpoint string `json:"endpoint"` - // Insecure allows connecting to a non-TLS S3 HTTP endpoint. + // Insecure allows connecting to a non-TLS HTTP Endpoint. // +optional Insecure bool `json:"insecure,omitempty"` - // The bucket region. + // Region of the Endpoint where the BucketName is located in. // +optional Region string `json:"region,omitempty"` - // The name of the secret containing authentication credentials + // SecretRef specifies the Secret containing authentication credentials // for the Bucket. // +optional SecretRef *meta.LocalObjectReference `json:"secretRef,omitempty"` - // The interval at which to check for bucket updates. + // Interval at which to check the Endpoint for updates. // +required Interval metav1.Duration `json:"interval"` - // The timeout for fetch operations, defaults to 60s. + // Timeout for fetch operations, defaults to 60s. // +kubebuilder:default="60s" // +optional Timeout *metav1.Duration `json:"timeout,omitempty"` @@ -81,18 +92,21 @@ type BucketSpec struct { // +optional Ignore *string `json:"ignore,omitempty"` - // This flag tells the controller to suspend the reconciliation of this source. + // Suspend tells the controller to suspend the reconciliation of this + // Bucket. // +optional Suspend bool `json:"suspend,omitempty"` - // AccessFrom defines an Access Control List for allowing cross-namespace references to this object. + // AccessFrom specifies an Access Control List for allowing cross-namespace + // references to this object. + // NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092 // +optional AccessFrom *acl.AccessFrom `json:"accessFrom,omitempty"` } -// BucketStatus defines the observed state of a bucket +// BucketStatus records the observed state of a Bucket. type BucketStatus struct { - // ObservedGeneration is the last observed generation. + // ObservedGeneration is the last observed generation of the Bucket object. // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` @@ -100,11 +114,13 @@ type BucketStatus struct { // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` - // URL is the fetch link for the artifact output of the last Bucket sync. + // URL is the dynamic fetch link for the latest Artifact. + // It is provided on a "best effort" basis, and using the precise + // BucketStatus.Artifact data is recommended. // +optional URL string `json:"url,omitempty"` - // Artifact represents the output of the last successful Bucket sync. + // Artifact represents the last successful Bucket reconciliation. // +optional Artifact *Artifact `json:"artifact,omitempty"` @@ -112,12 +128,12 @@ type BucketStatus struct { } const ( - // BucketOperationSucceededReason represents the fact that the bucket listing and - // fetch operations succeeded. + // BucketOperationSucceededReason signals that the Bucket listing and fetch + // operations succeeded. BucketOperationSucceededReason string = "BucketOperationSucceeded" - // BucketOperationFailedReason represents the fact that the bucket listing or - // fetch operations failed. + // BucketOperationFailedReason signals that the Bucket listing or fetch + // operations failed. BucketOperationFailedReason string = "BucketOperationFailed" ) @@ -136,23 +152,11 @@ func (in Bucket) GetRequeueAfter() time.Duration { return in.Spec.Interval.Duration } -// GetInterval returns the interval at which the source is reconciled. -// Deprecated: use GetRequeueAfter instead. -func (in Bucket) GetInterval() metav1.Duration { - return in.Spec.Interval -} - // GetArtifact returns the latest artifact from the source if present in the status sub-resource. func (in *Bucket) GetArtifact() *Artifact { return in.Status.Artifact } -// GetStatusConditions returns a pointer to the Status.Conditions slice. -// Deprecated: use GetConditions instead. -func (in *Bucket) GetStatusConditions() *[]metav1.Condition { - return &in.Status.Conditions -} - // +genclient // +genclient:Namespaced // +kubebuilder:storageversion @@ -163,7 +167,7 @@ func (in *Bucket) GetStatusConditions() *[]metav1.Condition { // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="" // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="" -// Bucket is the Schema for the buckets API +// Bucket is the Schema for the buckets API. type Bucket struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -173,9 +177,8 @@ type Bucket struct { Status BucketStatus `json:"status,omitempty"` } +// BucketList contains a list of Bucket objects. // +kubebuilder:object:root=true - -// BucketList contains a list of Bucket type BucketList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/api/v1beta2/condition_types.go b/api/v1beta2/condition_types.go index 1e6ff992d..1c68c621c 100644 --- a/api/v1beta2/condition_types.go +++ b/api/v1beta2/condition_types.go @@ -19,33 +19,41 @@ package v1beta2 const SourceFinalizer = "finalizers.fluxcd.io" const ( - // ArtifactOutdatedCondition indicates the current Artifact of the Source is outdated. - // This is a "negative polarity" or "abnormal-true" type, and is only present on the resource if it is True. + // ArtifactOutdatedCondition indicates the current Artifact of the Source + // is outdated. + // This is a "negative polarity" or "abnormal-true" type, and is only + // present on the resource if it is True. ArtifactOutdatedCondition string = "ArtifactOutdated" - // SourceVerifiedCondition indicates the integrity of the Source has been verified. If True, the integrity check - // succeeded. If False, it failed. The Condition is only present on the resource if the integrity has been verified. + // SourceVerifiedCondition indicates the integrity of the Source has been + // verified. If True, the integrity check succeeded. If False, it failed. + // The Condition is only present on the resource if the integrity has been + // verified. SourceVerifiedCondition string = "SourceVerified" - // FetchFailedCondition indicates a transient or persistent fetch failure of an upstream Source. - // If True, observations on the upstream Source revision may be impossible, and the Artifact available for the - // Source may be outdated. - // This is a "negative polarity" or "abnormal-true" type, and is only present on the resource if it is True. + // FetchFailedCondition indicates a transient or persistent fetch failure + // of an upstream Source. + // If True, observations on the upstream Source revision may be impossible, + // and the Artifact available for the Source may be outdated. + // This is a "negative polarity" or "abnormal-true" type, and is only + // present on the resource if it is True. FetchFailedCondition string = "FetchFailed" - // BuildFailedCondition indicates a transient or persistent build failure of a Source's Artifact. - // If True, the Source can be in an ArtifactOutdatedCondition + // BuildFailedCondition indicates a transient or persistent build failure + // of a Source's Artifact. + // If True, the Source can be in an ArtifactOutdatedCondition. BuildFailedCondition string = "BuildFailed" ) const ( - // URLInvalidReason represents the fact that a given source has an invalid URL. + // URLInvalidReason signals that a given Source has an invalid URL. URLInvalidReason string = "URLInvalid" - // StorageOperationFailedReason signals a failure caused by a storage operation. + // StorageOperationFailedReason signals a failure caused by a storage + // operation. StorageOperationFailedReason string = "StorageOperationFailed" - // AuthenticationFailedReason represents the fact that a given secret does not - // have the required fields or the provided credentials do not match. + // AuthenticationFailedReason signals that a Secret does not have the + // required fields, or the provided credentials do not match. AuthenticationFailedReason string = "AuthenticationFailed" ) diff --git a/api/v1beta2/source.go b/api/v1beta2/source.go index a8db640d9..76e2cc21e 100644 --- a/api/v1beta2/source.go +++ b/api/v1beta2/source.go @@ -19,26 +19,27 @@ package v1beta2 import ( "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) const ( - // SourceIndexKey is the key used for indexing resources - // resources based on their Source. + // SourceIndexKey is the key used for indexing objects based on their + // referenced Source. SourceIndexKey string = ".metadata.source" ) // Source interface must be supported by all API types. +// Source is the interface that provides generic access to the Artifact and +// interval. It must be supported by all kinds of the source.toolkit.fluxcd.io +// API group. +// // +k8s:deepcopy-gen=false type Source interface { runtime.Object - // GetRequeueAfter returns the duration after which the source must be reconciled again. + // GetRequeueAfter returns the duration after which the source must be + // reconciled again. GetRequeueAfter() time.Duration - // GetArtifact returns the latest artifact from the source if present in the - // status sub-resource. + // GetArtifact returns the latest artifact from the source if present in + // the status sub-resource. GetArtifact() *Artifact - // GetInterval returns the interval at which the source is updated. - // Deprecated: use GetRequeueAfter instead. - GetInterval() metav1.Duration } diff --git a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml index 113c6ab76..762e67931 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml @@ -266,7 +266,7 @@ spec: name: v1beta2 schema: openAPIV3Schema: - description: Bucket is the Schema for the buckets API + description: Bucket is the Schema for the buckets API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -281,12 +281,13 @@ spec: metadata: type: object spec: - description: BucketSpec defines the desired state of an S3 compatible - bucket + description: BucketSpec specifies the required configuration to produce + an Artifact for an object storage bucket. properties: accessFrom: - description: AccessFrom defines an Access Control List for allowing - cross-namespace references to this object. + description: 'AccessFrom specifies an Access Control List for allowing + cross-namespace references to this object. NOTE: Not implemented, + provisional as of https://github.com/fluxcd/flux2/pull/2092' properties: namespaceSelectors: description: NamespaceSelectors is the list of namespace selectors @@ -312,10 +313,11 @@ spec: - namespaceSelectors type: object bucketName: - description: The bucket name. + description: BucketName is the name of the object storage bucket. type: string endpoint: - description: The bucket endpoint address. + description: Endpoint is the object storage address the BucketName + is located at. type: string ignore: description: Ignore overrides the set of excluded patterns in the @@ -324,14 +326,15 @@ spec: to find out what those are. type: string insecure: - description: Insecure allows connecting to a non-TLS S3 HTTP endpoint. + description: Insecure allows connecting to a non-TLS HTTP Endpoint. type: boolean interval: - description: The interval at which to check for bucket updates. + description: Interval at which to check the Endpoint for updates. type: string provider: default: generic - description: The S3 compatible storage provider name, default ('generic'). + description: Provider of the object storage bucket. Defaults to 'generic', + which expects an S3 (API) compatible object storage. enum: - generic - aws @@ -339,11 +342,12 @@ spec: - azure type: string region: - description: The bucket region. + description: Region of the Endpoint where the BucketName is located + in. type: string secretRef: - description: The name of the secret containing authentication credentials - for the Bucket. + description: SecretRef specifies the Secret containing authentication + credentials for the Bucket. properties: name: description: Name of the referent. @@ -352,12 +356,12 @@ spec: - name type: object suspend: - description: This flag tells the controller to suspend the reconciliation - of this source. + description: Suspend tells the controller to suspend the reconciliation + of this Bucket. type: boolean timeout: default: 60s - description: The timeout for fetch operations, defaults to 60s. + description: Timeout for fetch operations, defaults to 60s. type: string required: - bucketName @@ -367,39 +371,38 @@ spec: status: default: observedGeneration: -1 - description: BucketStatus defines the observed state of a bucket + description: BucketStatus records the observed state of a Bucket. properties: artifact: - description: Artifact represents the output of the last successful - Bucket sync. + description: Artifact represents the last successful Bucket reconciliation. properties: checksum: - description: Checksum is the SHA256 checksum of the artifact. + description: Checksum is the SHA256 checksum of the Artifact file. type: string lastUpdateTime: description: LastUpdateTime is the timestamp corresponding to - the last update of this artifact. + the last update of the Artifact. format: date-time type: string path: - description: Path is the relative file path of this Artifact. - It can be used to locate the Artifact file in the root of the - Artifact storage on the local file system of the controller - managing the Source. + description: Path is the relative file path of the Artifact. It + can be used to locate the file in the root of the Artifact storage + on the local file system of the controller managing the Source. type: string revision: - description: Revision is a human readable identifier traceable + description: Revision is a human-readable identifier traceable in the origin source system. It can be a Git commit SHA, Git - tag, a Helm index timestamp, a Helm chart version, etc. + tag, a Helm chart version, etc. type: string size: description: Size is the number of bytes in the file. format: int64 type: integer url: - description: URL is the HTTP address of this artifact. It is used - by the consumers of the artifacts to fetch and use the artifacts. - It is expected to be resolvable from within the cluster. + description: URL is the HTTP address of the Artifact as exposed + by the controller managing the Source. It can be used to retrieve + the Artifact for consumption, e.g. by another controller applying + the Artifact contents. type: string required: - path @@ -481,12 +484,14 @@ spec: be detected. type: string observedGeneration: - description: ObservedGeneration is the last observed generation. + description: ObservedGeneration is the last observed generation of + the Bucket object. format: int64 type: integer url: - description: URL is the fetch link for the artifact output of the - last Bucket sync. + description: URL is the dynamic fetch link for the latest Artifact. + It is provided on a "best effort" basis, and using the precise BucketStatus.Artifact + data is recommended. type: string type: object type: object diff --git a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml index 7d445f7cb..113b62b1b 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml @@ -541,32 +541,32 @@ spec: repository sync. properties: checksum: - description: Checksum is the SHA256 checksum of the artifact. + description: Checksum is the SHA256 checksum of the Artifact file. type: string lastUpdateTime: description: LastUpdateTime is the timestamp corresponding to - the last update of this artifact. + the last update of the Artifact. format: date-time type: string path: - description: Path is the relative file path of this Artifact. - It can be used to locate the Artifact file in the root of the - Artifact storage on the local file system of the controller - managing the Source. + description: Path is the relative file path of the Artifact. It + can be used to locate the file in the root of the Artifact storage + on the local file system of the controller managing the Source. type: string revision: - description: Revision is a human readable identifier traceable + description: Revision is a human-readable identifier traceable in the origin source system. It can be a Git commit SHA, Git - tag, a Helm index timestamp, a Helm chart version, etc. + tag, a Helm chart version, etc. type: string size: description: Size is the number of bytes in the file. format: int64 type: integer url: - description: URL is the HTTP address of this artifact. It is used - by the consumers of the artifacts to fetch and use the artifacts. - It is expected to be resolvable from within the cluster. + description: URL is the HTTP address of the Artifact as exposed + by the controller managing the Source. It can be used to retrieve + the Artifact for consumption, e.g. by another controller applying + the Artifact contents. type: string required: - path @@ -646,36 +646,37 @@ spec: description: IncludedArtifacts represents the included artifacts from the last successful repository sync. items: - description: Artifact represents the output of a Source synchronisation. + description: Artifact represents the output of a Source reconciliation. properties: checksum: - description: Checksum is the SHA256 checksum of the artifact. + description: Checksum is the SHA256 checksum of the Artifact + file. type: string lastUpdateTime: description: LastUpdateTime is the timestamp corresponding to - the last update of this artifact. + the last update of the Artifact. format: date-time type: string path: - description: Path is the relative file path of this Artifact. - It can be used to locate the Artifact file in the root of - the Artifact storage on the local file system of the controller - managing the Source. + description: Path is the relative file path of the Artifact. + It can be used to locate the file in the root of the Artifact + storage on the local file system of the controller managing + the Source. type: string revision: - description: Revision is a human readable identifier traceable + description: Revision is a human-readable identifier traceable in the origin source system. It can be a Git commit SHA, Git - tag, a Helm index timestamp, a Helm chart version, etc. + tag, a Helm chart version, etc. type: string size: description: Size is the number of bytes in the file. format: int64 type: integer url: - description: URL is the HTTP address of this artifact. It is - used by the consumers of the artifacts to fetch and use the - artifacts. It is expected to be resolvable from within the - cluster. + description: URL is the HTTP address of the Artifact as exposed + by the controller managing the Source. It can be used to retrieve + the Artifact for consumption, e.g. by another controller applying + the Artifact contents. type: string required: - path diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml index 75b6bfee6..819549f55 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml @@ -420,32 +420,32 @@ spec: chart sync. properties: checksum: - description: Checksum is the SHA256 checksum of the artifact. + description: Checksum is the SHA256 checksum of the Artifact file. type: string lastUpdateTime: description: LastUpdateTime is the timestamp corresponding to - the last update of this artifact. + the last update of the Artifact. format: date-time type: string path: - description: Path is the relative file path of this Artifact. - It can be used to locate the Artifact file in the root of the - Artifact storage on the local file system of the controller - managing the Source. + description: Path is the relative file path of the Artifact. It + can be used to locate the file in the root of the Artifact storage + on the local file system of the controller managing the Source. type: string revision: - description: Revision is a human readable identifier traceable + description: Revision is a human-readable identifier traceable in the origin source system. It can be a Git commit SHA, Git - tag, a Helm index timestamp, a Helm chart version, etc. + tag, a Helm chart version, etc. type: string size: description: Size is the number of bytes in the file. format: int64 type: integer url: - description: URL is the HTTP address of this artifact. It is used - by the consumers of the artifacts to fetch and use the artifacts. - It is expected to be resolvable from within the cluster. + description: URL is the HTTP address of the Artifact as exposed + by the controller managing the Source. It can be used to retrieve + the Artifact for consumption, e.g. by another controller applying + the Artifact contents. type: string required: - path diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml index cd687f6bb..a2c6d9195 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml @@ -346,32 +346,32 @@ spec: repository sync. properties: checksum: - description: Checksum is the SHA256 checksum of the artifact. + description: Checksum is the SHA256 checksum of the Artifact file. type: string lastUpdateTime: description: LastUpdateTime is the timestamp corresponding to - the last update of this artifact. + the last update of the Artifact. format: date-time type: string path: - description: Path is the relative file path of this Artifact. - It can be used to locate the Artifact file in the root of the - Artifact storage on the local file system of the controller - managing the Source. + description: Path is the relative file path of the Artifact. It + can be used to locate the file in the root of the Artifact storage + on the local file system of the controller managing the Source. type: string revision: - description: Revision is a human readable identifier traceable + description: Revision is a human-readable identifier traceable in the origin source system. It can be a Git commit SHA, Git - tag, a Helm index timestamp, a Helm chart version, etc. + tag, a Helm chart version, etc. type: string size: description: Size is the number of bytes in the file. format: int64 type: integer url: - description: URL is the HTTP address of this artifact. It is used - by the consumers of the artifacts to fetch and use the artifacts. - It is expected to be resolvable from within the cluster. + description: URL is the HTTP address of the Artifact as exposed + by the controller managing the Source. It can be used to retrieve + the Artifact for consumption, e.g. by another controller applying + the Artifact contents. type: string required: - path diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 278722e37..a05c04eff 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -69,9 +69,9 @@ import ( // -> s > 100 const maxConcurrentBucketFetches = 100 -// bucketReadyConditions contains all the conditions information needed -// for Bucket Ready status conditions summary calculation. -var bucketReadyConditions = summarize.Conditions{ +// bucketReadyCondition contains the information required to summarize a +// v1beta2.Bucket Ready Condition. +var bucketReadyCondition = summarize.Conditions{ Target: meta.ReadyCondition, Owned: []string{ sourcev1.ArtifactOutdatedCondition, @@ -99,7 +99,7 @@ var bucketReadyConditions = summarize.Conditions{ // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets/finalizers,verbs=get;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch -// BucketReconciler reconciles a Bucket object +// BucketReconciler reconciles a v1beta2.Bucket object. type BucketReconciler struct { client.Client kuberecorder.EventRecorder @@ -135,9 +135,10 @@ type BucketProvider interface { Close(context.Context) } -// bucketReconcilerFunc is the function type for all the bucket reconciler -// functions. -type bucketReconcilerFunc func(ctx context.Context, obj *sourcev1.Bucket, index *etagIndex, dir string) (sreconcile.Result, error) +// bucketReconcileFunc is the function type for all the v1beta2.Bucket +// (sub)reconcile functions. The type implementations are grouped and +// executed serially to perform the complete reconcile of the object. +type bucketReconcileFunc func(ctx context.Context, obj *sourcev1.Bucket, index *etagIndex, dir string) (sreconcile.Result, error) // etagIndex is an index of storage object keys and their Etag values. type etagIndex struct { @@ -260,7 +261,7 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res defer func() { summarizeHelper := summarize.NewHelper(r.EventRecorder, patchHelper) summarizeOpts := []summarize.Option{ - summarize.WithConditions(bucketReadyConditions), + summarize.WithConditions(bucketReadyCondition), summarize.WithReconcileResult(recResult), summarize.WithReconcileError(retErr), summarize.WithIgnoreNotFound(), @@ -268,7 +269,7 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res summarize.RecordContextualError, summarize.RecordReconcileReq, ), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetInterval().Duration}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), summarize.WithPatchFieldOwner(r.ControllerName), } result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) @@ -292,7 +293,7 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res } // Reconcile actual object - reconcilers := []bucketReconcilerFunc{ + reconcilers := []bucketReconcileFunc{ r.reconcileStorage, r.reconcileSource, r.reconcileArtifact, @@ -301,10 +302,10 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res return } -// reconcile steps iterates through the actual reconciliation tasks for objec, -// it returns early on the first step that returns ResultRequeue or produces an -// error. -func (r *BucketReconciler) reconcile(ctx context.Context, obj *sourcev1.Bucket, reconcilers []bucketReconcilerFunc) (sreconcile.Result, error) { +// reconcile iterates through the gitRepositoryReconcileFunc tasks for the +// object. It returns early on the first call that returns +// reconcile.ResultRequeue, or produces an error. +func (r *BucketReconciler) reconcile(ctx context.Context, obj *sourcev1.Bucket, reconcilers []bucketReconcileFunc) (sreconcile.Result, error) { if obj.Generation != obj.Status.ObservedGeneration { conditions.MarkReconciling(obj, "NewGeneration", "reconciling new object generation (%d)", obj.Generation) } @@ -317,7 +318,11 @@ func (r *BucketReconciler) reconcile(ctx context.Context, obj *sourcev1.Bucket, Reason: sourcev1.StorageOperationFailedReason, } } - defer os.RemoveAll(tmpDir) + defer func() { + if err = os.RemoveAll(tmpDir); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "failed to remove temporary working directory") + } + }() // Run the sub-reconcilers and build the result of reconciliation. var ( @@ -345,11 +350,17 @@ func (r *BucketReconciler) reconcile(ctx context.Context, obj *sourcev1.Bucket, return res, resErr } -// reconcileStorage ensures the current state of the storage matches the desired and previously observed state. +// reconcileStorage ensures the current state of the storage matches the +// desired and previously observed state. // -// All artifacts for the resource except for the current one are garbage collected from the storage. -// If the artifact in the Status object of the resource disappeared from storage, it is removed from the object. -// If the hostname of the URLs on the object do not match the current storage server hostname, they are updated. +// All Artifacts for the object except for the current one in the Status are +// garbage collected from the Storage. +// If the Artifact in the Status of the object disappeared from the Storage, +// it is removed from the object. +// If the object does not have an Artifact in its Status, a Reconciling +// condition is added. +// The hostname of any URL in the Status of the object are updated, to ensure +// they match the Storage server hostname of current runtime. func (r *BucketReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.Bucket, _ *etagIndex, _ string) (sreconcile.Result, error) { // Garbage collect previous advertised artifact(s) from storage _ = r.garbageCollect(ctx, obj) @@ -374,10 +385,11 @@ func (r *BucketReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.B return sreconcile.ResultSuccess, nil } -// reconcileSource reconciles the upstream bucket with the client for the given object's Provider, and returns the -// result. -// If a SecretRef is defined, it attempts to fetch the Secret before calling the provider. If the fetch of the Secret -// fails, it records v1beta1.FetchFailedCondition=True and returns early. +// reconcileSource fetches the upstream bucket contents with the client for the +// given object's Provider, and returns the result. +// When a SecretRef is defined, it attempts to fetch the Secret before calling +// the provider. If this fails, it records v1beta2.FetchFailedCondition=True on +// the object and returns early. func (r *BucketReconciler) reconcileSource(ctx context.Context, obj *sourcev1.Bucket, index *etagIndex, dir string) (sreconcile.Result, error) { secret, err := r.getBucketSecret(ctx, obj) if err != nil { @@ -470,13 +482,15 @@ func (r *BucketReconciler) reconcileSource(ctx context.Context, obj *sourcev1.Bu return sreconcile.ResultSuccess, nil } -// reconcileArtifact archives a new artifact to the storage, if the current observation on the object does not match the -// given data. +// reconcileArtifact archives a new Artifact to the Storage, if the current +// (Status) data on the object does not match the given. // -// The inspection of the given data to the object is differed, ensuring any stale observations as -// If the given artifact does not differ from the object's current, it returns early. -// On a successful archive, the artifact in the status of the given object is set, and the symlink in the storage is -// updated to its path. +// The inspection of the given data to the object is differed, ensuring any +// stale observations like v1beta2.ArtifactOutdatedCondition are removed. +// If the given Artifact does not differ from the object's current, it returns +// early. +// On a successful archive, the Artifact in the Status of the object is set, +// and the symlink in the Storage is updated to its path. func (r *BucketReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.Bucket, index *etagIndex, dir string) (sreconcile.Result, error) { // Calculate revision revision, err := index.Revision() @@ -561,8 +575,9 @@ func (r *BucketReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1. return sreconcile.ResultSuccess, nil } -// reconcileDelete handles the deletion of an object. It first garbage collects all artifacts for the object from the -// artifact storage, if successful, the finalizer is removed from the object. +// reconcileDelete handles the deletion of the object. +// It first garbage collects all Artifacts for the object from the Storage. +// Removing the finalizer from the object if successful. func (r *BucketReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.Bucket) (sreconcile.Result, error) { // Garbage collect the resource's artifacts if err := r.garbageCollect(ctx, obj); err != nil { @@ -577,9 +592,11 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.Bu return sreconcile.ResultEmpty, nil } -// garbageCollect performs a garbage collection for the given v1beta1.Bucket. It removes all but the current -// artifact except for when the deletion timestamp is set, which will result in the removal of all artifacts for the -// resource. +// garbageCollect performs a garbage collection for the given object. +// +// It removes all but the current Artifact from the Storage, unless the +// deletion timestamp on the object is set. Which will result in the +// removal of all Artifacts for the objects. func (r *BucketReconciler) garbageCollect(ctx context.Context, obj *sourcev1.Bucket) error { if !obj.DeletionTimestamp.IsZero() { if deleted, err := r.Storage.RemoveAll(r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), "", "*")); err != nil { @@ -626,6 +643,10 @@ func (r *BucketReconciler) getBucketSecret(ctx context.Context, obj *sourcev1.Bu } // eventLogf records event and logs at the same time. +// +// This log is different from the debug log in the EventRecorder, in the sense +// that this is a simple log. While the debug log contains complete details +// about the event. func (r *BucketReconciler) eventLogf(ctx context.Context, obj runtime.Object, eventType string, reason string, messageFmt string, args ...interface{}) { r.annotatedEventLogf(ctx, obj, nil, eventType, reason, messageFmt, args...) } diff --git a/controllers/bucket_controller_test.go b/controllers/bucket_controller_test.go index 060b6e12c..8f783e629 100644 --- a/controllers/bucket_controller_test.go +++ b/controllers/bucket_controller_test.go @@ -124,7 +124,7 @@ func TestBucketReconciler_Reconcile(t *testing.T) { }, timeout).Should(BeTrue()) // Check if the object status is valid. - condns := &status.Conditions{NegativePolarity: bucketReadyConditions.NegativePolarity} + condns := &status.Conditions{NegativePolarity: bucketReadyCondition.NegativePolarity} checker := status.NewChecker(testEnv.Client, testEnv.GetScheme(), condns) checker.CheckErr(ctx, obj) diff --git a/docs/api/source.md b/docs/api/source.md index 83392ee9b..129d061bc 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -19,7 +19,7 @@ Resource Types:

Bucket

-

Bucket is the Schema for the buckets API

+

Bucket is the Schema for the buckets API.

@@ -83,7 +83,9 @@ string @@ -94,7 +96,7 @@ string @@ -105,7 +107,7 @@ string @@ -117,7 +119,7 @@ bool @@ -129,7 +131,7 @@ string @@ -143,7 +145,7 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference @@ -157,7 +159,7 @@ Kubernetes meta/v1.Duration @@ -171,7 +173,7 @@ Kubernetes meta/v1.Duration @@ -197,7 +199,8 @@ bool @@ -211,7 +214,9 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom
(Optional) -

The S3 compatible storage provider name, default (‘generic’).

+

Provider of the object storage bucket. +Defaults to ‘generic’, which expects an S3 (API) compatible object +storage.

-

The bucket name.

+

BucketName is the name of the object storage bucket.

-

The bucket endpoint address.

+

Endpoint is the object storage address the BucketName is located at.

(Optional) -

Insecure allows connecting to a non-TLS S3 HTTP endpoint.

+

Insecure allows connecting to a non-TLS HTTP Endpoint.

(Optional) -

The bucket region.

+

Region of the Endpoint where the BucketName is located in.

(Optional) -

The name of the secret containing authentication credentials +

SecretRef specifies the Secret containing authentication credentials for the Bucket.

-

The interval at which to check for bucket updates.

+

Interval at which to check the Endpoint for updates.

(Optional) -

The timeout for fetch operations, defaults to 60s.

+

Timeout for fetch operations, defaults to 60s.

(Optional) -

This flag tells the controller to suspend the reconciliation of this source.

+

Suspend tells the controller to suspend the reconciliation of this +Bucket.

(Optional) -

AccessFrom defines an Access Control List for allowing cross-namespace references to this object.

+

AccessFrom specifies an Access Control List for allowing cross-namespace +references to this object. +NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092

@@ -854,7 +859,7 @@ HelmRepositoryStatus HelmChartStatus, HelmRepositoryStatus)

-

Artifact represents the output of a Source synchronisation.

+

Artifact represents the output of a Source reconciliation.

@@ -873,9 +878,9 @@ string @@ -886,9 +891,9 @@ string @@ -900,9 +905,8 @@ string @@ -914,7 +918,7 @@ string @@ -927,8 +931,8 @@ Kubernetes meta/v1.Time @@ -953,7 +957,8 @@ int64 (Appears on:Bucket)

-

BucketSpec defines the desired state of an S3 compatible bucket

+

BucketSpec specifies the required configuration to produce an Artifact for +an object storage bucket.

-

Path is the relative file path of this Artifact. -It can be used to locate the Artifact file in the root of the Artifact -storage on the local file system of the controller managing the Source.

+

Path is the relative file path of the Artifact. It can be used to locate +the file in the root of the Artifact storage on the local file system of +the controller managing the Source.

-

URL is the HTTP address of this artifact. -It is used by the consumers of the artifacts to fetch and use the -artifacts. It is expected to be resolvable from within the cluster.

+

URL is the HTTP address of the Artifact as exposed by the controller +managing the Source. It can be used to retrieve the Artifact for +consumption, e.g. by another controller applying the Artifact contents.

(Optional) -

Revision is a human readable identifier traceable in the origin source -system. It can be a Git commit SHA, Git tag, a Helm index timestamp, a Helm -chart version, etc.

+

Revision is a human-readable identifier traceable in the origin source +system. It can be a Git commit SHA, Git tag, a Helm chart version, etc.

(Optional) -

Checksum is the SHA256 checksum of the artifact.

+

Checksum is the SHA256 checksum of the Artifact file.

-

LastUpdateTime is the timestamp corresponding to the last update of this -artifact.

+

LastUpdateTime is the timestamp corresponding to the last update of the +Artifact.

@@ -973,7 +978,9 @@ string @@ -984,7 +991,7 @@ string @@ -995,7 +1002,7 @@ string @@ -1007,7 +1014,7 @@ bool @@ -1019,7 +1026,7 @@ string @@ -1033,7 +1040,7 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference @@ -1047,7 +1054,7 @@ Kubernetes meta/v1.Duration @@ -1061,7 +1068,7 @@ Kubernetes meta/v1.Duration @@ -1087,7 +1094,8 @@ bool @@ -1101,7 +1109,9 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom @@ -1114,7 +1124,7 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom (Appears on:Bucket)

-

BucketStatus defines the observed state of a bucket

+

BucketStatus records the observed state of a Bucket.

(Optional) -

The S3 compatible storage provider name, default (‘generic’).

+

Provider of the object storage bucket. +Defaults to ‘generic’, which expects an S3 (API) compatible object +storage.

-

The bucket name.

+

BucketName is the name of the object storage bucket.

-

The bucket endpoint address.

+

Endpoint is the object storage address the BucketName is located at.

(Optional) -

Insecure allows connecting to a non-TLS S3 HTTP endpoint.

+

Insecure allows connecting to a non-TLS HTTP Endpoint.

(Optional) -

The bucket region.

+

Region of the Endpoint where the BucketName is located in.

(Optional) -

The name of the secret containing authentication credentials +

SecretRef specifies the Secret containing authentication credentials for the Bucket.

-

The interval at which to check for bucket updates.

+

Interval at which to check the Endpoint for updates.

(Optional) -

The timeout for fetch operations, defaults to 60s.

+

Timeout for fetch operations, defaults to 60s.

(Optional) -

This flag tells the controller to suspend the reconciliation of this source.

+

Suspend tells the controller to suspend the reconciliation of this +Bucket.

(Optional) -

AccessFrom defines an Access Control List for allowing cross-namespace references to this object.

+

AccessFrom specifies an Access Control List for allowing cross-namespace +references to this object. +NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092

@@ -1134,7 +1144,7 @@ int64 @@ -1160,7 +1170,9 @@ string @@ -1174,7 +1186,7 @@ Artifact @@ -2178,7 +2190,10 @@ string

Source

-

Source interface must be supported by all API types.

+

Source interface must be supported by all API types. +Source is the interface that provides generic access to the Artifact and +interval. It must be supported by all kinds of the source.toolkit.fluxcd.io +API group.

This page was automatically generated with gen-crd-api-reference-docs

diff --git a/docs/spec/README.md b/docs/spec/README.md index 16ca15fd5..4af0eb04b 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -20,6 +20,7 @@ of the components using them. ## API Specification +* [v1beta2](v1beta2/README.md) * [v1beta1](v1beta1/README.md) ## Implementation diff --git a/docs/spec/v1beta2/README.md b/docs/spec/v1beta2/README.md new file mode 100644 index 000000000..bdb9b1adb --- /dev/null +++ b/docs/spec/v1beta2/README.md @@ -0,0 +1,20 @@ +# source.toolkit.fluxcd.io/v1beta2 + +This is the v1beta2 API specification for defining the desired state sources of Kubernetes clusters. + +## Specification + +* Source kinds: + + GitRepository + + HelmRepository + + HelmChart + + [Bucket](buckets.md) + +## Implementation + +* [source-controller](https://github.com/fluxcd/source-controller/) + +## Consumers + +* [kustomize-controller](https://github.com/fluxcd/kustomize-controller/) +* [helm-controller](https://github.com/fluxcd/helm-controller/) diff --git a/docs/spec/v1beta2/buckets.md b/docs/spec/v1beta2/buckets.md new file mode 100644 index 000000000..a74e0a50a --- /dev/null +++ b/docs/spec/v1beta2/buckets.md @@ -0,0 +1,780 @@ +# Buckets + +The `Bucket` API defines a Source to produce an Artifact for objects from storage +solutions like Amazon S3, Google Cloud Storage buckets, or any other solution +with a S3 compatible API such as Minio, Alibaba Cloud OSS and others. + +## Example + +The following is an example of a Bucket. It creates a tarball (`.tar.gz`) +Artifact with the fetched objects from an object storage with an S3 +compatible API (e.g. [Minio](https://min.io)): + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: minio-bucket + namespace: default +spec: + interval: 5m0s + endpoint: minio.example.com + insecure: true + secretRef: + name: minio-bucket-secret + bucketName: example +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-bucket-secret + namespace: default +type: Opaque +stringData: + accesskey: + secretkey: +``` + +In the above example: + +- A Bucket named `minio-bucket` is created, indicated by the + `.metadata.name` field. +- The source-controller checks the object storage bucket every five minutes, + indicated by the `.spec.interval` field. +- It authenticates to the `minio.example.com` endpoint with + the static credentials from the `minio-secret` Secret data, indicated by + the `.spec.endpoint` and `.spec.secretRef.name` fields. +- A list of object keys and their [etags](https://en.wikipedia.org/wiki/HTTP_ETag) + in the `.spec.bucketName` bucket is compiled, while filtering the keys using + [default ignore rules](#default-exclusions). +- The SHA256 sum of the list is used as Artifact revision, reported + in-cluster in the `.status.artifact.revision` field. +- When the current Bucket revision differs from the latest calculated revision, + all objects are fetched and archived. +- The new Artifact is reported in the `.status.artifact` field. + +You can run this example by saving the manifest into `bucket.yaml`, and +changing the Bucket and Secret values to target a Minio instance you have +control over. + +**Note:** For more advanced examples targeting e.g. Amazon S3 or GCP, see +[Provider](#provider). + +1. Apply the resource on the cluster: + + ```sh + kubectl apply -f bucket.yaml + ``` + +2. Run `kubectl get buckets` to see the Bucket: + + ```console + NAME ENDPOINT READY STATUS AGE + minio-bucket minio.example.com True stored artifact for revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' 34s + ``` + +3. Run `kubectl describe bucket minio-bucket` to see the [Artifact](#artifact) + and [Conditions](#conditions) in the Bucket's Status: + + ```console + ... + Status: + Artifact: + Checksum: 72aa638abb455ca5f9ef4825b949fd2de4d4be0a74895bf7ed2338622cd12686 + Last Update Time: 2022-02-01T23:43:38Z + Path: bucket/default/minio-bucket/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.tar.gz + Revision: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + URL: http://source-controller.source-system.svc.cluster.local./bucket/default/minio-bucket/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.tar.gz + Conditions: + Last Transition Time: 2022-02-01T23:43:38Z + Message: stored artifact for revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' + Observed Generation: 1 + Reason: Succeeded + Status: True + Type: Ready + Observed Generation: 1 + URL: http://source-controller.source-system.svc.cluster.local./bucket/default/minio-bucket/latest.tar.gz + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal BucketOperationSucceed 43s source-controller downloaded 16 files with revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' from 'my-minio-bucket' + Normal NewArtifact 43s source-controller stored artifact for revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' + ``` + +## Writing a Bucket spec + +As with all other Kubernetes config, a Bucket needs `apiVersion`, `kind`, and +`metadata` fields. The name of a Bucket object must be a valid +[DNS subdomain name](https://kubernetes.io/docs/concepts/overview/working-with-objects/names#dns-subdomain-names). + +A Bucket also needs a +[`.spec` section](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status). + +### Provider + +The `.spec.provider` field allows for specifying a Provider to enable provider +specific configurations, for example to communicate with a non-S3 compatible +API endpoint, or to change the authentication method. + +Supported options are: + +- [Generic](#generic) +- [AWS](#aws) +- [GCP](#gcp) + +If you do not specify `.spec.provider`, it defaults to `generic`. + +#### Generic + +When a Bucket's `spec.provider` is set to `generic`, the controller will +attempt to communicate with the specified [Endpoint](#endpoint) using the +[Minio Client SDK](https://github.com/minio/minio-go), which can communicate +with any Amazon S3 compatible object storage (including +[GCS](https://cloud.google.com/storage/docs/interoperability), +[Wasabi](https://wasabi-support.zendesk.com/hc/en-us/articles/360002079671-How-do-I-use-Minio-Client-with-Wasabi-), +and many others). + +The `generic` Provider _requires_ a [Secret reference](#secret-reference) to a +Secret with `.data.accesskey` and `.data.secretkey` values, used to +authenticate with static credentials. + +The Provider allows for specifying a region the bucket is in using the +[`.spec.region` field](#region), if required by the [Endpoint](#endpoint). + +##### Generic example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: generic-insecure + namespace: default +spec: + provider: generic + interval: 5m0s + bucketName: podinfo + endpoint: minio.minio.svc.cluster.local:9000 + timeout: 60s + insecure: true + secretRef: + name: minio-credentials +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-credentials + namespace: default +type: Opaque +data: + accesskey: + secretkey: +``` + +#### AWS + +When a Bucket's `.spec.provider` field is set to `aws`, the source-controller +will attempt to communicate with the specified [Endpoint](#endpoint) using the +[Minio Client SDK](https://github.com/minio/minio-go). + +Without a [Secret reference](#secret-reference), authorization using +credentials retrieved from the AWS EC2 service is attempted by default. When +a reference is specified, it expects a Secret with `.data.accesskey` and +`.data.secretkey` values, used to authenticate with static credentials. + +The Provider allows for specifying the +[Amazon AWS Region](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) +using the [`.spec.region` field](#region). + +##### AWS EC2 example + +**Note:** On EKS you have to create an [IAM role](#aws-iam-role-example) for +the source-controller service account that grants access to the bucket. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: aws + namespace: default +spec: + interval: 5m0s + provider: aws + bucketName: podinfo + endpoint: s3.amazonaws.com + region: us-east-1 + timeout: 30s +``` + +##### AWS IAM role example + +Replace `` with the specified `.spec.bucketName`. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::/*" + }, + { + "Sid": "", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::" + } + ] +} +``` + +##### AWS static auth example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: aws + namespace: default +spec: + interval: 5m0s + provider: aws + bucketName: podinfo + endpoint: s3.amazonaws.com + region: us-east-1 + secretRef: + name: aws-credentials +--- +apiVersion: v1 +kind: Secret +metadata: + name: aws-credentials + namespace: default +type: Opaque +data: + accesskey: + secretkey: +``` + +#### GCP + +When a Bucket's `.spec.provider` is set to `gcp`, the source-controller will +attempt to communicate with the specified [Endpoint](#endpoint) using the +[Google Client SDK](https://github.com/googleapis/google-api-go-client). + +Without a [Secret reference](#secret-reference), authorization using a +workload identity is attempted by default. The workload identity is obtained +using the `GOOGLE_APPLICATION_CREDENTIALS` environment variable, falling back +to the Google Application Credential file in the config directory. +When a reference is specified, it expects a Secret with a `.data.serviceaccount` +value with a GCP service account JSON file. + +The Provider allows for specifying the +[Bucket location](https://cloud.google.com/storage/docs/locations) using the +[`.spec.region` field](#region). + +##### GCP example + +```yaml +--- +apiVersion: source.toolkit.fluccd.io/v1beta2 +kind: Bucket +metadata: + name: gcp-workload-identity + namespace: default +spec: + interval: 5m0s + provider: gcp + bucketName: podinfo + endpoint: storage.googleapis.com + region: us-east-1 + timeout: 30s +``` + +##### GCP static auth example + +```yaml +--- +apiVersion: source.toolkit.fluccd.io/v1beta1 +kind: Bucket +metadata: + name: gcp-secret + namespace: default +spec: + interval: 5m0s + provider: gcp + bucketName: + endpoint: storage.googleapis.com + region: + secretRef: + name: gcp-service-account +--- +apiVersion: v1 +kind: Secret +metadata: + name: gcp-service-account + namespace: default +type: Opaque +data: + serviceaccount: +``` + +Where the (base64 decoded) value of `.data.serviceaccount` looks like this: + +```json +{ + "type": "service_account", + "project_id": "example", + "private_key_id": "28qwgh3gdf5hj3gb5fj3gsu5yfgh34f45324568hy2", + "private_key": "-----BEGIN PRIVATE KEY-----\nHwethgy123hugghhhbdcu6356dgyjhsvgvGFDHYgcdjbvcdhbsx63c\n76tgycfehuhVGTFYfw6t7ydgyVgydheyhuggycuhejwy6t35fthyuhegvcetf\nTFUHGTygghubhxe65ygt6tgyedgy326hucyvsuhbhcvcsjhcsjhcsvgdtHFCGi\nHcye6tyyg3gfyuhchcsbhygcijdbhyyTF66tuhcevuhdcbhuhhvftcuhbh3uh7t6y\nggvftUHbh6t5rfthhuGVRtfjhbfcrd5r67yuhuvgFTYjgvtfyghbfcdrhyjhbfctfdfyhvfg\ntgvggtfyghvft6tugvTF5r66tujhgvfrtyhhgfct6y7ytfr5ctvghbhhvtghhjvcttfycf\nffxfghjbvgcgyt67ujbgvctfyhVC7uhvgcyjvhhjvyujc\ncgghgvgcfhgg765454tcfthhgftyhhvvyvvffgfryyu77reredswfthhgfcftycfdrttfhf/\n-----END PRIVATE KEY-----\n", + "client_email": "test@example.iam.gserviceaccount.com", + "client_id": "32657634678762536746", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test%40podinfo.iam.gserviceaccount.com" +} +``` + +### Interval + +`.spec.interval` is a required field that specifices the interval which the +object storage bucket must be consulted at. + +After successfully reconciling a Bucket object, the source-controller requeues +the object for inspection after the specified interval. The value must be in a +[Go recognized duration string format](https://pkg.go.dev/time#ParseDuration), +e.g. `10m0s` to look at the object storage bucket every 10 minutes. + +If the `.metadata.generation` of a resource changes (due to e.g. the apply of a +change to the spec), this is handled instantly outside of the interval window. + +### Endpoint + +`.spec.endpoint` is a required field that specifies the HTTP/S object storage +endpoint to connect to and fetch objects from. Connecting to an (insecure) +HTTP endpoint requires enabling [`.spec.insecure`](#insecure). + +Some endpoints require the specification of a [`.spec.region`](#region), +see [Provider](#provider) for more (provider specific) examples. + +### Bucket name + +`.spec.bucketName` is a required field that specifies which object storage +bucket on the [Endpoint](#endpoint) objects should be fetched from. + +See [Provider](#provider) for more (provider specific) examples. + +### Region + +`.spec.region` is an optional field to specify the region a +[`.spec.bucketName`](#bucket-name) is located in. + +See [Provider](#provider) for more (provider specific) examples. + +### Insecure + +`.spec.insecure` is an optional field to allow connecting to an insecure (HTTP) +[endpoint](#endpoint), if set to `true`. The default value is `false`, +denying insecure (HTTP) connections. + +### Timeout + +`.spec.timeout` is an optional field to specify a timeout for object storage +fetch operations. The value must be in a +[Go recognized duration string format](https://pkg.go.dev/time#ParseDuration), +e.g. `1m30s` for a timeout of one minute and thirty seconds. +The default value is `60s`. + +### Secret reference + +`.spec.secretRef.name` is an optional field to specify a name reference to a +Secret in the same namespace as the Bucket, containing authentication +credentials for the object storage. For some `.spec.provider` implementations +the presence of the field is required, see [Provider](#provider) for more +details and examples. + +### Ignore + +`.spec.ignore` is an optional field to specify rules in [the `.gitignore` +pattern format](https://git-scm.com/docs/gitignore#_pattern_format). Storage +objects which keys match the defined rules are excluded while fetching. + +When specified, `.spec.ignore` overrides the [default exclusion +list](#default-exclusions), and may overrule the [`.sourceignore` file +exclusions](#sourceignore-file). See [excluding files](#excluding-files) +for more information. + +### Suspend + +`.spec.suspend` is an optional field to suspend the reconciliation of a Bucket. +When set to `true`, the controller will stop reconciling the Bucket, and changes +to the resource or in the object storage bucket will not result in a new +Artifact. When the field is set to `false` or removed, it will resume. + +For practical information, see +[suspending and resuming](#suspending-and-resuming). + +## Working with Buckets + +### Excluding files + +By default, storage bucket objects which match the [default exclusion +rules](#default-exclusions) are excluded while fetching. It is possible to +overwrite and/or overrule the default exclusions using a file in the bucket +and/or an in-spec set of rules. + +#### `.sourceignore` file + +Excluding files is possible by adding a `.sourceignore` file in the root of the +object storage bucket. The `.sourceignore` file follows [the `.gitignore` +pattern format](https://git-scm.com/docs/gitignore#_pattern_format), and +pattern entries may overrule [default exclusions](#default-exclusions). + +#### Ignore spec + +Another option is to define the exclusions within the Bucket spec, using the +[`.spec.ignore` field](#ignore). Specified rules override the +[default exclusion list](#default-exclusions), and may overrule `.sourceignore` +file exclusions. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: +spec: + ignore: | + # exclude all + /* + # include deploy dir + !/deploy + # exclude file extensions from deploy dir + /deploy/**/*.md + /deploy/**/*.txt +``` + +### Triggering a reconcile + +To manually tell the source-controller to reconcile a Bucket outside of the +[specified interval window](#interval), a Bucket can be annotated with +`reconcile.fluxcd.io/requestedAt: `. Annotating the resource +queues the Bucket for reconciliation if the `` differs from +the last value the controller acted on, as reported in +[`.status.lastHandledReconcileAt`](#last-handled-reconcile-at). + +Using `kubectl`: + +```sh +kubectl annotate --overwrite bucket/ reconcile.fluxcd.io/requestedAt="$(date +%s)" +``` + +Using `flux`: + +```sh +flux reconcile source bucket +``` + +### Waiting for `Ready` + +When a change is applied, it is possible to wait for the Bucket to reach a +[ready state](#ready-bucket) using `kubectl`: + +```sh +kubectl wait bucket/ --for=condition=ready --timeout=1m +``` + +### Suspending and resuming + +When you find yourself in a situation where you temporarily want to pause the +reconciliation of a Bucket, you can suspend it using the [`.spec.suspend` +field](#suspend). + +#### Suspend a Bucket + +In your YAML declaration: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: +spec: + suspend: true +``` + +Using `kubectl`: + +```sh +kubectl patch bucket -p '{\"spec\": {\"suspend\" : true }}' +``` + +Using `flux`: + +```sh +flux suspend source bucket +``` + +**Note:** When a Bucket has an Artifact and is suspended, and this Artifact +later disappears from the storage due to e.g. the source-controller Pod being +evicted from a Node, this will not be reflected in the Bucket's Status until it +is resumed. + +#### Resume a Bucket + +In your YAML declaration, comment out (or remove) the field: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: +spec: + # suspend: true +``` + +**Note:** Setting the field value to `false` has the same effect as removing +it, but does not allow for "hot patching" using e.g. `kubectl` while practicing +GitOps; as the manually applied patch would be overwritten by the declared +state in Git. + +Using `kubectl`: + +```sh +kubectl patch bucket -p '{\"spec\" : {\"suspend\" : false }}' +``` + +Using `flux`: + +```sh +flux resume source bucket +``` + +### Debugging a Bucket + +There are several ways to gather information about a Bucket for debugging +purposes. + +#### Describe the Bucket + +Describing a Bucket using `kubectl describe bucket ` displays the +latest recorded information for the resource in the `Status` and `Events` +sections: + +```console +... +Status: +... + Conditions: + Last Transition Time: 2022-02-02T13:26:55Z + Message: reconciling new generation 2 + Observed Generation: 2 + Reason: NewGeneration + Status: True + Type: Reconciling + Last Transition Time: 2022-02-02T13:26:55Z + Message: bucket 'my-new-bucket' does not exist + Observed Generation: 2 + Reason: BucketOperationFailed + Status: False + Type: Ready + Last Transition Time: 2022-02-02T13:26:55Z + Message: bucket 'my-new-bucket' does not exist + Observed Generation: 2 + Reason: BucketOperationFailed + Status: True + Type: FetchFailed + Observed Generation: 1 + URL: http://source-controller.source-system.svc.cluster.local./bucket/default/minio-bucket/latest.tar.gz +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning BucketOperationFailed 37s (x11 over 42s) source-controller bucket 'my-new-bucket' does not exist +``` + +#### Trace emitted Events + +To view events for specific Bucket(s), `kubectl get events` can be used in +combination with `--field-sector` to list the Events for specific objects. +For example, running + +```sh +kubectl get events --field-selector involvedObject.kind=Bucket,involvedObject.name= +``` + +lists + +```console +LAST SEEN TYPE REASON OBJECT MESSAGE +2m30s Normal BucketOperationSucceed bucket/ downloaded 16 files with revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' from 'my-minio-bucket' +2m30s Normal NewArtifact bucket/ stored artifact for revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' +18s Warning BucketOperationFailed bucket/ bucket 'my-new-bucket' does not exist +``` + +Besides being reported in Events, the reconciliation errors are also logged by +the controller. The Flux CLI offer commands for filtering the logs for a +specific Bucket, e.g. `flux logs --level=error --kind=Bucket --name=`. + +## Bucket Status + +### Artifact + +The Bucket reports the latest synchronized state from the object storage +bucket as an Artifact object in the `.status.artifact` of the resource. + +The Artifact file is a gzip compressed TAR archive +(`.tar.gz`), and can be retrieved in-cluster from the +`.status.artifact.url` HTTP address. + +#### Artifact example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: +status: + artifact: + checksum: cbec34947cc2f36dee8adcdd12ee62ca6a8a36699fc6e56f6220385ad5bd421a + lastUpdateTime: "2022-01-28T10:30:30Z" + path: bucket///c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2.tar.gz + revision: c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2 + url: http://source-controller..svc.cluster.local./bucket///c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2.tar.gz +``` + +#### Default exclusions + +The following files and extensions are excluded from the Artifact by +default: + +- Git files (`.git/, .gitignore, .gitmodules, .gitattributes`) +- File extensions (`.jpg, .jpeg, .gif, .png, .wmv, .flv, .tar.gz, .zip`) +- CI configs (`.github/, .circleci/, .travis.yml, .gitlab-ci.yml, appveyor.yml, .drone.yml, cloudbuild.yaml, codeship-services.yml, codeship-steps.yml`) +- CLI configs (`.goreleaser.yml, .sops.yaml`) +- Flux v1 config (`.flux.yaml`) + +To define your own exclusion rules, see [excluding files](#excluding-files). + +### Conditions + +A Bucket enters various states during its lifecycle, reflected as +[Kubernetes Conditions][typical-status-properties]. +It can be [reconciling](#reconciling-bucket) while fetching storage objects, +it can be [ready](#ready-bucket), or it can [fail during +reconciliation](#failed-bucket). + +The Bucket API is compatible with the [kstatus specification][kstatus-spec], +and reports `Reconciling` and `Stalled` conditions where applicable to +provide better (timeout) support to solutions polling the Bucket to become +`Ready`. + +#### Reconciling Bucket + +The source-controller marks a Bucket as _reconciling_ when one of the following +is true: + +- There is no current Artifact for the Bucket, or the reported Artifact is + determined to have disappeared from the storage. +- The generation of the Bucket is newer than the [Observed Generation](#observed-generation). +- The newly calculated Artifact revision differs from the current Artifact. + +When the Bucket is "reconciling", the `Ready` Condition status becomes `False`, +and the controller adds a Condition with the following attributes to the +Bucket's `.status.conditions`: + +- `type: Reconciling` +- `status: "True"` +- `reason: NewGeneration` | `reason: NoArtifact` | `reason: NewRevision` + +If the reconciling state is due to a new revision, an additional Condition is +added with the following attributes: + +- `type: ArtifactOutdated` +- `status: "True"` +- `reason: NewRevision` + +Both Conditions have a ["negative polarity"][typical-status-properties], +and are only present on the Bucket while their status value is `"True"`. + +#### Ready Bucket + +The source-controller marks a Bucket as _ready_ when it has the following +characteristics: + +- The Bucket reports an [Artifact](#artifact). +- The reported Artifact exists in the controller's Artifact storage. +- The Bucket was able to communicate with the Bucket's object storage endpoint + using the current spec. +- The revision of the reported Artifact is up-to-date with the latest + calculated revision of the object storage bucket. + +When the Bucket is "ready", the controller sets a Condition with the following +attributes in the Bucket's `.status.conditions`: + +- `type: Ready` +- `status: "True"` +- `reason: Succeeded` + +This `Ready` Condition will retain a status value of `"True"` until the Bucket +is marked as [reconciling](#reconciling-bucket), or e.g. a +[transient error](#failed-bucket) occurs due to a temporary network issue. + +#### Failed Bucket + +The source-controller may get stuck trying to produce an Artifact for a Bucket +without completing. This can occur due to some of the following factors: + +- The object storage [Endpoint](#endpoint) is temporarily unavailable. +- The specified object storage bucket does not exist. +- The [Secret reference](#secret-reference) contains a reference to a + non-existing Secret. +- The credentials in the referenced Secret are invalid. +- The Bucket spec contains a generic misconfiguration. + +When this happens, the controller sets the `Ready` Condition status to `False`, +and adds a Condition with the following attributes to the Bucket's +`.status.conditions`: + +- `type: FetchFailed` +- `status: "True"` +- `reason: AuthenticationFailed` | `reason: BucketOperationFailed` + +This condition has a ["negative polarity"][typical-status-properties], +and is only present on the Bucket while the status value is `"True"`. + +While the Bucket has this Condition, the controller will continue to attempt +to produce an Artifact for the resource with an exponential backoff, until +it succeeds and the Bucket is marked as [ready](#ready-bucket). + +Note that a Bucket can be [reconciling](#reconciling-bucket) while failing at +the same time, for example due to a newly introduced configuration issue in the +Bucket spec. + +### Observed Generation + +The source-controller reports an +[observed generation][typical-status-properties] +in the Bucket's `.status.observedGeneration`. The observed generation is the +latest `.metadata.generation` which resulted in either a [ready state](#ready-bucket), +or stalled due to error it can not recover from without human +intervention. + +### Last Handled Reconcile At + +The source-controller reports the last `reconcile.fluxcd.io/requestedAt` +annotation value it acted on in the `.status.lastHandledReconcileAt` field. + +For practical information about this field, see [triggering a +reconcile](#triggering-a-reconcile). + +[typical-status-properties]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties +[kstatus-spec]: https://github.com/kubernetes-sigs/cli-utils/tree/master/pkg/kstatus From 86d1d80bf2c7debc26888081bf53e0cfbed59aeb Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 7 Feb 2022 12:34:43 +0100 Subject: [PATCH 2/9] Document HelmRepository API v1beta2 spec Signed-off-by: Hidde Beydals --- api/v1beta2/helmrepository_types.go | 86 ++- ...ce.toolkit.fluxcd.io_helmrepositories.yaml | 41 +- controllers/helmrepository_controller.go | 100 ++-- controllers/helmrepository_controller_test.go | 20 +- docs/api/source.md | 92 +-- docs/spec/v1beta2/README.md | 2 +- docs/spec/v1beta2/helmrepositories.md | 534 ++++++++++++++++++ 7 files changed, 716 insertions(+), 159 deletions(-) create mode 100644 docs/spec/v1beta2/helmrepositories.md diff --git a/api/v1beta2/helmrepository_types.go b/api/v1beta2/helmrepository_types.go index c57c8563e..1601885c5 100644 --- a/api/v1beta2/helmrepository_types.go +++ b/api/v1beta2/helmrepository_types.go @@ -28,56 +28,62 @@ import ( const ( // HelmRepositoryKind is the string representation of a HelmRepository. HelmRepositoryKind = "HelmRepository" - // HelmRepositoryURLIndexKey is the key to use for indexing HelmRepository - // resources by their HelmRepositorySpec.URL. + // HelmRepositoryURLIndexKey is the key used for indexing HelmRepository + // objects by their HelmRepositorySpec.URL. HelmRepositoryURLIndexKey = ".metadata.helmRepositoryURL" ) -// HelmRepositorySpec defines the reference to a Helm repository. +// HelmRepositorySpec specifies the required configuration to produce an +// Artifact for a Helm repository index YAML. type HelmRepositorySpec struct { - // The Helm repository URL, a valid URL contains at least a protocol and host. + // URL of the Helm repository, a valid URL contains at least a protocol and + // host. // +required URL string `json:"url"` - // The name of the secret containing authentication credentials for the Helm - // repository. - // For HTTP/S basic auth the secret must contain username and - // password fields. - // For TLS the secret must contain a certFile and keyFile, and/or - // caCert fields. + // SecretRef specifies the Secret containing authentication credentials + // for the HelmRepository. + // For HTTP/S basic auth the secret must contain 'username' and 'password' + // fields. + // For TLS the secret must contain a 'certFile' and 'keyFile', and/or + // 'caCert' fields. // +optional SecretRef *meta.LocalObjectReference `json:"secretRef,omitempty"` - // PassCredentials allows the credentials from the SecretRef to be passed on to - // a host that does not match the host as defined in URL. - // This may be required if the host of the advertised chart URLs in the index - // differ from the defined URL. - // Enabling this should be done with caution, as it can potentially result in - // credentials getting stolen in a MITM-attack. + // PassCredentials allows the credentials from the SecretRef to be passed + // on to a host that does not match the host as defined in URL. + // This may be required if the host of the advertised chart URLs in the + // index differ from the defined URL. + // Enabling this should be done with caution, as it can potentially result + // in credentials getting stolen in a MITM-attack. // +optional PassCredentials bool `json:"passCredentials,omitempty"` - // The interval at which to check the upstream for updates. + // Interval at which to check the URL for updates. // +required Interval metav1.Duration `json:"interval"` - // The timeout of index fetching, defaults to 60s. + // Timeout of the index fetch operation, defaults to 60s. // +kubebuilder:default:="60s" // +optional Timeout *metav1.Duration `json:"timeout,omitempty"` - // This flag tells the controller to suspend the reconciliation of this source. + // Suspend tells the controller to suspend the reconciliation of this + // HelmRepository. // +optional Suspend bool `json:"suspend,omitempty"` - // AccessFrom defines an Access Control List for allowing cross-namespace references to this object. + // AccessFrom specifies an Access Control List for allowing cross-namespace + // references to this object. + // NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092 // +optional AccessFrom *acl.AccessFrom `json:"accessFrom,omitempty"` } -// HelmRepositoryStatus defines the observed state of the HelmRepository. +// HelmRepositoryStatus records the observed state of the HelmRepository. type HelmRepositoryStatus struct { - // ObservedGeneration is the last observed generation. + // ObservedGeneration is the last observed generation of the HelmRepository + // object. // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` @@ -85,11 +91,13 @@ type HelmRepositoryStatus struct { // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` - // URL is the fetch link for the last index fetched. + // URL is the dynamic fetch link for the latest Artifact. + // It is provided on a "best effort" basis, and using the precise + // HelmRepositoryStatus.Artifact data is recommended. // +optional URL string `json:"url,omitempty"` - // Artifact represents the output of the last successful repository sync. + // Artifact represents the last successful HelmRepository reconciliation. // +optional Artifact *Artifact `json:"artifact,omitempty"` @@ -97,13 +105,9 @@ type HelmRepositoryStatus struct { } const ( - // IndexationFailedReason represents the fact that the indexation of the given - // Helm repository failed. + // IndexationFailedReason signals that the HelmRepository index fetch + // failed. IndexationFailedReason string = "IndexationFailed" - - // IndexationSucceededReason represents the fact that the indexation of the - // given Helm repository succeeded. - IndexationSucceededReason string = "IndexationSucceed" ) // GetConditions returns the status conditions of the object. @@ -116,28 +120,18 @@ func (in *HelmRepository) SetConditions(conditions []metav1.Condition) { in.Status.Conditions = conditions } -// GetRequeueAfter returns the duration after which the source must be reconciled again. +// GetRequeueAfter returns the duration after which the source must be +// reconciled again. func (in HelmRepository) GetRequeueAfter() time.Duration { return in.Spec.Interval.Duration } -// GetInterval returns the interval at which the source is reconciled. -// Deprecated: use GetRequeueAfter instead. -func (in HelmRepository) GetInterval() metav1.Duration { - return in.Spec.Interval -} - -// GetArtifact returns the latest artifact from the source if present in the status sub-resource. +// GetArtifact returns the latest artifact from the source if present in the +// status sub-resource. func (in *HelmRepository) GetArtifact() *Artifact { return in.Status.Artifact } -// GetStatusConditions returns a pointer to the Status.Conditions slice. -// Deprecated: use GetConditions instead. -func (in *HelmRepository) GetStatusConditions() *[]metav1.Condition { - return &in.Status.Conditions -} - // +genclient // +genclient:Namespaced // +kubebuilder:storageversion @@ -149,7 +143,7 @@ func (in *HelmRepository) GetStatusConditions() *[]metav1.Condition { // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="" // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="" -// HelmRepository is the Schema for the helmrepositories API +// HelmRepository is the Schema for the helmrepositories API. type HelmRepository struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -159,7 +153,7 @@ type HelmRepository struct { Status HelmRepositoryStatus `json:"status,omitempty"` } -// HelmRepositoryList contains a list of HelmRepository +// HelmRepositoryList contains a list of HelmRepository objects. // +kubebuilder:object:root=true type HelmRepositoryList struct { metav1.TypeMeta `json:",inline"` diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml index a2c6d9195..a2308eef6 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml @@ -253,7 +253,7 @@ spec: name: v1beta2 schema: openAPIV3Schema: - description: HelmRepository is the Schema for the helmrepositories API + description: HelmRepository is the Schema for the helmrepositories API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -268,11 +268,13 @@ spec: metadata: type: object spec: - description: HelmRepositorySpec defines the reference to a Helm repository. + description: HelmRepositorySpec specifies the required configuration to + produce an Artifact for a Helm repository index YAML. properties: accessFrom: - description: AccessFrom defines an Access Control List for allowing - cross-namespace references to this object. + description: 'AccessFrom specifies an Access Control List for allowing + cross-namespace references to this object. NOTE: Not implemented, + provisional as of https://github.com/fluxcd/flux2/pull/2092' properties: namespaceSelectors: description: NamespaceSelectors is the list of namespace selectors @@ -298,7 +300,7 @@ spec: - namespaceSelectors type: object interval: - description: The interval at which to check the upstream for updates. + description: Interval at which to check the URL for updates. type: string passCredentials: description: PassCredentials allows the credentials from the SecretRef @@ -309,10 +311,10 @@ spec: getting stolen in a MITM-attack. type: boolean secretRef: - description: The name of the secret containing authentication credentials - for the Helm repository. For HTTP/S basic auth the secret must contain - username and password fields. For TLS the secret must contain a - certFile and keyFile, and/or caCert fields. + description: SecretRef specifies the Secret containing authentication + credentials for the HelmRepository. For HTTP/S basic auth the secret + must contain 'username' and 'password' fields. For TLS the secret + must contain a 'certFile' and 'keyFile', and/or 'caCert' fields. properties: name: description: Name of the referent. @@ -321,15 +323,15 @@ spec: - name type: object suspend: - description: This flag tells the controller to suspend the reconciliation - of this source. + description: Suspend tells the controller to suspend the reconciliation + of this HelmRepository. type: boolean timeout: default: 60s - description: The timeout of index fetching, defaults to 60s. + description: Timeout of the index fetch operation, defaults to 60s. type: string url: - description: The Helm repository URL, a valid URL contains at least + description: URL of the Helm repository, a valid URL contains at least a protocol and host. type: string required: @@ -339,11 +341,11 @@ spec: status: default: observedGeneration: -1 - description: HelmRepositoryStatus defines the observed state of the HelmRepository. + description: HelmRepositoryStatus records the observed state of the HelmRepository. properties: artifact: - description: Artifact represents the output of the last successful - repository sync. + description: Artifact represents the last successful HelmRepository + reconciliation. properties: checksum: description: Checksum is the SHA256 checksum of the Artifact file. @@ -453,11 +455,14 @@ spec: be detected. type: string observedGeneration: - description: ObservedGeneration is the last observed generation. + description: ObservedGeneration is the last observed generation of + the HelmRepository object. format: int64 type: integer url: - description: URL is the fetch link for the last index fetched. + description: URL is the dynamic fetch link for the latest Artifact. + It is provided on a "best effort" basis, and using the precise HelmRepositoryStatus.Artifact + data is recommended. type: string type: object type: object diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go index 9bae915a3..8658f1dd3 100644 --- a/controllers/helmrepository_controller.go +++ b/controllers/helmrepository_controller.go @@ -52,9 +52,9 @@ import ( "github.com/fluxcd/source-controller/internal/reconcile/summarize" ) -// helmRepoReadyConditions contains all the conditions information needed -// for HelmRepository Ready status conditions summary calculation. -var helmRepoReadyConditions = summarize.Conditions{ +// helmRepositoryReadyCondition contains the information required to summarize a +// v1beta2.HelmRepository Ready Condition. +var helmRepositoryReadyCondition = summarize.Conditions{ Target: meta.ReadyCondition, Owned: []string{ sourcev1.FetchFailedCondition, @@ -82,7 +82,7 @@ var helmRepoReadyConditions = summarize.Conditions{ // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=helmrepositories/finalizers,verbs=get;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch -// HelmRepositoryReconciler reconciles a HelmRepository object +// HelmRepositoryReconciler reconciles a v1beta2.HelmRepository object. type HelmRepositoryReconciler struct { client.Client kuberecorder.EventRecorder @@ -97,10 +97,11 @@ type HelmRepositoryReconcilerOptions struct { MaxConcurrentReconciles int } -// helmRepoReconcilerFunc is the function type for all the helm repository -// reconciler functions. The reconciler functions are grouped together and -// executed serially to perform the main operation of the reconciler. -type helmRepoReconcilerFunc func(ctx context.Context, obj *sourcev1.HelmRepository, artifact *sourcev1.Artifact, repo *repository.ChartRepository) (sreconcile.Result, error) +// helmRepositoryReconcileFunc is the function type for all the +// v1beta2.HelmRepository (sub)reconcile functions. The type implementations +// are grouped and executed serially to perform the complete reconcile of the +// object. +type helmRepositoryReconcileFunc func(ctx context.Context, obj *sourcev1.HelmRepository, artifact *sourcev1.Artifact, repo *repository.ChartRepository) (sreconcile.Result, error) func (r *HelmRepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error { return r.SetupWithManagerAndOptions(mgr, HelmRepositoryReconcilerOptions{}) @@ -147,7 +148,7 @@ func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reque defer func() { summarizeHelper := summarize.NewHelper(r.EventRecorder, patchHelper) summarizeOpts := []summarize.Option{ - summarize.WithConditions(helmRepoReadyConditions), + summarize.WithConditions(helmRepositoryReadyCondition), summarize.WithReconcileResult(recResult), summarize.WithReconcileError(retErr), summarize.WithIgnoreNotFound(), @@ -155,7 +156,7 @@ func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reque summarize.RecordContextualError, summarize.RecordReconcileReq, ), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetInterval().Duration}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), summarize.WithPatchFieldOwner(r.ControllerName), } result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) @@ -180,7 +181,7 @@ func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reque } // Reconcile actual object - reconcilers := []helmRepoReconcilerFunc{ + reconcilers := []helmRepositoryReconcileFunc{ r.reconcileStorage, r.reconcileSource, r.reconcileArtifact, @@ -189,12 +190,10 @@ func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reque return } -// reconcile iterates through the sub-reconcilers and processes the source -// object. The sub-reconcilers are run sequentially. The result and error of -// the sub-reconciliation are collected and returned. For multiple results -// from different sub-reconcilers, the results are combined to return the -// result with the shortest requeue period. -func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.HelmRepository, reconcilers []helmRepoReconcilerFunc) (sreconcile.Result, error) { +// reconcile iterates through the gitRepositoryReconcileFunc tasks for the +// object. It returns early on the first call that returns +// reconcile.ResultRequeue, or produces an error. +func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.HelmRepository, reconcilers []helmRepositoryReconcileFunc) (sreconcile.Result, error) { if obj.Generation != obj.Status.ObservedGeneration { conditions.MarkReconciling(obj, "NewGeneration", "reconciling new object generation (%d)", obj.Generation) } @@ -224,12 +223,18 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1. return res, resErr } -// reconcileStorage ensures the current state of the storage matches the desired and previously observed state. +// reconcileStorage ensures the current state of the storage matches the +// desired and previously observed state. // -// All artifacts for the resource except for the current one are garbage collected from the storage. -// If the artifact in the Status object of the resource disappeared from storage, it is removed from the object. -// If the hostname of the URLs on the object do not match the current storage server hostname, they are updated. -func (r *HelmRepositoryReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.HelmRepository, artifact *sourcev1.Artifact, chartRepo *repository.ChartRepository) (sreconcile.Result, error) { +// All Artifacts for the object except for the current one in the Status are +// garbage collected from the Storage. +// If the Artifact in the Status of the object disappeared from the Storage, +// it is removed from the object. +// If the object does not have an Artifact in its Status, a Reconciling +// condition is added. +// The hostname of any URL in the Status of the object are updated, to ensure +// they match the Storage server hostname of current runtime. +func (r *HelmRepositoryReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.HelmRepository, _ *sourcev1.Artifact, _ *repository.ChartRepository) (sreconcile.Result, error) { // Garbage collect previous advertised artifact(s) from storage _ = r.garbageCollect(ctx, obj) @@ -253,13 +258,14 @@ func (r *HelmRepositoryReconciler) reconcileStorage(ctx context.Context, obj *so return sreconcile.ResultSuccess, nil } -// reconcileSource ensures the upstream Helm repository can be reached and downloaded out using the declared -// configuration, and stores a new artifact in the storage. +// reconcileSource attempts to fetch the Helm repository index using the +// specified configuration on the v1beta2.HelmRepository object. // -// The Helm repository index is downloaded using the defined configuration, and in case of an error during this process -// (including transient errors), it records v1beta1.FetchFailedCondition=True and returns early. -// If the download is successful, the given artifact pointer is set to a new artifact with the available metadata, and -// the index pointer is set to the newly downloaded index. +// When the fetch fails, it records v1beta2.FetchFailedCondition=True and +// returns early. +// If successful and the index is valid, any previous +// v1beta2.FetchFailedCondition is removed, and the repository.ChartRepository +// pointer is set to the newly fetched index. func (r *HelmRepositoryReconciler) reconcileSource(ctx context.Context, obj *sourcev1.HelmRepository, artifact *sourcev1.Artifact, chartRepo *repository.ChartRepository) (sreconcile.Result, error) { var tlsConfig *tls.Config @@ -373,14 +379,15 @@ func (r *HelmRepositoryReconciler) reconcileSource(ctx context.Context, obj *sou return sreconcile.ResultSuccess, nil } -// reconcileArtifact stores a new artifact in the storage, if the current observation on the object does not match the -// given data. +// reconcileArtifact archives a new Artifact to the Storage, if the current +// (Status) data on the object does not match the given. // -// The inspection of the given data to the object is differed, ensuring any stale observations as -// v1beta1.ArtifactUnavailableCondition and v1beta1.ArtifactOutdatedCondition are always deleted. -// If the given artifact does not differ from the object's current, it returns early. -// On a successful write of a new artifact, the artifact in the status of the given object is set, and the symlink in -// the storage is updated to its path. +// The inspection of the given data to the object is differed, ensuring any +// stale observations like v1beta2.ArtifactOutdatedCondition are removed. +// If the given Artifact does not differ from the object's current, it returns +// early. +// On a successful archive, the Artifact in the Status of the object is set, +// and the symlink in the Storage is updated to its path. func (r *HelmRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.HelmRepository, artifact *sourcev1.Artifact, chartRepo *repository.ChartRepository) (sreconcile.Result, error) { // Always restore the Ready condition in case it got removed due to a transient error. defer func() { @@ -450,15 +457,16 @@ func (r *HelmRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *s r.eventLogf(ctx, obj, corev1.EventTypeWarning, sourcev1.StorageOperationFailedReason, "failed to update status URL symlink: %s", err) } - if indexURL != "" { obj.Status.URL = indexURL } + return sreconcile.ResultSuccess, nil } -// reconcileDelete handles the delete of an object. It first garbage collects all artifacts for the object from the -// artifact storage, if successful, the finalizer is removed from the object. +// reconcileDelete handles the deletion of the object. +// It first garbage collects all Artifacts for the object from the Storage. +// Removing the finalizer from the object if successful. func (r *HelmRepositoryReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.HelmRepository) (sreconcile.Result, error) { // Garbage collect the resource's artifacts if err := r.garbageCollect(ctx, obj); err != nil { @@ -473,9 +481,11 @@ func (r *HelmRepositoryReconciler) reconcileDelete(ctx context.Context, obj *sou return sreconcile.ResultEmpty, nil } -// garbageCollect performs a garbage collection for the given v1beta1.HelmRepository. It removes all but the current -// artifact except for when the deletion timestamp is set, which will result in the removal of all artifacts for the -// resource. +// garbageCollect performs a garbage collection for the given object. +// +// It removes all but the current Artifact from the Storage, unless the +// deletion timestamp on the object is set. Which will result in the +// removal of all Artifacts for the objects. func (r *HelmRepositoryReconciler) garbageCollect(ctx context.Context, obj *sourcev1.HelmRepository) error { if !obj.DeletionTimestamp.IsZero() { if deleted, err := r.Storage.RemoveAll(r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), "", "*")); err != nil { @@ -504,9 +514,11 @@ func (r *HelmRepositoryReconciler) garbageCollect(ctx context.Context, obj *sour return nil } -// eventLog records event and logs at the same time. This log is different from -// the debug log in the event recorder in the sense that this is a simple log, -// the event recorder debug log contains complete details about the event. +// eventLogf records event and logs at the same time. +// +// This log is different from the debug log in the EventRecorder, in the sense +// that this is a simple log. While the debug log contains complete details +// about the event. func (r *HelmRepositoryReconciler) eventLogf(ctx context.Context, obj runtime.Object, eventType string, reason string, messageFmt string, args ...interface{}) { msg := fmt.Sprintf(messageFmt, args...) // Log and emit event. diff --git a/controllers/helmrepository_controller_test.go b/controllers/helmrepository_controller_test.go index 570bfb004..83cd57bb2 100644 --- a/controllers/helmrepository_controller_test.go +++ b/controllers/helmrepository_controller_test.go @@ -94,7 +94,7 @@ func TestHelmRepositoryReconciler_Reconcile(t *testing.T) { }, timeout).Should(BeTrue()) // Check if the object status is valid. - condns := &status.Conditions{NegativePolarity: helmRepoReadyConditions.NegativePolarity} + condns := &status.Conditions{NegativePolarity: helmRepositoryReadyCondition.NegativePolarity} checker := status.NewChecker(testEnv.Client, testEnv.GetScheme(), condns) checker.CheckErr(ctx, obj) @@ -633,8 +633,8 @@ func TestHelmRepositoryReconciler_reconcileArtifact(t *testing.T) { } func TestHelmRepositoryReconciler_reconcileSubRecs(t *testing.T) { - // Helper to build simple helmRepoReconcilerFunc with result and error. - buildReconcileFuncs := func(r sreconcile.Result, e error) helmRepoReconcilerFunc { + // Helper to build simple helmRepositoryReconcileFunc with result and error. + buildReconcileFuncs := func(r sreconcile.Result, e error) helmRepositoryReconcileFunc { return func(ctx context.Context, obj *sourcev1.HelmRepository, artifact *sourcev1.Artifact, repo *repository.ChartRepository) (sreconcile.Result, error) { return r, e } @@ -644,14 +644,14 @@ func TestHelmRepositoryReconciler_reconcileSubRecs(t *testing.T) { name string generation int64 observedGeneration int64 - reconcileFuncs []helmRepoReconcilerFunc + reconcileFuncs []helmRepositoryReconcileFunc wantResult sreconcile.Result wantErr bool assertConditions []metav1.Condition }{ { name: "successful reconciliations", - reconcileFuncs: []helmRepoReconcilerFunc{ + reconcileFuncs: []helmRepositoryReconcileFunc{ buildReconcileFuncs(sreconcile.ResultSuccess, nil), }, wantResult: sreconcile.ResultSuccess, @@ -661,7 +661,7 @@ func TestHelmRepositoryReconciler_reconcileSubRecs(t *testing.T) { name: "successful reconciliation with generation difference", generation: 3, observedGeneration: 2, - reconcileFuncs: []helmRepoReconcilerFunc{ + reconcileFuncs: []helmRepositoryReconcileFunc{ buildReconcileFuncs(sreconcile.ResultSuccess, nil), }, wantResult: sreconcile.ResultSuccess, @@ -672,7 +672,7 @@ func TestHelmRepositoryReconciler_reconcileSubRecs(t *testing.T) { }, { name: "failed reconciliation", - reconcileFuncs: []helmRepoReconcilerFunc{ + reconcileFuncs: []helmRepositoryReconcileFunc{ buildReconcileFuncs(sreconcile.ResultEmpty, fmt.Errorf("some error")), }, wantResult: sreconcile.ResultEmpty, @@ -680,7 +680,7 @@ func TestHelmRepositoryReconciler_reconcileSubRecs(t *testing.T) { }, { name: "multiple object status conditions mutations", - reconcileFuncs: []helmRepoReconcilerFunc{ + reconcileFuncs: []helmRepositoryReconcileFunc{ func(ctx context.Context, obj *sourcev1.HelmRepository, artifact *sourcev1.Artifact, repo *repository.ChartRepository) (sreconcile.Result, error) { conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", "new index revision") return sreconcile.ResultSuccess, nil @@ -699,7 +699,7 @@ func TestHelmRepositoryReconciler_reconcileSubRecs(t *testing.T) { }, { name: "subrecs with one result=Requeue, no error", - reconcileFuncs: []helmRepoReconcilerFunc{ + reconcileFuncs: []helmRepositoryReconcileFunc{ buildReconcileFuncs(sreconcile.ResultSuccess, nil), buildReconcileFuncs(sreconcile.ResultRequeue, nil), buildReconcileFuncs(sreconcile.ResultSuccess, nil), @@ -709,7 +709,7 @@ func TestHelmRepositoryReconciler_reconcileSubRecs(t *testing.T) { }, { name: "subrecs with error before result=Requeue", - reconcileFuncs: []helmRepoReconcilerFunc{ + reconcileFuncs: []helmRepositoryReconcileFunc{ buildReconcileFuncs(sreconcile.ResultSuccess, nil), buildReconcileFuncs(sreconcile.ResultEmpty, fmt.Errorf("some error")), buildReconcileFuncs(sreconcile.ResultRequeue, nil), diff --git a/docs/api/source.md b/docs/api/source.md index 129d061bc..8b93f1815 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -676,7 +676,7 @@ HelmChartStatus

HelmRepository

-

HelmRepository is the Schema for the helmrepositories API

+

HelmRepository is the Schema for the helmrepositories API.

(Optional) -

ObservedGeneration is the last observed generation.

+

ObservedGeneration is the last observed generation of the Bucket object.

(Optional) -

URL is the fetch link for the artifact output of the last Bucket sync.

+

URL is the dynamic fetch link for the latest Artifact. +It is provided on a “best effort” basis, and using the precise +BucketStatus.Artifact data is recommended.

(Optional) -

Artifact represents the output of the last successful Bucket sync.

+

Artifact represents the last successful Bucket reconciliation.

@@ -739,7 +739,8 @@ string @@ -753,12 +754,12 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference @@ -770,12 +771,12 @@ bool @@ -788,7 +789,7 @@ Kubernetes meta/v1.Duration @@ -802,7 +803,7 @@ Kubernetes meta/v1.Duration @@ -814,7 +815,8 @@ bool @@ -828,7 +830,9 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom
-

The Helm repository URL, a valid URL contains at least a protocol and host.

+

URL of the Helm repository, a valid URL contains at least a protocol and +host.

(Optional) -

The name of the secret containing authentication credentials for the Helm -repository. -For HTTP/S basic auth the secret must contain username and -password fields. -For TLS the secret must contain a certFile and keyFile, and/or -caCert fields.

+

SecretRef specifies the Secret containing authentication credentials +for the HelmRepository. +For HTTP/S basic auth the secret must contain ‘username’ and ‘password’ +fields. +For TLS the secret must contain a ‘certFile’ and ‘keyFile’, and/or +‘caCert’ fields.

(Optional) -

PassCredentials allows the credentials from the SecretRef to be passed on to -a host that does not match the host as defined in URL. -This may be required if the host of the advertised chart URLs in the index -differ from the defined URL. -Enabling this should be done with caution, as it can potentially result in -credentials getting stolen in a MITM-attack.

+

PassCredentials allows the credentials from the SecretRef to be passed +on to a host that does not match the host as defined in URL. +This may be required if the host of the advertised chart URLs in the +index differ from the defined URL. +Enabling this should be done with caution, as it can potentially result +in credentials getting stolen in a MITM-attack.

-

The interval at which to check the upstream for updates.

+

Interval at which to check the URL for updates.

(Optional) -

The timeout of index fetching, defaults to 60s.

+

Timeout of the index fetch operation, defaults to 60s.

(Optional) -

This flag tells the controller to suspend the reconciliation of this source.

+

Suspend tells the controller to suspend the reconciliation of this +HelmRepository.

(Optional) -

AccessFrom defines an Access Control List for allowing cross-namespace references to this object.

+

AccessFrom specifies an Access Control List for allowing cross-namespace +references to this object. +NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092

@@ -1928,7 +1932,8 @@ github.com/fluxcd/pkg/apis/meta.ReconcileRequestStatus (Appears on: HelmRepository)

-

HelmRepositorySpec defines the reference to a Helm repository.

+

HelmRepositorySpec specifies the required configuration to produce an +Artifact for a Helm repository index YAML.

@@ -1947,7 +1952,8 @@ string @@ -1961,12 +1967,12 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference @@ -1978,12 +1984,12 @@ bool @@ -1996,7 +2002,7 @@ Kubernetes meta/v1.Duration @@ -2010,7 +2016,7 @@ Kubernetes meta/v1.Duration @@ -2022,7 +2028,8 @@ bool @@ -2036,7 +2043,9 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom @@ -2049,7 +2058,7 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom (Appears on:HelmRepository)

-

HelmRepositoryStatus defines the observed state of the HelmRepository.

+

HelmRepositoryStatus records the observed state of the HelmRepository.

-

The Helm repository URL, a valid URL contains at least a protocol and host.

+

URL of the Helm repository, a valid URL contains at least a protocol and +host.

(Optional) -

The name of the secret containing authentication credentials for the Helm -repository. -For HTTP/S basic auth the secret must contain username and -password fields. -For TLS the secret must contain a certFile and keyFile, and/or -caCert fields.

+

SecretRef specifies the Secret containing authentication credentials +for the HelmRepository. +For HTTP/S basic auth the secret must contain ‘username’ and ‘password’ +fields. +For TLS the secret must contain a ‘certFile’ and ‘keyFile’, and/or +‘caCert’ fields.

(Optional) -

PassCredentials allows the credentials from the SecretRef to be passed on to -a host that does not match the host as defined in URL. -This may be required if the host of the advertised chart URLs in the index -differ from the defined URL. -Enabling this should be done with caution, as it can potentially result in -credentials getting stolen in a MITM-attack.

+

PassCredentials allows the credentials from the SecretRef to be passed +on to a host that does not match the host as defined in URL. +This may be required if the host of the advertised chart URLs in the +index differ from the defined URL. +Enabling this should be done with caution, as it can potentially result +in credentials getting stolen in a MITM-attack.

-

The interval at which to check the upstream for updates.

+

Interval at which to check the URL for updates.

(Optional) -

The timeout of index fetching, defaults to 60s.

+

Timeout of the index fetch operation, defaults to 60s.

(Optional) -

This flag tells the controller to suspend the reconciliation of this source.

+

Suspend tells the controller to suspend the reconciliation of this +HelmRepository.

(Optional) -

AccessFrom defines an Access Control List for allowing cross-namespace references to this object.

+

AccessFrom specifies an Access Control List for allowing cross-namespace +references to this object. +NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092

@@ -2069,7 +2078,8 @@ int64 @@ -2095,7 +2105,9 @@ string @@ -2109,7 +2121,7 @@ Artifact diff --git a/docs/spec/v1beta2/README.md b/docs/spec/v1beta2/README.md index bdb9b1adb..dca99b0e2 100644 --- a/docs/spec/v1beta2/README.md +++ b/docs/spec/v1beta2/README.md @@ -6,7 +6,7 @@ This is the v1beta2 API specification for defining the desired state sources of * Source kinds: + GitRepository - + HelmRepository + + [HelmRepository](helmrepositories.md) + HelmChart + [Bucket](buckets.md) diff --git a/docs/spec/v1beta2/helmrepositories.md b/docs/spec/v1beta2/helmrepositories.md new file mode 100644 index 000000000..d92a7ab56 --- /dev/null +++ b/docs/spec/v1beta2/helmrepositories.md @@ -0,0 +1,534 @@ +# Helm Repositories + +The `HelmRepository` API defines a Source to produce an Artifact for a Helm +repository index YAML (`index.yaml`). + +## Example + +The following is an example of a HelmRepository. It creates a YAML (`.yaml`) +Artifact from the fetched Helm repository index (in this example the [podinfo +repository](https://github.com/stefanprodan/podinfo)): + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: podinfo + namespace: default +spec: + interval: 5m0s + url: https://stefanprodan.github.io/podinfo +``` + +In the above example: + +- A HelmRepository named `podinfo` is created, indicated by the + `.metadata.name` field. +- The source-controller fetches the Helm repository index YAML every five + minutes from `https://stefanprodan.github.io/podinfo`, indicated by the + `.spec.interval` and `.spec.url` fields. +- The SHA256 sum of the Helm repository index after stable sorting the entries + is used as Artifact revision, reported in-cluster in the + `.status.artifact.revision` field. +- When the current HelmRepository revision differs from the latest fetched + revision, it is stored as a new Artifact. +- The new Artifact is reported in the `.status.artifact` field. + +You can run this example by saving the manifest into `helmrepository.yaml`. + +1. Apply the resource on the cluster: + + ```sh + kubectl apply -f helmrepository.yaml + ``` + +2. Run `kubectl get helmrepository` to see the HelmRepository: + + ```console + NAME URL READY STATUS AGE + podinfo https://stefanprodan.github.io/podinfo True stored artifact for revision '83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111' 4s + ``` + +3. Run `kubectl describe helmrepository podinfo` to see the [Artifact](#artifact) + and [Conditions](#conditions) in the HelmRepository's Status: + + ```console + ... + Status: + Artifact: + Checksum: 83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111 + Last Update Time: 2022-02-04T09:55:58Z + Path: helmrepository/default/podinfo/index-83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111.yaml + Revision: 83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111 + URL: http://source-controller.flux-system.svc.cluster.local./helmrepository/default/podinfo/index-83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111.yaml + Conditions: + Last Transition Time: 2022-02-04T09:55:58Z + Message: stored artifact for revision '83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111' + Observed Generation: 1 + Reason: Succeeded + Status: True + Type: Ready + Observed Generation: 1 + URL: http://source-controller.flux-system.svc.cluster.local./helmrepository/default/podinfo/index.yaml + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal NewArtifact 1m source-controller stored artifact for revision '83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111' + ``` + +## Writing a HelmRepository spec + +As with all other Kubernetes config, a HelmRepository needs `apiVersion`, +`kind`, and `metadata` fields. The name of a HelmRepository object must be a +valid [DNS subdomain name](https://kubernetes.io/docs/concepts/overview/working-with-objects/names#dns-subdomain-names). + +A HelmRepository also needs a +[`.spec` section](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status). + +### Interval + +`.spec.interval` is a required field that specifies the interval which the +Helm repository index must be consulted at. + +After successfully reconciling a HelmRepository object, the source-controller +requeues the object for inspection after the specified interval. The value +must be in a [Go recognized duration string format](https://pkg.go.dev/time#ParseDuration), +e.g. `10m0s` to fetch the HelmRepository index YAML every 10 minutes. + +If the `.metadata.generation` of a resource changes (due to e.g. applying a +change to the spec), this is handled instantly outside the interval window. + +### URL + +`.spec.url` is a required field that specifies the HTTP/S address of the Helm +repository. For Helm repositories which require authentication, see +[Secret reference](#secret-reference). + +### Timeout + +`.spec.timeout` is an optional field to specify a timeout for the fetch +operation. The value must be in a +[Go recognized duration string format](https://pkg.go.dev/time#ParseDuration), +e.g. `1m30s` for a timeout of one minute and thirty seconds. The default value +is `60s`. + +### Secret reference + +`.spec.secretRef.name` is an optional field to specify a name reference to a +Secret in the same namespace as the HelmRepository, containing authentication +credentials for the repository. + +#### Basic access authentication + +To authenticate towards a Helm repository using basic access authentication +(in other words: using a username and password), the referenced Secret is +expected to contain `.data.username` and `.data.password` values. + +For example: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: example + namespace: default +spec: + interval: 5m0s + url: https://example.com + secretRef: + name: example-user +--- +apiVersion: v1 +kind: Secret +metadata: + name: example-user + namespace: default +stringData: + username: example + password: 123456 +``` + +#### TLS authentication + +To provide TLS credentials to use while connecting with the Helm repository, +the referenced Secret is expected to contain `.data.certFile` and +`.data.keyFile`, and/or `.data.caFile` values. + +For example: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: example + namespace: default +spec: + interval: 5m0s + url: https://example.com + secretRef: + name: example-tls +--- +apiVersion: v1 +kind: Secret +metadata: + name: example-tls + namespace: default +data: + certFile: + keyFile: + # NOTE: Can be supplied without the above values + caFile: +``` + +### Pass credentials + +`.spec.passCredentials` is an optional field to allow the credentials from the +[Secret reference](#secret-reference) to be passed on to a host that does not +match the host as defined in URL. This may for example be required if the host +advertised chart URLs in the index differ from the specified URL. + +Enabling this should be done with caution, as it can potentially result in +credentials getting stolen in a man-in-the-middle attack. + +### Suspend + +`.spec.suspend` is an optional field to suspend the reconciliation of a +HelmRepository. When set to `true`, the controller will stop reconciling the +HelmRepository, and changes to the resource or the Helm repository index will +not result in a new Artifact. When the field is set to `false` or removed, it +will resume. + +For practical information, see +[suspending and resuming](#suspending-and-resuming). + +## Working with HelmRepositories + +### Triggering a reconcile + +To manually tell the source-controller to reconcile a HelmRepository outside the +[specified interval window](#interval), a HelmRepository can be annotated with +`reconcile.fluxcd.io/requestedAt: `. Annotating the resource +queues the object for reconciliation if the `` differs from +the last value the controller acted on, as reported in +[`.status.lastHandledReconcileAt`](#last-handled-reconcile-at). + +Using `kubectl`: + +```sh +kubectl annotate --overwrite helmrepository/ reconcile.fluxcd.io/requestedAt="$(date +%s)" +``` + +Using `flux`: + +```sh +flux reconcile source helm +``` + +### Waiting for `Ready` + +When a change is applied, it is possible to wait for the HelmRepository to +reach a [ready state](#ready-helmrepository) using `kubectl`: + +```sh +kubectl wait helmrepository/ --for=condition=ready --timeout=1m +``` + +### Suspending and resuming + +When you find yourself in a situation where you temporarily want to pause the +reconciliation of a HelmRepository, you can suspend it using the +[`.spec.suspend` field](#suspend). + +#### Suspend a HelmRepository + +In your YAML declaration: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: +spec: + suspend: true +``` + +Using `kubectl`: + +```sh +kubectl patch helmrepository -p '{\"spec\": {\"suspend\" : true }}' +``` + +Using `flux`: + +```sh +flux suspend source helm +``` + +**Note:** When a HelmRepository has an Artifact and is suspended, and this +Artifact later disappears from the storage due to e.g. the source-controller +Pod being evicted from a Node, this will not be reflected in the +HelmRepository's Status until it is resumed. + +#### Resume a HelmRepository + +In your YAML declaration, comment out (or remove) the field: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: +spec: + # suspend: true +``` + +**Note:** Setting the field value to `false` has the same effect as removing +it, but does not allow for "hot patching" using e.g. `kubectl` while practicing +GitOps; as the manually applied patch would be overwritten by the declared +state in Git. + +Using `kubectl`: + +```sh +kubectl patch helmrepository -p '{\"spec\" : {\"suspend\" : false }}' +``` + +Using `flux`: + +```sh +flux resume source helm +``` + +### Debugging a HelmRepository + +There are several ways to gather information about a HelmRepository for debugging +purposes. + +#### Describe the HelmRepository + +Describing a HelmRepository using `kubectl describe helmrepository ` +displays the latest recorded information for the resource in the `Status` and +`Events` sections: + +```console +... +Status: +... + Conditions: + Last Transition Time: 2022-02-04T13:41:56Z + Message: failed to construct Helm client: scheme "invalid" not supported + Observed Generation: 2 + Reason: Failed + Status: True + Type: Stalled + Last Transition Time: 2022-02-04T13:41:56Z + Message: failed to construct Helm client: scheme "invalid" not supported + Observed Generation: 2 + Reason: Failed + Status: False + Type: Ready + Last Transition Time: 2022-02-04T13:41:56Z + Message: failed to construct Helm client: scheme "invalid" not supported + Observed Generation: 2 + Reason: Failed + Status: True + Type: FetchFailed + Observed Generation: 2 + URL: http://source-controller.source-system.svc.cluster.local./helmrepository/default/podinfo/index.yaml +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Failed 6s source-controller failed to construct Helm client: scheme "invalid" not supported +``` + +#### Trace emitted Events + +To view events for specific HelmRepository(s), `kubectl get events` can be used in +combination with `--field-sector` to list the Events for specific objects. +For example, running + +```sh +kubectl get events --field-selector involvedObject.kind=HelmRepository,involvedObject.name= +``` + +lists + +```console +LAST SEEN TYPE REASON OBJECT MESSAGE +107s Warning Failed helmrepository/ failed to construct Helm client: scheme "invalid" not supported +7s Normal NewArtifact helmrepository/ stored artifact for revision '83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111' +``` + +Besides being reported in Events, the reconciliation errors are also logged by +the controller. The Flux CLI offer commands for filtering the logs for a +specific HelmRepository, e.g. `flux logs --level=error --kind=HelmRepository --name=`. + +## HelmRepository Status + +### Artifact + +The HelmRepository reports the last fetched repository index as an Artifact +object in the `.status.artifact` of the resource. + +The Artifact file is an exact copy of the Helm repository index YAML +(`index-.yaml`) as fetched, and can be retrieved in-cluster from the +`.status.artifact.url` HTTP address. + +#### Artifact example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: +status: + artifact: + checksum: 83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111 + lastUpdateTime: "2022-02-04T09:55:58Z" + path: helmrepository///index-83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111.yaml + revision: 83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111 + url: http://source-controller.flux-system.svc.cluster.local./helmrepository///index-83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111.yaml +``` + +### Conditions + +A HelmRepository enters various states during its lifecycle, reflected as [Kubernetes +Conditions][typical-status-properties]. +It can be [reconciling](#reconciling-helmrepository) while fetching the +repository index, it can be [ready](#ready-helmrepository), it can +[fail during reconciliation](#failed-helmrepository), or it can +[stall](#stalled-helmrepository). + +The HelmRepository API is compatible with the [kstatus +specification][kstatus-spec], +and reports `Reconciling` and `Stalled` conditions where applicable to +provide better (timeout) support to solutions polling the HelmRepository to become +`Ready`. + +#### Reconciling HelmRepository + +The source-controller marks a HelmRepository as _reconciling_ when one of the following +is true: + +- There is no current Artifact for the HelmRepository, or the reported Artifact + is determined to have disappeared from the storage. +- The generation of the HelmRepository is newer than the [Observed + Generation](#observed-generation). +- The newly fetched Artifact revision differs from the current Artifact. + +When the HelmRepository is "reconciling", the `Ready` Condition status becomes +`False`, and the controller adds a Condition with the following attributes to +the HelmRepository's `.status.conditions`: + +- `type: Reconciling` +- `status: "True"` +- `reason: NewGeneration` | `reason: NoArtifact` | `reason: NewRevision` + +If the reconciling state is due to a new revision, it adds an additional +Condition with the following attributes: + +- `type: ArtifactOutdated` +- `status: "True"` +- `reason: NewRevision` + +Both Conditions have a ["negative polarity"][typical-status-properties], +and are only present on the HelmRepository while their status value is `"True"`. + +#### Ready HelmRepository + +The source-controller marks a HelmRepository as _ready_ when it has the following +characteristics: + +- The HelmRepository reports an [Artifact](#artifact). +- The reported Artifact exists in the controller's Artifact storage. +- The controller was able to fetch the Helm repository index using the current + spec. +- The revision of the reported Artifact is up-to-date with the latest + revision of the Helm repository. + +When the HelmRepository is "ready", the controller sets a Condition with the following +attributes in the HelmRepository's `.status.conditions`: + +- `type: Ready` +- `status: "True"` +- `reason: Succeeded` + +This `Ready` Condition will retain a status value of `"True"` until the +HelmRepository is marked as [reconciling](#reconciling-helmrepository), or e.g. +a [transient error](#failed-helmrepository) occurs due to a temporary network +issue. + +#### Failed HelmRepository + +The source-controller may get stuck trying to produce an Artifact for a +HelmRepository without completing. This can occur due to some of the following +factors: + +- The Helm repository [URL](#url) is temporarily unavailable. +- The [Secret reference](#secret-reference) contains a reference to a + non-existing Secret. +- The credentials in the referenced Secret are invalid. +- The HelmRepository spec contains a generic misconfiguration. + +When this happens, the controller sets the `Ready` Condition status to `False`, +and adds a Condition with the following attributes to the HelmRepository's +`.status.conditions`: + +- `type: FetchFailed` +- `status: "True"` +- `reason: AuthenticationFailed` | `reason: IndexationFailed` | `reason: Failed` + +This condition has a ["negative polarity"][typical-status-properties], +and is only present on the HelmRepository while the status value is `"True"`. + +While the HelmRepository has this Condition, the controller will continue to +attempt to produce an Artifact for the resource with an exponential backoff, +until it succeeds and the HelmRepository is marked as [ready](#ready-helmrepository). + +Note that a HelmRepository can be [reconciling](#reconciling-helmrepository) +while failing at the same time, for example due to a newly introduced +configuration issue in the HelmRepository spec. + +#### Stalled HelmRepository + +The source-controller can mark a HelmRepository as _stalled_ when it determines +that without changes to the spec, the reconciliation can not succeed. +For example because a Helm repository URL with an unsupported protocol is +specified. + +When this happens, the controller sets the same Conditions as when it +[fails](#failed-helmrepository), but adds another Condition with the following +attributes to the HelmRepository's +`.status.conditions`: + +- `type: Stalled` +- `status: "True"` +- `reason: URLInvalid` + +While the HelmRepository has this Condition, the controller will not requeue +the resource any further, and will stop reconciling the resource until a change +to the spec is made. + +### Observed Generation + +The source-controller reports an [observed generation][typical-status-properties] +in the HelmRepository's `.status.observedGeneration`. The observed generation is +the latest `.metadata.generation` which resulted in either a [ready state](#ready-helmrepository), +or stalled due to error it can not recover from without human intervention. + +### Last Handled Reconcile At + +The source-controller reports the last `reconcile.fluxcd.io/requestedAt` +annotation value it acted on in the `.status.lastHandledReconcileAt` field. + +For practical information about this field, see [triggering a +reconcile](#triggering-a-reconcile). + +[typical-status-properties]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties +[kstatus-spec]: https://github.com/kubernetes-sigs/cli-utils/tree/master/pkg/kstatus From cbffd82d8fa49768fa4ac83040892e4724f192cc Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 8 Feb 2022 16:51:27 +0100 Subject: [PATCH 3/9] Document GitRepository API v1beta2 spec Signed-off-by: Hidde Beydals --- api/v1beta2/gitrepository_types.go | 165 ++-- ...rce.toolkit.fluxcd.io_gitrepositories.yaml | 112 +-- controllers/gitrepository_controller.go | 184 ++-- controllers/gitrepository_controller_test.go | 2 +- docs/api/source.md | 146 ++-- docs/spec/v1beta2/README.md | 2 +- docs/spec/v1beta2/gitrepositories.md | 814 ++++++++++++++++++ 7 files changed, 1173 insertions(+), 252 deletions(-) create mode 100644 docs/spec/v1beta2/gitrepositories.md diff --git a/api/v1beta2/gitrepository_types.go b/api/v1beta2/gitrepository_types.go index c88f08f4c..06e951da5 100644 --- a/api/v1beta2/gitrepository_types.go +++ b/api/v1beta2/gitrepository_types.go @@ -29,85 +29,117 @@ const ( // GitRepositoryKind is the string representation of a GitRepository. GitRepositoryKind = "GitRepository" - // GoGitImplementation represents the go-git Git implementation kind. + // GoGitImplementation for performing Git operations using go-git. GoGitImplementation = "go-git" - // LibGit2Implementation represents the git2go Git implementation kind. + // LibGit2Implementation for performing Git operations using libgit2. LibGit2Implementation = "libgit2" ) const ( - // IncludeUnavailableCondition indicates one of the includes is not available. For example, because it does not - // exist, or does not have an Artifact. - // This is a "negative polarity" or "abnormal-true" type, and is only present on the resource if it is True. + // IncludeUnavailableCondition indicates one of the includes is not + // available. For example, because it does not exist, or does not have an + // Artifact. + // This is a "negative polarity" or "abnormal-true" type, and is only + // present on the resource if it is True. IncludeUnavailableCondition string = "IncludeUnavailable" ) -// GitRepositorySpec defines the desired state of a Git repository. +// GitRepositorySpec specifies the required configuration to produce an +// Artifact for a Git repository. type GitRepositorySpec struct { - // The repository URL, can be a HTTP/S or SSH address. + // URL specifies the Git repository URL, it can be an HTTP/S or SSH address. // +kubebuilder:validation:Pattern="^(http|https|ssh)://" // +required URL string `json:"url"` - // The secret name containing the Git credentials. - // For HTTPS repositories the secret must contain username and password fields. - // For SSH repositories the secret must contain 'identity', 'identity.pub' and 'known_hosts' fields. + // SecretRef specifies the Secret containing authentication credentials for + // the GitRepository. + // For HTTPS repositories the Secret must contain 'username' and 'password' + // fields. + // For SSH repositories the Secret must contain 'identity', 'identity.pub' + // and 'known_hosts' fields. // +optional SecretRef *meta.LocalObjectReference `json:"secretRef,omitempty"` - // The interval at which to check for repository updates. + // Interval at which to check the GitRepository for updates. // +required Interval metav1.Duration `json:"interval"` - // The timeout for remote Git operations like cloning, defaults to 60s. + // Timeout for Git operations like cloning, defaults to 60s. // +kubebuilder:default="60s" // +optional Timeout *metav1.Duration `json:"timeout,omitempty"` - // The Git reference to checkout and monitor for changes, defaults to - // master branch. + // Reference specifies the Git reference to resolve and monitor for + // changes, defaults to the 'master' branch. // +optional Reference *GitRepositoryRef `json:"ref,omitempty"` - // Verification defines the configuration to verify the OpenPGP signature for the Git commit HEAD points to. + // Verification specifies the configuration to verify the Git commit + // signature(s). // +optional Verification *GitRepositoryVerification `json:"verify,omitempty"` - // Ignore overrides the set of excluded patterns in the .sourceignore format (which is the same as .gitignore). - // If not provided, a default will be used, consult the documentation for your version to find out what those are. + // Ignore overrides the set of excluded patterns in the .sourceignore format + // (which is the same as .gitignore). If not provided, a default will be used, + // consult the documentation for your version to find out what those are. // +optional Ignore *string `json:"ignore,omitempty"` - // Suspend tells the controller to suspend the reconciliation of this source. - // This flag tells the controller to suspend the reconciliation of this source. + // Suspend tells the controller to suspend the reconciliation of this + // GitRepository. // +optional Suspend bool `json:"suspend,omitempty"` - // Determines which git client library to use. - // Defaults to go-git, valid values are ('go-git', 'libgit2'). + // GitImplementation specifies which Git client library implementation to + // use. Defaults to 'go-git', valid values are ('go-git', 'libgit2'). // +kubebuilder:validation:Enum=go-git;libgit2 // +kubebuilder:default:=go-git // +optional GitImplementation string `json:"gitImplementation,omitempty"` - // When enabled, after the clone is created, initializes all submodules within, using their default settings. + // RecurseSubmodules enables the initialization of all submodules within + // the GitRepository as cloned from the URL, using their default settings. // This option is available only when using the 'go-git' GitImplementation. // +optional RecurseSubmodules bool `json:"recurseSubmodules,omitempty"` - // Include defines a list of GitRepository resources which artifacts should be included in the artifact produced for - // this resource. + // Include specifies a list of GitRepository resources which Artifacts + // should be included in the Artifact produced for this GitRepository. Include []GitRepositoryInclude `json:"include,omitempty"` - // AccessFrom defines an Access Control List for allowing cross-namespace references to this object. + // AccessFrom specifies an Access Control List for allowing cross-namespace + // references to this object. + // NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092 // +optional AccessFrom *acl.AccessFrom `json:"accessFrom,omitempty"` } +// GitRepositoryInclude specifies a local reference to a GitRepository which +// Artifact (sub-)contents must be included, and where they should be placed. +type GitRepositoryInclude struct { + // GitRepositoryRef specifies the GitRepository which Artifact contents + // must be included. + GitRepositoryRef meta.LocalObjectReference `json:"repository"` + + // FromPath specifies the path to copy contents from, defaults to the root + // of the Artifact. + // +optional + FromPath string `json:"fromPath"` + + // ToPath specifies the path to copy contents to, defaults to the name of + // the GitRepositoryRef. + // +optional + ToPath string `json:"toPath"` +} + +// GetFromPath returns the specified FromPath. func (in *GitRepositoryInclude) GetFromPath() string { return in.FromPath } +// GetToPath returns the specified ToPath, falling back to the name of the +// GitRepositoryRef. func (in *GitRepositoryInclude) GetToPath() string { if in.ToPath == "" { return in.GitRepositoryRef.Name @@ -115,52 +147,48 @@ func (in *GitRepositoryInclude) GetToPath() string { return in.ToPath } -// GitRepositoryInclude defines a source with a from and to path. -type GitRepositoryInclude struct { - // Reference to a GitRepository to include. - GitRepositoryRef meta.LocalObjectReference `json:"repository"` - - // The path to copy contents from, defaults to the root directory. - // +optional - FromPath string `json:"fromPath"` - - // The path to copy contents to, defaults to the name of the source ref. - // +optional - ToPath string `json:"toPath"` -} - -// GitRepositoryRef defines the Git ref used for pull and checkout operations. +// GitRepositoryRef specifies the Git reference to resolve and checkout. type GitRepositoryRef struct { - // The Git branch to checkout, defaults to master. + // Branch to check out, defaults to 'master' if no other field is defined. + // + // When GitRepositorySpec.GitImplementation is set to 'go-git', a shallow + // clone of the specified branch is performed. // +optional Branch string `json:"branch,omitempty"` - // The Git tag to checkout, takes precedence over Branch. + // Tag to check out, takes precedence over Branch. // +optional Tag string `json:"tag,omitempty"` - // The Git tag semver expression, takes precedence over Tag. + // SemVer tag expression to check out, takes precedence over Tag. // +optional SemVer string `json:"semver,omitempty"` - // The Git commit SHA to checkout, if specified Tag filters will be ignored. + // Commit SHA to check out, takes precedence over all reference fields. + // + // When GitRepositorySpec.GitImplementation is set to 'go-git', this can be + // combined with Branch to shallow clone the branch, in which the commit is + // expected to exist. // +optional Commit string `json:"commit,omitempty"` } -// GitRepositoryVerification defines the OpenPGP signature verification process. +// GitRepositoryVerification specifies the Git commit signature verification +// strategy. type GitRepositoryVerification struct { - // Mode describes what Git object should be verified, currently ('head'). + // Mode specifies what Git object should be verified, currently ('head'). // +kubebuilder:validation:Enum=head Mode string `json:"mode"` - // SecretRef containing the public keys of all trusted Git authors. + // SecretRef specifies the Secret containing the public keys of trusted Git + // authors. SecretRef meta.LocalObjectReference `json:"secretRef,omitempty"` } -// GitRepositoryStatus defines the observed state of a Git repository. +// GitRepositoryStatus records the observed state of a Git repository. type GitRepositoryStatus struct { - // ObservedGeneration is the last observed generation. + // ObservedGeneration is the last observed generation of the GitRepository + // object. // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` @@ -168,15 +196,18 @@ type GitRepositoryStatus struct { // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` - // URL is the fetch link for the artifact output of the last repository sync. + // URL is the dynamic fetch link for the latest Artifact. + // It is provided on a "best effort" basis, and using the precise + // GitRepositoryStatus.Artifact data is recommended. // +optional URL string `json:"url,omitempty"` - // Artifact represents the output of the last successful repository sync. + // Artifact represents the last successful GitRepository reconciliation. // +optional Artifact *Artifact `json:"artifact,omitempty"` - // IncludedArtifacts represents the included artifacts from the last successful repository sync. + // IncludedArtifacts contains a list of the last successfully included + // Artifacts as instructed by GitRepositorySpec.Include. // +optional IncludedArtifacts []*Artifact `json:"includedArtifacts,omitempty"` @@ -184,10 +215,12 @@ type GitRepositoryStatus struct { } const ( - // GitOperationSucceedReason represents the fact that the git clone, pull and checkout operations succeeded. - GitOperationSucceedReason string = "GitOperationSucceed" + // GitOperationSucceedReason signals that a Git operation (e.g. clone, + // checkout, etc.) succeeded. + GitOperationSucceedReason string = "GitOperationSucceeded" - // GitOperationFailedReason represents the fact that the git clone, pull or checkout operations failed. + // GitOperationFailedReason signals that a Git operation (e.g. clone, + // checkout, etc.) failed. GitOperationFailedReason string = "GitOperationFailed" ) @@ -201,28 +234,18 @@ func (in *GitRepository) SetConditions(conditions []metav1.Condition) { in.Status.Conditions = conditions } -// GetRequeueAfter returns the duration after which the source must be reconciled again. +// GetRequeueAfter returns the duration after which the GitRepository must be +// reconciled again. func (in GitRepository) GetRequeueAfter() time.Duration { return in.Spec.Interval.Duration } -// GetInterval returns the interval at which the source is reconciled. -// Deprecated: use GetRequeueAfter instead. -func (in GitRepository) GetInterval() metav1.Duration { - return in.Spec.Interval -} - -// GetArtifact returns the latest artifact from the source if present in the status sub-resource. +// GetArtifact returns the latest Artifact from the GitRepository if present in +// the status sub-resource. func (in *GitRepository) GetArtifact() *Artifact { return in.Status.Artifact } -// GetStatusConditions returns a pointer to the Status.Conditions slice. -// Deprecated: use GetConditions instead. -func (in *GitRepository) GetStatusConditions() *[]metav1.Condition { - return &in.Status.Conditions -} - // +genclient // +genclient:Namespaced // +kubebuilder:storageversion @@ -234,7 +257,7 @@ func (in *GitRepository) GetStatusConditions() *[]metav1.Condition { // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="" // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="" -// GitRepository is the Schema for the gitrepositories API +// GitRepository is the Schema for the gitrepositories API. type GitRepository struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -244,7 +267,7 @@ type GitRepository struct { Status GitRepositoryStatus `json:"status,omitempty"` } -// GitRepositoryList contains a list of GitRepository +// GitRepositoryList contains a list of GitRepository objects. // +kubebuilder:object:root=true type GitRepositoryList struct { metav1.TypeMeta `json:",inline"` diff --git a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml index 113b62b1b..612b2a048 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml @@ -365,7 +365,7 @@ spec: name: v1beta2 schema: openAPIV3Schema: - description: GitRepository is the Schema for the gitrepositories API + description: GitRepository is the Schema for the gitrepositories API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -380,11 +380,13 @@ spec: metadata: type: object spec: - description: GitRepositorySpec defines the desired state of a Git repository. + description: GitRepositorySpec specifies the required configuration to + produce an Artifact for a Git repository. properties: accessFrom: - description: AccessFrom defines an Access Control List for allowing - cross-namespace references to this object. + description: 'AccessFrom specifies an Access Control List for allowing + cross-namespace references to this object. NOTE: Not implemented, + provisional as of https://github.com/fluxcd/flux2/pull/2092' properties: namespaceSelectors: description: NamespaceSelectors is the list of namespace selectors @@ -411,8 +413,9 @@ spec: type: object gitImplementation: default: go-git - description: Determines which git client library to use. Defaults - to go-git, valid values are ('go-git', 'libgit2'). + description: GitImplementation specifies which Git client library + implementation to use. Defaults to 'go-git', valid values are ('go-git', + 'libgit2'). enum: - go-git - libgit2 @@ -424,18 +427,20 @@ spec: to find out what those are. type: string include: - description: Include defines a list of GitRepository resources which - artifacts should be included in the artifact produced for this resource. + description: Include specifies a list of GitRepository resources which + Artifacts should be included in the Artifact produced for this GitRepository. items: - description: GitRepositoryInclude defines a source with a from and - to path. + description: GitRepositoryInclude specifies a local reference to + a GitRepository which Artifact (sub-)contents must be included, + and where they should be placed. properties: fromPath: - description: The path to copy contents from, defaults to the - root directory. + description: FromPath specifies the path to copy contents from, + defaults to the root of the Artifact. type: string repository: - description: Reference to a GitRepository to include. + description: GitRepositoryRef specifies the GitRepository which + Artifact contents must be included. properties: name: description: Name of the referent. @@ -444,45 +449,52 @@ spec: - name type: object toPath: - description: The path to copy contents to, defaults to the name - of the source ref. + description: ToPath specifies the path to copy contents to, + defaults to the name of the GitRepositoryRef. type: string required: - repository type: object type: array interval: - description: The interval at which to check for repository updates. + description: Interval at which to check the GitRepository for updates. type: string recurseSubmodules: - description: When enabled, after the clone is created, initializes - all submodules within, using their default settings. This option - is available only when using the 'go-git' GitImplementation. + description: RecurseSubmodules enables the initialization of all submodules + within the GitRepository as cloned from the URL, using their default + settings. This option is available only when using the 'go-git' + GitImplementation. type: boolean ref: - description: The Git reference to checkout and monitor for changes, - defaults to master branch. + description: Reference specifies the Git reference to resolve and + monitor for changes, defaults to the 'master' branch. properties: branch: - description: The Git branch to checkout, defaults to master. + description: "Branch to check out, defaults to 'master' if no + other field is defined. \n When GitRepositorySpec.GitImplementation + is set to 'go-git', a shallow clone of the specified branch + is performed." type: string commit: - description: The Git commit SHA to checkout, if specified Tag - filters will be ignored. + description: "Commit SHA to check out, takes precedence over all + reference fields. \n When GitRepositorySpec.GitImplementation + is set to 'go-git', this can be combined with Branch to shallow + clone the branch, in which the commit is expected to exist." type: string semver: - description: The Git tag semver expression, takes precedence over - Tag. + description: SemVer tag expression to check out, takes precedence + over Tag. type: string tag: - description: The Git tag to checkout, takes precedence over Branch. + description: Tag to check out, takes precedence over Branch. type: string type: object secretRef: - description: The secret name containing the Git credentials. For HTTPS - repositories the secret must contain username and password fields. - For SSH repositories the secret must contain 'identity', 'identity.pub' - and 'known_hosts' fields. + description: SecretRef specifies the Secret containing authentication + credentials for the GitRepository. For HTTPS repositories the Secret + must contain 'username' and 'password' fields. For SSH repositories + the Secret must contain 'identity', 'identity.pub' and 'known_hosts' + fields. properties: name: description: Name of the referent. @@ -492,31 +504,31 @@ spec: type: object suspend: description: Suspend tells the controller to suspend the reconciliation - of this source. This flag tells the controller to suspend the reconciliation - of this source. + of this GitRepository. type: boolean timeout: default: 60s - description: The timeout for remote Git operations like cloning, defaults - to 60s. + description: Timeout for Git operations like cloning, defaults to + 60s. type: string url: - description: The repository URL, can be a HTTP/S or SSH address. + description: URL specifies the Git repository URL, it can be an HTTP/S + or SSH address. pattern: ^(http|https|ssh):// type: string verify: - description: Verification defines the configuration to verify the - OpenPGP signature for the Git commit HEAD points to. + description: Verification specifies the configuration to verify the + Git commit signature(s). properties: mode: - description: Mode describes what Git object should be verified, + description: Mode specifies what Git object should be verified, currently ('head'). enum: - head type: string secretRef: - description: SecretRef containing the public keys of all trusted - Git authors. + description: SecretRef specifies the Secret containing the public + keys of trusted Git authors. properties: name: description: Name of the referent. @@ -534,11 +546,11 @@ spec: status: default: observedGeneration: -1 - description: GitRepositoryStatus defines the observed state of a Git repository. + description: GitRepositoryStatus records the observed state of a Git repository. properties: artifact: - description: Artifact represents the output of the last successful - repository sync. + description: Artifact represents the last successful GitRepository + reconciliation. properties: checksum: description: Checksum is the SHA256 checksum of the Artifact file. @@ -643,8 +655,8 @@ spec: type: object type: array includedArtifacts: - description: IncludedArtifacts represents the included artifacts from - the last successful repository sync. + description: IncludedArtifacts contains a list of the last successfully + included Artifacts as instructed by GitRepositorySpec.Include. items: description: Artifact represents the output of a Source reconciliation. properties: @@ -689,12 +701,14 @@ spec: be detected. type: string observedGeneration: - description: ObservedGeneration is the last observed generation. + description: ObservedGeneration is the last observed generation of + the GitRepository object. format: int64 type: integer url: - description: URL is the fetch link for the artifact output of the - last repository sync. + description: URL is the dynamic fetch link for the latest Artifact. + It is provided on a "best effort" basis, and using the precise GitRepositoryStatus.Artifact + data is recommended. type: string type: object type: object diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index f53a835d5..83f9f931c 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -54,9 +54,9 @@ import ( "github.com/fluxcd/source-controller/pkg/sourceignore" ) -// gitRepoReadyConditions contains all the conditions information needed -// for GitRepository Ready status conditions summary calculation. -var gitRepoReadyConditions = summarize.Conditions{ +// gitRepositoryReadyCondition contains the information required to summarize a +// v1beta2.GitRepository Ready Condition. +var gitRepositoryReadyCondition = summarize.Conditions{ Target: meta.ReadyCondition, Owned: []string{ sourcev1.SourceVerifiedCondition, @@ -89,7 +89,7 @@ var gitRepoReadyConditions = summarize.Conditions{ // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=gitrepositories/finalizers,verbs=get;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch -// GitRepositoryReconciler reconciles a GitRepository object +// GitRepositoryReconciler reconciles a v1beta2.GitRepository object. type GitRepositoryReconciler struct { client.Client kuberecorder.EventRecorder @@ -106,9 +106,9 @@ type GitRepositoryReconcilerOptions struct { DependencyRequeueInterval time.Duration } -// gitRepoReconcilerFunc is the function type for all the Git repository -// reconciler functions. -type gitRepoReconcilerFunc func(ctx context.Context, obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) +// gitRepositoryReconcileFunc is the function type for all the +// v1beta2.GitRepository (sub)reconcile functions. +type gitRepositoryReconcileFunc func(ctx context.Context, obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) func (r *GitRepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error { return r.SetupWithManagerAndOptions(mgr, GitRepositoryReconcilerOptions{}) @@ -158,7 +158,7 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques defer func() { summarizeHelper := summarize.NewHelper(r.EventRecorder, patchHelper) summarizeOpts := []summarize.Option{ - summarize.WithConditions(gitRepoReadyConditions), + summarize.WithConditions(gitRepositoryReadyCondition), summarize.WithReconcileResult(recResult), summarize.WithReconcileError(retErr), summarize.WithIgnoreNotFound(), @@ -166,7 +166,7 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques summarize.RecordContextualError, summarize.RecordReconcileReq, ), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetInterval().Duration}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), summarize.WithPatchFieldOwner(r.ControllerName), } result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) @@ -191,7 +191,7 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // Reconcile actual object - reconcilers := []gitRepoReconcilerFunc{ + reconcilers := []gitRepositoryReconcileFunc{ r.reconcileStorage, r.reconcileSource, r.reconcileInclude, @@ -201,17 +201,15 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques return } -// reconcile steps iterates through the actual reconciliation tasks for objec, -// it returns early on the first step that returns ResultRequeue or produces an -// error. -func (r *GitRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.GitRepository, reconcilers []gitRepoReconcilerFunc) (sreconcile.Result, error) { +// reconcile iterates through the gitRepositoryReconcileFunc tasks for the +// object. It returns early on the first call that returns +// reconcile.ResultRequeue, or produces an error. +func (r *GitRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.GitRepository, reconcilers []gitRepositoryReconcileFunc) (sreconcile.Result, error) { + // Mark as reconciling if generation differs if obj.Generation != obj.Status.ObservedGeneration { conditions.MarkReconciling(obj, "NewGeneration", "reconciling new object generation (%d)", obj.Generation) } - var commit git.Commit - var includes artifactSet - // Create temp dir for Git clone tmpDir, err := util.TempDirForObj("", obj) if err != nil { @@ -220,11 +218,20 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.G Reason: sourcev1.StorageOperationFailedReason, } } - defer os.RemoveAll(tmpDir) + defer func() { + if err = os.RemoveAll(tmpDir); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "failed to remove temporary working directory") + } + }() // Run the sub-reconcilers and build the result of reconciliation. - var res sreconcile.Result - var resErr error + var ( + commit git.Commit + includes artifactSet + + res sreconcile.Result + resErr error + ) for _, rec := range reconcilers { recResult, err := rec(ctx, obj, &commit, &includes, tmpDir) // Exit immediately on ResultRequeue. @@ -244,14 +251,19 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.G return res, resErr } -// reconcileStorage ensures the current state of the storage matches the desired and previously observed state. +// reconcileStorage ensures the current state of the storage matches the +// desired and previously observed state. // -// All artifacts for the resource except for the current one are garbage collected from the storage. -// If the artifact in the Status object of the resource disappeared from storage, it is removed from the object. -// If the object does not have an artifact in its Status object, a v1beta1.ArtifactUnavailableCondition is set. -// If the hostname of any of the URLs on the object do not match the current storage server hostname, they are updated. +// All Artifacts for the object except for the current one in the Status are +// garbage collected from the Storage. +// If the Artifact in the Status of the object disappeared from the Storage, +// it is removed from the object. +// If the object does not have an Artifact in its Status, a Reconciling +// condition is added. +// The hostname of any URL in the Status of the object are updated, to ensure +// they match the Storage server hostname of current runtime. func (r *GitRepositoryReconciler) reconcileStorage(ctx context.Context, - obj *sourcev1.GitRepository, _ *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) { + obj *sourcev1.GitRepository, _ *git.Commit, _ *artifactSet, _ string) (sreconcile.Result, error) { // Garbage collect previous advertised artifact(s) from storage _ = r.garbageCollect(ctx, obj) @@ -275,17 +287,24 @@ func (r *GitRepositoryReconciler) reconcileStorage(ctx context.Context, return sreconcile.ResultSuccess, nil } -// reconcileSource ensures the upstream Git repository can be reached and checked out using the declared configuration, -// and observes its state. +// reconcileSource ensures the upstream Git repository and reference can be +// cloned and checked out using the specified configuration, and observes its +// state. // -// The repository is checked out to the given dir using the defined configuration, and in case of an error during the -// checkout process (including transient errors), it records v1beta1.FetchFailedCondition=True and returns early. -// On a successful checkout it removes v1beta1.FetchFailedCondition, and compares the current revision of HEAD to the -// artifact on the object, and records v1beta1.ArtifactOutdatedCondition if they differ. -// If instructed, the signature of the commit is verified if and recorded as v1beta1.SourceVerifiedCondition. If the -// signature can not be verified or the verification fails, the Condition=False and it returns early. -// If both the checkout and signature verification are successful, the given artifact pointer is set to a new artifact -// with the available metadata. +// The repository is cloned to the given dir, using the specified configuration +// to check out the reference. In case of an error during this process +// (including transient errors), it records v1beta2.FetchFailedCondition=True +// and returns early. +// On a successful checkout, it removes v1beta2.FetchFailedCondition and +// compares the current revision of HEAD to the revision of the Artifact in the +// Status of the object. It records v1beta2.ArtifactOutdatedCondition=True when +// they differ. +// If specified, the signature of the Git commit is verified. If the signature +// can not be verified or the verification fails, it records +// v1beta2.SourceVerifiedCondition=False and returns early. When successful, +// it records v1beta2.SourceVerifiedCondition=True. +// When all the above is successful, the given Commit pointer is set to the +// commit of the checked out Git repository. func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, obj *sourcev1.GitRepository, commit *git.Commit, _ *artifactSet, dir string) (sreconcile.Result, error) { // Configure authentication strategy to access the source @@ -376,15 +395,17 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, return sreconcile.ResultSuccess, nil } -// reconcileArtifact archives a new artifact to the storage, if the current observation on the object does not match the -// given data. +// reconcileArtifact archives a new Artifact to the Storage, if the current +// (Status) data on the object does not match the given. // -// The inspection of the given data to the object is differed, ensuring any stale observations as -// v1beta1.ArtifactUnavailableCondition and v1beta1.ArtifactOutdatedCondition are always deleted. -// If the given artifact and/or includes do not differ from the object's current, it returns early. -// Source ignore patterns are loaded, and the given directory is archived. -// On a successful archive, the artifact and includes in the status of the given object are set, and the symlink in the -// storage is updated to its path. +// The inspection of the given data to the object is differed, ensuring any +// stale observations like v1beta2.ArtifactOutdatedCondition are removed. +// If the given Artifact and/or artifactSet (includes) do not differ from the +// object's current, it returns early. +// Source ignore patterns are loaded, and the given directory is archived while +// taking these patterns into account. +// On a successful archive, the Artifact and Includes in the Status of the +// object are set, and the symlink in the Storage is updated to its path. func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) { // Create potential new artifact with current available metadata @@ -477,14 +498,19 @@ func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context, return sreconcile.ResultSuccess, nil } -// reconcileInclude reconciles the declared includes from the object by copying their artifact (sub)contents to the -// declared paths in the given directory. +// reconcileInclude reconciles the on the object specified +// v1beta2.GitRepositoryInclude list by copying their Artifact (sub)contents to +// the specified paths in the given directory. // -// If an include is unavailable, it marks the object with v1beta1.IncludeUnavailableCondition and returns early. -// If the copy operations are successful, it deletes the v1beta1.IncludeUnavailableCondition from the object. -// If the artifactSet differs from the current set, it marks the object with v1beta1.ArtifactOutdatedCondition. +// When one of the includes is unavailable, it marks the object with +// v1beta2.IncludeUnavailableCondition=True and returns early. +// When the copy operations are successful, it removes the +// v1beta2.IncludeUnavailableCondition from the object. +// When the composed artifactSet differs from the current set in the Status of +// the object, it marks the object with v1beta2.ArtifactOutdatedCondition=True. func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context, obj *sourcev1.GitRepository, _ *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) { + artifacts := make(artifactSet, len(obj.Spec.Include)) for i, incl := range obj.Spec.Include { // Do this first as it is much cheaper than copy operations @@ -546,25 +572,16 @@ func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context, return sreconcile.ResultSuccess, nil } -// reconcileDelete handles the delete of an object. It first garbage collects all artifacts for the object from the -// artifact storage, if successful, the finalizer is removed from the object. -func (r *GitRepositoryReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.GitRepository) (sreconcile.Result, error) { - // Garbage collect the resource's artifacts - if err := r.garbageCollect(ctx, obj); err != nil { - // Return the error so we retry the failed garbage collection - return sreconcile.ResultEmpty, err - } - - // Remove our finalizer from the list - controllerutil.RemoveFinalizer(obj, sourcev1.SourceFinalizer) - - // Stop reconciliation as the object is being deleted - return sreconcile.ResultEmpty, nil -} - -// verifyCommitSignature verifies the signature of the given commit if a verification mode is configured on the object. +// verifyCommitSignature verifies the signature of the given Git commit, if a +// verification mode is specified on the object. +// If the signature can not be verified or the verification fails, it records +// v1beta2.SourceVerifiedCondition=False and returns. +// When successful, it records v1beta2.SourceVerifiedCondition=True. +// If no verification mode is specified on the object, the +// v1beta2.SourceVerifiedCondition Condition is removed. func (r *GitRepositoryReconciler) verifyCommitSignature(ctx context.Context, obj *sourcev1.GitRepository, commit git.Commit) (sreconcile.Result, error) { - // Check if there is a commit verification is configured and remove any old observations if there is none + // Check if there is a commit verification is configured and remove any old + // observations if there is none if obj.Spec.Verification == nil || obj.Spec.Verification.Mode == "" { conditions.Delete(obj, sourcev1.SourceVerifiedCondition) return sreconcile.ResultSuccess, nil @@ -607,9 +624,28 @@ func (r *GitRepositoryReconciler) verifyCommitSignature(ctx context.Context, obj return sreconcile.ResultSuccess, nil } -// garbageCollect performs a garbage collection for the given v1beta1.GitRepository. It removes all but the current -// artifact except for when the deletion timestamp is set, which will result in the removal of all artifacts for the -// resource. +// reconcileDelete handles the deletion of the object. +// It first garbage collects all Artifacts for the object from the Storage. +// Removing the finalizer from the object if successful. +func (r *GitRepositoryReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.GitRepository) (sreconcile.Result, error) { + // Garbage collect the resource's artifacts + if err := r.garbageCollect(ctx, obj); err != nil { + // Return the error so we retry the failed garbage collection + return sreconcile.ResultEmpty, err + } + + // Remove our finalizer from the list + controllerutil.RemoveFinalizer(obj, sourcev1.SourceFinalizer) + + // Stop reconciliation as the object is being deleted + return sreconcile.ResultEmpty, nil +} + +// garbageCollect performs a garbage collection for the given object. +// +// It removes all but the current Artifact from the Storage, unless the +// deletion timestamp on the object is set. Which will result in the +// removal of all Artifacts for the objects. func (r *GitRepositoryReconciler) garbageCollect(ctx context.Context, obj *sourcev1.GitRepository) error { if !obj.DeletionTimestamp.IsZero() { if deleted, err := r.Storage.RemoveAll(r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), "", "*")); err != nil { @@ -637,9 +673,11 @@ func (r *GitRepositoryReconciler) garbageCollect(ctx context.Context, obj *sourc return nil } -// eventLog records event and logs at the same time. This log is different from -// the debug log in the event recorder in the sense that this is a simple log, -// the event recorder debug log contains complete details about the event. +// eventLogf records event and logs at the same time. +// +// This log is different from the debug log in the EventRecorder, in the sense +// that this is a simple log. While the debug log contains complete details +// about the event. func (r *GitRepositoryReconciler) eventLogf(ctx context.Context, obj runtime.Object, eventType string, reason string, messageFmt string, args ...interface{}) { msg := fmt.Sprintf(messageFmt, args...) // Log and emit event. diff --git a/controllers/gitrepository_controller_test.go b/controllers/gitrepository_controller_test.go index 5f20e18ae..8117e8d7c 100644 --- a/controllers/gitrepository_controller_test.go +++ b/controllers/gitrepository_controller_test.go @@ -193,7 +193,7 @@ func TestGitRepositoryReconciler_Reconcile(t *testing.T) { }, timeout).Should(BeTrue()) // Check if the object status is valid. - condns := &status.Conditions{NegativePolarity: gitRepoReadyConditions.NegativePolarity} + condns := &status.Conditions{NegativePolarity: gitRepositoryReadyCondition.NegativePolarity} checker := status.NewChecker(testEnv.Client, testEnv.GetScheme(), condns) checker.CheckErr(ctx, obj) diff --git a/docs/api/source.md b/docs/api/source.md index 8b93f1815..6bc762eac 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -240,7 +240,7 @@ BucketStatus

GitRepository

-

GitRepository is the Schema for the gitrepositories API

+

GitRepository is the Schema for the gitrepositories API.

(Optional) -

ObservedGeneration is the last observed generation.

+

ObservedGeneration is the last observed generation of the HelmRepository +object.

(Optional) -

URL is the fetch link for the last index fetched.

+

URL is the dynamic fetch link for the latest Artifact. +It is provided on a “best effort” basis, and using the precise +HelmRepositoryStatus.Artifact data is recommended.

(Optional) -

Artifact represents the output of the last successful repository sync.

+

Artifact represents the last successful HelmRepository reconciliation.

@@ -303,7 +303,7 @@ string @@ -317,9 +317,12 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference @@ -332,7 +335,7 @@ Kubernetes meta/v1.Duration @@ -346,7 +349,7 @@ Kubernetes meta/v1.Duration @@ -360,8 +363,8 @@ GitRepositoryRef @@ -375,7 +378,8 @@ GitRepositoryVerification @@ -387,8 +391,9 @@ string @@ -400,8 +405,8 @@ bool @@ -413,8 +418,8 @@ string @@ -426,7 +431,8 @@ bool @@ -440,8 +446,8 @@ This option is available only when using the ‘go-git’ GitImplementat @@ -455,7 +461,9 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom
-

The repository URL, can be a HTTP/S or SSH address.

+

URL specifies the Git repository URL, it can be an HTTP/S or SSH address.

(Optional) -

The secret name containing the Git credentials. -For HTTPS repositories the secret must contain username and password fields. -For SSH repositories the secret must contain ‘identity’, ‘identity.pub’ and ‘known_hosts’ fields.

+

SecretRef specifies the Secret containing authentication credentials for +the GitRepository. +For HTTPS repositories the Secret must contain ‘username’ and ‘password’ +fields. +For SSH repositories the Secret must contain ‘identity’, ‘identity.pub’ +and ‘known_hosts’ fields.

-

The interval at which to check for repository updates.

+

Interval at which to check the GitRepository for updates.

(Optional) -

The timeout for remote Git operations like cloning, defaults to 60s.

+

Timeout for Git operations like cloning, defaults to 60s.

(Optional) -

The Git reference to checkout and monitor for changes, defaults to -master branch.

+

Reference specifies the Git reference to resolve and monitor for +changes, defaults to the ‘master’ branch.

(Optional) -

Verification defines the configuration to verify the OpenPGP signature for the Git commit HEAD points to.

+

Verification specifies the configuration to verify the Git commit +signature(s).

(Optional) -

Ignore overrides the set of excluded patterns in the .sourceignore format (which is the same as .gitignore). -If not provided, a default will be used, consult the documentation for your version to find out what those are.

+

Ignore overrides the set of excluded patterns in the .sourceignore format +(which is the same as .gitignore). If not provided, a default will be used, +consult the documentation for your version to find out what those are.

(Optional) -

Suspend tells the controller to suspend the reconciliation of this source. -This flag tells the controller to suspend the reconciliation of this source.

+

Suspend tells the controller to suspend the reconciliation of this +GitRepository.

(Optional) -

Determines which git client library to use. -Defaults to go-git, valid values are (‘go-git’, ‘libgit2’).

+

GitImplementation specifies which Git client library implementation to +use. Defaults to ‘go-git’, valid values are (‘go-git’, ‘libgit2’).

(Optional) -

When enabled, after the clone is created, initializes all submodules within, using their default settings. +

RecurseSubmodules enables the initialization of all submodules within +the GitRepository as cloned from the URL, using their default settings. This option is available only when using the ‘go-git’ GitImplementation.

-

Include defines a list of GitRepository resources which artifacts should be included in the artifact produced for -this resource.

+

Include specifies a list of GitRepository resources which Artifacts +should be included in the Artifact produced for this GitRepository.

(Optional) -

AccessFrom defines an Access Control List for allowing cross-namespace references to this object.

+

AccessFrom specifies an Access Control List for allowing cross-namespace +references to this object. +NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092

@@ -1218,7 +1226,8 @@ github.com/fluxcd/pkg/apis/meta.ReconcileRequestStatus (Appears on: GitRepositorySpec)

-

GitRepositoryInclude defines a source with a from and to path.

+

GitRepositoryInclude specifies a local reference to a GitRepository which +Artifact (sub-)contents must be included, and where they should be placed.

@@ -1239,7 +1248,8 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference @@ -1251,7 +1261,8 @@ string @@ -1263,7 +1274,8 @@ string @@ -1276,7 +1288,7 @@ string (Appears on:GitRepositorySpec)

-

GitRepositoryRef defines the Git ref used for pull and checkout operations.

+

GitRepositoryRef specifies the Git reference to resolve and checkout.

-

Reference to a GitRepository to include.

+

GitRepositoryRef specifies the GitRepository which Artifact contents +must be included.

(Optional) -

The path to copy contents from, defaults to the root directory.

+

FromPath specifies the path to copy contents from, defaults to the root +of the Artifact.

(Optional) -

The path to copy contents to, defaults to the name of the source ref.

+

ToPath specifies the path to copy contents to, defaults to the name of +the GitRepositoryRef.

@@ -1296,7 +1308,9 @@ string @@ -1308,7 +1322,7 @@ string @@ -1320,7 +1334,7 @@ string @@ -1332,7 +1346,10 @@ string @@ -1345,7 +1362,8 @@ string (Appears on:GitRepository)

-

GitRepositorySpec defines the desired state of a Git repository.

+

GitRepositorySpec specifies the required configuration to produce an +Artifact for a Git repository.

(Optional) -

The Git branch to checkout, defaults to master.

+

Branch to checkout, defaults to ‘master’ if no other field is defined.

+

When GitRepositorySpec.GitImplementation is set to ‘go-git’, a shallow +clone of the specified branch is performed.

(Optional) -

The Git tag to checkout, takes precedence over Branch.

+

Tag to checkout, takes precedence over Branch.

(Optional) -

The Git tag semver expression, takes precedence over Tag.

+

SemVer tag expression to checkout, takes precedence over Tag.

(Optional) -

The Git commit SHA to checkout, if specified Tag filters will be ignored.

+

Commit SHA to checkout, takes precedence over all reference fields.

+

When GitRepositorySpec.GitImplementation is set to ‘go-git’, this can be +combined with Branch to shallow clone the branch, in which the commit is +expected to exist.

@@ -1364,7 +1382,7 @@ string @@ -1378,9 +1396,12 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference @@ -1393,7 +1414,7 @@ Kubernetes meta/v1.Duration @@ -1407,7 +1428,7 @@ Kubernetes meta/v1.Duration @@ -1421,8 +1442,8 @@ GitRepositoryRef @@ -1436,7 +1457,8 @@ GitRepositoryVerification @@ -1448,8 +1470,9 @@ string @@ -1461,8 +1484,8 @@ bool @@ -1474,8 +1497,8 @@ string @@ -1487,7 +1510,8 @@ bool @@ -1501,8 +1525,8 @@ This option is available only when using the ‘go-git’ GitImplementat @@ -1516,7 +1540,9 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom @@ -1529,7 +1555,7 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom (Appears on:GitRepository)

-

GitRepositoryStatus defines the observed state of a Git repository.

+

GitRepositoryStatus records the observed state of a Git repository.

-

The repository URL, can be a HTTP/S or SSH address.

+

URL specifies the Git repository URL, it can be an HTTP/S or SSH address.

(Optional) -

The secret name containing the Git credentials. -For HTTPS repositories the secret must contain username and password fields. -For SSH repositories the secret must contain ‘identity’, ‘identity.pub’ and ‘known_hosts’ fields.

+

SecretRef specifies the Secret containing authentication credentials for +the GitRepository. +For HTTPS repositories the Secret must contain ‘username’ and ‘password’ +fields. +For SSH repositories the Secret must contain ‘identity’, ‘identity.pub’ +and ‘known_hosts’ fields.

-

The interval at which to check for repository updates.

+

Interval at which to check the GitRepository for updates.

(Optional) -

The timeout for remote Git operations like cloning, defaults to 60s.

+

Timeout for Git operations like cloning, defaults to 60s.

(Optional) -

The Git reference to checkout and monitor for changes, defaults to -master branch.

+

Reference specifies the Git reference to resolve and monitor for +changes, defaults to the ‘master’ branch.

(Optional) -

Verification defines the configuration to verify the OpenPGP signature for the Git commit HEAD points to.

+

Verification specifies the configuration to verify the Git commit +signature(s).

(Optional) -

Ignore overrides the set of excluded patterns in the .sourceignore format (which is the same as .gitignore). -If not provided, a default will be used, consult the documentation for your version to find out what those are.

+

Ignore overrides the set of excluded patterns in the .sourceignore format +(which is the same as .gitignore). If not provided, a default will be used, +consult the documentation for your version to find out what those are.

(Optional) -

Suspend tells the controller to suspend the reconciliation of this source. -This flag tells the controller to suspend the reconciliation of this source.

+

Suspend tells the controller to suspend the reconciliation of this +GitRepository.

(Optional) -

Determines which git client library to use. -Defaults to go-git, valid values are (‘go-git’, ‘libgit2’).

+

GitImplementation specifies which Git client library implementation to +use. Defaults to ‘go-git’, valid values are (‘go-git’, ‘libgit2’).

(Optional) -

When enabled, after the clone is created, initializes all submodules within, using their default settings. +

RecurseSubmodules enables the initialization of all submodules within +the GitRepository as cloned from the URL, using their default settings. This option is available only when using the ‘go-git’ GitImplementation.

-

Include defines a list of GitRepository resources which artifacts should be included in the artifact produced for -this resource.

+

Include specifies a list of GitRepository resources which Artifacts +should be included in the Artifact produced for this GitRepository.

(Optional) -

AccessFrom defines an Access Control List for allowing cross-namespace references to this object.

+

AccessFrom specifies an Access Control List for allowing cross-namespace +references to this object. +NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092

@@ -1549,7 +1575,8 @@ int64 @@ -1575,7 +1602,9 @@ string @@ -1589,7 +1618,7 @@ Artifact @@ -1603,7 +1632,8 @@ Artifact @@ -1631,7 +1661,8 @@ github.com/fluxcd/pkg/apis/meta.ReconcileRequestStatus (Appears on:GitRepositorySpec)

-

GitRepositoryVerification defines the OpenPGP signature verification process.

+

GitRepositoryVerification specifies the Git commit signature verification +strategy.

(Optional) -

ObservedGeneration is the last observed generation.

+

ObservedGeneration is the last observed generation of the GitRepository +object.

(Optional) -

URL is the fetch link for the artifact output of the last repository sync.

+

URL is the dynamic fetch link for the latest Artifact. +It is provided on a “best effort” basis, and using the precise +GitRepositoryStatus.Artifact data is recommended.

(Optional) -

Artifact represents the output of the last successful repository sync.

+

Artifact represents the last successful GitRepository reconciliation.

(Optional) -

IncludedArtifacts represents the included artifacts from the last successful repository sync.

+

IncludedArtifacts contains a list of the last successfully included +Artifacts as instructed by GitRepositorySpec.Include.

@@ -1650,7 +1681,7 @@ string @@ -1663,7 +1694,8 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference diff --git a/docs/spec/v1beta2/README.md b/docs/spec/v1beta2/README.md index dca99b0e2..7e5ab30ba 100644 --- a/docs/spec/v1beta2/README.md +++ b/docs/spec/v1beta2/README.md @@ -5,7 +5,7 @@ This is the v1beta2 API specification for defining the desired state sources of ## Specification * Source kinds: - + GitRepository + + [GitRepository](gitrepositories.md) + [HelmRepository](helmrepositories.md) + HelmChart + [Bucket](buckets.md) diff --git a/docs/spec/v1beta2/gitrepositories.md b/docs/spec/v1beta2/gitrepositories.md new file mode 100644 index 000000000..594e3da2d --- /dev/null +++ b/docs/spec/v1beta2/gitrepositories.md @@ -0,0 +1,814 @@ +# Git Repositories + +The `GitRepository` API defines a Source to produce an Artifact for a Git +repository revision. + +## Example + +The following is an example of a GitRepository. It creates a tarball +(`.tar.gz`) Artifact with the fetched data from a Git repository for the +resolved reference. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: podinfo + namespace: default +spec: + interval: 5m0s + url: https://github.com/stefanprodan/podinfo + ref: + branch: master +``` + +In the above example: + +- A GitRepository named `podinfo` is created, indicated by the + `.metadata.name` field. +- The source-controller checks the Git repository every five minutes, indicated + by the `.spec.interval` field. +- It clones the `master` branch of the `https://github.com/stefanprodan/podinfo` + repository, indicated by the `.spec.ref.branch` and `.spec.url` fields. +- The specified branch and resolved HEAD revision are used as the Artifact + revision, reported in-cluster in the `.status.artifact.revision` field. +- When the current GitRepository revision differs from the latest fetched + revision, a new Artifact is archived. +- The new Artifact is reported in the `.status.artifact` field. + +You can run this example by saving the manifest into `gitrepository.yaml`. + +1. Apply the resource on the cluster: + + ```sh + kubectl apply -f gitrepository.yaml + ``` + +2. Run `kubectl get gitrepository` to see the GitRepository: + + ```console + NAME URL READY STATUS AGE + podinfo https://github.com/stefanprodan/podinfo True stored artifact for revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' 5s + ``` + +3. Run `kubectl describe gitrepository podinfo` to see the [Artifact](#artifact) + and [Conditions](#conditions) in the GitRepository's Status: + + ```console + ... + Status: + Artifact: + Checksum: 95e386f421272710c4cedbbd8607dbbaa019d500e7a5a0b6720bc7bebefc7bf2 + Last Update Time: 2022-02-14T11:23:36Z + Path: gitrepository/default/podinfo/132f4e719209eb10b9485302f8593fc0e680f4fc.tar.gz + Revision: master/132f4e719209eb10b9485302f8593fc0e680f4fc + URL: http://source-controller.source-system.svc.cluster.local./gitrepository/default/podinfo/132f4e719209eb10b9485302f8593fc0e680f4fc.tar.gz + Conditions: + Last Transition Time: 2022-02-14T11:23:36Z + Message: stored artifact for revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' + Observed Generation: 1 + Reason: Succeeded + Status: True + Type: Ready + Observed Generation: 1 + URL: http://source-controller.source-system.svc.cluster.local./gitrepository/default/podinfo/latest.tar.gz + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal GitOperationSucceed 62s source-controller cloned 'https://github.com/stefanprodan/podinfo' and checked out revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' + Normal NewArtifact 62s source-controller stored artifact for revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' + ``` + +## Writing a GitRepository spec + +As with all other Kubernetes config, a GitRepository needs `apiVersion`, +`kind`, and `metadata` fields. The name of a GitRepository object must be a +valid [DNS subdomain name](https://kubernetes.io/docs/concepts/overview/working-with-objects/names#dns-subdomain-names). + +A GitRepository also needs a +[`.spec` section](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status). + +### URL + +`.spec.url` is a required field that specifies the HTTP/S or SSH address of the +Git repository. + +**Note:** Unlike using `git`, the +[shorter scp-like syntax](https://git-scm.com/book/en/v2/Git-on-the-Server-The-Protocols#_the_ssh_protocol) +is not supported for SSH addresses (e.g. `user@example.com:repository.git`). +Instead, the valid URL format is `ssh://user@example.com:22/repository.git`. + +### Secret reference + +`.spec.secretRef.name` is an optional field to specify a name reference to a +Secret in the same namespace as the GitRepository, containing authentication +credentials for the Git repository. + +The required fields in the Secret depend on the specified protocol in the +[URL](#url). + +#### Basic access authentication + +To authenticate towards a Git repository over HTTPS using basic access +authentication (in other words: using a username and password), the referenced +Secret is expected to contain `.data.username` and `.data.password` values. + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: basic-access-auth +type: Opaque +data: + username: + password: +``` + +#### HTTPS Certificate Authority + +To provide a Certificate Authority to trust while connecting with a Git +repository over HTTPS, the referenced Secret can contain a `.data.caFile` +value. + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: https-ca-credentials + namespace: default +type: Opaque +data: + caFile: +``` + +#### SSH authentication + +To authenticate towards a Git repository over SSH, the referenced Secret is +expected to contain `.data.identity`, `.data.identity.pub` and `known_hosts` +fields. With the respective private and public key of the SSH key pair, and the +host keys of the Git repository. + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: ssh-credentials +type: Opaque +data: + identity: + identity.pub: + known_hosts: +``` + +### Interval + +`.spec.interval` is a required field that specifies the interval at which the +Git repository must be fetched. + +After successfully reconciling the object, the source-controller requeues it +for inspection after the specified interval. The value must be in a +[Go recognized duration string format](https://pkg.go.dev/time#ParseDuration), +e.g. `10m0s` to reconcile the object every 10 minutes. + +If the `.metadata.generation` of a resource changes (due to e.g. a change to +the spec), this is handled instantly outside the interval window. + +### Timeout + +`.spec.timeout` is an optional field to specify a timeout for Git operations +like cloning. The value must be in a +[Go recognized duration string format](https://pkg.go.dev/time#ParseDuration), +e.g. `1m30s` for a timeout of one minute and thirty seconds. The default value +is `60s`. + +### Reference + +`.spec.ref` is an optional field to specify the Git reference to resolve and +watch for changes. References are specified in one or more subfields +(`.branch`, `.tag`, `.semver`, `.commit`), with latter listed fields taking +precedence over earlier ones. If not specified, it defaults to a `master` +branch reference. + +#### Branch example + +To Git checkout a specified branch, use `.spec.ref.branch`: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + ref: + branch: +``` + +Using the [`go-git` Git implementation](#git-implementation), this will perform +a shallow clone to only fetch the specified branch. + +#### Tag example + +To Git checkout a specified tag, use `.spec.ref.tag`: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + ref: + tag: +``` + +This field takes precedence over [`.branch`](#branch-example). + +#### SemVer example + +To Git checkout a tag based on a +[SemVer range](https://github.com/Masterminds/semver#checking-version-constraints), +use `.spec.ref.semver`: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + ref: + # SemVer range reference: https://github.com/Masterminds/semver#checking-version-constraints + semver: "" +``` + +This field takes precedence over [`.branch`](#branch-example) and +[`.tag`](#tag-example). + +#### Commit example + +To Git checkout a specified commit, use `.spec.ref.commit`: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + ref: + commit: "" +``` + +This field takes precedence over all other fields. Using the [`go-git` Git +implementation](#git-implementation), it can be combined with `.spec.ref.branch` +to perform a shallow clone of the branch, in which the commit must exist: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + ref: + branch: + commit: "" +``` + +### Verification + +`.spec.verify` is an optional field to enable the verification of Git commit +signatures. The field offers two subfields: + +- `.mode`, to specify what Git commit object should be verified. Only supports + `head` at present. +- `.secretRef.name`, to specify a reference to a Secret in the same namespace as + the GitRepository. Containing the (PGP) public keys of trusted Git authors. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: GitRepository +metadata: + name: podinfo + namespace: default +spec: + interval: 1m + url: https://github.com/stefanprodan/podinfo + ref: + branch: master + verify: + mode: head + secretRef: + name: pgp-public-keys +``` + +When the verification succeeds, the controller adds a Condition with the +following attributes to the GitRepository's `.status.conditions`: + +- `type: SourceVerifiedCondition` +- `status: "True"` +- `reason: Succeeded` + +#### Verification Secret example + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: pgp-public-keys + namespace: default +type: Opaque +data: + author1.asc: + author2.asc: +``` + +Exporting armored public keys (`.asc` files) using `gpg`, and generating a +Secret: + +```sh +# Export armored public keys +gpg --export --armor 3CB12BA185C47B67 > author1.asc +gpg --export --armor 6A7436E8790F8689 > author2.asc +# Generate secret +kubectl create secret generic pgp-public-keys \ + --from-file=author1.asc \ + --from-file=author2.asc \ + -o yaml +``` + +### Ignore + +`.spec.ignore` is an optional field to specify rules in [the `.gitignore` +pattern format](https://git-scm.com/docs/gitignore#_pattern_format). Paths +matching the defined rules are excluded while archiving. + +When specified, `.spec.ignore` overrides the [default exclusion +list](#default-exclusions), and may overrule the [`.sourceignore` file +exclusions](#sourceignore-file). See [excluding files](#excluding-files) +for more information. + +### Suspend + +`.spec.suspend` is an optional field to suspend the reconciliation of a +GitRepository. When set to `true`, the controller will stop reconciling the +GitRepository, and changes to the resource or in the Git repository will not +result in a new Artifact. When the field is set to `false` or removed, it will +resume. + +### Git implementation + +`.spec.gitImplementation` is an optional field to change the client library +implementation used for Git operations (e.g. clone, checkout). The default +value is `go-git`. + +Unless you need support for a specific Git wire protocol functionality not +supported by the default implementation (as documented below), changing the +implementation is generally not recommended as it can come with its own set of +drawbacks. For example, not being able to make use of shallow clones forces the +controller to fetch the whole Git history tree instead of a specific one, +resulting in an increase of disk space and traffic usage. + +| Git Implementation | Shallow Clones | Git Submodules | V2 Protocol Support | +|--------------------|----------------|----------------|---------------------| +| `go-git` | true | true | false | +| `libgit2` | false | false | true | + +Some Git providers like Azure DevOps _require_ the `libgit2` implementation, as +their Git servers provide only support for the +[v2 protocol](https://git-scm.com/docs/protocol-v2). + +#### Proxy support + +When a proxy is configured in the source-controller Pod through the appropriate +environment variables, for example `HTTPS_PROXY`, `NO_PROXY`, etc. There may be +some limitations in the proxy support based on the Git implementation. + +| Git Implementation | HTTP_PROXY | HTTPS_PROXY | NO_PROXY | Self-signed Certs | +|--------------------|------------|-------------|----------|-------------------| +| `go-git` | true | true | true | false |n +| `libgit2` | true | true | true | true | + +### Recurse submodules + +`.spec.recurseSubmodules` is an optional field to enable the initialization of +all submodules within the cloned Git repository, using their default settings. +This option is only available when using the (default) `go-git` [Git +implementation](#git-implementation), and defaults to `false`. + +Note that for most Git providers (e.g. GitHub and GitLab), deploy keys can not +be used as reusing a key across multiple repositories is not allowed. You have +to use either [HTTPS token-based authentication](#basic-access-authentication), +or an SSH key belonging to a (bot) user who has access to the main repository +and all submodules. + +### Include + +`.spec.include` is an optional field to map the contents of GitRepository +Artifacts into another. This may look identical to Git submodules but has +multiple benefits over regular submodules: + +- Including a `GitRepository` allows you to use different authentication + methods for different repositories. +- A change in the included repository will trigger an update of the including + repository. +- Multiple `GitRepository` objects could include the same repository, which + decreases the amount of cloning done compared to using submodules. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: include-example +spec: + include: + - repository: + name: other-repository + fromPath: deploy/kubernetes + toPath: base/app +``` + +The `.fromPath` and `.toPath` fields allow you to limit the files included, and +where they will be copied to. If you do not specify a value for `.fromPath`, +all files from the referenced GitRepository Artifact will be included. The +`.toPath` defaults to the `.repository.name` (e.g. `./other-repository/*`). + +## Working with GitRepositories + +### Excluding files + +By default, files which match the [default exclusion rules](#default-exclusions) +are excluded while archiving the Git repository contents as an Artifact. It is +possible to overwrite and/or overrule the default exclusions using a file in +the Git repository and/or an in-spec set of rules. + +#### `.sourceignore` file + +Excluding files is possible by adding a `.sourceignore` file in the Git +repository. The `.sourceignore` file follows [the `.gitignore` pattern +format](https://git-scm.com/docs/gitignore#_pattern_format), and +pattern entries may overrule [default exclusions](#default-exclusions). + +#### Ignore spec + +Another option is to define the exclusions within the GitRepository spec, using +the [`.spec.ignore` field](#ignore). Specified rules override the [default +exclusion list](#default-exclusions), and may overrule `.sourceignore` file +exclusions. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + ignore: | + # exclude all + /* + # include deploy dir + !/deploy + # exclude file extensions from deploy dir + /deploy/**/*.md + /deploy/**/*.txt +``` + +### Triggering a reconcile + +To manually tell the source-controller to reconcile a GitRepository outside the +[specified interval window](#interval), a GitRepository can be annotated with +`reconcile.fluxcd.io/requestedAt: `. Annotating the resource +queues the GitRepository for reconciliation if the `` differs +from the last value the controller acted on, as reported in +[`.status.lastHandledReconcileAt`](#last-handled-reconcile-at). + +Using `kubectl`: + +```sh +kubectl annotate --overwrite gitrepository/ reconcile.fluxcd.io/requestedAt="$(date +%s)" +``` + +Using `flux`: + +```sh +flux reconcile source git +``` + +### Waiting for `Ready` + +When a change is applied, it is possible to wait for the GitRepository to reach +a [ready state](#ready-gitrepository) using `kubectl`: + +```sh +kubectl wait gitrepository/ --for=condition=ready --timeout=1m +``` + +### Suspending and resuming + +When you find yourself in a situation where you temporarily want to pause the +reconciliation of a GitRepository, you can suspend it using the +[`.spec.suspend` field](#suspend). + +#### Suspend a GitRepository + +In your YAML declaration: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + suspend: true +``` + +Using `kubectl`: + +```sh +kubectl patch gitrepository -p '{\"spec\": {\"suspend\" : true }}' +``` + +Using `flux`: + +```sh +flux suspend source git +``` + +**Note:** When a GitRepository has an Artifact and is suspended, and this +Artifact later disappears from the storage due to e.g. the source-controller +Pod being evicted from a Node, this will not be reflected in the +GitRepository's Status until it is resumed. + +#### Resume a GitRepository + +In your YAML declaration, comment out (or remove) the field: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +spec: + # suspend: true +``` + +**Note:** Setting the field value to `false` has the same effect as removing +it, but does not allow for "hot patching" using e.g. `kubectl` while practicing +GitOps; as the manually applied patch would be overwritten by the declared +state in Git. + +Using `kubectl`: + +```sh +kubectl patch gitrepository -p '{\"spec\" : {\"suspend\" : false }}' +``` + +Using `flux`: + +```sh +flux resume source git +``` + +### Debugging a GitRepository + +There are several ways to gather information about a GitRepository for +debugging purposes. + +#### Describe the GitRepository + +Describing a GitRepository using +`kubectl describe gitrepository ` +displays the latest recorded information for the resource in the `Status` and +`Events` sections: + +```console +... +Status: +... + Conditions: + Last Transition Time: 2022-02-14T09:40:27Z + Message: reconciling new generation 2 + Observed Generation: 2 + Reason: NewGeneration + Status: True + Type: Reconciling + Last Transition Time: 2022-02-14T09:40:27Z + Message: failed to checkout and determine revision: unable to clone 'https://github.com/stefanprodan/podinfo': couldn't find remote ref "refs/heads/invalid" + Observed Generation: 2 + Reason: GitOperationFailed + Status: False + Type: Ready + Last Transition Time: 2022-02-14T09:40:27Z + Message: failed to checkout and determine revision: unable to clone 'https://github.com/stefanprodan/podinfo': couldn't find remote ref "refs/heads/invalid" + Observed Generation: 2 + Reason: GitOperationFailed + Status: True + Type: FetchFailed + Observed Generation: 1 + URL: http://source-controller.source-system.svc.cluster.local./gitrepository/default/gitrepository-sample/latest.tar.gz +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning GitOperationFailed 2s (x9 over 4s) source-controller failed to checkout and determine revision: unable to clone 'https://github.com/stefanprodan/podinfo': couldn't find remote ref "refs/heads/invalid" +``` + +#### Trace emitted Events + +To view events for specific GitRepository(s), `kubectl get events` can be used +in combination with `--field-sector` to list the Events for specific objects. +For example, running + +```sh +kubectl get events --field-selector involvedObject.kind=GitRepository,involvedObject.name= +``` + +lists + +```console +LAST SEEN TYPE REASON OBJECT MESSAGE +2m14s Normal GitOperationSucceed gitrepository/ cloned 'https://github.com/stefanprodan/podinfo' and checked out revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' +2m14s Normal NewArtifact gitrepository/ stored artifact for revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' +94s Warning GitOperationFailed gitrepository/gitrepository-sample failed to checkout and determine revision: unable to clone 'https://github.com/stefanprodan/podinfo': couldn't find remote ref "refs/heads/invalid" +``` + +Besides being reported in Events, the reconciliation errors are also logged by +the controller. The Flux CLI offer commands for filtering the logs for a +specific GitRepository, e.g. +`flux logs --level=error --kind=GitRepository --name=`. + +## GitRepository Status + +### Artifact + +The GitRepository reports the latest synchronized state from the Git repository +as an Artifact object in the `.status.artifact` of the resource. + +The Artifact file is a gzip compressed TAR archive (`.tar.gz`), and +can be retrieved in-cluster from the `.status.artifact.url` HTTP address. + +#### Artifact example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: +status: + artifact: + checksum: e750c7a46724acaef8f8aa926259af30bbd9face2ae065ae8896ba5ee5ab832b + lastUpdateTime: "2022-01-29T06:59:23Z" + path: gitrepository///c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2.tar.gz + revision: master/363a6a8fe6a7f13e05d34c163b0ef02a777da20a + url: http://source-controller..svc.cluster.local./gitrepository///363a6a8fe6a7f13e05d34c163b0ef02a777da20a.tar.gz +``` + +#### Default exclusions + +The following files and extensions are excluded from the Artifact by +default: + +- Git files (`.git/, .gitignore, .gitmodules, .gitattributes`) +- File extensions (`.jpg, .jpeg, .gif, .png, .wmv, .flv, .tar.gz, .zip`) +- CI configs (`.github/, .circleci/, .travis.yml, .gitlab-ci.yml, appveyor.yml, .drone.yml, cloudbuild.yaml, codeship-services.yml, codeship-steps.yml`) +- CLI configs (`.goreleaser.yml, .sops.yaml`) +- Flux v1 config (`.flux.yaml`) + +To define your own exclusion rules, see [excluding files](#excluding-files). + +### Conditions + +A GitRepository enters various states during its lifecycle, reflected as +[Kubernetes Conditions][typical-status-properties]. +It can be [reconciling](#reconciling-gitrepository) while fetching the Git +state, it can be [ready](#ready-gitrepository), or it can [fail during +reconciliation](#failed-gitrepository). + +The GitRepository API is compatible with the [kstatus specification][kstatus-spec], +and reports `Reconciling` and `Stalled` conditions where applicable to +provide better (timeout) support to solutions polling the GitRepository to +become `Ready`. + +#### Reconciling GitRepository + +The source-controller marks a GitRepository as _reconciling_ when one of the +following is true: + +- There is no current Artifact for the GitRepository, or the reported Artifact + is determined to have disappeared from the storage. +- The generation of the GitRepository is newer than the [Observed + Generation](#observed-generation). +- The newly resolved Artifact revision differs from the current Artifact. + +When the GitRepository is "reconciling", the `Ready` Condition status becomes +`False`, and the controller adds a Condition with the following attributes to +the GitRepository's `.status.conditions`: + +- `type: Reconciling` +- `status: "True"` +- `reason: NewGeneration` | `reason: NoArtifact` | `reason: NewRevision` + +If the reconciling state is due to a new revision, an additional Condition is +added with the following attributes: + +- `type: ArtifactOutdated` +- `status: "True"` +- `reason: NewRevision` + +Both Conditions have a ["negative polarity"][typical-status-properties], +and are only present on the GitRepository while their status value is `"True"`. + +#### Ready GitRepository + +The source-controller marks a GitRepository as _ready_ when it has the +following characteristics: + +- The GitRepository reports an [Artifact](#artifact). +- The reported Artifact exists in the controller's Artifact storage. +- The controller was able to communicate with the remote Git repository using + the current spec. +- The revision of the reported Artifact is up-to-date with the latest + resolved revision of the remote Git repository. + +When the GitRepository is "ready", the controller sets a Condition with the +following attributes in the GitRepository's `.status.conditions`: + +- `type: Ready` +- `status: "True"` +- `reason: Succeeded` + +This `Ready` Condition will retain a status value of `"True"` until the +GitRepository is marked as [reconciling](#reconciling-gitrepository), or e.g. a +[transient error](#failed-gitrepository) occurs due to a temporary network issue. + +#### Failed GitRepository + +The source-controller may get stuck trying to produce an Artifact for a +GitRepository without completing. This can occur due to some of the following +factors: + +- The remote Git repository [URL](#url) is temporarily unavailable. +- The Git repository does not exist. +- The [Secret reference](#secret-reference) contains a reference to a + non-existing Secret. +- A specified Include is unavailable. +- The verification of the Git commit signature failed. +- The credentials in the referenced Secret are invalid. +- The GitRepository spec contains a generic misconfiguration. + +When this happens, the controller sets the `Ready` Condition status to `False`, +and adds a Condition with the following attributes to the GitRepository's +`.status.conditions`: + +- `type: FetchFailed` | `type: IncludeUnavailableCondition` +- `status: "True"` +- `reason: AuthenticationFailed` | `reason: GitOperationFailed` | `reason: StorageOperationFailed` + +This condition has a ["negative polarity"][typical-status-properties], +and is only present on the GitRepository while the status value is `"True"`. + +In addition to the above Condition types, when the +[verification of a Git commit signature](#verification) fails. A condition with +the following attributes is added to the GitRepository's `.status.conditions`: + +- `type: SourceVerifiedCondition` +- `status: "False"` +- `reason: Failed` + +While the GitRepository has one or more of these Conditions, the controller +will continue to attempt to produce an Artifact for the resource with an +exponential backoff, until it succeeds and the GitRepository is marked as +[ready](#ready-gitrepository). + +Note that a GitRepository can be [reconciling](#reconciling-gitrepository) +while failing at the same time, for example due to a newly introduced +configuration issue in the GitRepository spec. + +### Observed Generation + +The source-controller reports an [observed generation][typical-status-properties] +in the GitRepository's `.status.observedGeneration`. The observed generation is +the latest `.metadata.generation` which resulted in either a [ready state](#ready-gitrepository), +or stalled due to error it can not recover from without human +intervention. + +### Last Handled Reconcile At + +The source-controller reports the last `reconcile.fluxcd.io/requestedAt` +annotation value it acted on in the `.status.lastHandledReconcileAt` field. + +For practical information about this field, see [triggering a +reconcile](#triggering-a-reconcile). + +[typical-status-properties]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties +[kstatus-spec]: https://github.com/kubernetes-sigs/cli-utils/tree/master/pkg/kstatus From f59b48c339b065ab3d5bfcac75e79b6b58de8735 Mon Sep 17 00:00:00 2001 From: Sunny Date: Mon, 14 Feb 2022 14:36:55 +0530 Subject: [PATCH 4/9] Document HelmChart API v1beta2 spec Signed-off-by: Sunny --- api/v1beta2/helmchart_types.go | 93 ++-- .../source.toolkit.fluxcd.io_helmcharts.yaml | 68 +-- controllers/helmchart_controller.go | 80 ++- controllers/helmchart_controller_test.go | 20 +- docs/api/source.md | 105 ++-- docs/spec/v1beta2/README.md | 2 +- docs/spec/v1beta2/helmcharts.md | 481 ++++++++++++++++++ 7 files changed, 684 insertions(+), 165 deletions(-) create mode 100644 docs/spec/v1beta2/helmcharts.md diff --git a/api/v1beta2/helmchart_types.go b/api/v1beta2/helmchart_types.go index af15dbc31..2ce5a942f 100644 --- a/api/v1beta2/helmchart_types.go +++ b/api/v1beta2/helmchart_types.go @@ -28,28 +28,29 @@ import ( // HelmChartKind is the string representation of a HelmChart. const HelmChartKind = "HelmChart" -// HelmChartSpec defines the desired state of a Helm chart. +// HelmChartSpec specifies the desired state of a Helm chart. type HelmChartSpec struct { - // The name or path the Helm chart is available at in the SourceRef. + // Chart is the name or path the Helm chart is available at in the + // SourceRef. // +required Chart string `json:"chart"` - // The chart version semver expression, ignored for charts from GitRepository - // and Bucket sources. Defaults to latest when omitted. + // Version is the chart version semver expression, ignored for charts from + // GitRepository and Bucket sources. Defaults to latest when omitted. // +kubebuilder:default:=* // +optional Version string `json:"version,omitempty"` - // The reference to the Source the chart is available at. + // SourceRef is the reference to the Source the chart is available at. // +required SourceRef LocalHelmChartSourceReference `json:"sourceRef"` - // The interval at which to check the Source for updates. + // Interval is the interval at which to check the Source for updates. // +required Interval metav1.Duration `json:"interval"` - // Determines what enables the creation of a new artifact. Valid values are - // ('ChartVersion', 'Revision'). + // ReconcileStrategy determines what enables the creation of a new artifact. + // Valid values are ('ChartVersion', 'Revision'). // See the documentation of the values for an explanation on their behavior. // Defaults to ChartVersion when omitted. // +kubebuilder:validation:Enum=ChartVersion;Revision @@ -57,26 +58,30 @@ type HelmChartSpec struct { // +optional ReconcileStrategy string `json:"reconcileStrategy,omitempty"` - // Alternative list of values files to use as the chart values (values.yaml - // is not included by default), expected to be a relative path in the SourceRef. - // Values files are merged in the order of this list with the last file overriding - // the first. Ignored when omitted. + // ValuesFiles is an alternative list of values files to use as the chart + // values (values.yaml is not included by default), expected to be a + // relative path in the SourceRef. + // Values files are merged in the order of this list with the last file + // overriding the first. Ignored when omitted. // +optional ValuesFiles []string `json:"valuesFiles,omitempty"` - // Alternative values file to use as the default chart values, expected to - // be a relative path in the SourceRef. Deprecated in favor of ValuesFiles, - // for backwards compatibility the file defined here is merged before the - // ValuesFiles items. Ignored when omitted. + // ValuesFile is an alternative values file to use as the default chart + // values, expected to be a relative path in the SourceRef. Deprecated in + // favor of ValuesFiles, for backwards compatibility the file specified here + // is merged before the ValuesFiles items. Ignored when omitted. // +optional // +deprecated ValuesFile string `json:"valuesFile,omitempty"` - // This flag tells the controller to suspend the reconciliation of this source. + // Suspend tells the controller to suspend the reconciliation of this + // source. // +optional Suspend bool `json:"suspend,omitempty"` - // AccessFrom defines an Access Control List for allowing cross-namespace references to this object. + // AccessFrom specifies an Access Control List for allowing cross-namespace + // references to this object. + // NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092 // +optional AccessFrom *acl.AccessFrom `json:"accessFrom,omitempty"` } @@ -107,18 +112,19 @@ type LocalHelmChartSourceReference struct { Name string `json:"name"` } -// HelmChartStatus defines the observed state of the HelmChart. +// HelmChartStatus records the observed state of the HelmChart. type HelmChartStatus struct { - // ObservedGeneration is the last observed generation. + // ObservedGeneration is the last observed generation of the HelmChart + // object. // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` // ObservedSourceArtifactRevision is the last observed Artifact.Revision - // of the Source reference. + // of the HelmChartSpec.SourceRef. // +optional ObservedSourceArtifactRevision string `json:"observedSourceArtifactRevision,omitempty"` - // ObservedChartName is the last observed chart name as defined by the + // ObservedChartName is the last observed chart name as specified by the // resolved chart reference. // +optional ObservedChartName string `json:"observedChartName,omitempty"` @@ -127,11 +133,13 @@ type HelmChartStatus struct { // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` - // URL is the fetch link for the last chart pulled. + // URL is the dynamic fetch link for the latest Artifact. + // It is provided on a "best effort" basis, and using the precise + // BucketStatus.Artifact data is recommended. // +optional URL string `json:"url,omitempty"` - // Artifact represents the output of the last successful chart sync. + // Artifact represents the output of the last successful reconciliation. // +optional Artifact *Artifact `json:"artifact,omitempty"` @@ -139,19 +147,11 @@ type HelmChartStatus struct { } const ( - // ChartPullFailedReason represents the fact that the pull of the Helm chart - // failed. - ChartPullFailedReason string = "ChartPullFailed" - - // ChartPullSucceededReason represents the fact that the pull of the Helm chart + // ChartPullSucceededReason signals that the pull of the Helm chart // succeeded. ChartPullSucceededReason string = "ChartPullSucceeded" - // ChartPackageFailedReason represent the fact that the package of the Helm - // chart failed. - ChartPackageFailedReason string = "ChartPackageFailed" - - // ChartPackageSucceededReason represents the fact that the package of the Helm + // ChartPackageSucceededReason signals that the package of the Helm // chart succeeded. ChartPackageSucceededReason string = "ChartPackageSucceeded" ) @@ -166,23 +166,19 @@ func (in *HelmChart) SetConditions(conditions []metav1.Condition) { in.Status.Conditions = conditions } -// GetRequeueAfter returns the duration after which the source must be reconciled again. +// GetRequeueAfter returns the duration after which the source must be +// reconciled again. func (in HelmChart) GetRequeueAfter() time.Duration { return in.Spec.Interval.Duration } -// GetInterval returns the interval at which the source is reconciled. -// Deprecated: use GetRequeueAfter instead. -func (in HelmChart) GetInterval() metav1.Duration { - return in.Spec.Interval -} - -// GetArtifact returns the latest artifact from the source if present in the status sub-resource. +// GetArtifact returns the latest artifact from the source if present in the +// status sub-resource. func (in *HelmChart) GetArtifact() *Artifact { return in.Status.Artifact } -// GetValuesFiles returns a merged list of ValuesFiles. +// GetValuesFiles returns a merged list of HelmChartSpec.ValuesFiles. func (in *HelmChart) GetValuesFiles() []string { valuesFiles := in.Spec.ValuesFiles @@ -193,12 +189,6 @@ func (in *HelmChart) GetValuesFiles() []string { return valuesFiles } -// GetStatusConditions returns a pointer to the Status.Conditions slice. -// Deprecated: use GetConditions instead. -func (in *HelmChart) GetStatusConditions() *[]metav1.Condition { - return &in.Status.Conditions -} - // +genclient // +genclient:Namespaced // +kubebuilder:storageversion @@ -213,7 +203,7 @@ func (in *HelmChart) GetStatusConditions() *[]metav1.Condition { // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="" // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="" -// HelmChart is the Schema for the helmcharts API +// HelmChart is the Schema for the helmcharts API. type HelmChart struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -223,9 +213,8 @@ type HelmChart struct { Status HelmChartStatus `json:"status,omitempty"` } +// HelmChartList contains a list of HelmChart objects. // +kubebuilder:object:root=true - -// HelmChartList contains a list of HelmChart type HelmChartList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml index 819549f55..a45d0370b 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml @@ -299,7 +299,7 @@ spec: name: v1beta2 schema: openAPIV3Schema: - description: HelmChart is the Schema for the helmcharts API + description: HelmChart is the Schema for the helmcharts API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -314,11 +314,12 @@ spec: metadata: type: object spec: - description: HelmChartSpec defines the desired state of a Helm chart. + description: HelmChartSpec specifies the desired state of a Helm chart. properties: accessFrom: - description: AccessFrom defines an Access Control List for allowing - cross-namespace references to this object. + description: 'AccessFrom specifies an Access Control List for allowing + cross-namespace references to this object. NOTE: Not implemented, + provisional as of https://github.com/fluxcd/flux2/pull/2092' properties: namespaceSelectors: description: NamespaceSelectors is the list of namespace selectors @@ -344,24 +345,26 @@ spec: - namespaceSelectors type: object chart: - description: The name or path the Helm chart is available at in the - SourceRef. + description: Chart is the name or path the Helm chart is available + at in the SourceRef. type: string interval: - description: The interval at which to check the Source for updates. + description: Interval is the interval at which to check the Source + for updates. type: string reconcileStrategy: default: ChartVersion - description: Determines what enables the creation of a new artifact. - Valid values are ('ChartVersion', 'Revision'). See the documentation - of the values for an explanation on their behavior. Defaults to - ChartVersion when omitted. + description: ReconcileStrategy determines what enables the creation + of a new artifact. Valid values are ('ChartVersion', 'Revision'). + See the documentation of the values for an explanation on their + behavior. Defaults to ChartVersion when omitted. enum: - ChartVersion - Revision type: string sourceRef: - description: The reference to the Source the chart is available at. + description: SourceRef is the reference to the Source the chart is + available at. properties: apiVersion: description: APIVersion of the referent. @@ -382,28 +385,30 @@ spec: - name type: object suspend: - description: This flag tells the controller to suspend the reconciliation + description: Suspend tells the controller to suspend the reconciliation of this source. type: boolean valuesFile: - description: Alternative values file to use as the default chart values, - expected to be a relative path in the SourceRef. Deprecated in favor - of ValuesFiles, for backwards compatibility the file defined here - is merged before the ValuesFiles items. Ignored when omitted. + description: ValuesFile is an alternative values file to use as the + default chart values, expected to be a relative path in the SourceRef. + Deprecated in favor of ValuesFiles, for backwards compatibility + the file specified here is merged before the ValuesFiles items. + Ignored when omitted. type: string valuesFiles: - description: Alternative list of values files to use as the chart - values (values.yaml is not included by default), expected to be - a relative path in the SourceRef. Values files are merged in the - order of this list with the last file overriding the first. Ignored - when omitted. + description: ValuesFiles is an alternative list of values files to + use as the chart values (values.yaml is not included by default), + expected to be a relative path in the SourceRef. Values files are + merged in the order of this list with the last file overriding the + first. Ignored when omitted. items: type: string type: array version: default: '*' - description: The chart version semver expression, ignored for charts - from GitRepository and Bucket sources. Defaults to latest when omitted. + description: Version is the chart version semver expression, ignored + for charts from GitRepository and Bucket sources. Defaults to latest + when omitted. type: string required: - chart @@ -413,11 +418,11 @@ spec: status: default: observedGeneration: -1 - description: HelmChartStatus defines the observed state of the HelmChart. + description: HelmChartStatus records the observed state of the HelmChart. properties: artifact: description: Artifact represents the output of the last successful - chart sync. + reconciliation. properties: checksum: description: Checksum is the SHA256 checksum of the Artifact file. @@ -528,18 +533,21 @@ spec: type: string observedChartName: description: ObservedChartName is the last observed chart name as - defined by the resolved chart reference. + specified by the resolved chart reference. type: string observedGeneration: - description: ObservedGeneration is the last observed generation. + description: ObservedGeneration is the last observed generation of + the HelmChart object. format: int64 type: integer observedSourceArtifactRevision: description: ObservedSourceArtifactRevision is the last observed Artifact.Revision - of the Source reference. + of the HelmChartSpec.SourceRef. type: string url: - description: URL is the fetch link for the last chart pulled. + description: URL is the dynamic fetch link for the latest Artifact. + It is provided on a "best effort" basis, and using the precise BucketStatus.Artifact + data is recommended. type: string type: object type: object diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index 216e4e648..9f5e8dcda 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -64,9 +64,9 @@ import ( "github.com/fluxcd/source-controller/internal/util" ) -// helmChartReadyConditions contains all the conditions information +// helmChartReadyCondition contains all the conditions information // needed for HelmChart Ready status conditions summary calculation. -var helmChartReadyConditions = summarize.Conditions{ +var helmChartReadyCondition = summarize.Conditions{ Target: meta.ReadyCondition, Owned: []string{ sourcev1.BuildFailedCondition, @@ -116,7 +116,10 @@ type HelmChartReconcilerOptions struct { MaxConcurrentReconciles int } -type helmChartReconcilerFunc func(ctx context.Context, obj *sourcev1.HelmChart, build *chart.Build) (sreconcile.Result, error) +// helmChartReconcileFunc is the function type for all the v1beta2.HelmChart +// (sub)reconcile functions. The type implementations are grouped and +// executed serially to perform the complete reconcile of the object. +type helmChartReconcileFunc func(ctx context.Context, obj *sourcev1.HelmChart, build *chart.Build) (sreconcile.Result, error) func (r *HelmChartReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, opts HelmChartReconcilerOptions) error { if err := mgr.GetCache().IndexField(context.TODO(), &sourcev1.HelmRepository{}, sourcev1.HelmRepositoryURLIndexKey, @@ -184,7 +187,7 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( defer func() { summarizeHelper := summarize.NewHelper(r.EventRecorder, patchHelper) summarizeOpts := []summarize.Option{ - summarize.WithConditions(helmChartReadyConditions), + summarize.WithConditions(helmChartReadyCondition), summarize.WithReconcileResult(recResult), summarize.WithReconcileError(retErr), summarize.WithIgnoreNotFound(), @@ -192,7 +195,7 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( summarize.RecordContextualError, summarize.RecordReconcileReq, ), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetInterval().Duration}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), summarize.WithPatchFieldOwner(r.ControllerName), } result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) @@ -217,7 +220,7 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // Reconcile actual object - reconcilers := []helmChartReconcilerFunc{ + reconcilers := []helmChartReconcileFunc{ r.reconcileStorage, r.reconcileSource, r.reconcileArtifact, @@ -226,9 +229,10 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return } -// reconcile steps through the actual reconciliation tasks for the object, it returns early on the first step that -// produces an error. -func (r *HelmChartReconciler) reconcile(ctx context.Context, obj *sourcev1.HelmChart, reconcilers []helmChartReconcilerFunc) (sreconcile.Result, error) { +// reconcile iterates through the gitRepositoryReconcileFunc tasks for the +// object. It returns early on the first call that returns +// reconcile.ResultRequeue, or produces an error. +func (r *HelmChartReconciler) reconcile(ctx context.Context, obj *sourcev1.HelmChart, reconcilers []helmChartReconcileFunc) (sreconcile.Result, error) { if obj.Generation != obj.Status.ObservedGeneration { conditions.MarkReconciling(obj, "NewGeneration", "reconciling new object generation (%d)", obj.Generation) } @@ -258,14 +262,17 @@ func (r *HelmChartReconciler) reconcile(ctx context.Context, obj *sourcev1.HelmC return res, resErr } -// reconcileStorage ensures the current state of the storage matches the desired and previously observed state. +// reconcileStorage ensures the current state of the storage matches the +// desired and previously observed state. // -// All artifacts for the resource except for the current one are garbage collected from the storage. -// If the artifact in the Status object of the resource disappeared from storage, it is removed from the object. -// If the object does not have an artifact in its Status object, a v1beta1.ArtifactUnavailableCondition is set. -// If the hostname of the URLs on the object do not match the current storage server hostname, they are updated. -// -// The caller should assume a failure if an error is returned, or the BuildResult is zero. +// All Artifacts for the object except for the current one in the Status are +// garbage collected from the Storage. +// If the Artifact in the Status of the object disappeared from the Storage, +// it is removed from the object. +// If the object does not have an Artifact in its Status, a Reconciling +// condition is added. +// The hostname of any URL in the Status of the object are updated, to ensure +// they match the Storage server hostname of current runtime. func (r *HelmChartReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.HelmChart, build *chart.Build) (sreconcile.Result, error) { // Garbage collect previous advertised artifact(s) from storage _ = r.garbageCollect(ctx, obj) @@ -367,6 +374,11 @@ func (r *HelmChartReconciler) reconcileSource(ctx context.Context, obj *sourcev1 } } +// buildFromHelmRepository attempts to pull and/or package a Helm chart with +// the specified data from the v1beta2.HelmRepository and v1beta2.HelmChart +// objects. +// In case of a failure it records v1beta2.FetchFailedCondition on the chart +// object, and returns early. func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj *sourcev1.HelmChart, repo *sourcev1.HelmRepository, b *chart.Build) (sreconcile.Result, error) { var tlsConfig *tls.Config @@ -463,6 +475,10 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj * return sreconcile.ResultSuccess, nil } +// buildFromHelmRepository attempts to pull and/or package a Helm chart with +// the specified data v1beta2.HelmChart object and the given v1beta2.Artifact. +// In case of a failure it records v1beta2.FetchFailedCondition on the chart +// object, and returns early. func (r *HelmChartReconciler) buildFromTarballArtifact(ctx context.Context, obj *sourcev1.HelmChart, source sourcev1.Artifact, b *chart.Build) (sreconcile.Result, error) { // Create temporary working directory tmpDir, err := util.TempDirForObj("", obj) @@ -585,8 +601,15 @@ func (r *HelmChartReconciler) buildFromTarballArtifact(ctx context.Context, obj return sreconcile.ResultSuccess, nil } -// reconcileArtifact reconciles the given chart.Build to an v1beta1.Artifact in the Storage, and records it -// on the object. +// reconcileArtifact archives a new Artifact to the Storage, if the current +// (Status) data on the object does not match the given. +// +// The inspection of the given data to the object is differed, ensuring any +// stale observations like v1beta2.ArtifactOutdatedCondition are removed. +// If the given Artifact does not differ from the object's current, it returns +// early. +// On a successful archive, the Artifact in the Status of the object is set, +// and the symlink in the Storage is updated to its path. func (r *HelmChartReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.HelmChart, b *chart.Build) (sreconcile.Result, error) { // Without a complete chart build, there is little to reconcile if !b.Complete() { @@ -693,8 +716,9 @@ func (r *HelmChartReconciler) getSource(ctx context.Context, obj *sourcev1.HelmC return s, nil } -// reconcileDelete handles the delete of an object. It first garbage collects all artifacts for the object from the -// artifact storage, if successful, the finalizer is removed from the object. +// reconcileDelete handles the deletion of the object. +// It first garbage collects all Artifacts for the object from the Storage. +// Removing the finalizer from the object if successful. func (r *HelmChartReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.HelmChart) (sreconcile.Result, error) { // Garbage collect the resource's artifacts if err := r.garbageCollect(ctx, obj); err != nil { @@ -709,9 +733,11 @@ func (r *HelmChartReconciler) reconcileDelete(ctx context.Context, obj *sourcev1 return sreconcile.ResultEmpty, nil } -// garbageCollect performs a garbage collection for the given v1beta1.HelmChart. It removes all but the current -// artifact, unless the deletion timestamp is set. Which will result in the removal of all artifacts for the -// resource. +// garbageCollect performs a garbage collection for the given object. +// +// It removes all but the current Artifact from the Storage, unless the +// deletion timestamp on the object is set. Which will result in the +// removal of all Artifacts for the objects. func (r *HelmChartReconciler) garbageCollect(ctx context.Context, obj *sourcev1.HelmChart) error { if !obj.DeletionTimestamp.IsZero() { if deleted, err := r.Storage.RemoveAll(r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), "", "*")); err != nil { @@ -925,9 +951,11 @@ func (r *HelmChartReconciler) requestsForBucketChange(o client.Object) []reconci return reqs } -// eventLogf records event and logs at the same time. This log is different from -// the debug log in the event recorder in the sense that this is a simple log, -// the event recorder debug log contains complete details about the event. +// eventLogf records event and logs at the same time. +// +// This log is different from the debug log in the EventRecorder, in the sense +// that this is a simple log. While the debug log contains complete details +// about the event. func (r *HelmChartReconciler) eventLogf(ctx context.Context, obj runtime.Object, eventType string, reason string, messageFmt string, args ...interface{}) { msg := fmt.Sprintf(messageFmt, args...) // Log and emit event. diff --git a/controllers/helmchart_controller_test.go b/controllers/helmchart_controller_test.go index 154eed083..43d568b85 100644 --- a/controllers/helmchart_controller_test.go +++ b/controllers/helmchart_controller_test.go @@ -126,7 +126,7 @@ func TestHelmChartReconciler_Reconcile(t *testing.T) { }, timeout).Should(BeTrue()) // Check if the object status is valid. - condns := &status.Conditions{NegativePolarity: helmChartReadyConditions.NegativePolarity} + condns := &status.Conditions{NegativePolarity: helmChartReadyCondition.NegativePolarity} checker := status.NewChecker(testEnv.Client, testEnv.GetScheme(), condns) checker.CheckErr(ctx, obj) @@ -1352,8 +1352,8 @@ func TestHelmChartReconciler_reconcileDelete(t *testing.T) { } func TestHelmChartReconciler_reconcileSubRecs(t *testing.T) { - // Helper to build simple helmChartReconcilerFunc with result and error. - buildReconcileFuncs := func(r sreconcile.Result, e error) helmChartReconcilerFunc { + // Helper to build simple helmChartReconcileFunc with result and error. + buildReconcileFuncs := func(r sreconcile.Result, e error) helmChartReconcileFunc { return func(_ context.Context, _ *sourcev1.HelmChart, _ *chart.Build) (sreconcile.Result, error) { return r, e } @@ -1363,14 +1363,14 @@ func TestHelmChartReconciler_reconcileSubRecs(t *testing.T) { name string generation int64 observedGeneration int64 - reconcileFuncs []helmChartReconcilerFunc + reconcileFuncs []helmChartReconcileFunc wantResult sreconcile.Result wantErr bool assertConditions []metav1.Condition }{ { name: "successful reconciliations", - reconcileFuncs: []helmChartReconcilerFunc{ + reconcileFuncs: []helmChartReconcileFunc{ buildReconcileFuncs(sreconcile.ResultSuccess, nil), }, wantResult: sreconcile.ResultSuccess, @@ -1380,7 +1380,7 @@ func TestHelmChartReconciler_reconcileSubRecs(t *testing.T) { name: "successful reconciliation with generation difference", generation: 3, observedGeneration: 2, - reconcileFuncs: []helmChartReconcilerFunc{ + reconcileFuncs: []helmChartReconcileFunc{ buildReconcileFuncs(sreconcile.ResultSuccess, nil), }, wantResult: sreconcile.ResultSuccess, @@ -1391,7 +1391,7 @@ func TestHelmChartReconciler_reconcileSubRecs(t *testing.T) { }, { name: "failed reconciliation", - reconcileFuncs: []helmChartReconcilerFunc{ + reconcileFuncs: []helmChartReconcileFunc{ buildReconcileFuncs(sreconcile.ResultEmpty, fmt.Errorf("some error")), }, wantResult: sreconcile.ResultEmpty, @@ -1399,7 +1399,7 @@ func TestHelmChartReconciler_reconcileSubRecs(t *testing.T) { }, { name: "multiple object status conditions mutations", - reconcileFuncs: []helmChartReconcilerFunc{ + reconcileFuncs: []helmChartReconcileFunc{ func(_ context.Context, obj *sourcev1.HelmChart, _ *chart.Build) (sreconcile.Result, error) { conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", "new index revision") return sreconcile.ResultSuccess, nil @@ -1418,7 +1418,7 @@ func TestHelmChartReconciler_reconcileSubRecs(t *testing.T) { }, { name: "subrecs with one result=Requeue, no error", - reconcileFuncs: []helmChartReconcilerFunc{ + reconcileFuncs: []helmChartReconcileFunc{ buildReconcileFuncs(sreconcile.ResultSuccess, nil), buildReconcileFuncs(sreconcile.ResultRequeue, nil), buildReconcileFuncs(sreconcile.ResultSuccess, nil), @@ -1428,7 +1428,7 @@ func TestHelmChartReconciler_reconcileSubRecs(t *testing.T) { }, { name: "subrecs with error before result=Requeue", - reconcileFuncs: []helmChartReconcilerFunc{ + reconcileFuncs: []helmChartReconcileFunc{ buildReconcileFuncs(sreconcile.ResultSuccess, nil), buildReconcileFuncs(sreconcile.ResultEmpty, fmt.Errorf("some error")), buildReconcileFuncs(sreconcile.ResultRequeue, nil), diff --git a/docs/api/source.md b/docs/api/source.md index 6bc762eac..6f0d1621b 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -487,7 +487,7 @@ GitRepositoryStatus

HelmChart

-

HelmChart is the Schema for the helmcharts API

+

HelmChart is the Schema for the helmcharts API.

-

Mode describes what Git object should be verified, currently (‘head’).

+

Mode specifies what Git object should be verified, currently (‘head’).

-

SecretRef containing the public keys of all trusted Git authors.

+

SecretRef specifies the Secret containing the public keys of trusted Git +authors.

@@ -550,7 +550,8 @@ string @@ -562,8 +563,8 @@ string @@ -576,7 +577,7 @@ LocalHelmChartSourceReference @@ -589,7 +590,7 @@ Kubernetes meta/v1.Duration @@ -601,8 +602,8 @@ string @@ -616,10 +617,11 @@ Defaults to ChartVersion when omitted.

@@ -631,10 +633,10 @@ string @@ -646,7 +648,8 @@ bool @@ -660,7 +663,9 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom
-

The name or path the Helm chart is available at in the SourceRef.

+

Chart is the name or path the Helm chart is available at in the +SourceRef.

(Optional) -

The chart version semver expression, ignored for charts from GitRepository -and Bucket sources. Defaults to latest when omitted.

+

Version is the chart version semver expression, ignored for charts from +GitRepository and Bucket sources. Defaults to latest when omitted.

-

The reference to the Source the chart is available at.

+

SourceRef is the reference to the Source the chart is available at.

-

The interval at which to check the Source for updates.

+

Interval is the interval at which to check the Source for updates.

(Optional) -

Determines what enables the creation of a new artifact. Valid values are -(‘ChartVersion’, ‘Revision’). +

ReconcileStrategy determines what enables the creation of a new artifact. +Valid values are (‘ChartVersion’, ‘Revision’). See the documentation of the values for an explanation on their behavior. Defaults to ChartVersion when omitted.

(Optional) -

Alternative list of values files to use as the chart values (values.yaml -is not included by default), expected to be a relative path in the SourceRef. -Values files are merged in the order of this list with the last file overriding -the first. Ignored when omitted.

+

ValuesFiles is an alternative list of values files to use as the chart +values (values.yaml is not included by default), expected to be a +relative path in the SourceRef. +Values files are merged in the order of this list with the last file +overriding the first. Ignored when omitted.

(Optional) -

Alternative values file to use as the default chart values, expected to -be a relative path in the SourceRef. Deprecated in favor of ValuesFiles, -for backwards compatibility the file defined here is merged before the -ValuesFiles items. Ignored when omitted.

+

ValuesFile is an alternative values file to use as the default chart +values, expected to be a relative path in the SourceRef. Deprecated in +favor of ValuesFiles, for backwards compatibility the file specified here +is merged before the ValuesFiles items. Ignored when omitted.

(Optional) -

This flag tells the controller to suspend the reconciliation of this source.

+

Suspend tells the controller to suspend the reconciliation of this +source.

(Optional) -

AccessFrom defines an Access Control List for allowing cross-namespace references to this object.

+

AccessFrom specifies an Access Control List for allowing cross-namespace +references to this object. +NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092

@@ -1308,7 +1313,7 @@ string (Optional) -

Branch to checkout, defaults to ‘master’ if no other field is defined.

+

Branch to check out, defaults to ‘master’ if no other field is defined.

When GitRepositorySpec.GitImplementation is set to ‘go-git’, a shallow clone of the specified branch is performed.

@@ -1322,7 +1327,7 @@ string (Optional) -

Tag to checkout, takes precedence over Branch.

+

Tag to check out, takes precedence over Branch.

@@ -1334,7 +1339,7 @@ string (Optional) -

SemVer tag expression to checkout, takes precedence over Tag.

+

SemVer tag expression to check out, takes precedence over Tag.

@@ -1346,7 +1351,7 @@ string (Optional) -

Commit SHA to checkout, takes precedence over all reference fields.

+

Commit SHA to check out, takes precedence over all reference fields.

When GitRepositorySpec.GitImplementation is set to ‘go-git’, this can be combined with Branch to shallow clone the branch, in which the commit is expected to exist.

@@ -1708,7 +1713,7 @@ authors.

(Appears on: HelmChart)

-

HelmChartSpec defines the desired state of a Helm chart.

+

HelmChartSpec specifies the desired state of a Helm chart.

@@ -1727,7 +1732,8 @@ string @@ -1739,8 +1745,8 @@ string @@ -1753,7 +1759,7 @@ LocalHelmChartSourceReference @@ -1766,7 +1772,7 @@ Kubernetes meta/v1.Duration @@ -1778,8 +1784,8 @@ string @@ -1793,10 +1799,11 @@ Defaults to ChartVersion when omitted.

@@ -1808,10 +1815,10 @@ string @@ -1823,7 +1830,8 @@ bool @@ -1837,7 +1845,9 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom @@ -1850,7 +1860,7 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom (Appears on:HelmChart)

-

HelmChartStatus defines the observed state of the HelmChart.

+

HelmChartStatus records the observed state of the HelmChart.

-

The name or path the Helm chart is available at in the SourceRef.

+

Chart is the name or path the Helm chart is available at in the +SourceRef.

(Optional) -

The chart version semver expression, ignored for charts from GitRepository -and Bucket sources. Defaults to latest when omitted.

+

Version is the chart version semver expression, ignored for charts from +GitRepository and Bucket sources. Defaults to latest when omitted.

-

The reference to the Source the chart is available at.

+

SourceRef is the reference to the Source the chart is available at.

-

The interval at which to check the Source for updates.

+

Interval is the interval at which to check the Source for updates.

(Optional) -

Determines what enables the creation of a new artifact. Valid values are -(‘ChartVersion’, ‘Revision’). +

ReconcileStrategy determines what enables the creation of a new artifact. +Valid values are (‘ChartVersion’, ‘Revision’). See the documentation of the values for an explanation on their behavior. Defaults to ChartVersion when omitted.

(Optional) -

Alternative list of values files to use as the chart values (values.yaml -is not included by default), expected to be a relative path in the SourceRef. -Values files are merged in the order of this list with the last file overriding -the first. Ignored when omitted.

+

ValuesFiles is an alternative list of values files to use as the chart +values (values.yaml is not included by default), expected to be a +relative path in the SourceRef. +Values files are merged in the order of this list with the last file +overriding the first. Ignored when omitted.

(Optional) -

Alternative values file to use as the default chart values, expected to -be a relative path in the SourceRef. Deprecated in favor of ValuesFiles, -for backwards compatibility the file defined here is merged before the -ValuesFiles items. Ignored when omitted.

+

ValuesFile is an alternative values file to use as the default chart +values, expected to be a relative path in the SourceRef. Deprecated in +favor of ValuesFiles, for backwards compatibility the file specified here +is merged before the ValuesFiles items. Ignored when omitted.

(Optional) -

This flag tells the controller to suspend the reconciliation of this source.

+

Suspend tells the controller to suspend the reconciliation of this +source.

(Optional) -

AccessFrom defines an Access Control List for allowing cross-namespace references to this object.

+

AccessFrom specifies an Access Control List for allowing cross-namespace +references to this object. +NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092

@@ -1870,7 +1880,8 @@ int64 @@ -1883,7 +1894,7 @@ string @@ -1895,7 +1906,7 @@ string @@ -1922,7 +1933,9 @@ string @@ -1936,7 +1949,7 @@ Artifact diff --git a/docs/spec/v1beta2/README.md b/docs/spec/v1beta2/README.md index 7e5ab30ba..917848055 100644 --- a/docs/spec/v1beta2/README.md +++ b/docs/spec/v1beta2/README.md @@ -7,7 +7,7 @@ This is the v1beta2 API specification for defining the desired state sources of * Source kinds: + [GitRepository](gitrepositories.md) + [HelmRepository](helmrepositories.md) - + HelmChart + + [HelmChart](helmcharts.md) + [Bucket](buckets.md) ## Implementation diff --git a/docs/spec/v1beta2/helmcharts.md b/docs/spec/v1beta2/helmcharts.md new file mode 100644 index 000000000..2e78131fb --- /dev/null +++ b/docs/spec/v1beta2/helmcharts.md @@ -0,0 +1,481 @@ +# Helm Charts + +The `HelmChart` API defines a Source to produce an Artifact for a Helm chart +archive with a set of specific configurations. + +## Example + +The following is an example of a HelmChart. It fetches and/or packages a Helm +chart and exposes it as a tarball (`.tgz`) Artifact for the specified +configuration: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmChart +metadata: + name: podinfo + namespace: default +spec: + interval: 5m0s + chart: podinfo + reconcileStrategy: ChartVersion + sourceRef: + kind: HelmRepository + name: podinfo + version: '5.*' +``` + +In the above example: + +- A HelmChart named `podinfo` is created, indicated by the `.metadata.name` + field. +- The source-controller fetches the Helm chart every five minutes from the + `podinfo` HelmRepository source reference, indicated by the + `.spec.sourceRef.kind` and `.spec.sourceRef.name` fields. +- The fetched Helm chart version is the latest available chart + version in the range specified in `spec.version`. This version is also used as + Artifact revision, reported in-cluster in the `.status.artifact.revision` + field. +- When the current Helm Chart version differs from the latest available chart + in the version range, it is fetched and/or packaged as a new Artifact. +- The new Artifact is reported in the `.status.artifact` field. + +You can run this example by saving the manifest into `helmchart.yaml`. + +**NOTE:** HelmChart is usually used by the helm-controller. Based on the +HelmRelease configuration, an associated HelmChart is created by the +helm-controller. + +1. Apply the resource on the cluster: + + ```sh + kubectl apply -f helmchart.yaml + ``` + +2. Run `kubectl get helmchart` to see the HelmChart: + + ```console + NAME CHART VERSION SOURCE KIND SOURCE NAME READY STATUS AGE + podinfo podinfo 5.* HelmRepository podinfo True pulled 'podinfo' chart with version '5.2.1' 53s + ``` + +3. Run `kubectl describe helmchart podinfo` to see the [Artifact](#artifact) and + [Conditions](#conditions) in the HelmChart's Status: + + ```console + Status: + Observed Source Artifact Revision: 83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111 + Artifact: + Checksum: 6c3cc3b955bce1686036ae6822ee2ca0ef6ecb994e3f2d19eaf3ec03dcba84b3 + Last Update Time: 2022-02-13T11:24:10Z + Path: helmchart/default/podinfo/podinfo-5.2.1.tgz + Revision: 5.2.1 + URL: http://source-controller.flux-system.svc.cluster.local./helmchart/default/podinfo/podinfo-5.2.1.tgz + Conditions: + Last Transition Time: 2022-02-13T11:24:10Z + Message: pulled 'podinfo' chart with version '5.2.1' + Observed Generation: 1 + Reason: ChartPullSucceeded + Status: True + Type: Ready + Observed Chart Name: podinfo + Observed Generation: 1 + URL: http://source-controller.flux-system.svc.cluster.local./helmchart/default/podinfo/latest.tar.gz + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal ChartPullSucceeded 2m51s source-controller pulled 'podinfo' chart with version '5.2.1' + ``` + +## Writing a HelmChart spec + +As with all other Kubernetes config, a HelmChart needs `apiVersion`, `kind`, and +`metadata` fields. The name of a HelmChart object must be a valid +[DNS subdomain name](https://kubernetes.io/docs/concepts/overview/working-with-objects/names#dns-subdomain-names). + +A HelmChart also needs a +[`.spec` section](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status). + +### Interval + +`.spec.interval` is a required field that specifies the interval at which the +Helm Chart source must be checked for updates. + +After successfully reconciling a HelmChart object, the source-controller +requeues the object for inspection after the specified interval. The value must +be in a [Go recognized duration string format](https://pkg.go.dev/time#ParseDuration), +e.g. `10m0s` to look at the source for updates every 10 minutes. + +If the `.metadata.generation` of a resource changes (due to e.g. applying a +change to the spec), this is handled instantly outside the interval window. + +### Source reference + +`.spec.sourceRef` is a required field that specifies a reference to the Source +the chart is available at. `.spec.sourceRef.kind` must be one of +`HelmRepository`, `GitRepository` or `Bucket`. `.spec.sourceRef.name` is the +name of the referred kind. + +### Chart + +`.spec.chart` is a required field that specifies the name or path the Helm chart +is available at in the [Source reference](#source-reference). For HelmRepository +Source reference, it'll be just the name of the chart. For GitRepository and +Bucket Source reference, it'll be the path to the Helm chart directory. + +### Version + +`.spec.version` is an optional field to specify the version of the chart in +semver. It is applicable only when the Source reference is a HelmRepository. It +is ignored for GitRepository and Bucket Source reference. It defaults to the +latest version of the chart with value `*`. + +### Values files + +`.spec.valuesFiles` is an optional field to specify an alternative list of +values files to use as the chart values (values.yaml). The file paths are +expected to be relative to the Source reference. Values files are merged in the +order of the list with the last file overriding the first. It is ignored when +omitted. + +### Reconcile strategy + +`.spec.reconcileStrategy` is an optional field to specify what enables the +creation of a new Artifact. Valid values are `ChartVersion` and `Revision`. +`ChartVersion` is used for creating a new artifact when the chart version +changes in the HelmRepository. `Revision` is used for creating a new artifact +when the source revision changes in GitRepository or Bucket Source references. +It defaults to `ChartVersion`. + +### Suspend + +`.spec.suspend` is an optional field to suspend the reconciliation of a +HelmChart. When set to `true`, the controller will stop reconciling the +HelmChart, and changes to the resource or the Helm chart Source will not result +in a new Artifact. When the field is set to `false` or removed, it will resume. + +For practical information, see +[suspending and resuming](#suspending-and-resuming). + +## Working with HelmCharts + +### Triggering a reconcile + +To manually tell the source-controller to reconcile a HelmChart outside the +[specified interval window](#interval), a HelmCHart can be annotated with +`reconcile.fluxcd.io/requestedAt: `. Annotating the resource +queues the object for reconciliation if the `` differs from +the last value the controller acted on, as reported in +[`.status.lastHandledReconcileAt`](#last-handled-reconcile-at). + +Using `kubectl`: + +```sh +kubectl annotate --overwrite helmchart/ reconcile.fluxcd.io/requestedAt="$(date +%s)" +``` + +### Waiting for `Ready` + +When a change is applied, it is possible to wait for the HelmChart to reach a +[ready state](#ready-helmchart) using `kubectl`: + +```sh +kubectl wait helmchart/ --for=condition=ready --timeout=1m +``` + +### Suspending and resuming + +When you find yourself in a situation where you temporarily want to pause the +reconciliation of a HelmChart, you can suspend it using the +[`.spec.suspend` field](#suspend). + +#### Suspend a HelmChart + +In your YAML declaration: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmChart +metadata: + name: +spec: + suspend: true +``` + +Using `kubectl`: + +```sh +kubectl patch helmchart -p '{\"spec\": {\"suspend\" : true }}' +``` + +**Note:** When a HelmChart has an Artifact and is suspended, and this +Artifact later disappears from the storage due to e.g. the source-controller +Pod being evicted from a Node, this will not be reflected in the +HelmChart's Status until it is resumed. + +#### Resume a HelmChart + +In your YAML declaration, comment out (or remove) the field: + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmChart +metadata: + name: +spec: + # suspend: true +``` + +**Note:** Setting the field value to `false` has the same effect as removing +it, but does not allow for "hot patching" using e.g. `kubectl` while practicing +GitOps; as the manually applied patch would be overwritten by the declared +state in Git. + +Using `kubectl`: + +```sh +kubectl patch helmchart -p '{\"spec\" : {\"suspend\" : false }}' +``` + +### Debugging a HelmChart + +There are several ways to gather information about a HelmChart for debugging +purposes. + +#### Describe the HelmChart + +Describing a HelmChart using `kubectl describe helmchart ` displays +the latest recorded information for the resource in the `Status` and `Events` +sections: + +```console +... +Status: +... + Conditions: + Last Transition Time: 2022-02-13T14:06:27Z + Message: invalid chart reference: failed to get chart version for remote reference: no 'podinfo' chart with version matching '9.*' found + Observed Generation: 3 + Reason: InvalidChartReference + Status: True + Type: Stalled + Last Transition Time: 2022-02-13T14:06:27Z + Message: invalid chart reference: failed to get chart version for remote reference: no 'podinfo' chart with version matching '9.*' found + Observed Generation: 3 + Reason: InvalidChartReference + Status: False + Type: Ready + Last Transition Time: 2022-02-13T14:06:27Z + Message: invalid chart reference: failed to get chart version for remote reference: no 'podinfo' chart with version matching '9.*' found + Observed Generation: 3 + Reason: InvalidChartReference + Status: True + Type: FetchFailed + Last Handled Reconcile At: 1644759954 + Observed Chart Name: podinfo + Observed Generation: 3 + URL: http://source-controller.flux-system.svc.cluster.local./helmchart/default/podinfo/latest.tar.gz +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning InvalidChartReference 11s source-controller invalid chart reference: failed to get chart version for remote reference: no 'podinfo' chart with ver +sion matching '9.*' found +``` + +#### Trace emitted Events + +To view events for specific HelmChart(s), `kubectl get events` can be used in +combination with `--field-selector` to list the Events for specific objects. +For example, running + +```sh +kubectl get events --field-selector involvedObject.kind=HelmChart,involvedObject.name= +``` + +lists + +```console +LAST SEEN TYPE REASON OBJECT MESSAGE +22s Warning InvalidChartReference helmchart/ invalid chart reference: failed to get chart version for remote reference: no 'podinfo' chart with version matching '9.*' found +2s Normal ChartPullSucceeded helmchart/ pulled 'podinfo' chart with version '6.0.3' +``` + +Besides being reported in Events, the reconciliation errors are also logged by +the controller. The Flux CLI offer commands for filtering the logs for a +specific HelmChart, e.g. `flux logs --level=error --kind=HelmChart --name=`. + +## HelmChart Status + +### Artifact + +The HelmChart reports the last built chart as an Artifact object in the +`.status.artifact` of the resource. + +The Artifact file is a gzip compressed TAR archive (`-.tgz`), +and can be retrieved in-cluster from the `.status.artifact.url` HTTP address. + +### Artifact example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmChart +metadata: + name: +status: + artifact: + checksum: e30b95a08787de69ffdad3c232d65cfb131b5b50c6fd44295f48a078fceaa44e + lastUpdateTime: "2022-02-10T18:53:47Z" + path: helmchart///-.tgz + revision: 6.0.3 + url: http://source-controller.flux-system.svc.cluster.local./helmchart///-.tgz +``` + +### Conditions + +A HelmChart enters various states during its lifecycle, reflected as [Kubernetes +Conditions][typical-status-properties]. +It can be [reconciling](#reconciling-helmchart) while fetching or building the +chart, it can be [ready](#ready-helmchart), it can +[fail during reconciliation](#failed-helmchart), or it can +[stall](#stalled-helmchart). + +The HelmChart API is compatible with the [kstatus +specification][kstatus-spec], +and reports `Reconciling` and `Stalled` conditions where applicable to +provide better (timeout) support to solutions polling the HelmChart to become +`Ready`. + +#### Reconciling HelmChart + +The source-controller marks a HelmChart as _reconciling_ when one of the +following is true: + +- There is no current Artifact for the HelmChart, or the reported Artifact is + determined to have disappeared from the storage. +- The generation of the HelmChart is newer than the [Observed + Generation](#observed-generation). +- The newly fetched Artifact revision differs from the current Artifact. + +When the HelmChart is "reconciling", the `Ready` Condition status becomes +`False`, and the controller adds a Condition with the following attributes to +the HelmChart's `.status.conditions`: + +- `type: Reconciling` +- `status: "True"` +- `reason: NewGeneration` | `reason: NoArtifact` + +If the reconciling state is due to a new version, it adds an additional +Condition with the following attributes: + +- `type: ArtifactOutdated` +- `status: "True"` +- `reason: NewChart` + +Both Conditions have a ["negative polarity"][typical-status-properties], +and are only present on the HelmChart while their status value is `"True"`. + +#### Ready HelmChart + +The source-controller marks a HelmChart as _ready_ when it has the following +characteristics: + +- The HelmChart reports an [Artifact](#artifact). +- The reported Artifact exists in the controller's Artifact storage. +- The controller was able to fetch and build the Helm chart using the current + spec. +- The version/revision of the reported Artifact is up-to-date with the + latest version/revision of the Helm chart. + +When the HelmChart is "ready", the controller sets a Condition with the +following attributes in the HelmChart's `.status.conditions`: + +- `type: Ready` +- `status: "True"` +- `reason: Succeeded` + +This `Ready` Condition will retain a status value of `"True"` until the +HelmChart is marked as [reconciling](#reconciling-helmchart), or e.g. +a [transient error](#failed-helmchart) occurs due to a temporary network issue. + +#### Failed HelmChart + +The source-controller may get stuck trying to produce an Artifact for a +HelmChart without completing. This can occur due to some of the following +factors: + +- The Helm chart Source is temporarily unavailable. +- The credentials in the [Source reference](#source-reference) Secret are + invalid. +- The HelmChart spec contains a generic misconfiguration. + +When this happens, the controller sets the `Ready` Condition status to `False`, +and adds a Condition with the following attributes to the HelmChart's +`.status.conditions`: + +- `type: FetchFailed` +- `status: "True"` +- `reason: AuthenticationFailed` | `reason: StorageOperationFailed` | `reason: URLInvalid` | `reason: IllegalPath` | `reason: Failed` + +This condition has a ["negative polarity"][typical-status-properties], +and is only present on the HelmChart while the status value is `"True"`. + +While the HelmChart has this Condition, the controller will continue to +attempt to produce an Artifact for the resource with an exponential backoff, +until it succeeds and the HelmChart is marked as [ready](#ready-helmchart). + +Note that a HelmChart can be [reconciling](#reconciling-helmchart) +while failing at the same time, for example due to a newly introduced +configuration issue in the HelmChart spec. + +#### Stalled HelmChart + +The source-controller can mark a HelmChart as _stalled_ when it determines that +without changes to the spec, the reconciliation can not succeed. +For example because a HelmChart Version is set to a non-existing version. + +When this happens, the controller sets the same Conditions as when it +[fails](#failed-helmchart), but adds another Condition with the following +attributes to the HelmChart's `.status.conditions`: + +- `type: Stalled` +- `status: "True"` +- `reason: InvalidChartReference` + +While the HelmChart has this Condition, the controller will not requeue the +resource any further, and will stop reconciling the resource until a change to +the spec is made. + +### Observed Source Artifact Revision + +The source-controller reports the revision of the +[Source reference's](#source-reference) Artifact the current chart was fetched +from in the HelmChart's `.status.observedSourceArtifactRevision`. + +### Observed Chart Name + +The source-controller reports the resolved chart name of the current Artifact +for the [`.spec.chart` field](#chart) in the HelmChart's +`.status.observedChartName`. + +### Observed Generation + +The source-controller reports an [observed generation][typical-status-properties] +in the HelmChart's `.status.observedGeneration`. The observed generation is the +latest `.metadata.generation` which resulted in either a [ready state](#ready-helmchart), +or stalled due to error it can not recover from without human +intervention. + +### Last Handled Reconcile At + +The source-controller reports the last `reconcile.fluxcd.io/requestedAt` +annotation value it acted on in the `.status.lastHandledReconcileAt` field. + +For practical information about this field, see [triggering a +reconcile](#triggering-a-reconcile). + +[typical-status-properties]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties +[kstatus-spec]: https://github.com/kubernetes-sigs/cli-utils/tree/master/pkg/kstatus From 9e73bfab9eb33aa7e18315e9073a9b2c0617b556 Mon Sep 17 00:00:00 2001 From: Sunny Date: Fri, 25 Feb 2022 17:45:46 +0530 Subject: [PATCH 5/9] docs/spec: Update the outputs Signed-off-by: Sunny --- docs/spec/v1beta2/buckets.md | 12 +++++------- docs/spec/v1beta2/gitrepositories.md | 14 ++++++-------- docs/spec/v1beta2/helmcharts.md | 4 ++-- docs/spec/v1beta2/helmrepositories.md | 8 ++++---- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/docs/spec/v1beta2/buckets.md b/docs/spec/v1beta2/buckets.md index a74e0a50a..f265ac208 100644 --- a/docs/spec/v1beta2/buckets.md +++ b/docs/spec/v1beta2/buckets.md @@ -70,8 +70,8 @@ control over. 2. Run `kubectl get buckets` to see the Bucket: ```console - NAME ENDPOINT READY STATUS AGE - minio-bucket minio.example.com True stored artifact for revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' 34s + NAME ENDPOINT AGE READY STATUS + minio-bucket minio.example.com 34s True stored artifact for revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' ``` 3. Run `kubectl describe bucket minio-bucket` to see the [Artifact](#artifact) @@ -98,8 +98,7 @@ control over. Events: Type Reason Age From Message ---- ------ ---- ---- ------- - Normal BucketOperationSucceed 43s source-controller downloaded 16 files with revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' from 'my-minio-bucket' - Normal NewArtifact 43s source-controller stored artifact for revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' + Normal NewArtifact 82s source-controller fetched 16 files from 'example' ``` ## Writing a Bucket spec @@ -575,7 +574,7 @@ Status: ... Conditions: Last Transition Time: 2022-02-02T13:26:55Z - Message: reconciling new generation 2 + Message: reconciling new object generation (2) Observed Generation: 2 Reason: NewGeneration Status: True @@ -614,8 +613,7 @@ lists ```console LAST SEEN TYPE REASON OBJECT MESSAGE -2m30s Normal BucketOperationSucceed bucket/ downloaded 16 files with revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' from 'my-minio-bucket' -2m30s Normal NewArtifact bucket/ stored artifact for revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' +2m30s Normal NewArtifact bucket/ fetched 16 files with revision from 'my-new-bucket' 18s Warning BucketOperationFailed bucket/ bucket 'my-new-bucket' does not exist ``` diff --git a/docs/spec/v1beta2/gitrepositories.md b/docs/spec/v1beta2/gitrepositories.md index 594e3da2d..25976266a 100644 --- a/docs/spec/v1beta2/gitrepositories.md +++ b/docs/spec/v1beta2/gitrepositories.md @@ -48,8 +48,8 @@ You can run this example by saving the manifest into `gitrepository.yaml`. 2. Run `kubectl get gitrepository` to see the GitRepository: ```console - NAME URL READY STATUS AGE - podinfo https://github.com/stefanprodan/podinfo True stored artifact for revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' 5s + NAME URL AGE READY STATUS + podinfo https://github.com/stefanprodan/podinfo 5s True stored artifact for revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' ``` 3. Run `kubectl describe gitrepository podinfo` to see the [Artifact](#artifact) @@ -76,8 +76,7 @@ You can run this example by saving the manifest into `gitrepository.yaml`. Events: Type Reason Age From Message ---- ------ ---- ---- ------- - Normal GitOperationSucceed 62s source-controller cloned 'https://github.com/stefanprodan/podinfo' and checked out revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' - Normal NewArtifact 62s source-controller stored artifact for revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' + Normal NewArtifact 62s source-controller stored artifact for commit 'Merge pull request #160 from stefanprodan/release-6.0.3' ``` ## Writing a GitRepository spec @@ -597,7 +596,7 @@ Status: ... Conditions: Last Transition Time: 2022-02-14T09:40:27Z - Message: reconciling new generation 2 + Message: reconciling new object generation (2) Observed Generation: 2 Reason: NewGeneration Status: True @@ -636,9 +635,8 @@ lists ```console LAST SEEN TYPE REASON OBJECT MESSAGE -2m14s Normal GitOperationSucceed gitrepository/ cloned 'https://github.com/stefanprodan/podinfo' and checked out revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' -2m14s Normal NewArtifact gitrepository/ stored artifact for revision 'master/132f4e719209eb10b9485302f8593fc0e680f4fc' -94s Warning GitOperationFailed gitrepository/gitrepository-sample failed to checkout and determine revision: unable to clone 'https://github.com/stefanprodan/podinfo': couldn't find remote ref "refs/heads/invalid" +2m14s Normal NewArtifact gitrepository/ stored artifact for commit 'Merge pull request #160 from stefanprodan/release-6.0.3' +94s Warning GitOperationFailed gitrepository/ failed to checkout and determine revision: unable to clone 'https://github.com/stefanprodan/podinfo': couldn't find remote ref "refs/heads/invalid" ``` Besides being reported in Events, the reconciliation errors are also logged by diff --git a/docs/spec/v1beta2/helmcharts.md b/docs/spec/v1beta2/helmcharts.md index 2e78131fb..fa5f0af47 100644 --- a/docs/spec/v1beta2/helmcharts.md +++ b/docs/spec/v1beta2/helmcharts.md @@ -56,8 +56,8 @@ helm-controller. 2. Run `kubectl get helmchart` to see the HelmChart: ```console - NAME CHART VERSION SOURCE KIND SOURCE NAME READY STATUS AGE - podinfo podinfo 5.* HelmRepository podinfo True pulled 'podinfo' chart with version '5.2.1' 53s + NAME CHART VERSION SOURCE KIND SOURCE NAME AGE READY STATUS + podinfo podinfo 5.* HelmRepository podinfo 53s True pulled 'podinfo' chart with version '5.2.1' ``` 3. Run `kubectl describe helmchart podinfo` to see the [Artifact](#artifact) and diff --git a/docs/spec/v1beta2/helmrepositories.md b/docs/spec/v1beta2/helmrepositories.md index d92a7ab56..0d503cc82 100644 --- a/docs/spec/v1beta2/helmrepositories.md +++ b/docs/spec/v1beta2/helmrepositories.md @@ -46,8 +46,8 @@ You can run this example by saving the manifest into `helmrepository.yaml`. 2. Run `kubectl get helmrepository` to see the HelmRepository: ```console - NAME URL READY STATUS AGE - podinfo https://stefanprodan.github.io/podinfo True stored artifact for revision '83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111' 4s + NAME URL AGE READY STATUS + podinfo https://stefanprodan.github.io/podinfo 4s True stored artifact for revision '83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111' ``` 3. Run `kubectl describe helmrepository podinfo` to see the [Artifact](#artifact) @@ -74,7 +74,7 @@ You can run this example by saving the manifest into `helmrepository.yaml`. Events: Type Reason Age From Message ---- ------ ---- ---- ------- - Normal NewArtifact 1m source-controller stored artifact for revision '83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111' + Normal NewArtifact 1m source-controller fetched index of size 30.88kB from 'https://stefanprodan.github.io/podinfo' ``` ## Writing a HelmRepository spec @@ -361,7 +361,7 @@ lists ```console LAST SEEN TYPE REASON OBJECT MESSAGE 107s Warning Failed helmrepository/ failed to construct Helm client: scheme "invalid" not supported -7s Normal NewArtifact helmrepository/ stored artifact for revision '83a3c595163a6ff0333e0154c790383b5be441b9db632cb36da11db1c4ece111' +7s Normal NewArtifact helmrepository/ fetched index of size 30.88kB from 'https://stefanprodan.github.io/podinfo' ``` Besides being reported in Events, the reconciliation errors are also logged by From 8d4fce1955a7c5a63ce60605a39f1ea844a3b813 Mon Sep 17 00:00:00 2001 From: Sunny Date: Fri, 25 Feb 2022 20:37:04 +0530 Subject: [PATCH 6/9] Add more details in helmchart spec doc Details about the source reference, reconcile strategy and artifact revision value based on the reconcile strategy. Signed-off-by: Sunny --- controllers/helmchart_controller.go | 5 +- docs/spec/v1beta2/helmcharts.md | 182 +++++++++++++++++++++++----- 2 files changed, 156 insertions(+), 31 deletions(-) diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index 9f5e8dcda..a63145c75 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -475,8 +475,9 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj * return sreconcile.ResultSuccess, nil } -// buildFromHelmRepository attempts to pull and/or package a Helm chart with -// the specified data v1beta2.HelmChart object and the given v1beta2.Artifact. +// buildFromTarballArtifact attempts to pull and/or package a Helm chart with +// the specified data from the v1beta2.HelmChart object and the given +// v1beta2.Artifact. // In case of a failure it records v1beta2.FetchFailedCondition on the chart // object, and returns early. func (r *HelmChartReconciler) buildFromTarballArtifact(ctx context.Context, obj *sourcev1.HelmChart, source sourcev1.Artifact, b *chart.Build) (sreconcile.Result, error) { diff --git a/docs/spec/v1beta2/helmcharts.md b/docs/spec/v1beta2/helmcharts.md index fa5f0af47..b554b95a6 100644 --- a/docs/spec/v1beta2/helmcharts.md +++ b/docs/spec/v1beta2/helmcharts.md @@ -97,39 +97,78 @@ As with all other Kubernetes config, a HelmChart needs `apiVersion`, `kind`, and A HelmChart also needs a [`.spec` section](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status). -### Interval +### Source reference -`.spec.interval` is a required field that specifies the interval at which the -Helm Chart source must be checked for updates. +`.spec.sourceRef` is a required field that specifies a reference to the Source +the chart is available at. -After successfully reconciling a HelmChart object, the source-controller -requeues the object for inspection after the specified interval. The value must -be in a [Go recognized duration string format](https://pkg.go.dev/time#ParseDuration), -e.g. `10m0s` to look at the source for updates every 10 minutes. +Supported references are: +- [`HelmRepository`](helmrepositories.md) +- [`GitRepository`](gitrepositories.md) +- [`Bucket`](buckets.md) -If the `.metadata.generation` of a resource changes (due to e.g. applying a -change to the spec), this is handled instantly outside the interval window. +Although there are three kinds of source references, there are only two +underlying implementations. The artifact building process for `GitRepository` +and `Bucket` are the same as they are already built source artifacts. In case +of `HelmRepository`, a chart is fetched and/or packaged based on the +configuration of the Helm chart. -### Source reference +For a `HelmChart` to be reconciled, the associated artifact in the source +reference must be ready. If the source artifact is not ready, the `HelmChart` +reconciliation is retried. -`.spec.sourceRef` is a required field that specifies a reference to the Source -the chart is available at. `.spec.sourceRef.kind` must be one of -`HelmRepository`, `GitRepository` or `Bucket`. `.spec.sourceRef.name` is the -name of the referred kind. +When the `metadata.generation` of the `HelmChart` don't match with the +`status.observedGeneration`, the chart is fetched from source and/or packaged. +If there's no `.spec.valuesFiles` specified, the chart is only fetched from the +source, and not packaged. If `.spec.valuesFiles` are specified, the chart is +fetched and packaged with the values files. When the `metadata.generation` +matches the `status.observedGeneration`, the chart is only fetched from source +or from the cache if available, and not packaged. + +When using a `HelmRepository` source reference, the secret reference defined in +the Helm repository is used to fetch the chart. + +The HelmChart reconciliation behavior varies depending on the source reference +kind, see [reconcile strategy](#reconcile-strategy). + +The attributes of the generated artifact also varies depending on the source +reference kind, see [artifact](#artifact). ### Chart `.spec.chart` is a required field that specifies the name or path the Helm chart -is available at in the [Source reference](#source-reference). For HelmRepository -Source reference, it'll be just the name of the chart. For GitRepository and -Bucket Source reference, it'll be the path to the Helm chart directory. +is available at in the [Source reference](#source-reference). + +For `HelmRepository` Source reference, it'll be just the name of the chart. + +```yaml +spec: + chart: podinfo + sourceRef: + name: podinfo + kind: HelmRepository +``` + +For `GitRepository` and `Bucket` Source reference, it'll be the path to the +Helm chart directory. + +```yaml +spec: + chart: ./charts/podinfo + sourceRef: + name: podinfo + kind: +``` ### Version `.spec.version` is an optional field to specify the version of the chart in -semver. It is applicable only when the Source reference is a HelmRepository. It -is ignored for GitRepository and Bucket Source reference. It defaults to the -latest version of the chart with value `*`. +semver. It is applicable only when the Source reference is a `HelmRepository`. +It is ignored for `GitRepository` and `Bucket` Source reference. It defaults to +the latest version of the chart with value `*`. + +Version can be a fixed semver, minor or patch semver range of a specific +version (i.e. `4.0.x`) or any semver range (i.e. `>=4.0.0 <5.0.0`). ### Values files @@ -137,16 +176,53 @@ latest version of the chart with value `*`. values files to use as the chart values (values.yaml). The file paths are expected to be relative to the Source reference. Values files are merged in the order of the list with the last file overriding the first. It is ignored when -omitted. +omitted. When values files are specified, the chart is fetched and packaged +with the provided values. + +```yaml +spec: + chart: + spec: + chart: podinfo + ... + valuesFiles: + - values.yaml + - values-production.yaml +``` + +Values files also affect the generated artifact revision, see +[artifact](#artifact). ### Reconcile strategy `.spec.reconcileStrategy` is an optional field to specify what enables the creation of a new Artifact. Valid values are `ChartVersion` and `Revision`. `ChartVersion` is used for creating a new artifact when the chart version -changes in the HelmRepository. `Revision` is used for creating a new artifact -when the source revision changes in GitRepository or Bucket Source references. -It defaults to `ChartVersion`. +changes in a `HelmRepository`. `Revision` is used for creating a new artifact +when the source revision changes in a `GitRepository` or a `Bucket` Source. It +defaults to `ChartVersion`. + +**NOTE:** If the reconcile strategy is `ChartVersion` and the source reference +is a `GitRepository` or a `Bucket`, no new chart artifact is produced on updates +to the source unless the `version` in `Chart.yaml` is incremented. To produce +new chart artifact on change in source revision, set the reconcile strategy to +`Revision`. + +Reconcile strategy also affects the artifact version, see [artifact](#artifact) +for more details. + +### Interval + +`.spec.interval` is a required field that specifies the interval at which the +Helm Chart source must be checked for updates. + +After successfully reconciling a HelmChart object, the source-controller +requeues the object for inspection after the specified interval. The value must +be in a [Go recognized duration string format](https://pkg.go.dev/time#ParseDuration), +e.g. `10m0s` to look at the source for updates every 10 minutes. + +If the `.metadata.generation` of a resource changes (due to e.g. applying a +change to the spec), this is handled instantly outside the interval window. ### Suspend @@ -317,7 +393,7 @@ The HelmChart reports the last built chart as an Artifact object in the The Artifact file is a gzip compressed TAR archive (`-.tgz`), and can be retrieved in-cluster from the `.status.artifact.url` HTTP address. -### Artifact example +#### Artifact example ```yaml --- @@ -334,6 +410,51 @@ status: url: http://source-controller.flux-system.svc.cluster.local./helmchart///-.tgz ``` +When using a `HelmRepository` as the source reference and values files are +provided, the value of `status.artifact.revision` is the chart version combined +with the `HelmChart` object generation. For example, if the chart version is +`6.0.3` and the `HelmChart` object generation is `1`, the +`status.artifact.revision` value will be `6.0.3+1`. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmChart +metadata: + name: +status: + artifact: + checksum: ee68224ded207ebb18a8e9730cf3313fa6bc1f31e6d8d3943ab541113559bb52 + lastUpdateTime: "2022-02-28T08:07:12Z" + path: helmchart///-6.0.3+1.tgz + revision: 6.0.3+1 + url: http://source-controller.flux-system.svc.cluster.local./helmchart///-6.0.3+1.tgz + observedGeneration: 1 + ... +``` + +When using a `GitRepository` or a `Bucket` as the source reference and +`Revision` as the reconcile strategy, the value of `status.artifact.revision` is +the chart version combined with the first 12 characters of the revision of the +`GitRepository` or `Bucket`. For example if the chart version is `6.0.3` and the +revision of the `Bucket` is `4e5cbb7b97d00a8039b8810b90b922f4256fd3bd8f78b934b4892dae13f7ca87`, +the `status.artifact.revision` value will be `6.0.3+4e5cbb7b97d0`. + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmChart +metadata: + name: +status: + artifact: + checksum: 8d1f0ac3f4b0e8759a32180086f17ac87ca04e5d46c356e67f97e97616ef4718 + lastUpdateTime: "2022-02-28T08:07:12Z" + path: helmchart///-6.0.3+4e5cbb7b97d0.tgz + revision: 6.0.3+4e5cbb7b97d0 + url: http://source-controller.flux-system.svc.cluster.local./helmchart///-6.0.3+4e5cbb7b97d0.tgz +``` + ### Conditions A HelmChart enters various states during its lifecycle, reflected as [Kubernetes @@ -451,15 +572,18 @@ the spec is made. ### Observed Source Artifact Revision -The source-controller reports the revision of the +The source-controller reports the revision of the last [Source reference's](#source-reference) Artifact the current chart was fetched -from in the HelmChart's `.status.observedSourceArtifactRevision`. +from in the HelmChart's `.status.observedSourceArtifactRevision`. It is used to +keep track of the source artifact revision and detect when a new source +artifact is available. ### Observed Chart Name -The source-controller reports the resolved chart name of the current Artifact +The source-controller reports the last resolved chart name of the Artifact for the [`.spec.chart` field](#chart) in the HelmChart's -`.status.observedChartName`. +`.status.observedChartName`. It is used to keep track of the chart and detect +when a new chart is found. ### Observed Generation From abf29da106adaa040de7dc38b4e63906a85a5ef1 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 8 Mar 2022 14:17:43 +0100 Subject: [PATCH 7/9] docs/spec: document Bucket's Azure Blob support Signed-off-by: Hidde Beydals --- docs/spec/v1beta2/buckets.md | 170 ++++++++++++++++++++++++++++++++++- 1 file changed, 168 insertions(+), 2 deletions(-) diff --git a/docs/spec/v1beta2/buckets.md b/docs/spec/v1beta2/buckets.md index f265ac208..c40bbd02a 100644 --- a/docs/spec/v1beta2/buckets.md +++ b/docs/spec/v1beta2/buckets.md @@ -120,6 +120,7 @@ Supported options are: - [Generic](#generic) - [AWS](#aws) +- [Azure](#azure) - [GCP](#gcp) If you do not specify `.spec.provider`, it defaults to `generic`. @@ -260,6 +261,171 @@ data: secretkey: ``` +#### Azure + +When a Bucket's `.spec.provider` is set to `azure`, the source-controller will +attempt to communicate with the specified [Endpoint](#endpoint) using the +[Azure Blob Storage SDK for Go](https://github.com/Azure/azure-sdk-for-go/tree/main/sdk/storage/azblob). + +Without a [Secret reference](#secret-reference), authentication using a chain +with: + +- [Environment credentials](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#EnvironmentCredential) +- [Managed Identity](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#ManagedIdentityCredential) + with the `AZURE_CLIENT_ID` +- Managed Identity with a system-assigned identity + +is attempted by default. If no chain can be established, the bucket +is assumed to be publicly reachable. + +When a reference is specified, it expects a Secret with one of the following +sets of `.data` fields: + +- `tenantId`, `clientId` and `clientSecret` for authenticating a Service + Principal with a secret. +- `tenantId`, `clientId` and `clientCertificate` (plus optionally + `clientCertificatePassword` and/or `clientCertificateSendChain`) for + authenticating a Service Principal with a certificate. +- `clientId` for authenticating using a Managed Identity. +- `accountKey` for authenticating using a + [Shared Key](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob#SharedKeyCredential). + +For any Managed Identity and/or Azure Active Directory authentication method, +the base URL can be configured using `.data.authorityHost`. If not supplied, +[`AzurePublicCloud` is assumed](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#AuthorityHost). + +##### Azure example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: azure-public + namespace: default +spec: + interval: 5m0s + provider: azure + bucketName: podinfo + endpoint: https://podinfoaccount.blob.core.windows.net + timeout: 30s +``` + +##### Azure Service Principal Secret example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: azure-service-principal-secret + namespace: default +spec: + interval: 5m0s + provider: azure + bucketName: + endpoint: https://.blob.core.windows.net + secretRef: + name: azure-sp-auth +--- +apiVersion: v1 +kind: Secret +metadata: + name: azure-sp-auth + namespace: default +type: Opaque +data: + tenantId: + clientId: + clientSecret: +``` + +##### Azure Service Principal Certificate example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: azure-service-principal-cert + namespace: default +spec: + interval: 5m0s + provider: azure + bucketName: + endpoint: https://.blob.core.windows.net + secretRef: + name: azure-sp-auth +--- +apiVersion: v1 +kind: Secret +metadata: + name: azure-sp-auth + namespace: default +type: Opaque +data: + tenantId: + clientId: + clientCertificate: + # Plus optionally + clientCertificatePassword: + clientCertificateSendChain: # either "1" or "true" +``` + +##### Azure Managed Identity with Client ID example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: azure-managed-identity + namespace: default +spec: + interval: 5m0s + provider: azure + bucketName: + endpoint: https://.blob.core.windows.net + secretRef: + name: azure-smi-auth +--- +apiVersion: v1 +kind: Secret +metadata: + name: azure-smi-auth + namespace: default +type: Opaque +data: + clientId: +``` + +##### Azure Blob Shared Key example + +```yaml +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: Bucket +metadata: + name: azure-shared-key + namespace: default +spec: + interval: 5m0s + provider: azure + bucketName: + endpoint: https://.blob.core.windows.net + secretRef: + name: azure-key +--- +apiVersion: v1 +kind: Secret +metadata: + name: azure-key + namespace: default +type: Opaque +data: + accountKey: +``` + #### GCP When a Bucket's `.spec.provider` is set to `gcp`, the source-controller will @@ -281,7 +447,7 @@ The Provider allows for specifying the ```yaml --- -apiVersion: source.toolkit.fluccd.io/v1beta2 +apiVersion: source.toolkit.fluxcd.io/v1beta2 kind: Bucket metadata: name: gcp-workload-identity @@ -299,7 +465,7 @@ spec: ```yaml --- -apiVersion: source.toolkit.fluccd.io/v1beta1 +apiVersion: source.toolkit.fluxcd.io/v1beta2 kind: Bucket metadata: name: gcp-secret From ff536e10bb5c13a98eec9c30b69775e55d10829d Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Wed, 9 Mar 2022 12:44:29 +0100 Subject: [PATCH 8/9] doc/spec: use --field-manager in kubectl examples To align with: https://fluxcd.io/docs/faq/#why-are-kubectl-edits-rolled-back-by-flux Signed-off-by: Hidde Beydals --- docs/spec/v1beta2/buckets.md | 6 +++--- docs/spec/v1beta2/gitrepositories.md | 6 +++--- docs/spec/v1beta2/helmcharts.md | 6 +++--- docs/spec/v1beta2/helmrepositories.md | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/spec/v1beta2/buckets.md b/docs/spec/v1beta2/buckets.md index c40bbd02a..7d75d342e 100644 --- a/docs/spec/v1beta2/buckets.md +++ b/docs/spec/v1beta2/buckets.md @@ -637,7 +637,7 @@ the last value the controller acted on, as reported in Using `kubectl`: ```sh -kubectl annotate --overwrite bucket/ reconcile.fluxcd.io/requestedAt="$(date +%s)" +kubectl annotate --field-manager=flux-client-side-apply --overwrite bucket/ reconcile.fluxcd.io/requestedAt="$(date +%s)" ``` Using `flux`: @@ -678,7 +678,7 @@ spec: Using `kubectl`: ```sh -kubectl patch bucket -p '{\"spec\": {\"suspend\" : true }}' +kubectl patch bucket --field-manager=flux-client-side-apply -p '{\"spec\": {\"suspend\" : true }}' ``` Using `flux`: @@ -714,7 +714,7 @@ state in Git. Using `kubectl`: ```sh -kubectl patch bucket -p '{\"spec\" : {\"suspend\" : false }}' +kubectl patch bucket --field-manager=flux-client-side-apply -p '{\"spec\" : {\"suspend\" : false }}' ``` Using `flux`: diff --git a/docs/spec/v1beta2/gitrepositories.md b/docs/spec/v1beta2/gitrepositories.md index 25976266a..7e59c294a 100644 --- a/docs/spec/v1beta2/gitrepositories.md +++ b/docs/spec/v1beta2/gitrepositories.md @@ -492,7 +492,7 @@ from the last value the controller acted on, as reported in Using `kubectl`: ```sh -kubectl annotate --overwrite gitrepository/ reconcile.fluxcd.io/requestedAt="$(date +%s)" +kubectl annotate --field-manager=flux-client-side-apply --overwrite gitrepository/ reconcile.fluxcd.io/requestedAt="$(date +%s)" ``` Using `flux`: @@ -533,7 +533,7 @@ spec: Using `kubectl`: ```sh -kubectl patch gitrepository -p '{\"spec\": {\"suspend\" : true }}' +kubectl patch gitrepository --field-manager=flux-client-side-apply -p '{\"spec\": {\"suspend\" : true }}' ``` Using `flux`: @@ -569,7 +569,7 @@ state in Git. Using `kubectl`: ```sh -kubectl patch gitrepository -p '{\"spec\" : {\"suspend\" : false }}' +kubectl patch gitrepository --field-manager=flux-client-side-apply -p '{\"spec\" : {\"suspend\" : false }}' ``` Using `flux`: diff --git a/docs/spec/v1beta2/helmcharts.md b/docs/spec/v1beta2/helmcharts.md index b554b95a6..ead65545e 100644 --- a/docs/spec/v1beta2/helmcharts.md +++ b/docs/spec/v1beta2/helmcharts.md @@ -248,7 +248,7 @@ the last value the controller acted on, as reported in Using `kubectl`: ```sh -kubectl annotate --overwrite helmchart/ reconcile.fluxcd.io/requestedAt="$(date +%s)" +kubectl annotate --field-manager=flux-client-side-apply --overwrite helmchart/ reconcile.fluxcd.io/requestedAt="$(date +%s)" ``` ### Waiting for `Ready` @@ -283,7 +283,7 @@ spec: Using `kubectl`: ```sh -kubectl patch helmchart -p '{\"spec\": {\"suspend\" : true }}' +kubectl patch helmchart --field-manager=flux-client-side-apply -p '{\"spec\": {\"suspend\" : true }}' ``` **Note:** When a HelmChart has an Artifact and is suspended, and this @@ -313,7 +313,7 @@ state in Git. Using `kubectl`: ```sh -kubectl patch helmchart -p '{\"spec\" : {\"suspend\" : false }}' +kubectl patch helmchart --field-manager=flux-client-side-apply -p '{\"spec\" : {\"suspend\" : false }}' ``` ### Debugging a HelmChart diff --git a/docs/spec/v1beta2/helmrepositories.md b/docs/spec/v1beta2/helmrepositories.md index 0d503cc82..e59fcb978 100644 --- a/docs/spec/v1beta2/helmrepositories.md +++ b/docs/spec/v1beta2/helmrepositories.md @@ -218,7 +218,7 @@ the last value the controller acted on, as reported in Using `kubectl`: ```sh -kubectl annotate --overwrite helmrepository/ reconcile.fluxcd.io/requestedAt="$(date +%s)" +kubectl annotate --field-manager=flux-client-side-apply --overwrite helmrepository/ reconcile.fluxcd.io/requestedAt="$(date +%s)" ``` Using `flux`: @@ -259,7 +259,7 @@ spec: Using `kubectl`: ```sh -kubectl patch helmrepository -p '{\"spec\": {\"suspend\" : true }}' +kubectl patch helmrepository --field-manager=flux-client-side-apply -p '{\"spec\": {\"suspend\" : true }}' ``` Using `flux`: @@ -295,7 +295,7 @@ state in Git. Using `kubectl`: ```sh -kubectl patch helmrepository -p '{\"spec\" : {\"suspend\" : false }}' +kubectl patch helmrepository --field-manager=flux-client-side-apply -p '{\"spec\" : {\"suspend\" : false }}' ``` Using `flux`: From 971caf92d579b05c153ac8e5bb3d33dcac688345 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 10 Mar 2022 18:26:52 +0100 Subject: [PATCH 9/9] controllers: finetune `eventLogf` (variant) docs Signed-off-by: Hidde Beydals --- controllers/bucket_controller.go | 9 +++++---- controllers/gitrepository_controller.go | 2 +- controllers/helmchart_controller.go | 2 +- controllers/helmrepository_controller.go | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index a05c04eff..7c8b40516 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -642,7 +642,7 @@ func (r *BucketReconciler) getBucketSecret(ctx context.Context, obj *sourcev1.Bu return secret, nil } -// eventLogf records event and logs at the same time. +// eventLogf records events, and logs at the same time. // // This log is different from the debug log in the EventRecorder, in the sense // that this is a simple log. While the debug log contains complete details @@ -651,9 +651,10 @@ func (r *BucketReconciler) eventLogf(ctx context.Context, obj runtime.Object, ev r.annotatedEventLogf(ctx, obj, nil, eventType, reason, messageFmt, args...) } -// annotatedEventLogf records annotated event and logs at the same time. This -// log is different from the debug log in the event recorder in the sense that -// this is a simple log, the event recorder debug log contains complete details +// annotatedEventLogf records annotated events, and logs at the same time. +// +// This log is different from the debug log in the EventRecorder, in the sense +// that this is a simple log. While the debug log contains complete details // about the event. func (r *BucketReconciler) annotatedEventLogf(ctx context.Context, obj runtime.Object, annotations map[string]string, eventType string, reason string, messageFmt string, args ...interface{}) { diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index 83f9f931c..5564b836a 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -673,7 +673,7 @@ func (r *GitRepositoryReconciler) garbageCollect(ctx context.Context, obj *sourc return nil } -// eventLogf records event and logs at the same time. +// eventLogf records events, and logs at the same time. // // This log is different from the debug log in the EventRecorder, in the sense // that this is a simple log. While the debug log contains complete details diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index a63145c75..7ea13ac60 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -952,7 +952,7 @@ func (r *HelmChartReconciler) requestsForBucketChange(o client.Object) []reconci return reqs } -// eventLogf records event and logs at the same time. +// eventLogf records events, and logs at the same time. // // This log is different from the debug log in the EventRecorder, in the sense // that this is a simple log. While the debug log contains complete details diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go index 8658f1dd3..b4f68c0f5 100644 --- a/controllers/helmrepository_controller.go +++ b/controllers/helmrepository_controller.go @@ -514,7 +514,7 @@ func (r *HelmRepositoryReconciler) garbageCollect(ctx context.Context, obj *sour return nil } -// eventLogf records event and logs at the same time. +// eventLogf records events, and logs at the same time. // // This log is different from the debug log in the EventRecorder, in the sense // that this is a simple log. While the debug log contains complete details
(Optional) -

ObservedGeneration is the last observed generation.

+

ObservedGeneration is the last observed generation of the HelmChart +object.

(Optional)

ObservedSourceArtifactRevision is the last observed Artifact.Revision -of the Source reference.

+of the HelmChartSpec.SourceRef.

(Optional) -

ObservedChartName is the last observed chart name as defined by the +

ObservedChartName is the last observed chart name as specified by the resolved chart reference.

(Optional) -

URL is the fetch link for the last chart pulled.

+

URL is the dynamic fetch link for the latest Artifact. +It is provided on a “best effort” basis, and using the precise +BucketStatus.Artifact data is recommended.

(Optional) -

Artifact represents the output of the last successful chart sync.

+

Artifact represents the output of the last successful reconciliation.