From bfa94a93a1cd7e2e0a6513570c307ff31ab9eb2c Mon Sep 17 00:00:00 2001 From: Johnathan Kupferer Date: Thu, 16 Apr 2026 17:43:53 -0400 Subject: [PATCH] Add feature to set labels and annotations on resources --- helm/crds/resourcehandles.yaml | 13 +++ helm/crds/resourcepools.yaml | 13 +++ helm/templates/crds/resourcehandles.yaml | 13 +++ helm/templates/crds/resourcepools.yaml | 13 +++ operator/operator.py | 4 +- operator/resourcehandle.py | 85 +++++++++------ operator/resourcepool.py | 12 +++ operator/resourceprovider.py | 6 ++ .../tasks/test-labels-annotations-01.yaml | 101 ++++++++++++++++++ .../roles/poolboy_test_simple/tasks/test.yaml | 1 + 10 files changed, 227 insertions(+), 34 deletions(-) create mode 100644 test/roles/poolboy_test_simple/tasks/test-labels-annotations-01.yaml diff --git a/helm/crds/resourcehandles.yaml b/helm/crds/resourcehandles.yaml index 2b08ca7..54a01f9 100644 --- a/helm/crds/resourcehandles.yaml +++ b/helm/crds/resourcehandles.yaml @@ -123,6 +123,12 @@ spec: Parameter values used with the ResourceProvider to generate resources list. type: object x-kubernetes-preserve-unknown-fields: true + resourceAnnotations: + description: >- + Name/value pairs to apply as annotations on resources created for this ResourceHandle. + type: object + additionalProperties: + type: string resourceClaim: description: >- ResourceClaim reference for claim matched to this ResourceHandle when the handle has been claimed. @@ -141,6 +147,13 @@ spec: type: string namespace: type: string + resourceLabels: + description: >- + Name/value pairs to apply as labels on resources created for this ResourceHandle. + type: object + additionalProperties: + type: string + pattern: '^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$' resourcePool: description: >- ResourcePool reference for pool that created this handle. diff --git a/helm/crds/resourcepools.yaml b/helm/crds/resourcepools.yaml index ef691ed..a11aed7 100644 --- a/helm/crds/resourcepools.yaml +++ b/helm/crds/resourcepools.yaml @@ -127,6 +127,19 @@ spec: Parameter values used with the ResourceProvider to generate resources list. type: object x-kubernetes-preserve-unknown-fields: true + resourceAnnotations: + description: >- + Name/value pairs to apply as annotations on resources created for ResourceHandles created for this ResourcePool. + type: object + additionalProperties: + type: string + resourceLabels: + description: >- + Name/value pairs to apply as labels on resources created for ResourceHandles created for this ResourcePool. + type: object + additionalProperties: + type: string + pattern: '^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$' resources: description: >- Resources description to apply to ResourceHandles for the pool. diff --git a/helm/templates/crds/resourcehandles.yaml b/helm/templates/crds/resourcehandles.yaml index 4520d96..68cdf8c 100644 --- a/helm/templates/crds/resourcehandles.yaml +++ b/helm/templates/crds/resourcehandles.yaml @@ -120,6 +120,12 @@ spec: Parameter values used with the ResourceProvider to generate resources list. type: object x-kubernetes-preserve-unknown-fields: true + resourceAnnotations: + description: >- + Name/value pairs to apply as annotations on resources created for this ResourceHandle. + type: object + additionalProperties: + type: string resourceClaim: description: >- ResourceClaim reference for claim matched to this ResourceHandle when the handle has been claimed. @@ -138,6 +144,13 @@ spec: type: string namespace: type: string + resourceLabels: + description: >- + Name/value pairs to apply as labels on resources created for this ResourceHandle. + type: object + additionalProperties: + type: string + pattern: '^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$' resourcePool: description: >- ResourcePool reference for pool that created this handle. diff --git a/helm/templates/crds/resourcepools.yaml b/helm/templates/crds/resourcepools.yaml index 189686e..053174d 100644 --- a/helm/templates/crds/resourcepools.yaml +++ b/helm/templates/crds/resourcepools.yaml @@ -128,6 +128,19 @@ spec: Parameter values used with the ResourceProvider to generate resources list. type: object x-kubernetes-preserve-unknown-fields: true + resourceAnnotations: + description: >- + Name/value pairs to apply as annotations on resources created for ResourceHandles created for this ResourcePool. + type: object + additionalProperties: + type: string + resourceLabels: + description: >- + Name/value pairs to apply as labels on resources created for ResourceHandles created for this ResourcePool. + type: object + additionalProperties: + type: string + pattern: '^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$' resources: description: >- Resources description to apply to ResourceHandles for the pool. diff --git a/operator/operator.py b/operator/operator.py index 1956372..a693f3e 100755 --- a/operator/operator.py +++ b/operator/operator.py @@ -280,7 +280,7 @@ async def resource_claim_daemon( except K8sApiException as exception: if exception.status != 404: raise - logger.info(f"{resource_claim} found deleted in daemon") + logger.info("%s found deleted in daemon", resource_claim) return if not resource_claim.ignore: await resource_claim.manage(logger=logger) @@ -396,7 +396,7 @@ async def resource_handle_daemon( except K8sApiException as exception: if exception.status != 404: raise - logger.info(f"{description} found deleted in daemon") + logger.info("%s found deleted in daemon", resource_handle) return if not resource_handle.ignore: await resource_handle.manage(logger=logger) diff --git a/operator/resourcehandle.py b/operator/resourcehandle.py index 86b56ba..ad14abd 100644 --- a/operator/resourcehandle.py +++ b/operator/resourcehandle.py @@ -115,12 +115,12 @@ async def bind_handle_to_claim( if resource_handle: try: await resource_handle.refetch() - logger.warning(f"Rebinding {resource_handle} to {resource_claim}") + logger.warning("Rebinding %s to %s", resource_handle, resource_claim) return resource_handle except K8sApiException as exception: if exception.status != 404: raise - logger.warning(f"Deleted {resource_handle} was still in memory cache") + logger.warning("Deleted %s was still in memory cache", resource_handle) claim_status_resources = resource_claim.status_resources @@ -275,16 +275,19 @@ async def bind_handle_to_claim( matched_resource_handle.__register() except K8sApiException as exception: if exception.status == 404: - logger.warning(f"Attempt to bind deleted {matched_resource_handle} to {resource_claim}") + logger.warning("Attempt to bind deleted %s to %s", matched_resource_handle, resource_claim) matched_resource_handle.__unregister() matched_resource_handle = None if exception.status == 422: - logger.warning(f"Attempt to bind {matched_resource_handle} to {resource_claim} failed, most likely handle already bound") + logger.warning( + "Attempt to bind %s to %s failed, most likely handle already bound", + matched_resource_handle, resource_claim + ) matched_resource_handle = None else: raise if matched_resource_handle: - logger.info(f"Bound {matched_resource_handle} to {resource_claim}") + logger.info("Bound %s to %s", matched_resource_handle, resource_claim) break else: # No unbound resource handle matched @@ -296,8 +299,8 @@ async def bind_handle_to_claim( await resource_pool.manage(logger=logger) else: logger.warning( - f"Unable to find ResourcePool {matched_resource_handle.resource_pool_name} for " - f"{matched_resource_handle} claimed by {resource_claim}" + "Unable to find ResourcePool %s for %s claimed by %s", + matched_resource_handle.resource_pool_name, matched_resource_handle, resource_claim ) return matched_resource_handle @@ -499,6 +502,12 @@ async def create_for_pool( if resource_pool.preference_score is not None: definition['spec']['preferenceScore'] = resource_pool.preference_score + if resource_pool.resource_annotations is not None: + definition['spec']['resourceAnnotations'] = resource_pool.resource_annotations + + if resource_pool.resource_labels is not None: + definition['spec']['resourceLabels'] = resource_pool.resource_labels + definition = await Poolboy.custom_objects_api.create_namespaced_custom_object( body = definition, group = Poolboy.operator_domain, @@ -514,7 +523,7 @@ async def create_for_pool( ) ): resource_handle.__register() - logger.info(f"Created ResourceHandle {resource_handle.name} for ResourcePool {resource_pool.name}") + logger.info("Created ResourceHandle %s for %s", resource_handle, resource_pool) return resource_handle @classmethod @@ -529,10 +538,7 @@ async def delete_unbound_handles_for_pool( for resource_handle in list(cls.unbound_instances.values()): if resource_handle.resource_pool_name == resource_pool.name \ and resource_handle.resource_pool_namespace == resource_pool.namespace: - logger.info( - f"Deleting unbound ResourceHandle {resource_handle.name} " - f"for ResourcePool {resource_pool.name}" - ) + logger.info("Deleting unbound %s for %s", resource_handle, resource_pool) resource_handle.__unregister() await resource_handle.delete() return resource_handles @@ -542,10 +548,7 @@ async def delete_unbound_handles_for_pool( logger=logger, ) for resource_handle in resource_handles: - logger.info( - f"Deleting unbound ResourceHandle {resource_handle.name} " - f"for ResourcePool {resource_pool.name}" - ) + logger.info("Deleting unbound %s for %s", resource_handle, resource_pool) await resource_handle.delete() return resource_handles @@ -866,6 +869,12 @@ def preference_score(self) -> float: """Preference score for match preference""" return self.spec.get('preferenceScore', 0) + @property + def resource_annotations(self) -> Mapping[str,str]: + """Name/value pairs to apply as annotations on resources created for + this ResourceHandle.""" + return self.spec.get('resourceAnnotations', {}) + @property def resource_claim_description(self) -> str|None: """ResourceClaim descriptive string if bound to ResourceClaim""" @@ -888,6 +897,12 @@ def resource_handler_idx(self) -> int: """Label value used to select which resource handler pod should manage this ResourceHandle.""" return int(UUID(self.uid)) % Poolboy.resource_handler_count + @property + def resource_labels(self) -> Mapping[str,str]: + """Name/value pairs to apply as labels on resources created for + this ResourceHandle.""" + return self.spec.get('resourceLabels', {}) + @property def resource_pool_name(self) -> str|None: """ResourcePool name if from a ResourcePool""" @@ -1014,7 +1029,7 @@ async def __manage_init_status_resources(self, return except K8sApiException as exception: if attempt > 2: - logger.exception(f"{self} failed status patch: {patch}") + logger.exception("%s failed status patch: %s", self, patch) raise await self.refresh() attempt += 1 @@ -1028,11 +1043,11 @@ async def __manage_check_delete(self, - Is bound to resource claim that has been deleted. """ if self.is_past_lifespan_end: - logger.info(f"Deleting {self} at end of lifespan ({self.lifespan_end_timestamp})") + logger.info("Deleting %s at end of lifespan (%s)", self, self.lifespan_end_timestamp) await self.delete() return True if self.is_bound and not resource_claim: - logger.warning(f"Propagating deletion of {self.resource_claim_description} to {self}") + logger.warning("Propagating deletion of %s to %s", self.resource_claim_description, self) await self.delete() return True @@ -1065,8 +1080,8 @@ async def __manage_update_spec_resources(self, updated_provider = resource['provider']['name'] if current_provider != updated_provider: logger.warning( - f"Refusing update resources in {self} as it would change " - f"ResourceProvider from {current_provider} to {updated_provider}" + "Refusing update resources in %s as it would change ResourceProvider from %s to %s", + self, current_provider, updated_provider ) current_template = self.spec['resources'][idx].get('template') updated_template = resource.get('template') @@ -1085,7 +1100,7 @@ async def __manage_update_spec_resources(self, if patch: await self.json_patch(patch) - logger.info(f"Updated resources for {self} from {resource_provider}") + logger.info("Updated resources for %s from %s", self, resource_provider) def get_lifespan_default(self, resource_claim=None): return self.__lifespan_value('default', resource_claim=resource_claim) @@ -1246,7 +1261,7 @@ async def handle_delete(self, logger: kopf.ObjectLogger) -> None: f"{reference['name']} in {reference['namespace']}" if 'namespace' in reference else reference['name'] ) - logger.info(f"Propagating delete of {self} to {resource_description}") + logger.info("Propagating delete of %s to %s", self, resource_description) # Annotate managed resource to indicate resource handle deletion. await poolboy_k8s.patch_object( api_version = reference['apiVersion'], @@ -1273,7 +1288,7 @@ async def handle_delete(self, logger: kopf.ObjectLogger) -> None: resource_claim = await self.get_resource_claim(not_found_okay=True) if resource_claim and not resource_claim.is_detached: await resource_claim.delete() - logger.info(f"Propagated delete of {self} to ResourceClaim {resource_claim}") + logger.info("Propagated delete of %s to %s", self, resource_claim) if self.is_from_resource_pool: resource_pool = await resourcepool.ResourcePool.get(self.resource_pool_name) @@ -1458,7 +1473,7 @@ async def manage(self, logger: kopf.ObjectLogger) -> None: ) if updated_state: resource_states[resource_index] = updated_state - logger.info(f"Updated {resource_description} for ResourceHandle {self.name}") + logger.info("Updated %s for %s", resource_description, self) else: resources_to_create.append((resource_index, resource_definition)) @@ -1467,7 +1482,7 @@ async def manage(self, logger: kopf.ObjectLogger) -> None: await self.json_patch_status(patch) except K8sApiException as exception: if exception.status == 422: - logger.error(f"Failed to apply {patch}") + logger.error("Failed to apply patch %s", patch) raise for resource_index, resource_definition in resources_to_create: @@ -1482,7 +1497,7 @@ async def manage(self, logger: kopf.ObjectLogger) -> None: created_resource = await poolboy_k8s.create_object(resource_definition) if created_resource: resource_states[resource_index] = created_resource - logger.info(f"Created {resource_description} for {self}") + logger.info("Created %s for %s", resource_description, self) except K8sApiException as exception: if exception.status != 409: raise @@ -1503,7 +1518,10 @@ async def update_status(self, status = self.status while len(self.resources) < len(resource_states): - logger.warning(f"{self} update status with resource states longer that list of resources, attempting refetch: {len(self.resources)} < {len(resource_states)}") + logger.warning( + "%s update status with resource states longer that list of resources, attempting refetch: %s < %s", + self, len(self.resources), len(resource_states) + ) await asyncio.sleep(0.2) try: await self.refetch() @@ -1514,7 +1532,10 @@ async def update_status(self, raise if len(self.resources) < len(resource_states): - logger.error(f"{self} update status with resource states longer that list of resources after refetch: {len(self.resources)} < {len(resource_states)}") + logger.error( + "%s update status with resource states longer that list of resources after refetch: %s < %s", + self, len(self.resources), len(resource_states) + ) return # Create consolidated information about resources @@ -1640,10 +1661,10 @@ async def update_status(self, }) except K8sApiException: logger.exception( - f"Failed to get ResourceProvider {resource_provider_name} for {self}" + "Failed to get ResourceProvider %s for %s", resource_provider_name, self ) except Exception: - logger.exception(f"Failed to generate status summary for {self}") + logger.exception("Failed to generate status summary for %s", self) if patch: patch_attempt = 0 @@ -1654,7 +1675,7 @@ async def update_status(self, except K8sApiException: patch_attempt += 1 if patch_attempt > 5: - logger.exception(f"Failed to patch status on {self}") + logger.exception("Failed to patch status on %s", self) return await asyncio.sleep(0.2) diff --git a/operator/resourcepool.py b/operator/resourcepool.py index 8364a83..285beea 100644 --- a/operator/resourcepool.py +++ b/operator/resourcepool.py @@ -138,11 +138,23 @@ def preference_score(self) -> float|None: """Preference score to apply to ResourceHandles created for ResourcePool""" return self.spec.get('preferenceScore') + @property + def resource_annotations(self) -> Mapping[str,str]|None: + """Name/value pairs to apply as annotations on resources created for + ResourceHandles created for this ResourcePool.""" + return self.spec.get('resourceAnnotations') + @property def resource_handler_idx(self) -> int: """Label value used to select which resource handler pod should manage this ResourcePool.""" return int(UUID(self.uid)) % Poolboy.resource_handler_count + @property + def resource_labels(self) -> Mapping[str,str]|None: + """Name/value pairs to apply as labels on resources created for + ResourceHandles created for this ResourcePool.""" + return self.spec.get('resourceLabels') + @property def resource_provider_name(self) -> str|None: return self.spec.get('provider', {}).get('name') diff --git a/operator/resourceprovider.py b/operator/resourceprovider.py index e8cdd83..4468789 100644 --- a/operator/resourceprovider.py +++ b/operator/resourceprovider.py @@ -738,6 +738,7 @@ async def resource_definition_from_template(self, resource_definition['metadata']['annotations'] = {} resource_definition['metadata']['annotations'].update({ + **resource_handle.resource_annotations, Poolboy.resource_provider_name_annotation: self.name, Poolboy.resource_provider_namespace_annotation: self.namespace, Poolboy.resource_handle_name_annotation: resource_handle.name, @@ -770,6 +771,11 @@ async def resource_definition_from_template(self, Poolboy.resource_requester_preferred_username_annotation: requester_identity.get('extra', {}).get('preferred_username', ''), }) + if resource_handle.resource_labels: + if 'labels' not in resource_definition['metadata']: + resource_definition['metadata']['labels'] = {} + resource_definition['metadata']['labels'].update(resource_handle.resource_labels) + return resource_definition async def update_resource(self, diff --git a/test/roles/poolboy_test_simple/tasks/test-labels-annotations-01.yaml b/test/roles/poolboy_test_simple/tasks/test-labels-annotations-01.yaml new file mode 100644 index 0000000..4e6fd5d --- /dev/null +++ b/test/roles/poolboy_test_simple/tasks/test-labels-annotations-01.yaml @@ -0,0 +1,101 @@ +--- +- name: Create ResourceProvider test-labels-annotations-01 + kubernetes.core.k8s: + definition: + apiVersion: "{{ poolboy_domain }}/v1" + kind: ResourceProvider + metadata: + name: test-labels-annotations-01 + namespace: "{{ poolboy_namespace }}" + labels: >- + {{ { + poolboy_domain ~ "/test": "simple" + } }} + spec: + override: + apiVersion: "{{ poolboy_domain }}/v1" + kind: ResourceClaimTest + metadata: + name: "test-labels-annotations-01-{% raw %}{{ guid }}{% endraw %}" + namespace: "{{ poolboy_test_namespace }}" + template: + enable: true + validation: + openAPIV3Schema: + additionalProperties: false + properties: + spec: + additionalProperties: false + properties: + value: + type: string + required: + - value + type: object + +- name: Create ResourcePool test-labels-annotations-01 + kubernetes.core.k8s: + definition: + apiVersion: "{{ poolboy_domain }}/v1" + kind: ResourcePool + metadata: + name: test-labels-annotations-01 + namespace: "{{ poolboy_namespace }}" + labels: >- + {{ { + poolboy_domain ~ "/test": "simple" + } }} + spec: + minAvailable: 1 + resourceAnnotations: + test-annotation: test-annotation-value + resourceLabels: + test-label: test-label-value + resources: + - provider: + apiVersion: "{{ poolboy_domain }}/v1" + kind: ResourceProvider + name: test-labels-annotations-01 + namespace: "{{ poolboy_namespace }}" + template: + spec: + value: foo + +- name: Verify ResourceHandles for test-labels-annotations-01 + kubernetes.core.k8s_info: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourceHandle + namespace: "{{ poolboy_namespace }}" + label_selectors: + - "{{ poolboy_domain }}/resource-pool-name = test-labels-annotations-01" + register: r_get_resource_handles + failed_when: >- + r_get_resource_handles.resources | length != 1 or + r_get_resource_handles.resources[0].spec.resourceAnnotations['test-annotation'] != 'test-annotation-value' or + r_get_resource_handles.resources[0].spec.resourceLabels['test-label'] != 'test-label-value' + retries: 5 + delay: 1 + +- name: Verify annotation and label applied to resource for ResourceHandle + vars: + _ref: "{{ r_get_resource_handles.resources[0].status.resources[0].reference }}" + kubernetes.core.k8s_info: + api_version: "{{ _ref.apiVersion }}" + kind: "{{ _ref.kind }}" + name: "{{ _ref.name }}" + namespace: "{{ _ref.namespace }}" + register: r_get_resource + failed_when: >- + r_get_resource.resources | length != 1 or + r_get_resource.resources[0].metadata.annotations['test-annotation'] != 'test-annotation-value' or + r_get_resource.resources[0].metadata.labels['test-label'] != 'test-label-value' + retries: 5 + delay: 1 + +- name: Delete ResourcePool test-labels-annotations-01 + kubernetes.core.k8s: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourcePool + name: test-labels-annotations-01 + namespace: "{{ poolboy_namespace }}" + state: absent diff --git a/test/roles/poolboy_test_simple/tasks/test.yaml b/test/roles/poolboy_test_simple/tasks/test.yaml index 243937c..ed8f2fc 100644 --- a/test/roles/poolboy_test_simple/tasks/test.yaml +++ b/test/roles/poolboy_test_simple/tasks/test.yaml @@ -10,6 +10,7 @@ - test-disable-creation-02.yaml - test-ignore-01.yaml - test-finalizers-01.yaml + - test-labels-annotations-01.yaml - test-lifespan-start-01.yaml - test-linked-01.yaml - test-linked-02.yaml