From a4d26860ed01a5159068026c82a01a0bc45283b8 Mon Sep 17 00:00:00 2001 From: upodroid Date: Sun, 30 Nov 2025 11:34:48 +0300 Subject: [PATCH] create the remaining AR registries and start using terraform to manage the k8s-artifacts-prod project --- infra/gcp/bash/ensure-prod-storage.sh | 103 ------------------ .../terraform/k8s-artifacts-prod/buckets.tf | 41 +++++++ infra/gcp/terraform/k8s-artifacts-prod/iam.tf | 62 +++++++++++ .../gcp/terraform/k8s-artifacts-prod/main.tf | 38 +++++++ .../terraform/k8s-artifacts-prod/provider.tf | 35 ++++++ .../k8s-artifacts-prod/registries.tf | 89 +++++++++++++++ 6 files changed, 265 insertions(+), 103 deletions(-) create mode 100644 infra/gcp/terraform/k8s-artifacts-prod/buckets.tf create mode 100644 infra/gcp/terraform/k8s-artifacts-prod/iam.tf create mode 100644 infra/gcp/terraform/k8s-artifacts-prod/main.tf create mode 100644 infra/gcp/terraform/k8s-artifacts-prod/provider.tf create mode 100644 infra/gcp/terraform/k8s-artifacts-prod/registries.tf diff --git a/infra/gcp/bash/ensure-prod-storage.sh b/infra/gcp/bash/ensure-prod-storage.sh index 7c4b098af19..bc0e9cf98c0 100755 --- a/infra/gcp/bash/ensure-prod-storage.sh +++ b/infra/gcp/bash/ensure-prod-storage.sh @@ -89,109 +89,6 @@ readonly PROD_PROJECT_DISABLED_SERVICES=( containerscanning.googleapis.com ) -# Regions for prod GCR. -GCR_PROD_REGIONS=(us eu asia) -# Regions for prod AR. gcloud artifacts locations list --format json | jq '.[] | select(.name!="europe" and .name!="asia" and .name!="us") | .name' -r | xargs -AR_PROD_REGIONS=(asia-east1 asia-east2 asia-south1 asia-northeast1 asia-northeast2 australia-southeast1 europe-north1 europe-southwest1 europe-west1 europe-west10 europe-west12 europe-west2 europe-west3 europe-west4 europe-west8 europe-west9 southamerica-east1 southamerica-west1 us-central1 us-east1 us-east4 us-east5 us-south1 us-west1 us-west2 us-west3 us-west4) - -# Minimum time we expect to keep prod GCS artifacts. -PROD_RETENTION="10y" - -# Make a prod GCR repository and grant access to it. -# -# $1: The GCP project name (GCR names == project names) -function ensure_prod_gcr() { - if [ $# != 1 ] || [ -z "$1" ]; then - echo "ensure_prod_gcr(project) requires 1 argument" >&2 - return 1 - fi - local project="${1}" - - color 6 "Ensuring prod GCR for regions: ${GCR_PROD_REGIONS[*]}" - for region in "${GCR_PROD_REGIONS[@]}"; do - local gcr_bucket="gs://${region}.artifacts.${project}.appspot.com" - - color 3 "region: ${region}" - color 6 "Ensuring a GCR repo exists in region: ${region} for project: ${project}" - ensure_gcr_repo "${project}" "${region}" - - color 6 "Ensuring GCR admins can admin GCR in region: ${region} for project: ${project}" - empower_gcr_admins "${project}" "${region}" - - color 6 "Empowering image promoter for region: ${region} in project: ${project}" - empower_image_promoter "${project}" "${region}" - - color 6 "Ensuring GCS access logs enabled for GCR bucket in region: ${region} in project: ${project}" - ensure_gcs_bucket_logging "${gcr_bucket}" - done 2>&1 | indent -} - -# Make a prod AR repository and grant access to it. -# -# $1: The GCP project name (GCR names == project names) -function ensure_prod_ar() { - if [ $# != 1 ] || [ -z "$1" ]; then - echo "ensure_prod_ar(project) requires 1 argument" >&2 - return 1 - fi - local project="${1}" - local serviceaccount - - color 6 "Ensuring prod AR registry for locations: ${AR_PROD_REGIONS[*]}" - for region in "${AR_PROD_REGIONS[@]}"; do - - color 3 "region: ${region}" - color 6 "Ensuring an AR repo exists in location: ${region} for project: ${project}" - ensure_ar_repo "${project}" "${region}" - - color 6 "Ensuring GCR admins can admin AR in location: ${region} for project: ${project}" - empower_ar_admins "${project}" "${region}" - - color 6 "Empowering image promoter with roles/artifactregistry.repoAdmin in project: ${project}" - serviceaccount=$(svc_acct_email "${project}" "${IMAGE_PROMOTER_SVCACCT}") - ensure_project_role_binding "${project}" "serviceAccount:$serviceaccount" "roles/artifactregistry.repoAdmin" - done 2>&1 | indent -} - -# Make a prod GCS bucket and grant access to it. We need whole buckets for -# this because we want to grant minimal permissions, but there's no concept of -# permissions on a "subdirectory" of a bucket. If we had a GCS promoter akin -# to the GCR promoter, we might have used a single bucket, but we don't have -# that yet. -# -# $1: The GCP project to make the bucket -# $2: The bucket, including gs:// prefix -# $3: The group email to empower (optional) -function ensure_prod_gcs_bucket() { - if [ $# -lt 2 ] || [ $# -gt 3 ] || [ -z "$1" ] || [ -z "$2" ]; then - echo "ensure_prod_gcs_bucket(project, bucket, [group]) requires 2 or 3 arguments" >&2 - return 1 - fi - local project="${1}" - local bucket="${2}" - local group="${3:-}" - - color 6 "Ensuring the GCS bucket exists and is readable" - ensure_public_gcs_bucket "${project}" "${bucket}" - - color 6 "Ensuring the bucket retention policy is set" - ensure_gcs_bucket_retention "${bucket}" "${PROD_RETENTION}" - - color 6 "Empowering GCS admins" - empower_gcs_admins "${project}" "${bucket}" - - color 6 "Empowering file promoter in project: ${project}" - empower_file_promoter "${project}" "${bucket}" - - color 6 "Ensuring GCS access logs enabled for ${bucket} in project: ${project}" - ensure_gcs_bucket_logging "${bucket}" - - if [ -n "${group}" ]; then - color 6 "Empowering ${group} to write to the bucket" - empower_group_to_write_gcs_bucket "${group}" "${bucket}" - fi -} - # Grant access to "fake prod" projects for tol testing # $1: The GCP project # $2: The googlegroups group diff --git a/infra/gcp/terraform/k8s-artifacts-prod/buckets.tf b/infra/gcp/terraform/k8s-artifacts-prod/buckets.tf new file mode 100644 index 00000000000..daae4748e9f --- /dev/null +++ b/infra/gcp/terraform/k8s-artifacts-prod/buckets.tf @@ -0,0 +1,41 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +# module "gcb_bucket" { +# source = "terraform-google-modules/cloud-storage/google//modules/simple_bucket" +# version = "~> 8.0" + +# name = "k8s-staging-images-gcb" +# project_id = module.project.project_id +# location = "us" + +# lifecycle_rules = [{ +# action = { +# type = "Delete" +# } +# condition = { +# age = 90 # 90d +# with_state = "ANY" +# } +# }] + +# iam_members = [ +# { +# role = "roles/storage.admin" +# member = "serviceAccount:gcb-builder@k8s-infra-prow-build-trusted.iam.gserviceaccount.com" +# } +# ] +# } diff --git a/infra/gcp/terraform/k8s-artifacts-prod/iam.tf b/infra/gcp/terraform/k8s-artifacts-prod/iam.tf new file mode 100644 index 00000000000..f7987fd2489 --- /dev/null +++ b/infra/gcp/terraform/k8s-artifacts-prod/iam.tf @@ -0,0 +1,62 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +module "iam" { + source = "terraform-google-modules/iam/google//modules/projects_iam" + version = "~> 8.2" + + projects = [module.project.project_id] + + mode = "authoritative" + + bindings = { + "roles/artifactregistry.admin" = [ + "group:k8s-infra-artifact-admins@kubernetes.io", + ] + "roles/artifactregistry.repoAdmin" = [ + "serviceAccount:k8s-infra-gcr-promoter@k8s-artifacts-prod.iam.gserviceaccount.com" + ] + "roles/errorreporting.user" = [ + "group:k8s-infra-artifact-admins@kubernetes.io", + ] + "roles/serviceusage.serviceUsageConsumer" = [ + "group:k8s-infra-artifact-admins@kubernetes.io", + ] + "roles/viewer" = [ + "group:k8s-infra-artifact-admins@kubernetes.io", + "group:k8s-infra-artifact-security@kubernetes.io", + ] + } +} + +module "audit_logs" { + source = "terraform-google-modules/iam/google//modules/audit_config" + version = "~> 8.2" + + project = module.project.project_id + audit_log_config = [ + { + service = "artifactregistry.googleapis.com" + log_type = "DATA_READ" + exempted_members = null + }, + { + service = "artifactregistry.googleapis.com" + log_type = "DATA_WRITE" + exempted_members = null + }, + ] +} diff --git a/infra/gcp/terraform/k8s-artifacts-prod/main.tf b/infra/gcp/terraform/k8s-artifacts-prod/main.tf new file mode 100644 index 00000000000..a66498914bf --- /dev/null +++ b/infra/gcp/terraform/k8s-artifacts-prod/main.tf @@ -0,0 +1,38 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +module "project" { + source = "terraform-google-modules/project-factory/google" + version = "~> 18.2" + + name = "k8s-artifacts-prod" + project_id = "k8s-artifacts-prod" + org_id = "758905017065" + billing_account = "018801-93540E-22A20E" + + # Sane project defaults + default_service_account = "keep" + disable_services_on_destroy = false + create_project_sa = false + random_project_id = false + auto_create_network = false + + + activate_apis = [ + "artifactregistry.googleapis.com", + "run.googleapis.com", + ] +} diff --git a/infra/gcp/terraform/k8s-artifacts-prod/provider.tf b/infra/gcp/terraform/k8s-artifacts-prod/provider.tf new file mode 100644 index 00000000000..98598395270 --- /dev/null +++ b/infra/gcp/terraform/k8s-artifacts-prod/provider.tf @@ -0,0 +1,35 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +terraform { + required_version = "1.13.5" + + backend "gcs" { + bucket = "k8s-infra-terraform" + prefix = "k8s-artifacts-prod" + } + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 7.12" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 7.12" + } + } +} diff --git a/infra/gcp/terraform/k8s-artifacts-prod/registries.tf b/infra/gcp/terraform/k8s-artifacts-prod/registries.tf new file mode 100644 index 00000000000..6e89af9de33 --- /dev/null +++ b/infra/gcp/terraform/k8s-artifacts-prod/registries.tf @@ -0,0 +1,89 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +locals { + // We want to have a registry in every location except the multi region ones + // gcloud artifacts locations list --format json | jq '.[] | select(.name!="europe" and .name!="asia" and .name!="us") | .name' | awk '{print $0","}' | --version-sort + registries = [ + "africa-south1", + # "asia-east1", + "asia-east2", + # "asia-northeast1", + # "asia-northeast2", + "asia-northeast3", + # "asia-south1", + "asia-south2", + "asia-southeast1", + "asia-southeast2", + # "australia-southeast1", + "australia-southeast2", + "europe-central2", + # "europe-north1", + "europe-north2", + # "europe-southwest1", + # "europe-west1", + # "europe-west2", + # "europe-west3", + # "europe-west4", + "europe-west6", + # "europe-west8", + # "europe-west9", + # "europe-west10", + "europe-west12", + "me-central1", + # "me-central2", # THIS REGION REQUIRES SUPPORT APPROVAL WHICH I STARTED + "me-west1", + "northamerica-northeast1", + "northamerica-northeast2", + "northamerica-south1", + "southamerica-east1", + # "southamerica-west1", + # "us-central1", + # "us-east1", + # "us-east4", + # "us-east5", + # "us-south1", + # "us-west1", + # "us-west2", + "us-west3", + "us-west4", + ] + +} + +module "artifact_registry" { + for_each = toset(local.registries) + source = "GoogleCloudPlatform/artifact-registry/google" + version = "~> 0.2" + + project_id = module.project.project_id + location = each.key + format = "DOCKER" + repository_id = "images" + # docker_config = { + # immutable_tags = true // APPLY THIS SOON + # } + members = { + readers = ["allUsers"], + } +} + +# # DELETE THIS AFTER ALL THE REGISTRIES ARE IMPORTED +# import { +# for_each = toset(local.registries) +# to = module.artifact_registry[each.key].google_artifact_registry_repository.repo +# id = "k8s-artifacts-prod/${each.key}/images" +# }