core-services/prow/02_config/generate-boskos: For spitting out lots of names

Quotas for some providers depend on both account-scoped resources
(e.g. AWS elastic IPs default to five per account [1]), region-scoped
resources (e.g. AWS VPCs default to five per region [1]), potentialy
other scoping (e.g. AWS NAT gateways default to five per availability
zone [1]).  This commit moves us away from account-scoped leases and
towards region-scoped leases in account-scoped buckets.  That allows
us to say things like "on Azure, we have more capacity in centralus
than we do in our other regions".

The generating script is because typing out "us-east-1" 50 times is
tedious and hard to review, and today there is apparently no support
in the Boskos config directly to ask for $N copies of a given name.

I'm also adding myself as an owner, so DPTP doesn't have to bother
with lease adjustment.  Other CI-platform maintainers should feel free
to add themselves as well, if they want to approve lease adjustments
to their platforms.

Currently a mostly-no-op reshuffling of the exisiting Boskos config.

The python-validation image follows the pattern set by the existing
prow-config-semantics job.

This script works with both Python 2 and Python 3, but I'm using
python3 in the shebang because the verification image currently has
'python' pointed at Python 2 but only installs PyYAML for Python 3,
and I've been unable to land a pivot to point it at Python 3 [2].

[1]: https://docs.openshift.com/container-platform/4.5/installing/installing_aws/installing-aws-account.html#installation-aws-limits_installing-aws-account
[2]: #11869

Author: wking

Makefile

@@ -1,6 +1,6 @@
 SHELL=/usr/bin/env bash -o errexit
 
-.PHONY: help check check-core check-services dry-core core dry-services services all
+.PHONY: help check check-boskos check-core check-services dry-core core dry-services services all
 
 CONTAINER_ENGINE ?= docker
 
@@ -9,9 +9,13 @@ help:
 
 all: core services
 
-check: check-core check-services
+check: check-core check-services check-boskos
 	@echo "Service config check: PASS"
 
+check-boskos:
+	hack/validate-boskos.sh
+	@echo "Boskos config check: PASS"
+
 check-core:
 	core-services/_hack/validate-core-services.sh core-services
 	@echo "Core service config check: PASS"
@@ -38,6 +42,7 @@ services:
 update:
 	$(MAKE) jobs
 	$(MAKE) ci-operator-config
+	$(MAKE) boskos-config
 	$(MAKE) prow-config
 	$(MAKE) registry-metadata
 
@@ -59,6 +64,10 @@ registry-metadata:
 	$(CONTAINER_ENGINE) pull registry.svc.ci.openshift.org/ci/generate-registry-metadata:latest
 	$(CONTAINER_ENGINE) run --rm -v "$(CURDIR)/ci-operator/step-registry:/ci-operator/step-registry:z" registry.svc.ci.openshift.org/ci/generate-registry-metadata:latest --registry /ci-operator/step-registry
 
+boskos-config:
+	cd core-services/prow/02_config && ./generate-boskos.py
+.PHONY: boskos-config
+
 prow-config:
 	$(CONTAINER_ENGINE) pull registry.svc.ci.openshift.org/ci/determinize-prow-config:latest
 	$(CONTAINER_ENGINE) run --rm -v "$(CURDIR)/core-services/prow/02_config:/config:z" registry.svc.ci.openshift.org/ci/determinize-prow-config:latest --prow-config-dir /config

ci-operator/jobs/openshift/release/openshift-release-master-presubmits.yaml

@@ -38,6 +38,28 @@ presubmits:
             cpu: 10m
       serviceAccountName: config-updater
     trigger: (?m)^/test( | .* )app-ci-config-dry,?($|\s.*)
+  - agent: kubernetes
+    always_run: true
+    branches:
+    - master
+    cluster: build01
+    context: ci/prow/boskos-config
+    decorate: true
+    labels:
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-release-master-boskos-config
+    rerun_command: /test boskos-config
+    spec:
+      containers:
+      - command:
+        - hack/validate-boskos.sh
+        image: registry.svc.ci.openshift.org/ci/python-validation
+        imagePullPolicy: Always
+        name: ""
+        resources:
+          requests:
+            cpu: 10m
+    trigger: (?m)^/test( | .* )boskos-config,?($|\s.*)
   - agent: kubernetes
     always_run: true
     branches:

core-services/prow/02_config/OWNERS

@@ -0,0 +1,3 @@
+approvers:
+  - dptp
+  - wking

core-services/prow/02_config/_boskos.yaml

@@ -1,78 +1,78 @@
----
+# generated with generate-boskos.py; do not edit directly
 resources:
-- type: aws-quota-slice
-  state: free
+- max-count: 150
   min-count: 150
-  max-count: 150
-- type: azure4-quota-slice
   state: free
+  type: aws-quota-slice
+- max-count: 30
   min-count: 30
-  max-count: 30
-- type: gcp-quota-slice
   state: free
+  type: azure4-quota-slice
+- max-count: 120
   min-count: 120
-  max-count: 120
-- type: libvirt-s390x-quota-slice
   state: free
-  names:
-  - "libvirt-s390x-0-0"
-  - "libvirt-s390x-0-1"
-  - "libvirt-s390x-0-2"
-  - "libvirt-s390x-0-3"
-  - "libvirt-s390x-0-4"
-- type: libvirt-ppc64le-quota-slice
+  type: gcp-quota-slice
+- names:
+  - libvirt-ppc64le-0-0
+  - libvirt-ppc64le-0-1
+  - libvirt-ppc64le-0-2
+  - libvirt-ppc64le-0-3
+  - libvirt-ppc64le-1-0
+  - libvirt-ppc64le-1-1
+  - libvirt-ppc64le-1-2
+  - libvirt-ppc64le-1-3
   state: free
-  names:
-  - "libvirt-ppc64le-0-0"
-  - "libvirt-ppc64le-0-1"
-  - "libvirt-ppc64le-0-2"
-  - "libvirt-ppc64le-0-3"
-  - "libvirt-ppc64le-1-0"
-  - "libvirt-ppc64le-1-1"
-  - "libvirt-ppc64le-1-2"
-  - "libvirt-ppc64le-1-3"
-- type: metal-quota-slice
+  type: libvirt-ppc64le-quota-slice
+- names:
+  - libvirt-s390x-0-0
+  - libvirt-s390x-0-1
+  - libvirt-s390x-0-2
+  - libvirt-s390x-0-3
+  - libvirt-s390x-0-4
   state: free
+  type: libvirt-s390x-quota-slice
+- max-count: 1000
   min-count: 1000
-  max-count: 1000
-- type: openstack-quota-slice
   state: free
+  type: metal-quota-slice
+- names:
+  - openstack-osuosl-ppc64le-01
+  - openstack-osuosl-ppc64le-02
+  - openstack-osuosl-ppc64le-03
+  - openstack-osuosl-ppc64le-04
+  state: free
+  type: openstack-osuosl-ppc64le-quota-slice
+- names:
+  - openstack-ppc64le-00
+  - openstack-ppc64le-01
+  - openstack-ppc64le-02
+  - openstack-ppc64le-03
+  state: free
+  type: openstack-ppc64le-quota-slice
+- max-count: 7
   min-count: 7
-  max-count: 7
-- type: openstack-vexxhost-quota-slice
   state: free
+  type: openstack-quota-slice
+- max-count: 3
   min-count: 3
-  max-count: 3
-- type: openstack-ppc64le-quota-slice
   state: free
-  names:
-  - "openstack-ppc64le-00"
-  - "openstack-ppc64le-01"
-  - "openstack-ppc64le-02"
-  - "openstack-ppc64le-03"
-- type: openstack-osuosl-ppc64le-quota-slice
+  type: openstack-vexxhost-quota-slice
+- names:
+  - ovirt-10
+  - ovirt-11
+  - ovirt-12
+  - ovirt-13
+  - ovirt-14
+  - ovirt-15
+  - ovirt-16
+  - ovirt-17
   state: free
-  names:
-  - "openstack-osuosl-ppc64le-01"
-  - "openstack-osuosl-ppc64le-02"
-  - "openstack-osuosl-ppc64le-03"
-  - "openstack-osuosl-ppc64le-04"
-- type: vsphere-quota-slice
+  type: ovirt-quota-slice
+- max-count: 20
+  min-count: 20
   state: free
+  type: packet-quota-slice
+- max-count: 10
   min-count: 10
-  max-count: 10
-- type: ovirt-quota-slice
-  state: free
-  names:
-  - "ovirt-10"
-  - "ovirt-11"
-  - "ovirt-12"
-  - "ovirt-13"
-  - "ovirt-14"
-  - "ovirt-15"
-  - "ovirt-16"
-  - "ovirt-17"
-- type: packet-quota-slice
   state: free
-  min-count: 20
-  max-count: 20
+  type: vsphere-quota-slice

core-services/prow/02_config/generate-boskos.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+import yaml
+
+
+CONFIG = {
+    'aws-quota-slice': {
+        'default': 150,
+    },
+    'azure4-quota-slice': {
+        'default': 30,
+    },
+    'gcp-quota-slice': {
+        'default': 120,
+    },
+    'libvirt-s390x-quota-slice': {},
+    'libvirt-ppc64le-quota-slice': {},
+    'metal-quota-slice': {
+        # Wild guesses.  We'll see when we hit quota issues
+        'default': 1000,
+    },
+    'openstack-osuosl-ppc64le-quota-slice': {},
+    'openstack-quota-slice': {
+        'default': 7,
+    },
+    'openstack-vexxhost-quota-slice': {
+        'default': 3,
+    },
+    'openstack-ppc64le-quota-slice': {},
+    'ovirt-quota-slice': {},
+    'packet-quota-slice': {
+        'default': 20,
+    },
+    'vsphere-quota-slice': {
+        'default': 10,
+    },
+}
+
+for i in range(1):
+    for j in range(5):
+        CONFIG['libvirt-s390x-quota-slice']['libvirt-s390x-{}-{}'.format(i, j)] = 1
+
+for i in range(2):
+    for j in range(4):
+        CONFIG['libvirt-ppc64le-quota-slice']['libvirt-ppc64le-{}-{}'.format(i, j)] = 1
+
+for i in range(1, 5):
+    CONFIG['openstack-osuosl-ppc64le-quota-slice']['openstack-osuosl-ppc64le-{0:0>2}'.format(i)] = 1
+
+for i in range(4):
+    CONFIG['openstack-ppc64le-quota-slice']['openstack-ppc64le-{0:0>2}'.format(i)] = 1
+
+for i in range(10, 18):
+    CONFIG['ovirt-quota-slice']['ovirt-{}'.format(i)] = 1
+
+config = {
+    'resources': [],
+}
+
+for typeName, data in sorted(CONFIG.items()):
+    resource = {
+        'type': typeName,
+        'state': 'free',
+    }
+    if set(data.keys()) == {'default'}:
+        resource['min-count'] = resource['max-count'] = data['default']
+    else:
+        resource['names'] = []
+        for name, count in sorted(data.items()):
+            resource['names'].extend([name]*count)
+    config['resources'].append(resource)
+
+with open('_boskos.yaml', 'w') as f:
+    f.write('# generated with generate-boskos.py; do not edit directly\n')
+    yaml.dump(config, f, default_flow_style=False)

hack/validate-boskos.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# This script ensures that the Boskos configuration checked into git is up-to-date
+# with the generator. If it is not, re-generate the configuration to update it.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+base_dir=.
+
+cd "${base_dir}/core-services/prow/02_config"
+./generate-boskos.py
+DIFF="$(git diff)"
+if test -n "${DIFF}"
+then
+	cat << EOF
+ERROR: This check enforces that the Boskos configuration is generated
+ERROR: correctly. We have automation in place that updates the configuration and
+ERROR: new changes to the configuration should be followed with a re-generation.
+
+ERROR: Run the following command to re-generate the Boskos configuration:
+ERROR: $ make boskos-config
+
+ERROR: The following errors were found:
+
+EOF
+	echo "${DIFF}"
+	exit 1
+fi