From 9b6ce8825459919b79a7ff544565f04c70c033c3 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 26 Jun 2023 16:58:49 -0400 Subject: [PATCH 01/10] update verify-shell output format and default exclusions --- scripts/verify/verify-shell.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/verify/verify-shell.sh b/scripts/verify/verify-shell.sh index d6e36d0291..f7a4706c76 100755 --- a/scripts/verify/verify-shell.sh +++ b/scripts/verify/verify-shell.sh @@ -20,5 +20,18 @@ CHECK_FILE_LIST=$(find . \( -type d \( ${IGNORE_PATHS} \) -o -name "${IGNORE_FIL for f in ${CHECK_FILE_LIST} ; do echo "shellcheck: ${f}" - "${SHELL_CHECK}" -x "${f}" + # Use format=gcc so integration with editors allows a developer to + # jump right to the file with an issue. + # + # Exclude SC1091 because we source other scripts using variables + # and the linter cannot interpret the variables and find the + # files. + # + # Add --external-sources to allow `source` calls from outside + # the list of checked files. + "${SHELL_CHECK}" \ + --format=gcc \ + --exclude=SC1091 \ + --external-sources \ + "${f}" done From 18ed064884dedd5f9b578cdbdb738e5d813a21ef Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sun, 25 Jun 2023 09:55:42 -0400 Subject: [PATCH 02/10] test: make kustomize tests compatible with ostree-based systems --- test/suites/kustomize.robot | 62 ++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/test/suites/kustomize.robot b/test/suites/kustomize.robot index 2759139596..326b685d39 100644 --- a/test/suites/kustomize.robot +++ b/test/suites/kustomize.robot @@ -14,8 +14,8 @@ Test Tags restart slow *** Variables *** ${CONFIGMAP_NAME} test-configmap -${NON_DEFAULT_DIR} /usr/lib/microshift/test-manifests -${UNCONFIGURED_DIR} /usr/lib/microshift/test-manifests.d/unconfigured +${NON_DEFAULT_DIR} /home/${USHIFT_USER}/test-manifests +${UNCONFIGURED_DIR} /home/${USHIFT_USER}/test-manifests.d/unconfigured ${YAML_PATH} /etc/microshift/manifests.d/yaml-ext ${YML_PATH} /etc/microshift/manifests.d/yml-ext ${NOEXT_PATH} /etc/microshift/manifests.d/no-ext @@ -31,18 +31,21 @@ Load From /etc/microshift/manifestsd [Documentation] Subdir of /etc/microshift/manifests.d ConfigMap Path Should Match ${ETC_SUBDIR_NAMESPACE} ${ETC_SUBDIR} -Load From /usr/lib/microshift/manifests - [Documentation] /usr/lib/microshift/manifests - ConfigMap Path Should Match ${USR_NAMESPACE} /usr/lib/microshift/manifests - -Load From /usr/lib/microshift/manifestsd - # Keyword names cannot have '.' in them - [Documentation] Subdir of /usr/lib/microshift/manifests.d - ConfigMap Path Should Match ${USR_SUBDIR_NAMESPACE} ${USR_SUBDIR} +# The metal CI automation currently _only_ works for ostree deployments, +# and /usr/lib is not writable there. +# +# Load From /usr/lib/microshift/manifests +# [Documentation] /usr/lib/microshift/manifests +# ConfigMap Path Should Match ${USR_NAMESPACE} /usr/lib/microshift/manifests +# +# Load From /usr/lib/microshift/manifestsd +# # Keyword names cannot have '.' in them +# [Documentation] Subdir of /usr/lib/microshift/manifests.d +# ConfigMap Path Should Match ${USR_SUBDIR_NAMESPACE} ${USR_SUBDIR} Load From Configured Dir [Documentation] Non-default directory - ConfigMap Path Should Match ${NON_DEFAULT_NAMESPACE} /usr/lib/microshift/test-manifests + ConfigMap Path Should Match ${NON_DEFAULT_NAMESPACE} ${NON_DEFAULT_DIR} Do Not Load From Unconfigured Dir [Documentation] Manifests from a directory not in the config should not be loaded @@ -78,15 +81,18 @@ Setup Suite # robocop: disable=too-long-keyword ${ns}= Generate Manifests ${ETC_SUBDIR} Set Suite Variable \${ETC_SUBDIR_NAMESPACE} ${ns} - # Used by "Load From /usr/lib/microshift/manifests" - ${ns}= Generate Manifests /usr/lib/microshift/manifests - Set Suite Variable \${USR_NAMESPACE} ${ns} - - # Used by "Load From /usr/lib/microshift/manifestsd" - ${rand}= Generate Random String - Set Suite Variable \${USR_SUBDIR} /usr/lib/microshift/manifests.d/${rand} - ${ns}= Generate Manifests ${USR_SUBDIR} - Set Suite Variable \${USR_SUBDIR_NAMESPACE} ${ns} + # The metal CI automation currently _only_ works for ostree deployments, + # and /usr/lib is not writable there. + # + # # Used by "Load From /usr/lib/microshift/manifests" + # ${ns}= Generate Manifests /usr/lib/microshift/manifests + # Set Suite Variable \${USR_NAMESPACE} ${ns} + # + # # Used by "Load From /usr/lib/microshift/manifestsd" + # ${rand}= Generate Random String + # Set Suite Variable \${USR_SUBDIR} /usr/lib/microshift/manifests.d/${rand} + # ${ns}= Generate Manifests ${USR_SUBDIR} + # Set Suite Variable \${USR_SUBDIR_NAMESPACE} ${ns} # Used by "Load From Configured Dir" ${ns}= Generate Manifests ${NON_DEFAULT_DIR} @@ -119,7 +125,7 @@ Setup Suite # robocop: disable=too-long-keyword ... \ \ \ \ - /usr/lib/microshift/manifests.d/* ... \ \ \ \ - ${NON_DEFAULT_DIR} ... \ \ \ \ # Add a directory _without_ the glob for unconfigured test - ... \ \ \ \ - /usr/lib/microshift/test-manifests.d + ... \ \ \ \ - /home/${USHIFT_USER}/test-manifests.d ${merged}= Extend MicroShift Config ${config_content} Upload MicroShift Config ${merged} @@ -130,8 +136,11 @@ Teardown Suite # robocop: disable=too-many-calls-in-keyword Clear Manifest Directory /etc/microshift/manifests Remove Manifest Directory ${ETC_SUBDIR} - Clear Manifest Directory /usr/lib/microshift/manifests - Remove Manifest Directory ${USR_SUBDIR} + # The metal CI automation currently _only_ works for ostree deployments, + # and /usr/lib is not writable there. + # + # Clear Manifest Directory /usr/lib/microshift/manifests + # Remove Manifest Directory ${USR_SUBDIR} Remove Manifest Directory ${NON_DEFAULT_DIR} Remove Manifest Directory ${UNCONFIGURED_DIR} Remove Manifest Directory ${YAML_PATH} @@ -140,8 +149,11 @@ Teardown Suite # robocop: disable=too-many-calls-in-keyword Run With Kubeconfig oc delete namespace ${ETC_NAMESPACE} allow_fail=True Run With Kubeconfig oc delete namespace ${ETC_SUBDIR_NAMESPACE} allow_fail=True - Run With Kubeconfig oc delete namespace ${USR_NAMESPACE} allow_fail=True - Run With Kubeconfig oc delete namespace ${USR_SUBDIR_NAMESPACE} allow_fail=True + # The metal CI automation currently _only_ works for ostree deployments, + # and /usr/lib is not writable there. + # + # Run With Kubeconfig oc delete namespace ${USR_NAMESPACE} allow_fail=True + # Run With Kubeconfig oc delete namespace ${USR_SUBDIR_NAMESPACE} allow_fail=True Run With Kubeconfig oc delete namespace ${NON_DEFAULT_NAMESPACE} allow_fail=True Run With Kubeconfig oc delete namespace ${UNCONFIGURED_NAMESPACE} allow_fail=True Run With Kubeconfig oc delete namespace ${YAML_NAMESPACE} allow_fail=True From eb5fc4b170bc74e9cc00b5613c9bc2555e5cc461 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 22 Jun 2023 17:56:43 -0400 Subject: [PATCH 03/10] scripts: add caddy as dependency for image builder --- scripts/image-builder/configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/image-builder/configure.sh b/scripts/image-builder/configure.sh index 198fb02934..38e67a848e 100755 --- a/scripts/image-builder/configure.sh +++ b/scripts/image-builder/configure.sh @@ -14,7 +14,7 @@ sudo firewall-cmd --add-service=cockpit --permanent # The mock utility comes from the EPEL repository sudo dnf install -y "https://dl.fedoraproject.org/pub/epel/epel-release-latest-${OSVERSION}.noarch.rpm" -sudo dnf install -y mock +sudo dnf install -y mock caddy sudo usermod -a -G mock "$(whoami)" # Verify umask and home directory permissions From 01566b12f75375dd10435760cf12e5f0bdfcddf1 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 29 Jun 2023 17:22:54 -0400 Subject: [PATCH 04/10] test: make kubeconfig resource compatible with running on a remote system When we have to get the kubeconfig from a remote system through a port-forwarded connection, we may have to update the connection URL. --- test/resources/DataFormats.py | 8 ++++++++ test/resources/kubeconfig.resource | 15 ++++++++++++++- test/variables.yaml.example | 5 ++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/test/resources/DataFormats.py b/test/resources/DataFormats.py index fd266d2610..42c307ef0f 100644 --- a/test/resources/DataFormats.py +++ b/test/resources/DataFormats.py @@ -1,6 +1,7 @@ """Provides helpers for common data formats """ import json + import yaml from robot.utils import DotDict @@ -79,3 +80,10 @@ def yaml_merge(base, addition): parsed_addition = yaml.safe_load(addition) _merge(combined, parsed_addition) return yaml.dump(combined) + + +def update_kubeconfig_server_url(kubeconfig_text, new_url): + """Change the server URL and return the new kubeconfig text.""" + parsed = yaml.safe_load(kubeconfig_text) + parsed['clusters'][0]['cluster']['server'] = new_url + return yaml.dump(parsed) diff --git a/test/resources/kubeconfig.resource b/test/resources/kubeconfig.resource index 54519886aa..85a63eda84 100644 --- a/test/resources/kubeconfig.resource +++ b/test/resources/kubeconfig.resource @@ -3,9 +3,14 @@ Documentation Keywords for accessing a kubeconfig file for the MicroShift Library OperatingSystem Library String +Library DataFormats.py Resource ../resources/microshift-host.resource +*** Variables *** +${API_PORT} ${EMPTY} # overridden by scenario.sh in CI + + *** Keywords *** Get Kubeconfig [Documentation] Get the kubeconfig file from the host argument and return contents @@ -18,9 +23,17 @@ Get Kubeconfig RETURN ${kubeconfig} Setup Kubeconfig - [Documentation] Get the kubeconfig file from the configured $USHIFT_HOST, create a temporary file + [Documentation] Get the kubeconfig file from the configured $USHIFT_HOST, + ... update the server URL based on the API_PORT, create a temporary file, ... and export it as $KUBECONFIG variable. ${kubeconfig}= Get Kubeconfig ${USHIFT_HOST} + # If we have been given an explicit API port to use, make sure that + # port appears in the kubeconfig file. + IF "${API_PORT}"!="${EMPTY}" + ${kubeconfig}= Update Kubeconfig Server Url + ... ${kubeconfig} + ... https://${USHIFT_HOST}:${API_PORT} + END ${rand}= Generate Random String ${path}= Join Path /tmp ${rand} Create File ${path} ${kubeconfig} diff --git a/test/variables.yaml.example b/test/variables.yaml.example index 6c9e386257..05ff2ef1a7 100644 --- a/test/variables.yaml.example +++ b/test/variables.yaml.example @@ -8,4 +8,7 @@ USHIFT_USER: microshift SSH_PRIV_KEY: /home/microshift/.ssh/id_rsa # SSH port to use when connecting to MicroShift's host. # Defaults to 22. -#SSH_PORT: 22 \ No newline at end of file +#SSH_PORT: 22 +# API port, in case the connection is through a forwarded port. +# Defaults to whatever is in the kubeconfig file. +#API_PORT: 6443 \ No newline at end of file From 22ec1c1cb7456cacde544df1a032be94a70d67ad Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 29 Jun 2023 17:23:35 -0400 Subject: [PATCH 05/10] test: make load-balancer tests compatible with running on a remote system When we run the tests on a remote system through a port-forwarded connection, we may have to modify some connection parameters. --- test/suites/load-balancer.robot | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/suites/load-balancer.robot b/test/suites/load-balancer.robot index b257d83ed3..bbb7e0b4bd 100644 --- a/test/suites/load-balancer.robot +++ b/test/suites/load-balancer.robot @@ -11,6 +11,7 @@ Suite Teardown Teardown Suite With Namespace *** Variables *** ${HELLO_USHIFT} assets/hello-microshift.yaml +${LB_PORT} ${EMPTY} *** Test Cases *** @@ -41,8 +42,13 @@ Expose Hello MicroShift Pod Via LB Access Hello Microshift Via LB [Documentation] Try to retrieve data from the "hello microshift" service end point + IF "${LB_PORT}"=="${EMPTY}" + ${connect_to}= Set Variable "hello-microshift.cluster.local:80:${USHIFT_HOST}:5678" + ELSE + ${connect_to}= Set Variable "hello-microshift.cluster.local:80:${USHIFT_HOST}:${LB_PORT}" + END ${result}= Run Process - ... curl -i http://hello-microshift.cluster.local --connect-to "hello-microshift.cluster.local:80:${USHIFT_HOST}:5678" + ... curl -i http://hello-microshift.cluster.local --connect-to ${connect_to} ... shell=True ... timeout=15s Log Many ${result.rc} ${result.stdout} ${result.stderr} From 782dc0befb5ed6adc058604b9ebbec6083d448c8 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 30 Jun 2023 14:50:40 -0400 Subject: [PATCH 06/10] add error options to manage-vm.sh for use in ci --- scripts/devenv-builder/manage-vm.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/devenv-builder/manage-vm.sh b/scripts/devenv-builder/manage-vm.sh index 95bc5ea648..8788618585 100755 --- a/scripts/devenv-builder/manage-vm.sh +++ b/scripts/devenv-builder/manage-vm.sh @@ -1,5 +1,7 @@ #!/usr/bin/bash +set -eo pipefail + # https://github.com/openshift/microshift/blob/main/docs/devenv_setup.md # https://github.com/openshift/microshift/blob/main/docs/devenv_setup_auto.md From 1b18a47c5f4ec70f6ad272c405d9f3f7db4d99a6 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sat, 1 Jul 2023 19:13:24 -0400 Subject: [PATCH 07/10] manage-vm: retry installing dependencies dnf is reporting different missing file errors in CI, so retry the command a few times --- scripts/devenv-builder/manage-vm.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/devenv-builder/manage-vm.sh b/scripts/devenv-builder/manage-vm.sh index 8788618585..0231eb5c05 100755 --- a/scripts/devenv-builder/manage-vm.sh +++ b/scripts/devenv-builder/manage-vm.sh @@ -61,7 +61,24 @@ function get_base_isofile { } function action_config() { - sudo dnf install -y libvirt virt-manager virt-install virt-viewer libvirt-client qemu-kvm qemu-img sshpass + local tries=0 + local failed=true + local deps="libvirt virt-manager virt-install virt-viewer libvirt-client qemu-kvm qemu-img sshpass" + + set +e # retry because we see errors caching different RPMs in CI + while [ ${tries} -lt 5 ]; do + # shellcheck disable=SC2086 + if sudo dnf install -y ${deps}; then + failed=false + break + fi + ((tries+=1)) + done + if ${failed}; then + exit 1 + fi + set -e + if [ "$(systemctl is-active libvirtd.socket)" != "active" ] ; then echo "Enabling libvirtd" sudo systemctl enable --now libvirtd From 36b736143a000e40cd107c5494784571f0525c76 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 29 Jun 2023 17:24:35 -0400 Subject: [PATCH 08/10] USHIFT-1363: add multi-vm test harness Define a test harness for running different "scenarios" defined by combining OS images, MicroShift versions, and test suites. --- .gitignore | 1 + scripts/image-builder/configure.sh | 2 +- test/README.md | 223 ++++++++++++++ test/bin/build_images.sh | 115 +++++++ test/bin/common.sh | 92 ++++++ test/bin/composer_cleanup.sh | 23 ++ test/bin/configure_hypervisor_firewall.sh | 15 + test/bin/create_local_repo.sh | 31 ++ test/bin/download_images.sh | 59 ++++ test/bin/force_vm_settings.sh | 24 ++ test/bin/manage_vm_connections.sh | 226 ++++++++++++++ test/bin/scenario.sh | 351 ++++++++++++++++++++++ test/bin/start_osbuild_workers.sh | 20 ++ test/bin/start_webserver.sh | 19 ++ test/bin/wait_images.py | 120 ++++++++ test/scenario_settings.sh.example | 14 + 16 files changed, 1334 insertions(+), 1 deletion(-) create mode 100755 test/bin/build_images.sh create mode 100755 test/bin/common.sh create mode 100755 test/bin/composer_cleanup.sh create mode 100755 test/bin/configure_hypervisor_firewall.sh create mode 100755 test/bin/create_local_repo.sh create mode 100755 test/bin/download_images.sh create mode 100755 test/bin/force_vm_settings.sh create mode 100755 test/bin/manage_vm_connections.sh create mode 100755 test/bin/scenario.sh create mode 100755 test/bin/start_osbuild_workers.sh create mode 100755 test/bin/start_webserver.sh create mode 100755 test/bin/wait_images.py create mode 100644 test/scenario_settings.sh.example diff --git a/.gitignore b/.gitignore index 2aae810de3..70935d154f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ _output sshfile ansible/*.txt test/variables.yaml +test/scenario_settings.sh diff --git a/scripts/image-builder/configure.sh b/scripts/image-builder/configure.sh index 38e67a848e..6c39641738 100755 --- a/scripts/image-builder/configure.sh +++ b/scripts/image-builder/configure.sh @@ -14,7 +14,7 @@ sudo firewall-cmd --add-service=cockpit --permanent # The mock utility comes from the EPEL repository sudo dnf install -y "https://dl.fedoraproject.org/pub/epel/epel-release-latest-${OSVERSION}.noarch.rpm" -sudo dnf install -y mock caddy +sudo dnf install -y mock caddy tomcli sudo usermod -a -G mock "$(whoami)" # Verify umask and home directory permissions diff --git a/test/README.md b/test/README.md index 17e3bc2c22..10152e4711 100644 --- a/test/README.md +++ b/test/README.md @@ -109,3 +109,226 @@ For more options, see the help output for the `robot` command. ``` $ ./_output/robotenv/bin/robot -h ``` + +## Test Scenarios in CI + +The test scenario tools in the `bin` directory are useful for running +more complex test cases that require VMs and different images. + +### Package Sources + +In order to build different images, Composer needs to be configured +with all of the right package sources to pull the required RPMs. The +sources used by all scenarios are in the `package-sources` directory. + +Package source definition files are templates using `envsubst`, +which means the files can have shell variables embedded. + +``` +id = "fast-datapath" +name = "Fast Datapath for RHEL 9" +type = "yum-baseurl" +url = "https://cdn.redhat.com/content/dist/layered/rhel9/${UNAME_M}/fast-datapath/os" +check_gpg = true +check_ssl = true +system = false +rhsm = true +``` + +Refer to `./bin/build_images.sh` for the set of known variables that can +be expanded. + +## Image Blueprints + +The image blueprints are independent of the test scenarios so that +images can be reused. Be careful making changes to specific images in +case the change affects the way the image is used in different +scenarios. Be careful adding unnecessary new images, since each image +takes time to build and may slow down the overall test job. + +Add blueprints as TOML files in the `image-blueprints` directory, then +add a short description of the image here for reference. + +Name | Purpose +---- | ------- +rhel-9.2 | A simple RHEL image without MicroShift. +rhel-9.2-microshift-4.13 | A RHEL 9.2 image with the latest MicroShift 4.13 z-stream installed and enabled. +rhel-9.2-microshift-source | A RHEL 9.2 image with the RPMs built from source. + +## Preparing to run test scenarios + +### Creating the local RPM repository + +After running `make rpm` at the top of the source tree, run +`./bin/create_local_repo.sh` from this directory to copy the necessary +files into a location that can be used as an RPM repository by +Composer. If you build new RPMs, you need to re-run +`create_local_repo.sh` *and* build new images. + +### Creating the images + +Use `./bin/start_osbuild_workers.sh` to create multiple workers for +building images in parallel. This is optional, and not necessarily +recommended on a laptop. + +Use `./bin/build_images.sh` to build all of the images for all of the +blueprints available. + +### Downloading the images for use by the test scenarios + +Use `./bin/download_images.sh` to download all of the images from +Composer's cache and set up the directory for the web server to host +the files needed to launch VMs and run test scenarios. + +### Global settings + +The test scenario tool uses several global settings that may be +configured before the tool is used in `./scenario_settings.sh`. You +can copy `./scenario_settings.sh.example` as a starting point. + +`PUBLIC_IP` -- The public IP of the hypervisor, when accessing VMs +remotely through port-forwarded connections. + +`SSH_PUBLIC_KEY` -- The name of the public key file to use for +providing password-less access to the VMs. + +`SSH_PRIVATE_KEY` -- The name of the private key file to use for +providing password-less access to the VMs. Set to an empty string to +use ssh-agent. + +### Creating test infrastructure + +Use `./bin/start_webserver.sh` to run a caddy web server to serve the +images needed for the test scenarios. + +Use `./bin/configure_hypervisor_firewall.sh` to set up the firewall +rules that allow VMs to access the web server on the hypervisor. + +Use `./bin/scenario.sh` to create test infrastructure for a scenario +with the `create` argument and a scenario directory name as input. + +``` +$ ./bin/scenario.sh create ./scenarios/rhel-9.2-microshift-source-standard-suite.sh +``` + +### Enabling connections to the VMs + +Run `./bin/manage_vm_connections.sh` on the hypervisor to set up the API +server and ssh port of each VM. The appropriate connection ports are +written to the `$SCENARIO_INFO_DIR` directory (refer to `common.sh` +for the setting for the variable), depending on the mode used. + +For CI integration or a remote hypervisor, use `remote` and pass the +starting ports for the API server and ssh server port forwarding. + +``` +$ ./bin/manage_vm_connections.sh remote -a 7000 -s 6000 -l 7500 +``` + +To run the tests from a local hypervisor, as in a local developer +configuration, use `local` with no other arguments. + +``` +$ ./bin/manage_vm_connections.sh local +``` + +### Run a scenario + +Use `./bin/scenario.sh run` with a scenario file to run the tests for +the scenario. + +``` +$ ./bin/scenario.sh run ./scenarios/rhel-9.2-microshift-source-standard-suite.sh +``` + +## Scenario definitions + +Scenarios are saved as shell scripts under `scenarios`. Each +scenario includes several functions that are combined +with the framework scripts to take the specific actions for the +combination of images and tests that make up the scenario. + +The scenario script should be defined with a combination of the RHEL +version(s), MicroShift version(s), and an indication of what sort of +tests are being run. For example, +`rhel-9.2-microshift-source-standard-suite.sh` runs the standard test +suite (not the ostree upgrade tests) against MicroShift built from +source running on a RHEL 9.2 image. + +Scenarios define VMs using short names, like `host1`, which are made +unique across the entire set of scenarios. VMs are not reused across +scenarios. + +Scenarios use images defined by the blueprints created +earlier. Blueprints and images are reused between scenarios. Refer to +"Image Blueprints" above for details. + +Scenarios use kickstart templates from the `kickstart-templates` +directory. Kickstart templates are reused between scenarios. + +All of the functions that act as the scenario API are run in the +context of `scenario.sh` and can therefore use any functions defined +there. + +### scenario_create_vms + +This function should do any work needed to boot all of the VMs needed +for the scenario, including producing kickstart files and launching +the VMs themselves. + +The `prepare_kickstart` function takes as input the VM name, kickstart +template file relative to the `kickstart-templates` directory, and the +initial edge image to boot. It produces a unique kickstart file _for +that VM_ in the `${SCENARIO_INFO_DIR}` directory. Use the function +multiple times to create kickstart files for additional VMs. + +The `launch_vm` function takes as input the VM name. It expects a +kickstart file to already exist, and it defines a new VM configured to +boot from the installer ISO and the kickstart file. + +### scenario_remove_vms + +This functions is used to remove any VMs defined by the scenario. It +should call `remove_vm` for each VM created by `scenario_create_vms`, +and take any other cleanup actions that might be unique to the +scenario based on other steps taken in `scenario_create_vms`. + +It is not necessary to explicitly clean up the scenario metadata in +`${SCENARIO_INFO_DIR}`. + +### scenario_run_tests + +This function runs the tests. It is invoked separately because the +same host may be used with multiple test runs when working on a local +developer system. + +The function `run_tests` should be invoked exactly one time, passing +the primary host to use for connecting and any arguments to be given +to `robot`, including the test suites and any unique variables that +are not saved to the default variables file by the framework. + +## Cleaning up + +Use `./bin/composer_cleanup.sh` to stop any running jobs, remove +everything from the queue, and delete existing builds. + +Use `./bin/scenario.sh cleanup` to remove the test infrastructure for a +scenario. + +``` +$ ./bin/scenario.sh cleanup ./scenarios/rhel-9.2-microshift-source-standard-suite/ +``` + +## CI Integration Scripts + +### ci_phase_iso_build_pre.sh + +Runs on the CI cluster, in a container. Copies everything needed to +build ISOs onto the hypervisor, then triggers `ci_phase_iso_build.sh`. + +### ci_phase_iso_build.sh + +Runs on the hypervisor. Responsible for all of the setup to build all +needed images. Rebuilds MicroShift RPMs from source, sets up RPM repo, +sets up osbuild workers, builds the images, and creates the web server +to host the images. diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh new file mode 100755 index 0000000000..85f2dad835 --- /dev/null +++ b/test/bin/build_images.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# +# This script should be run on the image build server (usually the +# same as the hypervisor). + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +mkdir -p "${IMAGEDIR}" + +# Determine the version of the RPM in the local repo so we can use it +# in the blueprint templates. +if [ ! -d "${LOCAL_REPO}" ]; then + error "Run ${SCRIPTDIR}/create_local_repo.sh before building images." + exit 1 +fi +release_info_rpm=$(find "${LOCAL_REPO}" -name 'microshift-release-info-*.rpm') +if [ -z "${release_info_rpm}" ]; then + error "Failed to find microshift-release-info RPM in ${LOCAL_REPO}" + exit 1 +fi +SOURCE_VERSION=$(rpm -q --queryformat '%{version}' "${release_info_rpm}") + +## TEMPLATE VARIABLES +# +# Machine platform type ("x86_64") +UNAME_M=$(uname -m) +export UNAME_M +export LOCAL_REPO # defined in common.sh +export SOURCE_VERSION # defined earlier + +# Add our sources. It is OK to run these steps repeatedly, if the +# details change they are updated in the service. +mkdir -p "${IMAGEDIR}/package-sources" +# shellcheck disable=SC2231 # allow glob expansion without quotes in for loop +for template in ${TESTDIR}/package-sources/*.toml; do + name=$(basename "${template}" .toml) + outfile="${IMAGEDIR}/package-sources/${name}.toml" + echo "Rendering ${template} to ${outfile}" + envsubst <"${template}" >"${outfile}" + echo "Adding package source from ${outfile}" + sudo composer-cli sources add "${outfile}" +done + +# Show details about the available sources to make debugging easier. +for name in $(sudo composer-cli sources list); do + echo + echo "Package source: ${name}" + sudo composer-cli sources info "${name}" | sed -e 's/gpgkeys.*/gpgkeys = .../g' +done + +# Given a blueprint filename, extract the name value. It does not have +# to match the filename, but some commands take the file and others +# take the name, so we need to be able to have both. +get_blueprint_name() { + local filename="${1}" + tomcli-get "${filename}" name +} + +# Track some dynamically created values +BUILDIDS="" + +# Upload the blueprint definitions +mkdir -p "${IMAGEDIR}/blueprints" +mkdir -p "${IMAGEDIR}/builds" +# shellcheck disable=SC2231 # allow glob expansion without quotes in for loop +for template in ${TESTDIR}/image-blueprints/*.toml; do + echo + echo "Blueprint ${template}" + + blueprint_file="${IMAGEDIR}/blueprints/$(basename "${template}")" + echo "Rendering ${template} to ${blueprint_file}" + envsubst <"${template}" >"${blueprint_file}" + + blueprint=$(get_blueprint_name "${blueprint_file}") + + if sudo composer-cli blueprints list | grep -q "^${blueprint}$"; then + echo "Removing existing definition of ${blueprint}" + sudo composer-cli blueprints delete "${blueprint}" + fi + + echo "Loading new definition of ${blueprint}" + sudo composer-cli blueprints push "${blueprint_file}" + + echo "Resolving dependencies for ${blueprint}" + sudo composer-cli blueprints depsolve "${blueprint}" + + echo "Building edge-commit from ${blueprint}" + buildid=$(sudo composer-cli compose start-ostree \ + --ref "${blueprint}" \ + "${blueprint}" \ + edge-commit \ + | awk '{print $2}') + echo "Build ID ${buildid}" + echo "${buildid}" > "${IMAGEDIR}/builds/${blueprint}.edge-commit" + BUILDIDS="${BUILDIDS} ${buildid}" +done + +# In the future we may need to build multiple images with different +# formats but for now we just have one special case to build an +# installer image in a different format. +echo "Building image-installer from ${INSTALLER_IMAGE_BLUEPRINT}" +buildid=$(sudo composer-cli compose start \ + "${INSTALLER_IMAGE_BLUEPRINT}" \ + image-installer \ + | awk '{print $2}') +echo "Build ID ${buildid}" +echo "${buildid}" > "${IMAGEDIR}/builds/${blueprint}.image-installer" +BUILDIDS="${BUILDIDS} ${buildid}" + +echo "Waiting for builds to complete..." +# shellcheck disable=SC2086 # pass command arguments quotes to allow word splitting +time "${SCRIPTDIR}/wait_images.py" ${BUILDIDS} diff --git a/test/bin/common.sh b/test/bin/common.sh new file mode 100755 index 0000000000..1d2aba126e --- /dev/null +++ b/test/bin/common.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +# The location of the test directory, relative to the script. +TESTDIR="$(cd "${SCRIPTDIR}/.." && pwd)" + +# The location of the root of the git repo, relative to the script. +ROOTDIR="$(cd "${TESTDIR}/.." && pwd)" + +# The location of shared kickstart templates +# shellcheck disable=SC2034 # used elsewhere +KICKSTART_TEMPLATE_DIR="${TESTDIR}/kickstart-templates" + +# The blueprint we should use for building an installer image. +# shellcheck disable=SC2034 # used elsewhere +INSTALLER_IMAGE_BLUEPRINT="rhel-9.2" + +# The location for downloading all of the image-related output. +# The location the web server should serve. +export IMAGEDIR="${ROOTDIR}/_output/test-images" + +# The location for storage for the VMs. +# shellcheck disable=SC2034 # used elsewhere +VM_DISK_DIR="${IMAGEDIR}/vm-storage" + +# Location of RPMs built from source +# shellcheck disable=SC2034 # used elsewhere +RPM_SOURCE="${ROOTDIR}/_output/rpmbuild" + +# Location of local repository used by composer +# shellcheck disable=SC2034 # used elsewhere +LOCAL_REPO="${IMAGEDIR}/microshift-local" + +# Location of data files created by the tools for managing scenarios +# as they are run. +# +# The CI system will override this, but we need a default for local +# use. Use the image directoy, since that is already served by a web +# server. +# +# shellcheck disable=SC2034 # used elsewhere +SCENARIO_INFO_DIR="${SCENARIO_INFO_DIR:-${IMAGEDIR}/scenario-info}" + +# The location of the robot framework virtualenv. +# The CI system will override this. +# shellcheck disable=SC2034 # used elsewhere +RF_VENV=${RF_VENV:-${ROOTDIR}/_output/robotenv} + +# Which port the web server should run on. +WEB_SERVER_PORT=${WEB_SERVER_PORT:-8080} + +error() { + local message="$*" + echo "ERROR: ${message} [$(caller)]" 1>&2 +} + +get_vm_bridge_interface() { + # $ sudo virsh net-info default + # Name: default + # UUID: eaac9592-2324-4ae6-b2ec-a5ae94272456 + # Active: yes + # Persistent: yes + # Autostart: yes + # Bridge: virbr0 + + sudo virsh net-info default | grep '^Bridge:' | awk '{print $2}' +} + +get_vm_bridge_ip() { + local bridge + + # When get_vm_bridge_interface is run on the CI cluster there is + # no bridge, and possibly no virsh command. Do not fail, but + # return an empty IP. + bridge="$(get_vm_bridge_interface || true)" + + if [ -z "${bridge}" ]; then + echo "" + return + fi + + # $ ip -f inet addr show virbr0 + # 10: virbr0: mtu 1500 qdisc noqueue state DOWN group default qlen 1000 + # inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0 + # valid_lft forever preferred_lft forever + + ip -f inet addr show "${bridge}" | grep inet | awk '{print $2}' | cut -d/ -f1 +} + +# The IP address of the current host on the bridge used for the +# default network for libvirt VMs. +# shellcheck disable=SC2034 # used elsewhere +VM_BRIDGE_IP="$(get_vm_bridge_ip)" diff --git a/test/bin/composer_cleanup.sh b/test/bin/composer_cleanup.sh new file mode 100755 index 0000000000..2843797d97 --- /dev/null +++ b/test/bin/composer_cleanup.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# This script cancels any composer jobs and deletes failed and completed jobs. + +cancel() { + local status="$1" + + for id in $(sudo composer-cli compose list | grep "${status}" | cut -f1 -d' '); do + echo "Cancelling ${id}" + sudo composer-cli compose cancel "${id}" + done +} + +do_delete() { + for id in $(sudo composer-cli compose list | grep -v "^ID" | cut -f1 -d' '); do + echo "Deleting ${id}" + sudo composer-cli compose delete "${id}" + done +} + +cancel WAITING +cancel RUNNING +do_delete diff --git a/test/bin/configure_hypervisor_firewall.sh b/test/bin/configure_hypervisor_firewall.sh new file mode 100755 index 0000000000..23ff92e93a --- /dev/null +++ b/test/bin/configure_hypervisor_firewall.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# +# This script should be run on the hypervisor to configure the +# firewall for incoming connections to the VM network. + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +VM_BRIDGE=$(sudo virsh net-info default | grep '^Bridge:' | awk '{print $2}') +VM_BRIDGE_CIDR=$(ip -f inet addr show "${VM_BRIDGE}" | grep inet | awk '{print $2}') +sudo firewall-cmd --permanent --zone=trusted --add-source="${VM_BRIDGE_CIDR}" +sudo firewall-cmd --permanent --zone=public --add-port="${WEB_SERVER_PORT}/tcp" +sudo firewall-cmd --reload diff --git a/test/bin/create_local_repo.sh b/test/bin/create_local_repo.sh new file mode 100755 index 0000000000..2d36a30e95 --- /dev/null +++ b/test/bin/create_local_repo.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# This script should be run on the image build server (usually the +# hypervisor) to create a local RPM repository containing the RPMs +# built from the source under test. + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +mkdir -p "${IMAGEDIR}" +cd "${IMAGEDIR}" + +if [ -d "${LOCAL_REPO}" ]; then + echo "Cleaning up existing repository" + rm -rf "${LOCAL_REPO}" +fi +mkdir -p "${LOCAL_REPO}" + +# Create the local RPM repository for whatever was built from source. +echo "Copying RPMs from ${RPM_SOURCE} to ${LOCAL_REPO}" +# shellcheck disable=SC2086 # no quotes for command arguments to allow word splitting +cp -R ${RPM_SOURCE}/{RPMS,SPECS,SRPMS} "${LOCAL_REPO}/" + +echo "Creating RPM repo at ${LOCAL_REPO}" +createrepo "${LOCAL_REPO}" + +echo "Fixing permissions of RPM repo contents" +find "${LOCAL_REPO}" -type f -exec chmod a+r {} \; +find "${LOCAL_REPO}" -type d -exec chmod a+rx {} \; diff --git a/test/bin/download_images.sh b/test/bin/download_images.sh new file mode 100755 index 0000000000..8ff8df7d55 --- /dev/null +++ b/test/bin/download_images.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# This script should be run on the hypervisor to download all of the +# images from composer. + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +mkdir -p "${VM_DISK_DIR}" + +download_image() { + local buildid="${1}" + # FIXME: These logs should go to the artifacts directory instead + # of the build web server. + # shellcheck disable=SC2086 # pass glob args without quotes + rm -f ${buildid}-*.tar + sudo composer-cli compose logs "${buildid}" + sudo composer-cli compose metadata "${buildid}" + sudo composer-cli compose image "${buildid}" + # shellcheck disable=SC2086 # pass glob args without quotes + sudo chown "$(whoami)." ${buildid}-* +} + +download_dir="${IMAGEDIR}/builds" +mkdir -p "${download_dir}" + +echo "Downloading installer images to ${download_dir}" +cd "${download_dir}" +for blueprint_build in *.image-installer; do + blueprint=$(basename "${blueprint_build}" .image-installer) + buildid=$(cat "${blueprint_build}") + download_image "${buildid}" + iso_file="${buildid}-installer.iso" + echo "Moving ${iso_file} to ${VM_DISK_DIR}/${blueprint}.iso" + mv -f "${iso_file}" "${VM_DISK_DIR}/${blueprint}.iso" +done + +if [ -d "${IMAGEDIR}/repo" ]; then + echo "Cleaning up existing images in ${IMAGEDIR}/repo" + rm -rf "${IMAGEDIR}/repo" +fi + +echo "Downloading ostree commits and metadata to ${download_dir}" +cd "${download_dir}" +for blueprint_build in *.edge-commit; do + blueprint=$(basename "${blueprint_build}" .edge-commit) + buildid=$(cat "${blueprint_build}") + download_image "${buildid}" + commit_file="${buildid}-commit.tar" + echo "Unpacking ${commit_file} ${blueprint}" + tar -C "${IMAGEDIR}" -xf "${commit_file}" +done + +echo "Updating references" +cd "${IMAGEDIR}" +ostree summary --update --repo=repo +ostree summary --view --repo=repo diff --git a/test/bin/force_vm_settings.sh b/test/bin/force_vm_settings.sh new file mode 100755 index 0000000000..81a15cc126 --- /dev/null +++ b/test/bin/force_vm_settings.sh @@ -0,0 +1,24 @@ +#!/bin/bash -x +# +# This script is used by scenario.sh when creating a new VM to force +# specific settings on the host, such as opening firewall ports. + +set -euo pipefail + +# Exit if the current user is not 'root' +if [ "$(id -u)" -ne 0 ] ; then + echo "The '${SCRIPT_NAME}' script must be run with the 'root' user privileges" + exit 1 +fi + +systemctl enable firewalld --now +firewall-cmd --permanent --zone=trusted --add-source=10.42.0.0/16 +firewall-cmd --permanent --zone=trusted --add-source=169.254.169.1 +firewall-cmd --permanent --zone=public --add-port=80/tcp +firewall-cmd --permanent --zone=public --add-port=443/tcp +firewall-cmd --permanent --zone=public --add-port=5353/udp +firewall-cmd --permanent --zone=public --add-port=30000-32767/tcp +firewall-cmd --permanent --zone=public --add-port=30000-32767/udp +firewall-cmd --permanent --zone=public --add-port=6443/tcp +firewall-cmd --permanent --zone=public --add-service=mdns +firewall-cmd --reload diff --git a/test/bin/manage_vm_connections.sh b/test/bin/manage_vm_connections.sh new file mode 100755 index 0000000000..f82a6cea00 --- /dev/null +++ b/test/bin/manage_vm_connections.sh @@ -0,0 +1,226 @@ +#!/bin/bash +# +# This script should be run on the hypervisor to set up port forwarding. + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +usage() { + cat - < The base port number for API server port forwarding. + + -l The base port number for load balancer port forwarding for + services running on the cluster. + + -s The base port number for ssh port forwarding. + +cleanup: Remove iptables rules created by 'remote' command. + +local: Set up connection settings for VMs for local access from the + hypervisor. + +EOF +} + +add_iptables_rules() { + local vm_ip="${1}" + local api_port="${2}" + local ssh_port="${3}" + local lb_port="${4}" + + # Setup external access with port forwarding to allow running commands and tests from the CI container. + + # MicroShift API server + sudo /sbin/iptables -I FORWARD -o virbr0 -p tcp -d "${vm_ip}" --dport 6443 -j ACCEPT + sudo /sbin/iptables -t nat -I PREROUTING -p tcp --dport "${api_port}" -j DNAT --to "${vm_ip}:6443" + + # SSH to the VM + sudo /sbin/iptables -I FORWARD -o virbr0 -p tcp -d "${vm_ip}" --dport 22 -j ACCEPT + sudo /sbin/iptables -t nat -I PREROUTING -p tcp --dport "${ssh_port}" -j DNAT --to "${vm_ip}:22" + + # Port usable for services running on MicroShift + sudo /sbin/iptables -I FORWARD -o virbr0 -p tcp -d "${vm_ip}" --dport 5678 -j ACCEPT + sudo /sbin/iptables -t nat -I PREROUTING -p tcp --dport "${lb_port}" -j DNAT --to "${vm_ip}:5678" + + # Show the rules we just added + sudo /sbin/iptables -L FORWARD | grep "${vm_ip}" + sudo /sbin/iptables -t nat -L PREROUTING | grep "${vm_ip}" +} + +delete_iptables_rules() { + local vm_ip="${1}" + + # If there are no rules grep exits with an error, so do the search + # with an expression that always returns true. Sort the results in + # reverse order so we remove rules from the end of the list and do + # not change the index values of the other rules that need to be + # removed. + + echo "Looking for rules using ${vm_ip}" + # shellcheck disable=SC2162 # read without -r + (sudo /sbin/iptables -L FORWARD --line-numbers | grep "${vm_ip}" | sort -n -r || true) | while read num rule; do + echo "Removing rule ${rule}" + sudo /sbin/iptables -D FORWARD "${num}" + done + + # shellcheck disable=SC2162 # read without -r + (sudo /sbin/iptables -t nat -L PREROUTING --line-numbers | grep "${vm_ip}" | sort -n -r || true) | while read num rule; do + echo "Removing rule ${rule}" + sudo /sbin/iptables -t nat -D PREROUTING "${num}" + done +} + +action_remote() { + + local api_base="" + local ssh_base="" + local lb_base="" + while getopts "a:hl:s:" opt; do + case "${opt}" in + a) + api_base="${OPTARG}" + ;; + l) + lb_base="${OPTARG}" + ;; + s) + ssh_base="${OPTARG}" + ;; + h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac + done + + if [ -z "${api_base}" ]; then + usage + error "Specify API base port with -a option" + exit 1 + fi + local api_port="${api_base}" + + if [ -z "${lb_base}" ]; then + usage + error "Specify LB base port with -l option" + exit 1 + fi + local lb_port="${lb_base}" + + if [ -z "${ssh_base}" ]; then + usage + error "Specify ssh base port with -s option" + exit 1 + fi + local ssh_port="${ssh_base}" + + cd "${SCENARIO_INFO_DIR}" + + local scenario + local vm_name + local vm_ip + for scenario in *; do + pushd "${scenario}" >/dev/null + for vm in vms/*; do + vm_name=$(basename "${vm}") + vm_ip=$(cat "${vm}/ip") + + echo "${scenario}: Configuring ${vm_name} at ${vm_ip} with API port ${api_port}, ssh port ${ssh_port}, and LB port ${lb_port}" + add_iptables_rules "${vm_ip}" "${api_port}" "${ssh_port}" "${lb_port}" + + # Record the ports used for the VM + echo "${api_port}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/api_port" + echo "${ssh_port}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/ssh_port" + echo "${lb_port}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/lb_port" + + # Increment the ports so they are unique + api_port=$((api_port+1)) + ssh_port=$((ssh_port+1)) + lb_port=$((lb_port+1)) + done + popd >/dev/null + done +} + +action_cleanup() { + cd "${SCENARIO_INFO_DIR}" + + local scenario + local vm_name + local vm_ip + for scenario in *; do + pushd "${scenario}" >/dev/null + for vm in vms/*; do + vm_name=$(basename "${vm}") + vm_ip=$(cat "${vm}/ip") + + echo "${scenario}: Cleaning up iptables rules for ${vm_name}" + delete_iptables_rules "${vm_ip}" + done + popd >/dev/null + done +} + +action_local() { + cd "${SCENARIO_INFO_DIR}" + + local api_port=6443 + local ssh_port=22 + local lb_port=5678 + + local scenario + local vm_name + local vm_ip + for scenario in *; do + pushd "${scenario}" >/dev/null + for vm in vms/*; do + vm_name=$(basename "${vm}") + vm_ip=$(cat "${vm}/ip") + + echo "${scenario}: Configuring ${vm_name} at ${vm_ip} with API port ${api_port} and ssh port ${ssh_port}" + + # Record the ports used for the VM + echo "${api_port}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/api_port" + echo "${ssh_port}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/ssh_port" + echo "${lb_port}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/lb_port" + echo "${vm_ip}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/public_ip" + done + popd >/dev/null + done +} + +if [ $# -eq 0 ]; then + usage + exit 1 +fi +action="${1}" +shift + +case "${action}" in + remote|cleanup|local) + "action_${action}" "$@" + ;; + -h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; +esac diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh new file mode 100755 index 0000000000..7656fa16cc --- /dev/null +++ b/test/bin/scenario.sh @@ -0,0 +1,351 @@ +#!/bin/bash +# +# This script should be run on the hypervisor to manage the VMs needed +# for a scenario. + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +DEFAULT_BOOT_BLUEPRINT="rhel-9.2" +LVM_SYSROOT_SIZE="10240" +WEB_SERVER_URL="http://${VM_BRIDGE_IP}:${WEB_SERVER_PORT}" +PULL_SECRET="$(jq -c . "${HOME}/.pull-secret.json")" +PUBLIC_IP=${PUBLIC_IP:-""} # may be overridden in global settings file +VM_BOOT_TIMEOUT=8m + +full_vm_name() { + local base="${1}" + echo "${SCENARIO}-${base}" +} + +# Public function to render a unique kickstart from a template for a +# VM in a scenario. +# +# Arguments: +# vmname -- The short name of the VM (e.g., "host1") +# template -- The path to the kickstart template file, relative to +# the scenario directory. +# boot_commit_ref -- The reference to the image that should be booted +# first on the host. This usually matches an image +# blueprint name. +prepare_kickstart() { + local vmname="$1" + local template="$2" + local boot_commit_ref="$3" + + local full_vmname + local output_file + local vm_hostname + + full_vmname="$(full_vm_name "${vmname}")" + output_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/kickstart.ks" + vm_hostname="${full_vmname/./-}" + + echo "Preparing kickstart file ${template} ${output_file}" + if [ ! -f "${KICKSTART_TEMPLATE_DIR}/${template}" ]; then + # FIXME: Perhaps we want a default kickstart to reduce duplication? + error "No ${template} in ${KICKSTART_TEMPLATE_DIR}" + exit 1 + fi + mkdir -p "$(dirname "${output_file}")" + # shellcheck disable=SC2002 # useless cat + cat "${KICKSTART_TEMPLATE_DIR}/${template}" \ + | sed -e "s/REPLACE_LVM_SYSROOT_SIZE/${LVM_SYSROOT_SIZE}/g" \ + -e "s|REPLACE_OSTREE_SERVER_URL|${WEB_SERVER_URL}/repo|g" \ + -e "s|REPLACE_BOOT_COMMIT_REF|${boot_commit_ref}|g" \ + -e "s|REPLACE_PULL_SECRET|${PULL_SECRET}|g" \ + -e "s|REPLACE_HOST_NAME|${vm_hostname}|g" \ + -e "s|REPLACE_REDHAT_AUTHORIZED_KEYS|${REDHAT_AUTHORIZED_KEYS}|g" \ + -e "s|REPLACE_PUBLIC_IP|${PUBLIC_IP}|g" \ + > "${output_file}" +} + +# Show the IP address of the VM +function get_vm_ip { + local vmname="${1}" + sudo virsh domifaddr "${vmname}" \ + | grep vnet \ + | awk '{print $4}' \ + | cut -f1 -d/ +} + +# Try to login to the host via ssh until the connection is accepted +wait_for_ssh() { + local ip="${1}" + + echo "Waiting ${VM_BOOT_TIMEOUT} for ssh access to ${ip}" + timeout "${VM_BOOT_TIMEOUT}" bash -c "until ssh -oBatchMode=yes -oStrictHostKeyChecking=accept-new redhat@${ip} 'echo host is up'; do date; sleep 5; done" +} + +# Add the microshift config file needed to allow remote access via IP address +enable_ip_access() { + local vmname="${1}" + local ip="${2}" + + echo "Waiting ${VM_BOOT_TIMEOUT} for ${full_vmname} to boot" + wait_for_ssh "${ip}" + + echo "Adjusting VM host settings" + scp "${SCRIPTDIR}/force_vm_settings.sh" "redhat@${ip}:/home/redhat/" + ssh "redhat@${ip}" "chmod +x /home/redhat/force_vm_settings.sh && sudo /home/redhat/force_vm_settings.sh" +} + +# Wait for greenboot health check to complete, without checking the results +wait_for_greenboot() { + local vmname="${1}" + local ip="${2}" + + echo "Waiting ${VM_BOOT_TIMEOUT} for greenboot on ${vmname} to complete" + timeout "${VM_BOOT_TIMEOUT}" bash -c "until ssh redhat@${ip} \"sudo journalctl -n 5 -u greenboot-healthcheck; sudo systemctl status greenboot-healthcheck | grep 'active (exited)'\"; do date; sleep 10; done" +} + +# Public function to start a VM. +# +# Creates a new VM using the scenario name and the vmname given to +# create a unique name. Uses the boot_blueprint argument to select the +# ISO from which to boot. If no boot_blueprint is specified, uses +# DEFAULT_BOOT_BLUEPRINT. +# +# Arguments +# vmname -- The short name of the VM in the scenario (e.g., "host1"). +# boot_blueprint -- The image blueprint used to create the ISO that +# should be used to boot the VM. This is _not_ +# necessarily the image to be installed (see +# prepare_kickstart). +launch_vm() { + local vmname="$1" + local boot_blueprint="${2:-${DEFAULT_BOOT_BLUEPRINT}}" + + local full_vmname + local kickstart_url + + full_vmname="$(full_vm_name "${vmname}")" + kickstart_url="${WEB_SERVER_URL}/scenario-info/${SCENARIO}/vms/${vmname}/kickstart.ks" + + # Checking web server for kickstart file + if ! curl -o /dev/null "${kickstart_url}" >/dev/null 2>&1; then + error "Failed to load kickstart file from ${kickstart_url}" + exit 1 + fi + + echo "Creating ${full_vmname}" + + # FIXME: variable for vcpus? + # FIXME: variable for memory? + # FIXME: variable for ISO + timeout "${VM_BOOT_TIMEOUT}" sudo virt-install \ + --noautoconsole \ + --name "${full_vmname}" \ + --vcpus 2 \ + --memory 4092 \ + --disk "path=${VM_DISK_DIR}/${full_vmname}.qcow2,size=30" \ + --network network=default,model=virtio \ + --events on_reboot=restart \ + --location "${VM_DISK_DIR}/${boot_blueprint}.iso" \ + --extra-args "inst.ks=${kickstart_url}" \ + --wait + + # Wait for an IP to be assigned + ip=$(get_vm_ip "${full_vmname}") + while [ -z "${ip}" ]; do + echo "Waiting for VM ${full_vmname} to have an IP" + sleep 30 + ip=$(get_vm_ip "${full_vmname}") + done + echo "VM ${full_vmname} has IP ${ip}" + + # Remove any previous key info for the host + if [ -f "${HOME}/.ssh/known_hosts" ]; then + echo "Clearing known_hosts entry for ${ip}" + ssh-keygen -R "${ip}" + fi + + # Record the IP of this VM so our caller can use it to configure + # port forwarding and the firewall. + mkdir -p "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}" + echo "${ip}" > "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/ip" + # Record the _public_ IP of the VM so the test suite can use it to + # access the host. This is useful when the public IP is the + # hypervisor forwarding connections. If we have no PUBLIC_IP, use + # the VM IP and assume a local connection. + if [ -n "${PUBLIC_IP}" ]; then + echo "${PUBLIC_IP}" > "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/public_ip" + else + echo "${ip}" > "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/public_ip" + fi + + enable_ip_access "${full_vmname}" "${ip}" + wait_for_greenboot "${full_vmname}" "${ip}" + + echo "${full_vmname} is up and ready" +} + +# Clean up the resources for one VM. +remove_vm() { + local vmname="${1}" + + local full_vmname + full_vmname="$(full_vm_name "${vmname}")" + + # Remove the actual VM and its storage + if sudo virsh dumpxml "${full_vmname}" >/dev/null; then + if ! sudo virsh dominfo "${full_vmname}" | grep '^State' | grep -q 'shut off'; then + sudo virsh destroy "${full_vmname}" + fi + sudo virsh undefine "${full_vmname}" + fi + if sudo virsh vol-dumpxml "${full_vmname}.qcow2" vm-storage >/dev/null; then + sudo virsh vol-delete "${full_vmname}.qcow2" vm-storage + fi + + # Remove the info file so something processing the VMs does not + # assume the file exists. This is most useful in a local setting. + rm -rf "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}" +} + +# Run the tests for the current scenario +run_tests() { + local vmname="${1}" + shift + + echo "Running tests with $# args" "$@" + + if [ ! -d "${RF_VENV}" ]; then + error "RF_VENV (${RF_VENV}) does not exist, create it with: ${ROOTDIR}/scripts/fetch_tools.sh robotframework" + exit 1 + fi + local rf_binary="${RF_VENV}/bin/robot" + if [ ! -f "${rf_binary}" ]; then + error "robot is not installed to ${rf_binary}" + exit 1 + fi + + + local ssh_port_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/ssh_port" + local api_port_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/api_port" + local lb_port_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/lb_port" + local public_ip_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/public_ip" + local ip_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/ip" + local api_port + local ssh_port + local public_ip + local vm_ip + local f + for f in "${ssh_port_file}" "${api_port_file}" "${lb_port_file}" "${public_ip_file}" "${ip_file}"; do + if [ ! -f "${f}" ]; then + error "Cannot read ${f}" + exit 1 + fi + done + ssh_port=$(cat "${ssh_port_file}") + api_port=$(cat "${api_port_file}") + lb_port=$(cat "${lb_port_file}") + public_ip=$(cat "${public_ip_file}") + vm_ip=$(cat "${ip_file}") + + local variable_file="${SCENARIO_INFO_DIR}/${SCENARIO}/variables.yaml" + echo "Writing variables to ${variable_file}" + mkdir -p "$(dirname "${variable_file}")" + cat - <caddy.log 2>&1 & diff --git a/test/bin/wait_images.py b/test/bin/wait_images.py new file mode 100755 index 0000000000..bc5d25e6f7 --- /dev/null +++ b/test/bin/wait_images.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# +# This script is run by build_images.sh after all of the builds have +# been enqueued. It waits for the jobs identified by the UUIDs +# provided as input to either fail or complete. + +import argparse +import json +import logging +import subprocess +import time + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s: %(message)s', +) + +STATUS_COMMAND = ['sudo', 'composer-cli', 'compose', 'status', '--json'] + + +# Convert the weird queue json data structure to a sequence of jobs. +# $ sudo composer-cli compose status --json +# [ +# { +# "method": "GET", +# "path": "/compose/queue", +# "status": 200, +# "body": { +# "new": [ +# { +# "blueprint": "rhel-9.2", +# "compose_type": "image-installer", +# "id": "7bdb78bf-c4b6-4f19-80df-073d38fade56", +# "image_size": 0, +# "job_created": 1687466891.2393456, +# "queue_status": "WAITING", +# "version": "0.0.1" +# } +# ], +# "run": [ +# { +# "blueprint": "rhel-9.2", +# "compose_type": "edge-commit", +# "id": "4aa19f32-54e3-42ce-a4ba-cf038a3df91c", +# "image_size": 0, +# "job_created": 1687466889.5383687, +# "job_started": 1687466889.5480232, +# "queue_status": "RUNNING", +# "version": "0.0.1" +# } +# ] +# } +# }, +# { +# "method": "GET", +# "path": "/compose/finished", +# "status": 200, +# "body": { +# "finished": [] +# } +# }, +# { +# "method": "GET", +# "path": "/compose/failed", +# "status": 200, +# "body": { +# "failed": [] +# } +# } +# ] +def flattened_status(): + result = subprocess.run(STATUS_COMMAND, stdout=subprocess.PIPE) + if result.returncode != 0: + raise SystemError(f'Status command returned {result.returncode}') + status = json.loads(result.stdout) + for result_set in status: + for state in result_set["body"]: + for job in result_set["body"][state]: + yield job + + +def main(build_ids): + ignore_ids = set() + known_ids = set(build_ids) + found_ids = set() + while build_ids: + logging.info(f'Waiting for {build_ids}') + for job in flattened_status(): + job_id = job["id"] + found_ids.add(job_id) + status_text = f'{job_id} {job["compose_type"]} for {job["blueprint"]} - {job["queue_status"]}' + if job_id in build_ids: + logging.info(status_text) + if job["queue_status"] in {"FAILED", "FINISHED"}: + # After a job fails or finishes, stop reporting its status. + build_ids.remove(job_id) + elif job_id not in ignore_ids and job_id not in known_ids: + # Report any unknown jobs one time, then ignore them. + logging.info(f'{status_text} (unknown job)') + ignore_ids.add(job_id) + to_ignore = [] + for build_id in build_ids: + if build_id not in found_ids: + logging.info(f'{build_id} is not a known build, ignoring') + to_ignore.append(build_id) + for i in to_ignore: + build_ids.remove(i) + if build_ids: + time.sleep(30) + + +if __name__ == '__main__': + cli_parser = argparse.ArgumentParser(add_help=True) + cli_parser.add_argument( + 'build_id', + nargs='+', + help="a build id (UUID) from composer", + ) + args = cli_parser.parse_args() + main(args.build_id) diff --git a/test/scenario_settings.sh.example b/test/scenario_settings.sh.example new file mode 100644 index 0000000000..8713499ef8 --- /dev/null +++ b/test/scenario_settings.sh.example @@ -0,0 +1,14 @@ +#!/bin/bash + +# Set the PUBLIC_IP to the IP address used to access the hypervisor. +# shellcheck disable=SC2034 +PUBLIC_IP=10.8.1.133 + +# Set SSH_PUBLIC_KEY to the filename of the ssh key to be used to +# access the VMs created. +# shellcheck disable=SC2034 +SSH_PUBLIC_KEY=${SSH_PUBLIC_KEY:-${HOME}/.ssh/id_rsa.pub} + +# Set SSH_PRIVATE_KEY to the filename of the matching ssh key, +# if any. Set it to an empty string to use ssh-agent. +#SSH_PRIVATE_KEY=${SSH_PRIVATE_KEY:-${HOME}/.ssh/id_rsa} From e166b6d6f3aa01b94d69d69ec6b1bd0d9bc8903c Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 29 Jun 2023 17:25:26 -0400 Subject: [PATCH 09/10] USHIFT-1363: add initial microshift test scenarios Add image blueprint data for RHEL 9.2 images without MicroShift and with MicroShift 4.13 and the package built from source. Add package sources for fast-datapath and RHOCP 4.13 for dependencies. Add a kickstart template for booting an image from an ostree commit. Add a scenario for running the standard test suite against a host running RHEL 9.2 and MicroShift built from source. Add a scenario for running the upgrade test against a host running RHEL 9.2 and MicroShift built from source. --- test/bin/scenario.sh | 5 +- .../rhel92-microshift413.toml | 25 +++++ test/image-blueprints/rhel92-source.toml | 25 +++++ test/image-blueprints/rhel92.toml | 7 ++ .../kickstart-templates/kickstart.ks.template | 95 +++++++++++++++++++ test/package-sources/fast-datapath-rhel9.toml | 8 ++ test/package-sources/microshift-local.toml | 7 ++ test/package-sources/rhocp-4.13.toml | 8 ++ ...el-9.2-microshift-source-backup-restore.sh | 16 ++++ ...el-9.2-microshift-source-standard-suite.sh | 16 ++++ test/variables.yaml.example | 2 +- 11 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 test/image-blueprints/rhel92-microshift413.toml create mode 100644 test/image-blueprints/rhel92-source.toml create mode 100644 test/image-blueprints/rhel92.toml create mode 100644 test/kickstart-templates/kickstart.ks.template create mode 100644 test/package-sources/fast-datapath-rhel9.toml create mode 100644 test/package-sources/microshift-local.toml create mode 100644 test/package-sources/rhocp-4.13.toml create mode 100644 test/scenarios/rhel-9.2-microshift-source-backup-restore.sh create mode 100644 test/scenarios/rhel-9.2-microshift-source-standard-suite.sh diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 7656fa16cc..e2c823f139 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -11,7 +11,8 @@ source "${SCRIPTDIR}/common.sh" DEFAULT_BOOT_BLUEPRINT="rhel-9.2" LVM_SYSROOT_SIZE="10240" WEB_SERVER_URL="http://${VM_BRIDGE_IP}:${WEB_SERVER_PORT}" -PULL_SECRET="$(jq -c . "${HOME}/.pull-secret.json")" +PULL_SECRET="${PULL_SECRET:-${HOME}/.pull-secret.json}" +PULL_SECRET_CONTENT="$(jq -c . "${PULL_SECRET}")" PUBLIC_IP=${PUBLIC_IP:-""} # may be overridden in global settings file VM_BOOT_TIMEOUT=8m @@ -55,7 +56,7 @@ prepare_kickstart() { | sed -e "s/REPLACE_LVM_SYSROOT_SIZE/${LVM_SYSROOT_SIZE}/g" \ -e "s|REPLACE_OSTREE_SERVER_URL|${WEB_SERVER_URL}/repo|g" \ -e "s|REPLACE_BOOT_COMMIT_REF|${boot_commit_ref}|g" \ - -e "s|REPLACE_PULL_SECRET|${PULL_SECRET}|g" \ + -e "s|REPLACE_PULL_SECRET|${PULL_SECRET_CONTENT}|g" \ -e "s|REPLACE_HOST_NAME|${vm_hostname}|g" \ -e "s|REPLACE_REDHAT_AUTHORIZED_KEYS|${REDHAT_AUTHORIZED_KEYS}|g" \ -e "s|REPLACE_PUBLIC_IP|${PUBLIC_IP}|g" \ diff --git a/test/image-blueprints/rhel92-microshift413.toml b/test/image-blueprints/rhel92-microshift413.toml new file mode 100644 index 0000000000..23de492b62 --- /dev/null +++ b/test/image-blueprints/rhel92-microshift413.toml @@ -0,0 +1,25 @@ +name = "rhel-9.2-microshift-4.13" +description = "" +version = "0.0.1" +modules = [] +groups = [] +distro = "rhel-92" + +[[packages]] +name = "microshift" +version = "4.13*" + +[[packages]] +name = "microshift-greenboot" +version = "4.13*" + +[[packages]] +name = "microshift-networking" +version = "4.13*" + +[[packages]] +name = "microshift-selinux" +version = "4.13*" + +[customizations.services] +enabled = ["microshift"] diff --git a/test/image-blueprints/rhel92-source.toml b/test/image-blueprints/rhel92-source.toml new file mode 100644 index 0000000000..1fa0d5ca67 --- /dev/null +++ b/test/image-blueprints/rhel92-source.toml @@ -0,0 +1,25 @@ +name = "rhel-9.2-microshift-source" +description = "" +version = "0.0.1" +modules = [] +groups = [] +distro = "rhel-92" + +[[packages]] +name = "microshift" +version = "${SOURCE_VERSION}" + +[[packages]] +name = "microshift-greenboot" +version = "${SOURCE_VERSION}" + +[[packages]] +name = "microshift-networking" +version = "${SOURCE_VERSION}" + +[[packages]] +name = "microshift-selinux" +version = "${SOURCE_VERSION}" + +[customizations.services] +enabled = ["microshift"] diff --git a/test/image-blueprints/rhel92.toml b/test/image-blueprints/rhel92.toml new file mode 100644 index 0000000000..a59191514c --- /dev/null +++ b/test/image-blueprints/rhel92.toml @@ -0,0 +1,7 @@ +name = "rhel-9.2" +description = "" +version = "0.0.1" +distro = "rhel-92" +modules = [] +groups = [] +packages = [] diff --git a/test/kickstart-templates/kickstart.ks.template b/test/kickstart-templates/kickstart.ks.template new file mode 100644 index 0000000000..5c938bd515 --- /dev/null +++ b/test/kickstart-templates/kickstart.ks.template @@ -0,0 +1,95 @@ +lang en_US.UTF-8 +keyboard us +timezone UTC +text +reboot + +# Configure network to use DHCP and activate on boot +network --bootproto=dhcp --device=link --activate --onboot=on --hostname=REPLACE_HOST_NAME + +# Partition disk with a 1GB boot XFS partition and an LVM volume containing a 10GB+ system root +# The remainder of the volume will be used by the CSI driver for storing data +# +# For example, a 20GB disk would be partitioned in the following way: +# +# NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT +# sda 8:0 0 20G 0 disk +# ├─sda1 8:1 0 200M 0 part /boot/efi +# ├─sda1 8:1 0 800M 0 part /boot +# └─sda2 8:2 0 19G 0 part +# └─rhel-root 253:0 0 10G 0 lvm /sysroot +# +zerombr +clearpart --all --initlabel +part /boot/efi --fstype=efi --size=200 +part /boot --fstype=xfs --asprimary --size=800 +# Uncomment this line to add a SWAP partition of the recommended size +#part swap --fstype=swap --recommended +part pv.01 --grow +volgroup rhel pv.01 +logvol / --vgname=rhel --fstype=xfs --size=REPLACE_LVM_SYSROOT_SIZE --name=root +rootpw --lock + +# Configure ostree +ostreesetup --nogpg --osname=rhel --remote=edge --url=REPLACE_OSTREE_SERVER_URL --ref=REPLACE_BOOT_COMMIT_REF + +%post --log=/var/log/anaconda/post-install.log --erroronfail + +# Set the configuration of MicroShift to include subjectAltNames +cat - >/etc/microshift/config.yaml <>/etc/microshift/config.yaml +fi + +# Replace the ostree server URL +echo -e 'url=REPLACE_OSTREE_SERVER_URL' >> /etc/ostree/remotes.d/edge.conf + +# The pull secret is mandatory for MicroShift builds on top of OpenShift, but not OKD +# The /etc/crio/crio.conf.d/microshift.conf references the /etc/crio/openshift-pull-secret file +cat > /etc/crio/openshift-pull-secret < /etc/sudoers.d/microshift + +# # Add authorized ssh keys +mkdir -m 700 /home/redhat/.ssh +cat >> /home/redhat/.ssh/authorized_keys <> /root/.profile + +# Configure systemd journal service to persist logs between boots and limit their size to 1G +sudo mkdir -p /etc/systemd/journald.conf.d +cat > /etc/systemd/journald.conf.d/microshift.conf < Date: Sat, 1 Jul 2023 16:29:53 -0400 Subject: [PATCH 10/10] USHIFT-1363: add ci integration scripts for scenario tests --- test/README.md | 10 +++---- test/bin/ci_phase_iso_boot.sh | 41 +++++++++++++++++++++++++ test/bin/ci_phase_iso_build.sh | 55 ++++++++++++++++++++++++++++++++++ test/bin/cleanup.sh | 16 ++++++++++ 4 files changed, 117 insertions(+), 5 deletions(-) create mode 100755 test/bin/ci_phase_iso_boot.sh create mode 100755 test/bin/ci_phase_iso_build.sh create mode 100755 test/bin/cleanup.sh diff --git a/test/README.md b/test/README.md index 10152e4711..55d78342f4 100644 --- a/test/README.md +++ b/test/README.md @@ -321,14 +321,14 @@ $ ./bin/scenario.sh cleanup ./scenarios/rhel-9.2-microshift-source-standard-sui ## CI Integration Scripts -### ci_phase_iso_build_pre.sh - -Runs on the CI cluster, in a container. Copies everything needed to -build ISOs onto the hypervisor, then triggers `ci_phase_iso_build.sh`. - ### ci_phase_iso_build.sh Runs on the hypervisor. Responsible for all of the setup to build all needed images. Rebuilds MicroShift RPMs from source, sets up RPM repo, sets up osbuild workers, builds the images, and creates the web server to host the images. + +### ci_phase_iso_boot.sh + +Runs on the hypervisor. Responsible for launching all of the VMs that +are used in the test step. diff --git a/test/bin/ci_phase_iso_boot.sh b/test/bin/ci_phase_iso_boot.sh new file mode 100755 index 0000000000..032cc7b2f6 --- /dev/null +++ b/test/bin/ci_phase_iso_boot.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# This script runs on the hypervisor, from the iso-build step. + +set -xeuo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +# Log output automatically +LOGDIR="${ROOTDIR}/_output/ci-logs" +LOGFILE="${LOGDIR}/$(basename "$0" .sh).log" +if [ ! -d "${LOGDIR}" ]; then + mkdir -p "${LOGDIR}" +fi +echo "Logging to ${LOGFILE}" +# Set fd 1 and 2 to write to the log file +exec &> >(tee >(awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; fflush() }' >"${LOGFILE}")) + +API_EXTERNAL_BASE_PORT="${1}" +SSH_EXTERNAL_BASE_PORT="${2}" +LB_EXTERNAL_BASE_PORT="${3}" + +cd ~/microshift/test + +# Start the web server to host the kickstart files and ostree commit +# repository. +bash -x ./bin/start_webserver.sh + +# Build all of the needed VMs +for scenario in scenarios/*.sh; do + time bash -x ./bin/scenario.sh create "${scenario}" +done + +# Set up port forwarding +bash -x ./bin/manage_vm_connections.sh remote -a "${API_EXTERNAL_BASE_PORT}" -s "${SSH_EXTERNAL_BASE_PORT}" -l "${LB_EXTERNAL_BASE_PORT}" + +# Kill the web server +pkill caddy || true + +echo "Boot phase complete" diff --git a/test/bin/ci_phase_iso_build.sh b/test/bin/ci_phase_iso_build.sh new file mode 100755 index 0000000000..f10b6cbf42 --- /dev/null +++ b/test/bin/ci_phase_iso_build.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# This script runs on the hypervisor, from the iso-build step. + +set -xeuo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# Cannot use common.sh because virsh is not installed, but we only +# need ROOTDIR to set up logging in this script. +ROOTDIR="$(cd "${SCRIPTDIR}/../.." && pwd)" + +# Log output automatically +LOGDIR="${ROOTDIR}/_output/ci-logs" +LOGFILE="${LOGDIR}/$(basename "$0" .sh).log" +if [ ! -d "${LOGDIR}" ]; then + mkdir -p "${LOGDIR}" +fi +echo "Logging to ${LOGFILE}" +# Set fd 1 and 2 to write to the log file +exec &> >(tee >(awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; fflush() }' >"${LOGFILE}")) + +PULL_SECRET=${PULL_SECRET:-${HOME}/.pull-secret.json} + +cd ~/microshift + +# Get firewalld and repos in place. Use scripts to get the right repos +# for each branch. +bash -x ./scripts/devenv-builder/configure-vm.sh --no-build --force-firewall "${PULL_SECRET}" +bash -x ./scripts/image-builder/configure.sh + +# Make sure libvirtd is running. We do this here, because some of the +# other scripts use virsh. +bash -x ./scripts/devenv-builder/manage-vm.sh config + +# Fix up firewall so the VMs on their NAT network can talk to the +# server running on the hypervisor. We do this here, before creating +# any VMs, because the iptables rules added when the VMs are created +# are not persistent and are lost when this script reloads the +# firewall. +cd ~/microshift/test/ +bash -x ./bin/configure_hypervisor_firewall.sh + +# Re-build from source. +cd ~/microshift/ +rm -rf ./_output/rpmbuild +make rpm + +# Set up for scenario tests +cd ~/microshift/test/ +timeout 20m bash -x ./bin/create_local_repo.sh +timeout 20m bash -x ./bin/start_osbuild_workers.sh 5 +timeout 20m bash -x ./bin/build_images.sh +timeout 20m bash -x ./bin/download_images.sh + +echo "Build phase complete" diff --git a/test/bin/cleanup.sh b/test/bin/cleanup.sh new file mode 100755 index 0000000000..5d2a1bbc29 --- /dev/null +++ b/test/bin/cleanup.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -xeuo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +"${SCRIPTDIR}/composer_cleanup.sh" + +for scenario in scenarios/*.sh; do + ./bin/scenario.sh cleanup "${scenario}" +done + +pkill caddy || true + +rm -rf "${IMAGEDIR}"