diff --git a/.gitignore b/.gitignore index 2aae810de3..70935d154f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ _output sshfile ansible/*.txt test/variables.yaml +test/scenario_settings.sh diff --git a/scripts/devenv-builder/manage-vm.sh b/scripts/devenv-builder/manage-vm.sh index 95bc5ea648..0231eb5c05 100755 --- a/scripts/devenv-builder/manage-vm.sh +++ b/scripts/devenv-builder/manage-vm.sh @@ -1,5 +1,7 @@ #!/usr/bin/bash +set -eo pipefail + # https://github.com/openshift/microshift/blob/main/docs/devenv_setup.md # https://github.com/openshift/microshift/blob/main/docs/devenv_setup_auto.md @@ -59,7 +61,24 @@ function get_base_isofile { } function action_config() { - sudo dnf install -y libvirt virt-manager virt-install virt-viewer libvirt-client qemu-kvm qemu-img sshpass + local tries=0 + local failed=true + local deps="libvirt virt-manager virt-install virt-viewer libvirt-client qemu-kvm qemu-img sshpass" + + set +e # retry because we see errors caching different RPMs in CI + while [ ${tries} -lt 5 ]; do + # shellcheck disable=SC2086 + if sudo dnf install -y ${deps}; then + failed=false + break + fi + ((tries+=1)) + done + if ${failed}; then + exit 1 + fi + set -e + if [ "$(systemctl is-active libvirtd.socket)" != "active" ] ; then echo "Enabling libvirtd" sudo systemctl enable --now libvirtd diff --git a/scripts/image-builder/configure.sh b/scripts/image-builder/configure.sh index 198fb02934..6c39641738 100755 --- a/scripts/image-builder/configure.sh +++ b/scripts/image-builder/configure.sh @@ -14,7 +14,7 @@ sudo firewall-cmd --add-service=cockpit --permanent # The mock utility comes from the EPEL repository sudo dnf install -y "https://dl.fedoraproject.org/pub/epel/epel-release-latest-${OSVERSION}.noarch.rpm" -sudo dnf install -y mock +sudo dnf install -y mock caddy tomcli sudo usermod -a -G mock "$(whoami)" # Verify umask and home directory permissions diff --git a/scripts/verify/verify-shell.sh b/scripts/verify/verify-shell.sh index d6e36d0291..f7a4706c76 100755 --- a/scripts/verify/verify-shell.sh +++ b/scripts/verify/verify-shell.sh @@ -20,5 +20,18 @@ CHECK_FILE_LIST=$(find . \( -type d \( ${IGNORE_PATHS} \) -o -name "${IGNORE_FIL for f in ${CHECK_FILE_LIST} ; do echo "shellcheck: ${f}" - "${SHELL_CHECK}" -x "${f}" + # Use format=gcc so integration with editors allows a developer to + # jump right to the file with an issue. + # + # Exclude SC1091 because we source other scripts using variables + # and the linter cannot interpret the variables and find the + # files. + # + # Add --external-sources to allow `source` calls from outside + # the list of checked files. + "${SHELL_CHECK}" \ + --format=gcc \ + --exclude=SC1091 \ + --external-sources \ + "${f}" done diff --git a/test/README.md b/test/README.md index 17e3bc2c22..55d78342f4 100644 --- a/test/README.md +++ b/test/README.md @@ -109,3 +109,226 @@ For more options, see the help output for the `robot` command. ``` $ ./_output/robotenv/bin/robot -h ``` + +## Test Scenarios in CI + +The test scenario tools in the `bin` directory are useful for running +more complex test cases that require VMs and different images. + +### Package Sources + +In order to build different images, Composer needs to be configured +with all of the right package sources to pull the required RPMs. The +sources used by all scenarios are in the `package-sources` directory. + +Package source definition files are templates using `envsubst`, +which means the files can have shell variables embedded. + +``` +id = "fast-datapath" +name = "Fast Datapath for RHEL 9" +type = "yum-baseurl" +url = "https://cdn.redhat.com/content/dist/layered/rhel9/${UNAME_M}/fast-datapath/os" +check_gpg = true +check_ssl = true +system = false +rhsm = true +``` + +Refer to `./bin/build_images.sh` for the set of known variables that can +be expanded. + +## Image Blueprints + +The image blueprints are independent of the test scenarios so that +images can be reused. Be careful making changes to specific images in +case the change affects the way the image is used in different +scenarios. Be careful adding unnecessary new images, since each image +takes time to build and may slow down the overall test job. + +Add blueprints as TOML files in the `image-blueprints` directory, then +add a short description of the image here for reference. + +Name | Purpose +---- | ------- +rhel-9.2 | A simple RHEL image without MicroShift. +rhel-9.2-microshift-4.13 | A RHEL 9.2 image with the latest MicroShift 4.13 z-stream installed and enabled. +rhel-9.2-microshift-source | A RHEL 9.2 image with the RPMs built from source. + +## Preparing to run test scenarios + +### Creating the local RPM repository + +After running `make rpm` at the top of the source tree, run +`./bin/create_local_repo.sh` from this directory to copy the necessary +files into a location that can be used as an RPM repository by +Composer. If you build new RPMs, you need to re-run +`create_local_repo.sh` *and* build new images. + +### Creating the images + +Use `./bin/start_osbuild_workers.sh` to create multiple workers for +building images in parallel. This is optional, and not necessarily +recommended on a laptop. + +Use `./bin/build_images.sh` to build all of the images for all of the +blueprints available. + +### Downloading the images for use by the test scenarios + +Use `./bin/download_images.sh` to download all of the images from +Composer's cache and set up the directory for the web server to host +the files needed to launch VMs and run test scenarios. + +### Global settings + +The test scenario tool uses several global settings that may be +configured before the tool is used in `./scenario_settings.sh`. You +can copy `./scenario_settings.sh.example` as a starting point. + +`PUBLIC_IP` -- The public IP of the hypervisor, when accessing VMs +remotely through port-forwarded connections. + +`SSH_PUBLIC_KEY` -- The name of the public key file to use for +providing password-less access to the VMs. + +`SSH_PRIVATE_KEY` -- The name of the private key file to use for +providing password-less access to the VMs. Set to an empty string to +use ssh-agent. + +### Creating test infrastructure + +Use `./bin/start_webserver.sh` to run a caddy web server to serve the +images needed for the test scenarios. + +Use `./bin/configure_hypervisor_firewall.sh` to set up the firewall +rules that allow VMs to access the web server on the hypervisor. + +Use `./bin/scenario.sh` to create test infrastructure for a scenario +with the `create` argument and a scenario directory name as input. + +``` +$ ./bin/scenario.sh create ./scenarios/rhel-9.2-microshift-source-standard-suite.sh +``` + +### Enabling connections to the VMs + +Run `./bin/manage_vm_connections.sh` on the hypervisor to set up the API +server and ssh port of each VM. The appropriate connection ports are +written to the `$SCENARIO_INFO_DIR` directory (refer to `common.sh` +for the setting for the variable), depending on the mode used. + +For CI integration or a remote hypervisor, use `remote` and pass the +starting ports for the API server and ssh server port forwarding. + +``` +$ ./bin/manage_vm_connections.sh remote -a 7000 -s 6000 -l 7500 +``` + +To run the tests from a local hypervisor, as in a local developer +configuration, use `local` with no other arguments. + +``` +$ ./bin/manage_vm_connections.sh local +``` + +### Run a scenario + +Use `./bin/scenario.sh run` with a scenario file to run the tests for +the scenario. + +``` +$ ./bin/scenario.sh run ./scenarios/rhel-9.2-microshift-source-standard-suite.sh +``` + +## Scenario definitions + +Scenarios are saved as shell scripts under `scenarios`. Each +scenario includes several functions that are combined +with the framework scripts to take the specific actions for the +combination of images and tests that make up the scenario. + +The scenario script should be defined with a combination of the RHEL +version(s), MicroShift version(s), and an indication of what sort of +tests are being run. For example, +`rhel-9.2-microshift-source-standard-suite.sh` runs the standard test +suite (not the ostree upgrade tests) against MicroShift built from +source running on a RHEL 9.2 image. + +Scenarios define VMs using short names, like `host1`, which are made +unique across the entire set of scenarios. VMs are not reused across +scenarios. + +Scenarios use images defined by the blueprints created +earlier. Blueprints and images are reused between scenarios. Refer to +"Image Blueprints" above for details. + +Scenarios use kickstart templates from the `kickstart-templates` +directory. Kickstart templates are reused between scenarios. + +All of the functions that act as the scenario API are run in the +context of `scenario.sh` and can therefore use any functions defined +there. + +### scenario_create_vms + +This function should do any work needed to boot all of the VMs needed +for the scenario, including producing kickstart files and launching +the VMs themselves. + +The `prepare_kickstart` function takes as input the VM name, kickstart +template file relative to the `kickstart-templates` directory, and the +initial edge image to boot. It produces a unique kickstart file _for +that VM_ in the `${SCENARIO_INFO_DIR}` directory. Use the function +multiple times to create kickstart files for additional VMs. + +The `launch_vm` function takes as input the VM name. It expects a +kickstart file to already exist, and it defines a new VM configured to +boot from the installer ISO and the kickstart file. + +### scenario_remove_vms + +This functions is used to remove any VMs defined by the scenario. It +should call `remove_vm` for each VM created by `scenario_create_vms`, +and take any other cleanup actions that might be unique to the +scenario based on other steps taken in `scenario_create_vms`. + +It is not necessary to explicitly clean up the scenario metadata in +`${SCENARIO_INFO_DIR}`. + +### scenario_run_tests + +This function runs the tests. It is invoked separately because the +same host may be used with multiple test runs when working on a local +developer system. + +The function `run_tests` should be invoked exactly one time, passing +the primary host to use for connecting and any arguments to be given +to `robot`, including the test suites and any unique variables that +are not saved to the default variables file by the framework. + +## Cleaning up + +Use `./bin/composer_cleanup.sh` to stop any running jobs, remove +everything from the queue, and delete existing builds. + +Use `./bin/scenario.sh cleanup` to remove the test infrastructure for a +scenario. + +``` +$ ./bin/scenario.sh cleanup ./scenarios/rhel-9.2-microshift-source-standard-suite/ +``` + +## CI Integration Scripts + +### ci_phase_iso_build.sh + +Runs on the hypervisor. Responsible for all of the setup to build all +needed images. Rebuilds MicroShift RPMs from source, sets up RPM repo, +sets up osbuild workers, builds the images, and creates the web server +to host the images. + +### ci_phase_iso_boot.sh + +Runs on the hypervisor. Responsible for launching all of the VMs that +are used in the test step. diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh new file mode 100755 index 0000000000..85f2dad835 --- /dev/null +++ b/test/bin/build_images.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# +# This script should be run on the image build server (usually the +# same as the hypervisor). + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +mkdir -p "${IMAGEDIR}" + +# Determine the version of the RPM in the local repo so we can use it +# in the blueprint templates. +if [ ! -d "${LOCAL_REPO}" ]; then + error "Run ${SCRIPTDIR}/create_local_repo.sh before building images." + exit 1 +fi +release_info_rpm=$(find "${LOCAL_REPO}" -name 'microshift-release-info-*.rpm') +if [ -z "${release_info_rpm}" ]; then + error "Failed to find microshift-release-info RPM in ${LOCAL_REPO}" + exit 1 +fi +SOURCE_VERSION=$(rpm -q --queryformat '%{version}' "${release_info_rpm}") + +## TEMPLATE VARIABLES +# +# Machine platform type ("x86_64") +UNAME_M=$(uname -m) +export UNAME_M +export LOCAL_REPO # defined in common.sh +export SOURCE_VERSION # defined earlier + +# Add our sources. It is OK to run these steps repeatedly, if the +# details change they are updated in the service. +mkdir -p "${IMAGEDIR}/package-sources" +# shellcheck disable=SC2231 # allow glob expansion without quotes in for loop +for template in ${TESTDIR}/package-sources/*.toml; do + name=$(basename "${template}" .toml) + outfile="${IMAGEDIR}/package-sources/${name}.toml" + echo "Rendering ${template} to ${outfile}" + envsubst <"${template}" >"${outfile}" + echo "Adding package source from ${outfile}" + sudo composer-cli sources add "${outfile}" +done + +# Show details about the available sources to make debugging easier. +for name in $(sudo composer-cli sources list); do + echo + echo "Package source: ${name}" + sudo composer-cli sources info "${name}" | sed -e 's/gpgkeys.*/gpgkeys = .../g' +done + +# Given a blueprint filename, extract the name value. It does not have +# to match the filename, but some commands take the file and others +# take the name, so we need to be able to have both. +get_blueprint_name() { + local filename="${1}" + tomcli-get "${filename}" name +} + +# Track some dynamically created values +BUILDIDS="" + +# Upload the blueprint definitions +mkdir -p "${IMAGEDIR}/blueprints" +mkdir -p "${IMAGEDIR}/builds" +# shellcheck disable=SC2231 # allow glob expansion without quotes in for loop +for template in ${TESTDIR}/image-blueprints/*.toml; do + echo + echo "Blueprint ${template}" + + blueprint_file="${IMAGEDIR}/blueprints/$(basename "${template}")" + echo "Rendering ${template} to ${blueprint_file}" + envsubst <"${template}" >"${blueprint_file}" + + blueprint=$(get_blueprint_name "${blueprint_file}") + + if sudo composer-cli blueprints list | grep -q "^${blueprint}$"; then + echo "Removing existing definition of ${blueprint}" + sudo composer-cli blueprints delete "${blueprint}" + fi + + echo "Loading new definition of ${blueprint}" + sudo composer-cli blueprints push "${blueprint_file}" + + echo "Resolving dependencies for ${blueprint}" + sudo composer-cli blueprints depsolve "${blueprint}" + + echo "Building edge-commit from ${blueprint}" + buildid=$(sudo composer-cli compose start-ostree \ + --ref "${blueprint}" \ + "${blueprint}" \ + edge-commit \ + | awk '{print $2}') + echo "Build ID ${buildid}" + echo "${buildid}" > "${IMAGEDIR}/builds/${blueprint}.edge-commit" + BUILDIDS="${BUILDIDS} ${buildid}" +done + +# In the future we may need to build multiple images with different +# formats but for now we just have one special case to build an +# installer image in a different format. +echo "Building image-installer from ${INSTALLER_IMAGE_BLUEPRINT}" +buildid=$(sudo composer-cli compose start \ + "${INSTALLER_IMAGE_BLUEPRINT}" \ + image-installer \ + | awk '{print $2}') +echo "Build ID ${buildid}" +echo "${buildid}" > "${IMAGEDIR}/builds/${blueprint}.image-installer" +BUILDIDS="${BUILDIDS} ${buildid}" + +echo "Waiting for builds to complete..." +# shellcheck disable=SC2086 # pass command arguments quotes to allow word splitting +time "${SCRIPTDIR}/wait_images.py" ${BUILDIDS} diff --git a/test/bin/ci_phase_iso_boot.sh b/test/bin/ci_phase_iso_boot.sh new file mode 100755 index 0000000000..032cc7b2f6 --- /dev/null +++ b/test/bin/ci_phase_iso_boot.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# This script runs on the hypervisor, from the iso-build step. + +set -xeuo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +# Log output automatically +LOGDIR="${ROOTDIR}/_output/ci-logs" +LOGFILE="${LOGDIR}/$(basename "$0" .sh).log" +if [ ! -d "${LOGDIR}" ]; then + mkdir -p "${LOGDIR}" +fi +echo "Logging to ${LOGFILE}" +# Set fd 1 and 2 to write to the log file +exec &> >(tee >(awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; fflush() }' >"${LOGFILE}")) + +API_EXTERNAL_BASE_PORT="${1}" +SSH_EXTERNAL_BASE_PORT="${2}" +LB_EXTERNAL_BASE_PORT="${3}" + +cd ~/microshift/test + +# Start the web server to host the kickstart files and ostree commit +# repository. +bash -x ./bin/start_webserver.sh + +# Build all of the needed VMs +for scenario in scenarios/*.sh; do + time bash -x ./bin/scenario.sh create "${scenario}" +done + +# Set up port forwarding +bash -x ./bin/manage_vm_connections.sh remote -a "${API_EXTERNAL_BASE_PORT}" -s "${SSH_EXTERNAL_BASE_PORT}" -l "${LB_EXTERNAL_BASE_PORT}" + +# Kill the web server +pkill caddy || true + +echo "Boot phase complete" diff --git a/test/bin/ci_phase_iso_build.sh b/test/bin/ci_phase_iso_build.sh new file mode 100755 index 0000000000..f10b6cbf42 --- /dev/null +++ b/test/bin/ci_phase_iso_build.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# This script runs on the hypervisor, from the iso-build step. + +set -xeuo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# Cannot use common.sh because virsh is not installed, but we only +# need ROOTDIR to set up logging in this script. +ROOTDIR="$(cd "${SCRIPTDIR}/../.." && pwd)" + +# Log output automatically +LOGDIR="${ROOTDIR}/_output/ci-logs" +LOGFILE="${LOGDIR}/$(basename "$0" .sh).log" +if [ ! -d "${LOGDIR}" ]; then + mkdir -p "${LOGDIR}" +fi +echo "Logging to ${LOGFILE}" +# Set fd 1 and 2 to write to the log file +exec &> >(tee >(awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; fflush() }' >"${LOGFILE}")) + +PULL_SECRET=${PULL_SECRET:-${HOME}/.pull-secret.json} + +cd ~/microshift + +# Get firewalld and repos in place. Use scripts to get the right repos +# for each branch. +bash -x ./scripts/devenv-builder/configure-vm.sh --no-build --force-firewall "${PULL_SECRET}" +bash -x ./scripts/image-builder/configure.sh + +# Make sure libvirtd is running. We do this here, because some of the +# other scripts use virsh. +bash -x ./scripts/devenv-builder/manage-vm.sh config + +# Fix up firewall so the VMs on their NAT network can talk to the +# server running on the hypervisor. We do this here, before creating +# any VMs, because the iptables rules added when the VMs are created +# are not persistent and are lost when this script reloads the +# firewall. +cd ~/microshift/test/ +bash -x ./bin/configure_hypervisor_firewall.sh + +# Re-build from source. +cd ~/microshift/ +rm -rf ./_output/rpmbuild +make rpm + +# Set up for scenario tests +cd ~/microshift/test/ +timeout 20m bash -x ./bin/create_local_repo.sh +timeout 20m bash -x ./bin/start_osbuild_workers.sh 5 +timeout 20m bash -x ./bin/build_images.sh +timeout 20m bash -x ./bin/download_images.sh + +echo "Build phase complete" diff --git a/test/bin/cleanup.sh b/test/bin/cleanup.sh new file mode 100755 index 0000000000..5d2a1bbc29 --- /dev/null +++ b/test/bin/cleanup.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -xeuo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +"${SCRIPTDIR}/composer_cleanup.sh" + +for scenario in scenarios/*.sh; do + ./bin/scenario.sh cleanup "${scenario}" +done + +pkill caddy || true + +rm -rf "${IMAGEDIR}" diff --git a/test/bin/common.sh b/test/bin/common.sh new file mode 100755 index 0000000000..1d2aba126e --- /dev/null +++ b/test/bin/common.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +# The location of the test directory, relative to the script. +TESTDIR="$(cd "${SCRIPTDIR}/.." && pwd)" + +# The location of the root of the git repo, relative to the script. +ROOTDIR="$(cd "${TESTDIR}/.." && pwd)" + +# The location of shared kickstart templates +# shellcheck disable=SC2034 # used elsewhere +KICKSTART_TEMPLATE_DIR="${TESTDIR}/kickstart-templates" + +# The blueprint we should use for building an installer image. +# shellcheck disable=SC2034 # used elsewhere +INSTALLER_IMAGE_BLUEPRINT="rhel-9.2" + +# The location for downloading all of the image-related output. +# The location the web server should serve. +export IMAGEDIR="${ROOTDIR}/_output/test-images" + +# The location for storage for the VMs. +# shellcheck disable=SC2034 # used elsewhere +VM_DISK_DIR="${IMAGEDIR}/vm-storage" + +# Location of RPMs built from source +# shellcheck disable=SC2034 # used elsewhere +RPM_SOURCE="${ROOTDIR}/_output/rpmbuild" + +# Location of local repository used by composer +# shellcheck disable=SC2034 # used elsewhere +LOCAL_REPO="${IMAGEDIR}/microshift-local" + +# Location of data files created by the tools for managing scenarios +# as they are run. +# +# The CI system will override this, but we need a default for local +# use. Use the image directoy, since that is already served by a web +# server. +# +# shellcheck disable=SC2034 # used elsewhere +SCENARIO_INFO_DIR="${SCENARIO_INFO_DIR:-${IMAGEDIR}/scenario-info}" + +# The location of the robot framework virtualenv. +# The CI system will override this. +# shellcheck disable=SC2034 # used elsewhere +RF_VENV=${RF_VENV:-${ROOTDIR}/_output/robotenv} + +# Which port the web server should run on. +WEB_SERVER_PORT=${WEB_SERVER_PORT:-8080} + +error() { + local message="$*" + echo "ERROR: ${message} [$(caller)]" 1>&2 +} + +get_vm_bridge_interface() { + # $ sudo virsh net-info default + # Name: default + # UUID: eaac9592-2324-4ae6-b2ec-a5ae94272456 + # Active: yes + # Persistent: yes + # Autostart: yes + # Bridge: virbr0 + + sudo virsh net-info default | grep '^Bridge:' | awk '{print $2}' +} + +get_vm_bridge_ip() { + local bridge + + # When get_vm_bridge_interface is run on the CI cluster there is + # no bridge, and possibly no virsh command. Do not fail, but + # return an empty IP. + bridge="$(get_vm_bridge_interface || true)" + + if [ -z "${bridge}" ]; then + echo "" + return + fi + + # $ ip -f inet addr show virbr0 + # 10: virbr0: mtu 1500 qdisc noqueue state DOWN group default qlen 1000 + # inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0 + # valid_lft forever preferred_lft forever + + ip -f inet addr show "${bridge}" | grep inet | awk '{print $2}' | cut -d/ -f1 +} + +# The IP address of the current host on the bridge used for the +# default network for libvirt VMs. +# shellcheck disable=SC2034 # used elsewhere +VM_BRIDGE_IP="$(get_vm_bridge_ip)" diff --git a/test/bin/composer_cleanup.sh b/test/bin/composer_cleanup.sh new file mode 100755 index 0000000000..2843797d97 --- /dev/null +++ b/test/bin/composer_cleanup.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# This script cancels any composer jobs and deletes failed and completed jobs. + +cancel() { + local status="$1" + + for id in $(sudo composer-cli compose list | grep "${status}" | cut -f1 -d' '); do + echo "Cancelling ${id}" + sudo composer-cli compose cancel "${id}" + done +} + +do_delete() { + for id in $(sudo composer-cli compose list | grep -v "^ID" | cut -f1 -d' '); do + echo "Deleting ${id}" + sudo composer-cli compose delete "${id}" + done +} + +cancel WAITING +cancel RUNNING +do_delete diff --git a/test/bin/configure_hypervisor_firewall.sh b/test/bin/configure_hypervisor_firewall.sh new file mode 100755 index 0000000000..23ff92e93a --- /dev/null +++ b/test/bin/configure_hypervisor_firewall.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# +# This script should be run on the hypervisor to configure the +# firewall for incoming connections to the VM network. + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +VM_BRIDGE=$(sudo virsh net-info default | grep '^Bridge:' | awk '{print $2}') +VM_BRIDGE_CIDR=$(ip -f inet addr show "${VM_BRIDGE}" | grep inet | awk '{print $2}') +sudo firewall-cmd --permanent --zone=trusted --add-source="${VM_BRIDGE_CIDR}" +sudo firewall-cmd --permanent --zone=public --add-port="${WEB_SERVER_PORT}/tcp" +sudo firewall-cmd --reload diff --git a/test/bin/create_local_repo.sh b/test/bin/create_local_repo.sh new file mode 100755 index 0000000000..2d36a30e95 --- /dev/null +++ b/test/bin/create_local_repo.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# This script should be run on the image build server (usually the +# hypervisor) to create a local RPM repository containing the RPMs +# built from the source under test. + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +mkdir -p "${IMAGEDIR}" +cd "${IMAGEDIR}" + +if [ -d "${LOCAL_REPO}" ]; then + echo "Cleaning up existing repository" + rm -rf "${LOCAL_REPO}" +fi +mkdir -p "${LOCAL_REPO}" + +# Create the local RPM repository for whatever was built from source. +echo "Copying RPMs from ${RPM_SOURCE} to ${LOCAL_REPO}" +# shellcheck disable=SC2086 # no quotes for command arguments to allow word splitting +cp -R ${RPM_SOURCE}/{RPMS,SPECS,SRPMS} "${LOCAL_REPO}/" + +echo "Creating RPM repo at ${LOCAL_REPO}" +createrepo "${LOCAL_REPO}" + +echo "Fixing permissions of RPM repo contents" +find "${LOCAL_REPO}" -type f -exec chmod a+r {} \; +find "${LOCAL_REPO}" -type d -exec chmod a+rx {} \; diff --git a/test/bin/download_images.sh b/test/bin/download_images.sh new file mode 100755 index 0000000000..8ff8df7d55 --- /dev/null +++ b/test/bin/download_images.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# This script should be run on the hypervisor to download all of the +# images from composer. + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +mkdir -p "${VM_DISK_DIR}" + +download_image() { + local buildid="${1}" + # FIXME: These logs should go to the artifacts directory instead + # of the build web server. + # shellcheck disable=SC2086 # pass glob args without quotes + rm -f ${buildid}-*.tar + sudo composer-cli compose logs "${buildid}" + sudo composer-cli compose metadata "${buildid}" + sudo composer-cli compose image "${buildid}" + # shellcheck disable=SC2086 # pass glob args without quotes + sudo chown "$(whoami)." ${buildid}-* +} + +download_dir="${IMAGEDIR}/builds" +mkdir -p "${download_dir}" + +echo "Downloading installer images to ${download_dir}" +cd "${download_dir}" +for blueprint_build in *.image-installer; do + blueprint=$(basename "${blueprint_build}" .image-installer) + buildid=$(cat "${blueprint_build}") + download_image "${buildid}" + iso_file="${buildid}-installer.iso" + echo "Moving ${iso_file} to ${VM_DISK_DIR}/${blueprint}.iso" + mv -f "${iso_file}" "${VM_DISK_DIR}/${blueprint}.iso" +done + +if [ -d "${IMAGEDIR}/repo" ]; then + echo "Cleaning up existing images in ${IMAGEDIR}/repo" + rm -rf "${IMAGEDIR}/repo" +fi + +echo "Downloading ostree commits and metadata to ${download_dir}" +cd "${download_dir}" +for blueprint_build in *.edge-commit; do + blueprint=$(basename "${blueprint_build}" .edge-commit) + buildid=$(cat "${blueprint_build}") + download_image "${buildid}" + commit_file="${buildid}-commit.tar" + echo "Unpacking ${commit_file} ${blueprint}" + tar -C "${IMAGEDIR}" -xf "${commit_file}" +done + +echo "Updating references" +cd "${IMAGEDIR}" +ostree summary --update --repo=repo +ostree summary --view --repo=repo diff --git a/test/bin/force_vm_settings.sh b/test/bin/force_vm_settings.sh new file mode 100755 index 0000000000..81a15cc126 --- /dev/null +++ b/test/bin/force_vm_settings.sh @@ -0,0 +1,24 @@ +#!/bin/bash -x +# +# This script is used by scenario.sh when creating a new VM to force +# specific settings on the host, such as opening firewall ports. + +set -euo pipefail + +# Exit if the current user is not 'root' +if [ "$(id -u)" -ne 0 ] ; then + echo "The '${SCRIPT_NAME}' script must be run with the 'root' user privileges" + exit 1 +fi + +systemctl enable firewalld --now +firewall-cmd --permanent --zone=trusted --add-source=10.42.0.0/16 +firewall-cmd --permanent --zone=trusted --add-source=169.254.169.1 +firewall-cmd --permanent --zone=public --add-port=80/tcp +firewall-cmd --permanent --zone=public --add-port=443/tcp +firewall-cmd --permanent --zone=public --add-port=5353/udp +firewall-cmd --permanent --zone=public --add-port=30000-32767/tcp +firewall-cmd --permanent --zone=public --add-port=30000-32767/udp +firewall-cmd --permanent --zone=public --add-port=6443/tcp +firewall-cmd --permanent --zone=public --add-service=mdns +firewall-cmd --reload diff --git a/test/bin/manage_vm_connections.sh b/test/bin/manage_vm_connections.sh new file mode 100755 index 0000000000..f82a6cea00 --- /dev/null +++ b/test/bin/manage_vm_connections.sh @@ -0,0 +1,226 @@ +#!/bin/bash +# +# This script should be run on the hypervisor to set up port forwarding. + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +usage() { + cat - < The base port number for API server port forwarding. + + -l The base port number for load balancer port forwarding for + services running on the cluster. + + -s The base port number for ssh port forwarding. + +cleanup: Remove iptables rules created by 'remote' command. + +local: Set up connection settings for VMs for local access from the + hypervisor. + +EOF +} + +add_iptables_rules() { + local vm_ip="${1}" + local api_port="${2}" + local ssh_port="${3}" + local lb_port="${4}" + + # Setup external access with port forwarding to allow running commands and tests from the CI container. + + # MicroShift API server + sudo /sbin/iptables -I FORWARD -o virbr0 -p tcp -d "${vm_ip}" --dport 6443 -j ACCEPT + sudo /sbin/iptables -t nat -I PREROUTING -p tcp --dport "${api_port}" -j DNAT --to "${vm_ip}:6443" + + # SSH to the VM + sudo /sbin/iptables -I FORWARD -o virbr0 -p tcp -d "${vm_ip}" --dport 22 -j ACCEPT + sudo /sbin/iptables -t nat -I PREROUTING -p tcp --dport "${ssh_port}" -j DNAT --to "${vm_ip}:22" + + # Port usable for services running on MicroShift + sudo /sbin/iptables -I FORWARD -o virbr0 -p tcp -d "${vm_ip}" --dport 5678 -j ACCEPT + sudo /sbin/iptables -t nat -I PREROUTING -p tcp --dport "${lb_port}" -j DNAT --to "${vm_ip}:5678" + + # Show the rules we just added + sudo /sbin/iptables -L FORWARD | grep "${vm_ip}" + sudo /sbin/iptables -t nat -L PREROUTING | grep "${vm_ip}" +} + +delete_iptables_rules() { + local vm_ip="${1}" + + # If there are no rules grep exits with an error, so do the search + # with an expression that always returns true. Sort the results in + # reverse order so we remove rules from the end of the list and do + # not change the index values of the other rules that need to be + # removed. + + echo "Looking for rules using ${vm_ip}" + # shellcheck disable=SC2162 # read without -r + (sudo /sbin/iptables -L FORWARD --line-numbers | grep "${vm_ip}" | sort -n -r || true) | while read num rule; do + echo "Removing rule ${rule}" + sudo /sbin/iptables -D FORWARD "${num}" + done + + # shellcheck disable=SC2162 # read without -r + (sudo /sbin/iptables -t nat -L PREROUTING --line-numbers | grep "${vm_ip}" | sort -n -r || true) | while read num rule; do + echo "Removing rule ${rule}" + sudo /sbin/iptables -t nat -D PREROUTING "${num}" + done +} + +action_remote() { + + local api_base="" + local ssh_base="" + local lb_base="" + while getopts "a:hl:s:" opt; do + case "${opt}" in + a) + api_base="${OPTARG}" + ;; + l) + lb_base="${OPTARG}" + ;; + s) + ssh_base="${OPTARG}" + ;; + h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac + done + + if [ -z "${api_base}" ]; then + usage + error "Specify API base port with -a option" + exit 1 + fi + local api_port="${api_base}" + + if [ -z "${lb_base}" ]; then + usage + error "Specify LB base port with -l option" + exit 1 + fi + local lb_port="${lb_base}" + + if [ -z "${ssh_base}" ]; then + usage + error "Specify ssh base port with -s option" + exit 1 + fi + local ssh_port="${ssh_base}" + + cd "${SCENARIO_INFO_DIR}" + + local scenario + local vm_name + local vm_ip + for scenario in *; do + pushd "${scenario}" >/dev/null + for vm in vms/*; do + vm_name=$(basename "${vm}") + vm_ip=$(cat "${vm}/ip") + + echo "${scenario}: Configuring ${vm_name} at ${vm_ip} with API port ${api_port}, ssh port ${ssh_port}, and LB port ${lb_port}" + add_iptables_rules "${vm_ip}" "${api_port}" "${ssh_port}" "${lb_port}" + + # Record the ports used for the VM + echo "${api_port}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/api_port" + echo "${ssh_port}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/ssh_port" + echo "${lb_port}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/lb_port" + + # Increment the ports so they are unique + api_port=$((api_port+1)) + ssh_port=$((ssh_port+1)) + lb_port=$((lb_port+1)) + done + popd >/dev/null + done +} + +action_cleanup() { + cd "${SCENARIO_INFO_DIR}" + + local scenario + local vm_name + local vm_ip + for scenario in *; do + pushd "${scenario}" >/dev/null + for vm in vms/*; do + vm_name=$(basename "${vm}") + vm_ip=$(cat "${vm}/ip") + + echo "${scenario}: Cleaning up iptables rules for ${vm_name}" + delete_iptables_rules "${vm_ip}" + done + popd >/dev/null + done +} + +action_local() { + cd "${SCENARIO_INFO_DIR}" + + local api_port=6443 + local ssh_port=22 + local lb_port=5678 + + local scenario + local vm_name + local vm_ip + for scenario in *; do + pushd "${scenario}" >/dev/null + for vm in vms/*; do + vm_name=$(basename "${vm}") + vm_ip=$(cat "${vm}/ip") + + echo "${scenario}: Configuring ${vm_name} at ${vm_ip} with API port ${api_port} and ssh port ${ssh_port}" + + # Record the ports used for the VM + echo "${api_port}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/api_port" + echo "${ssh_port}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/ssh_port" + echo "${lb_port}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/lb_port" + echo "${vm_ip}" > "${SCENARIO_INFO_DIR}/${scenario}/vms/${vm_name}/public_ip" + done + popd >/dev/null + done +} + +if [ $# -eq 0 ]; then + usage + exit 1 +fi +action="${1}" +shift + +case "${action}" in + remote|cleanup|local) + "action_${action}" "$@" + ;; + -h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; +esac diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh new file mode 100755 index 0000000000..e2c823f139 --- /dev/null +++ b/test/bin/scenario.sh @@ -0,0 +1,352 @@ +#!/bin/bash +# +# This script should be run on the hypervisor to manage the VMs needed +# for a scenario. + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +DEFAULT_BOOT_BLUEPRINT="rhel-9.2" +LVM_SYSROOT_SIZE="10240" +WEB_SERVER_URL="http://${VM_BRIDGE_IP}:${WEB_SERVER_PORT}" +PULL_SECRET="${PULL_SECRET:-${HOME}/.pull-secret.json}" +PULL_SECRET_CONTENT="$(jq -c . "${PULL_SECRET}")" +PUBLIC_IP=${PUBLIC_IP:-""} # may be overridden in global settings file +VM_BOOT_TIMEOUT=8m + +full_vm_name() { + local base="${1}" + echo "${SCENARIO}-${base}" +} + +# Public function to render a unique kickstart from a template for a +# VM in a scenario. +# +# Arguments: +# vmname -- The short name of the VM (e.g., "host1") +# template -- The path to the kickstart template file, relative to +# the scenario directory. +# boot_commit_ref -- The reference to the image that should be booted +# first on the host. This usually matches an image +# blueprint name. +prepare_kickstart() { + local vmname="$1" + local template="$2" + local boot_commit_ref="$3" + + local full_vmname + local output_file + local vm_hostname + + full_vmname="$(full_vm_name "${vmname}")" + output_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/kickstart.ks" + vm_hostname="${full_vmname/./-}" + + echo "Preparing kickstart file ${template} ${output_file}" + if [ ! -f "${KICKSTART_TEMPLATE_DIR}/${template}" ]; then + # FIXME: Perhaps we want a default kickstart to reduce duplication? + error "No ${template} in ${KICKSTART_TEMPLATE_DIR}" + exit 1 + fi + mkdir -p "$(dirname "${output_file}")" + # shellcheck disable=SC2002 # useless cat + cat "${KICKSTART_TEMPLATE_DIR}/${template}" \ + | sed -e "s/REPLACE_LVM_SYSROOT_SIZE/${LVM_SYSROOT_SIZE}/g" \ + -e "s|REPLACE_OSTREE_SERVER_URL|${WEB_SERVER_URL}/repo|g" \ + -e "s|REPLACE_BOOT_COMMIT_REF|${boot_commit_ref}|g" \ + -e "s|REPLACE_PULL_SECRET|${PULL_SECRET_CONTENT}|g" \ + -e "s|REPLACE_HOST_NAME|${vm_hostname}|g" \ + -e "s|REPLACE_REDHAT_AUTHORIZED_KEYS|${REDHAT_AUTHORIZED_KEYS}|g" \ + -e "s|REPLACE_PUBLIC_IP|${PUBLIC_IP}|g" \ + > "${output_file}" +} + +# Show the IP address of the VM +function get_vm_ip { + local vmname="${1}" + sudo virsh domifaddr "${vmname}" \ + | grep vnet \ + | awk '{print $4}' \ + | cut -f1 -d/ +} + +# Try to login to the host via ssh until the connection is accepted +wait_for_ssh() { + local ip="${1}" + + echo "Waiting ${VM_BOOT_TIMEOUT} for ssh access to ${ip}" + timeout "${VM_BOOT_TIMEOUT}" bash -c "until ssh -oBatchMode=yes -oStrictHostKeyChecking=accept-new redhat@${ip} 'echo host is up'; do date; sleep 5; done" +} + +# Add the microshift config file needed to allow remote access via IP address +enable_ip_access() { + local vmname="${1}" + local ip="${2}" + + echo "Waiting ${VM_BOOT_TIMEOUT} for ${full_vmname} to boot" + wait_for_ssh "${ip}" + + echo "Adjusting VM host settings" + scp "${SCRIPTDIR}/force_vm_settings.sh" "redhat@${ip}:/home/redhat/" + ssh "redhat@${ip}" "chmod +x /home/redhat/force_vm_settings.sh && sudo /home/redhat/force_vm_settings.sh" +} + +# Wait for greenboot health check to complete, without checking the results +wait_for_greenboot() { + local vmname="${1}" + local ip="${2}" + + echo "Waiting ${VM_BOOT_TIMEOUT} for greenboot on ${vmname} to complete" + timeout "${VM_BOOT_TIMEOUT}" bash -c "until ssh redhat@${ip} \"sudo journalctl -n 5 -u greenboot-healthcheck; sudo systemctl status greenboot-healthcheck | grep 'active (exited)'\"; do date; sleep 10; done" +} + +# Public function to start a VM. +# +# Creates a new VM using the scenario name and the vmname given to +# create a unique name. Uses the boot_blueprint argument to select the +# ISO from which to boot. If no boot_blueprint is specified, uses +# DEFAULT_BOOT_BLUEPRINT. +# +# Arguments +# vmname -- The short name of the VM in the scenario (e.g., "host1"). +# boot_blueprint -- The image blueprint used to create the ISO that +# should be used to boot the VM. This is _not_ +# necessarily the image to be installed (see +# prepare_kickstart). +launch_vm() { + local vmname="$1" + local boot_blueprint="${2:-${DEFAULT_BOOT_BLUEPRINT}}" + + local full_vmname + local kickstart_url + + full_vmname="$(full_vm_name "${vmname}")" + kickstart_url="${WEB_SERVER_URL}/scenario-info/${SCENARIO}/vms/${vmname}/kickstart.ks" + + # Checking web server for kickstart file + if ! curl -o /dev/null "${kickstart_url}" >/dev/null 2>&1; then + error "Failed to load kickstart file from ${kickstart_url}" + exit 1 + fi + + echo "Creating ${full_vmname}" + + # FIXME: variable for vcpus? + # FIXME: variable for memory? + # FIXME: variable for ISO + timeout "${VM_BOOT_TIMEOUT}" sudo virt-install \ + --noautoconsole \ + --name "${full_vmname}" \ + --vcpus 2 \ + --memory 4092 \ + --disk "path=${VM_DISK_DIR}/${full_vmname}.qcow2,size=30" \ + --network network=default,model=virtio \ + --events on_reboot=restart \ + --location "${VM_DISK_DIR}/${boot_blueprint}.iso" \ + --extra-args "inst.ks=${kickstart_url}" \ + --wait + + # Wait for an IP to be assigned + ip=$(get_vm_ip "${full_vmname}") + while [ -z "${ip}" ]; do + echo "Waiting for VM ${full_vmname} to have an IP" + sleep 30 + ip=$(get_vm_ip "${full_vmname}") + done + echo "VM ${full_vmname} has IP ${ip}" + + # Remove any previous key info for the host + if [ -f "${HOME}/.ssh/known_hosts" ]; then + echo "Clearing known_hosts entry for ${ip}" + ssh-keygen -R "${ip}" + fi + + # Record the IP of this VM so our caller can use it to configure + # port forwarding and the firewall. + mkdir -p "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}" + echo "${ip}" > "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/ip" + # Record the _public_ IP of the VM so the test suite can use it to + # access the host. This is useful when the public IP is the + # hypervisor forwarding connections. If we have no PUBLIC_IP, use + # the VM IP and assume a local connection. + if [ -n "${PUBLIC_IP}" ]; then + echo "${PUBLIC_IP}" > "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/public_ip" + else + echo "${ip}" > "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/public_ip" + fi + + enable_ip_access "${full_vmname}" "${ip}" + wait_for_greenboot "${full_vmname}" "${ip}" + + echo "${full_vmname} is up and ready" +} + +# Clean up the resources for one VM. +remove_vm() { + local vmname="${1}" + + local full_vmname + full_vmname="$(full_vm_name "${vmname}")" + + # Remove the actual VM and its storage + if sudo virsh dumpxml "${full_vmname}" >/dev/null; then + if ! sudo virsh dominfo "${full_vmname}" | grep '^State' | grep -q 'shut off'; then + sudo virsh destroy "${full_vmname}" + fi + sudo virsh undefine "${full_vmname}" + fi + if sudo virsh vol-dumpxml "${full_vmname}.qcow2" vm-storage >/dev/null; then + sudo virsh vol-delete "${full_vmname}.qcow2" vm-storage + fi + + # Remove the info file so something processing the VMs does not + # assume the file exists. This is most useful in a local setting. + rm -rf "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}" +} + +# Run the tests for the current scenario +run_tests() { + local vmname="${1}" + shift + + echo "Running tests with $# args" "$@" + + if [ ! -d "${RF_VENV}" ]; then + error "RF_VENV (${RF_VENV}) does not exist, create it with: ${ROOTDIR}/scripts/fetch_tools.sh robotframework" + exit 1 + fi + local rf_binary="${RF_VENV}/bin/robot" + if [ ! -f "${rf_binary}" ]; then + error "robot is not installed to ${rf_binary}" + exit 1 + fi + + + local ssh_port_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/ssh_port" + local api_port_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/api_port" + local lb_port_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/lb_port" + local public_ip_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/public_ip" + local ip_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}/ip" + local api_port + local ssh_port + local public_ip + local vm_ip + local f + for f in "${ssh_port_file}" "${api_port_file}" "${lb_port_file}" "${public_ip_file}" "${ip_file}"; do + if [ ! -f "${f}" ]; then + error "Cannot read ${f}" + exit 1 + fi + done + ssh_port=$(cat "${ssh_port_file}") + api_port=$(cat "${api_port_file}") + lb_port=$(cat "${lb_port_file}") + public_ip=$(cat "${public_ip_file}") + vm_ip=$(cat "${ip_file}") + + local variable_file="${SCENARIO_INFO_DIR}/${SCENARIO}/variables.yaml" + echo "Writing variables to ${variable_file}" + mkdir -p "$(dirname "${variable_file}")" + cat - <caddy.log 2>&1 & diff --git a/test/bin/wait_images.py b/test/bin/wait_images.py new file mode 100755 index 0000000000..bc5d25e6f7 --- /dev/null +++ b/test/bin/wait_images.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# +# This script is run by build_images.sh after all of the builds have +# been enqueued. It waits for the jobs identified by the UUIDs +# provided as input to either fail or complete. + +import argparse +import json +import logging +import subprocess +import time + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s: %(message)s', +) + +STATUS_COMMAND = ['sudo', 'composer-cli', 'compose', 'status', '--json'] + + +# Convert the weird queue json data structure to a sequence of jobs. +# $ sudo composer-cli compose status --json +# [ +# { +# "method": "GET", +# "path": "/compose/queue", +# "status": 200, +# "body": { +# "new": [ +# { +# "blueprint": "rhel-9.2", +# "compose_type": "image-installer", +# "id": "7bdb78bf-c4b6-4f19-80df-073d38fade56", +# "image_size": 0, +# "job_created": 1687466891.2393456, +# "queue_status": "WAITING", +# "version": "0.0.1" +# } +# ], +# "run": [ +# { +# "blueprint": "rhel-9.2", +# "compose_type": "edge-commit", +# "id": "4aa19f32-54e3-42ce-a4ba-cf038a3df91c", +# "image_size": 0, +# "job_created": 1687466889.5383687, +# "job_started": 1687466889.5480232, +# "queue_status": "RUNNING", +# "version": "0.0.1" +# } +# ] +# } +# }, +# { +# "method": "GET", +# "path": "/compose/finished", +# "status": 200, +# "body": { +# "finished": [] +# } +# }, +# { +# "method": "GET", +# "path": "/compose/failed", +# "status": 200, +# "body": { +# "failed": [] +# } +# } +# ] +def flattened_status(): + result = subprocess.run(STATUS_COMMAND, stdout=subprocess.PIPE) + if result.returncode != 0: + raise SystemError(f'Status command returned {result.returncode}') + status = json.loads(result.stdout) + for result_set in status: + for state in result_set["body"]: + for job in result_set["body"][state]: + yield job + + +def main(build_ids): + ignore_ids = set() + known_ids = set(build_ids) + found_ids = set() + while build_ids: + logging.info(f'Waiting for {build_ids}') + for job in flattened_status(): + job_id = job["id"] + found_ids.add(job_id) + status_text = f'{job_id} {job["compose_type"]} for {job["blueprint"]} - {job["queue_status"]}' + if job_id in build_ids: + logging.info(status_text) + if job["queue_status"] in {"FAILED", "FINISHED"}: + # After a job fails or finishes, stop reporting its status. + build_ids.remove(job_id) + elif job_id not in ignore_ids and job_id not in known_ids: + # Report any unknown jobs one time, then ignore them. + logging.info(f'{status_text} (unknown job)') + ignore_ids.add(job_id) + to_ignore = [] + for build_id in build_ids: + if build_id not in found_ids: + logging.info(f'{build_id} is not a known build, ignoring') + to_ignore.append(build_id) + for i in to_ignore: + build_ids.remove(i) + if build_ids: + time.sleep(30) + + +if __name__ == '__main__': + cli_parser = argparse.ArgumentParser(add_help=True) + cli_parser.add_argument( + 'build_id', + nargs='+', + help="a build id (UUID) from composer", + ) + args = cli_parser.parse_args() + main(args.build_id) diff --git a/test/image-blueprints/rhel92-microshift413.toml b/test/image-blueprints/rhel92-microshift413.toml new file mode 100644 index 0000000000..23de492b62 --- /dev/null +++ b/test/image-blueprints/rhel92-microshift413.toml @@ -0,0 +1,25 @@ +name = "rhel-9.2-microshift-4.13" +description = "" +version = "0.0.1" +modules = [] +groups = [] +distro = "rhel-92" + +[[packages]] +name = "microshift" +version = "4.13*" + +[[packages]] +name = "microshift-greenboot" +version = "4.13*" + +[[packages]] +name = "microshift-networking" +version = "4.13*" + +[[packages]] +name = "microshift-selinux" +version = "4.13*" + +[customizations.services] +enabled = ["microshift"] diff --git a/test/image-blueprints/rhel92-source.toml b/test/image-blueprints/rhel92-source.toml new file mode 100644 index 0000000000..1fa0d5ca67 --- /dev/null +++ b/test/image-blueprints/rhel92-source.toml @@ -0,0 +1,25 @@ +name = "rhel-9.2-microshift-source" +description = "" +version = "0.0.1" +modules = [] +groups = [] +distro = "rhel-92" + +[[packages]] +name = "microshift" +version = "${SOURCE_VERSION}" + +[[packages]] +name = "microshift-greenboot" +version = "${SOURCE_VERSION}" + +[[packages]] +name = "microshift-networking" +version = "${SOURCE_VERSION}" + +[[packages]] +name = "microshift-selinux" +version = "${SOURCE_VERSION}" + +[customizations.services] +enabled = ["microshift"] diff --git a/test/image-blueprints/rhel92.toml b/test/image-blueprints/rhel92.toml new file mode 100644 index 0000000000..a59191514c --- /dev/null +++ b/test/image-blueprints/rhel92.toml @@ -0,0 +1,7 @@ +name = "rhel-9.2" +description = "" +version = "0.0.1" +distro = "rhel-92" +modules = [] +groups = [] +packages = [] diff --git a/test/kickstart-templates/kickstart.ks.template b/test/kickstart-templates/kickstart.ks.template new file mode 100644 index 0000000000..5c938bd515 --- /dev/null +++ b/test/kickstart-templates/kickstart.ks.template @@ -0,0 +1,95 @@ +lang en_US.UTF-8 +keyboard us +timezone UTC +text +reboot + +# Configure network to use DHCP and activate on boot +network --bootproto=dhcp --device=link --activate --onboot=on --hostname=REPLACE_HOST_NAME + +# Partition disk with a 1GB boot XFS partition and an LVM volume containing a 10GB+ system root +# The remainder of the volume will be used by the CSI driver for storing data +# +# For example, a 20GB disk would be partitioned in the following way: +# +# NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT +# sda 8:0 0 20G 0 disk +# ├─sda1 8:1 0 200M 0 part /boot/efi +# ├─sda1 8:1 0 800M 0 part /boot +# └─sda2 8:2 0 19G 0 part +# └─rhel-root 253:0 0 10G 0 lvm /sysroot +# +zerombr +clearpart --all --initlabel +part /boot/efi --fstype=efi --size=200 +part /boot --fstype=xfs --asprimary --size=800 +# Uncomment this line to add a SWAP partition of the recommended size +#part swap --fstype=swap --recommended +part pv.01 --grow +volgroup rhel pv.01 +logvol / --vgname=rhel --fstype=xfs --size=REPLACE_LVM_SYSROOT_SIZE --name=root +rootpw --lock + +# Configure ostree +ostreesetup --nogpg --osname=rhel --remote=edge --url=REPLACE_OSTREE_SERVER_URL --ref=REPLACE_BOOT_COMMIT_REF + +%post --log=/var/log/anaconda/post-install.log --erroronfail + +# Set the configuration of MicroShift to include subjectAltNames +cat - >/etc/microshift/config.yaml <>/etc/microshift/config.yaml +fi + +# Replace the ostree server URL +echo -e 'url=REPLACE_OSTREE_SERVER_URL' >> /etc/ostree/remotes.d/edge.conf + +# The pull secret is mandatory for MicroShift builds on top of OpenShift, but not OKD +# The /etc/crio/crio.conf.d/microshift.conf references the /etc/crio/openshift-pull-secret file +cat > /etc/crio/openshift-pull-secret < /etc/sudoers.d/microshift + +# # Add authorized ssh keys +mkdir -m 700 /home/redhat/.ssh +cat >> /home/redhat/.ssh/authorized_keys <> /root/.profile + +# Configure systemd journal service to persist logs between boots and limit their size to 1G +sudo mkdir -p /etc/systemd/journald.conf.d +cat > /etc/systemd/journald.conf.d/microshift.conf <