From 82e434c8b9a62940679949514412ecd2ae6badd1 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 10 Jul 2023 15:14:01 -0400 Subject: [PATCH 01/11] pin openvswitch3.1 to 3.1.0-14 See OCPBUGS-15948 --- test/image-blueprints/rhel92-microshift413.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/image-blueprints/rhel92-microshift413.toml b/test/image-blueprints/rhel92-microshift413.toml index 23de492b62..5e6c1b6196 100644 --- a/test/image-blueprints/rhel92-microshift413.toml +++ b/test/image-blueprints/rhel92-microshift413.toml @@ -5,6 +5,11 @@ modules = [] groups = [] distro = "rhel-92" +# FIXME: Temporarily pin the openvswitch version. +[[packages]] +name = "openvswitch3.1" +version = "3.1.0-14.el9fdp" + [[packages]] name = "microshift" version = "4.13*" From b1a56d95e6f6da40fae8e05b3571f96be7a25e42 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 3 Jul 2023 15:56:50 -0400 Subject: [PATCH 02/11] USHIFT-1098: make image definition for previous y-stream dynamic Compute the version number of the previous minor release and build the image dynamically so we do not have to update the image definition for every new release branch. --- test/bin/build_images.sh | 4 ++++ ...t413.toml => rhel92-microshift-previous-minor.toml} | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) rename test/image-blueprints/{rhel92-microshift413.toml => rhel92-microshift-previous-minor.toml} (65%) diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh index b7859b4fdf..2838b42334 100755 --- a/test/bin/build_images.sh +++ b/test/bin/build_images.sh @@ -33,6 +33,8 @@ if [ -z "${release_info_rpm}" ]; then fi SOURCE_VERSION=$(rpm -q --queryformat '%{version}' "${release_info_rpm}") FAKE_NEXT_MINOR_VERSION=$(cd "${SCRIPTDIR}/.." && make fake-next-minor) +MINOR_VERSION=$(echo "${SOURCE_VERSION}" | cut -f2 -d.) +PREVIOUS_MINOR_VERSION=$(( "${MINOR_VERSION}" - 1 )) ## TEMPLATE VARIABLES # @@ -42,6 +44,8 @@ export UNAME_M export LOCAL_REPO # defined in common.sh export SOURCE_VERSION # defined earlier export FAKE_NEXT_MINOR_VERSION # defined earlier +export MINOR_VERSION # defined earlier +export PREVIOUS_MINOR_VERSION # defined earlier # Add our sources. It is OK to run these steps repeatedly, if the # details change they are updated in the service. diff --git a/test/image-blueprints/rhel92-microshift413.toml b/test/image-blueprints/rhel92-microshift-previous-minor.toml similarity index 65% rename from test/image-blueprints/rhel92-microshift413.toml rename to test/image-blueprints/rhel92-microshift-previous-minor.toml index 5e6c1b6196..d54c6348ff 100644 --- a/test/image-blueprints/rhel92-microshift413.toml +++ b/test/image-blueprints/rhel92-microshift-previous-minor.toml @@ -1,4 +1,4 @@ -name = "rhel-9.2-microshift-4.13" +name = "rhel-9.2-microshift-4.${PREVIOUS_MINOR_VERSION}" description = "" version = "0.0.1" modules = [] @@ -12,19 +12,19 @@ version = "3.1.0-14.el9fdp" [[packages]] name = "microshift" -version = "4.13*" +version = "4.${PREVIOUS_MINOR_VERSION}*" [[packages]] name = "microshift-greenboot" -version = "4.13*" +version = "4.${PREVIOUS_MINOR_VERSION}*" [[packages]] name = "microshift-networking" -version = "4.13*" +version = "4.${PREVIOUS_MINOR_VERSION}*" [[packages]] name = "microshift-selinux" -version = "4.13*" +version = "4.${PREVIOUS_MINOR_VERSION}*" [customizations.services] enabled = ["microshift"] From 2bf57db98f9ca83448ac32b264a0ca1adfbdfb3f Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 3 Jul 2023 16:17:58 -0400 Subject: [PATCH 03/11] better error handling in manage_vm_connection.sh --- test/bin/manage_vm_connections.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/bin/manage_vm_connections.sh b/test/bin/manage_vm_connections.sh index f82a6cea00..34887caa0b 100755 --- a/test/bin/manage_vm_connections.sh +++ b/test/bin/manage_vm_connections.sh @@ -177,6 +177,10 @@ action_cleanup() { } action_local() { + if [ ! -d "${SCENARIO_INFO_DIR}" ]; then + error "No scenario info directory at ${SCENARIO_INFO_DIR}, did you boot the VMs for a scenario?" + exit 1 + fi cd "${SCENARIO_INFO_DIR}" local api_port=6443 From 0c5f6ce20355baff1c8f4f061cc718bf201ef586 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 3 Jul 2023 16:36:01 -0400 Subject: [PATCH 04/11] add functions to scenario.sh for manipulating version strings --- test/bin/scenario.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 3f7166bafe..7a2c6b6c34 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -15,6 +15,7 @@ PULL_SECRET="${PULL_SECRET:-${HOME}/.pull-secret.json}" PULL_SECRET_CONTENT="$(jq -c . "${PULL_SECRET}")" PUBLIC_IP=${PUBLIC_IP:-""} # may be overridden in global settings file VM_BOOT_TIMEOUT=8m +MAKE_VAR_FILE="${TESTDIR}/../Makefile.version.$(uname -m).var" full_vm_name() { local base="${1}" @@ -220,6 +221,25 @@ remove_vm() { rm -rf "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}" } + +# Function to report the full current version, e.g. "4.13.5" +current_version() { + # OCP_VERSION := 4.14.0-0.nightly-2023-06-30-131338 + grep OCP_VERSION "${MAKE_VAR_FILE}" | awk '{print $3}' | cut -f1 -d- +} + +# Function to report only the minor portion of the current version, +# e.g. from "4.13.5" reports "13" +current_minor_version() { + current_version | cut -f2 -d. +} + +# Function to report the *previous* minor version. If the current +# version is "4.13.5", reports "12". +previous_minor_version() { + echo $(( $(current_minor_version) - 1 )) +} + # Run the tests for the current scenario run_tests() { local vmname="${1}" From 0a8596b4599f8d32af9904a7cacfcba41925c605 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 3 Jul 2023 16:36:16 -0400 Subject: [PATCH 05/11] USHIFT-1098: add minor-version upgrade scenario --- .../upgrade-minor-version-healthy.sh | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 test/scenarios/upgrade-minor-version-healthy.sh diff --git a/test/scenarios/upgrade-minor-version-healthy.sh b/test/scenarios/upgrade-minor-version-healthy.sh new file mode 100644 index 0000000000..95172e76c9 --- /dev/null +++ b/test/scenarios/upgrade-minor-version-healthy.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Sourced from cleanup_scenario.sh and uses functions defined there. + +scenario_create_vms() { + # Determine the starting image based on the source tree current + # version, minus one. + local start_image + start_image="rhel-9.2-microshift-4.$(previous_minor_version)" + + prepare_kickstart host1 kickstart.ks.template "${start_image}" + launch_vm host1 +} + +scenario_remove_vms() { + remove_vm host1 +} + +scenario_run_tests() { + run_tests host1 \ + --variable "TARGET_REF:rhel-9.2-microshift-source" \ + suites-ostree/healthy-upgrade.robot +} From 06a187190d756092cac4d50061b5d434be10ac7d Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 13 Jul 2023 13:48:57 -0400 Subject: [PATCH 06/11] USHIFT-1089: use the same source for current version information --- test/bin/get_latest_rpm_version.sh | 16 ++++++++++++++++ test/bin/scenario.sh | 4 +--- 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100755 test/bin/get_latest_rpm_version.sh diff --git a/test/bin/get_latest_rpm_version.sh b/test/bin/get_latest_rpm_version.sh new file mode 100755 index 0000000000..cc0b47d6dc --- /dev/null +++ b/test/bin/get_latest_rpm_version.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# This script reports the version of the latest RPM in the local +# repository. + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${SCRIPTDIR}/common.sh" + +release_info_rpm=$(find "${LOCAL_REPO}" -name 'microshift-release-info-*.rpm' | sort | tail -n 1) +if [ -z "${release_info_rpm}" ]; then + error "Failed to find microshift-release-info RPM in ${LOCAL_REPO}" + exit 1 +fi +rpm -q --queryformat '%{version}' "${release_info_rpm}" 2>/dev/null diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 7a2c6b6c34..3bfb674aaf 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -15,7 +15,6 @@ PULL_SECRET="${PULL_SECRET:-${HOME}/.pull-secret.json}" PULL_SECRET_CONTENT="$(jq -c . "${PULL_SECRET}")" PUBLIC_IP=${PUBLIC_IP:-""} # may be overridden in global settings file VM_BOOT_TIMEOUT=8m -MAKE_VAR_FILE="${TESTDIR}/../Makefile.version.$(uname -m).var" full_vm_name() { local base="${1}" @@ -224,8 +223,7 @@ remove_vm() { # Function to report the full current version, e.g. "4.13.5" current_version() { - # OCP_VERSION := 4.14.0-0.nightly-2023-06-30-131338 - grep OCP_VERSION "${MAKE_VAR_FILE}" | awk '{print $3}' | cut -f1 -d- + "${SCRIPTDIR}/get_latest_rpm_version.sh" } # Function to report only the minor portion of the current version, From f033e4927fcf963b5cb8e11b5d590f7a26c21071 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sat, 15 Jul 2023 10:01:18 -0400 Subject: [PATCH 07/11] make it possible to build rpms to different destinations --- packaging/rpm/make-rpm.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/rpm/make-rpm.sh b/packaging/rpm/make-rpm.sh index 49812044a8..f201ed03e3 100755 --- a/packaging/rpm/make-rpm.sh +++ b/packaging/rpm/make-rpm.sh @@ -22,7 +22,8 @@ GIT_SHA=$(git rev-parse HEAD) GITHUB_SHA="${GIT_SHA:0:7}" TARBALL_FILE="microshift-${GITHUB_SHA}.tar.gz" -RPMBUILD_DIR="$(git rev-parse --show-toplevel)/_output/rpmbuild/" +DEFAULT_RPMBUILD_DIR="$(git rev-parse --show-toplevel)/_output/rpmbuild/" +RPMBUILD_DIR="${RPMBUILD_DIR:-${DEFAULT_RPMBUILD_DIR}}" title() { echo -e "\E[34m\n# $1\E[00m"; @@ -87,6 +88,7 @@ if [ -n "${TARGET_ARCH}" ]; then fi # prepare the rpmbuild env +echo "Building to ${RPMBUILD_DIR}" mkdir -p "${RPMBUILD_DIR}"/{BUILD,RPMS,SOURCES,SPECS,SRPMS} case $1 in From 75324333973d3b174e3aa0b9167633589bd165f9 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sat, 15 Jul 2023 10:01:42 -0400 Subject: [PATCH 08/11] build fake next minor rpm to different directory than the real rpm --- test/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/Makefile b/test/Makefile index a13bc89d54..15c33f7498 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,6 +1,8 @@ REPO := $(shell git rev-parse --show-toplevel) include $(REPO)/Makefile +RPMBUILD_DIR := $(REPO)/_output/rpmbuild-fake-next-minor/ + NEXT_MINOR = $(shell expr $(MINOR) + 1) .PHONY: fake-next-minor fake-next-minor-rpm @@ -8,5 +10,7 @@ fake-next-minor: @echo $(MAJOR).$(NEXT_MINOR).$(PATCH)_fake_next_minor fake-next-minor-rpm: + rm -rf $(RPMBUILD_DIR) MICROSHIFT_VERSION="$(MAJOR).$(NEXT_MINOR).$(PATCH)_fake_next_minor" \ + RPMBUILD_DIR=$(RPMBUILD_DIR) \ $(MAKE) -C $(REPO) rpm From 61c819320200506eedfd00c30ab63c29dd16cb85 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sat, 15 Jul 2023 10:02:13 -0400 Subject: [PATCH 09/11] set up separate rpm repository for fake next minor version --- test/bin/build_images.sh | 1 + test/bin/ci_phase_iso_build.sh | 2 +- test/bin/common.sh | 10 ++++- test/bin/create_local_repo.sh | 40 +++++++++++-------- .../microshift-fake-next-minor.toml | 7 ++++ 5 files changed, 42 insertions(+), 18 deletions(-) create mode 100644 test/package-sources/microshift-fake-next-minor.toml diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh index 2838b42334..86c7a016ae 100755 --- a/test/bin/build_images.sh +++ b/test/bin/build_images.sh @@ -42,6 +42,7 @@ PREVIOUS_MINOR_VERSION=$(( "${MINOR_VERSION}" - 1 )) UNAME_M=$(uname -m) export UNAME_M export LOCAL_REPO # defined in common.sh +export NEXT_REPO # defined in common.sh export SOURCE_VERSION # defined earlier export FAKE_NEXT_MINOR_VERSION # defined earlier export MINOR_VERSION # defined earlier diff --git a/test/bin/ci_phase_iso_build.sh b/test/bin/ci_phase_iso_build.sh index 53f105262b..912e5936f1 100755 --- a/test/bin/ci_phase_iso_build.sh +++ b/test/bin/ci_phase_iso_build.sh @@ -49,7 +49,7 @@ bash -x ./bin/configure_hypervisor_firewall.sh # Re-build from source. cd ~/microshift/ -rm -rf ./_output/rpmbuild +rm -rf ./_output/rpmbuild* time make rpm time make -C test/ fake-next-minor-rpm diff --git a/test/bin/common.sh b/test/bin/common.sh index 23649e880a..cf9fb0caaf 100755 --- a/test/bin/common.sh +++ b/test/bin/common.sh @@ -29,9 +29,17 @@ VM_DISK_DIR="${IMAGEDIR}/${VM_STORAGE_POOL}" # shellcheck disable=SC2034 # used elsewhere RPM_SOURCE="${ROOTDIR}/_output/rpmbuild" +# Location of RPMs built from source +# shellcheck disable=SC2034 # used elsewhere +NEXT_RPM_SOURCE="${ROOTDIR}/_output/rpmbuild-fake-next-minor" + +# Location of local repository used by composer +# shellcheck disable=SC2034 # used elsewhere +LOCAL_REPO="${IMAGEDIR}/rpm-repos/microshift-local" + # Location of local repository used by composer # shellcheck disable=SC2034 # used elsewhere -LOCAL_REPO="${IMAGEDIR}/microshift-local" +NEXT_REPO="${IMAGEDIR}/rpm-repos/microshift-fake-next-minor" # Location of data files created by the tools for managing scenarios # as they are run. diff --git a/test/bin/create_local_repo.sh b/test/bin/create_local_repo.sh index 2d36a30e95..78d286a28a 100755 --- a/test/bin/create_local_repo.sh +++ b/test/bin/create_local_repo.sh @@ -9,23 +9,31 @@ set -euo pipefail SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source "${SCRIPTDIR}/common.sh" -mkdir -p "${IMAGEDIR}" -cd "${IMAGEDIR}" +make_repo() { + local repodir="$1" + local builddir="$2" + + if [ -d "${repodir}" ]; then + echo "Cleaning up existing repository" + rm -rf "${repodir}" + fi + mkdir -p "${repodir}" -if [ -d "${LOCAL_REPO}" ]; then - echo "Cleaning up existing repository" - rm -rf "${LOCAL_REPO}" -fi -mkdir -p "${LOCAL_REPO}" + # Create the local RPM repository for whatever was built from source. + echo "Copying RPMs from ${builddir} to ${repodir}" + # shellcheck disable=SC2086 # no quotes for command arguments to allow word splitting + cp -R ${builddir}/{RPMS,SPECS,SRPMS} "${repodir}/" -# Create the local RPM repository for whatever was built from source. -echo "Copying RPMs from ${RPM_SOURCE} to ${LOCAL_REPO}" -# shellcheck disable=SC2086 # no quotes for command arguments to allow word splitting -cp -R ${RPM_SOURCE}/{RPMS,SPECS,SRPMS} "${LOCAL_REPO}/" + echo "Creating RPM repo at ${repodir}" + createrepo "${repodir}" -echo "Creating RPM repo at ${LOCAL_REPO}" -createrepo "${LOCAL_REPO}" + echo "Fixing permissions of RPM repo contents" + find "${repodir}" -type f -print -exec chmod a+r {} \; + find "${repodir}" -type d -exec chmod a+rx {} \; +} + +mkdir -p "${IMAGEDIR}" +cd "${IMAGEDIR}" -echo "Fixing permissions of RPM repo contents" -find "${LOCAL_REPO}" -type f -exec chmod a+r {} \; -find "${LOCAL_REPO}" -type d -exec chmod a+rx {} \; +make_repo "${LOCAL_REPO}" "${RPM_SOURCE}" +make_repo "${NEXT_REPO}" "${NEXT_RPM_SOURCE}" diff --git a/test/package-sources/microshift-fake-next-minor.toml b/test/package-sources/microshift-fake-next-minor.toml new file mode 100644 index 0000000000..c56a3b0c55 --- /dev/null +++ b/test/package-sources/microshift-fake-next-minor.toml @@ -0,0 +1,7 @@ +id = "microshift-fake-next-minor" +name = "MicroShift Fake Next Minor Version Repo" +type = "yum-baseurl" +url = "file://${NEXT_REPO}/" +check_gpg = false +check_ssl = false +system = false From 698847c81e7ab5e95cca928299983258ccad5c8f Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sat, 15 Jul 2023 10:26:09 -0400 Subject: [PATCH 10/11] standardize on shell functions for determining versions --- test/Makefile | 5 +---- test/bin/build_images.sh | 4 ++-- test/bin/scenario.sh | 6 ++++++ test/image-blueprints/rhel92-source-fake-next-minor.toml | 8 ++++---- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/test/Makefile b/test/Makefile index 15c33f7498..23463c5772 100644 --- a/test/Makefile +++ b/test/Makefile @@ -5,10 +5,7 @@ RPMBUILD_DIR := $(REPO)/_output/rpmbuild-fake-next-minor/ NEXT_MINOR = $(shell expr $(MINOR) + 1) -.PHONY: fake-next-minor fake-next-minor-rpm -fake-next-minor: - @echo $(MAJOR).$(NEXT_MINOR).$(PATCH)_fake_next_minor - +.PHONY: fake-next-minor-rpm fake-next-minor-rpm: rm -rf $(RPMBUILD_DIR) MICROSHIFT_VERSION="$(MAJOR).$(NEXT_MINOR).$(PATCH)_fake_next_minor" \ diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh index 86c7a016ae..a965400ec1 100755 --- a/test/bin/build_images.sh +++ b/test/bin/build_images.sh @@ -26,15 +26,15 @@ if [ ! -d "${LOCAL_REPO}" ]; then error "Run ${SCRIPTDIR}/create_local_repo.sh before building images." exit 1 fi -release_info_rpm=$(find "${LOCAL_REPO}" -name 'microshift-release-info-*.rpm' -a -not -name '*_fake_*') +release_info_rpm=$(find "${LOCAL_REPO}" -name 'microshift-release-info-*.rpm' | sort | tail -n 1) if [ -z "${release_info_rpm}" ]; then error "Failed to find microshift-release-info RPM in ${LOCAL_REPO}" exit 1 fi SOURCE_VERSION=$(rpm -q --queryformat '%{version}' "${release_info_rpm}") -FAKE_NEXT_MINOR_VERSION=$(cd "${SCRIPTDIR}/.." && make fake-next-minor) MINOR_VERSION=$(echo "${SOURCE_VERSION}" | cut -f2 -d.) PREVIOUS_MINOR_VERSION=$(( "${MINOR_VERSION}" - 1 )) +FAKE_NEXT_MINOR_VERSION=$(( "${MINOR_VERSION}" + 1 )) ## TEMPLATE VARIABLES # diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 3bfb674aaf..76c4b9db38 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -238,6 +238,12 @@ previous_minor_version() { echo $(( $(current_minor_version) - 1 )) } +# Function to report the *next* minor version. If the current +# version is "4.14.5", reports "15". +next_minor_version() { + echo $(( $(current_minor_version) + 1 )) +} + # Run the tests for the current scenario run_tests() { local vmname="${1}" diff --git a/test/image-blueprints/rhel92-source-fake-next-minor.toml b/test/image-blueprints/rhel92-source-fake-next-minor.toml index edf5b86c0c..99f7a30592 100644 --- a/test/image-blueprints/rhel92-source-fake-next-minor.toml +++ b/test/image-blueprints/rhel92-source-fake-next-minor.toml @@ -7,19 +7,19 @@ distro = "rhel-92" [[packages]] name = "microshift" -version = "${FAKE_NEXT_MINOR_VERSION}" +version = "4.${FAKE_NEXT_MINOR_VERSION}.*" [[packages]] name = "microshift-greenboot" -version = "${FAKE_NEXT_MINOR_VERSION}" +version = "4.${FAKE_NEXT_MINOR_VERSION}.*" [[packages]] name = "microshift-networking" -version = "${FAKE_NEXT_MINOR_VERSION}" +version = "4.${FAKE_NEXT_MINOR_VERSION}.*" [[packages]] name = "microshift-selinux" -version = "${FAKE_NEXT_MINOR_VERSION}" +version = "4.${FAKE_NEXT_MINOR_VERSION}.*" [customizations.services] enabled = ["microshift"] From 1796ec31fed511ccef13eda97d6906d8f30a30a1 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sat, 15 Jul 2023 13:20:50 -0400 Subject: [PATCH 11/11] make sure we reset any package source on rebuild --- test/bin/build_images.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh index a965400ec1..81b4c0fdee 100755 --- a/test/bin/build_images.sh +++ b/test/bin/build_images.sh @@ -58,6 +58,9 @@ for template in ${TESTDIR}/package-sources/*.toml; do echo "Rendering ${template} to ${outfile}" envsubst <"${template}" >"${outfile}" echo "Adding package source from ${outfile}" + if sudo composer-cli sources list | grep "^${name}\$"; then + sudo composer-cli sources delete "${name}" + fi sudo composer-cli sources add "${outfile}" done