Skip to content

USHIFT-1423: Only test upgrades on metal #41143

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-robot merged 4 commits into from
Jul 13, 2023
Merged

USHIFT-1423: Only test upgrades on metal #41143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c145b8a
USHIFT-1423: Remove execution of e2e and cncf tests from metal
pacevedom Jul 11, 2023
e3af910
USHIFT-1423: Remove dead code
pacevedom Jul 11, 2023
d04afb3
USHIFT-1423: Remove build phase for cncf and old e2e on metal
pacevedom Jul 11, 2023
a71a93c
USHIFT-1423: Remove boot phase for cncf and old e2e on metal
pacevedom Jul 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 0 additions & 165 deletions ...try/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,153 +19,6 @@ Host ${IP_ADDRESS}
EOF
chmod 0600 "${HOME}/.ssh/config"

# Print test output on exit
print_test_output() {
echo "##### START TEST OUTPUT #####"

for log in /tmp/run_*_test.log ; do
echo "##### OUTPUT OF ${log} TEST #####"
cat "${log}"
done

echo "##### FINISH TEST OUTPUT #####"
}

# Bash e2e tests
run_e2e() {
local -r VM_IP="${IP_ADDRESS}"
local -r VM_PORT="$(cat "${SHARED_DIR}"/vm_ssh_port_0)"
local -r VM_USER="$(cat "${SHARED_DIR}"/vm_user_0)"

cat << EOF >/tmp/e2e.yaml
USHIFT_HOST: ${VM_IP}
USHIFT_USER: ${VM_USER}
SSH_PRIV_KEY: ${CLUSTER_PROFILE_DIR}/ssh-privatekey
SSH_PORT: ${VM_PORT}
EOF
/microshift/test/run.sh -o "${ARTIFACT_DIR}/e2e" -i /tmp/e2e.yaml -v /tmp/venv /microshift/test/suites-ostree/backup-restore.robot
}

# Bash CNCF Tests
# See https://github.com/openshift/microshift/blob/main/docs/multinode/setup.md
run_cncf() {
local -r PRI_HOST="$(cat "${SHARED_DIR}"/vm_ssh_host_1)"
local -r PRI_ADDR="$(cat "${SHARED_DIR}"/vm_int_ip_1)"
local -r PRI_PORT="$(cat "${SHARED_DIR}"/vm_ssh_port_1)"
local -r PRI_USER="$(cat "${SHARED_DIR}"/vm_user_1)"

local -r SEC_HOST="$(cat "${SHARED_DIR}"/vm_ssh_host_2)"
local -r SEC_ADDR="$(cat "${SHARED_DIR}"/vm_int_ip_2)"
local -r SEC_PORT="$(cat "${SHARED_DIR}"/vm_ssh_port_2)"
local -r SEC_USER="$(cat "${SHARED_DIR}"/vm_user_2)"

local -r SSH_CMD="ssh -i ${CLUSTER_PROFILE_DIR}/ssh-privatekey"
local -r SCP_CMD="scp -i ${CLUSTER_PROFILE_DIR}/ssh-privatekey"

local -r RUN_SCRIPT=/tmp/run_cncf.sh
local -r KUBECONFIG=/tmp/kubeconfig-cncf

cd /microshift/
# Configure the primary host
${SCP_CMD} -P "${PRI_PORT}" ./scripts/multinode/configure-pri.sh "${PRI_USER}@${IP_ADDRESS}:"
${SSH_CMD} -p "${PRI_PORT}" "${PRI_USER}@${IP_ADDRESS}" \
./configure-pri.sh "${PRI_HOST}" "${PRI_ADDR}" "${SEC_HOST}" "${SEC_ADDR}"

# Copy the kubelet configuration from the primary to the secondary host in two steps
${SCP_CMD} -P "${PRI_PORT}" \
"${PRI_USER}@${IP_ADDRESS}:/home/redhat/kubelet-${SEC_HOST}".{key,crt} \
"${PRI_USER}@${IP_ADDRESS}:/home/redhat/kubeconfig-${PRI_HOST}" \
/tmp/
${SCP_CMD} -P "${SEC_PORT}" \
"/tmp/kubelet-${SEC_HOST}".{key,crt} \
"/tmp/kubeconfig-${PRI_HOST}" \
"${SEC_USER}@${IP_ADDRESS}":

# Configure the secondary host
${SCP_CMD} -P "${SEC_PORT}" ./scripts/multinode/configure-sec.sh "${SEC_USER}@${IP_ADDRESS}:"
${SSH_CMD} -p "${SEC_PORT}" "${SEC_USER}@${IP_ADDRESS}" \
./configure-sec.sh "${PRI_HOST}" "${PRI_ADDR}" "${SEC_HOST}" "${SEC_ADDR}"

cat <<EOF > "${RUN_SCRIPT}"
#!/bin/bash
set -xeuo pipefail

cd \${HOME}/microshift

# Resolve primary host name locally
echo "${PRI_ADDR} ${PRI_HOST}" | sudo tee -a /etc/hosts &>/dev/null

export KUBECONFIG="${KUBECONFIG}"
oc get pods -A -o wide

# Wait up to 5m until both nodes are ready
NREADY=1
for _ in \$(seq 1 30) ; do
NREADY=\$(oc get nodes --no-headers | awk '\$2=="Ready" {print \$1}' | wc -l)
[ "\${NREADY}" = 2 ] && break
sleep 10
done
oc get nodes -o wide
[ "\${NREADY}" != 2 ] && exit 1

# Configure cluster prerequisites
oc adm policy add-scc-to-group privileged system:authenticated system:serviceaccounts
oc adm policy add-scc-to-group anyuid system:authenticated system:serviceaccounts

# Install the tests
sudo dnf install -y golang
go install github.com/vmware-tanzu/sonobuoy@latest

# Run the tests
~/go/bin/sonobuoy run \
--mode=certified-conformance \
--dns-namespace=openshift-dns \
--dns-pod-labels=dns.operator.openshift.io/daemonset-dns=default

# Wait for up to 1m until tests start
WAIT_FAILURE=true
for _ in \$(seq 1 30) ; do
if ~/go/bin/sonobuoy status --json | jq '.status' &>/dev/null ; then
WAIT_FAILURE=false
break
fi
sleep 2
done

# Exit with error on wait failure
\$WAIT_FAILURE && exit 1

# Wait until test complete (exit as soon as one of the tests failed)
TEST_FAILURE=false
while [ "\$(~/go/bin/sonobuoy status --json | jq -r '.status')" = "running" ] ; do
~/go/bin/sonobuoy status --json | jq '.plugins[] | select(.plugin=="e2e") | .progress'
if [ "\$(~/go/bin/sonobuoy status --json | jq -r '.plugins[] | select(.plugin=="e2e") | .progress.failed')" != "null" ] ; then
TEST_FAILURE=true
break
fi
sleep 60
done

# Exit with error on test failure
\$TEST_FAILURE && exit 1
# Normal exit
exit 0
EOF

# Download the kubeconfig from the primary host
${SCP_CMD} -P "${PRI_PORT}" "${PRI_USER}@${IP_ADDRESS}:/home/redhat/kubeconfig-${PRI_HOST}" "${KUBECONFIG}"
cat "${KUBECONFIG}"

# Copy and run the script, waiting up to 2h for it to complete
chmod +x "${RUN_SCRIPT}"
scp "${RUN_SCRIPT}" "${KUBECONFIG}" "${INSTANCE_PREFIX}:/tmp"
timeout 120m ssh "${INSTANCE_PREFIX}" "${RUN_SCRIPT}"
}

######################################################################
# If more tests are to be run in parallel the code should go in here #
######################################################################

trap 'scp -r ${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/_output/test-images/scenario-info ${ARTIFACT_DIR}' EXIT

# Run the scenario tests, if the phase script exists
Expand All @@ -174,21 +27,3 @@ cd /microshift/test || true
if [ -f ./bin/ci_phase_test.sh ]; then
./bin/ci_phase_test.sh
fi

# VM mapping
# e2e: vm0
# cncf: vm1, vm2
trap print_test_output EXIT
run_e2e &> /tmp/run_e2e_test.log &
run_cncf &> /tmp/run_cncf_test.log &

FAIL=0
for job in $(jobs -p) ; do
echo "Waiting for job: $job"
wait "$job" || ((FAIL+=1))
done

if [ "$FAIL" != "0" ]; then
echo "Tests failed. Check junit for details"
exit 1
fi
106 changes: 1 addition & 105 deletions ...istry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,110 +26,6 @@ Host ${IP_ADDRESS}
EOF
chmod 0600 "${HOME}/.ssh/config"

# Number of VMs to create.
# This will change over time as we add more tests.
NUM_VMS=3
echo "${NUM_VMS}" > "${SHARED_DIR}/num_vms"

# Run the boot VM loop
# TODO: run the boot.sh script in parallel
# Note that 'dnf' command fails when in parallel, so it needs to be put in
# critical section in the scripts/image-builder/create-vm.sh script
for (( i=0; i<NUM_VMS; i++ ))
do
API_EXTERNAL_PORT=$((API_EXTERNAL_BASE_PORT+i))
SSH_EXTERNAL_PORT=$((SSH_EXTERNAL_BASE_PORT+i))
VM_NAME="ushift-${i}"
cat <<EOF > /tmp/boot.sh
#!/bin/bash
set -xeuo pipefail

cd ~/microshift

ISO_FILE=\$(find ~/microshift/_output/image-builder -name "*.iso")
sudo bash -c " \
cd /var/lib/libvirt/images/ && \
virt-install \
--name ${VM_NAME} \
--vcpus 2 \
--memory 3072 \
--disk path=./${VM_NAME}.qcow2,size=20 \
--network network=default,model=virtio \
--events on_reboot=restart \
--cdrom "\${ISO_FILE}" \
--noautoconsole \
--wait \
"

VM_IP=\$(./scripts/devenv-builder/manage-vm.sh ip -n ${VM_NAME})
timeout 8m bash -c "until ssh -oStrictHostKeyChecking=accept-new redhat@\${VM_IP} 'echo hello'; do sleep 5; done"

cat << EOF2 > /tmp/config.yaml
apiServer:
subjectAltNames:
- "${IP_ADDRESS}"
EOF2

scp /tmp/config.yaml "redhat@\${VM_IP}":/tmp/
set +e
ssh "redhat@\${VM_IP}" "sudo mv /tmp/config.yaml /etc/microshift/config.yaml && sudo reboot"
set -e
EOF

chmod +x /tmp/boot.sh
scp /tmp/boot.sh "${INSTANCE_PREFIX}:/tmp"
ssh "${INSTANCE_PREFIX}" "/tmp/boot.sh"
done

# Run the wait VM loop
for (( i=0; i<NUM_VMS; i++ ))
do
API_EXTERNAL_PORT=$((API_EXTERNAL_BASE_PORT+i))
SSH_EXTERNAL_PORT=$((SSH_EXTERNAL_BASE_PORT+i))
VM_NAME="ushift-${i}"

cat <<EOF > /tmp/wait.sh
#!/bin/bash
set -xeuo pipefail

cd ~/microshift
VM_IP=\$(./scripts/devenv-builder/manage-vm.sh ip -n ${VM_NAME})

timeout 8m bash -c "until ssh redhat@\${VM_IP} hostname; do sleep 5; done"
timeout 8m bash -c "date; until ssh redhat@\${VM_IP} \"sudo systemctl status greenboot-healthcheck | grep 'active (exited)'\"; do sleep 5; done; date"

# Setup external access with port forwarding to allow running commands and tests from the CI container.
sudo /sbin/iptables -I FORWARD -o virbr0 -p tcp -d \${VM_IP} --dport 6443 -j ACCEPT
sudo /sbin/iptables -t nat -I PREROUTING -p tcp --dport "${API_EXTERNAL_PORT}" -j DNAT --to \${VM_IP}:6443
sudo /sbin/iptables -I FORWARD -o virbr0 -p tcp -d \${VM_IP} --dport 22 -j ACCEPT
sudo /sbin/iptables -t nat -I PREROUTING -p tcp --dport "${SSH_EXTERNAL_PORT}" -j DNAT --to \${VM_IP}:22
EOF

chmod +x /tmp/wait.sh
scp /tmp/wait.sh "${INSTANCE_PREFIX}:/tmp"
ssh "${INSTANCE_PREFIX}" "/tmp/wait.sh"
done

# Save the name, ip, port, user, etc. information about the VMs
for (( i=0; i<NUM_VMS; i++ ))
do
API_EXTERNAL_PORT=$((API_EXTERNAL_BASE_PORT+i))
SSH_EXTERNAL_PORT=$((SSH_EXTERNAL_BASE_PORT+i))
VM_NAME="ushift-${i}"

ssh "redhat@${IP_ADDRESS}" -p ${SSH_EXTERNAL_PORT} \
"sudo cat /var/lib/microshift/resources/kubeadmin/${IP_ADDRESS}/kubeconfig" > "${SHARED_DIR}/kubeconfig_${i}"
sed -i "s,:6443,:${API_EXTERNAL_PORT}," "${SHARED_DIR}/kubeconfig_${i}"

# shellcheck disable=SC2029
ssh "${INSTANCE_PREFIX}" \
"microshift/scripts/devenv-builder/manage-vm.sh ip -n ${VM_NAME}" > "${SHARED_DIR}/vm_int_ip_${i}"

echo "${VM_NAME}" > "${SHARED_DIR}/vm_ssh_host_${i}"
echo "${SSH_EXTERNAL_PORT}" > "${SHARED_DIR}/vm_ssh_port_${i}"
echo "redhat" > "${SHARED_DIR}/vm_user_${i}"
done

# Install the settings for the scenario runner. The ssh keys have
# already been copied into place in the iso-build step.
SETTINGS_FILE="${SHARED_DIR}/scenario_settings.sh"
Expand All @@ -143,4 +39,4 @@ scp "${SETTINGS_FILE}" "${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/test/"
trap 'scp -r ${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/_output/test-images/scenario-info ${ARTIFACT_DIR}' EXIT

# Run the in-repo ci phase script to create the VMs for the test scenarios.
ssh "${INSTANCE_PREFIX}" "/home/${HOST_USER}/microshift/test/bin/ci_phase_iso_boot.sh ${API_EXTERNAL_PORT} ${SSH_EXTERNAL_PORT} ${LB_EXTERNAL_BASE_PORT}"
ssh "${INSTANCE_PREFIX}" "/home/${HOST_USER}/microshift/test/bin/ci_phase_iso_boot.sh ${SSH_EXTERNAL_BASE_PORT} ${API_EXTERNAL_BASE_PORT} ${LB_EXTERNAL_BASE_PORT}"
5 changes: 0 additions & 5 deletions ...try/openshift/microshift/infra/iso-build/openshift-microshift-infra-iso-build-commands.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@ if ! sudo subscription-manager status >&/dev/null; then
fi

chmod 0755 ~
mkdir ~/rpms
tar -xf /tmp/rpms.tar -C ~/rpms
tar -xf /tmp/microshift.tgz -C ~

cp /tmp/ssh-publickey ~/.ssh/id_rsa.pub
Expand All @@ -46,15 +44,12 @@ cp /tmp/pull-secret "\${PULL_SECRET}"
cd ~/microshift

./test/bin/ci_phase_iso_build.sh

./scripts/image-builder/build.sh -pull_secret_file "\${PULL_SECRET}" -microshift_rpms ~/rpms -authorized_keys_file ~/.ssh/id_rsa.pub -open_firewall_ports 6443:tcp
Copy link
Copy Markdown
Contributor

@dhellmann dhellmann Jul 12, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm seeing an error on openshift/microshift#1989 that I think will be resolved by removing this step. See https://prow.ci.openshift.org/view/gs/origin-ci-test/pr-logs/pull/openshift_microshift/1989/pull-ci-openshift-microshift-main-microshift-metal-tests/1679202926033113088#1:build-log.txt%3A4889 for example

Copy link
Copy Markdown
Contributor Author

@pacevedom pacevedom Jul 13, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This started happening in all builds. The issue is that there is a directory missing, but we didnt change the logic. Not sure if its worth fixing as merging this PR will make it disappear.

EOF
chmod +x /tmp/iso.sh

tar czf /tmp/microshift.tgz /microshift

scp \
/rpms.tar \
/tmp/iso.sh \
/var/run/rhsm/subscription-manager-org \
/var/run/rhsm/subscription-manager-act-key \
Expand Down