From 12842586cf7ccf005332ae1657e8cc22114ee0e5 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Thu, 6 Jul 2023 00:14:28 +0200 Subject: [PATCH 1/5] wip --- ...openshift-microshift-e2e-metal-tests-commands.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh b/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh index a534140430440..968de5cc29ec2 100644 --- a/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh +++ b/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh @@ -172,10 +172,19 @@ scp -r "${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/_output/test-images/sce # Run the scenario tests, if the phase script exists # (we can clean this up after the main PR lands) -cd /microshift/test || true + +#TODO temporary clone. Remove to use the one from home dir. +cat > /tmp/tests.sh << EOF +git clone -b USHIFT-1387 https://github.com/pacevedom/microshift /tmp/microshift +cd /tmp/microshift/test || true if [ -f ./bin/ci_phase_test.sh ]; then - ./bin/ci_phase_test.sh + ./bin/ci_phase_test.sh fi +EOF +chmod +x /tmp/tests.sh + +scp /tmp/tests.sh "${INSTANCE_PREFIX}:/tmp" +ssh "${INSTANCE_PREFIX}" "/tmp/tests.sh" # Copy the scenario settings from the remote host here *again* to # include the log files and other outputs. From 07a1ceb6ddccf18151810279aae61c94fa2396ad Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Fri, 7 Jul 2023 09:15:22 +0200 Subject: [PATCH 2/5] temporarily disable non-scenario vms --- ...ift-microshift-e2e-metal-tests-commands.sh | 6 +- ...hift-microshift-infra-iso-boot-commands.sh | 206 +++++++++--------- ...ift-microshift-infra-iso-build-commands.sh | 2 +- 3 files changed, 107 insertions(+), 107 deletions(-) diff --git a/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh b/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh index 968de5cc29ec2..acc7ab642b785 100644 --- a/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh +++ b/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh @@ -193,9 +193,9 @@ scp -r "${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/_output/test-images/sce # VM mapping # e2e: vm0 # cncf: vm1, vm2 -trap print_test_output EXIT -run_e2e &> /tmp/run_e2e_test.log & -run_cncf &> /tmp/run_cncf_test.log & +# trap print_test_output EXIT +# run_e2e &> /tmp/run_e2e_test.log & +# run_cncf &> /tmp/run_cncf_test.log & FAIL=0 for job in $(jobs -p) ; do diff --git a/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh b/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh index bde233fe40bb4..033dd45763066 100644 --- a/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh +++ b/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh @@ -26,109 +26,109 @@ Host ${IP_ADDRESS} EOF chmod 0600 "${HOME}/.ssh/config" -# Number of VMs to create. -# This will change over time as we add more tests. -NUM_VMS=3 -echo "${NUM_VMS}" > "${SHARED_DIR}/num_vms" - -# Run the boot VM loop -# TODO: run the boot.sh script in parallel -# Note that 'dnf' command fails when in parallel, so it needs to be put in -# critical section in the scripts/image-builder/create-vm.sh script -for (( i=0; i /tmp/boot.sh -#!/bin/bash -set -xeuo pipefail - -cd ~/microshift - -ISO_FILE=\$(find ~/microshift/_output/image-builder -name "*.iso") -sudo bash -c " \ - cd /var/lib/libvirt/images/ && \ - virt-install \ - --name ${VM_NAME} \ - --vcpus 2 \ - --memory 3072 \ - --disk path=./${VM_NAME}.qcow2,size=20 \ - --network network=default,model=virtio \ - --events on_reboot=restart \ - --cdrom "\${ISO_FILE}" \ - --noautoconsole \ - --wait \ -" - -VM_IP=\$(./scripts/devenv-builder/manage-vm.sh ip -n ${VM_NAME}) -timeout 8m bash -c "until ssh -oStrictHostKeyChecking=accept-new redhat@\${VM_IP} 'echo hello'; do sleep 5; done" - -cat << EOF2 > /tmp/config.yaml -apiServer: - subjectAltNames: - - "${IP_ADDRESS}" -EOF2 - -scp /tmp/config.yaml "redhat@\${VM_IP}":/tmp/ -set +e -ssh "redhat@\${VM_IP}" "sudo mv /tmp/config.yaml /etc/microshift/config.yaml && sudo reboot" -set -e -EOF - - chmod +x /tmp/boot.sh - scp /tmp/boot.sh "${INSTANCE_PREFIX}:/tmp" - ssh "${INSTANCE_PREFIX}" "/tmp/boot.sh" -done - -# Run the wait VM loop -for (( i=0; i /tmp/wait.sh -#!/bin/bash -set -xeuo pipefail - -cd ~/microshift -VM_IP=\$(./scripts/devenv-builder/manage-vm.sh ip -n ${VM_NAME}) - -timeout 8m bash -c "until ssh redhat@\${VM_IP} hostname; do sleep 5; done" -timeout 8m bash -c "date; until ssh redhat@\${VM_IP} \"sudo systemctl status greenboot-healthcheck | grep 'active (exited)'\"; do sleep 5; done; date" - -# Setup external access with port forwarding to allow running commands and tests from the CI container. -sudo /sbin/iptables -I FORWARD -o virbr0 -p tcp -d \${VM_IP} --dport 6443 -j ACCEPT -sudo /sbin/iptables -t nat -I PREROUTING -p tcp --dport "${API_EXTERNAL_PORT}" -j DNAT --to \${VM_IP}:6443 -sudo /sbin/iptables -I FORWARD -o virbr0 -p tcp -d \${VM_IP} --dport 22 -j ACCEPT -sudo /sbin/iptables -t nat -I PREROUTING -p tcp --dport "${SSH_EXTERNAL_PORT}" -j DNAT --to \${VM_IP}:22 -EOF - - chmod +x /tmp/wait.sh - scp /tmp/wait.sh "${INSTANCE_PREFIX}:/tmp" - ssh "${INSTANCE_PREFIX}" "/tmp/wait.sh" -done - -# Save the name, ip, port, user, etc. information about the VMs -for (( i=0; i "${SHARED_DIR}/kubeconfig_${i}" - sed -i "s,:6443,:${API_EXTERNAL_PORT}," "${SHARED_DIR}/kubeconfig_${i}" - - # shellcheck disable=SC2029 - ssh "${INSTANCE_PREFIX}" \ - "microshift/scripts/devenv-builder/manage-vm.sh ip -n ${VM_NAME}" > "${SHARED_DIR}/vm_int_ip_${i}" - - echo "${VM_NAME}" > "${SHARED_DIR}/vm_ssh_host_${i}" - echo "${SSH_EXTERNAL_PORT}" > "${SHARED_DIR}/vm_ssh_port_${i}" - echo "redhat" > "${SHARED_DIR}/vm_user_${i}" -done +# # Number of VMs to create. +# # This will change over time as we add more tests. +# NUM_VMS=3 +# echo "${NUM_VMS}" > "${SHARED_DIR}/num_vms" + +# # Run the boot VM loop +# # TODO: run the boot.sh script in parallel +# # Note that 'dnf' command fails when in parallel, so it needs to be put in +# # critical section in the scripts/image-builder/create-vm.sh script +# for (( i=0; i /tmp/boot.sh +# #!/bin/bash +# set -xeuo pipefail + +# cd ~/microshift + +# ISO_FILE=\$(find ~/microshift/_output/image-builder -name "*.iso") +# sudo bash -c " \ +# cd /var/lib/libvirt/images/ && \ +# virt-install \ +# --name ${VM_NAME} \ +# --vcpus 2 \ +# --memory 3072 \ +# --disk path=./${VM_NAME}.qcow2,size=20 \ +# --network network=default,model=virtio \ +# --events on_reboot=restart \ +# --cdrom "\${ISO_FILE}" \ +# --noautoconsole \ +# --wait \ +# " + +# VM_IP=\$(./scripts/devenv-builder/manage-vm.sh ip -n ${VM_NAME}) +# timeout 8m bash -c "until ssh -oStrictHostKeyChecking=accept-new redhat@\${VM_IP} 'echo hello'; do sleep 5; done" + +# cat << EOF2 > /tmp/config.yaml +# apiServer: +# subjectAltNames: +# - "${IP_ADDRESS}" +# EOF2 + +# scp /tmp/config.yaml "redhat@\${VM_IP}":/tmp/ +# set +e +# ssh "redhat@\${VM_IP}" "sudo mv /tmp/config.yaml /etc/microshift/config.yaml && sudo reboot" +# set -e +# EOF + +# chmod +x /tmp/boot.sh +# scp /tmp/boot.sh "${INSTANCE_PREFIX}:/tmp" +# ssh "${INSTANCE_PREFIX}" "/tmp/boot.sh" +# done + +# # Run the wait VM loop +# for (( i=0; i /tmp/wait.sh +# #!/bin/bash +# set -xeuo pipefail + +# cd ~/microshift +# VM_IP=\$(./scripts/devenv-builder/manage-vm.sh ip -n ${VM_NAME}) + +# timeout 8m bash -c "until ssh redhat@\${VM_IP} hostname; do sleep 5; done" +# timeout 8m bash -c "date; until ssh redhat@\${VM_IP} \"sudo systemctl status greenboot-healthcheck | grep 'active (exited)'\"; do sleep 5; done; date" + +# # Setup external access with port forwarding to allow running commands and tests from the CI container. +# sudo /sbin/iptables -I FORWARD -o virbr0 -p tcp -d \${VM_IP} --dport 6443 -j ACCEPT +# sudo /sbin/iptables -t nat -I PREROUTING -p tcp --dport "${API_EXTERNAL_PORT}" -j DNAT --to \${VM_IP}:6443 +# sudo /sbin/iptables -I FORWARD -o virbr0 -p tcp -d \${VM_IP} --dport 22 -j ACCEPT +# sudo /sbin/iptables -t nat -I PREROUTING -p tcp --dport "${SSH_EXTERNAL_PORT}" -j DNAT --to \${VM_IP}:22 +# EOF + +# chmod +x /tmp/wait.sh +# scp /tmp/wait.sh "${INSTANCE_PREFIX}:/tmp" +# ssh "${INSTANCE_PREFIX}" "/tmp/wait.sh" +# done + +# # Save the name, ip, port, user, etc. information about the VMs +# for (( i=0; i "${SHARED_DIR}/kubeconfig_${i}" +# sed -i "s,:6443,:${API_EXTERNAL_PORT}," "${SHARED_DIR}/kubeconfig_${i}" + +# # shellcheck disable=SC2029 +# ssh "${INSTANCE_PREFIX}" \ +# "microshift/scripts/devenv-builder/manage-vm.sh ip -n ${VM_NAME}" > "${SHARED_DIR}/vm_int_ip_${i}" + +# echo "${VM_NAME}" > "${SHARED_DIR}/vm_ssh_host_${i}" +# echo "${SSH_EXTERNAL_PORT}" > "${SHARED_DIR}/vm_ssh_port_${i}" +# echo "redhat" > "${SHARED_DIR}/vm_user_${i}" +# done # Install the settings for the scenario runner. The ssh keys have # already been copied into place in the iso-build step. diff --git a/ci-operator/step-registry/openshift/microshift/infra/iso-build/openshift-microshift-infra-iso-build-commands.sh b/ci-operator/step-registry/openshift/microshift/infra/iso-build/openshift-microshift-infra-iso-build-commands.sh index bdacb45221b77..f5db396f3fdab 100644 --- a/ci-operator/step-registry/openshift/microshift/infra/iso-build/openshift-microshift-infra-iso-build-commands.sh +++ b/ci-operator/step-registry/openshift/microshift/infra/iso-build/openshift-microshift-infra-iso-build-commands.sh @@ -47,7 +47,7 @@ cd ~/microshift ./test/bin/ci_phase_iso_build.sh -./scripts/image-builder/build.sh -pull_secret_file "\${PULL_SECRET}" -microshift_rpms ~/rpms -authorized_keys_file ~/.ssh/id_rsa.pub -open_firewall_ports 6443:tcp +# ./scripts/image-builder/build.sh -pull_secret_file "\${PULL_SECRET}" -microshift_rpms ~/rpms -authorized_keys_file ~/.ssh/id_rsa.pub -open_firewall_ports 6443:tcp EOF chmod +x /tmp/iso.sh From 76b9d0d682736106b9e38faf14e369d3bf8341dc Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Fri, 7 Jul 2023 10:31:08 +0200 Subject: [PATCH 3/5] fix boot base ports --- .../iso-boot/openshift-microshift-infra-iso-boot-commands.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh b/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh index 033dd45763066..9ca1eb2ac8101 100644 --- a/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh +++ b/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh @@ -141,7 +141,7 @@ EOF scp "${SETTINGS_FILE}" "${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/test/" # Run the in-repo ci phase script to create the VMs for the test scenarios. -ssh "${INSTANCE_PREFIX}" "/home/${HOST_USER}/microshift/test/bin/ci_phase_iso_boot.sh ${API_EXTERNAL_PORT} ${SSH_EXTERNAL_PORT} ${LB_EXTERNAL_BASE_PORT}" +ssh "${INSTANCE_PREFIX}" "/home/${HOST_USER}/microshift/test/bin/ci_phase_iso_boot.sh ${API_EXTERNAL_BASE_PORT} ${SSH_EXTERNAL_BASE_PORT} ${LB_EXTERNAL_BASE_PORT}" # Copy the scenario info directory from the host to our artifact directory. scp -r "${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/_output/test-images/scenario-info" "${ARTIFACT_DIR}" From 366114d97a3a0877f0f21d666b09914cccb9c92e Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Fri, 7 Jul 2023 12:03:28 +0200 Subject: [PATCH 4/5] use my repo temporarily from build step --- .../openshift-microshift-e2e-metal-tests-commands.sh | 3 +-- .../openshift-microshift-infra-iso-boot-commands.sh | 1 + .../openshift-microshift-infra-iso-build-commands.sh | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh b/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh index acc7ab642b785..e0c4ed259d73a 100644 --- a/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh +++ b/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh @@ -175,8 +175,7 @@ scp -r "${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/_output/test-images/sce #TODO temporary clone. Remove to use the one from home dir. cat > /tmp/tests.sh << EOF -git clone -b USHIFT-1387 https://github.com/pacevedom/microshift /tmp/microshift -cd /tmp/microshift/test || true +cd ~/microshift/test || true if [ -f ./bin/ci_phase_test.sh ]; then ./bin/ci_phase_test.sh fi diff --git a/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh b/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh index 9ca1eb2ac8101..7ad54e6b2bfe5 100644 --- a/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh +++ b/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh @@ -26,6 +26,7 @@ Host ${IP_ADDRESS} EOF chmod 0600 "${HOME}/.ssh/config" +#TODO commenting for speed. # # Number of VMs to create. # # This will change over time as we add more tests. # NUM_VMS=3 diff --git a/ci-operator/step-registry/openshift/microshift/infra/iso-build/openshift-microshift-infra-iso-build-commands.sh b/ci-operator/step-registry/openshift/microshift/infra/iso-build/openshift-microshift-infra-iso-build-commands.sh index f5db396f3fdab..434ed8cc3b4ca 100644 --- a/ci-operator/step-registry/openshift/microshift/infra/iso-build/openshift-microshift-infra-iso-build-commands.sh +++ b/ci-operator/step-registry/openshift/microshift/infra/iso-build/openshift-microshift-infra-iso-build-commands.sh @@ -31,9 +31,9 @@ if ! sudo subscription-manager status >&/dev/null; then fi chmod 0755 ~ -mkdir ~/rpms -tar -xf /tmp/rpms.tar -C ~/rpms -tar -xf /tmp/microshift.tgz -C ~ +#mkdir ~/rpms +#tar -xf /tmp/rpms.tar -C ~/rpms +#tar -xf /tmp/microshift.tgz -C ~ cp /tmp/ssh-publickey ~/.ssh/id_rsa.pub cp /tmp/ssh-privatekey ~/.ssh/id_rsa @@ -43,6 +43,9 @@ chmod 0400 ~/.ssh/id_rsa* export PULL_SECRET="\${HOME}/.pull-secret.json" cp /tmp/pull-secret "\${PULL_SECRET}" +#TODO remove this and use the tar above +sudo dnf install -y git +git clone -b USHIFT-1387 https://github.com/pacevedom/microshift ~/microshift cd ~/microshift ./test/bin/ci_phase_iso_build.sh From 01cc3decb9267022844087a41d32d1479c5d9bcd Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Fri, 7 Jul 2023 15:09:41 +0200 Subject: [PATCH 5/5] enable sos report log scp --- .../openshift-microshift-e2e-metal-tests-commands.sh | 9 +++------ .../openshift-microshift-infra-iso-boot-commands.sh | 5 ++--- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh b/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh index e0c4ed259d73a..fcd2bef87912e 100644 --- a/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh +++ b/ci-operator/step-registry/openshift/microshift/e2e/metal-tests/openshift-microshift-e2e-metal-tests-commands.sh @@ -166,6 +166,8 @@ EOF # If more tests are to be run in parallel the code should go in here # ###################################################################### +trap "scp -r ${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/_output/test-images/scenario-info ${ARTIFACT_DIR}" EXIT + # Copy the scenario settings from the remote host back here so the # test runner has access to them. scp -r "${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/_output/test-images/scenario-info" "${ARTIFACT_DIR}" @@ -173,9 +175,8 @@ scp -r "${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/_output/test-images/sce # Run the scenario tests, if the phase script exists # (we can clean this up after the main PR lands) -#TODO temporary clone. Remove to use the one from home dir. cat > /tmp/tests.sh << EOF -cd ~/microshift/test || true +cd ~/microshift/test if [ -f ./bin/ci_phase_test.sh ]; then ./bin/ci_phase_test.sh fi @@ -185,10 +186,6 @@ chmod +x /tmp/tests.sh scp /tmp/tests.sh "${INSTANCE_PREFIX}:/tmp" ssh "${INSTANCE_PREFIX}" "/tmp/tests.sh" -# Copy the scenario settings from the remote host here *again* to -# include the log files and other outputs. -scp -r "${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/_output/test-images/scenario-info" "${ARTIFACT_DIR}" - # VM mapping # e2e: vm0 # cncf: vm1, vm2 diff --git a/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh b/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh index 7ad54e6b2bfe5..63f3db64e6fdf 100644 --- a/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh +++ b/ci-operator/step-registry/openshift/microshift/infra/iso-boot/openshift-microshift-infra-iso-boot-commands.sh @@ -141,8 +141,7 @@ SSH_PRIVATE_KEY=~/.ssh/id_rsa EOF scp "${SETTINGS_FILE}" "${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/test/" +trap "scp -r ${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/_output/test-images/scenario-info ${ARTIFACT_DIR}" EXIT + # Run the in-repo ci phase script to create the VMs for the test scenarios. ssh "${INSTANCE_PREFIX}" "/home/${HOST_USER}/microshift/test/bin/ci_phase_iso_boot.sh ${API_EXTERNAL_BASE_PORT} ${SSH_EXTERNAL_BASE_PORT} ${LB_EXTERNAL_BASE_PORT}" - -# Copy the scenario info directory from the host to our artifact directory. -scp -r "${INSTANCE_PREFIX}:/home/${HOST_USER}/microshift/_output/test-images/scenario-info" "${ARTIFACT_DIR}"