diff --git a/ci-operator/config/openshift/installer/openshift-installer-master.yaml b/ci-operator/config/openshift/installer/openshift-installer-master.yaml index 045ce71386098..c311874eb5ef4 100644 --- a/ci-operator/config/openshift/installer/openshift-installer-master.yaml +++ b/ci-operator/config/openshift/installer/openshift-installer-master.yaml @@ -199,3 +199,7 @@ tests: steps: cluster_profile: aws workflow: origin-e2e +- as: e2e-baremetalds + steps: + cluster_profile: packet + workflow: baremetalds-e2e \ No newline at end of file diff --git a/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml b/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml index 840312cfc873a..16e73b2fa6565 100644 --- a/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml +++ b/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml @@ -1222,6 +1222,93 @@ presubmits: secret: secretName: sentry-dsn trigger: (?m)^/test( | .* )e2e-azure-upi,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - master + context: ci/prow/e2e-baremetalds + decorate: true + decoration_config: + skip_cloning: true + labels: + ci-operator.openshift.io/prowgen-controlled: "true" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-installer-master-e2e-baremetalds + optional: true + rerun_command: /test e2e-baremetalds + skip_report: true + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --branch=master + - --give-pr-author-access-to-namespace=true + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --kubeconfig=/etc/apici/kubeconfig + - --lease-server-password-file=/etc/boskos/password + - --lease-server-username=ci + - --lease-server=https://boskos-ci.svc.ci.openshift.org + - --org=openshift + - --repo=installer + - --resolver-address=http://ci-operator-configresolver-ci.svc.ci.openshift.org + - --secret-dir=/usr/local/e2e-baremetalds-cluster-profile + - --sentry-dsn-path=/etc/sentry-dsn/ci-operator + - --target=e2e-baremetalds + command: + - ci-operator + env: + - name: CONFIG_SPEC + valueFrom: + configMapKeyRef: + key: openshift-installer-master.yaml + name: ci-operator-master-configs + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/apici + name: apici-ci-operator-credentials + readOnly: true + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /usr/local/e2e-baremetalds-cluster-profile + name: cluster-profile + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/sentry-dsn + name: sentry-dsn + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: apici-ci-operator-credentials + secret: + items: + - key: sa.ci-operator.apici.config + path: kubeconfig + secretName: apici-ci-operator-credentials + - name: boskos + secret: + items: + - key: password + path: password + secretName: boskos-credentials + - name: cluster-profile + projected: + sources: + - secret: + name: cluster-secrets-metal + - name: pull-secret + secret: + secretName: regcred + - name: sentry-dsn + secret: + secretName: sentry-dsn + trigger: (?m)^/test( | .* )e2e-baremetalds,?($|\s.*) - agent: kubernetes always_run: false branches: diff --git a/ci-operator/step-registry/baremetalds/OWNERS b/ci-operator/step-registry/baremetalds/OWNERS new file mode 100644 index 0000000000000..3ba207e71e301 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/OWNERS @@ -0,0 +1,5 @@ +approvers: +- andfasano +- akiselev1 +- derekhiggins +- honza diff --git a/ci-operator/step-registry/baremetalds/devscripts/OWNERS b/ci-operator/step-registry/baremetalds/devscripts/OWNERS new file mode 100644 index 0000000000000..3ba207e71e301 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/devscripts/OWNERS @@ -0,0 +1,5 @@ +approvers: +- andfasano +- akiselev1 +- derekhiggins +- honza diff --git a/ci-operator/step-registry/baremetalds/devscripts/setup/OWNERS b/ci-operator/step-registry/baremetalds/devscripts/setup/OWNERS new file mode 100644 index 0000000000000..3ba207e71e301 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/devscripts/setup/OWNERS @@ -0,0 +1,5 @@ +approvers: +- andfasano +- akiselev1 +- derekhiggins +- honza diff --git a/ci-operator/step-registry/baremetalds/devscripts/setup/baremetalds-devscripts-setup-commands.sh b/ci-operator/step-registry/baremetalds/devscripts/setup/baremetalds-devscripts-setup-commands.sh new file mode 100644 index 0000000000000..c205d662f9246 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/devscripts/setup/baremetalds-devscripts-setup-commands.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +cluster_profile=/var/run/secrets/ci.openshift.io/cluster-profile + +export SSH_PRIV_KEY_PATH=${cluster_profile}/ssh-privatekey +export PULL_SECRET_PATH=${cluster_profile}/pull-secret +export SSHOPTS="-o ConnectTimeout=5 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=90 -i ${SSH_PRIV_KEY_PATH}" + +echo "************ baremetalds devscripts setup command ************" +env | sort + +# Ensure our UID, which is randomly generated, is in /etc/passwd. This is required +# to be able to SSH. +if ! whoami &> /dev/null; then + if [[ -w /etc/passwd ]]; then + echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd + else + echo "/etc/passwd is not writeable, and user matching this uid is not found." + exit 1 + fi +fi + +# Initial check +if [ "${CLUSTER_TYPE}" != "packet" ] ; then + echo >&2 "Unsupported cluster type '${CLUSTER_TYPE}'" + exit 1 +fi + +echo "-------[ $SHARED_DIR ]" +ls -ll ${SHARED_DIR} + +# Fetch packet server IP +IP=$(cat ${SHARED_DIR}/server-ip) +export IP +echo "Packet server IP is ${IP}" + +# Checkout dev-scripts and make +for x in $(seq 10) ; do + test $x == 10 && exit 1 + ssh $SSHOPTS root@$IP hostname && break + sleep 10 +done + +# Get dev-scripts logs +finished() +{ + set +e + + # Get dev-scripts logs + echo "dev-scripts setup completed, fetching logs" + ssh $SSHOPTS root@$IP tar -czf - /root/dev-scripts/logs | tar -C ${ARTIFACT_DIR} -xzf - + sed -i -e 's/.*auths.*/*** PULL_SECRET ***/g' ${ARTIFACT_DIR}/root/dev-scripts/logs/* +} +trap finished EXIT TERM + +# Prepare configuration and run dev-scripts +scp $SSHOPTS ${PULL_SECRET_PATH} root@$IP:pull-secret + +timeout -s 9 175m ssh $SSHOPTS root@$IP bash - << EOF |& sed -e 's/.*auths.*/*** PULL_SECRET ***/g' + +set -ex + +#### For debug only, to be removed #################################### +curl https://github.com/derekhiggins.keys >> /root/.ssh/authorized_keys +curl https://github.com/andfasano.keys >> /root/.ssh/authorized_keys +curl https://github.com/russellb.keys >> /root/.ssh/authorized_keys +curl https://github.com/stbenjam.keys >> /root/.ssh/authorized_keys +curl https://github.com/honza.keys >> /root/.ssh/authorized_keys +####################################################################### + +yum install -y git + +mkdir -p /tmp/artifacts + +if [ ! -e dev-scripts ] ; then + git clone https://github.com/openshift-metal3/dev-scripts.git +fi +cd dev-scripts + +set +x +echo "export PULL_SECRET='\$(cat /root/pull-secret)'" > /root/dev-scripts/config_root.sh +set -x + +curl https://mirror.openshift.com/pub/openshift-v4/clients/oc/4.4/linux/oc.tar.gz | tar -C /usr/bin -xzf - + +#echo "export OPENSHIFT_RELEASE_IMAGE=registry.svc.ci.openshift.org/ocp/release:4.5.0-0.ci-2020-03-11-134608" >> /root/dev-scripts/config_root.sh +#echo "export OPENSHIFT_RELEASE_IMAGE=registry.svc.ci.openshift.org/${NAMESPACE}/release:latest" >> /root/dev-scripts/config_root.sh +echo "export OPENSHIFT_RELEASE_IMAGE=${RELEASE_IMAGE_LATEST}" >> /root/dev-scripts/config_root.sh + +echo "export ADDN_DNS=\$(awk '/nameserver/ { print \$2;exit; }' /etc/resolv.conf)" >> /root/dev-scripts/config_root.sh +echo "export OPENSHIFT_CI=true" >> /root/dev-scripts/config_root.sh +echo "export MIRROR_IMAGES=true" >> /root/dev-scripts/config_root.sh + +echo 'export KUBECONFIG=/root/dev-scripts/ocp/ostest/auth/kubeconfig' >> /root/.bashrc + +if [ ! -e /opt/dev-scripts/pool ] ; then + mkdir -p /opt/dev-scripts/pool + mount -t tmpfs -o size=100G tmpfs /opt/dev-scripts/pool +fi + +timeout -s 9 105m make + +EOF + + + + + + + diff --git a/ci-operator/step-registry/baremetalds/devscripts/setup/baremetalds-devscripts-setup-ref.yaml b/ci-operator/step-registry/baremetalds/devscripts/setup/baremetalds-devscripts-setup-ref.yaml new file mode 100644 index 0000000000000..2f34b25663d75 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/devscripts/setup/baremetalds-devscripts-setup-ref.yaml @@ -0,0 +1,10 @@ +ref: + as: baremetalds-devscripts-setup + from: stable:baremetal-installer + commands: baremetalds-devscripts-setup-commands.sh + resources: + requests: + cpu: 1000m + memory: 500Mi + documentation: |- + Checkout and execute dev-scripts on a packet server diff --git a/ci-operator/step-registry/baremetalds/e2e/OWNERS b/ci-operator/step-registry/baremetalds/e2e/OWNERS new file mode 100644 index 0000000000000..3ba207e71e301 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/e2e/OWNERS @@ -0,0 +1,5 @@ +approvers: +- andfasano +- akiselev1 +- derekhiggins +- honza diff --git a/ci-operator/step-registry/baremetalds/e2e/baremetalds-e2e-workflow.yaml b/ci-operator/step-registry/baremetalds/e2e/baremetalds-e2e-workflow.yaml new file mode 100644 index 0000000000000..74a013bfde9ad --- /dev/null +++ b/ci-operator/step-registry/baremetalds/e2e/baremetalds-e2e-workflow.yaml @@ -0,0 +1,13 @@ +workflow: + as: baremetalds-e2e + steps: + pre: + - ref: baremetalds-packet-setup + - ref: ipi-install-rbac + - ref: baremetalds-devscripts-setup + test: + - ref: baremetalds-e2e-test + post: + - ref: baremetalds-packet-teardown + documentation: |- + This workflow executes the common end-to-end test suite on a cluster provisioned by running dev-scripts on a packet server. diff --git a/ci-operator/step-registry/baremetalds/e2e/test/OWNERS b/ci-operator/step-registry/baremetalds/e2e/test/OWNERS new file mode 100644 index 0000000000000..3ba207e71e301 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/e2e/test/OWNERS @@ -0,0 +1,5 @@ +approvers: +- andfasano +- akiselev1 +- derekhiggins +- honza diff --git a/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-commands.sh b/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-commands.sh new file mode 100644 index 0000000000000..1085fcffeb0fe --- /dev/null +++ b/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-commands.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +cluster_profile=/var/run/secrets/ci.openshift.io/cluster-profile + +export SSH_PRIV_KEY_PATH=${cluster_profile}/ssh-privatekey +export SSHOPTS="-o ConnectTimeout=5 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=90 -i ${SSH_PRIV_KEY_PATH}" + +echo "************ baremetalds test command ************" +env | sort + +# Ensure our UID, which is randomly generated, is in /etc/passwd. This is required +# to be able to SSH. +if ! whoami &> /dev/null; then + if [[ -w /etc/passwd ]]; then + echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd + else + echo "/etc/passwd is not writeable, and user matching this uid is not found." + exit 1 + fi +fi + +# Initial check +if [ "${CLUSTER_TYPE}" != "packet" ] ; then + echo >&2 "Unsupported cluster type '${CLUSTER_TYPE}'" + exit 0 +fi + +echo "-------[ $SHARED_DIR ]" +ls -ll ${SHARED_DIR} + +IP=$(cat ${SHARED_DIR}/server-ip) +export IP + +# Copy test binaries on packet server +echo "### Copying test binaries" +scp $SSHOPTS /usr/bin/openshift-tests /usr/bin/kubectl root@$IP:/usr/local/bin + +# # List of exclude cases +# echo "### Preparing filter" +# read -d '*' EXCL << EOF +# sig-storage +# custom build with buildah being created from new-build +# docker build using a pull secret Building from a template +# prune builds based on settings in the buildconfig +# result image should have proper labels set +# Image policy +# deploymentconfigs adoption +# Alerts +# templateinstance readiness test +# oc adm must-gather +# capture build stages and durations +# deploymentconfigs with multiple image change triggers +# Managed cluster should +# forcePull should affect pulling builder images +# s2i build with a root user image +# Networking Granular Checks: Services +# Image layer subresource +# openshift mongodb image creating from a template +# capture build stages and durations +# process valueFrom in build strategy environment variables +# result image should have proper labels set S2I build from a template +# oc new-app +# Image append +# oc tag +# forcePull should affect pulling builder images +# templateinstance readiness test +# Multi-stage image builds +# Image extract +# TestDockercfgTokenDeletedController +# process valueFrom in build strategy environment variables +# Prometheus when installed on the cluster +# build can reference a cluster service with a build being created from new-build +# deploymentconfigs with multiple image change triggers +# deploymentconfigs should respect image stream tag reference policy +# * +# EOF + +# Tests execution +set +e +echo "### Running tests" +ssh $SSHOPTS root@$IP openshift-tests run "openshift/conformance/parallel" --dry-run \| grep 'Feature:ProjectAPI' \| openshift-tests run -o /tmp/artifacts/e2e.log --junit-dir /tmp/artifacts/junit -f - +rv=$? + +echo "### Fetching results" +ssh $SSHOPTS root@$IP tar -czf - /tmp/artifacts | tar -C ${ARTIFACT_DIR} -xzf - +set -e +echo "### Done! (${rv})" +exit $rv diff --git a/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-ref.yaml b/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-ref.yaml new file mode 100644 index 0000000000000..72d2789a63e99 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/e2e/test/baremetalds-e2e-test-ref.yaml @@ -0,0 +1,12 @@ +ref: + as: baremetalds-e2e-test + from: stable:tests + commands: baremetalds-e2e-test-commands.sh + resources: + requests: + cpu: "3" + memory: 600Mi + limits: + memory: 2Gi + documentation: |- + The Baremtal DS E2E step executes the common end-to-end test suite. diff --git a/ci-operator/step-registry/baremetalds/packet/OWNERS b/ci-operator/step-registry/baremetalds/packet/OWNERS new file mode 100644 index 0000000000000..3ba207e71e301 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/packet/OWNERS @@ -0,0 +1,5 @@ +approvers: +- andfasano +- akiselev1 +- derekhiggins +- honza diff --git a/ci-operator/step-registry/baremetalds/packet/setup/OWNERS b/ci-operator/step-registry/baremetalds/packet/setup/OWNERS new file mode 100644 index 0000000000000..3ba207e71e301 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/packet/setup/OWNERS @@ -0,0 +1,5 @@ +approvers: +- andfasano +- akiselev1 +- derekhiggins +- honza diff --git a/ci-operator/step-registry/baremetalds/packet/setup/baremetalds-packet-setup-commands.sh b/ci-operator/step-registry/baremetalds/packet/setup/baremetalds-packet-setup-commands.sh new file mode 100644 index 0000000000000..a0f25a450b85f --- /dev/null +++ b/ci-operator/step-registry/baremetalds/packet/setup/baremetalds-packet-setup-commands.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +cluster_profile=/var/run/secrets/ci.openshift.io/cluster-profile + +export CLUSTER_NAME=${NAMESPACE}-${JOB_NAME_HASH} + +set +x +export PACKET_PROJECT_ID=b3c1623c-ce0b-45cf-9757-c61a71e06eac +PACKET_AUTH_TOKEN=$(cat ${cluster_profile}/.packetcred) +export PACKET_AUTH_TOKEN +set -x + +echo "************ baremetalds packet setup command ************" +env | sort + +# Initial check +if [ "${CLUSTER_TYPE}" != "packet" ] ; then + echo >&2 "Unsupported cluster type '${CLUSTER_TYPE}'" + exit 1 +fi + +# Terraform setup and init for packet server +terraform_home=${ARTIFACT_DIR}/terraform +mkdir -p ${terraform_home} +cd ${terraform_home} + +cat > ${terraform_home}/terraform.tf <<-EOF +provider "packet" { +} + +resource "packet_device" "server" { + count = "1" + project_id = "$PACKET_PROJECT_ID" + hostname = "ipi-$CLUSTER_NAME" + plan = "m2.xlarge.x86" + facilities = ["sjc1", "ewr1"] + operating_system = "centos_8" + billing_cycle = "hourly" +} +EOF + +terraform init + +# Packet returns transients errors when creating devices. +# example, `Oh snap, something went wrong! We've logged the error and will take a look - please reach out to us if you continue having trouble.` +# therefore the terraform apply needs to be retried a few time before giving up. +rc=1 +# shellcheck disable=SC20347 +for _ in {1..5}; do terraform apply -auto-approve && rc=0 && break ; done +if test "${rc}" -eq 1; then + echo >&2 "Failed to create packet server" + exit 1 +fi + +# Sharing terraform artifacts required by teardown +cp ${terraform_home}/terraform.* ${SHARED_DIR} + +# Sharing artifacts required by other steps +jq -r '.modules[0].resources["packet_device.server"].primary.attributes.access_public_ipv4' terraform.tfstate > /tmp/server-ip +cp /tmp/server-ip ${SHARED_DIR} + + + diff --git a/ci-operator/step-registry/baremetalds/packet/setup/baremetalds-packet-setup-ref.yaml b/ci-operator/step-registry/baremetalds/packet/setup/baremetalds-packet-setup-ref.yaml new file mode 100644 index 0000000000000..c6b8a3f7fc3f2 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/packet/setup/baremetalds-packet-setup-ref.yaml @@ -0,0 +1,12 @@ +ref: + as: baremetalds-packet-setup + from: stable:upi-installer + commands: baremetalds-packet-setup-commands.sh + resources: + requests: + cpu: 1000m + memory: 600Mi + limits: + memory: 2Gi + documentation: |- + This step performs the setup of the packet server diff --git a/ci-operator/step-registry/baremetalds/packet/teardown/OWNERS b/ci-operator/step-registry/baremetalds/packet/teardown/OWNERS new file mode 100644 index 0000000000000..3ba207e71e301 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/packet/teardown/OWNERS @@ -0,0 +1,5 @@ +approvers: +- andfasano +- akiselev1 +- derekhiggins +- honza diff --git a/ci-operator/step-registry/baremetalds/packet/teardown/baremetalds-packet-teardown-commands.sh b/ci-operator/step-registry/baremetalds/packet/teardown/baremetalds-packet-teardown-commands.sh new file mode 100644 index 0000000000000..8b1782c8e5419 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/packet/teardown/baremetalds-packet-teardown-commands.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +cluster_profile=/var/run/secrets/ci.openshift.io/cluster-profile +export SSH_PRIV_KEY_PATH=${cluster_profile}/ssh-privatekey + +set +x +export SSHOPTS="-o ConnectTimeout=5 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=90 -i ${SSH_PRIV_KEY_PATH}" +PACKET_AUTH_TOKEN=$(cat ${cluster_profile}/.packetcred) +export PACKET_AUTH_TOKEN +set -x + +echo "************ baremetalds packet teardown command ************" +env | sort + +# Initial check +if [ "${CLUSTER_TYPE}" != "packet" ] ; then + echo >&2 "Unsupported cluster type '${CLUSTER_TYPE}'" + exit 1 +fi + +echo "-------[ $SHARED_DIR ]" +ls -ll ${SHARED_DIR} + +# Shutdown packet server +terraform_home=${ARTIFACT_DIR}/terraform +mkdir -p ${terraform_home} +cp ${SHARED_DIR}/terraform.* ${terraform_home} +echo "Deprovisioning cluster..." +cd ${terraform_home} +terraform init +# shellcheck disable=SC2034 +for r in {1..5}; do terraform destroy -auto-approve && break ; done + + + diff --git a/ci-operator/step-registry/baremetalds/packet/teardown/baremetalds-packet-teardown-ref.yaml b/ci-operator/step-registry/baremetalds/packet/teardown/baremetalds-packet-teardown-ref.yaml new file mode 100644 index 0000000000000..2533f83f50f2f --- /dev/null +++ b/ci-operator/step-registry/baremetalds/packet/teardown/baremetalds-packet-teardown-ref.yaml @@ -0,0 +1,12 @@ +ref: + as: baremetalds-packet-teardown + from: stable:upi-installer + commands: baremetalds-packet-teardown-commands.sh + resources: + requests: + cpu: 1000m + memory: 600Mi + limits: + memory: 2Gi + documentation: |- + This step deprovisions the packet server