From d69407a003cf16106f20fe9a880fdf3e3258cc76 Mon Sep 17 00:00:00 2001 From: Derek Higgins Date: Thu, 12 Sep 2019 11:15:57 +0100 Subject: [PATCH] WIP: Add e2e ipi job Add a job to test installer end to end on a baremetal packet host. --- ...openshift-installer-master-presubmits.yaml | 64 +++++ .../cluster-launch-installer-ipi-e2e.yaml | 263 ++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 ci-operator/templates/openshift/installer/cluster-launch-installer-ipi-e2e.yaml diff --git a/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml b/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml index bba4649ca0edf..90111d37f0011 100644 --- a/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml +++ b/ci-operator/jobs/openshift/installer/openshift-installer-master-presubmits.yaml @@ -726,6 +726,70 @@ presubmits: secret: secretName: sentry-dsn trigger: (?m)^/test( | .* )e2e-gcp,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - master + context: ci/prow/e2e-ipi + decorate: true + decoration_config: + skip_cloning: true + labels: + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-installer-master-e2e-ipi + rerun_command: /test e2e-ipi + spec: + containers: + - args: + - --artifact-dir=$(ARTIFACTS) + - --give-pr-author-access-to-namespace=true + - --secret-dir=/usr/local/e2e-ipi-cluster-profile + - --sentry-dsn-path=/etc/sentry-dsn/ci-operator + - --target=e2e-ipi + - --template=/usr/local/e2e-ipi + command: + - ci-operator + env: + - name: CLUSTER_TYPE + value: aws + - name: CONFIG_SPEC + valueFrom: + configMapKeyRef: + key: openshift-installer-master.yaml + name: ci-operator-master-configs + - name: JOB_NAME_SAFE + value: e2e-ipi + - name: TEST_COMMAND + value: echo 1 + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /usr/local/e2e-ipi-cluster-profile + name: cluster-profile + - mountPath: /usr/local/e2e-ipi + name: job-definition + subPath: cluster-launch-installer-ipi-e2e.yaml + - mountPath: /etc/sentry-dsn + name: sentry-dsn + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: cluster-profile + projected: + sources: + - secret: + name: cluster-secrets-metal + - configMap: + name: prow-job-cluster-launch-installer-upi-e2e + name: job-definition + - name: sentry-dsn + secret: + secretName: sentry-dsn + trigger: (?m)^/test( | .* )e2e-ipi,?($|\s.*) - agent: kubernetes always_run: false branches: diff --git a/ci-operator/templates/openshift/installer/cluster-launch-installer-ipi-e2e.yaml b/ci-operator/templates/openshift/installer/cluster-launch-installer-ipi-e2e.yaml new file mode 100644 index 0000000000000..dd409aae95217 --- /dev/null +++ b/ci-operator/templates/openshift/installer/cluster-launch-installer-ipi-e2e.yaml @@ -0,0 +1,263 @@ +kind: Template +apiVersion: template.openshift.io/v1 + +parameters: +- name: JOB_NAME_SAFE + required: true +- name: JOB_NAME_HASH + required: true +- name: NAMESPACE + required: true +- name: IMAGE_LIBVIRT_INSTALLER + required: true +- name: IMAGE_UPI_INSTALLER + required: true +- name: CLUSTER_TYPE + required: true +- name: RELEASE_IMAGE_LATEST + required: true +- name: BUILD_ID + required: false + +objects: + +# We want the cluster to be able to access these images +- kind: RoleBinding + apiVersion: authorization.openshift.io/v1 + metadata: + name: ${JOB_NAME_SAFE}-image-puller + namespace: ${NAMESPACE} + roleRef: + name: system:image-puller + subjects: + - kind: SystemGroup + name: system:unauthenticated + - kind: SystemGroup + name: system:authenticated + +# Give admin access to a known bot +- kind: RoleBinding + apiVersion: authorization.openshift.io/v1 + metadata: + name: ${JOB_NAME_SAFE}-namespace-admins + namespace: ${NAMESPACE} + roleRef: + name: admin + subjects: + - kind: ServiceAccount + namespace: ci + name: ci-chat-bot + +# The e2e pod spins up a cluster, runs e2e tests, and then cleans up the cluster. +- kind: Pod + apiVersion: v1 + metadata: + name: ${JOB_NAME_SAFE} + namespace: ${NAMESPACE} + annotations: + # we want to gather the teardown logs no matter what + ci-operator.openshift.io/wait-for-container-artifacts: teardown + ci-operator.openshift.io/save-container-logs: "true" + spec: + restartPolicy: Never + activeDeadlineSeconds: 14400 + terminationGracePeriodSeconds: 900 + volumes: + - name: shared-ignition-files + emptyDir: {} + - name: artifacts + emptyDir: {} + - name: shared-tmp + emptyDir: {} + - name: cluster-profile + secret: + secretName: ${JOB_NAME_SAFE}-cluster-profile + + containers: + + # Runs an install + - name: setup + image: ${IMAGE_UPI_INSTALLER} + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: shared-tmp + mountPath: /tmp + - name: cluster-profile + mountPath: /etc/openshift-installer + - name: artifacts + mountPath: /tmp/artifacts + env: + - name: CLUSTER_NAME + value: ${NAMESPACE}-${JOB_NAME_HASH} + - name: SSH_PUB_KEY_PATH + value: /etc/openshift-installer/ssh-publickey + - name: SSH_PRIVATE_KEY_PATH + value: /etc/openshift-installer/ssh-privatekey + - name: PACKET_PROJECT_ID + value: b3c1623c-ce0b-45cf-9757-c61a71e06eac + - name: PULL_SECRET_PATH + value: /etc/openshift-installer/pull-secret + command: + - /bin/sh + - -c + - | + #!/bin/sh + set -e + + set -x + + finished() + { + set +e + + echo "Deprovisioning cluster ..." + cd /tmp/artifacts/terraform + terraform init + for r in {1..5}; do terraform destroy -auto-approve && break ; done + } + trap finished EXIT TERM + + mkdir -p /tmp/artifacts/terraform /tmp/shared || true + cd /tmp/artifacts/terraform + + set +x + export PACKET_AUTH_TOKEN=$(cat /etc/openshift-installer/.packetcred) + set -x + + cat > /tmp/artifacts/terraform/terraform.tf <<-EOF + provider "packet" { + } + + resource "packet_device" "server" { + count = "1" + project_id = "$PACKET_PROJECT_ID" + hostname = "ipi-$CLUSTER_NAME" + plan = "c2.medium.x86" + facilities = ["ewr1", "ewr1", "sjc1"] + operating_system = "centos_7" + billing_cycle = "hourly" + } + + EOF + + terraform init + # Packet returns transients errors when creating devices. + # example, `Oh snap, something went wrong! We've logged the error and will take a look - please reach out to us if you continue having trouble.` + # therefore the terraform apply needs to be retried a few time before giving up. + rc=1 + for r in {1..5}; do terraform apply -auto-approve && rc=0 && break ; done + if test "${rc}" -eq 1; then echo "failed to create the infra resources"; sleep 1; fi + + jq -r '.modules[0].resources["packet_device.server"].primary.attributes.access_public_ipv4' terraform.tfstate > /tmp/IP + + touch /tmp/ready + while [ ! -f /tmp/exit ] ; do sleep 1 ; done + + # ssh container + - name: dotest + image: ${IMAGE_LIBVIRT_INSTALLER} + terminationMessagePolicy: FallbackToLogsOnError + resources: + requests: + cpu: 1 + memory: 300Mi + limits: + memory: 3Gi + volumeMounts: + - name: shared-tmp + mountPath: /tmp + - name: cluster-profile + mountPath: /etc/openshift-installer + - name: artifacts + mountPath: /tmp/artifacts + env: + - name: HOME + value: /tmp/packer + - name: SSH_PUB_KEY_PATH + value: /etc/openshift-installer/ssh-publickey + - name: SSH_PRIVATE_KEY_PATH + value: /etc/openshift-installer/ssh-privatekey + - name: NAMESPACE + value: ${NAMESPACE} + - name: PULL_SECRET_PATH + value: /etc/openshift-installer/pull-secret + - name: NSS_WRAPPER_PASSWD + value: /tmp/packer/passwd + - name: NSS_WRAPPER_GROUP + value: /tmp/packer/group + - name: NSS_USERNAME + value: packer + - name: NSS_GROUPNAME + value: packer + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -xeuo pipefail + + mkdir -p /tmp/packer + mock-nss.sh + + export LD_PRELOAD=/usr/lib64/libnss_wrapper.so + + for x in $(seq 120) ; do + test $x == 120 && exit 1 + test -f /tmp/ready && break + sleep 10 + done + + finished() + { + set +e + + if [ -n "$IP" ] ; then + echo "Getting logs" + ssh $SSHOPTS root@$IP tar -czf - /root/dev-scripts/logs | tar -C /tmp/artifacts -xzf - + sed -i -e 's/.*auths.*/*** PULL_SECRET ***/g' /tmp/artifacts/root/dev-scripts/logs/* + fi + + touch /tmp/exit + } + + SSHOPTS="-o ConnectTimeout=5 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i $SSH_PRIVATE_KEY_PATH" + trap finished EXIT + + + set +x + export PACKET_AUTH_TOKEN=$(cat /etc/openshift-installer/.packetcred) + set -x + + export IP=$(cat /tmp/IP) + + for x in $(seq 10) ; do + test $x == 10 && exit 1 + ssh $SSHOPTS root@$IP hostname && break + sleep 10 + done + + scp $SSHOPTS ${PULL_SECRET_PATH} root@$IP:pull-secret + timeout -s 9 175m ssh $SSHOPTS root@$IP bash - << EOF + set -ex + + yum install -y git + + export OPENSHIFT_RELEASE_IMAGE=registry.svc.ci.openshift.org/$NAMESPACE/release:latest + set +x + export PULL_SECRET=\$(cat pull-secret) + set -x + + # python2-cryptography needs to come from delorean-master-testing, priority of packet.repo overrides it + # remove the priority and instead ensure the packet repo is named first alphabetically + # this way it is prefered but it isn't a hard override when newer versions are found elsewhere + sed -i -e 's/priority.*//g' /etc/yum.repos.d/packet.repo + sed -i -e 's/packet-/a_packet-/g' /etc/yum.repos.d/packet.repo + + export ADDN_DNS=\$(awk '/nameserver/ { print \$2;exit; }' /etc/resolv.conf) + + git clone https://github.com/openshift-metal3/dev-scripts.git + cd dev-scripts + touch /root/dev-scripts/config_root.sh + timeout -s 9 105m make |& sed -e 's/.*auths.*/*** PULL_SECRET ***/g' + + EOF