diff --git a/docs/contributor/multinode/setup.md b/docs/contributor/multinode/setup.md index 459033307e..c9fd867c05 100644 --- a/docs/contributor/multinode/setup.md +++ b/docs/contributor/multinode/setup.md @@ -34,18 +34,17 @@ and run it remotely. ``` git clone https://github.com/openshift/microshift.git ~/microshift cd ~/microshift/ -scp -o StrictHostKeyChecking=no ./scripts/multinode/configure-pri.sh redhat@${PRI_ADDR}: -ssh redhat@${PRI_ADDR} ./configure-pri.sh "${PRI_HOST}" "${PRI_ADDR}" "${SEC_HOST}" "${SEC_ADDR}" +scp -o StrictHostKeyChecking=no ./scripts/multinode/configure-node.sh redhat@${PRI_ADDR}: +ssh redhat@${PRI_ADDR} ./configure-node.sh ``` -If the configuration script runs successfully, it prints the list of the -`kubelet` configuration files that need to be copied to the secondary host. +If the configuration script runs successfully, it prints the location of a +bootstrap kubeconfig that will be used in the secondary host. -Copy the `kubelet` configuration files from the primary to the secondary host. +Copy the kubeconfig configuration file from the primary to the secondary host. ``` -scp -3 -o StrictHostKeyChecking=no \ - redhat@${PRI_ADDR}:/home/redhat/kubelet-${SEC_HOST}.{key,crt} \ - redhat@${PRI_ADDR}:/home/redhat/kubeconfig-${PRI_HOST} \ +scp -o StrictHostKeyChecking=no \ + redhat@${PRI_ADDR}:/home/redhat/kubeconfig-bootstrap \ redhat@${SEC_ADDR}: ``` @@ -53,18 +52,18 @@ Run the following commands to copy the configuration script to the secondary hos and run it remotely. ``` cd ~/microshift/ -scp -o StrictHostKeyChecking=no ./scripts/multinode/configure-sec.sh redhat@${SEC_ADDR}: -ssh redhat@${SEC_ADDR} ./configure-sec.sh "${PRI_HOST}" "${PRI_ADDR}" "${SEC_HOST}" "${SEC_ADDR}" +scp -o StrictHostKeyChecking=no ./scripts/multinode/configure-node.sh redhat@${SEC_ADDR}: +ssh redhat@${SEC_ADDR} "BOOTSTRAP_KUBECONFIG=~/kubeconfig-bootstrap ./configure-node.sh" ``` ## Run Tests -Before running tests, make sure that the `microshift-pri` host name is resolved -and accessible from the `hypervisor host`. +Before running tests, make sure that the `microshift-pri` host is accessible +from the `hypervisor host`. Set the `KUBECONFIG` variable using the configuration file from the primary host. ``` export KUBECONFIG=$(mktemp /tmp/microshift-kubeconfig.XXXXXXXXXX) -scp redhat@${PRI_ADDR}:/home/redhat/kubeconfig-${PRI_HOST} ${KUBECONFIG} +scp redhat@${PRI_ADDR}:/home/redhat/kubeconfig-bootstrap ${KUBECONFIG} ``` Verify that the cluster has **two** nodes in the `Ready` status and wait until @@ -76,4 +75,4 @@ watch oc get pods -A Run your test suite of choice using the primary and secondary MicroShift instances. -If you want to run [openshift-tests](https://github.com/openshift/origin) suite, follow [this](../openshift_ci.md#running-tests-manually) document. \ No newline at end of file +If you want to run [openshift-tests](https://github.com/openshift/origin) suite, follow [this](../openshift_ci.md#running-tests-manually) document. diff --git a/scripts/auto-rebase/rebase.sh b/scripts/auto-rebase/rebase.sh index 6241c5cdf3..3ffd6c6c74 100755 --- a/scripts/auto-rebase/rebase.sh +++ b/scripts/auto-rebase/rebase.sh @@ -1289,33 +1289,10 @@ rebase_to() { echo "No changes to buildfiles." fi - update_cncf_kubelet_version - if [[ -n "$(git status -s scripts/multinode/configure-sec.sh)" ]]; then - title "## Committing changes to scripts/multinode/configure-sec.sh" - git add scripts/multinode/configure-sec.sh - git commit -m "update kubernetes version in CNCF scripts" - else - echo "No changes to Kubernetes version." - fi - title "# Removing staging directory" rm -rf "${STAGING_DIR}" } -update_cncf_kubelet_version() { - title "Updating Kubernetes version in CNCF scripts" - - source "${REPOROOT}/Makefile.kube_git.var" - local -r kube_hash_amd64="$(curl -L https://dl.k8s.io/release/${KUBE_GIT_VERSION}/bin/linux/amd64/kubelet.sha256 2>/dev/null)" - local -r kube_hash_arm64="$(curl -L https://dl.k8s.io/release/${KUBE_GIT_VERSION}/bin/linux/arm64/kubelet.sha256 2>/dev/null)" - - local -r target="${REPOROOT}/scripts/multinode/configure-sec.sh" - sed -i "s,# version=v1\.[0-9]*\.[0-9]*;,# version=${KUBE_GIT_VERSION};,g" "${target}" - sed -i "s,local -r version=.*,local -r version=\"${KUBE_GIT_VERSION}\",g" "${target}" - sed -i "s,local -r kube_hash_amd64=.*,local -r kube_hash_amd64=\"${kube_hash_amd64}\",g" "${target}" - sed -i "s,local -r kube_hash_arm64=.*,local -r kube_hash_arm64=\"${kube_hash_arm64}\",g" "${target}" -} - to_just_images() { local release_image_amd64=$1 local release_image_arm64=$2 @@ -1353,7 +1330,6 @@ usage() { echo "$(basename "$0") generated-apis Regenerates OpenAPIs" echo "$(basename "$0") images Rebases the component images to the downloaded release" echo "$(basename "$0") manifests Rebases the component manifests to the downloaded release" - echo "$(basename "$0") cncf-kube-version Updates kubelet version in configure-sec.sh to match version in Makefile.kube_git.var" exit 1 } @@ -1385,8 +1361,5 @@ case "$command" in copy_manifests update_openshift_manifests ;; - cncf-kube-version) - update_cncf_kubelet_version - ;; *) usage;; esac diff --git a/scripts/multinode/configure-node.sh b/scripts/multinode/configure-node.sh new file mode 100755 index 0000000000..7aef43ebf6 --- /dev/null +++ b/scripts/multinode/configure-node.sh @@ -0,0 +1,161 @@ +#!/bin/bash +set -euo pipefail + +# Variables for node configuration +NODE_ADDR="" +BOOTSTRAP_KUBECONFIG="" + +function usage() { + echo "This script configures a node to run a MicroShift cluster." + echo "Optionally, it can also be used to configure MicroShift to join an existing cluster." + echo "Usage: $(basename "$0") [OPTIONS]" + echo "Options:" + echo " --bootstrap-kubeconfig PATH Path to kubeconfig file for joining existing cluster (optional)" + echo " -h, --help Show this help message" + exit 1 +} + +function configure_system() { + # TODO: Edit firewall rules instead of stopping firewall + sudo systemctl stop firewalld + sudo systemctl disable firewalld + + sudo systemctl stop greenboot-healthcheck + sudo systemctl reset-failed greenboot-healthcheck + sudo systemctl disable greenboot-healthcheck +} + +function configure_microshift() { + # Clean the current MicroShift configuration and stop the service + echo 1 | sudo microshift-cleanup-data --all --keep-images + + get_node_ip_from_config + + # Configure MicroShift to disable telemetry + cat </dev/null +apiServer: + subjectAltNames: + - ${NODE_ADDR} +telemetry: + status: Disabled +EOF + + sudo mkdir -p /etc/systemd/system/microshift.service.d + cat </dev/null +[Service] +# Clear previous ExecStart, otherwise systemd would try to run both. +ExecStart= +ExecStart=microshift run --multinode +EOF + sudo systemctl daemon-reload + sudo systemctl enable microshift.service +} + +function start_microshift() { + sudo systemctl start microshift.service +} + +function run_add_node_commands() { + if ! sudo microshift add-node --kubeconfig="${BOOTSTRAP_KUBECONFIG}"; then + echo "Error: Failed to add node using kubeconfig: ${BOOTSTRAP_KUBECONFIG}" + exit 1 + fi + echo "Successfully added node using kubeconfig: ${BOOTSTRAP_KUBECONFIG}" +} + +function get_node_ip_from_config() { + # Extract nodeIP from MicroShift running configuration and store in global variable + local node_ip="" + + # Use microshift show-config to get the IP address of the node + node_ip=$(sudo microshift show-config 2>/dev/null | awk '/^\s*nodeIP\s*:/ {print $NF; exit}') + + if [ -z "${node_ip}" ]; then + echo "Warning: nodeIP not found in MicroShift config" + exit 1 + fi + + NODE_ADDR="${node_ip}" +} + +function copy_bootstrap_kubeconfig() { + local kubeconfig_source="/var/lib/microshift/resources/kubeadmin/${NODE_ADDR}/kubeconfig" + local kubeconfig_dest="${HOME}/kubeconfig-bootstrap" + + if ! sudo test -f "${kubeconfig_source}"; then + echo "Error: Kubeconfig file not found at ${kubeconfig_source}" + exit 1 + fi + + if sudo cp "${kubeconfig_source}" "${kubeconfig_dest}"; then + sudo chown "$(whoami):" "${kubeconfig_dest}" + echo "Kubeconfig copied successfully to ${kubeconfig_dest}" + else + echo "Error: Failed to copy kubeconfig file" + exit 1 + fi +} + +function run_healthcheck() { + if ! sudo systemctl start greenboot-healthcheck; then + echo "Error: Failed to start greenboot-healthcheck service" + exit 1 + fi + + greenboot_status=$(systemctl show -p Result --value greenboot-healthcheck) + if [ "${greenboot_status}" != "success" ]; then + echo "Error: greenboot-healthcheck did not complete successfully (Result: ${greenboot_status})" + exit 1 + fi +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --bootstrap-kubeconfig) + if [ $# -lt 2 ]; then + echo "Error: --bootstrap-kubeconfig requires an argument" + usage + fi + BOOTSTRAP_KUBECONFIG="$2" + shift 2 + ;; + -h|--help) + usage + ;; + *) + echo "Error: Unknown option '$1'" + usage + ;; + esac +done + +# Validate BOOTSTRAP_KUBECONFIG if provided +if [ -n "${BOOTSTRAP_KUBECONFIG}" ]; then + if [ ! -f "${BOOTSTRAP_KUBECONFIG}" ]; then + echo "Error: Bootstrap kubeconfig file '${BOOTSTRAP_KUBECONFIG}' does not exist" + exit 1 + fi + echo "Using bootstrap kubeconfig: ${BOOTSTRAP_KUBECONFIG}" +fi + +configure_system +configure_microshift +if [ -n "${BOOTSTRAP_KUBECONFIG}" ]; then + run_add_node_commands + run_healthcheck +else + start_microshift +fi +echo +echo "Node configuration completed" +if [ -z "${BOOTSTRAP_KUBECONFIG}" ]; then + copy_bootstrap_kubeconfig + echo + echo "To add other nodes to this cluster, copy the following kubeconfig file to other nodes:" + echo " ${HOME}/kubeconfig-bootstrap" + echo + echo "Then run the following command on each node you want to add:" + echo " $(basename "$0") --bootstrap-kubeconfig /path/to/kubeconfig" +fi +echo "Done" diff --git a/scripts/multinode/configure-pri.sh b/scripts/multinode/configure-pri.sh deleted file mode 100755 index fdfd12d86a..0000000000 --- a/scripts/multinode/configure-pri.sh +++ /dev/null @@ -1,225 +0,0 @@ -#!/bin/bash -set -euo pipefail - -OC_CMD="sudo -i oc --kubeconfig /var/lib/microshift/resources/kubeadmin/kubeconfig" -KUBECTL_CMD="sudo -i kubectl --kubeconfig /var/lib/microshift/resources/kubeadmin/kubeconfig" - -PRI_NODE_HOST= -PRI_NODE_ADDR= -SEC_NODE_HOST= -SEC_NODE_ADDR= - -function usage() { - echo "Usage: $(basename "$0") " - exit 1 -} - -function configure_system() { - # Disable selinux - # TODO: remove once selinux is working properly again - sudo setenforce 0 || true - - # TODO: Edit firewall rules instead of stopping firewall - sudo systemctl stop firewalld - sudo systemctl disable firewalld - - # Greenboot checks are tuned for a single node - sudo systemctl stop greenboot-healthcheck - sudo systemctl reset-failed greenboot-healthcheck - sudo systemctl disable greenboot-healthcheck - - # Configure the MicroShift host name - sudo hostnamectl hostname "${PRI_NODE_HOST}" - - # Update /etc/hosts to resolve primary and secondary host names if not already resolved - if [ "$(getent ahostsv4 "${PRI_NODE_HOST}" | head -1 | awk '{print $1}')" != "${PRI_NODE_ADDR}" ] ; then - echo "${PRI_NODE_ADDR} ${PRI_NODE_HOST}" | sudo tee -a /etc/hosts &>/dev/null - fi - if [ "$(getent ahostsv4 "${SEC_NODE_HOST}" | head -1 | awk '{print $1}')" != "${SEC_NODE_ADDR}" ] ; then - echo "${SEC_NODE_ADDR} ${SEC_NODE_HOST}" | sudo tee -a /etc/hosts &>/dev/null - fi -} - -function configure_microshift() { - # Clean the current MicroShift configuration and stop the service - echo 1 | sudo microshift-cleanup-data --all --keep-images - - # Run OVN initialization script - sleep 5 - sudo systemctl start --wait microshift-ovs-init.service - - # OVN-K expects br-ex to have IP address assigned, add dummy IP to br-ex. - if ! ip addr show br-ex 2>/dev/null | grep -q '10.44.0.0/32'; then - sudo ip addr add 10.44.0.0/32 dev br-ex - fi - - # Configure MicroShift to advertise apiserver in the node IP - cat </dev/null -apiServer: - advertiseAddress: ${PRI_NODE_ADDR} -telemetry: - status: Disabled -EOF -} - -function wait_for_pod_ready() { - local pod_namespace=$1 - local pod_name=$2 - local service_ready=false - - echo "Waiting for MicroShift ${pod_name}@${pod_namespace} pod to be ready" - for _ in $(seq 300) ; do - if ${OC_CMD} wait --timeout=0s --for=condition=ready pod -n "${pod_namespace}" -l app="${pod_name}" &>/dev/null ; then - service_ready=true - break - fi - echo -n "." - sleep 1 - done - echo - - if ! ${service_ready} ; then - echo "Error: timed out waiting for MicroShift ${pod_name}@${pod_namespace} pod to be ready" - exit 1 - fi -} - -function generate_service_certs() { - local -r cfssl=$(mktemp /tmp/cfssl.XXXXXXXXXX) - local -r cfssl_json=$(mktemp /tmp/cfssl_json.XXXXXXXXXX) - local -r cfssl_sha=$(mktemp /tmp/cfssl_sha.XXXXXXXXXX) - local -r csr_file=$(mktemp /tmp/csrfile.XXXXXXXXXX) - local -r kubelet_csr=$(mktemp /tmp/kubelet.XXXXXXXXXX) - local cfssl_arch="" - - # Cleanup temporary files on exit - # shellcheck disable=SC2064 - trap "rm -f ${cfssl} ${cfssl_json} ${cfssl_sha} ${csr_file} ${kubelet_csr}*" EXIT - - # Install cfssl utilities - declare -A cfssl_map - case "$(uname -m)" in - x86_64) - cfssl_arch="amd64" - cfssl_map[cfssl]="b947d073e677189f8533704c44b2b1eae4042f5cefd2b8347d4d9b4c6a5008cf" - cfssl_map[cfssl_json]="d7c52a815f96ebd4fc857b012cee70b44751edabb55ae60c4b743ee09e67f4de" - ;; - aarch64) - cfssl_arch="arm64" - cfssl_map[cfssl]="453495690f9b4e811d195d1f214ae58ad281e2e50e6dc3ffb19a8c58ddbc8a51" - cfssl_map[cfssl_json]="4f68110f5a88a8b60382ff6b96008f55714636c31e5a06ab9c60855a1bd9bf47" - ;; - *) - echo "Unsupported cfssl architecture $(uname -m)" - exit 1 - esac - - curl -s -L -o "${cfssl}" "https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl_1.6.4_linux_${cfssl_arch}" - curl -s -L -o "${cfssl_json}" "https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssljson_1.6.4_linux_${cfssl_arch}" - cat < "${cfssl_sha}" -${cfssl_map[cfssl]} ${cfssl} -${cfssl_map[cfssl_json]} ${cfssl_json} -EOF - sha256sum --check "${cfssl_sha}" - chmod +x "${cfssl}" "${cfssl_json}" - - # Generate serving certs for secondary node's kubelet - cat < "${csr_file}" -{ -"CN": "system:node:${SEC_NODE_ADDR}", -"key": { - "algo": "rsa", - "size": 2048 -}, -"hosts": [ - "${SEC_NODE_HOST}", - "${SEC_NODE_ADDR}" -], -"names": [ - { - "O": "system:nodes" - } -] -} -EOF - - # Generate and apply the certificate - ${cfssl} genkey "${csr_file}" | ${cfssl_json} -bare "${kubelet_csr}" - - ${OC_CMD} apply -f - < "${KUBELET_HOME}/kubelet-${SEC_NODE_HOST}.crt" - cp "${kubelet_csr}-key.pem" "${KUBELET_HOME}/kubelet-${SEC_NODE_HOST}.key" - - # Copy the bootstrap kube configuration files - sudo cp "/var/lib/microshift/resources/kubeadmin/${PRI_NODE_HOST}/kubeconfig" "${KUBELET_HOME}/kubeconfig-${PRI_NODE_HOST}" - sudo chown "$(whoami)." "${KUBELET_HOME}/kubeconfig-${PRI_NODE_HOST}" - - # Copy lvmd configuration files for the second node - sudo cp /var/lib/microshift/lvms/lvmd.yaml "${KUBELET_HOME}/lvmd-${PRI_NODE_HOST}.yaml" - sudo chown "$(whoami)." "${KUBELET_HOME}/lvmd-${PRI_NODE_HOST}.yaml" -} - -# -# Main function -# -if [ $# -ne 4 ] ; then - usage -fi -PRI_NODE_HOST=$1 -PRI_NODE_ADDR=$2 -SEC_NODE_HOST=$3 -SEC_NODE_ADDR=$4 - -KUBELET_HOME="${HOME}" -if [ ! -w "${KUBELET_HOME}" ] ; then - echo "The ${KUBELET_HOME} directory is not writable" - exit 1 -fi - -# Configure system for the multinode environment -configure_system - -# Configure MicroShift for the multinode environment -configure_microshift - -# Run MicroShift in the multinode mode -sudo mkdir -p /etc/systemd/system/microshift.service.d -cat </dev/null -[Service] -# Clear previous ExecStart, otherwise systemd would try to run both. -ExecStart= -ExecStart=microshift run --multinode -EOF -sudo systemctl daemon-reload -sudo systemctl start microshift.service - -# Wait for the service-ca pod to be ready -wait_for_pod_ready "openshift-service-ca" "service-ca" - -# Generate the certificates for the multinode environment -generate_service_certs - -# Print the file names to be copied to the secondary host -echo -echo "Copy the following files to the ${SEC_NODE_HOST} host" -ls -1 "${KUBELET_HOME}/kubeconfig-${PRI_NODE_HOST}" -ls -1 "${KUBELET_HOME}/kubelet-${SEC_NODE_HOST}".{key,crt} -ls -1 "${KUBELET_HOME}/lvmd-${PRI_NODE_HOST}.yaml" - -echo -echo "Done" diff --git a/scripts/multinode/configure-sec.sh b/scripts/multinode/configure-sec.sh deleted file mode 100755 index 194703a097..0000000000 --- a/scripts/multinode/configure-sec.sh +++ /dev/null @@ -1,239 +0,0 @@ -#!/bin/bash -set -euo pipefail - -OC_CMD="sudo -i oc --kubeconfig /var/lib/microshift/resources/kubeadmin/kubeconfig" - -PRI_NODE_HOST= -PRI_NODE_ADDR= -SEC_NODE_HOST= -SEC_NODE_ADDR= - -function usage() { - echo "Usage: $(basename "$0") " - exit 1 -} - -function configure_system() { - # Disable selinux - # TODO: remove once selinux is working properly again - sudo setenforce 0 || true - - # TODO: Edit firewall rules instead of stopping firewall - sudo systemctl stop firewalld - sudo systemctl disable firewalld - - # Greenboot checks are tuned for a single node - sudo systemctl stop greenboot-healthcheck - sudo systemctl reset-failed greenboot-healthcheck - sudo systemctl disable greenboot-healthcheck - - # Configure the MicroShift host name - sudo hostnamectl hostname "${SEC_NODE_HOST}" - - # Update /etc/hosts to resolve primary and secondary host names if not already resolved - if [ "$(getent ahostsv4 "${PRI_NODE_HOST}" | head -1 | awk '{print $1}')" != "${PRI_NODE_ADDR}" ] ; then - echo "${PRI_NODE_ADDR} ${PRI_NODE_HOST}" | sudo tee -a /etc/hosts &>/dev/null - fi - if [ "$(getent ahostsv4 "${SEC_NODE_HOST}" | head -1 | awk '{print $1}')" != "${SEC_NODE_ADDR}" ] ; then - echo "${SEC_NODE_ADDR} ${SEC_NODE_HOST}" | sudo tee -a /etc/hosts &>/dev/null - fi -} - -function configure_microshift() { - # Clean the current MicroShift configuration and stop the service - echo 1 | sudo microshift-cleanup-data --all --keep-images - - # Run OVN initialization script - sleep 5 - sudo systemctl start --wait microshift-ovs-init.service - - # OVN-K expects br-ex to have IP address assigned, add dummy IP to br-ex. - if ! ip addr show br-ex 2>/dev/null | grep -q '10.44.0.0/32'; then - sudo ip addr add 10.44.0.0/32 dev br-ex - fi - - # Stop and unload the kubelet service - if [ "$(systemctl is-active kubelet.service)" = "active" ] ; then - sudo systemctl stop --now kubelet - fi - sudo systemctl reset-failed kubelet || true - # Make sure the kubelet process is terminated - sudo pkill -9 --exact kubelet || true - until ! pidof kubelet &>/dev/null ; do - sleep 1 - done - # Clean up the old kubelet data - for dir in $(mount | awk '{print $3}' | grep ^/var/lib/kubelet/) ; do - sudo umount "${dir}" - done - sudo rm -rf /var/lib/kubelet - # Remove the kubelet service unit - sudo find /run/systemd -name kubelet.service -exec rm -f {} \; - sudo systemctl daemon-reload -} - -function configure_kubelet() { - # Download the kubelet executable - - # Checksums can be obtained from https://www.downloadkubernetes.com/ - # or by downloading a "${url}.sha256" file (see below for ${url}). For example: - # version=v1.33.5; for kube_arch in amd64 arm64; do echo "${kube_arch}: $(curl -L https://dl.k8s.io/release/${version}/bin/linux/${kube_arch}/kubelet.sha256 2>/dev/null)"; done - local -r version="v1.33.5" - local -r kube_hash_amd64="8f6106b970259486c5af5cbee404d4f23406d96d99dfb92a6965b299c2a4db0e" - local -r kube_hash_arm64="c6ad0510c089d49244eede2638b4a4ff125258fd29a0649e7eef05c7f79c737f" - - local kube_arch="" - local kube_hash="" - - case $(uname -m) in - x86_64) - kube_arch="amd64" - kube_hash="${kube_hash_amd64}" - ;; - aarch64) - kube_arch="arm64" - kube_hash="${kube_hash_arm64}" - ;; - *) - echo "Unsupported kubelet architecture $(uname -m)" - exit 1 - esac - - local -r url="https://dl.k8s.io/release/${version}/bin/linux/${kube_arch}/kubelet" - curl -sLO "${url}" --output-dir "${KUBELET_HOME}" - - cat < "${KUBELET_HOME}/kubelet.sha256" -${kube_hash} ${KUBELET_HOME}/kubelet -EOF - sha256sum --check "${KUBELET_HOME}/kubelet.sha256" - chmod +x "${KUBELET_HOME}/kubelet" - - # OVN requires kubeconfig at this path - # It must be a hard link or copy to be accessed from the container - sudo mkdir -p /var/lib/microshift/resources/kubeadmin - sudo ln "${KUBELET_HOME}/kubeconfig-${PRI_NODE_HOST}" /var/lib/microshift/resources/kubeadmin/kubeconfig - # Remove the old kubelet configuration file so that it is recreated - sudo rm -f "${KUBELET_HOME}/kubeconfig" - - # LVMS vg-manager requires presence of the lvmd.yaml file at a specific location - sudo mkdir -p /var/lib/microshift/lvms - sudo ln "${KUBELET_HOME}/lvmd-${PRI_NODE_HOST}.yaml" /var/lib/microshift/lvms/lvmd.yaml - - # Start crio & kubelet - sudo systemctl enable --now crio - sudo systemd-run --unit=kubelet --description="Kubelet" \ - --property=Environment="PATH=/sbin:/bin:/usr/sbin:/usr/bin:/opt/bin" \ - "${KUBELET_HOME}"/kubelet \ - --container-runtime-endpoint=/var/run/crio/crio.sock \ - --resolv-conf=/etc/resolv.conf \ - --rotate-certificates=true \ - --kubeconfig="${KUBELET_HOME}/kubeconfig" \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --fail-swap-on=false \ - --max-pods=250 \ - --cgroup-driver=systemd \ - --tls-cert-file="${KUBELET_HOME}/kubelet-${SEC_NODE_HOST}.crt" \ - --tls-private-key-file="${KUBELET_HOME}/kubelet-${SEC_NODE_HOST}.key" \ - --bootstrap-kubeconfig="${KUBELET_HOME}/kubeconfig-${PRI_NODE_HOST}" \ - --cluster-dns=10.43.0.10 \ - --cluster-domain=cluster.local -} - -function configure_node() { - local service_ready=false - - echo "Waiting for MicroShift nodes to be ready" - for _ in $(seq 300) ; do - if ${OC_CMD} wait --for=condition=Ready nodes "${PRI_NODE_HOST}" --timeout=0s &>/dev/null ; then - if ${OC_CMD} wait --for=condition=Ready nodes "${SEC_NODE_HOST}" --timeout=0s &>/dev/null ; then - service_ready=true - break - fi - fi - echo -n "." - sleep 1 - done - echo - - if ! ${service_ready} ; then - echo "Error: timed out waiting for MicroShift nodes to be ready" - exit 1 - fi - - # Check all the nodes have the same kubelet version - if ! ${OC_CMD} get node -o json | jq -e '[.items[].status.nodeInfo.kubeletVersion] | unique | length == 1' > /dev/null; then - echo "Error: kubelet versions do not match" - exit 1 - fi - - # Labeling the second node as a worker - ${OC_CMD} label nodes "${SEC_NODE_HOST}" node-role.kubernetes.io/worker= -} - -function wait_namespace_resources_ready() { - local -r wait_secs=600 - local -r custom_json=$(cat </dev/null return ${rc} } diff --git a/test/scenarios/periodics/el96-src@cncf-conformance.sh b/test/scenarios/periodics/el96-src@cncf-conformance.sh index 337a266f15..5c6c951d21 100644 --- a/test/scenarios/periodics/el96-src@cncf-conformance.sh +++ b/test/scenarios/periodics/el96-src@cncf-conformance.sh @@ -19,15 +19,11 @@ prepare_hosts() { local -r secondary_host_ip=$(cat "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/host2/ip") local -r primary_host_ssh_port=$(cat "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/host1/ssh_port") local -r secondary_host_ssh_port=$(cat "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/host2/ssh_port") - local -r primary_host_name="$(full_vm_name host1)" - local -r secondary_host_name="$(full_vm_name host2)" local rc=0 # Configure primary host - scp -P "${primary_host_ssh_port}" "${ROOTDIR}/scripts/multinode/configure-pri.sh" "redhat@${primary_host_ip}": - ssh -p "${primary_host_ssh_port}" "redhat@${primary_host_ip}" ./configure-pri.sh \ - "${primary_host_name}" "${primary_host_ip}" \ - "${secondary_host_name}" "${secondary_host_ip}" || rc=$? + scp -P "${primary_host_ssh_port}" "${ROOTDIR}/scripts/multinode/configure-node.sh" "redhat@${primary_host_ip}": + ssh -p "${primary_host_ssh_port}" "redhat@${primary_host_ip}" ./configure-node.sh || rc=$? if [ ${rc} -ne 0 ] ; then record_junit "prepare_hosts" "configure_primary" "FAILED" return ${rc} @@ -35,27 +31,20 @@ prepare_hosts() { record_junit "prepare_hosts" "configure_primary" "OK" # Configure secondary host - scp -3 -P "${primary_host_ssh_port}" \ - "redhat@${primary_host_ip}:/home/redhat/kubelet-${secondary_host_name}.key" \ - "redhat@${primary_host_ip}:/home/redhat/kubelet-${secondary_host_name}.crt" \ - "redhat@${primary_host_ip}:/home/redhat/kubeconfig-${primary_host_name}" \ - "redhat@${primary_host_ip}:/home/redhat/lvmd-${primary_host_name}.yaml" \ + scp -P "${primary_host_ssh_port}" \ + "redhat@${primary_host_ip}:/home/redhat/kubeconfig-bootstrap" \ "redhat@${secondary_host_ip}": - scp -P "${secondary_host_ssh_port}" "${ROOTDIR}/scripts/multinode/configure-sec.sh" "redhat@${secondary_host_ip}": - ssh -p "${secondary_host_ssh_port}" "redhat@${secondary_host_ip}" ./configure-sec.sh \ - "${primary_host_name}" "${primary_host_ip}" \ - "${secondary_host_name}" "${secondary_host_ip}" || rc=$? + scp -P "${secondary_host_ssh_port}" "${ROOTDIR}/scripts/multinode/configure-node.sh" "redhat@${secondary_host_ip}": + ssh -p "${secondary_host_ssh_port}" "redhat@${secondary_host_ip}" "./configure-node.sh --bootstrap-kubeconfig /home/redhat/kubeconfig-bootstrap" || rc=$? if [ ${rc} -ne 0 ] ; then record_junit "prepare_hosts" "configure_secondary" "FAILED" return ${rc} fi record_junit "prepare_hosts" "configure_secondary" "OK" - # Configure kubeconfig and host name resolution - scp -P "${primary_host_ssh_port}" "redhat@${primary_host_ip}:/home/redhat/kubeconfig-${primary_host_name}" "${KUBECONFIG}" + scp -P "${primary_host_ssh_port}" "redhat@${primary_host_ip}:/home/redhat/kubeconfig-bootstrap" "${KUBECONFIG}" export KUBECONFIG="${KUBECONFIG}" - echo "${primary_host_ip} ${primary_host_name}" | sudo tee -a /etc/hosts &>/dev/null return ${rc} }