Feature: Add multinode support

.github/actions/build-deb/action.yaml

@@ -51,7 +51,7 @@ runs:
     - name: Start the MicroShift service
       shell: bash
       run: |
-        make _topolvm_create
+        ./src/cluster_manager.sh topolvm-create || exit 1
         sudo systemctl start --no-block microshift.service
 
     - name: Run a test to verify that MicroShift is functioning properly

.github/actions/build/action.yaml

@@ -30,6 +30,11 @@ inputs:
     required: false
     default: 0
     type: integer
+  node-count:
+    description: Number of nodes in the MicroShift cluster
+    required: false
+    default: 1
+    type: integer
   build:
     type: choice
     description: Types of artifacts to build
@@ -111,11 +116,15 @@ runs:
           done
         fi
 
-        # Wait until the MicroShift service is ready and healthy
+        # Start-stop test with readiness check
         make run-ready
         make run-healthy
 
-        # Stop the MicroShift container
+        for i in $(seq 2 ${{ inputs.node-count }}); do
+          make add-node
+        done
+
+        make run-healthy
         make stop
 
     # Uncomment this to enable tmate-debug on failure

.github/actions/quick-start-clean/action.yaml

@@ -18,8 +18,14 @@ runs:
         sudo IMAGE_REF=${{ inputs.image-ref }} bash -xeuo pipefail < ./src/quickstart.sh
 
         # Wait until the MicroShift service is ready and healthy
-        make run-ready
-        make run-healthy
+        for _ in $(seq 100); do
+          state=$(sudo podman exec -i microshift-okd systemctl show --property=SubState --value greenboot-healthcheck 2>/dev/null || echo "unknown")
+          if [ "${state}" = "exited" ]; then
+            exit 0
+          fi
+          sleep 10
+        done
+        exit 1
 
     - name: Run the quick clean script
       shell: bash

.github/workflows/builders.yaml

@@ -23,6 +23,7 @@ jobs:
           bootc-image-url: quay.io/centos-bootc/centos-bootc
           bootc-image-tag: stream9
           build: bootc-image
+          node-count: 2
 
   centos10-bootc:
     runs-on: ubuntu-24.04
@@ -42,6 +43,7 @@ jobs:
           bootc-image-url: quay.io/centos-bootc/centos-bootc
           bootc-image-tag: stream10
           build: bootc-image
+          node-count: 2
 
   fedora-bootc:
     runs-on: ubuntu-24.04
@@ -61,6 +63,7 @@ jobs:
           bootc-image-url: registry.fedoraproject.org/fedora-bootc
           bootc-image-tag: latest
           build: bootc-image
+          node-count: 2
 
   ubuntu-rpm2deb:
     runs-on: ubuntu-24.04

Makefile

@@ -30,19 +30,21 @@ VG_NAME := myvg1
 #
 .PHONY: all
 all:
-	@echo "make <rpm | image | run | login | stop | clean | check>"
+	@echo "make <rpm | image | run | add-node | start | stop | clean | check>"
 	@echo "   rpm:       	build the MicroShift RPMs"
 	@echo "   image:     	build the MicroShift bootc container image"
-	@echo "   run:       	run the MicroShift bootc container"
-	@echo "   login:     	login to the MicroShift bootc container"
-	@echo "   stop:      	stop the MicroShift bootc container"
-	@echo "   clean:     	clean up the MicroShift container and the LVM backend"
+	@echo "   run:       	create and run a MicroShift cluster (1 node) in a bootc container"
+	@echo "   add-node:  	add a new node to the MicroShift cluster in a bootc container"
+	@echo "   start:     	start the MicroShift cluster that was already created"
+	@echo "   stop:      	stop the MicroShift cluster"
+	@echo "   clean:     	clean up the MicroShift cluster and the LVM backend"
 	@echo "   check:     	run the presubmit checks"
 	@echo ""
 	@echo "Sub-targets:"
 	@echo "   rpm-to-deb:	convert the MicroShift RPMs to Debian packages"
-	@echo "   run-ready: 	wait until the MicroShift service is ready"
-	@echo "   run-healthy:	wait until the MicroShift service is healthy"
+	@echo "   run-ready: 	wait until the MicroShift service is ready across the cluster"
+	@echo "   run-healthy:	wait until the MicroShift service is healthy across the cluster"
+	@echo "   run-status:	show the status of the MicroShift cluster"
 	@echo "   clean-all:	perform a full cleanup, including the container images"
 	@echo ""
 
@@ -105,68 +107,49 @@ image:
 #   which is less efficient than the default driver
 .PHONY: run
 run:
-	@echo "Running the MicroShift container"
-	sudo modprobe openvswitch
-	$(MAKE) _topolvm_create
-
-	NETWORK_OPTS="" ; \
-	if [ "${ISOLATED_NETWORK}" = "1" ] ; then \
-		NETWORK_OPTS="--network none" ; \
-	fi ; \
-	VOL_OPTS="--tty --volume /dev:/dev" ; \
-	for device in input snd dri; do \
-		[ -d "/dev/$${device}" ] && VOL_OPTS="$${VOL_OPTS} --tmpfs /dev/$${device}" ; \
-	done ; \
-	sudo podman run --privileged --rm -d \
-		--replace \
-		$${NETWORK_OPTS} \
-		$${VOL_OPTS} \
-		--tmpfs /var/lib/containers \
-		--name "${USHIFT_IMAGE}" \
-		--hostname 127.0.0.1.nip.io \
-		"${USHIFT_IMAGE}" ; \
-	$(MAKE) _isolated_network_config
+	@USHIFT_IMAGE=${USHIFT_IMAGE} ISOLATED_NETWORK=${ISOLATED_NETWORK} LVM_DISK=${LVM_DISK} LVM_VOLSIZE=${LVM_VOLSIZE} VG_NAME=${VG_NAME} ./src/cluster_manager.sh create
+
+.PHONY: add-node
+add-node:
+	@USHIFT_IMAGE=${USHIFT_IMAGE} ISOLATED_NETWORK=${ISOLATED_NETWORK} LVM_DISK=${LVM_DISK} LVM_VOLSIZE=${LVM_VOLSIZE} VG_NAME=${VG_NAME} ./src/cluster_manager.sh add-node
+
+.PHONY: start
+start:
+	@USHIFT_IMAGE=${USHIFT_IMAGE} ISOLATED_NETWORK=${ISOLATED_NETWORK} LVM_DISK=${LVM_DISK} LVM_VOLSIZE=${LVM_VOLSIZE} VG_NAME=${VG_NAME} ./src/cluster_manager.sh start
+
+.PHONY: stop
+stop:
+	@USHIFT_IMAGE=${USHIFT_IMAGE} ISOLATED_NETWORK=${ISOLATED_NETWORK} LVM_DISK=${LVM_DISK} LVM_VOLSIZE=${LVM_VOLSIZE} VG_NAME=${VG_NAME} ./src/cluster_manager.sh stop
 
 .PHONY: run-ready
 run-ready:
 	@echo "Waiting 5m for the MicroShift service to be ready"
 	@for _ in $$(seq 60); do \
-		if sudo podman exec -i "${USHIFT_IMAGE}" systemctl -q is-active microshift.service ; then \
+		if USHIFT_IMAGE=${USHIFT_IMAGE} ISOLATED_NETWORK=${ISOLATED_NETWORK} LVM_DISK=${LVM_DISK} LVM_VOLSIZE=${LVM_VOLSIZE} VG_NAME=${VG_NAME} ./src/cluster_manager.sh ready ; then \
 			printf "\nOK\n" && exit 0; \
 		fi ; \
-		echo -n "." && sleep 5 ; \
+		sleep 5 ; \
 	done ; \
 	printf "\nFAILED\n" && exit 1
 
 .PHONY: run-healthy
 run-healthy:
 	@echo "Waiting 15m for the MicroShift service to be healthy"
 	@for _ in $$(seq 60); do \
-		state=$$(sudo podman exec -i "${USHIFT_IMAGE}" systemctl show --property=SubState --value greenboot-healthcheck) ; \
-		if [ "$${state}" = "exited" ] ; then \
+		if USHIFT_IMAGE=${USHIFT_IMAGE} ISOLATED_NETWORK=${ISOLATED_NETWORK} LVM_DISK=${LVM_DISK} LVM_VOLSIZE=${LVM_VOLSIZE} VG_NAME=${VG_NAME} ./src/cluster_manager.sh healthy ; then \
 			printf "\nOK\n" && exit 0; \
 		fi ; \
-		echo -n "." && sleep 15 ; \
+		sleep 5 ; \
 	done ; \
-	printf "\nThe state of the greenboot-healthcheck service is '$${state}'" && \
 	printf "\nFAILED\n" && exit 1
 
-.PHONY: login
-login:
-	@echo "Logging into the MicroShift container"
-	sudo podman exec -it "${USHIFT_IMAGE}" bash -l
-
-.PHONY: stop
-stop:
-	@echo "Stopping the MicroShift container"
-	sudo podman stop --time 0 "${USHIFT_IMAGE}" || true
+.PHONY: run-status
+run-status:
+	@USHIFT_IMAGE=${USHIFT_IMAGE} ISOLATED_NETWORK=${ISOLATED_NETWORK} LVM_DISK=${LVM_DISK} LVM_VOLSIZE=${LVM_VOLSIZE} VG_NAME=${VG_NAME} ./src/cluster_manager.sh status
 
 .PHONY: clean
 clean:
-	@echo "Cleaning up the MicroShift container and the TopoLVM CSI backend"
-	$(MAKE) stop
-	sudo rmmod openvswitch || true
-	$(MAKE) _topolvm_delete
+	@USHIFT_IMAGE=${USHIFT_IMAGE} ISOLATED_NETWORK=${ISOLATED_NETWORK} LVM_DISK=${LVM_DISK} LVM_VOLSIZE=${LVM_VOLSIZE} VG_NAME=${VG_NAME} ./src/cluster_manager.sh delete
 
 .PHONY: clean-all
 clean-all:
@@ -181,35 +164,6 @@ check: _hadolint _shellcheck
 #
 # Define the private targets
 #
-# The configurations for the isolated network are done inside the container
-.PHONY: _isolated_network_config
-_isolated_network_config:
-	if [ "${ISOLATED_NETWORK}" = "1" ] ; then \
-		sudo podman cp ./src/config_isolated_net.sh "${USHIFT_IMAGE}:/tmp/config_isolated_net.sh" && \
-		sudo podman exec -i "${USHIFT_IMAGE}" /tmp/config_isolated_net.sh && \
-		sudo podman exec -i "${USHIFT_IMAGE}" rm -vf /tmp/config_isolated_net.sh ; \
-	fi
-
-.PHONY: _topolvm_create
-_topolvm_create:
-	if [ ! -f "${LVM_DISK}" ] ; then \
-		echo "Creating the TopoLVM CSI backend" ; \
-		sudo mkdir -p "$$(dirname "${LVM_DISK}")" ; \
-		sudo truncate --size="${LVM_VOLSIZE}" "${LVM_DISK}" ; \
-		DEVICE_NAME="$$(sudo losetup --find --show --nooverlap "${LVM_DISK}")" && \
-		sudo vgcreate -f -y "${VG_NAME}" "$${DEVICE_NAME}" ; \
-	fi
-
-.PHONY: _topolvm_delete
-_topolvm_delete:
-	if [ -f "${LVM_DISK}" ] ; then \
-		echo "Deleting the TopoLVM CSI backend" ; \
-		sudo lvremove -y "${VG_NAME}" || true ; \
-		sudo vgremove -y "${VG_NAME}" || true ; \
-		DEVICE_NAME="$$(sudo losetup -j "${LVM_DISK}" | cut -d: -f1)" ; \
-		[ -n "$${DEVICE_NAME}" ] && sudo losetup -d $${DEVICE_NAME} || true ; \
-		sudo rm -rf "$$(dirname "${LVM_DISK}")" ; \
-	fi
 
 # When run inside a container, the file contents are redirected via stdin and
 # the output of errors does not contain the file path. Work around this issue

docs/run.md

@@ -98,14 +98,17 @@ The following options can be specified in the make command line using the `NAME=
 
 | Name              | Required | Default  | Comments
 |-------------------|----------|----------|---------
-| LVM_VOLUME_SIZE   | no       | 1G       | TopoLVM CSI backend volume
+| LVM_VOLSIZE       | no       | 1G       | TopoLVM CSI backend volume
 | ISOLATED_NETWORK  | no       | 0        | Use `--network none` podman option
 
+This step creates a single-node MicroShift cluster. The cluster can be extended using `make add-node` to add one node at a time.
+
 This step includes:
 * Loading the `openvswitch` module required when OVN-K CNI driver is used
   when compiled with the non-default `WITH_KINDNET=0` image build option.
-* Preparing a 1GB TopoLVM CSI backend on the host to be used by MicroShift when
+* Preparing a TopoLVM CSI backend (default 1GB, configurable via `LVM_VOLSIZE`) on the host to be used by MicroShift when
   compiled with the default `WITH_TOPOLVM=1` image build option.
+* Creating a podman network for easier multi-node cluster support with name resolution.
 
 ```bash
 make run
@@ -124,20 +127,29 @@ make run
 
 ### Container Login
 
-Log into the container by running the following command.
+Log into the container by running the following command. The commands for doing so are displayed as
+part of the summary from `make run` and `make add-node`.
 
+For example, the first node in a cluster is named `microshift-okd-1`:
 ```bash
-make login
+sudo podman exec -it microshift-okd-1 /bin/bash -l
 ```
 
 Verify that all the MicroShift services are up and running successfully.
-
 ```bash
 export KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig
 oc get nodes
 oc get pods -A
 ```
 
+### Start the Container
+
+If you have stopped the MicroShift cluster, you can start it again using the following command.
+
+```bash
+make start
+```
+
 ### Stop the Container
 
 Run the following command to stop the MicroShift Bootc container.
@@ -146,6 +158,31 @@ Run the following command to stop the MicroShift Bootc container.
 make stop
 ```
 
+### Add Node to Cluster
+
+To create a multi-node cluster, you can add additional nodes after creating the initial cluster with `make run`.
+
+```bash
+make add-node
+```
+
+> Note: The `add-node` target requires a non-isolated network (`ISOLATED_NETWORK=0`). Each additional node will be automatically joined to the cluster.
+
+### Check Cluster Status
+
+Run the following commands to check the status of your MicroShift cluster.
+
+```bash
+# Wait until the MicroShift service is ready (checks all nodes)
+make run-ready
+
+# Wait until the MicroShift service is healthy (checks all nodes)
+make run-healthy
+
+# Show current cluster status including nodes and pods
+make run-status
+```
+
 ## Cleanup
 
 ### RPM