From 02dd4450029c31a08802f8b94f600a111743ee19 Mon Sep 17 00:00:00 2001
From: Benedikt Schackenberg
 <6381261+BenediktSchackenberg@users.noreply.github.com>
Date: Mon, 6 Apr 2026 13:59:00 +0000
Subject: [PATCH 1/2] fix(preflight): detect conflicting host kubelet before
 gateway start

Signed-off-by: Benedikt Schackenberg <6381261+BenediktSchackenberg@users.noreply.github.com>
---
 scripts/lib/runtime.sh |  44 +++++++++++
 scripts/setup-spark.sh | 166 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 210 insertions(+)
 create mode 100755 scripts/setup-spark.sh

diff --git a/scripts/lib/runtime.sh b/scripts/lib/runtime.sh
index 3de8e16fe2..5bbf77cdec 100755
--- a/scripts/lib/runtime.sh
+++ b/scripts/lib/runtime.sh
@@ -282,3 +282,47 @@ check_local_provider_health() {
       ;;
   esac
 }
+
+# ── Kubelet conflict detection ────────────────────────────────────
+# Returns 0 if a conflicting kubelet is detected, 1 otherwise.
+# Sets KUBELET_CONFLICT_DETAIL to a human-readable description.
+# See: https://github.com/NVIDIA/NemoClaw/issues/431
+detect_kubelet_conflict() {
+  KUBELET_CONFLICT_DETAIL=""
+
+  if pgrep -x kubelet > /dev/null 2>&1 || pgrep -x kubelite > /dev/null 2>&1 || pgrep -x k3s > /dev/null 2>&1; then
+    KUBELET_CONFLICT_DETAIL="kubelet process detected"
+    return 0
+  fi
+
+  if command -v microk8s > /dev/null 2>&1; then
+    if microk8s status 2>/dev/null | grep -q "microk8s is running"; then
+      KUBELET_CONFLICT_DETAIL="MicroK8s is running"
+      return 0
+    fi
+  fi
+
+  if systemctl is-active --quiet k3s 2>/dev/null || systemctl is-active --quiet k3s-agent 2>/dev/null; then
+    KUBELET_CONFLICT_DETAIL="k3s service is active"
+    return 0
+  fi
+
+  return 1
+}
+
+# Emit standardized warning for kubelet conflicts.
+warn_kubelet_conflict() {
+  local detail="${1:-${KUBELET_CONFLICT_DETAIL:-}}"
+  warn "⚠️  Conflicting Kubernetes detected: $detail"
+  warn ""
+  warn "The gateway runs k3s inside Docker with cgroupns=host, which will"
+  warn "conflict with the host kubelet over /sys/fs/cgroup/kubepods."
+  warn "This causes all pods to enter CrashLoopBackOff."
+  warn ""
+  warn "Options:"
+  warn "  1. Stop the host Kubernetes first:"
+  warn "     sudo microk8s stop        # for MicroK8s"
+  warn "     sudo systemctl stop k3s   # for k3s"
+  warn "     sudo systemctl stop kubelet  # for kubeadm"
+  warn "  2. Continue anyway (gateway will likely fail)"
+}
diff --git a/scripts/setup-spark.sh b/scripts/setup-spark.sh
new file mode 100755
index 0000000000..e06e4870d7
--- /dev/null
+++ b/scripts/setup-spark.sh
@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# NemoClaw setup for DGX Spark devices.
+#
+# Spark ships Ubuntu 24.04 (cgroup v2) + Docker 28.x but no k3s.
+# OpenShell's gateway starts k3s inside a Docker container, which
+# needs cgroup host namespace access. This script configures Docker
+# for that.
+#
+# Usage:
+#   sudo nemoclaw setup-spark
+#   # or directly:
+#   sudo bash scripts/setup-spark.sh
+#
+# What it does:
+#   1. Adds current user to docker group (avoids sudo for everything else)
+#   2. Configures Docker daemon for cgroupns=host (k3s-in-Docker on cgroup v2)
+#   3. Restarts Docker
+
+set -euo pipefail
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+info() { echo -e "${GREEN}>>>${NC} $1"; }
+warn() { echo -e "${YELLOW}>>>${NC} $1"; }
+fail() {
+  echo -e "${RED}>>>${NC} $1"
+  exit 1
+}
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=lib/runtime.sh
+source "$SCRIPT_DIR/lib/runtime.sh"
+
+# ── Pre-flight checks ─────────────────────────────────────────────
+
+if [ "$(uname -s)" != "Linux" ]; then
+  fail "This script is for DGX Spark (Linux). Use 'nemoclaw setup' for macOS."
+fi
+
+if [ "$(id -u)" -ne 0 ]; then
+  fail "Must run as root: sudo nemoclaw setup-spark"
+fi
+
+# Detect the real user (not root) for docker group add
+REAL_USER="${SUDO_USER:-$(logname 2>/dev/null || echo "")}"
+if [ -z "$REAL_USER" ]; then
+  warn "Could not detect non-root user. Docker group will not be configured."
+fi
+
+command -v docker >/dev/null || fail "Docker not found. DGX Spark should have Docker pre-installed."
+
+# ── 1. Docker group ───────────────────────────────────────────────
+
+if [ -n "$REAL_USER" ]; then
+  if id -nG "$REAL_USER" | grep -qw docker; then
+    info "User '$REAL_USER' already in docker group"
+  else
+    info "Adding '$REAL_USER' to docker group..."
+    usermod -aG docker "$REAL_USER"
+    info "Added. Group will take effect on next login (or use 'newgrp docker')."
+  fi
+fi
+
+# ── 1b. Check for conflicting Kubernetes installations ────────────
+#
+# If another kubelet is running on the host, cgroupns=host causes
+# cgroup path conflicts → CrashLoopBackOff.
+# See: https://github.com/NVIDIA/NemoClaw/issues/431
+
+if detect_kubelet_conflict; then
+  warn_kubelet_conflict "$KUBELET_CONFLICT_DETAIL"
+  warn ""
+
+  if [ -t 0 ]; then
+    if ! read -rp "Continue anyway? [y/N] " reply; then
+      fail "Aborted (no input). Stop the conflicting Kubernetes service and retry."
+    fi
+    if [[ ! "$reply" =~ ^[Yy] ]]; then
+      fail "Aborted. Stop the conflicting Kubernetes service and retry."
+    fi
+  else
+    fail "Conflicting Kubernetes detected. Stop it first or run interactively to override."
+  fi
+fi
+
+# ── 2. Docker cgroup namespace ────────────────────────────────────
+#
+# Spark runs cgroup v2 (Ubuntu 24.04). OpenShell's gateway embeds
+# k3s in a Docker container, which needs --cgroupns=host to manage
+# cgroup hierarchies. Without this, kubelet fails with:
+#   "openat2 /sys/fs/cgroup/kubepods/pids.max: no"
+#
+# Setting default-cgroupns-mode=host in daemon.json makes all
+# containers use the host cgroup namespace. This is safe — it's
+# the Docker default on cgroup v1 hosts anyway.
+
+DAEMON_JSON="/etc/docker/daemon.json"
+NEEDS_RESTART=false
+
+if [ -f "$DAEMON_JSON" ]; then
+  # Check if already configured
+  if grep -q '"default-cgroupns-mode"' "$DAEMON_JSON" 2>/dev/null; then
+    CURRENT_MODE=$(python3 -c "import json; print(json.load(open('$DAEMON_JSON')).get('default-cgroupns-mode',''))" 2>/dev/null || echo "")
+    if [ "$CURRENT_MODE" = "host" ]; then
+      info "Docker daemon already configured for cgroupns=host"
+    else
+      info "Updating Docker daemon cgroupns mode to 'host'..."
+      python3 -c "
+import json
+with open('$DAEMON_JSON') as f:
+    d = json.load(f)
+d['default-cgroupns-mode'] = 'host'
+with open('$DAEMON_JSON', 'w') as f:
+    json.dump(d, f, indent=2)
+"
+      NEEDS_RESTART=true
+    fi
+  else
+    info "Adding cgroupns=host to Docker daemon config..."
+    python3 -c "
+import json
+try:
+    with open('$DAEMON_JSON') as f:
+        d = json.load(f)
+except:
+    d = {}
+d['default-cgroupns-mode'] = 'host'
+with open('$DAEMON_JSON', 'w') as f:
+    json.dump(d, f, indent=2)
+"
+    NEEDS_RESTART=true
+  fi
+else
+  info "Creating Docker daemon config with cgroupns=host..."
+  mkdir -p "$(dirname "$DAEMON_JSON")"
+  echo '{ "default-cgroupns-mode": "host" }' >"$DAEMON_JSON"
+  NEEDS_RESTART=true
+fi
+
+# ── 3. Restart Docker if needed ───────────────────────────────────
+
+if [ "$NEEDS_RESTART" = true ]; then
+  info "Restarting Docker daemon..."
+  systemctl restart docker
+  # Wait for Docker to be ready
+  for i in 1 2 3 4 5 6 7 8 9 10; do
+    if docker info >/dev/null 2>&1; then
+      break
+    fi
+    [ "$i" -eq 10 ] && fail "Docker didn't come back after restart. Check 'systemctl status docker'."
+    sleep 2
+  done
+  info "Docker restarted with cgroupns=host"
+fi
+
+# ── 4. Run normal setup ──────────────────────────────────────────
+
+echo ""
+info "DGX Spark Docker configuration complete."
+info ""

From 2cfe78d01d5dc5c839d32b191e22528f54380219 Mon Sep 17 00:00:00 2001
From: Benedikt Schackenberg
 <6381261+BenediktSchackenberg@users.noreply.github.com>
Date: Tue, 14 Apr 2026 07:46:25 +0000
Subject: [PATCH 2/2] style: apply shfmt formatting to runtime.sh

Pipe continuation style and redirect spacing to match project
shfmt config (-i 2 -ci).

Signed-off-by: Benedikt Schackenberg <6381261+BenediktSchackenberg@users.noreply.github.com>
---
 scripts/lib/runtime.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/lib/runtime.sh b/scripts/lib/runtime.sh
index 5bbf77cdec..ec6c957f16 100755
--- a/scripts/lib/runtime.sh
+++ b/scripts/lib/runtime.sh
@@ -156,8 +156,8 @@ first_non_loopback_nameserver() {
     return 1
   fi
 
-  printf '%s\n' "$resolv_conf" \
-    | awk '$1 == "nameserver" && $2 !~ /^127\./ { print $2; exit }'
+  printf '%s\n' "$resolv_conf" |
+    awk '$1 == "nameserver" && $2 !~ /^127\./ { print $2; exit }'
 }
 
 get_colima_vm_nameserver() {
@@ -290,12 +290,12 @@ check_local_provider_health() {
 detect_kubelet_conflict() {
   KUBELET_CONFLICT_DETAIL=""
 
-  if pgrep -x kubelet > /dev/null 2>&1 || pgrep -x kubelite > /dev/null 2>&1 || pgrep -x k3s > /dev/null 2>&1; then
+  if pgrep -x kubelet >/dev/null 2>&1 || pgrep -x kubelite >/dev/null 2>&1 || pgrep -x k3s >/dev/null 2>&1; then
     KUBELET_CONFLICT_DETAIL="kubelet process detected"
     return 0
   fi
 
-  if command -v microk8s > /dev/null 2>&1; then
+  if command -v microk8s >/dev/null 2>&1; then
     if microk8s status 2>/dev/null | grep -q "microk8s is running"; then
       KUBELET_CONFLICT_DETAIL="MicroK8s is running"
       return 0