Add watchdog process to auto-restart crashed server processes

.github/workflows/ci.yml

@@ -0,0 +1,23 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Shellcheck
+        run: shellcheck --severity=warning opencode-server.sh
+
+  test:
+    needs: validate
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run tests
+        run: ./tests/test-devcontainer.sh

opencode-server.sh

@@ -51,6 +51,7 @@ fi
 # Configuration
 PASSWORD_FILE="$HOME/.config/opencode-server-local"
 PID_FILE="$HOME/.config/opencode-server.pid"
+STOP_FLAG="$HOME/.config/opencode-server.stop"
 CERT_DIR="$HOME/.config/opencode-certs"
 CERT_FILE="$CERT_DIR/cert.pem"
 KEY_FILE="$CERT_DIR/key.pem"
@@ -122,30 +123,91 @@ generate_cert() {
 # Check if server is already running
 is_running() {
   if [ -f "$PID_FILE" ]; then
-    read OPENCODE_PID CADDY_PID < "$PID_FILE"
+    read OPENCODE_PID CADDY_PID WATCHDOG_PID < "$PID_FILE"
     if kill -0 "$OPENCODE_PID" 2>/dev/null && kill -0 "$CADDY_PID" 2>/dev/null; then
       return 0
     fi
+    # Also consider the server running if the watchdog is alive (it will restart processes)
+    if [ -n "$WATCHDOG_PID" ] && kill -0 "$WATCHDOG_PID" 2>/dev/null; then
+      return 0
+    fi
   fi
   return 1
 }
 
 # Stop the running server
 stop_server() {
   if [ -f "$PID_FILE" ]; then
-    read OPENCODE_PID CADDY_PID < "$PID_FILE"
+    read OPENCODE_PID CADDY_PID WATCHDOG_PID < "$PID_FILE"
+    # Signal the watchdog to stop before killing processes
+    touch "$STOP_FLAG"
+    if [ -n "$WATCHDOG_PID" ]; then
+      echo "Stopping watchdog (PID: $WATCHDOG_PID)..."
+      kill "$WATCHDOG_PID" 2>/dev/null
+    fi
     echo "Stopping OpenCode server (PID: $OPENCODE_PID)..."
     kill "$OPENCODE_PID" 2>/dev/null
     echo "Stopping Caddy proxy (PID: $CADDY_PID)..."
     kill "$CADDY_PID" 2>/dev/null
     rm -f "$PID_FILE"
     rm -f "$CADDYFILE"
+    rm -f "$HOME/.config/opencode-watchdog.sh"
     echo "Server stopped."
   else
     echo "No running server found."
   fi
 }
 
+# Watchdog: monitors and restarts processes if they die
+# Written to a script file and run detached so it survives parent shell exit
+start_watchdog() {
+  local watchdog_script="$HOME/.config/opencode-watchdog.sh"
+  cat > "$watchdog_script" <<WATCHDOG_EOF
+#!/usr/bin/env bash
+export PATH="$HOME/.opencode/bin:\$PATH"
+STOP_FLAG="$STOP_FLAG"
+PID_FILE="$PID_FILE"
+LOG_FILE="$LOG_FILE"
+OPENCODE_INTERNAL_PORT="$OPENCODE_INTERNAL_PORT"
+CADDYFILE="$CADDYFILE"
+while true; do
+  sleep 3
+  # Check if stop was requested
+  if [ -f "\$STOP_FLAG" ]; then
+    exit 0
+  fi
+  if [ ! -f "\$PID_FILE" ]; then
+    exit 0
+  fi
+  read OPENCODE_PID CADDY_PID _WATCHDOG_PID < "\$PID_FILE"
+  NEED_UPDATE=false
+  # Restart opencode if it died
+  if ! kill -0 "\$OPENCODE_PID" 2>/dev/null; then
+    echo "\$(date): OpenCode process died. Restarting..." >> "\$LOG_FILE"
+    opencode serve --hostname 127.0.0.1 --port \$OPENCODE_INTERNAL_PORT >> "\$LOG_FILE" 2>&1 &
+    OPENCODE_PID=\$!
+    NEED_UPDATE=true
+    sleep 3
+  fi
+  # Restart caddy if it died
+  if ! kill -0 "\$CADDY_PID" 2>/dev/null; then
+    echo "\$(date): Caddy process died. Restarting..." >> "\$LOG_FILE"
+    caddy run --config "\$CADDYFILE" --adapter caddyfile >> "\$LOG_FILE" 2>&1 &
+    CADDY_PID=\$!
+    NEED_UPDATE=true
+  fi
+  if [ "\$NEED_UPDATE" = true ]; then
+    echo "\$OPENCODE_PID \$CADDY_PID \$\$" > "\$PID_FILE"
+  fi
+done
+WATCHDOG_EOF
+  chmod +x "$watchdog_script"
+  nohup bash "$watchdog_script" >> "$LOG_FILE" 2>&1 &
+  local pid=$!
+  disown $pid 2>/dev/null
+  echo $pid
+}
+
 # Handle --stop flag
 if [ "$1" = "--stop" ]; then
   stop_server
@@ -196,23 +258,32 @@ cat > "$CADDYFILE" <<EOF
 }
 EOF
 
+# Clear any previous stop flag
+rm -f "$STOP_FLAG"
+
 # Start opencode on localhost only (not exposed to network)
 opencode serve --hostname 127.0.0.1 --port $OPENCODE_INTERNAL_PORT >> "$LOG_FILE" 2>&1 &
 OPENCODE_PID=$!
+disown $OPENCODE_PID 2>/dev/null
 
 # Wait a moment for opencode to start
 sleep 1
 
 # Start Caddy natively
 caddy run --config "$CADDYFILE" --adapter caddyfile >> "$LOG_FILE" 2>&1 &
 CADDY_PID=$!
+disown $CADDY_PID 2>/dev/null
+
+# Start watchdog to monitor and restart processes
+WATCHDOG_PID=$(start_watchdog)
 
-# Save PIDs to file
-echo "$OPENCODE_PID $CADDY_PID" > "$PID_FILE"
+# Save PIDs to file (including watchdog)
+echo "$OPENCODE_PID $CADDY_PID $WATCHDOG_PID" > "$PID_FILE"
 
 echo "OpenCode server started in background."
 echo "OpenCode PID: $OPENCODE_PID"
 echo "Caddy PID: $CADDY_PID"
+echo "Watchdog PID: $WATCHDOG_PID"
 echo "Logs: $LOG_FILE"
 echo
 print_info

tests/test-devcontainer.sh

@@ -133,6 +133,67 @@ else
   fi
 fi
 
+# ------------------------------------------------------------------------------
+echo "=== Test 7c: Watchdog restarts killed OpenCode process ==="
+# Start server fresh
+docker exec "$CONTAINER_NAME" bash -c \
+  "OPENCODE_PORT=$TEST_PORT bash /workspace/opencode-server.sh --skip-auth" >/dev/null 2>&1
+sleep 2
+# Kill only the opencode process (not via --stop)
+docker exec "$CONTAINER_NAME" bash -c \
+  'OPENCODE_PID=$(cut -d" " -f1 ~/.config/opencode-server.pid); kill "$OPENCODE_PID" 2>/dev/null || true'
+# Wait for watchdog to detect and restart, then retry checking the server
+RESTART_OK=false
+for _i in $(seq 1 8); do
+  sleep 5
+  HTTP_CODE=$(docker exec "$CONTAINER_NAME" bash -c \
+    "curl -k -s -o /dev/null -w '%{http_code}' https://127.0.0.1:$TEST_PORT" 2>&1 || true)
+  if [ "$HTTP_CODE" = "401" ]; then
+    RESTART_OK=true
+    break
+  fi
+done
+if [ "$RESTART_OK" = true ]; then
+  pass "Server auto-restarted after OpenCode process was killed"
+else
+  fail "Server did not auto-restart after kill (got: $HTTP_CODE)"
+fi
+
+# ------------------------------------------------------------------------------
+echo "=== Test 7d: Watchdog restarts killed Caddy process ==="
+# Kill only the caddy process (not via --stop)
+docker exec "$CONTAINER_NAME" bash -c \
+  'CADDY_PID=$(cut -d" " -f2 ~/.config/opencode-server.pid); kill "$CADDY_PID" 2>/dev/null || true'
+# Wait for watchdog to detect and restart, then retry checking the server
+RESTART_OK=false
+for _i in $(seq 1 8); do
+  sleep 5
+  HTTP_CODE=$(docker exec "$CONTAINER_NAME" bash -c \
+    "curl -k -s -o /dev/null -w '%{http_code}' https://127.0.0.1:$TEST_PORT" 2>&1 || true)
+  if [ "$HTTP_CODE" = "401" ]; then
+    RESTART_OK=true
+    break
+  fi
+done
+if [ "$RESTART_OK" = true ]; then
+  pass "Server auto-restarted after Caddy process was killed"
+else
+  fail "Server did not auto-restart after Caddy kill (got: $HTTP_CODE)"
+fi
+
+# ------------------------------------------------------------------------------
+echo "=== Test 7e: Stop command permanently stops server (no restart) ==="
+docker exec "$CONTAINER_NAME" bash /workspace/opencode-server.sh --stop >/dev/null 2>&1
+# Wait longer than the watchdog interval to confirm it stays down
+sleep 10
+LISTENING=$(docker exec "$CONTAINER_NAME" bash -c "ss -tlnp | grep $TEST_PORT || echo 'not listening'" 2>&1)
+if [[ "$LISTENING" == *"not listening"* ]]; then
+  pass "Server stays stopped after --stop (watchdog does not restart)"
+else
+  fail "Server restarted after --stop (watchdog should not restart)"
+  echo "$LISTENING"
+fi
+
 # ------------------------------------------------------------------------------
 echo "=== Test 8: Password persists after restart ==="
 docker exec "$CONTAINER_NAME" bash -c \