fix(gastown): route MR bead failures through full review lifecycle to unblock convoys

cloudflare-gastown/AGENTS.md

@@ -8,6 +8,37 @@
 ## Durable Objects
 
 - Each DO module must export a `get{ClassName}Stub` helper function (e.g. `getRigDOStub`) that centralizes how that DO namespace creates instances. Callers should use this helper instead of accessing the namespace binding directly.
+- **Sub-modules for large DOs**: When a Durable Object grows beyond a few hundred lines, extract domain logic into sub-modules under a `<do-name>/` directory alongside the DO file. For example, `Town.do.ts` delegates to modules in `town/`:
+
+  ```
+  dos/
+    Town.do.ts            # Class definition, RPC methods, alarm loop
+    town/
+      agents.ts           # Agent CRUD, hook management
+      beads.ts            # Bead CRUD, convoy progress
+      scheduling.ts       # Agent dispatch, pending work scheduling
+      review-queue.ts     # Review lifecycle, recovery
+      patrol.ts           # Zombie detection, stale hook recovery
+      config.ts           # Town configuration
+      rigs.ts             # Rig registry
+      mail.ts             # Inter-agent mail
+      container-dispatch.ts  # Container start/stop/status
+  ```
+
+  Each sub-module exports plain functions (not classes) that accept `SqlStorage` and any other required context as arguments. The DO imports them with the `import * as X` pattern:
+
+  ```ts
+  import * as beadOps from './town/beads';
+  import * as agents from './town/agents';
+  import * as scheduling from './town/scheduling';
+
+  // In the DO class:
+  beadOps.updateBeadStatus(this.sql, beadId, 'closed', agentId);
+  agents.getOrCreateAgent(this.sql, 'polecat', rigId, this.townId);
+  await scheduling.schedulePendingWork(this.schedulingCtx);
+  ```
+
+  This keeps the DO class thin (RPC surface + orchestration) while sub-modules own the business logic. The `import * as X` pattern makes call sites self-documenting — you can always tell which domain a function belongs to.
 
 ## IO boundaries
 

cloudflare-gastown/container/Dockerfile

@@ -44,6 +44,14 @@ RUN cd /opt/gastown-plugin && npm install --omit=dev && \
     ln -s /opt/gastown-plugin/index.ts /home/agent/.config/kilo/plugins/gastown.ts && \
     chown -R agent:agent /home/agent/.config
 
+# ── Git config for agent user ───────────────────────────────────────
+# Skip LFS smudge filter: agents don't need binary assets and LFS
+# downloads can fail when credentials don't cover the batch endpoint.
+# Also disable LFS fetch entirely so clone/worktree never stalls.
+RUN printf '[filter "lfs"]\n\tsmudge = git-lfs smudge --skip -- %%f\n\tprocess = git-lfs filter-process --skip\n\tclean = git-lfs clean -- %%f\n\trequired = true\n[lfs]\n\tfetchexclude = *\n' \
+      > /home/agent/.gitconfig && \
+    chown agent:agent /home/agent/.gitconfig
+
 WORKDIR /app
 
 # ── Install production deps via pnpm ────────────────────────────────

cloudflare-gastown/container/src/git-manager.ts

@@ -1,4 +1,4 @@
-import { mkdir, realpath, rm, stat } from 'node:fs/promises';
+import { mkdir, realpath, rm, stat, writeFile } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
 import type { CloneOptions, WorktreeOptions } from './types';
 
@@ -105,6 +105,49 @@ function authenticateGitUrl(gitUrl: string, envVars?: Record<string, string>): s
   return gitUrl;
 }
 
+/**
+ * Configure a credential-store helper on the bare repo so that worktree
+ * operations (checkout, reset, lfs smudge) can resolve credentials
+ * through the standard git credential chain.
+ *
+ * Without this, git-lfs smudge filters triggered by `git worktree add`
+ * or `git reset --hard` fail with "Smudge error" because the LFS batch
+ * API request has no credentials. The token is embedded in the remote
+ * URL, but some git-lfs versions require the credential helper for the
+ * LFS batch endpoint (which uses a different URL path).
+ */
+async function configureRepoCredentials(
+  repoDir: string,
+  gitUrl: string,
+  envVars?: Record<string, string>
+): Promise<void> {
+  if (!envVars) return;
+
+  const token = envVars.GIT_TOKEN ?? envVars.GITHUB_TOKEN;
+  const gitlabToken = envVars.GITLAB_TOKEN;
+  if (!token && !gitlabToken) return;
+
+  try {
+    const url = new URL(gitUrl);
+    const credentialLine =
+      gitlabToken && (url.hostname.includes('gitlab') || envVars.GITLAB_INSTANCE_URL)
+        ? `https://oauth2:${gitlabToken}@${url.hostname}`
+        : token
+          ? `https://x-access-token:${token}@${url.hostname}`
+          : null;
+
+    if (!credentialLine) return;
+
+    // Write to a per-repo credential file outside the repo itself
+    const credFile = `/tmp/.git-credentials-repo-${repoDir.replace(/[^a-zA-Z0-9]/g, '-')}`;
+    await writeFile(credFile, credentialLine + '\n', { mode: 0o600 });
+
+    await exec('git', ['config', 'credential.helper', `store --file=${credFile}`], repoDir);
+  } catch (err) {
+    console.warn(`Failed to configure repo credentials for ${repoDir}:`, err);
+  }
+}
+
 /**
  * Validate a branch name — block control characters and shell metacharacters.
  */
@@ -148,6 +191,11 @@ async function exec(cmd: string, args: string[], cwd?: string): Promise<string>
       // Public repos clone without auth; private repos fail fast with
       // a clear error instead of hanging on a username prompt.
       GIT_TERMINAL_PROMPT: '0',
+      // Skip LFS smudge filter during checkout/worktree operations.
+      // Agents don't need binary assets (videos, images, etc.) and
+      // LFS downloads can fail when the credential helper doesn't
+      // cover the LFS batch endpoint, blocking worktree creation.
+      GIT_LFS_SKIP_SMUDGE: '1',
     },
   });
 
@@ -211,6 +259,7 @@ async function cloneRepoInner(
     await exec('git', ['remote', 'set-url', 'origin', authUrl], dir).catch(err => {
       console.warn(`Failed to update remote URL for rig ${options.rigId}:`, err);
     });
+    await configureRepoCredentials(dir, options.gitUrl, options.envVars);
     await exec('git', ['fetch', '--all', '--prune'], dir);
     console.log(`Fetched latest for rig ${options.rigId}`);
     return dir;
@@ -228,6 +277,7 @@ async function cloneRepoInner(
 
   await mkdir(dir, { recursive: true });
   await exec('git', ['clone', '--no-checkout', '--branch', options.defaultBranch, authUrl, dir]);
+  await configureRepoCredentials(dir, options.gitUrl, options.envVars);
   console.log(`Cloned repo for rig ${options.rigId}`);
   return dir;
 }

cloudflare-gastown/scripts/monitor-town.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+# Continuously monitor a town's state via the debug endpoint.
+# Usage: ./scripts/monitor-town.sh [townId] [interval_seconds]
+
+TOWN_ID="${1:-8a6f9375-b806-4ee0-ad6e-1697ea2dbfff}"
+INTERVAL="${2:-15}"
+BASE_URL="${GASTOWN_URL:-https://gastown.kiloapps.io}"
+URL="${BASE_URL}/debug/towns/${TOWN_ID}/status"
+
+echo "Monitoring town ${TOWN_ID} every ${INTERVAL}s"
+echo "Endpoint: ${URL}"
+echo "Press Ctrl+C to stop"
+echo "=========================================="
+
+while true; do
+  RESP=$(curl -s --max-time 10 "${URL}" 2>/dev/null)
+  if [ -z "$RESP" ]; then
+    echo "$(date -u +%H:%M:%S)  [ERROR] No response from ${URL}"
+    sleep "$INTERVAL"
+    continue
+  fi
+
+  echo "$RESP" | python3 -c "
+import sys, json, datetime
+
+try:
+    d = json.load(sys.stdin)
+except:
+    print('$(date -u +%H:%M:%S)  [ERROR] Invalid JSON response')
+    sys.exit(0)
+
+ts = datetime.datetime.utcnow().strftime('%H:%M:%S')
+alarm = d.get('alarmStatus', {})
+agents_info = alarm.get('agents', {})
+beads_info = alarm.get('beads', {})
+patrol_info = alarm.get('patrol', {})
+events = alarm.get('recentEvents', [])
+
+working = agents_info.get('working', 0)
+idle = agents_info.get('idle', 0)
+op = beads_info.get('open', 0)
+ip = beads_info.get('inProgress', 0)
+ir = beads_info.get('inReview', 0)
+failed = beads_info.get('failed', 0)
+orphaned = patrol_info.get('orphanedHooks', 0)
+
+# Agent details
+agents = d.get('agentMeta', [])
+hooked_agents = [a for a in agents if a.get('current_hook_bead_id')]
+refinery = [a for a in agents if a.get('role') == 'refinery']
+
+# Non-terminal beads
+beads = d.get('beadSummary', [])
+
+print(f'{ts}  W={working} I={idle} | open={op} prog={ip} review={ir} fail={failed} | hooks={orphaned} hooked={len(hooked_agents)}')
+
+# Show refinery state
+for r in refinery:
+    hook = r.get('current_hook_bead_id', 'NULL') or 'NULL'
+    print(f'         refinery: status={r.get(\"status\",\"?\"):8s} hook={hook[:12]:12s} dispatch={r.get(\"dispatch_attempts\",0)}')
+
+# Show non-terminal beads
+if beads:
+    for b in beads[:8]:
+        assignee = str(b.get('assignee_agent_bead_id', '') or '')[:8]
+        print(f'         {b.get(\"status\",\"?\"):12s} {b.get(\"type\",\"?\"):16s} {str(b.get(\"bead_id\",\"\"))[:8]}  agent={assignee:8s}  {str(b.get(\"title\",\"\"))[:50]}')
+    if len(beads) > 8:
+        print(f'         ... and {len(beads) - 8} more')
+
+# Show most recent event
+if events:
+    e = events[0]
+    print(f'         last: {e.get(\"time\",\"\")[:19]}  {e.get(\"type\",\"\"):20s}  {e.get(\"message\",\"\")[:70]}')
+
+# Show review outcomes
+review_events = [e for e in events if e.get('type') == 'review_completed']
+for e in review_events[:2]:
+    print(f'         REVIEW: {e.get(\"time\",\"\")[:19]}  {e.get(\"message\",\"\")[:70]}')
+
+print()
+" 2>/dev/null
+
+  sleep "$INTERVAL"
+done