refactor(dev): improve debug log organization

- Move raw console logs to debug/console/ subdirectory
- Rename pino logs to .jsonl extension for clarity
- Rename web.log to next.log to distinguish from pino web.jsonl
- Simplify log paths (removed debug/proc/ nesting)

New structure:
  debug/
  ├── cli.jsonl      (CLI app, structured JSON)
  ├── web.jsonl      (Web app, structured JSON)
  └── console/
      ├── next.log   (Next.js framework)
      ├── db.log     (Database startup)
      ├── sdk.log    (SDK build)
      └── studio.log (Drizzle Studio)

Author: brandonkachen

cli/src/utils/logger.ts

@@ -1,4 +1,4 @@
-import { existsSync, mkdirSync, unlinkSync } from 'fs'
+import { appendFileSync, existsSync, mkdirSync, unlinkSync } from 'fs'
 import path, { dirname } from 'path'
 import { format as stringFormat } from 'util'
 
@@ -49,7 +49,7 @@ function setLogPath(p: string): void {
   const fileStream = pino.destination({
     dest: p, // absolute or relative file path
     mkdir: true, // create parent dirs if they don’t exist
-    sync: false, // set true if you *must* block on every write
+    sync: true, // set true if you *must* block on every write
   })
 
   pinoLogger = pino(
@@ -66,7 +66,7 @@ function setLogPath(p: string): void {
 
 export function clearLogFile(): void {
   const projectRoot = getProjectRoot()
-  const defaultLog = path.join(projectRoot, 'debug', 'cli.log')
+  const defaultLog = path.join(projectRoot, 'debug', 'cli.jsonl')
   const targets = new Set<string>()
 
   if (logPath) {
@@ -102,7 +102,7 @@ function sendAnalyticsAndLog(
 
     const logTarget =
       env.NEXT_PUBLIC_CB_ENVIRONMENT === 'dev'
-        ? path.join(projectRoot, 'debug', 'cli.log')
+        ? path.join(projectRoot, 'debug', 'cli.jsonl')
         : path.join(getCurrentChatDir(), 'log.jsonl')
 
     setLogPath(logTarget)
@@ -143,7 +143,22 @@ function sendAnalyticsAndLog(
     trackEvent(analyticsEventId, toTrack)
   }
 
-  if (pinoLogger !== undefined) {
+  // In dev mode, use appendFileSync for real-time logging (Bun has issues with pino sync)
+  // In prod mode, use pino for better performance
+  if (env.NEXT_PUBLIC_CB_ENVIRONMENT === 'dev' && logPath) {
+    const logEntry = JSON.stringify({
+      level: level.toUpperCase(),
+      timestamp: new Date().toISOString(),
+      ...loggerContext,
+      ...(includeData ? { data: normalizedData } : {}),
+      msg: stringFormat(normalizedMsg ?? '', ...args),
+    })
+    try {
+      appendFileSync(logPath, logEntry + '\n')
+    } catch {
+      // Ignore write errors
+    }
+  } else if (pinoLogger !== undefined) {
     const base = { ...loggerContext }
     const obj = includeData ? { ...base, data: normalizedData } : base
     pinoLogger[level](obj, normalizedMsg as any, ...args)

scripts/dev.sh

@@ -8,7 +8,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
 export PATH="$PROJECT_ROOT/.bin:$PATH"
 
-LOG_DIR="$PROJECT_ROOT/debug/proc"
+LOG_DIR="$PROJECT_ROOT/debug/console"
 mkdir -p "$LOG_DIR"
 
 # =============================================================================
@@ -61,7 +61,7 @@ kill_proc() {
     local name="$1"
     local pattern="$2"
     if pkill -f "$pattern" 2>/dev/null; then
-        echo "[$(date '+%H:%M:%S')] Killed $name" >> "$LOG_DIR/$name.log"
+        echo "[$(date '+%H:%M:%S')] Killed $name" >> "$LOG_DIR/$name.log" 2>/dev/null
         return 0
     fi
     return 1
@@ -73,8 +73,9 @@ cleanup() {
     echo "Shutting down..."
     echo ""
 
-    kill_proc "web" 'bun.*--cwd web dev' || kill_proc "web" 'next-server'
-    ok "web" "stopped"
+    # Kill web server (tmux session or background process)
+    tmux kill-session -t codebuff-web 2>/dev/null || kill_proc "next" 'bun.*--cwd web dev' || kill_proc "next" 'next-server'
+    ok "next" "stopped"
 
     kill_proc "studio" 'drizzle-kit studio' && ok "studio" "stopped"
     kill_proc "sdk" 'bun.*--cwd sdk' && ok "sdk" "stopped"
@@ -95,18 +96,44 @@ printf "  %s %-10s starting...\r" "${SPINNER_FRAMES[0]}" "db"
 bun --cwd packages/internal db:start > "$LOG_DIR/db.log" 2>&1
 ok "db" "ready!"
 
-# 2. Background processes (fire and forget)
-bun --cwd sdk run build > "$LOG_DIR/sdk.log" 2>&1 &
-ok "sdk" "(background)"
+# 2. Background processes
+bun run --cwd sdk build > "$LOG_DIR/sdk.log" 2>&1 &
 
 bun --cwd packages/internal db:studio > "$LOG_DIR/studio.log" 2>&1 &
 ok "studio" "(background)"
 
 # 3. Web server (wait for ready)
-bun --cwd web dev > "$LOG_DIR/web.log" 2>&1 &
-wait_for_log "web" "Ready in" 60
+tmux kill-session -t codebuff-web 2>/dev/null
+pkill -f 'next-server' 2>/dev/null
+
+# Start web server in tmux with unbuffer for real-time logging
+# unbuffer creates a pseudo-TTY which prevents output buffering
+if command -v unbuffer &>/dev/null; then
+    tmux new-session -d -s codebuff-web "cd $PROJECT_ROOT && unbuffer $PROJECT_ROOT/.bin/bun --cwd web dev 2>&1 | tee $LOG_DIR/next.log"
+else
+    # Fallback without unbuffer (logs may be delayed)
+    tmux new-session -d -s codebuff-web "cd $PROJECT_ROOT && $PROJECT_ROOT/.bin/bun --cwd web dev 2>&1 | tee $LOG_DIR/next.log"
+fi
+
+# Wait for web server to be ready by polling the health endpoint
+printf "  %s %-10s starting..." "${SPINNER_FRAMES[0]}" "next"
+frame=0
+elapsed=0
+while ! curl -sf http://localhost:3000/api/healthz > /dev/null 2>&1; do
+    printf "\r  %s %-10s starting..." "${SPINNER_FRAMES[$frame]}" "next"
+    frame=$(( (frame + 1) % ${#SPINNER_FRAMES[@]} ))
+    sleep 0.5
+    elapsed=$(( elapsed + 1 ))
+    if (( elapsed > 120 )); then  # 60 second timeout (0.5s * 120)
+        echo ""
+        echo "Timeout waiting for web server" >&2
+        exit 1
+    fi
+done
+printf "\r"
+ok "next" "ready!"
 
-# 4. CLI
+# 4. CLI (SDK builds in background)
 echo ""
 echo "Starting CLI..."
 bun --cwd cli dev "$@"

sdk/package.json

@@ -39,7 +39,8 @@
     "fetch-ripgrep": "bun scripts/fetch-ripgrep.ts",
     "prepack": "bun run build",
     "typecheck": "tsc --noEmit -p .",
-    "test": "bun test"
+    "test": "bun test",
+    "dev": "bun run build"
   },
   "engines": {
     "node": ">=18.0.0"

web/src/util/logger.ts

@@ -1,4 +1,4 @@
-import fs from 'fs'
+import fs, { appendFileSync } from 'fs'
 import path from 'path'
 import { format } from 'util'
 
@@ -19,40 +19,32 @@ function getDebugDir(): string | null {
   if (debugDir !== undefined) {
     return debugDir
   }
-  debugDir = __dirname
-  while (true) {
-    if (fs.existsSync(path.join(debugDir, '.git'))) {
-      break
+  // Walk up from cwd to find the git root (where .git exists)
+  let dir = process.cwd()
+  while (dir !== path.dirname(dir)) {
+    if (fs.existsSync(path.join(dir, '.git'))) {
+      debugDir = path.join(dir, 'debug')
+      return debugDir
     }
-    const parent = path.dirname(debugDir)
-    if (parent === debugDir) {
-      debugDir = null
-      console.error('Failed to find git root directory for logger')
-      break
-    }
-    debugDir = parent
-  }
-
-  if (debugDir) {
-    debugDir = path.join(debugDir, 'debug')
+    dir = path.dirname(dir)
   }
-
+  debugDir = null
+  console.error('Failed to find git root directory for logger')
   return debugDir
 }
 
-setLocalDebugDir: if (
+// Initialize debug directory in dev environment
+if (
   env.NEXT_PUBLIC_CB_ENVIRONMENT === 'dev' &&
   process.env.CODEBUFF_GITHUB_ACTIONS !== 'true'
 ) {
-  const debugDir = getDebugDir()
-  if (!debugDir) {
-    break setLocalDebugDir
-  }
-
-  try {
-    fs.mkdirSync(debugDir, { recursive: true })
-  } catch (err) {
-    console.error('Failed to create debug directory:', err)
+  const dir = getDebugDir()
+  if (dir) {
+    try {
+      fs.mkdirSync(dir, { recursive: true })
+    } catch {
+      // Ignore errors when creating debug directory
+    }
   }
 }
 
@@ -67,12 +59,10 @@ const pinoLogger = pino(
     timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
   },
   debugDir
-    ? pino.transport({
-        target: 'pino/file',
-        options: {
-          destination: path.join(debugDir, 'web.log'),
-        },
-        level: 'debug',
+    ? pino.destination({
+        dest: path.join(debugDir, 'web.jsonl'),
+        mkdir: true,
+        sync: true, // sync writes for real-time logging
       })
     : undefined,
 )
@@ -108,18 +98,43 @@ function splitAndLog(
   })
 }
 
-export const logger: Record<LogLevel, pino.LogFn> =
-  env.NEXT_PUBLIC_CB_ENVIRONMENT === 'dev'
-    ? pinoLogger
-    : (Object.fromEntries(
-        loggingLevels.map((level) => {
-          return [
-            level,
-            (data: any, msg?: string, ...args: any[]) =>
-              splitAndLog(level, data, msg, ...args),
-          ]
-        }),
-      ) as Record<LogLevel, pino.LogFn>)
+// In dev mode, use appendFileSync for real-time file logging (Bun has issues with pino sync)
+// Also output to console via pinoLogger so logs remain visible in the terminal
+function logWithSync(level: LogLevel, data: any, msg?: string, ...args: any[]): void {
+  const formattedMsg = format(msg ?? '', ...args)
+  
+  if (env.NEXT_PUBLIC_CB_ENVIRONMENT === 'dev') {
+    // Write to file for real-time logging
+    if (debugDir) {
+      const logEntry = JSON.stringify({
+        level: level.toUpperCase(),
+        timestamp: new Date().toISOString(),
+        ...(data && typeof data === 'object' ? data : { data }),
+        msg: formattedMsg,
+      })
+      try {
+        appendFileSync(path.join(debugDir, 'web.jsonl'), logEntry + '\n')
+      } catch {
+        // Ignore write errors
+      }
+    }
+    // Also output to console for interactive debugging
+    pinoLogger[level](data, msg, ...args)
+  } else {
+    // In prod, use pino with splitAndLog for large payloads
+    splitAndLog(level, data, msg, ...args)
+  }
+}
+
+export const logger: Record<LogLevel, pino.LogFn> = Object.fromEntries(
+  loggingLevels.map((level) => {
+    return [
+      level,
+      (data: any, msg?: string, ...args: any[]) =>
+        logWithSync(level, data, msg, ...args),
+    ]
+  }),
+) as Record<LogLevel, pino.LogFn>
 
 export function loggerWithContext(
   context: ParamsOf<LoggerWithContextFn>,