feat(api-proxy): add startup API key validation

containers/api-proxy/server.js

@@ -884,6 +884,265 @@
 /**
  * Build the enhanced health response (superset of original format).
  */
+// ---------------------------------------------------------------------------
+// Startup key validation
+// ---------------------------------------------------------------------------
+
+/**
+ * Validation result for a single provider's API key.
+ * @typedef {'pending'|'valid'|'auth_rejected'|'network_error'|'inconclusive'|'skipped'} ValidationStatus
+ * @typedef {{ status: ValidationStatus, message: string }} ValidationResult
+ */
+
+/** @type {Record<string, ValidationResult>} */
+const keyValidationResults = {};
+
+/** Set to true once validateApiKeys() has finished (regardless of outcome). */
+let keyValidationComplete = false;
+
+/** Reset validation state (used in tests). */
+function resetKeyValidationState() {
+  for (const key of Object.keys(keyValidationResults)) {
+    delete keyValidationResults[key];
+  }
+  keyValidationComplete = false;
+}
+
+/**
+ * Perform a lightweight probe against the provider's API to check if the
+ * configured key is still accepted.  Results are logged and stored in
+ * `keyValidationResults` — the health endpoint exposes them.
+ *
+ * Validation is **non-blocking by default**: the proxy still serves traffic
+ * even if a key is rejected.  Set AWF_VALIDATE_KEYS=strict to exit(1) on
+ * any auth rejection.
+ *
+ * Only validates against known default targets.  Custom/enterprise targets
+ * are skipped because we don't know what probe endpoints they expose.
+ *
+ * @param {object} [overrides={}] - Optional key/target overrides (used in tests)
+ * @param {string} [overrides.openaiKey] - Override OPENAI_API_KEY
+ * @param {string} [overrides.openaiTarget] - Override OPENAI_API_TARGET
+ * @param {string} [overrides.anthropicKey] - Override ANTHROPIC_API_KEY
+ * @param {string} [overrides.anthropicTarget] - Override ANTHROPIC_API_TARGET
+ * @param {string} [overrides.copilotGithubToken] - Override COPILOT_GITHUB_TOKEN
+ * @param {string} [overrides.copilotApiKey] - Override COPILOT_API_KEY
+ * @param {string} [overrides.copilotAuthToken] - Override COPILOT_AUTH_TOKEN
+ * @param {string} [overrides.copilotTarget] - Override COPILOT_API_TARGET
+ * @param {string} [overrides.copilotIntegrationId] - Override COPILOT_INTEGRATION_ID
+ * @param {string} [overrides.geminiKey] - Override GEMINI_API_KEY
+ * @param {string} [overrides.geminiTarget] - Override GEMINI_API_TARGET
+ * @param {number} [overrides.timeoutMs] - Override probe timeout
+ */
+async function validateApiKeys(overrides = {}) {
+  const mode = (process.env.AWF_VALIDATE_KEYS || 'warn').toLowerCase(); // off | warn | strict
+  if (mode === 'off') {
+    logRequest('info', 'key_validation', { message: 'Key validation disabled (AWF_VALIDATE_KEYS=off)' });
+    keyValidationComplete = true;
+    return;
+  }
+
+  const ov = (key, fallback) => key in overrides ? overrides[key] : fallback;
+  const openaiKey = ov('openaiKey', OPENAI_API_KEY);
+  const openaiTarget = ov('openaiTarget', OPENAI_API_TARGET);
+  const anthropicKey = ov('anthropicKey', ANTHROPIC_API_KEY);
+  const anthropicTarget = ov('anthropicTarget', ANTHROPIC_API_TARGET);
+  const copilotGithubToken = ov('copilotGithubToken', COPILOT_GITHUB_TOKEN);
+  const copilotApiKey = ov('copilotApiKey', COPILOT_API_KEY);
+  const copilotAuthToken = ov('copilotAuthToken', COPILOT_AUTH_TOKEN);
+  const copilotTarget = ov('copilotTarget', COPILOT_API_TARGET);
+  const copilotIntegrationId = ov('copilotIntegrationId', COPILOT_INTEGRATION_ID);
+  const geminiKey = ov('geminiKey', GEMINI_API_KEY);
+  const geminiTarget = ov('geminiTarget', GEMINI_API_TARGET);
+  const TIMEOUT_MS = ov('timeoutMs', 10_000);
+
+  const probes = [];
+
+  // --- Copilot (COPILOT_GITHUB_TOKEN only — COPILOT_API_KEY has no probe endpoint) ---
+  if (copilotGithubToken) {
+    if (copilotTarget !== 'api.githubcopilot.com') {
+      keyValidationResults.copilot = { status: 'skipped', message: `Custom target ${copilotTarget}; validation skipped` };
+      logRequest('info', 'key_validation', { provider: 'copilot', ...keyValidationResults.copilot });
+    } else {
+      probes.push(probeProvider('copilot', `https://${copilotTarget}/models`, {
+        method: 'GET',
+        headers: {
+          'Authorization': `Bearer ${copilotGithubToken}`,
+          'Copilot-Integration-Id': copilotIntegrationId,
+        },
+      }, TIMEOUT_MS));
+    }
+  } else if (copilotApiKey && !copilotGithubToken) {
+    keyValidationResults.copilot = { status: 'skipped', message: 'COPILOT_API_KEY configured but startup validation is not supported for this auth mode' };
+    logRequest('info', 'key_validation', { provider: 'copilot', ...keyValidationResults.copilot });
+  }
+
+  // --- OpenAI ---
+  if (openaiKey) {
+    if (openaiTarget !== 'api.openai.com') {
+      keyValidationResults.openai = { status: 'skipped', message: `Custom target ${openaiTarget}; validation skipped` };
+      logRequest('info', 'key_validation', { provider: 'openai', ...keyValidationResults.openai });
+    } else {
+      probes.push(probeProvider('openai', `https://${openaiTarget}/v1/models`, {
+        method: 'GET',
+        headers: { 'Authorization': `Bearer ${openaiKey}` },
+      }, TIMEOUT_MS));
+    }
+  }
+
+  // --- Anthropic ---
+  if (anthropicKey) {
+    if (anthropicTarget !== 'api.anthropic.com') {
+      keyValidationResults.anthropic = { status: 'skipped', message: `Custom target ${anthropicTarget}; validation skipped` };
+      logRequest('info', 'key_validation', { provider: 'anthropic', ...keyValidationResults.anthropic });
+    } else {
+      // POST /v1/messages with an empty body — 400 = key valid (bad body), 401 = key invalid
+      probes.push(probeProvider('anthropic', `https://${anthropicTarget}/v1/messages`, {
+        method: 'POST',
+        headers: {
+          'x-api-key': anthropicKey,
+          'anthropic-version': '2023-06-01',
+          'content-type': 'application/json',
+        },
+        body: '{}',
+      }, TIMEOUT_MS));
+    }
+  }
+
+  // --- Gemini ---
+  if (geminiKey) {
+    if (geminiTarget !== 'generativelanguage.googleapis.com') {
+      keyValidationResults.gemini = { status: 'skipped', message: `Custom target ${geminiTarget}; validation skipped` };
+      logRequest('info', 'key_validation', { provider: 'gemini', ...keyValidationResults.gemini });
+    } else {
+      probes.push(probeProvider('gemini', `https://${geminiTarget}/v1beta/models`, {
+        method: 'GET',
+        headers: { 'x-goog-api-key': geminiKey },
+      }, TIMEOUT_MS));
+    }
+  }
+
+  if (probes.length === 0) {
+    logRequest('info', 'key_validation', { message: 'No providers to validate' });
+    keyValidationComplete = true;
+    return;
+  }
+
+  await Promise.allSettled(probes);
+  keyValidationComplete = true;
+
+  // Summarize
+  const failures = Object.entries(keyValidationResults)
+    .filter(([, r]) => r.status === 'auth_rejected');
+
+  if (failures.length > 0) {
+    for (const [provider, result] of failures) {
+      logRequest('error', 'key_validation_failed', {
+        provider,
+        message: `${provider.toUpperCase()} API key validation failed — ${result.message}. Rotate the secret and re-run.`,
+      });
+    }
+    if (mode === 'strict') {
+      logRequest('error', 'key_validation_strict_exit', {
+        message: `AWF_VALIDATE_KEYS=strict: exiting due to ${failures.length} auth failure(s)`,
+        providers: failures.map(([p]) => p),
+      });
+      process.exit(1);
+    }
+  } else {
+    logRequest('info', 'key_validation', { message: 'All configured API keys validated successfully' });
+  }
+}
+
+/**
+ * Probe a single provider to check if the API key is accepted.
+ *
+ * @param {string} provider - Provider name (copilot, openai, etc.)
+ * @param {string} url - Probe URL
+ * @param {{ method: string, headers: Record<string,string>, body?: string }} opts
+ * @param {number} timeoutMs
+ */
+async function probeProvider(provider, url, opts, timeoutMs) {
+  keyValidationResults[provider] = { status: 'pending', message: 'Validating...' };
+  try {
+    const status = await httpProbe(url, opts, timeoutMs);
+
+    if (status >= 200 && status < 300) {
+      keyValidationResults[provider] = { status: 'valid', message: `HTTP ${status}` };
+      logRequest('info', 'key_validation', { provider, status: 'valid', httpStatus: status });
+    } else if (status === 401 || status === 403) {
+      keyValidationResults[provider] = { status: 'auth_rejected', message: `HTTP ${status} — token expired or invalid` };
+      logRequest('warn', 'key_validation', { provider, status: 'auth_rejected', httpStatus: status });
+    } else if (status === 400) {
+      // 400 for Anthropic means key is valid but request body was bad — expected
+      keyValidationResults[provider] = { status: 'valid', message: `HTTP ${status} (auth accepted, probe body rejected)` };
+      logRequest('info', 'key_validation', { provider, status: 'valid', httpStatus: status, note: 'probe body rejected but auth accepted' });
+    } else {
+      keyValidationResults[provider] = { status: 'inconclusive', message: `HTTP ${status}` };
+      logRequest('warn', 'key_validation', { provider, status: 'inconclusive', httpStatus: status });
+    }
+  } catch (err) {
+    const message = err && err.message ? err.message : String(err);
+    keyValidationResults[provider] = { status: 'network_error', message };
+    logRequest('warn', 'key_validation', { provider, status: 'network_error', error: message });
+  }
+}
+
+/**
+ * Make an HTTPS request through the proxy and return the HTTP status code.
+ *
+ * @param {string} url
+ * @param {{ method: string, headers: Record<string,string>, body?: string }} opts
+ * @param {number} timeoutMs
+ * @returns {Promise<number>} HTTP status code
+ */
+function httpProbe(url, opts, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    const parsed = new URL(url);
+    const isHttps = parsed.protocol === 'https:';
+    const mod = isHttps ? https : http;
+    const reqOpts = {
+      hostname: parsed.hostname,
+      port: parsed.port || (isHttps ? 443 : 80),
+      path: parsed.pathname + parsed.search,
+      method: opts.method,
+      headers: { ...opts.headers },
+      ...(isHttps && proxyAgent ? { agent: proxyAgent } : {}),
+      timeout: timeoutMs,
+    };
+
+    let settled = false;
+    const resolveOnce = (statusCode) => {
+      if (settled) return;
+      settled = true;
+      resolve(statusCode);
+    };
+    const rejectOnce = (err) => {
+      if (settled) return;
+      settled = true;
+      reject(err);
+    };
+
+    const req = mod.request(reqOpts, (res) => {
+      // Consume body to free the socket
+      res.resume();
+      res.on('end', () => resolveOnce(res.statusCode));
+      res.on('error', rejectOnce);
+      res.on('close', () => resolveOnce(res.statusCode));
+    });
+
+    req.on('timeout', () => {
+      req.destroy(new Error(`Probe timed out after ${timeoutMs}ms`));
+    });
+    req.on('error', rejectOnce);
+
+    if (opts.body) {
+      req.write(opts.body);
+    }
+    req.end();
+  });
+}
+
 function healthResponse() {
   return {
     status: 'healthy',
@@ -895,6 +1154,10 @@
       gemini: !!GEMINI_API_KEY,
       copilot: !!COPILOT_AUTH_TOKEN,
     },
+    key_validation: {
+      complete: keyValidationComplete,
+      results: keyValidationResults,
+    },
     metrics_summary: metrics.getSummary(),
     rate_limits: limiter.getAllStatus(),
   };
@@ -923,6 +1186,26 @@
   // Health port is always 10000 — this is what Docker healthcheck hits
   const HEALTH_PORT = 10000;
 
+  // Startup latch: count listeners that participate in key validation.
+  // The no-key Gemini 503 handler binds port 10003 but doesn't participate
+  // in validation, so it's intentionally excluded from the count.
+  let expectedListeners = 1; // port 10000 (always)
+  if (ANTHROPIC_API_KEY) expectedListeners++;
+  if (COPILOT_AUTH_TOKEN) expectedListeners++;
+  if (GEMINI_API_KEY) expectedListeners++;
+  if (OPENAI_API_KEY || ANTHROPIC_API_KEY || COPILOT_AUTH_TOKEN) expectedListeners++; // OpenCode (10004)
+  let readyListeners = 0;
+  function onListenerReady() {
+    readyListeners++;
+    if (readyListeners === expectedListeners) {
+      logRequest('info', 'startup_complete', { message: `All ${expectedListeners} validation-participating listeners ready, starting key validation` });
+      validateApiKeys().catch((err) => {
+        logRequest('error', 'key_validation_error', { message: 'Unexpected error during key validation', error: String(err) });
+        keyValidationComplete = true;
+      });
+    }
+  }
+
   // OpenAI API proxy (port 10000)
   if (OPENAI_API_KEY) {
     const server = http.createServer((req, res) => {
@@ -943,6 +1226,7 @@
 
     server.listen(HEALTH_PORT, '0.0.0.0', () => {
       logRequest('info', 'server_start', { message: `OpenAI proxy listening on port ${HEALTH_PORT}`, target: OPENAI_API_TARGET });
+      onListenerReady();
     });
   } else {
     // No OpenAI key — still need a health endpoint on port 10000 for Docker healthcheck
@@ -960,6 +1244,7 @@
 
     server.listen(HEALTH_PORT, '0.0.0.0', () => {
       logRequest('info', 'server_start', { message: `Health endpoint listening on port ${HEALTH_PORT} (OpenAI not configured)` });
+      onListenerReady();
     });
   }
 
@@ -993,6 +1278,7 @@
 
     server.listen(10001, '0.0.0.0', () => {
       logRequest('info', 'server_start', { message: 'Anthropic proxy listening on port 10001', target: ANTHROPIC_API_TARGET });
+      onListenerReady();
     });
   }
 
@@ -1053,6 +1339,7 @@
 
     copilotServer.listen(10002, '0.0.0.0', () => {
       logRequest('info', 'server_start', { message: 'GitHub Copilot proxy listening on port 10002' });
+      onListenerReady();
     });
   }
 
@@ -1087,6 +1374,7 @@
 
     geminiServer.listen(10003, '0.0.0.0', () => {
       logRequest('info', 'server_start', { message: 'Google Gemini proxy listening on port 10003', target: GEMINI_API_TARGET });
+      onListenerReady();
     });
   } else {
     // No Gemini key — listen on port 10003 and return 503 so the Gemini CLI
@@ -1195,6 +1483,7 @@
 
     opencodeServer.listen(10004, '0.0.0.0', () => {
       logRequest('info', 'server_start', { message: `OpenCode proxy listening on port 10004 (-> ${opencodeStartupRoute.target})` });
+      onListenerReady();
     });
   }
 
@@ -1213,4 +1502,4 @@
 }
 
 // Export for testing
-module.exports = { normalizeApiTarget, deriveCopilotApiTarget, deriveGitHubApiTarget, deriveGitHubApiBasePath, normalizeBasePath, buildUpstreamPath, proxyWebSocket, resolveCopilotAuthToken, resolveOpenCodeRoute, shouldStripHeader, stripGeminiKeyParam };
+module.exports = { normalizeApiTarget, deriveCopilotApiTarget, deriveGitHubApiTarget, deriveGitHubApiBasePath, normalizeBasePath, buildUpstreamPath, proxyWebSocket, resolveCopilotAuthToken, resolveOpenCodeRoute, shouldStripHeader, stripGeminiKeyParam, validateApiKeys, probeProvider, httpProbe, keyValidationResults, resetKeyValidationState };