feat: add Vision Language Model (VLM) support

backend/src/adapters/request/anthropic.ts

@@ -4,6 +4,7 @@
  */
 
 import type {
+  ImageContentBlock,
   InternalContentBlock,
   InternalMessage,
   InternalRequest,
@@ -25,9 +26,10 @@ interface AnthropicContentBlock {
   text?: string;
   thinking?: string;
   source?: {
-    type: "base64";
-    media_type: string;
-    data: string;
+    type: "base64" | "url";
+    media_type?: string;
+    data?: string;
+    url?: string;
   };
   id?: string;
   name?: string;
@@ -137,7 +139,28 @@ function convertContentBlock(
     }
 
     case "image":
-      // Images not supported in MVP
+      // Handle both base64 and URL source types
+      if (block.source?.type === "url" && block.source.url) {
+        return {
+          type: "image",
+          source: {
+            type: "url",
+            url: block.source.url,
+          },
+        } as ImageContentBlock;
+      }
+      // Handle base64 - only if type matches and data is present
+      if (block.source?.type === "base64" && block.source.data) {
+        return {
+          type: "image",
+          source: {
+            type: "base64",
+            mediaType: block.source.media_type,
+            data: block.source.data,
+          },
+        } as ImageContentBlock;
+      }
+      // Skip images with missing data
       return null;
 
     default:

backend/src/adapters/request/openai-chat.ts

@@ -4,6 +4,8 @@
  */
 
 import type {
+  ImageContentBlock,
+  ImageSource,
   InternalContentBlock,
   InternalMessage,
   InternalRequest,
@@ -14,6 +16,35 @@ import type {
   ToolUseContentBlock,
 } from "../types";
 
+// =============================================================================
+// Helper Functions
+// =============================================================================
+
+/**
+ * Parse a data URL into base64 source, or return URL source for regular URLs
+ * Data URL format: data:[<mediatype>][;base64],<data>
+ */
+function parseImageUrl(url: string): ImageSource {
+  if (url.startsWith("data:")) {
+    // Parse data URL: data:image/jpeg;base64,/9j/4AAQ...
+    const match = url.match(/^data:([^;,]+)?(?:;base64)?,(.*)$/);
+    if (match) {
+      const mediaType = match[1] || "image/jpeg";
+      const data = match[2] || "";
+      return {
+        type: "base64",
+        mediaType,
+        data,
+      };
+    }
+  }
+  // Regular URL
+  return {
+    type: "url",
+    url,
+  };
+}
+
 // =============================================================================
 // OpenAI Chat Request Types
 // =============================================================================
@@ -122,13 +153,20 @@ function convertContent(
   if (typeof content === "string") {
     return content;
   }
-  // Array of content parts - currently only support text (images not supported yet)
+  // Array of content parts - support text and image_url
   const blocks: InternalContentBlock[] = [];
   for (const part of content) {
     if (part.type === "text" && part.text) {
       blocks.push({ type: "text", text: part.text });
+    } else if (part.type === "image_url" && part.image_url?.url) {
+      // Parse URL - handles both regular URLs and data URLs (base64)
+      const source = parseImageUrl(part.image_url.url);
+      blocks.push({
+        type: "image",
+        source,
+        detail: part.image_url.detail,
+      } as ImageContentBlock);
     }
-    // Skip image_url parts for now (not supported in MVP)
   }
 
   const [firstBlock] = blocks;

backend/src/adapters/request/openai-response.ts

@@ -4,6 +4,9 @@
  */
 
 import type {
+  ImageContentBlock,
+  ImageSource,
+  InternalContentBlock,
   InternalMessage,
   InternalRequest,
   InternalToolDefinition,
@@ -12,6 +15,35 @@ import type {
   ToolResultContentBlock,
 } from "../types";
 
+// =============================================================================
+// Helper Functions
+// =============================================================================
+
+/**
+ * Parse a data URL into base64 source, or return URL source for regular URLs
+ * Data URL format: data:[<mediatype>][;base64],<data>
+ */
+function parseImageUrl(url: string): ImageSource {
+  if (url.startsWith("data:")) {
+    // Parse data URL: data:image/jpeg;base64,/9j/4AAQ...
+    const match = url.match(/^data:([^;,]+)?(?:;base64)?,(.*)$/);
+    if (match) {
+      const mediaType = match[1] || "image/jpeg";
+      const data = match[2] || "";
+      return {
+        type: "base64",
+        mediaType,
+        data,
+      };
+    }
+  }
+  // Regular URL
+  return {
+    type: "url",
+    url,
+  };
+}
+
 // =============================================================================
 // OpenAI Response API Request Types
 // =============================================================================
@@ -102,13 +134,38 @@ const KNOWN_FIELDS = new Set([
 // =============================================================================
 
 /**
- * Convert Response API content parts to string
+ * Convert Response API content parts to string or content blocks
  */
-function convertContentParts(parts: ResponseApiContentPart[]): string {
-  return parts
-    .filter((p) => p.type === "input_text" || p.type === "text")
-    .map((p) => p.text || "")
-    .join("");
+function convertContentParts(
+  parts: ResponseApiContentPart[],
+): string | InternalContentBlock[] {
+  const hasImages = parts.some((p) => p.type === "input_image");
+
+  if (!hasImages) {
+    // Simple case: text only
+    return parts
+      .filter((p) => p.type === "input_text" || p.type === "text")
+      .map((p) => p.text || "")
+      .join("");
+  }
+
+  // Complex case: includes images
+  const blocks: InternalContentBlock[] = [];
+  for (const part of parts) {
+    if (part.type === "input_text" || part.type === "text") {
+      if (part.text) {
+        blocks.push({ type: "text", text: part.text });
+      }
+    } else if (part.type === "input_image" && part.image_url) {
+      // Parse URL - handles both regular URLs and data URLs (base64)
+      const source = parseImageUrl(part.image_url);
+      blocks.push({
+        type: "image",
+        source,
+      } as ImageContentBlock);
+    }
+  }
+  return blocks;
 }
 
 /**

backend/src/adapters/response/anthropic.ts

@@ -94,6 +94,9 @@ function convertContentBlock(
     case "tool_result":
       // Tool results are not included in Anthropic assistant responses
       return null;
+    case "image":
+      // Images are not included in assistant responses (only in requests)
+      return null;
   }
 }
 

backend/src/adapters/types.ts

@@ -45,14 +45,38 @@ export interface ToolResultContentBlock {
   isError?: boolean;
 }
 
+/**
+ * Image source types - discriminated union for type safety
+ */
+export type ImageSource =
+  | {
+      type: "base64";
+      mediaType?: string; // "image/jpeg", "image/png", etc.
+      data: string;
+    }
+  | {
+      type: "url";
+      url: string;
+    };
+
+/**
+ * Image content block - represents an image input for vision models
+ */
+export interface ImageContentBlock {
+  type: "image";
+  source: ImageSource;
+  detail?: "auto" | "low" | "high"; // OpenAI vision detail level
+}
+
 /**
  * Union type for all content blocks
  */
 export type InternalContentBlock =
   | TextContentBlock
   | ThinkingContentBlock
   | ToolUseContentBlock
-  | ToolResultContentBlock;
+  | ToolResultContentBlock
+  | ImageContentBlock;
 
 // =============================================================================
 // Message Types

backend/src/adapters/upstream/anthropic.ts

@@ -26,6 +26,12 @@ interface AnthropicContentBlock {
   type: "text" | "image" | "tool_use" | "tool_result" | "thinking";
   text?: string;
   thinking?: string;
+  source?: {
+    type: "base64" | "url";
+    media_type?: string;
+    data?: string;
+    url?: string;
+  };
   id?: string;
   name?: string;
   input?: Record<string, unknown>;
@@ -195,6 +201,27 @@ function convertMessage(msg: InternalMessage): AnthropicMessage | null {
         text: block.text,
         cache_control: block.cacheControl,
       });
+    } else if (block.type === "image") {
+      // Only push image blocks with valid data
+      if (block.source.type === "base64" && block.source.data) {
+        content.push({
+          type: "image",
+          source: {
+            type: "base64",
+            media_type: block.source.mediaType || "image/jpeg",
+            data: block.source.data,
+          },
+        });
+      } else if (block.source.type === "url" && block.source.url) {
+        // Anthropic also supports URL source type
+        content.push({
+          type: "image",
+          source: {
+            type: "url",
+            url: block.source.url,
+          },
+        });
+      }
     }
   }
 

backend/src/adapters/upstream/openai-responses.ts

@@ -16,14 +16,17 @@ import type {
   ToolUseContentBlock,
   UpstreamAdapter,
 } from "../types";
+import { convertImageToUrl, hasImages } from "./utils";
 
 // =============================================================================
 // Response API Types
 // =============================================================================
 
 interface ResponseApiContentPart {
-  type: "input_text" | "output_text" | "refusal";
+  type: "input_text" | "output_text" | "refusal" | "input_image";
   text?: string;
+  image_url?: string;
+  detail?: "auto" | "low" | "high";
 }
 
 interface ResponseApiInputItem {
@@ -126,14 +129,54 @@ function convertMessage(msg: InternalMessage): ResponseApiInputItem | null {
     };
   }
 
-  // Regular messages
-  const content =
-    typeof msg.content === "string"
-      ? msg.content
-      : msg.content
-          .filter((b) => b.type === "text")
-          .map((b) => b.text)
-          .join("");
+  // Handle string content
+  if (typeof msg.content === "string") {
+    return {
+      type: "message",
+      role: msg.role,
+      content: msg.content,
+    };
+  }
+
+  // Handle content array - check if it contains images
+  if (hasImages(msg.content)) {
+    // Build content array with input_text and input_image parts
+    const contentParts: ResponseApiContentPart[] = [];
+    for (const block of msg.content) {
+      if (block.type === "text") {
+        contentParts.push({ type: "input_text", text: block.text });
+      } else if (block.type === "image") {
+        // Only include images with valid data
+        const imageUrl = convertImageToUrl(block);
+        if (imageUrl) {
+          contentParts.push({
+            type: "input_image",
+            image_url: imageUrl,
+            detail: block.detail,
+          });
+        }
+      }
+    }
+    // Ensure we don't send empty content array to API
+    if (contentParts.length === 0) {
+      return {
+        type: "message",
+        role: msg.role,
+        content: "",
+      };
+    }
+    return {
+      type: "message",
+      role: msg.role,
+      content: contentParts,
+    };
+  }
+
+  // Text-only content - join as string
+  const content = msg.content
+    .filter((b) => b.type === "text")
+    .map((b) => b.text)
+    .join("");
 
   return {
     type: "message",