From 746b87743aa67a25016df3955be9569d513eeef2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 5 Aug 2025 15:04:50 -0700 Subject: [PATCH 01/13] feat(ollama): added streaming & tool call support for ollama, updated docs (#884) --- .github/CONTRIBUTING.md | 8 +- README.md | 24 +- apps/sim/.env.example | 3 + .../app/api/providers/ollama/models/route.ts | 52 ++ .../workflow-block/workflow-block.tsx | 40 +- apps/sim/blocks/blocks/agent.ts | 14 +- apps/sim/blocks/types.ts | 31 +- apps/sim/executor/resolver/resolver.ts | 45 +- apps/sim/providers/ollama/index.ts | 718 +++++++++++++----- apps/sim/providers/utils.ts | 7 + apps/sim/scripts/ollama_docker.sh | 25 - apps/sim/stores/ollama/store.ts | 63 +- apps/sim/stores/ollama/types.ts | 2 + docker-compose.ollama.yml | 143 +++- 14 files changed, 868 insertions(+), 307 deletions(-) create mode 100644 apps/sim/app/api/providers/ollama/models/route.ts delete mode 100755 apps/sim/scripts/ollama_docker.sh diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 4d0d8df90d0..469ce8aa0ff 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -164,10 +164,14 @@ Access the application at [http://localhost:3000/](http://localhost:3000/) To use local models with Sim: -1. Pull models using our helper script: +1. Install Ollama and pull models: ```bash -./apps/sim/scripts/ollama_docker.sh pull +# Install Ollama (if not already installed) +curl -fsSL https://ollama.ai/install.sh | sh + +# Pull a model (e.g., gemma3:4b) +ollama pull gemma3:4b ``` 2. Start Sim with local model support: diff --git a/README.md b/README.md index f9855815e9f..be3b0ec97cf 100644 --- a/README.md +++ b/README.md @@ -59,27 +59,21 @@ docker compose -f docker-compose.prod.yml up -d Access the application at [http://localhost:3000/](http://localhost:3000/) -#### Using Local Models +#### Using Local Models with Ollama -To use local models with Sim: - -1. Pull models using our helper script: +Run Sim with local AI models using [Ollama](https://ollama.ai) - no external APIs required: ```bash -./apps/sim/scripts/ollama_docker.sh pull -``` +# Start with GPU support (automatically downloads gemma3:4b model) +docker compose -f docker-compose.ollama.yml --profile setup up -d -2. Start Sim with local model support: +# For CPU-only systems: +docker compose -f docker-compose.ollama.yml --profile cpu --profile setup up -d +``` +Wait for the model to download, then visit [http://localhost:3000](http://localhost:3000). Add more models with: ```bash -# With NVIDIA GPU support -docker compose --profile local-gpu -f docker-compose.ollama.yml up -d - -# Without GPU (CPU only) -docker compose --profile local-cpu -f docker-compose.ollama.yml up -d - -# If hosting on a server, update the environment variables in the docker-compose.prod.yml file to include the server's public IP then start again (OLLAMA_URL to i.e. http://1.1.1.1:11434) -docker compose -f docker-compose.prod.yml up -d +docker compose -f docker-compose.ollama.yml exec ollama ollama pull llama3.1:8b ``` ### Option 3: Dev Containers diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 5c126fb3230..ee2c0f84d69 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -15,3 +15,6 @@ ENCRYPTION_KEY=your_encryption_key # Use `openssl rand -hex 32` to generate # RESEND_API_KEY= # Uncomment and add your key from https://resend.com to send actual emails # If left commented out, emails will be logged to console instead +# Local AI Models (Optional) +# OLLAMA_URL=http://localhost:11434 # URL for local Ollama server - uncomment if using local models + diff --git a/apps/sim/app/api/providers/ollama/models/route.ts b/apps/sim/app/api/providers/ollama/models/route.ts new file mode 100644 index 00000000000..7c184588b60 --- /dev/null +++ b/apps/sim/app/api/providers/ollama/models/route.ts @@ -0,0 +1,52 @@ +import { type NextRequest, NextResponse } from 'next/server' +import { env } from '@/lib/env' +import { createLogger } from '@/lib/logs/console/logger' +import type { ModelsObject } from '@/providers/ollama/types' + +const logger = createLogger('OllamaModelsAPI') +const OLLAMA_HOST = env.OLLAMA_URL || 'http://localhost:11434' + +export const dynamic = 'force-dynamic' + +/** + * Get available Ollama models + */ +export async function GET(request: NextRequest) { + try { + logger.info('Fetching Ollama models', { + host: OLLAMA_HOST, + }) + + const response = await fetch(`${OLLAMA_HOST}/api/tags`, { + headers: { + 'Content-Type': 'application/json', + }, + }) + + if (!response.ok) { + logger.warn('Ollama service is not available', { + status: response.status, + statusText: response.statusText, + }) + return NextResponse.json({ models: [] }) + } + + const data = (await response.json()) as ModelsObject + const models = data.models.map((model) => model.name) + + logger.info('Successfully fetched Ollama models', { + count: models.length, + models, + }) + + return NextResponse.json({ models }) + } catch (error) { + logger.error('Failed to fetch Ollama models', { + error: error instanceof Error ? error.message : 'Unknown error', + host: OLLAMA_HOST, + }) + + // Return empty array instead of error to avoid breaking the UI + return NextResponse.json({ models: [] }) + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx index 0a9907e271b..cb110085319 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx @@ -405,33 +405,37 @@ export function WorkflowBlock({ id, data }: NodeProps) { // If there's no condition, the block should be shown if (!block.condition) return true + // If condition is a function, call it to get the actual condition object + const actualCondition = + typeof block.condition === 'function' ? block.condition() : block.condition + // Get the values of the fields this block depends on from the appropriate state - const fieldValue = stateToUse[block.condition.field]?.value - const andFieldValue = block.condition.and - ? stateToUse[block.condition.and.field]?.value + const fieldValue = stateToUse[actualCondition.field]?.value + const andFieldValue = actualCondition.and + ? stateToUse[actualCondition.and.field]?.value : undefined // Check if the condition value is an array - const isValueMatch = Array.isArray(block.condition.value) + const isValueMatch = Array.isArray(actualCondition.value) ? fieldValue != null && - (block.condition.not - ? !block.condition.value.includes(fieldValue as string | number | boolean) - : block.condition.value.includes(fieldValue as string | number | boolean)) - : block.condition.not - ? fieldValue !== block.condition.value - : fieldValue === block.condition.value + (actualCondition.not + ? !actualCondition.value.includes(fieldValue as string | number | boolean) + : actualCondition.value.includes(fieldValue as string | number | boolean)) + : actualCondition.not + ? fieldValue !== actualCondition.value + : fieldValue === actualCondition.value // Check both conditions if 'and' is present const isAndValueMatch = - !block.condition.and || - (Array.isArray(block.condition.and.value) + !actualCondition.and || + (Array.isArray(actualCondition.and.value) ? andFieldValue != null && - (block.condition.and.not - ? !block.condition.and.value.includes(andFieldValue as string | number | boolean) - : block.condition.and.value.includes(andFieldValue as string | number | boolean)) - : block.condition.and.not - ? andFieldValue !== block.condition.and.value - : andFieldValue === block.condition.and.value) + (actualCondition.and.not + ? !actualCondition.and.value.includes(andFieldValue as string | number | boolean) + : actualCondition.and.value.includes(andFieldValue as string | number | boolean)) + : actualCondition.and.not + ? andFieldValue !== actualCondition.and.value + : andFieldValue === actualCondition.and.value) return isValueMatch && isAndValueMatch }) diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index f1eb04a96fc..4e02bf40adb 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -12,6 +12,12 @@ import { MODELS_WITH_TEMPERATURE_SUPPORT, providers, } from '@/providers/utils' + +// Get current Ollama models dynamically +const getCurrentOllamaModels = () => { + return useOllamaStore.getState().models +} + import { useOllamaStore } from '@/stores/ollama/store' import type { ToolResponse } from '@/tools/types' @@ -213,14 +219,18 @@ Create a system prompt appropriately detailed for the request, using clear langu password: true, connectionDroppable: false, required: true, - // Hide API key for all hosted models when running on hosted version + // Hide API key for hosted models and Ollama models condition: isHosted ? { field: 'model', value: getHostedModels(), not: true, // Show for all models EXCEPT those listed } - : undefined, // Show for all models in non-hosted environments + : () => ({ + field: 'model', + value: getCurrentOllamaModels(), + not: true, // Show for all models EXCEPT Ollama models + }), }, { id: 'azureEndpoint', diff --git a/apps/sim/blocks/types.ts b/apps/sim/blocks/types.ts index e1818e386bc..8cd1391742e 100644 --- a/apps/sim/blocks/types.ts +++ b/apps/sim/blocks/types.ts @@ -118,16 +118,27 @@ export interface SubBlockConfig { hidden?: boolean description?: string value?: (params: Record) => string - condition?: { - field: string - value: string | number | boolean | Array - not?: boolean - and?: { - field: string - value: string | number | boolean | Array | undefined - not?: boolean - } - } + condition?: + | { + field: string + value: string | number | boolean | Array + not?: boolean + and?: { + field: string + value: string | number | boolean | Array | undefined + not?: boolean + } + } + | (() => { + field: string + value: string | number | boolean | Array + not?: boolean + and?: { + field: string + value: string | number | boolean | Array | undefined + not?: boolean + } + }) // Props specific to 'code' sub-block type language?: 'javascript' | 'json' generationType?: GenerationType diff --git a/apps/sim/executor/resolver/resolver.ts b/apps/sim/executor/resolver/resolver.ts index 51101c3104f..13fbdd12109 100644 --- a/apps/sim/executor/resolver/resolver.ts +++ b/apps/sim/executor/resolver/resolver.ts @@ -58,7 +58,7 @@ export class InputResolver { /** * Evaluates if a sub-block should be active based on its condition - * @param condition - The condition to evaluate + * @param condition - The condition to evaluate (can be static object or function) * @param currentValues - Current values of all inputs * @returns True if the sub-block should be active */ @@ -70,37 +70,46 @@ export class InputResolver { not?: boolean and?: { field: string; value: any; not?: boolean } } + | (() => { + field: string + value: any + not?: boolean + and?: { field: string; value: any; not?: boolean } + }) | undefined, currentValues: Record ): boolean { if (!condition) return true + // If condition is a function, call it to get the actual condition object + const actualCondition = typeof condition === 'function' ? condition() : condition + // Get the field value - const fieldValue = currentValues[condition.field] + const fieldValue = currentValues[actualCondition.field] // Check if the condition value is an array - const isValueMatch = Array.isArray(condition.value) + const isValueMatch = Array.isArray(actualCondition.value) ? fieldValue != null && - (condition.not - ? !condition.value.includes(fieldValue) - : condition.value.includes(fieldValue)) - : condition.not - ? fieldValue !== condition.value - : fieldValue === condition.value + (actualCondition.not + ? !actualCondition.value.includes(fieldValue) + : actualCondition.value.includes(fieldValue)) + : actualCondition.not + ? fieldValue !== actualCondition.value + : fieldValue === actualCondition.value // Check both conditions if 'and' is present const isAndValueMatch = - !condition.and || + !actualCondition.and || (() => { - const andFieldValue = currentValues[condition.and!.field] - return Array.isArray(condition.and!.value) + const andFieldValue = currentValues[actualCondition.and!.field] + return Array.isArray(actualCondition.and!.value) ? andFieldValue != null && - (condition.and!.not - ? !condition.and!.value.includes(andFieldValue) - : condition.and!.value.includes(andFieldValue)) - : condition.and!.not - ? andFieldValue !== condition.and!.value - : andFieldValue === condition.and!.value + (actualCondition.and!.not + ? !actualCondition.and!.value.includes(andFieldValue) + : actualCondition.and!.value.includes(andFieldValue)) + : actualCondition.and!.not + ? andFieldValue !== actualCondition.and!.value + : andFieldValue === actualCondition.and!.value })() return isValueMatch && isAndValueMatch diff --git a/apps/sim/providers/ollama/index.ts b/apps/sim/providers/ollama/index.ts index 7dc2bbb409a..3bf99217b90 100644 --- a/apps/sim/providers/ollama/index.ts +++ b/apps/sim/providers/ollama/index.ts @@ -1,6 +1,7 @@ import OpenAI from 'openai' import { env } from '@/lib/env' import { createLogger } from '@/lib/logs/console/logger' +import type { StreamingExecution } from '@/executor/types' import type { ModelsObject } from '@/providers/ollama/types' import type { ProviderConfig, @@ -8,12 +9,57 @@ import type { ProviderResponse, TimeSegment, } from '@/providers/types' +import { + prepareToolExecution, + prepareToolsWithUsageControl, + trackForcedToolUsage, +} from '@/providers/utils' import { useOllamaStore } from '@/stores/ollama/store' import { executeTool } from '@/tools' const logger = createLogger('OllamaProvider') const OLLAMA_HOST = env.OLLAMA_URL || 'http://localhost:11434' +/** + * Helper function to convert an Ollama stream to a standard ReadableStream + * and collect completion metrics + */ +function createReadableStreamFromOllamaStream( + ollamaStream: any, + onComplete?: (content: string, usage?: any) => void +): ReadableStream { + let fullContent = '' + let usageData: any = null + + return new ReadableStream({ + async start(controller) { + try { + for await (const chunk of ollamaStream) { + // Check for usage data in the final chunk + if (chunk.usage) { + usageData = chunk.usage + } + + const content = chunk.choices[0]?.delta?.content || '' + if (content) { + fullContent += content + controller.enqueue(new TextEncoder().encode(content)) + } + } + + // Once stream is complete, call the completion callback with the final content and usage + if (onComplete) { + onComplete(fullContent, usageData) + } + + controller.close() + } catch (error) { + controller.error(error) + } + }, + }) +} + export const ollamaProvider: ProviderConfig = { id: 'ollama', name: 'Ollama', @@ -46,91 +92,238 @@ export const ollamaProvider: ProviderConfig = { } }, - executeRequest: async (request: ProviderRequest): Promise => { + executeRequest: async ( + request: ProviderRequest + ): Promise => { logger.info('Preparing Ollama request', { model: request.model, hasSystemPrompt: !!request.systemPrompt, - hasMessages: !!request.context, + hasMessages: !!request.messages?.length, hasTools: !!request.tools?.length, toolCount: request.tools?.length || 0, hasResponseFormat: !!request.responseFormat, + stream: !!request.stream, + }) + + // Create Ollama client using OpenAI-compatible API + const ollama = new OpenAI({ + apiKey: 'empty', + baseURL: `${OLLAMA_HOST}/v1`, }) - const startTime = Date.now() + // Start with an empty array for all messages + const allMessages = [] - try { - // Prepare messages array - const ollama = new OpenAI({ - apiKey: 'empty', - baseURL: `${OLLAMA_HOST}/v1`, + // Add system prompt if present + if (request.systemPrompt) { + allMessages.push({ + role: 'system', + content: request.systemPrompt, }) + } - // Start with an empty array for all messages - const allMessages = [] + // Add context if present + if (request.context) { + allMessages.push({ + role: 'user', + content: request.context, + }) + } - // Add system prompt if present - if (request.systemPrompt) { - allMessages.push({ role: 'system', content: request.systemPrompt }) - } + // Add remaining messages + if (request.messages) { + allMessages.push(...request.messages) + } - // Add context if present - if (request.context) { - allMessages.push({ role: 'user', content: request.context }) - } + // Transform tools to OpenAI format if provided + const tools = request.tools?.length + ? request.tools.map((tool) => ({ + type: 'function', + function: { + name: tool.id, + description: tool.description, + parameters: tool.parameters, + }, + })) + : undefined + + // Build the request payload + const payload: any = { + model: request.model, + messages: allMessages, + } - // Add remaining messages - if (request.messages) { - allMessages.push(...request.messages) + // Add optional parameters + if (request.temperature !== undefined) payload.temperature = request.temperature + if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens + + // Add response format for structured output if specified + if (request.responseFormat) { + // Use OpenAI's JSON schema format (Ollama supports this) + payload.response_format = { + type: 'json_schema', + json_schema: { + name: request.responseFormat.name || 'response_schema', + schema: request.responseFormat.schema || request.responseFormat, + strict: request.responseFormat.strict !== false, + }, } - // Build the basic payload - const payload: any = { - model: request.model, - messages: allMessages, - stream: false, + logger.info('Added JSON schema response format to Ollama request') + } + + // Handle tools and tool usage control + let preparedTools: ReturnType | null = null + + if (tools?.length) { + preparedTools = prepareToolsWithUsageControl(tools, request.tools, logger, 'ollama') + const { tools: filteredTools, toolChoice } = preparedTools + + if (filteredTools?.length && toolChoice) { + payload.tools = filteredTools + // Ollama supports 'auto' but not forced tool selection - convert 'force' to 'auto' + payload.tool_choice = typeof toolChoice === 'string' ? toolChoice : 'auto' + + logger.info('Ollama request configuration:', { + toolCount: filteredTools.length, + toolChoice: payload.tool_choice, + model: request.model, + }) } + } - // Add optional parameters - if (request.temperature !== undefined) payload.temperature = request.temperature - if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens - - // Transform tools to OpenAI format if provided - const tools = request.tools?.length - ? request.tools.map((tool) => ({ - type: 'function', - function: { - name: tool.id, - description: tool.description, - parameters: tool.parameters, - }, - })) - : undefined - - // Handle tools and tool usage control - if (tools?.length) { - // Filter out any tools with usageControl='none', but ignore 'force' since Ollama doesn't support it - const filteredTools = tools.filter((tool) => { - const toolId = tool.function?.name - const toolConfig = request.tools?.find((t) => t.id === toolId) - // Only filter out 'none', treat 'force' as 'auto' - return toolConfig?.usageControl !== 'none' + // Start execution timer for the entire provider execution + const providerStartTime = Date.now() + const providerStartTimeISO = new Date(providerStartTime).toISOString() + + try { + // Check if we can stream directly (no tools required) + if (request.stream && (!tools || tools.length === 0)) { + logger.info('Using streaming response for Ollama request') + + // Create a streaming request with token usage tracking + const streamResponse = await ollama.chat.completions.create({ + ...payload, + stream: true, + stream_options: { include_usage: true }, }) - if (filteredTools?.length) { - payload.tools = filteredTools - // Always use 'auto' for Ollama, regardless of the tool_choice setting - payload.tool_choice = 'auto' + // Start collecting token usage from the stream + const tokenUsage = { + prompt: 0, + completion: 0, + total: 0, + } + + // Create a StreamingExecution response with a callback to update content and tokens + const streamingResult = { + stream: createReadableStreamFromOllamaStream(streamResponse, (content, usage) => { + // Update the execution data with the final content and token usage + streamingResult.execution.output.content = content + + // Clean up the response content + if (content) { + streamingResult.execution.output.content = content + .replace(/```json\n?|\n?```/g, '') + .trim() + } + + // Update the timing information with the actual completion time + const streamEndTime = Date.now() + const streamEndTimeISO = new Date(streamEndTime).toISOString() + + if (streamingResult.execution.output.providerTiming) { + streamingResult.execution.output.providerTiming.endTime = streamEndTimeISO + streamingResult.execution.output.providerTiming.duration = + streamEndTime - providerStartTime + + // Update the time segment as well + if (streamingResult.execution.output.providerTiming.timeSegments?.[0]) { + streamingResult.execution.output.providerTiming.timeSegments[0].endTime = + streamEndTime + streamingResult.execution.output.providerTiming.timeSegments[0].duration = + streamEndTime - providerStartTime + } + } + + // Update token usage if available from the stream + if (usage) { + const newTokens = { + prompt: usage.prompt_tokens || tokenUsage.prompt, + completion: usage.completion_tokens || tokenUsage.completion, + total: usage.total_tokens || tokenUsage.total, + } + + streamingResult.execution.output.tokens = newTokens + } + }), + execution: { + success: true, + output: { + content: '', // Will be filled by the stream completion callback + model: request.model, + tokens: tokenUsage, + toolCalls: undefined, + providerTiming: { + startTime: providerStartTimeISO, + endTime: new Date().toISOString(), + duration: Date.now() - providerStartTime, + timeSegments: [ + { + type: 'model', + name: 'Streaming response', + startTime: providerStartTime, + endTime: Date.now(), + duration: Date.now() - providerStartTime, + }, + ], + }, + }, + logs: [], // No block logs for direct streaming + metadata: { + startTime: providerStartTimeISO, + endTime: new Date().toISOString(), + duration: Date.now() - providerStartTime, + }, + }, + } as StreamingExecution + + // Return the streaming execution object + return streamingResult as StreamingExecution + } - logger.info('Ollama request configuration:', { - toolCount: filteredTools.length, - toolChoice: 'auto', // Ollama always uses auto - model: request.model, - }) + // Make the initial API request + const initialCallTime = Date.now() + + // Track the original tool_choice for forced tool tracking + const originalToolChoice = payload.tool_choice + + // Track forced tools and their usage + const forcedTools = preparedTools?.forcedTools || [] + let usedForcedTools: string[] = [] + + // Helper function to check for forced tool usage in responses + const checkForForcedToolUsage = ( + response: any, + toolChoice: string | { type: string; function?: { name: string }; name?: string; any?: any } + ) => { + if (typeof toolChoice === 'object' && response.choices[0]?.message?.tool_calls) { + const toolCallsResponse = response.choices[0].message.tool_calls + const result = trackForcedToolUsage( + toolCallsResponse, + toolChoice, + logger, + 'ollama', + forcedTools, + usedForcedTools + ) + hasUsedForcedTool = result.hasUsedForcedTool + usedForcedTools = result.usedForcedTools } } let currentResponse = await ollama.chat.completions.create(payload) - const firstResponseTime = Date.now() - startTime + const firstResponseTime = Date.now() - initialCallTime let content = currentResponse.choices[0]?.message?.content || '' @@ -140,6 +333,7 @@ export const ollamaProvider: ProviderConfig = { content = content.trim() } + // Collect token information const tokens = { prompt: currentResponse.usage?.prompt_tokens || 0, completion: currentResponse.usage?.completion_tokens || 0, @@ -155,201 +349,307 @@ export const ollamaProvider: ProviderConfig = { let modelTime = firstResponseTime let toolsTime = 0 + // Track if a forced tool has been used + let hasUsedForcedTool = false + // Track each model and tool call segment with timestamps const timeSegments: TimeSegment[] = [ { type: 'model', name: 'Initial response', - startTime: startTime, - endTime: startTime + firstResponseTime, + startTime: initialCallTime, + endTime: initialCallTime + firstResponseTime, duration: firstResponseTime, }, ] - try { - while (iterationCount < MAX_ITERATIONS) { - // Check for tool calls - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls - if (!toolCallsInResponse || toolCallsInResponse.length === 0) { - break + // Check if a forced tool was used in the first response + checkForForcedToolUsage(currentResponse, originalToolChoice) + + while (iterationCount < MAX_ITERATIONS) { + // Check for tool calls + const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + if (!toolCallsInResponse || toolCallsInResponse.length === 0) { + break + } + + logger.info( + `Processing ${toolCallsInResponse.length} tool calls (iteration ${iterationCount + 1}/${MAX_ITERATIONS})` + ) + + // Track time for tool calls in this batch + const toolsStartTime = Date.now() + + // Process each tool call + for (const toolCall of toolCallsInResponse) { + try { + const toolName = toolCall.function.name + const toolArgs = JSON.parse(toolCall.function.arguments) + + // Get the tool from the tools registry + const tool = request.tools?.find((t) => t.id === toolName) + if (!tool) continue + + // Execute the tool + const toolCallStartTime = Date.now() + + const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) + const result = await executeTool(toolName, executionParams, true) + const toolCallEndTime = Date.now() + const toolCallDuration = toolCallEndTime - toolCallStartTime + + // Add to time segments for both success and failure + timeSegments.push({ + type: 'tool', + name: toolName, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallDuration, + }) + + // Prepare result content for the LLM + let resultContent: any + if (result.success) { + toolResults.push(result.output) + resultContent = result.output + } else { + // Include error information so LLM can respond appropriately + resultContent = { + error: true, + message: result.error || 'Tool execution failed', + tool: toolName, + } + } + + toolCalls.push({ + name: toolName, + arguments: toolParams, + startTime: new Date(toolCallStartTime).toISOString(), + endTime: new Date(toolCallEndTime).toISOString(), + duration: toolCallDuration, + result: resultContent, + success: result.success, + }) + + // Add the tool call and result to messages (both success and failure) + currentMessages.push({ + role: 'assistant', + content: null, + tool_calls: [ + { + id: toolCall.id, + type: 'function', + function: { + name: toolName, + arguments: toolCall.function.arguments, + }, + }, + ], + }) + + currentMessages.push({ + role: 'tool', + tool_call_id: toolCall.id, + content: JSON.stringify(resultContent), + }) + } catch (error) { + logger.error('Error processing tool call:', { + error, + toolName: toolCall?.function?.name, + }) } + } - // Track time for tool calls in this batch - const toolsStartTime = Date.now() + // Calculate tool call time for this iteration + const thisToolsTime = Date.now() - toolsStartTime + toolsTime += thisToolsTime - // Process each tool call - for (const toolCall of toolCallsInResponse) { - try { - const toolName = toolCall.function.name - const toolArgs = JSON.parse(toolCall.function.arguments) + // Make the next request with updated messages + const nextPayload = { + ...payload, + messages: currentMessages, + } - // Get the tool from the tools registry - const tool = request.tools?.find((t) => t.id === toolName) - if (!tool) continue + // Update tool_choice based on which forced tools have been used + if (typeof originalToolChoice === 'object' && hasUsedForcedTool && forcedTools.length > 0) { + // If we have remaining forced tools, get the next one to force + const remainingTools = forcedTools.filter((tool) => !usedForcedTools.includes(tool)) + + if (remainingTools.length > 0) { + // Ollama doesn't support forced tool selection, so we keep using 'auto' + nextPayload.tool_choice = 'auto' + logger.info(`Ollama doesn't support forced tools, using auto for: ${remainingTools[0]}`) + } else { + // All forced tools have been used, continue with auto + nextPayload.tool_choice = 'auto' + logger.info('All forced tools have been used, continuing with auto tool_choice') + } + } - // Execute the tool - const toolCallStartTime = Date.now() + // Time the next model call + const nextModelStartTime = Date.now() - // Only merge actual tool parameters for logging - const toolParams = { - ...tool.params, - ...toolArgs, - } + // Make the next request + currentResponse = await ollama.chat.completions.create(nextPayload) - // Add system parameters for execution - const executionParams = { - ...toolParams, - ...(request.workflowId - ? { - _context: { - workflowId: request.workflowId, - ...(request.chatId ? { chatId: request.chatId } : {}), - }, - } - : {}), - ...(request.environmentVariables ? { envVars: request.environmentVariables } : {}), - } + // Check if any forced tools were used in this response + checkForForcedToolUsage(currentResponse, nextPayload.tool_choice) - const result = await executeTool(toolName, executionParams, true) - const toolCallEndTime = Date.now() - const toolCallDuration = toolCallEndTime - toolCallStartTime - - // Add to time segments for both success and failure - timeSegments.push({ - type: 'tool', - name: toolName, - startTime: toolCallStartTime, - endTime: toolCallEndTime, - duration: toolCallDuration, - }) - - // Prepare result content for the LLM - let resultContent: any - if (result.success) { - toolResults.push(result.output) - resultContent = result.output - } else { - // Include error information so LLM can respond appropriately - resultContent = { - error: true, - message: result.error || 'Tool execution failed', - tool: toolName, - } - } + const nextModelEndTime = Date.now() + const thisModelTime = nextModelEndTime - nextModelStartTime - toolCalls.push({ - name: toolName, - arguments: toolParams, - startTime: new Date(toolCallStartTime).toISOString(), - endTime: new Date(toolCallEndTime).toISOString(), - duration: toolCallDuration, - result: resultContent, - success: result.success, - }) - - // Add the tool call and result to messages (both success and failure) - currentMessages.push({ - role: 'assistant', - content: null, - tool_calls: [ - { - id: toolCall.id, - type: 'function', - function: { - name: toolName, - arguments: toolCall.function.arguments, - }, - }, - ], - }) - - currentMessages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: JSON.stringify(resultContent), - }) - } catch (error) { - logger.error('Error processing tool call:', { error }) - } - } + // Add to time segments + timeSegments.push({ + type: 'model', + name: `Model response (iteration ${iterationCount + 1})`, + startTime: nextModelStartTime, + endTime: nextModelEndTime, + duration: thisModelTime, + }) - // Calculate tool call time for this iteration - const thisToolsTime = Date.now() - toolsStartTime - toolsTime += thisToolsTime + // Add to model time + modelTime += thisModelTime - // Make the next request with updated messages - const nextPayload = { - ...payload, - messages: currentMessages, - } + // Update content if we have a text response + if (currentResponse.choices[0]?.message?.content) { + content = currentResponse.choices[0].message.content + // Clean up the response content + content = content.replace(/```json\n?|\n?```/g, '') + content = content.trim() + } - // Time the next model call - const nextModelStartTime = Date.now() + // Update token counts + if (currentResponse.usage) { + tokens.prompt += currentResponse.usage.prompt_tokens || 0 + tokens.completion += currentResponse.usage.completion_tokens || 0 + tokens.total += currentResponse.usage.total_tokens || 0 + } - // Make the next request - currentResponse = await ollama.chat.completions.create(nextPayload) + iterationCount++ + } - const nextModelEndTime = Date.now() - const thisModelTime = nextModelEndTime - nextModelStartTime + // After all tool processing complete, if streaming was requested and we have messages, use streaming for the final response + if (request.stream && iterationCount > 0) { + logger.info('Using streaming for final response after tool calls') - // Add to time segments - timeSegments.push({ - type: 'model', - name: `Model response (iteration ${iterationCount + 1})`, - startTime: nextModelStartTime, - endTime: nextModelEndTime, - duration: thisModelTime, - }) + const streamingPayload = { + ...payload, + messages: currentMessages, + tool_choice: 'auto', // Always use 'auto' for the streaming response after tool calls + stream: true, + stream_options: { include_usage: true }, + } - // Add to model time - modelTime += thisModelTime + const streamResponse = await ollama.chat.completions.create(streamingPayload) + + // Create the StreamingExecution object with all collected data + const streamingResult = { + stream: createReadableStreamFromOllamaStream(streamResponse, (content, usage) => { + // Update the execution data with the final content and token usage + streamingResult.execution.output.content = content - // Update content if we have a text response - if (currentResponse.choices[0]?.message?.content) { - content = currentResponse.choices[0].message.content // Clean up the response content - content = content.replace(/```json\n?|\n?```/g, '') - content = content.trim() - } + if (content) { + streamingResult.execution.output.content = content + .replace(/```json\n?|\n?```/g, '') + .trim() + } - // Update token counts - if (currentResponse.usage) { - tokens.prompt += currentResponse.usage.prompt_tokens || 0 - tokens.completion += currentResponse.usage.completion_tokens || 0 - tokens.total += currentResponse.usage.total_tokens || 0 - } + // Update token usage if available from the stream + if (usage) { + const newTokens = { + prompt: usage.prompt_tokens || tokens.prompt, + completion: usage.completion_tokens || tokens.completion, + total: usage.total_tokens || tokens.total, + } - iterationCount++ - } - } catch (error) { - logger.error('Error in Ollama request:', { error }) + streamingResult.execution.output.tokens = newTokens + } + }), + execution: { + success: true, + output: { + content: '', // Will be filled by the callback + model: request.model, + tokens: { + prompt: tokens.prompt, + completion: tokens.completion, + total: tokens.total, + }, + toolCalls: + toolCalls.length > 0 + ? { + list: toolCalls, + count: toolCalls.length, + } + : undefined, + providerTiming: { + startTime: providerStartTimeISO, + endTime: new Date().toISOString(), + duration: Date.now() - providerStartTime, + modelTime: modelTime, + toolsTime: toolsTime, + firstResponseTime: firstResponseTime, + iterations: iterationCount + 1, + timeSegments: timeSegments, + }, + }, + logs: [], // No block logs at provider level + metadata: { + startTime: providerStartTimeISO, + endTime: new Date().toISOString(), + duration: Date.now() - providerStartTime, + }, + }, + } as StreamingExecution + + // Return the streaming execution object + return streamingResult as StreamingExecution } - const endTime = Date.now() + // Calculate overall timing + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime return { - content: content, + content, model: request.model, tokens, toolCalls: toolCalls.length > 0 ? toolCalls : undefined, toolResults: toolResults.length > 0 ? toolResults : undefined, timing: { - startTime: new Date(startTime).toISOString(), - endTime: new Date(endTime).toISOString(), - duration: endTime - startTime, + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, modelTime: modelTime, toolsTime: toolsTime, firstResponseTime: firstResponseTime, iterations: iterationCount + 1, - timeSegments, + timeSegments: timeSegments, }, } } catch (error) { - logger.error('Error in Ollama request', { - error: error instanceof Error ? error.message : 'Unknown error', - model: request.model, + // Include timing information even for errors + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime + + logger.error('Error in Ollama request:', { + error, + duration: totalDuration, }) - throw error + + // Create a new error with timing information + const enhancedError = new Error(error instanceof Error ? error.message : String(error)) + // @ts-ignore - Adding timing property to the error + enhancedError.timing = { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + } + + throw enhancedError } }, } diff --git a/apps/sim/providers/utils.ts b/apps/sim/providers/utils.ts index 6e7f759c9ba..6ab2650f0fd 100644 --- a/apps/sim/providers/utils.ts +++ b/apps/sim/providers/utils.ts @@ -27,6 +27,7 @@ import { openaiProvider } from '@/providers/openai' import type { ProviderConfig, ProviderId, ProviderToolConfig } from '@/providers/types' import { xAIProvider } from '@/providers/xai' import { useCustomToolsStore } from '@/stores/custom-tools/store' +import { useOllamaStore } from '@/stores/ollama/store' const logger = createLogger('ProviderUtils') @@ -548,6 +549,12 @@ export function getApiKey(provider: string, model: string, userProvidedKey?: str // If user provided a key, use it as a fallback const hasUserKey = !!userProvidedKey + // Ollama models don't require API keys - they run locally + const isOllamaModel = provider === 'ollama' || useOllamaStore.getState().models.includes(model) + if (isOllamaModel) { + return 'empty' // Ollama uses 'empty' as a placeholder API key + } + // Use server key rotation for all OpenAI models and Anthropic's Claude models on the hosted platform const isOpenAIModel = provider === 'openai' const isClaudeModel = provider === 'anthropic' diff --git a/apps/sim/scripts/ollama_docker.sh b/apps/sim/scripts/ollama_docker.sh deleted file mode 100755 index d8c99308512..00000000000 --- a/apps/sim/scripts/ollama_docker.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -e - -# Check that at least one argument is provided. If not, display the usage help. -if [ "$#" -eq 0 ]; then - echo "Usage: $(basename "$0") [args...]" - echo "Example: $(basename "$0") ps # This will run 'ollama ps' inside the container" - exit 1 -fi - -# Start a detached container from the ollama/ollama image, -# mounting the host's ~/.ollama directory directly into the container. -# Here we mount it to /root/.ollama, assuming that's where the image expects it. -CONTAINER_ID=$(docker run -d -v ~/.ollama:/root/.ollama -p 11434:11434 ollama/ollama -) - -# Define a cleanup function to stop the container regardless of how the script exits. -cleanup() { - docker stop "$CONTAINER_ID" >/dev/null -} -trap cleanup EXIT - -# Execute the command provided by the user within the running container. -# The command runs as: "ollama " -docker exec -it "$CONTAINER_ID" ollama "$@" diff --git a/apps/sim/stores/ollama/store.ts b/apps/sim/stores/ollama/store.ts index 672a91212b5..4d52d151601 100644 --- a/apps/sim/stores/ollama/store.ts +++ b/apps/sim/stores/ollama/store.ts @@ -5,11 +5,72 @@ import type { OllamaStore } from '@/stores/ollama/types' const logger = createLogger('OllamaStore') -export const useOllamaStore = create((set) => ({ +// Fetch models from the server API when on client side +const fetchOllamaModels = async (): Promise => { + try { + const response = await fetch('/api/providers/ollama/models') + if (!response.ok) { + logger.warn('Failed to fetch Ollama models from API', { + status: response.status, + statusText: response.statusText, + }) + return [] + } + const data = await response.json() + return data.models || [] + } catch (error) { + logger.error('Error fetching Ollama models', { + error: error instanceof Error ? error.message : 'Unknown error', + }) + return [] + } +} + +export const useOllamaStore = create((set, get) => ({ models: [], + isLoading: false, setModels: (models) => { set({ models }) // Update the providers when models change updateOllamaProviderModels(models) }, + + // Fetch models from API (client-side only) + fetchModels: async () => { + if (typeof window === 'undefined') { + logger.info('Skipping client-side model fetch on server') + return + } + + if (get().isLoading) { + logger.info('Model fetch already in progress') + return + } + + logger.info('Fetching Ollama models from API') + set({ isLoading: true }) + + try { + const models = await fetchOllamaModels() + logger.info('Successfully fetched Ollama models', { + count: models.length, + models, + }) + get().setModels(models) + } catch (error) { + logger.error('Failed to fetch Ollama models', { + error: error instanceof Error ? error.message : 'Unknown error', + }) + } finally { + set({ isLoading: false }) + } + }, })) + +// Auto-fetch models when the store is first accessed on the client +if (typeof window !== 'undefined') { + // Delay to avoid hydration issues + setTimeout(() => { + useOllamaStore.getState().fetchModels() + }, 1000) +} diff --git a/apps/sim/stores/ollama/types.ts b/apps/sim/stores/ollama/types.ts index 7c89f4ff9fa..77b0fa26cdd 100644 --- a/apps/sim/stores/ollama/types.ts +++ b/apps/sim/stores/ollama/types.ts @@ -1,4 +1,6 @@ export interface OllamaStore { models: string[] + isLoading: boolean setModels: (models: string[]) => void + fetchModels: () => Promise } diff --git a/docker-compose.ollama.yml b/docker-compose.ollama.yml index ca22447891c..e5b75cac024 100644 --- a/docker-compose.ollama.yml +++ b/docker-compose.ollama.yml @@ -1,11 +1,106 @@ +name: sim-with-ollama + services: - local-llm-gpu: - profiles: - - local-gpu # This profile requires both 'local' and 'gpu' + # Main Sim Studio Application + simstudio: + build: + context: . + dockerfile: docker/app.Dockerfile + ports: + - '3000:3000' + deploy: + resources: + limits: + memory: 8G + environment: + - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} + - BETTER_AUTH_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} + - NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} + - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-sim_auth_secret_$(openssl rand -hex 16)} + - ENCRYPTION_KEY=${ENCRYPTION_KEY:-$(openssl rand -hex 32)} + - OLLAMA_URL=http://ollama:11434 + - NEXT_PUBLIC_SOCKET_URL=${NEXT_PUBLIC_SOCKET_URL:-http://localhost:3002} + depends_on: + db: + condition: service_healthy + migrations: + condition: service_completed_successfully + realtime: + condition: service_healthy + ollama: + condition: service_healthy + healthcheck: + test: ['CMD', 'wget', '--spider', '--quiet', 'http://127.0.0.1:3000'] + interval: 90s + timeout: 5s + retries: 3 + start_period: 10s + restart: unless-stopped + + # Realtime Socket Server + realtime: + build: + context: . + dockerfile: docker/realtime.Dockerfile + environment: + - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} + - NEXT_PUBLIC_APP_URL=${NEXT_PUBLIC_APP_URL:-http://localhost:3000} + - BETTER_AUTH_URL=${BETTER_AUTH_URL:-http://localhost:3000} + - BETTER_AUTH_SECRET=${BETTER_AUTH_SECRET:-sim_auth_secret_$(openssl rand -hex 16)} + depends_on: + db: + condition: service_healthy + restart: unless-stopped + ports: + - '3002:3002' + deploy: + resources: + limits: + memory: 8G + healthcheck: + test: ['CMD', 'wget', '--spider', '--quiet', 'http://127.0.0.1:3002/health'] + interval: 90s + timeout: 5s + retries: 3 + start_period: 10s + + # Database Migrations + migrations: + build: + context: . + dockerfile: docker/db.Dockerfile + environment: + - DATABASE_URL=postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-simstudio} + depends_on: + db: + condition: service_healthy + command: ['bun', 'run', 'db:migrate'] + restart: 'no' + + # PostgreSQL Database with Vector Extension + db: + image: pgvector/pgvector:pg17 + restart: always + ports: + - '5432:5432' + environment: + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} + - POSTGRES_DB=${POSTGRES_DB:-simstudio} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U postgres'] + interval: 5s + timeout: 5s + retries: 5 + + # Ollama with GPU support (default) + ollama: image: ollama/ollama:latest pull_policy: always volumes: - - ${HOME}/.ollama:/root/.ollama + - ollama_data:/root/.ollama ports: - '11434:11434' environment: @@ -13,6 +108,7 @@ services: - OLLAMA_LOAD_TIMEOUT=-1 - OLLAMA_KEEP_ALIVE=-1 - OLLAMA_DEBUG=1 + - OLLAMA_HOST=0.0.0.0:11434 command: 'serve' deploy: resources: @@ -26,23 +122,56 @@ services: interval: 10s timeout: 5s retries: 5 + start_period: 30s + restart: unless-stopped - local-llm-cpu: + # Ollama CPU-only version (use with --profile cpu profile) + ollama-cpu: profiles: - - local-cpu # This profile requires both 'local' and 'cpu' + - cpu image: ollama/ollama:latest pull_policy: always volumes: - - ${HOME}/.ollama:/root/.ollama + - ollama_data:/root/.ollama ports: - '11434:11434' environment: - OLLAMA_LOAD_TIMEOUT=-1 - OLLAMA_KEEP_ALIVE=-1 - OLLAMA_DEBUG=1 + - OLLAMA_HOST=0.0.0.0:11434 command: 'serve' healthcheck: test: ['CMD', 'curl', '-f', 'http://localhost:11434/'] interval: 10s timeout: 5s retries: 5 + start_period: 30s + restart: unless-stopped + + # Helper container to pull models automatically + model-setup: + image: ollama/ollama:latest + profiles: + - setup + volumes: + - ollama_data:/root/.ollama + environment: + - OLLAMA_HOST=ollama:11434 + depends_on: + ollama: + condition: service_healthy + command: > + sh -c " + echo 'Waiting for Ollama to be ready...' && + sleep 10 && + echo 'Pulling gemma3:4b model (recommended starter model)...' && + ollama pull gemma3:4b && + echo 'Model setup complete! You can now use gemma3:4b in Sim.' && + echo 'To add more models, run: docker compose -f docker-compose.ollama.yml exec ollama ollama pull ' + " + restart: 'no' + +volumes: + postgres_data: + ollama_data: From 062e2a2c40c2f59c99a87afe885c0c37970390c5 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Tue, 5 Aug 2025 15:26:57 -0700 Subject: [PATCH 02/13] fix(deployed-state): use deployed state for API sync and async execs, deployed state modal visual for enabled/disabled (#885) * fix(deployments): use deployed state for API sync and async execs * fix deployed workflow modal visualization for enabled * fix tests --- .../api/workflows/[id]/execute/route.test.ts | 8 +-- .../app/api/workflows/[id]/execute/route.ts | 21 +++----- .../workflow-block/workflow-block.tsx | 5 +- apps/sim/lib/workflows/db-helpers.ts | 51 +++++++++++++++++-- apps/sim/trigger/workflow-execution.ts | 14 ++--- 5 files changed, 65 insertions(+), 34 deletions(-) diff --git a/apps/sim/app/api/workflows/[id]/execute/route.test.ts b/apps/sim/app/api/workflows/[id]/execute/route.test.ts index 237ddb4cfed..7be75d9f8ad 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.test.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.test.ts @@ -87,7 +87,7 @@ describe('Workflow Execution API Route', () => { })) vi.doMock('@/lib/workflows/db-helpers', () => ({ - loadWorkflowFromNormalizedTables: vi.fn().mockResolvedValue({ + loadDeployedWorkflowState: vi.fn().mockResolvedValue({ blocks: { 'starter-id': { id: 'starter-id', @@ -121,7 +121,7 @@ describe('Workflow Execution API Route', () => { ], loops: {}, parallels: {}, - isFromNormalizedTables: true, + isFromNormalizedTables: false, // Changed to false since it's from deployed state }), })) @@ -516,7 +516,7 @@ describe('Workflow Execution API Route', () => { })) vi.doMock('@/lib/workflows/db-helpers', () => ({ - loadWorkflowFromNormalizedTables: vi.fn().mockResolvedValue({ + loadDeployedWorkflowState: vi.fn().mockResolvedValue({ blocks: { 'starter-id': { id: 'starter-id', @@ -550,7 +550,7 @@ describe('Workflow Execution API Route', () => { ], loops: {}, parallels: {}, - isFromNormalizedTables: true, + isFromNormalizedTables: false, // Changed to false since it's from deployed state }), })) diff --git a/apps/sim/app/api/workflows/[id]/execute/route.ts b/apps/sim/app/api/workflows/[id]/execute/route.ts index 893a5efa68a..927ae1f067b 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.ts @@ -9,7 +9,7 @@ import { createLogger } from '@/lib/logs/console/logger' import { LoggingSession } from '@/lib/logs/execution/logging-session' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' import { decryptSecret } from '@/lib/utils' -import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/db-helpers' +import { loadDeployedWorkflowState } from '@/lib/workflows/db-helpers' import { createHttpResponseFromBlock, updateWorkflowRunCounts, @@ -111,20 +111,13 @@ async function executeWorkflow(workflow: any, requestId: string, input?: any): P runningExecutions.add(executionKey) logger.info(`[${requestId}] Starting workflow execution: ${workflowId}`) - // Load workflow data from normalized tables - logger.debug(`[${requestId}] Loading workflow ${workflowId} from normalized tables`) - const normalizedData = await loadWorkflowFromNormalizedTables(workflowId) + // Load workflow data from deployed state for API executions + const deployedData = await loadDeployedWorkflowState(workflowId) - if (!normalizedData) { - throw new Error( - `Workflow ${workflowId} has no normalized data available. Ensure the workflow is properly saved to normalized tables.` - ) - } - - // Use normalized data as primary source - const { blocks, edges, loops, parallels } = normalizedData - logger.info(`[${requestId}] Using normalized tables for workflow execution: ${workflowId}`) - logger.debug(`[${requestId}] Normalized data loaded:`, { + // Use deployed data as primary source for API executions + const { blocks, edges, loops, parallels } = deployedData + logger.info(`[${requestId}] Using deployed state for workflow execution: ${workflowId}`) + logger.debug(`[${requestId}] Deployed data loaded:`, { blocksCount: Object.keys(blocks || {}).length, edgesCount: (edges || []).length, loopsCount: Object.keys(loops || {}).length, diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx index cb110085319..64b702b1e00 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/workflow-block/workflow-block.tsx @@ -70,7 +70,10 @@ export function WorkflowBlock({ id, data }: NodeProps) { const currentWorkflow = useCurrentWorkflow() const currentBlock = currentWorkflow.getBlockById(id) - const isEnabled = currentBlock?.enabled ?? true + // In preview mode, use the blockState provided; otherwise use current workflow state + const isEnabled = data.isPreview + ? (data.blockState?.enabled ?? true) + : (currentBlock?.enabled ?? true) // Get diff status from the block itself (set by diff engine) const diffStatus = diff --git a/apps/sim/lib/workflows/db-helpers.ts b/apps/sim/lib/workflows/db-helpers.ts index ff4a334a558..2073dc65c88 100644 --- a/apps/sim/lib/workflows/db-helpers.ts +++ b/apps/sim/lib/workflows/db-helpers.ts @@ -1,8 +1,8 @@ import { eq } from 'drizzle-orm' import { createLogger } from '@/lib/logs/console/logger' import { db } from '@/db' -import { workflowBlocks, workflowEdges, workflowSubflows } from '@/db/schema' -import type { LoopConfig, WorkflowState } from '@/stores/workflows/workflow/types' +import { workflow, workflowBlocks, workflowEdges, workflowSubflows } from '@/db/schema' +import type { WorkflowState } from '@/stores/workflows/workflow/types' import { SUBFLOW_TYPES } from '@/stores/workflows/workflow/types' const logger = createLogger('WorkflowDBHelpers') @@ -12,7 +12,49 @@ export interface NormalizedWorkflowData { edges: any[] loops: Record parallels: Record - isFromNormalizedTables: true // Flag to indicate this came from new tables + isFromNormalizedTables: boolean // Flag to indicate source (true = normalized tables, false = deployed state) +} + +/** + * Load deployed workflow state for execution + * Returns deployed state if available, otherwise throws error + */ +export async function loadDeployedWorkflowState( + workflowId: string +): Promise { + try { + // First check if workflow is deployed and get deployed state + const [workflowResult] = await db + .select({ + isDeployed: workflow.isDeployed, + deployedState: workflow.deployedState, + }) + .from(workflow) + .where(eq(workflow.id, workflowId)) + .limit(1) + + if (!workflowResult) { + throw new Error(`Workflow ${workflowId} not found`) + } + + if (!workflowResult.isDeployed || !workflowResult.deployedState) { + throw new Error(`Workflow ${workflowId} is not deployed or has no deployed state`) + } + + const deployedState = workflowResult.deployedState as any + + // Convert deployed state to normalized format + return { + blocks: deployedState.blocks || {}, + edges: deployedState.edges || [], + loops: deployedState.loops || {}, + parallels: deployedState.parallels || {}, + isFromNormalizedTables: false, // Flag to indicate this came from deployed state + } + } catch (error) { + logger.error(`Error loading deployed workflow state ${workflowId}:`, error) + throw error + } } /** @@ -88,7 +130,6 @@ export async function loadWorkflowFromNormalizedTables( const config = subflow.config || {} if (subflow.type === SUBFLOW_TYPES.LOOP) { - const loopConfig = config as LoopConfig loops[subflow.id] = { id: subflow.id, ...config, @@ -126,7 +167,7 @@ export async function saveWorkflowToNormalizedTables( ): Promise<{ success: boolean; jsonBlob?: any; error?: string }> { try { // Start a transaction - const result = await db.transaction(async (tx) => { + await db.transaction(async (tx) => { // Clear existing data for this workflow await Promise.all([ tx.delete(workflowBlocks).where(eq(workflowBlocks.workflowId, workflowId)), diff --git a/apps/sim/trigger/workflow-execution.ts b/apps/sim/trigger/workflow-execution.ts index 3a84b91c68b..5a455310c35 100644 --- a/apps/sim/trigger/workflow-execution.ts +++ b/apps/sim/trigger/workflow-execution.ts @@ -6,7 +6,7 @@ import { createLogger } from '@/lib/logs/console/logger' import { LoggingSession } from '@/lib/logs/execution/logging-session' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' import { decryptSecret } from '@/lib/utils' -import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/db-helpers' +import { loadDeployedWorkflowState } from '@/lib/workflows/db-helpers' import { updateWorkflowRunCounts } from '@/lib/workflows/utils' import { db } from '@/db' import { environment as environmentTable, userStats } from '@/db/schema' @@ -60,16 +60,10 @@ export const workflowExecution = task({ ) } - // Load workflow data from normalized tables - const normalizedData = await loadWorkflowFromNormalizedTables(workflowId) - if (!normalizedData) { - logger.error(`[${requestId}] Workflow not found in normalized tables: ${workflowId}`) - throw new Error(`Workflow ${workflowId} data not found in normalized tables`) - } - - logger.info(`[${requestId}] Workflow loaded successfully: ${workflowId}`) + // Load workflow data from deployed state (this task is only used for API executions right now) + const workflowData = await loadDeployedWorkflowState(workflowId) - const { blocks, edges, loops, parallels } = normalizedData + const { blocks, edges, loops, parallels } = workflowData // Merge subblock states (server-safe version doesn't need workflowId) const mergedStates = mergeSubblockState(blocks, {}) From 94368eb1c2ac5b8cb1e67b23301c1a736d6e58fa Mon Sep 17 00:00:00 2001 From: Siddharth Ganesan <33737564+Sg312@users.noreply.github.com> Date: Tue, 5 Aug 2025 17:01:53 -0700 Subject: [PATCH 03/13] Feat/copilot files (#886) * Connects to s3 * Checkpoint * File shows in message * Make files clickable * User input image * Persist thumbnails * Drag and drop files * Lint * Fix isdev * Dont re-download files on rerender --- apps/sim/app/api/copilot/chat/file-utils.ts | 132 ++++++ apps/sim/app/api/copilot/chat/route.ts | 129 +++++- .../api/copilot/chat/update-messages/route.ts | 11 + apps/sim/app/api/files/presigned/route.ts | 69 ++- .../app/api/files/serve/[...path]/route.ts | 35 +- .../copilot-message/copilot-message.tsx | 125 ++++- .../components/user-input/user-input.tsx | 433 ++++++++++++++++-- .../panel/components/copilot/copilot.tsx | 15 +- apps/sim/lib/copilot/api.ts | 12 + apps/sim/lib/env.ts | 2 + apps/sim/lib/uploads/s3/s3-client.ts | 24 + apps/sim/lib/uploads/setup.server.ts | 6 + apps/sim/lib/uploads/setup.ts | 12 + apps/sim/stores/copilot/store.ts | 22 +- apps/sim/stores/copilot/types.ts | 13 + 15 files changed, 971 insertions(+), 69 deletions(-) create mode 100644 apps/sim/app/api/copilot/chat/file-utils.ts diff --git a/apps/sim/app/api/copilot/chat/file-utils.ts b/apps/sim/app/api/copilot/chat/file-utils.ts new file mode 100644 index 00000000000..48b81bafa6c --- /dev/null +++ b/apps/sim/app/api/copilot/chat/file-utils.ts @@ -0,0 +1,132 @@ +export interface FileAttachment { + id: string + s3_key: string + filename: string + media_type: string + size: number +} + +export interface AnthropicMessageContent { + type: 'text' | 'image' | 'document' + text?: string + source?: { + type: 'base64' + media_type: string + data: string + } +} + +/** + * Mapping of MIME types to Anthropic content types + */ +export const MIME_TYPE_MAPPING: Record = { + // Images + 'image/jpeg': 'image', + 'image/jpg': 'image', + 'image/png': 'image', + 'image/gif': 'image', + 'image/webp': 'image', + 'image/svg+xml': 'image', + + // Documents + 'application/pdf': 'document', + 'text/plain': 'document', + 'text/csv': 'document', + 'application/json': 'document', + 'application/xml': 'document', + 'text/xml': 'document', + 'text/html': 'document', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'document', // .docx + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'document', // .xlsx + 'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'document', // .pptx + 'application/msword': 'document', // .doc + 'application/vnd.ms-excel': 'document', // .xls + 'application/vnd.ms-powerpoint': 'document', // .ppt + 'text/markdown': 'document', + 'application/rtf': 'document', +} + +/** + * Get the Anthropic content type for a given MIME type + */ +export function getAnthropicContentType(mimeType: string): 'image' | 'document' | null { + return MIME_TYPE_MAPPING[mimeType.toLowerCase()] || null +} + +/** + * Check if a MIME type is supported by Anthropic + */ +export function isSupportedFileType(mimeType: string): boolean { + return mimeType.toLowerCase() in MIME_TYPE_MAPPING +} + +/** + * Convert a file buffer to base64 + */ +export function bufferToBase64(buffer: Buffer): string { + return buffer.toString('base64') +} + +/** + * Create Anthropic message content from file data + */ +export function createAnthropicFileContent( + fileBuffer: Buffer, + mimeType: string +): AnthropicMessageContent | null { + const contentType = getAnthropicContentType(mimeType) + if (!contentType) { + return null + } + + return { + type: contentType, + source: { + type: 'base64', + media_type: mimeType, + data: bufferToBase64(fileBuffer), + }, + } +} + +/** + * Extract file extension from filename + */ +export function getFileExtension(filename: string): string { + const lastDot = filename.lastIndexOf('.') + return lastDot !== -1 ? filename.slice(lastDot + 1).toLowerCase() : '' +} + +/** + * Get MIME type from file extension (fallback if not provided) + */ +export function getMimeTypeFromExtension(extension: string): string { + const extensionMimeMap: Record = { + // Images + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + png: 'image/png', + gif: 'image/gif', + webp: 'image/webp', + svg: 'image/svg+xml', + + // Documents + pdf: 'application/pdf', + txt: 'text/plain', + csv: 'text/csv', + json: 'application/json', + xml: 'application/xml', + html: 'text/html', + htm: 'text/html', + docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + doc: 'application/msword', + xls: 'application/vnd.ms-excel', + ppt: 'application/vnd.ms-powerpoint', + md: 'text/markdown', + rtf: 'application/rtf', + } + + return extensionMimeMap[extension.toLowerCase()] || 'application/octet-stream' +} diff --git a/apps/sim/app/api/copilot/chat/route.ts b/apps/sim/app/api/copilot/chat/route.ts index 6b5bfad14d0..9f042f855ed 100644 --- a/apps/sim/app/api/copilot/chat/route.ts +++ b/apps/sim/app/api/copilot/chat/route.ts @@ -13,12 +13,25 @@ import { getCopilotModel } from '@/lib/copilot/config' import { TITLE_GENERATION_SYSTEM_PROMPT, TITLE_GENERATION_USER_PROMPT } from '@/lib/copilot/prompts' import { env } from '@/lib/env' import { createLogger } from '@/lib/logs/console/logger' +import { downloadFile } from '@/lib/uploads' +import { downloadFromS3WithConfig } from '@/lib/uploads/s3/s3-client' +import { S3_COPILOT_CONFIG, USE_S3_STORAGE } from '@/lib/uploads/setup' import { db } from '@/db' import { copilotChats } from '@/db/schema' import { executeProviderRequest } from '@/providers' +import { createAnthropicFileContent, isSupportedFileType } from './file-utils' const logger = createLogger('CopilotChatAPI') +// Schema for file attachments +const FileAttachmentSchema = z.object({ + id: z.string(), + s3_key: z.string(), + filename: z.string(), + media_type: z.string(), + size: z.number(), +}) + // Schema for chat messages const ChatMessageSchema = z.object({ message: z.string().min(1, 'Message is required'), @@ -29,6 +42,7 @@ const ChatMessageSchema = z.object({ createNewChat: z.boolean().optional().default(false), stream: z.boolean().optional().default(true), implicitFeedback: z.string().optional(), + fileAttachments: z.array(FileAttachmentSchema).optional(), }) // Sim Agent API configuration @@ -145,6 +159,7 @@ export async function POST(req: NextRequest) { createNewChat, stream, implicitFeedback, + fileAttachments, } = ChatMessageSchema.parse(body) logger.info(`[${tracker.requestId}] Processing copilot chat request`, { @@ -195,15 +210,91 @@ export async function POST(req: NextRequest) { } } + // Process file attachments if present + const processedFileContents: any[] = [] + if (fileAttachments && fileAttachments.length > 0) { + logger.info(`[${tracker.requestId}] Processing ${fileAttachments.length} file attachments`) + + for (const attachment of fileAttachments) { + try { + // Check if file type is supported + if (!isSupportedFileType(attachment.media_type)) { + logger.warn(`[${tracker.requestId}] Unsupported file type: ${attachment.media_type}`) + continue + } + + // Download file from S3 + logger.info(`[${tracker.requestId}] Downloading file: ${attachment.s3_key}`) + let fileBuffer: Buffer + if (USE_S3_STORAGE) { + fileBuffer = await downloadFromS3WithConfig(attachment.s3_key, S3_COPILOT_CONFIG) + } else { + // Fallback to generic downloadFile for other storage providers + fileBuffer = await downloadFile(attachment.s3_key) + } + + // Convert to Anthropic format + const fileContent = createAnthropicFileContent(fileBuffer, attachment.media_type) + if (fileContent) { + processedFileContents.push(fileContent) + logger.info( + `[${tracker.requestId}] Processed file: ${attachment.filename} (${attachment.media_type})` + ) + } + } catch (error) { + logger.error( + `[${tracker.requestId}] Failed to process file ${attachment.filename}:`, + error + ) + // Continue processing other files + } + } + } + // Build messages array for sim agent with conversation history const messages = [] - // Add conversation history + // Add conversation history (need to rebuild these with file support if they had attachments) for (const msg of conversationHistory) { - messages.push({ - role: msg.role, - content: msg.content, - }) + if (msg.fileAttachments && msg.fileAttachments.length > 0) { + // This is a message with file attachments - rebuild with content array + const content: any[] = [{ type: 'text', text: msg.content }] + + // Process file attachments for historical messages + for (const attachment of msg.fileAttachments) { + try { + if (isSupportedFileType(attachment.media_type)) { + let fileBuffer: Buffer + if (USE_S3_STORAGE) { + fileBuffer = await downloadFromS3WithConfig(attachment.s3_key, S3_COPILOT_CONFIG) + } else { + // Fallback to generic downloadFile for other storage providers + fileBuffer = await downloadFile(attachment.s3_key) + } + const fileContent = createAnthropicFileContent(fileBuffer, attachment.media_type) + if (fileContent) { + content.push(fileContent) + } + } + } catch (error) { + logger.error( + `[${tracker.requestId}] Failed to process historical file ${attachment.filename}:`, + error + ) + } + } + + messages.push({ + role: msg.role, + content, + }) + } else { + // Regular text-only message + messages.push({ + role: msg.role, + content: msg.content, + }) + } } // Add implicit feedback if provided @@ -214,11 +305,27 @@ export async function POST(req: NextRequest) { }) } - // Add current user message - messages.push({ - role: 'user', - content: message, - }) + // Add current user message with file attachments + if (processedFileContents.length > 0) { + // Message with files - use content array format + const content: any[] = [{ type: 'text', text: message }] + + // Add file contents + for (const fileContent of processedFileContents) { + content.push(fileContent) + } + + messages.push({ + role: 'user', + content, + }) + } else { + // Text-only message + messages.push({ + role: 'user', + content: message, + }) + } // Start title generation in parallel if this is a new chat with first message if (actualChatId && !currentChat?.title && conversationHistory.length === 0) { @@ -270,6 +377,7 @@ export async function POST(req: NextRequest) { role: 'user', content: message, timestamp: new Date().toISOString(), + ...(fileAttachments && fileAttachments.length > 0 && { fileAttachments }), } // Create a pass-through stream that captures the response @@ -590,6 +698,7 @@ export async function POST(req: NextRequest) { role: 'user', content: message, timestamp: new Date().toISOString(), + ...(fileAttachments && fileAttachments.length > 0 && { fileAttachments }), } const assistantMessage = { diff --git a/apps/sim/app/api/copilot/chat/update-messages/route.ts b/apps/sim/app/api/copilot/chat/update-messages/route.ts index c7af5952516..598679e560c 100644 --- a/apps/sim/app/api/copilot/chat/update-messages/route.ts +++ b/apps/sim/app/api/copilot/chat/update-messages/route.ts @@ -24,6 +24,17 @@ const UpdateMessagesSchema = z.object({ timestamp: z.string(), toolCalls: z.array(z.any()).optional(), contentBlocks: z.array(z.any()).optional(), + fileAttachments: z + .array( + z.object({ + id: z.string(), + s3_key: z.string(), + filename: z.string(), + media_type: z.string(), + size: z.number(), + }) + ) + .optional(), }) ), }) diff --git a/apps/sim/app/api/files/presigned/route.ts b/apps/sim/app/api/files/presigned/route.ts index 30c33215446..a343d3a1eb3 100644 --- a/apps/sim/app/api/files/presigned/route.ts +++ b/apps/sim/app/api/files/presigned/route.ts @@ -9,9 +9,11 @@ import { getS3Client, sanitizeFilenameForMetadata } from '@/lib/uploads/s3/s3-cl import { BLOB_CHAT_CONFIG, BLOB_CONFIG, + BLOB_COPILOT_CONFIG, BLOB_KB_CONFIG, S3_CHAT_CONFIG, S3_CONFIG, + S3_COPILOT_CONFIG, S3_KB_CONFIG, } from '@/lib/uploads/setup' import { createErrorResponse, createOptionsResponse } from '@/app/api/files/utils' @@ -22,9 +24,11 @@ interface PresignedUrlRequest { fileName: string contentType: string fileSize: number + userId?: string + chatId?: string } -type UploadType = 'general' | 'knowledge-base' | 'chat' +type UploadType = 'general' | 'knowledge-base' | 'chat' | 'copilot' class PresignedUrlError extends Error { constructor( @@ -58,7 +62,7 @@ export async function POST(request: NextRequest) { throw new ValidationError('Invalid JSON in request body') } - const { fileName, contentType, fileSize } = data + const { fileName, contentType, fileSize, userId, chatId } = data if (!fileName?.trim()) { throw new ValidationError('fileName is required and cannot be empty') @@ -83,7 +87,16 @@ export async function POST(request: NextRequest) { ? 'knowledge-base' : uploadTypeParam === 'chat' ? 'chat' - : 'general' + : uploadTypeParam === 'copilot' + ? 'copilot' + : 'general' + + // Validate copilot-specific requirements + if (uploadType === 'copilot') { + if (!userId?.trim()) { + throw new ValidationError('userId is required for copilot uploads') + } + } if (!isUsingCloudStorage()) { throw new StorageConfigError( @@ -96,9 +109,9 @@ export async function POST(request: NextRequest) { switch (storageProvider) { case 's3': - return await handleS3PresignedUrl(fileName, contentType, fileSize, uploadType) + return await handleS3PresignedUrl(fileName, contentType, fileSize, uploadType, userId) case 'blob': - return await handleBlobPresignedUrl(fileName, contentType, fileSize, uploadType) + return await handleBlobPresignedUrl(fileName, contentType, fileSize, uploadType, userId) default: throw new StorageConfigError(`Unknown storage provider: ${storageProvider}`) } @@ -126,7 +139,8 @@ async function handleS3PresignedUrl( fileName: string, contentType: string, fileSize: number, - uploadType: UploadType + uploadType: UploadType, + userId?: string ) { try { const config = @@ -134,15 +148,26 @@ async function handleS3PresignedUrl( ? S3_KB_CONFIG : uploadType === 'chat' ? S3_CHAT_CONFIG - : S3_CONFIG + : uploadType === 'copilot' + ? S3_COPILOT_CONFIG + : S3_CONFIG if (!config.bucket || !config.region) { throw new StorageConfigError(`S3 configuration missing for ${uploadType} uploads`) } const safeFileName = fileName.replace(/\s+/g, '-').replace(/[^a-zA-Z0-9.-]/g, '_') - const prefix = uploadType === 'knowledge-base' ? 'kb/' : uploadType === 'chat' ? 'chat/' : '' - const uniqueKey = `${prefix}${Date.now()}-${uuidv4()}-${safeFileName}` + + let prefix = '' + if (uploadType === 'knowledge-base') { + prefix = 'kb/' + } else if (uploadType === 'chat') { + prefix = 'chat/' + } else if (uploadType === 'copilot') { + prefix = `${userId}/` + } + + const uniqueKey = `${prefix}${uuidv4()}-${safeFileName}` const sanitizedOriginalName = sanitizeFilenameForMetadata(fileName) @@ -155,6 +180,9 @@ async function handleS3PresignedUrl( metadata.purpose = 'knowledge-base' } else if (uploadType === 'chat') { metadata.purpose = 'chat' + } else if (uploadType === 'copilot') { + metadata.purpose = 'copilot' + metadata.userId = userId || '' } const command = new PutObjectCommand({ @@ -210,7 +238,8 @@ async function handleBlobPresignedUrl( fileName: string, contentType: string, fileSize: number, - uploadType: UploadType + uploadType: UploadType, + userId?: string ) { try { const config = @@ -218,7 +247,9 @@ async function handleBlobPresignedUrl( ? BLOB_KB_CONFIG : uploadType === 'chat' ? BLOB_CHAT_CONFIG - : BLOB_CONFIG + : uploadType === 'copilot' + ? BLOB_COPILOT_CONFIG + : BLOB_CONFIG if ( !config.accountName || @@ -229,8 +260,17 @@ async function handleBlobPresignedUrl( } const safeFileName = fileName.replace(/\s+/g, '-').replace(/[^a-zA-Z0-9.-]/g, '_') - const prefix = uploadType === 'knowledge-base' ? 'kb/' : uploadType === 'chat' ? 'chat/' : '' - const uniqueKey = `${prefix}${Date.now()}-${uuidv4()}-${safeFileName}` + + let prefix = '' + if (uploadType === 'knowledge-base') { + prefix = 'kb/' + } else if (uploadType === 'chat') { + prefix = 'chat/' + } else if (uploadType === 'copilot') { + prefix = `${userId}/` + } + + const uniqueKey = `${prefix}${uuidv4()}-${safeFileName}` const blobServiceClient = getBlobServiceClient() const containerClient = blobServiceClient.getContainerClient(config.containerName) @@ -282,6 +322,9 @@ async function handleBlobPresignedUrl( uploadHeaders['x-ms-meta-purpose'] = 'knowledge-base' } else if (uploadType === 'chat') { uploadHeaders['x-ms-meta-purpose'] = 'chat' + } else if (uploadType === 'copilot') { + uploadHeaders['x-ms-meta-purpose'] = 'copilot' + uploadHeaders['x-ms-meta-userid'] = encodeURIComponent(userId || '') } return NextResponse.json({ diff --git a/apps/sim/app/api/files/serve/[...path]/route.ts b/apps/sim/app/api/files/serve/[...path]/route.ts index 810bd58e108..c0c8973e0b2 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.ts @@ -58,7 +58,11 @@ export async function GET( if (isUsingCloudStorage() || isCloudPath) { // Extract the actual key (remove 's3/' or 'blob/' prefix if present) const cloudKey = isCloudPath ? path.slice(1).join('/') : fullPath - return await handleCloudProxy(cloudKey) + + // Get bucket type from query parameter + const bucketType = request.nextUrl.searchParams.get('bucket') + + return await handleCloudProxy(cloudKey, bucketType) } // Use local handler for local files @@ -152,12 +156,37 @@ async function downloadKBFile(cloudKey: string): Promise { /** * Proxy cloud file through our server */ -async function handleCloudProxy(cloudKey: string): Promise { +async function handleCloudProxy( + cloudKey: string, + bucketType?: string | null +): Promise { try { // Check if this is a KB file (starts with 'kb/') const isKBFile = cloudKey.startsWith('kb/') - const fileBuffer = isKBFile ? await downloadKBFile(cloudKey) : await downloadFile(cloudKey) + let fileBuffer: Buffer + + if (isKBFile) { + fileBuffer = await downloadKBFile(cloudKey) + } else if (bucketType === 'copilot') { + // Download from copilot-specific bucket + const storageProvider = getStorageProvider() + + if (storageProvider === 's3') { + const { downloadFromS3WithConfig } = await import('@/lib/uploads/s3/s3-client') + const { S3_COPILOT_CONFIG } = await import('@/lib/uploads/setup') + fileBuffer = await downloadFromS3WithConfig(cloudKey, S3_COPILOT_CONFIG) + } else if (storageProvider === 'blob') { + // For Azure Blob, use the default downloadFile for now + // TODO: Add downloadFromBlobWithConfig when needed + fileBuffer = await downloadFile(cloudKey) + } else { + fileBuffer = await downloadFile(cloudKey) + } + } else { + // Default bucket + fileBuffer = await downloadFile(cloudKey) + } // Extract the original filename from the key (last part after last /) const originalFilename = cloudKey.split('/').pop() || 'download' diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/copilot-message/copilot-message.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/copilot-message/copilot-message.tsx index bae6d96a348..e6e73b5afa2 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/copilot-message/copilot-message.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/copilot-message/copilot-message.tsx @@ -1,7 +1,17 @@ 'use client' import { type FC, memo, useEffect, useMemo, useRef, useState } from 'react' -import { Check, Clipboard, Loader2, RotateCcw, ThumbsDown, ThumbsUp, X } from 'lucide-react' +import { + Check, + Clipboard, + FileText, + Image, + Loader2, + RotateCcw, + ThumbsDown, + ThumbsUp, + X, +} from 'lucide-react' import { InlineToolCall } from '@/lib/copilot/tools/inline-tool-call' import { createLogger } from '@/lib/logs/console/logger' import { usePreviewStore } from '@/stores/copilot/preview-store' @@ -38,6 +48,107 @@ const StreamingIndicator = memo(() => ( StreamingIndicator.displayName = 'StreamingIndicator' +// File attachment display component +interface FileAttachmentDisplayProps { + fileAttachments: any[] +} + +const FileAttachmentDisplay = memo(({ fileAttachments }: FileAttachmentDisplayProps) => { + // Cache for file URLs to avoid re-fetching on every render + const [fileUrls, setFileUrls] = useState>({}) + + const formatFileSize = (bytes: number) => { + if (bytes === 0) return '0 B' + const k = 1024 + const sizes = ['B', 'KB', 'MB', 'GB'] + const i = Math.floor(Math.log(bytes) / Math.log(k)) + return `${Math.round((bytes / k ** i) * 10) / 10} ${sizes[i]}` + } + + const getFileIcon = (mediaType: string) => { + if (mediaType.startsWith('image/')) { + return + } + if (mediaType.includes('pdf')) { + return + } + if (mediaType.includes('text') || mediaType.includes('json') || mediaType.includes('xml')) { + return + } + return + } + + const getFileUrl = (file: any) => { + const cacheKey = file.s3_key + if (fileUrls[cacheKey]) { + return fileUrls[cacheKey] + } + + // Generate URL only once and cache it + const url = `/api/files/serve/s3/${encodeURIComponent(file.s3_key)}?bucket=copilot` + setFileUrls((prev) => ({ ...prev, [cacheKey]: url })) + return url + } + + const handleFileClick = (file: any) => { + // Use cached URL or generate it + const serveUrl = getFileUrl(file) + + // Open the file in a new tab + window.open(serveUrl, '_blank') + } + + const isImageFile = (mediaType: string) => { + return mediaType.startsWith('image/') + } + + return ( + <> + {fileAttachments.map((file) => ( +
handleFileClick(file)} + title={`${file.filename} (${formatFileSize(file.size)})`} + > + {isImageFile(file.media_type) ? ( + // For images, show actual thumbnail + {file.filename} { + // If image fails to load, replace with icon + const target = e.target as HTMLImageElement + target.style.display = 'none' + const parent = target.parentElement + if (parent) { + const iconContainer = document.createElement('div') + iconContainer.className = + 'flex items-center justify-center w-full h-full bg-background/50' + iconContainer.innerHTML = + '' + parent.appendChild(iconContainer) + } + }} + /> + ) : ( + // For other files, show icon centered +
+ {getFileIcon(file.media_type)} +
+ )} + + {/* Hover overlay effect */} +
+
+ ))} + + ) +}) + +FileAttachmentDisplay.displayName = 'FileAttachmentDisplay' + // Smooth streaming text component with typewriter effect interface SmoothStreamingTextProps { content: string @@ -481,8 +592,18 @@ const CopilotMessage: FC = memo( if (isUser) { return (
+ {/* File attachments displayed above the message, completely separate from message box width */} + {message.fileAttachments && message.fileAttachments.length > 0 && ( +
+
+ +
+
+ )} +
+ {/* Message content in purple box */}
= memo(
+ + {/* Checkpoints below message */} {hasCheckpoints && (
{showRestoreConfirmation ? ( diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/user-input.tsx index 1ffa1f397aa..b4e1dd12a26 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/user-input.tsx @@ -8,13 +8,43 @@ import { useRef, useState, } from 'react' -import { ArrowUp, Loader2, MessageCircle, Package, X } from 'lucide-react' +import { + ArrowUp, + FileText, + Image, + Loader2, + MessageCircle, + Package, + Paperclip, + X, +} from 'lucide-react' import { Button } from '@/components/ui/button' import { Textarea } from '@/components/ui/textarea' +import { useSession } from '@/lib/auth-client' import { cn } from '@/lib/utils' +import { useCopilotStore } from '@/stores/copilot/store' + +export interface MessageFileAttachment { + id: string + s3_key: string + filename: string + media_type: string + size: number +} + +interface AttachedFile { + id: string + name: string + size: number + type: string + path: string + key?: string // Add key field to store the actual S3 key + uploading: boolean + previewUrl?: string // For local preview of images before upload +} interface UserInputProps { - onSubmit: (message: string) => void + onSubmit: (message: string, fileAttachments?: MessageFileAttachment[]) => void onAbort?: () => void disabled?: boolean isLoading?: boolean @@ -49,7 +79,15 @@ const UserInput = forwardRef( ref ) => { const [internalMessage, setInternalMessage] = useState('') + const [attachedFiles, setAttachedFiles] = useState([]) + // Drag and drop state + const [isDragging, setIsDragging] = useState(false) + const [dragCounter, setDragCounter] = useState(0) const textareaRef = useRef(null) + const fileInputRef = useRef(null) + + const { data: session } = useSession() + const { currentChat, workflowId } = useCopilotStore() // Expose focus method to parent useImperativeHandle( @@ -76,17 +114,190 @@ const UserInput = forwardRef( } }, [message]) + // Cleanup preview URLs on unmount + useEffect(() => { + return () => { + attachedFiles.forEach((f) => { + if (f.previewUrl) { + URL.revokeObjectURL(f.previewUrl) + } + }) + } + }, []) + + // Drag and drop handlers + const handleDragEnter = (e: React.DragEvent) => { + e.preventDefault() + e.stopPropagation() + setDragCounter((prev) => { + const newCount = prev + 1 + if (newCount === 1) { + setIsDragging(true) + } + return newCount + }) + } + + const handleDragLeave = (e: React.DragEvent) => { + e.preventDefault() + e.stopPropagation() + setDragCounter((prev) => { + const newCount = prev - 1 + if (newCount === 0) { + setIsDragging(false) + } + return newCount + }) + } + + const handleDragOver = (e: React.DragEvent) => { + e.preventDefault() + e.stopPropagation() + // Add visual feedback for valid drop zone + e.dataTransfer.dropEffect = 'copy' + } + + const handleDrop = async (e: React.DragEvent) => { + e.preventDefault() + e.stopPropagation() + setIsDragging(false) + setDragCounter(0) + + if (e.dataTransfer.files && e.dataTransfer.files.length > 0) { + await processFiles(e.dataTransfer.files) + } + } + + // Process dropped or selected files + const processFiles = async (fileList: FileList) => { + const userId = session?.user?.id + + if (!userId) { + console.error('User ID not available for file upload') + return + } + + // Process files one by one + for (const file of Array.from(fileList)) { + // Create a preview URL for images + let previewUrl: string | undefined + if (file.type.startsWith('image/')) { + previewUrl = URL.createObjectURL(file) + } + + // Create a temporary file entry with uploading state + const tempFile: AttachedFile = { + id: crypto.randomUUID(), + name: file.name, + size: file.size, + type: file.type, + path: '', + uploading: true, + previewUrl, + } + + setAttachedFiles((prev) => [...prev, tempFile]) + + try { + // Request presigned URL + const presignedResponse = await fetch('/api/files/presigned?type=copilot', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + fileName: file.name, + contentType: file.type, + fileSize: file.size, + userId, + }), + }) + + if (!presignedResponse.ok) { + throw new Error('Failed to get presigned URL') + } + + const presignedData = await presignedResponse.json() + + // Upload file to S3 + console.log('Uploading to S3:', presignedData.presignedUrl) + const uploadResponse = await fetch(presignedData.presignedUrl, { + method: 'PUT', + headers: { + 'Content-Type': file.type, + }, + body: file, + }) + + console.log('S3 Upload response status:', uploadResponse.status) + + if (!uploadResponse.ok) { + const errorText = await uploadResponse.text() + console.error('S3 Upload failed:', errorText) + throw new Error(`Failed to upload file: ${uploadResponse.status} ${errorText}`) + } + + // Update file entry with success + setAttachedFiles((prev) => + prev.map((f) => + f.id === tempFile.id + ? { + ...f, + path: presignedData.fileInfo.path, + key: presignedData.fileInfo.key, // Store the actual S3 key + uploading: false, + } + : f + ) + ) + } catch (error) { + console.error('File upload failed:', error) + // Remove failed upload + setAttachedFiles((prev) => prev.filter((f) => f.id !== tempFile.id)) + } + } + } + const handleSubmit = () => { const trimmedMessage = message.trim() if (!trimmedMessage || disabled || isLoading) return - onSubmit(trimmedMessage) - // Clear the message after submit + // Check for failed uploads and show user feedback + const failedUploads = attachedFiles.filter((f) => !f.uploading && !f.key) + if (failedUploads.length > 0) { + console.error( + 'Some files failed to upload:', + failedUploads.map((f) => f.name) + ) + } + + // Convert attached files to the format expected by the API + const fileAttachments = attachedFiles + .filter((f) => !f.uploading && f.key) // Only include successfully uploaded files with keys + .map((f) => ({ + id: f.id, + s3_key: f.key!, // Use the actual S3 key stored from the upload response + filename: f.name, + media_type: f.type, + size: f.size, + })) + + onSubmit(trimmedMessage, fileAttachments) + + // Clean up preview URLs before clearing + attachedFiles.forEach((f) => { + if (f.previewUrl) { + URL.revokeObjectURL(f.previewUrl) + } + }) + + // Clear the message and files after submit if (controlledValue !== undefined) { onControlledChange?.('') } else { setInternalMessage('') } + setAttachedFiles([]) } const handleAbort = () => { @@ -111,6 +322,67 @@ const UserInput = forwardRef( } } + const handleFileSelect = () => { + fileInputRef.current?.click() + } + + const handleFileChange = async (e: React.ChangeEvent) => { + const files = e.target.files + if (!files || files.length === 0) return + + await processFiles(files) + + // Clear the input + if (fileInputRef.current) { + fileInputRef.current.value = '' + } + } + + const removeFile = (fileId: string) => { + // Clean up preview URL if it exists + const file = attachedFiles.find((f) => f.id === fileId) + if (file?.previewUrl) { + URL.revokeObjectURL(file.previewUrl) + } + setAttachedFiles((prev) => prev.filter((f) => f.id !== fileId)) + } + + const handleFileClick = (file: AttachedFile) => { + // If file has been uploaded and has an S3 key, open the S3 URL + if (file.key) { + const serveUrl = `/api/files/serve/s3/${encodeURIComponent(file.key)}?bucket=copilot` + window.open(serveUrl, '_blank') + } else if (file.previewUrl) { + // If file hasn't been uploaded yet but has a preview URL, open that + window.open(file.previewUrl, '_blank') + } + } + + const formatFileSize = (bytes: number) => { + if (bytes === 0) return '0 Bytes' + const k = 1024 + const sizes = ['Bytes', 'KB', 'MB', 'GB'] + const i = Math.floor(Math.log(bytes) / Math.log(k)) + return `${Math.round((bytes / k ** i) * 100) / 100} ${sizes[i]}` + } + + const isImageFile = (type: string) => { + return type.startsWith('image/') + } + + const getFileIcon = (mediaType: string) => { + if (mediaType.startsWith('image/')) { + return + } + if (mediaType.includes('pdf')) { + return + } + if (mediaType.includes('text') || mediaType.includes('json') || mediaType.includes('xml')) { + return + } + return + } + const canSubmit = message.trim().length > 0 && !disabled && !isLoading const showAbortButton = isLoading && onAbort @@ -130,23 +402,93 @@ const UserInput = forwardRef( return (
-
+
+ {/* Attached Files Display with Thumbnails */} + {attachedFiles.length > 0 && ( +
+ {attachedFiles.map((file) => ( +
handleFileClick(file)} + > + {isImageFile(file.type) && file.previewUrl ? ( + // For images, show actual thumbnail + {file.name} + ) : isImageFile(file.type) && file.key ? ( + // For uploaded images without preview URL, use S3 URL + {file.name} + ) : ( + // For other files, show icon centered +
+ {getFileIcon(file.type)} +
+ )} + + {/* Loading overlay */} + {file.uploading && ( +
+ +
+ )} + + {/* Remove button */} + {!file.uploading && ( + + )} + + {/* Hover overlay effect */} +
+
+ ))} +
+ )} + {/* Textarea Field */}