Skip to content

fix(voice): create ElevenLabs tools before agent setup #459

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Shujakuinkuraudo wants to merge 1 commit into from
Closed

fix(voice): create ElevenLabs tools before agent setup #459

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 126 additions & 4 deletions hub/src/web/routes/voice.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import type { WebAppEnv } from '../middleware/auth'
import {
ELEVENLABS_API_BASE,
VOICE_AGENT_NAME,
buildVoiceAgentConfig
buildVoiceAgentConfig,
buildVoiceToolRequests,
type VoiceToolConfig
} from '@hapi/protocol/voice'

const tokenRequestSchema = z.object({
Expand All @@ -20,6 +22,14 @@ interface ElevenLabsAgent {
name: string
}

interface ElevenLabsTool {
id: string
tool_config?: {
name?: string
type?: string
}
}

/**
* Find an existing "Hapi Voice Assistant" agent
*/
Expand Down Expand Up @@ -50,7 +60,7 @@ async function findHapiAgent(apiKey: string): Promise<string | null> {
/**
* Create a new "Hapi Voice Assistant" agent
*/
async function createHapiAgent(apiKey: string): Promise<string | null> {
async function createHapiAgent(apiKey: string, toolIds: string[]): Promise<string | null> {
try {
const response = await fetch(`${ELEVENLABS_API_BASE}/convai/agents/create`, {
method: 'POST',
Expand All @@ -59,7 +69,7 @@ async function createHapiAgent(apiKey: string): Promise<string | null> {
'Content-Type': 'application/json',
'Accept': 'application/json'
},
body: JSON.stringify(buildVoiceAgentConfig())
body: JSON.stringify(buildVoiceAgentConfig(toolIds))
})

if (!response.ok) {
Expand All @@ -79,6 +89,106 @@ async function createHapiAgent(apiKey: string): Promise<string | null> {
}
}

async function updateHapiAgent(apiKey: string, agentId: string, toolIds: string[]): Promise<boolean> {
try {
const response = await fetch(`${ELEVENLABS_API_BASE}/convai/agents/${agentId}`, {
method: 'PATCH',
headers: {
'xi-api-key': apiKey,
'Content-Type': 'application/json',
'Accept': 'application/json'
},
body: JSON.stringify(buildVoiceAgentConfig(toolIds))
})

if (!response.ok) {
const errorData = await response.json().catch(() => ({})) as { detail?: { message?: string } | string }
const errorMessage = typeof errorData.detail === 'string'
? errorData.detail
: (errorData.detail as { message?: string })?.message || `API error: ${response.status}`
console.error('[Voice] Failed to update agent:', errorMessage)
return false
}

return true
} catch (error) {
console.error('[Voice] Error updating agent:', error)
return false
}
}

async function listTools(apiKey: string): Promise<ElevenLabsTool[]> {
const response = await fetch(`${ELEVENLABS_API_BASE}/convai/tools`, {
method: 'GET',
headers: {
'xi-api-key': apiKey,
'Accept': 'application/json'
}
})

if (!response.ok) {
const errorData = await response.json().catch(() => ({})) as { detail?: { message?: string } | string }
const errorMessage = typeof errorData.detail === 'string'
? errorData.detail
: (errorData.detail as { message?: string })?.message || `API error: ${response.status}`
throw new Error(errorMessage)
}

const data = await response.json() as { tools?: ElevenLabsTool[] }
return data.tools || []
}

async function createTool(apiKey: string, toolConfig: VoiceToolConfig): Promise<string> {
const response = await fetch(`${ELEVENLABS_API_BASE}/convai/tools`, {
method: 'POST',
headers: {
'xi-api-key': apiKey,
'Content-Type': 'application/json',
'Accept': 'application/json'
},
body: JSON.stringify({ tool_config: toolConfig })
})

if (!response.ok) {
const errorData = await response.json().catch(() => ({})) as { detail?: { message?: string } | string }
const errorMessage = typeof errorData.detail === 'string'
? errorData.detail
: (errorData.detail as { message?: string })?.message || `API error: ${response.status}`
throw new Error(errorMessage)
}

const data = await response.json() as { id?: string }
if (!data.id) {
throw new Error('No tool id in ElevenLabs response')
}

return data.id
}

async function ensureHapiToolIds(apiKey: string): Promise<string[]> {
const existingTools = await listTools(apiKey)
const toolIdByName = new Map(
existingTools
.filter((tool) => tool.tool_config?.type === 'client' && typeof tool.tool_config?.name === 'string')
.map((tool) => [tool.tool_config!.name!, tool.id])
)

const toolIds: string[] = []
for (const request of buildVoiceToolRequests()) {
const existingId = toolIdByName.get(request.tool_config.name)
Copy link
Copy Markdown

@github-actions github-actions bot Apr 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[MAJOR] Reusing any existing tool ID here means previously migrated Hapi tools never receive later schema/config changes. This PR already mutates processPermissionRequest in shared/src/voice.ts, but accounts that already have that tool will keep the stale definition forever because there is no PATCH/update path.

Suggested fix:

const existingTool = existingToolsByName.get(request.tool_config.name)
const toolId = existingTool
    ? await updateToolIfNeeded(apiKey, existingTool, request.tool_config)
    : await createTool(apiKey, request.tool_config)
toolIds.push(toolId)

if (existingId) {
toolIds.push(existingId)
continue
}

const createdId = await createTool(apiKey, request.tool_config)
toolIds.push(createdId)
toolIdByName.set(request.tool_config.name, createdId)
}

return toolIds
}

/**
* Get or create agent ID - finds existing or creates new "Hapi Voice Assistant" agent
*/
Expand All @@ -90,16 +200,28 @@ async function getOrCreateAgentId(apiKey: string): Promise<string | null> {
return cached
}

let toolIds: string[]
try {
toolIds = await ensureHapiToolIds(apiKey)
} catch (error) {
console.error('[Voice] Failed to ensure Hapi tools:', error)
return null
}

// Try to find existing agent
console.log('[Voice] No agent ID configured, searching for existing agent...')
let agentId = await findHapiAgent(apiKey)

if (agentId) {
console.log('[Voice] Found existing agent:', agentId)
const updated = await updateHapiAgent(apiKey, agentId, toolIds)
if (!updated) {
return null
}
} else {
// Create new agent
console.log('[Voice] No existing agent found, creating new one...')
agentId = await createHapiAgent(apiKey)
agentId = await createHapiAgent(apiKey, toolIds)
if (agentId) {
console.log('[Voice] Created new agent:', agentId)
}
Expand Down
38 changes: 33 additions & 5 deletions shared/src/voice.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,24 @@ For builds, tests, or large file operations:

export const VOICE_FIRST_MESSAGE = "Hey! Hapi here."

export const VOICE_TOOLS = [
export interface VoiceToolConfig {
type: 'client'
name: string
description: string
expects_response: boolean
response_timeout_secs: number
parameters: {
type: 'object'
required: string[]
properties: Record<string, {
type: 'string'
description: string
enum?: string[]
}>
}
}

export const VOICE_TOOLS: VoiceToolConfig[] = [
{
type: 'client' as const,
name: 'messageCodingAgent',
Expand Down Expand Up @@ -170,7 +187,8 @@ export const VOICE_TOOLS = [
properties: {
decision: {
type: 'string',
description: "The user's decision: must be either 'allow' or 'deny'"
description: "The user's decision: must be either 'allow' or 'deny'",
enum: ['allow', 'deny']
}
}
}
Expand All @@ -188,7 +206,7 @@ export interface VoiceAgentConfig {
llm: string
temperature: number
max_tokens: number
tools: typeof VOICE_TOOLS
tool_ids: string[]
}
}
turn: {
Expand All @@ -213,11 +231,21 @@ export interface VoiceAgentConfig {
}
}

export interface VoiceToolCreateRequest {
tool_config: VoiceToolConfig
}

export function buildVoiceToolRequests(): VoiceToolCreateRequest[] {
return VOICE_TOOLS.map(tool => ({
tool_config: tool
}))
}

/**
* Build the agent configuration for Hapi Voice Assistant.
* Used by both server-side auto-creation and client-side configuration.
*/
export function buildVoiceAgentConfig(): VoiceAgentConfig {
export function buildVoiceAgentConfig(toolIds: string[] = []): VoiceAgentConfig {
return {
name: VOICE_AGENT_NAME,
conversation_config: {
Expand All @@ -229,7 +257,7 @@ export function buildVoiceAgentConfig(): VoiceAgentConfig {
llm: 'gemini-2.5-flash',
temperature: 0.7,
max_tokens: 1024,
tools: VOICE_TOOLS
tool_ids: toolIds
}
},
turn: {
Expand Down
87 changes: 85 additions & 2 deletions web/src/api/voice.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ import type { ApiClient } from './client'
import {
ELEVENLABS_API_BASE,
VOICE_AGENT_NAME,
buildVoiceAgentConfig
buildVoiceAgentConfig,
buildVoiceToolRequests,
type VoiceToolConfig
} from '@hapi/protocol/voice'

export interface VoiceTokenResponse {
Expand Down Expand Up @@ -55,6 +57,14 @@ export interface ElevenLabsAgent {
name: string
}

interface ElevenLabsTool {
id: string
tool_config?: {
name?: string
type?: string
}
}

export interface FindAgentResult {
success: boolean
agentId?: string
Expand All @@ -68,6 +78,78 @@ export interface CreateAgentResult {
created?: boolean
}

async function listTools(apiKey: string): Promise<ElevenLabsTool[]> {
const response = await fetch(`${ELEVENLABS_API_BASE}/convai/tools`, {
method: 'GET',
headers: {
'xi-api-key': apiKey,
'Accept': 'application/json'
}
})

if (!response.ok) {
const errorData = await response.json().catch(() => ({})) as { detail?: { message?: string } | string }
const errorMessage = typeof errorData.detail === 'string'
? errorData.detail
: errorData.detail?.message || `API error: ${response.status}`
throw new Error(errorMessage)
}

const data = await response.json() as { tools?: ElevenLabsTool[] }
return data.tools || []
}

async function createTool(apiKey: string, toolConfig: VoiceToolConfig): Promise<string> {
const response = await fetch(`${ELEVENLABS_API_BASE}/convai/tools`, {
method: 'POST',
headers: {
'xi-api-key': apiKey,
'Content-Type': 'application/json',
'Accept': 'application/json'
},
body: JSON.stringify({ tool_config: toolConfig })
})

if (!response.ok) {
const errorData = await response.json().catch(() => ({})) as { detail?: { message?: string } | string }
const errorMessage = typeof errorData.detail === 'string'
? errorData.detail
: errorData.detail?.message || `API error: ${response.status}`
throw new Error(errorMessage)
}

const data = await response.json() as { id?: string }
if (!data.id) {
throw new Error('Failed to get tool ID from response')
}

return data.id
}

async function ensureHapiToolIds(apiKey: string): Promise<string[]> {
const existingTools = await listTools(apiKey)
const toolIdByName = new Map(
existingTools
.filter((tool) => tool.tool_config?.type === 'client' && typeof tool.tool_config?.name === 'string')
.map((tool) => [tool.tool_config!.name!, tool.id])
)

const toolIds: string[] = []
for (const request of buildVoiceToolRequests()) {
const existingId = toolIdByName.get(request.tool_config.name)
if (existingId) {
toolIds.push(existingId)
continue
}

const createdId = await createTool(apiKey, request.tool_config)
toolIds.push(createdId)
toolIdByName.set(request.tool_config.name, createdId)
}

return toolIds
}

/**
* Find an existing "Hapi Voice Assistant" agent using the provided API key.
*/
Expand Down Expand Up @@ -112,7 +194,8 @@ export async function createOrUpdateHapiAgent(apiKey: string): Promise<CreateAge
const findResult = await findHapiAgent(apiKey)
const existingAgentId = findResult.success ? findResult.agentId : null

const agentConfig = buildVoiceAgentConfig()
const toolIds = await ensureHapiToolIds(apiKey)
const agentConfig = buildVoiceAgentConfig(toolIds)

let response: Response
let created = false
Expand Down
Loading