diff --git a/echo/frontend/src/components/participant/ParticipantConversationAudio.tsx b/echo/frontend/src/components/participant/ParticipantConversationAudio.tsx new file mode 100644 index 00000000..cbae46b7 --- /dev/null +++ b/echo/frontend/src/components/participant/ParticipantConversationAudio.tsx @@ -0,0 +1,712 @@ +import { useChat } from "@ai-sdk/react"; +import { t } from "@lingui/core/macro"; +import { Trans } from "@lingui/react/macro"; +import { + ActionIcon, + Box, + Button, + Divider, + Group, + LoadingOverlay, + Modal, + Stack, + Text, +} from "@mantine/core"; +import { useDisclosure, useWindowEvent } from "@mantine/hooks"; +import { + IconCheck, + IconMicrophone, + IconPlayerPause, + IconPlayerPlay, + IconPlayerStopFilled, + IconPlus, + IconQuestionMark, + IconReload, + IconTextCaption, +} from "@tabler/icons-react"; +import clsx from "clsx"; +import Cookies from "js-cookie"; +import { useCallback, useEffect, useState } from "react"; +import { useParams } from "react-router"; +import { API_BASE_URL } from "@/config"; +import { useElementOnScreen } from "@/hooks/useElementOnScreen"; +import { useI18nNavigate } from "@/hooks/useI18nNavigate"; +import { useLanguage } from "@/hooks/useLanguage"; +import { useWakeLock } from "@/hooks/useWakeLock"; +import { finishConversation } from "@/lib/api"; +import { checkPermissionError } from "@/lib/utils"; +import { I18nLink } from "../common/i18nLink"; +import { ScrollToBottomButton } from "../common/ScrollToBottom"; +import { toast } from "../common/Toaster"; +import { useProjectSharingLink } from "../project/ProjectQRCode"; +import { EchoErrorAlert } from "./EchoErrorAlert"; +import { + useConversationChunksQuery, + useConversationQuery, + useConversationRepliesQuery, + useParticipantProjectById, + useUploadConversationChunk, +} from "./hooks"; +import useChunkedAudioRecorder from "./hooks/useChunkedAudioRecorder"; +import { ParticipantBody } from "./ParticipantBody"; +import SpikeMessage from "./SpikeMessage"; + +const DEFAULT_REPLY_COOLDOWN = 120; // 2 minutes in seconds +const CONVERSATION_DELETION_STATUS_CODES = [404, 403, 410]; + +export const ParticipantConversationAudio = () => { + const { projectId, conversationId } = useParams(); + const textModeUrl = `/${projectId}/conversation/${conversationId}/text`; + const finishUrl = `/${projectId}/conversation/${conversationId}/finish`; + + // Get device ID from cookies for audio recording + const savedDeviceId = Cookies.get("micDeviceId"); + const deviceId = savedDeviceId || ""; + + const projectQuery = useParticipantProjectById(projectId ?? ""); + const conversationQuery = useConversationQuery(projectId, conversationId); + const chunks = useConversationChunksQuery(projectId, conversationId); + const repliesQuery = useConversationRepliesQuery(conversationId); + const uploadChunkMutation = useUploadConversationChunk(); + + const onChunk = (chunk: Blob) => { + uploadChunkMutation.mutate({ + chunk, + conversationId: conversationId ?? "", + runFinishHook: false, + source: "PORTAL_AUDIO", + timestamp: new Date(), + }); + }; + + const [scrollTargetRef, isVisible] = useElementOnScreen({ + root: null, + rootMargin: "-83px", + threshold: 0.1, + }); + + const [troubleShootingGuideOpened, setTroubleShootingGuideOpened] = + useState(false); + const [lastReplyTime, setLastReplyTime] = useState(null); + const [remainingCooldown, setRemainingCooldown] = useState(0); + const [showCooldownMessage, setShowCooldownMessage] = useState(false); + const [ + conversationDeletedDuringRecording, + setConversationDeletedDuringRecording, + ] = useState(false); + + const [isFinishing, _setIsFinishing] = useState(false); + const [isStopping, setIsStopping] = useState(false); + const [opened, { open, close }] = useDisclosure(false); + // Navigation and language + const navigate = useI18nNavigate(); + const { iso639_1 } = useLanguage(); + const newConversationLink = useProjectSharingLink(projectQuery.data); + + // Calculate remaining cooldown time + const getRemainingCooldown = useCallback(() => { + if (!lastReplyTime) return 0; + const cooldownSeconds = DEFAULT_REPLY_COOLDOWN; + const elapsedSeconds = Math.floor( + (new Date().getTime() - lastReplyTime.getTime()) / 1000, + ); + return Math.max(0, cooldownSeconds - elapsedSeconds); + }, [lastReplyTime]); + + // Update cooldown timer + useEffect(() => { + if (!lastReplyTime) return; + + const interval = setInterval(() => { + const remaining = getRemainingCooldown(); + setRemainingCooldown(remaining); + + if (remaining <= 0) { + clearInterval(interval); + } + }, 1000); + + return () => clearInterval(interval); + }, [lastReplyTime, getRemainingCooldown]); + + const audioRecorder = useChunkedAudioRecorder({ deviceId, onChunk }); + useWakeLock({ obtainWakeLockOnMount: true }); + + const { + startRecording, + stopRecording, + isRecording, + isPaused, + pauseRecording, + resumeRecording, + recordingTime, + errored, + permissionError, + } = audioRecorder; + + const handleMicrophoneDeviceChanged = async () => { + try { + stopRecording(); + } catch (error) { + toast.error( + t`Failed to stop recording on device change. Please try again.`, + ); + console.error("Failed to stop recording on device change:", error); + } + }; + + useWindowEvent("microphoneDeviceChanged", handleMicrophoneDeviceChanged); + + // Monitor conversation status during recording - handle deletion mid-recording + useEffect(() => { + if (!isRecording) return; + + if ( + conversationQuery.isError && + !conversationQuery.isFetching && + !conversationQuery.isLoading + ) { + const error = conversationQuery.error; + const httpStatus = error?.response?.status; + + if ( + httpStatus && + CONVERSATION_DELETION_STATUS_CODES.includes(httpStatus) + ) { + console.warn( + "Conversation was deleted or is no longer accessible during recording", + { message: error?.message, status: httpStatus }, + ); + stopRecording(); + setConversationDeletedDuringRecording(true); + } else { + console.warn( + "Error fetching conversation during recording - continuing", + { message: error?.message, status: httpStatus }, + ); + } + } + }, [ + isRecording, + conversationQuery.isError, + conversationQuery.isLoading, + conversationQuery.isFetching, + conversationQuery.error, + stopRecording, + ]); + + const { + messages: echoMessages, + isLoading, + status, + error, + handleSubmit, + } = useChat({ + api: `${API_BASE_URL}/conversations/${conversationId}/get-reply`, + body: { language: iso639_1 }, + experimental_prepareRequestBody() { + return { + language: iso639_1, + }; + }, + initialMessages: + repliesQuery.data?.map((msg) => ({ + content: msg.content_text ?? "", + id: String(msg.id), + role: msg.type === "assistant_reply" ? "assistant" : "user", + })) ?? [], + onError: (error) => { + console.error("onError", error); + }, + }); + + // Handlers + const handleCheckMicrophoneAccess = async () => { + const permissionError = await checkPermissionError(); + if (["granted", "prompt"].includes(permissionError ?? "")) { + window.location.reload(); + } else { + alert( + t`Microphone access is still denied. Please check your settings and try again.`, + ); + } + }; + + const handleReply = async (e: React.MouseEvent) => { + const remaining = getRemainingCooldown(); + if (remaining > 0) { + setShowCooldownMessage(true); + const minutes = Math.floor(remaining / 60); + const seconds = remaining % 60; + const timeStr = + minutes > 0 + ? t`${minutes} minutes and ${seconds} seconds` + : t`${seconds} seconds`; + + toast.info(t`Please wait ${timeStr} before requesting another ECHO.`); + return; + } + + try { + setShowCooldownMessage(false); + // Wait for pending uploads to complete + while (uploadChunkMutation.isPending) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + // scroll to bottom of the page + setTimeout(() => { + if (scrollTargetRef.current) { + scrollTargetRef.current.scrollIntoView({ behavior: "smooth" }); + } + }, 0); + + handleSubmit(e, { allowEmptySubmit: true }); + setLastReplyTime(new Date()); + setRemainingCooldown(DEFAULT_REPLY_COOLDOWN); + } catch (error) { + console.error("Error during echo:", error); + } + }; + + const handleStopRecording = () => { + if (isRecording) { + pauseRecording(); + open(); + } + }; + + const handleConfirmFinish = async () => { + setIsStopping(true); + try { + stopRecording(); + await finishConversation(conversationId ?? ""); + close(); + navigate(finishUrl); + } catch (error) { + console.error("Error finishing conversation:", error); + toast.error(t`Failed to finish conversation. Please try again.`); + setIsStopping(false); + } + }; + + if (conversationQuery.isLoading || projectQuery.isLoading) { + return ; + } + + // Check if conversation is not present or failed to load + if ( + conversationQuery.isError || + !conversationQuery.data || + conversationDeletedDuringRecording + ) { + return ( +
+
+ + {conversationDeletedDuringRecording ? ( + + Conversation Ended + + ) : ( + + Something went wrong + + )} + + + {conversationDeletedDuringRecording ? ( + + It looks like the conversation was deleted while you were + recording. We've stopped the recording to prevent any issues. + You can start a new one anytime. + + ) : ( + + The conversation could not be loaded. Please try again or + contact support. + + )} + + + + {newConversationLink && ( + + )} + +
+
+ ); + } + + return ( +
+ {/* modal for permissions error */} + true} + centered + fullScreen + radius={0} + transitionProps={{ duration: 200, transition: "fade" }} + withCloseButton={false} + > +
+ +
+ + Oops! It looks like microphone access was denied. No worries, + though! We've got a handy troubleshooting guide for you. Feel + free to check it out. Once you've resolved the issue, come back + and visit this page again to check if your microphone is ready. + +
+ + + + + +
+ + + {/* modal for stop recording confirmation */} + {} : close} + closeOnClickOutside={!isStopping} + closeOnEscape={!isStopping} + centered + title={ + + Finish Conversation + + } + size="sm" + radius="md" + padding="xl" + > + + + + Are you sure you want to finish the conversation? + + + + + + + + + + + {projectQuery.data && conversationQuery.data && ( + + )} + + + {echoMessages && echoMessages.length > 0 && ( + <> + {echoMessages.map((message, index) => ( + + ))} + {status !== "streaming" && status !== "ready" && !error && ( + + )} + + )} + + {error && } + +
+ + + {!errored && ( + + + + + + {/* Recording time indicator */} + {isRecording && ( +
+ + {isPaused ? ( + + ) : ( +
+ )} + + {Math.floor(recordingTime / 3600) > 0 && ( + <> + {Math.floor(recordingTime / 3600) + .toString() + .padStart(2, "0")} + : + + )} + {Math.floor((recordingTime % 3600) / 60) + .toString() + .padStart(2, "0")} + :{(recordingTime % 60).toString().padStart(2, "0")} + + +
+ )} + + + {!isRecording && ( + + {chunks?.data && + chunks.data.length > 0 && + !!projectQuery.data?.is_get_reply_enabled && ( + + + + )} + + + + + + + + + + {!isRecording && + !isStopping && + chunks?.data && + chunks.data.length > 0 && ( + + )} + + )} + + {isRecording && ( + <> + {chunks?.data && + chunks.data.length > 0 && + !!projectQuery.data?.is_get_reply_enabled && ( + + + + )} + + {isPaused ? ( + + ) : ( + + )} + + + + )} + + + )} +
+ ); +}; diff --git a/echo/frontend/src/components/participant/ParticipantConversationText.tsx b/echo/frontend/src/components/participant/ParticipantConversationText.tsx new file mode 100644 index 00000000..726a5a7c --- /dev/null +++ b/echo/frontend/src/components/participant/ParticipantConversationText.tsx @@ -0,0 +1,243 @@ +import { t } from "@lingui/core/macro"; +import { Trans } from "@lingui/react/macro"; +import { + ActionIcon, + Box, + Button, + Group, + LoadingOverlay, + Modal, + Stack, + Text, +} from "@mantine/core"; +import { useDisclosure } from "@mantine/hooks"; +import { + IconCheck, + IconMicrophone, + IconPlus, + IconReload, + IconUpload, +} from "@tabler/icons-react"; +import clsx from "clsx"; +import { useState } from "react"; +import { useParams } from "react-router"; +import { I18nLink } from "@/components/common/i18nLink"; +import { + useConversationChunksQuery, + useConversationQuery, + useParticipantProjectById, + useUploadConversationTextChunk, +} from "@/components/participant/hooks"; +import { ParticipantBody } from "@/components/participant/ParticipantBody"; +import { useProjectSharingLink } from "@/components/project/ProjectQRCode"; +import { useElementOnScreen } from "@/hooks/useElementOnScreen"; +import { useI18nNavigate } from "@/hooks/useI18nNavigate"; + +export const ParticipantConversationText = () => { + const { projectId, conversationId } = useParams(); + const projectQuery = useParticipantProjectById(projectId ?? ""); + const conversationQuery = useConversationQuery(projectId, conversationId); + const chunks = useConversationChunksQuery(projectId, conversationId); + const uploadChunkMutation = useUploadConversationTextChunk(); + const newConversationLink = useProjectSharingLink(projectQuery.data); + + const [text, setText] = useState(""); + const [ + finishModalOpened, + { open: openFinishModal, close: closeFinishModal }, + ] = useDisclosure(false); + + const [scrollTargetRef] = useElementOnScreen({ + root: null, + rootMargin: "-158px", + threshold: 0.1, + }); + + const onChunk = () => { + if (!text || text.trim() === "") { + return; + } + + setTimeout(() => { + if (scrollTargetRef.current) { + scrollTargetRef.current.scrollIntoView({ behavior: "smooth" }); + } + }, 0); + + uploadChunkMutation.mutate({ + content: text.trim(), + conversationId: conversationId ?? "", + source: "PORTAL_TEXT", + timestamp: new Date(), + }); + + setText(""); + }; + + const navigate = useI18nNavigate(); + + const audioModeUrl = `/${projectId}/conversation/${conversationId}`; + const finishUrl = `/${projectId}/conversation/${conversationId}/finish`; + + const handleConfirmFinishButton = () => { + navigate(finishUrl); + }; + + if (conversationQuery.isLoading || projectQuery.isLoading) { + return ; + } + + // Check if conversation is not present or failed to load + if (conversationQuery.isError || !conversationQuery.data) { + return ( +
+
+ + + Something went wrong + + + + + The conversation could not be loaded. Please try again or contact + support. + + + + + {newConversationLink && ( + + )} + +
+
+ ); + } + + return ( +
+ {/* modal for finish conversation confirmation */} + + + Finish Conversation + + + } + size="sm" + radius="md" + padding="xl" + > + + + + Are you sure you want to finish the conversation? + + + + + + + + + + + {projectQuery.data && conversationQuery.data && ( + + )} + +
+ + + + + {/* */} + +