diff --git a/frontend/bun.lock b/frontend/bun.lock index 3ab60275..c6f3e148 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -4,7 +4,7 @@ "": { "name": "maple", "dependencies": { - "@opensecret/react": "1.4.0", + "@opensecret/react": "1.4.3", "@radix-ui/react-alert-dialog": "^1.1.1", "@radix-ui/react-avatar": "^1.1.0", "@radix-ui/react-dialog": "^1.1.1", @@ -30,6 +30,7 @@ "react": "^18.3.1", "react-dom": "^18.3.1", "react-markdown": "^9.0.1", + "recordrtc": "^5.6.2", "rehype-highlight": "^7.0.0", "rehype-katex": "^7.0.1", "rehype-sanitize": "^6.0.0", @@ -51,6 +52,7 @@ "@types/node": "^22.3.0", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", + "@types/recordrtc": "^5.6.14", "@types/uuid": "^10.0.0", "@vitejs/plugin-react": "^4.3.1", "autoprefixer": "^10.4.20", @@ -217,7 +219,7 @@ "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], - "@opensecret/react": ["@opensecret/react@1.4.0", "", { "dependencies": { "@peculiar/x509": "^1.12.2", "@stablelib/base64": "^2.0.0", "@stablelib/chacha20poly1305": "^2.0.0", "@stablelib/random": "^2.0.0", "cbor2": "^1.7.0", "tweetnacl": "^1.0.3", "zod": "^3.23.8" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-21L4V1AWoKTzcMKwe4OrM+sj3BKl5ATMODdFwsh+wdRvmYrG/OXf9AXmO7cP9LfsN7Gb1nj3fTmnwU8Gbx//Kw=="], + "@opensecret/react": ["@opensecret/react@1.4.3", "", { "dependencies": { "@peculiar/x509": "^1.12.2", "@stablelib/base64": "^2.0.0", "@stablelib/chacha20poly1305": "^2.0.0", "@stablelib/random": "^2.0.0", "cbor2": "^1.7.0", "tweetnacl": "^1.0.3", "zod": "^3.23.8" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-lsBsPRM9tsY9C8y7hHxLe8MOKlvNVI2B3X0XLWUqn/Prm51iAl5anWsRbEhKBvvNvWjlW/gfgHBfx+i2B0tvAw=="], "@peculiar/asn1-cms": ["@peculiar/asn1-cms@2.3.15", "", { "dependencies": { "@peculiar/asn1-schema": "^2.3.15", "@peculiar/asn1-x509": "^2.3.15", "@peculiar/asn1-x509-attr": "^2.3.15", "asn1js": "^3.0.5", "tslib": "^2.8.1" } }, "sha512-B+DoudF+TCrxoJSTjjcY8Mmu+lbv8e7pXGWrhNp2/EGJp9EEcpzjBCar7puU57sGifyzaRVM03oD5L7t7PghQg=="], @@ -469,6 +471,8 @@ "@types/react-dom": ["@types/react-dom@18.3.5", "", { "peerDependencies": { "@types/react": "^18.0.0" } }, "sha512-P4t6saawp+b/dFrUr2cvkVsfvPguwsxtH6dNIYRllMsefqFzkZk5UIjzyDOv5g1dXIPdG4Sp1yCR4Z6RCUsG/Q=="], + "@types/recordrtc": ["@types/recordrtc@5.6.14", "", {}, "sha512-Reiy1sl11xP0r6w8DW3iQjc1BgXFyNC7aDuutysIjpFoqyftbQps9xPA2FoBkfVXpJM61betgYPNt+v65zvMhA=="], + "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], "@types/uuid": ["@types/uuid@10.0.0", "", {}, "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ=="], @@ -1037,6 +1041,8 @@ "readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="], + "recordrtc": ["recordrtc@5.6.2", "", {}, "sha512-1QNKKNtl7+KcwD1lyOgP3ZlbiJ1d0HtXnypUy7yq49xEERxk31PHvE9RCciDrulPCY7WJ+oz0R9hpNxgsIurGQ=="], + "reflect-metadata": ["reflect-metadata@0.2.2", "", {}, "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q=="], "rehype-highlight": ["rehype-highlight@7.0.2", "", { "dependencies": { "@types/hast": "^3.0.0", "hast-util-to-text": "^4.0.0", "lowlight": "^3.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" } }, "sha512-k158pK7wdC2qL3M5NcZROZ2tR/l7zOzjxXd5VGdcfIyoijjQqpHd3JKtYSBDpDZ38UI2WJWuFAtkMDxmx5kstA=="], diff --git a/frontend/package.json b/frontend/package.json index b2d1880c..1a687360 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -16,7 +16,7 @@ "mdast-util-gfm-autolink-literal": "2.0.0" }, "dependencies": { - "@opensecret/react": "1.4.0", + "@opensecret/react": "1.4.3", "@radix-ui/react-alert-dialog": "^1.1.1", "@radix-ui/react-avatar": "^1.1.0", "@radix-ui/react-dialog": "^1.1.1", @@ -42,6 +42,7 @@ "react": "^18.3.1", "react-dom": "^18.3.1", "react-markdown": "^9.0.1", + "recordrtc": "^5.6.2", "rehype-highlight": "^7.0.0", "rehype-katex": "^7.0.1", "rehype-sanitize": "^6.0.0", @@ -63,6 +64,7 @@ "@types/node": "^22.3.0", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", + "@types/recordrtc": "^5.6.14", "@types/uuid": "^10.0.0", "@vitejs/plugin-react": "^4.3.1", "autoprefixer": "^10.4.20", diff --git a/frontend/src-tauri/Entitlements.plist b/frontend/src-tauri/Entitlements.plist new file mode 100644 index 00000000..27c71b44 --- /dev/null +++ b/frontend/src-tauri/Entitlements.plist @@ -0,0 +1,19 @@ + + + + + + com.apple.security.device.audio-input + + + + com.apple.security.network.client + + com.apple.security.network.server + + + + com.apple.security.files.user-selected.read-write + + + diff --git a/frontend/src-tauri/Info.plist b/frontend/src-tauri/Info.plist new file mode 100644 index 00000000..73070f8e --- /dev/null +++ b/frontend/src-tauri/Info.plist @@ -0,0 +1,10 @@ + + + + + NSMicrophoneUsageDescription + Maple needs access to your microphone to record voice messages for your AI conversations. + NSCameraUsageDescription + Maple needs access to your camera to take photos for your AI conversations. + + \ No newline at end of file diff --git a/frontend/src-tauri/gen/apple/maple_iOS/Info.plist b/frontend/src-tauri/gen/apple/maple_iOS/Info.plist index 5a1056bf..526c015f 100644 --- a/frontend/src-tauri/gen/apple/maple_iOS/Info.plist +++ b/frontend/src-tauri/gen/apple/maple_iOS/Info.plist @@ -62,5 +62,11 @@ Maple needs access to your photo library to upload images to your AI conversations. NSCameraUsageDescription Maple needs access to your camera to take photos for your AI conversations. + NSMicrophoneUsageDescription + Maple needs access to your microphone to record voice messages for your AI conversations. + UIBackgroundModes + + audio + \ No newline at end of file diff --git a/frontend/src-tauri/tauri.conf.json b/frontend/src-tauri/tauri.conf.json index ef720958..a48f1aff 100644 --- a/frontend/src-tauri/tauri.conf.json +++ b/frontend/src-tauri/tauri.conf.json @@ -43,7 +43,7 @@ } ], "security": { - "csp": "default-src 'self'; connect-src 'self' https://opensecret.cloud https://*.opensecret.cloud https://trymaple.ai https://*.trymaple.ai https://secretgpt.ai https://*.secretgpt.ai https://*.maple-ca8.pages.dev https://raw.githubusercontent.com localhost:*; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; font-src 'self' data:" + "csp": "default-src 'self'; connect-src 'self' https://opensecret.cloud https://*.opensecret.cloud https://trymaple.ai https://*.trymaple.ai https://secretgpt.ai https://*.secretgpt.ai https://*.maple-ca8.pages.dev https://raw.githubusercontent.com localhost:*; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; font-src 'self' data:; worker-src 'self' blob:; media-src 'self' blob: mediastream:" } }, "bundle": { @@ -62,7 +62,7 @@ "minimumSystemVersion": "10.13", "exceptionDomain": "opensecret.cloud", "signingIdentity": null, - "entitlements": null + "entitlements": "./Entitlements.plist" }, "iOS": { "developmentTeam": "X773Y823TN" diff --git a/frontend/src/components/ChatBox.tsx b/frontend/src/components/ChatBox.tsx index aa1eb17a..a046d71d 100644 --- a/frontend/src/components/ChatBox.tsx +++ b/frontend/src/components/ChatBox.tsx @@ -1,12 +1,9 @@ -import { CornerRightUp, Bot, Image, X, FileText, Loader2, Plus } from "lucide-react"; +import { CornerRightUp, Bot, Image, X, FileText, Loader2, Mic } from "lucide-react"; +import RecordRTC from "recordrtc"; import { Button } from "@/components/ui/button"; import { Label } from "@/components/ui/label"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuTrigger -} from "@/components/ui/dropdown-menu"; +import { UpgradePromptDialog } from "@/components/UpgradePromptDialog"; +import { RecordingOverlay } from "@/components/RecordingOverlay"; import { useEffect, useRef, useState, useMemo } from "react"; import { useLocalState } from "@/state/useLocalState"; import { cn, useIsMobile } from "@/utils/utils"; @@ -205,8 +202,9 @@ export default function Component({ systemPrompt?: string, images?: File[], documentText?: string, - documentMetadata?: { filename: string; fullContent: string } - ) => void; + documentMetadata?: { filename: string; fullContent: string }, + sentViaVoice?: boolean + ) => void | Promise; startTall?: boolean; messages?: ChatMessage[]; isStreaming?: boolean; @@ -225,7 +223,8 @@ export default function Component({ clearDraftMessage, model, setModel, - availableModels + availableModels, + hasWhisperModel } = useLocalState(); const supportsVision = MODEL_CONFIG[model]?.supportsVision || false; @@ -241,8 +240,17 @@ export default function Component({ const [imageError, setImageError] = useState(null); const fileInputRef = useRef(null); const documentInputRef = useRef(null); + const [upgradeDialogOpen, setUpgradeDialogOpen] = useState(false); + const [upgradeFeature, setUpgradeFeature] = useState<"image" | "voice">("image"); const os = useOpenSecret(); - const navigate = useNavigate(); + + // Audio recording state + const [isRecording, setIsRecording] = useState(false); + const [isTranscribing, setIsTranscribing] = useState(false); + const [isProcessingSend, setIsProcessingSend] = useState(false); + const [audioError, setAudioError] = useState(null); + const recorderRef = useRef(null); + const streamRef = useRef(null); // Find the first vision-capable model the user has access to const findFirstVisionModel = () => { @@ -471,6 +479,201 @@ export default function Component({ setUploadedDocument(null); setDocumentError(null); }; + + // Audio recording functions + const startRecording = async () => { + // Prevent duplicate starts + if (isRecording || isTranscribing) return; + + try { + // Check if getUserMedia is available + if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { + setAudioError( + "Microphone access is blocked. Please check your browser permissions or disable Lockdown Mode for this site (Settings > Safari > Advanced > Lockdown Mode)." + ); + setTimeout(() => setAudioError(null), 8000); // Longer timeout for this important message + return; + } + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: false, // Disable to reduce processing overhead + noiseSuppression: true, + autoGainControl: false, // Disable AGC to prevent audio ducking + sampleRate: 16000 // Lower sample rate to match output + } + }); + + streamRef.current = stream; + + // Create RecordRTC instance configured for WAV + const recorder = new RecordRTC(stream, { + type: "audio", + mimeType: "audio/wav", + recorderType: RecordRTC.StereoAudioRecorder, + numberOfAudioChannels: 1, // Mono audio for smaller file size + desiredSampRate: 16000 // 16kHz is good for speech + }); + + recorderRef.current = recorder; + recorder.startRecording(); + setIsRecording(true); + setAudioError(null); // Clear any previous errors + } catch (error) { + console.error("Failed to start recording:", error); + const err = error as Error & { name?: string }; + console.error("Error name:", err.name); + console.error("Error message:", err.message); + + // Handle different error types + if (err.name === "NotAllowedError" || err.name === "PermissionDeniedError") { + setAudioError( + "Microphone access denied. Please enable microphone permissions in Settings > Maple." + ); + } else if (err.name === "NotFoundError" || err.name === "DevicesNotFoundError") { + setAudioError("No microphone found. Please check your device."); + } else if (err.name === "NotReadableError" || err.name === "TrackStartError") { + setAudioError("Microphone is already in use by another app."); + } else { + // Include error details for debugging + setAudioError( + `Failed to access microphone: ${err.name || "Unknown error"} - ${err.message || "Please try again"}` + ); + } + + // Clear error after 5 seconds + setTimeout(() => setAudioError(null), 5000); + } + }; + + const stopRecording = (shouldSend: boolean = false) => { + if (recorderRef.current && isRecording) { + // Only hide immediately if canceling, keep visible if sending + if (!shouldSend) { + setIsRecording(false); + } else { + setIsProcessingSend(true); // Show processing state + } + + recorderRef.current.stopRecording(async () => { + // Safely get blob (recorder might be null by now) + const blob = recorderRef.current?.getBlob(); + + if (!blob || blob.size === 0) { + console.error("No audio recorded or empty recording"); + if (shouldSend) { + setAudioError("No audio was recorded. Please try again."); + setTimeout(() => setAudioError(null), 5000); + } + // Still need to clean up + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + } + recorderRef.current = null; + setIsProcessingSend(false); + setIsRecording(false); + return; + } + + // Create a proper WAV file + const audioFile = new File([blob], "recording.wav", { + type: "audio/wav" + }); + + if (shouldSend) { + setIsTranscribing(true); + try { + const result = await os.transcribeAudio(audioFile, "whisper-large-v3"); + + // Set the transcribed text + const transcribedText = result.text.trim(); + + if (transcribedText) { + // Directly submit without updating the input field + const newValue = inputValue ? `${inputValue} ${transcribedText}` : transcribedText; + + if (newValue.trim()) { + // Wait for onSubmit to complete (in case it returns a Promise for navigation) + await onSubmit( + newValue.trim(), + messages.length === 0 ? systemPromptValue.trim() || undefined : undefined, + images, + uploadedDocument?.cleanedText, + uploadedDocument + ? { + filename: uploadedDocument.parsed.document.filename, + fullContent: + uploadedDocument.parsed.document.md_content || + uploadedDocument.parsed.document.json_content || + uploadedDocument.parsed.document.html_content || + uploadedDocument.parsed.document.text_content || + uploadedDocument.parsed.document.doctags_content || + "" + } + : undefined, + true // sentViaVoice flag + ); + + // Clear the input and other states + setInputValue(""); + imageUrls.forEach((url) => URL.revokeObjectURL(url)); + setImageUrls(new Map()); + setImages([]); + setUploadedDocument(null); + setDocumentError(null); + setImageError(null); + } + } + } catch (error) { + console.error("Transcription failed:", error); + setAudioError("Failed to transcribe audio. Please try again."); + // Clear error after 5 seconds + setTimeout(() => setAudioError(null), 5000); + } finally { + setIsTranscribing(false); + setIsProcessingSend(false); + setIsRecording(false); // Hide overlay after send is complete + } + } + + // Clean up + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + } + recorderRef.current = null; + }); + } + }; + + const toggleRecording = () => { + if (isRecording) { + stopRecording(); + } else { + startRecording(); + } + }; + + const handleRecordingSend = () => { + stopRecording(true); + }; + + const handleRecordingCancel = () => { + if (recorderRef.current && isRecording) { + setIsRecording(false); // Hide overlay immediately + + recorderRef.current.stopRecording(() => { + // Clean up without transcribing + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + } + recorderRef.current = null; + }); + } + }; + const [isFocused, setIsFocused] = useState(false); const inputRef = useRef(null); const systemPromptRef = useRef(null); @@ -511,7 +714,16 @@ export default function Component({ freshBillingStatus.product_name?.toLowerCase().includes("max") || freshBillingStatus.product_name?.toLowerCase().includes("team")); - const canUseDocuments = hasProTeamAccess; + // Check if user has access to Starter features (Starter plan and above) + const hasStarterAccess = + freshBillingStatus && + (freshBillingStatus.product_name?.toLowerCase().includes("starter") || + freshBillingStatus.product_name?.toLowerCase().includes("pro") || + freshBillingStatus.product_name?.toLowerCase().includes("max") || + freshBillingStatus.product_name?.toLowerCase().includes("team")); + + const canUseImages = hasStarterAccess; + const canUseVoice = hasProTeamAccess; const handleSubmit = (e?: React.FormEvent) => { e?.preventDefault(); @@ -709,6 +921,23 @@ export default function Component({ }; }, [imageUrls]); + // Cleanup audio recording on unmount + useEffect(() => { + return () => { + // Stop any active recording and release microphone + if (streamRef.current) { + streamRef.current.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + } + // Clean up recorder + if (recorderRef.current && isRecording) { + recorderRef.current.stopRecording(() => { + recorderRef.current = null; + }); + } + }; + }, []); + // No longer need token calculation or plan type check since we removed the hard limit // Just keeping the TokenWarning component which handles its own calculations const placeholderText = (() => { @@ -724,273 +953,289 @@ export default function Component({ })(); return ( -
- {/* Simple System Prompt Section - just a gear button and input when expanded */} -
-
- +
+ {isRecording && ( + + )} +
+ {/* Simple System Prompt Section - just a gear button and input when expanded */} +
+
+ +
+ + {isSystemPromptExpanded && ( +