From 85fca418cd6b687b0044f181932fd248f368a607 Mon Sep 17 00:00:00 2001 From: shobhit99 Date: Fri, 15 May 2026 19:19:34 +0530 Subject: [PATCH 1/2] fix the image not being copied --- src/main/clipboard-manager.ts | 95 +++++++++++++++++++---- src/native/fast-paste-addon/fast_paste.mm | 11 +++ 2 files changed, 91 insertions(+), 15 deletions(-) diff --git a/src/main/clipboard-manager.ts b/src/main/clipboard-manager.ts index a359e0d0..4fc8c303 100644 --- a/src/main/clipboard-manager.ts +++ b/src/main/clipboard-manager.ts @@ -14,6 +14,23 @@ import * as fs from 'fs'; import * as path from 'path'; import * as crypto from 'crypto'; +// Lazy-loaded native addon — provides getPasteboardChangeCount() which returns +// NSPasteboard.general.changeCount (an integer that increments on every write). +// Checking this is O(1) and avoids all pasteboard data reads when nothing changed. +type FastPasteAddon = { getPasteboardChangeCount?: () => number }; +let _fastPasteAddon: FastPasteAddon | null = null; +let _fastPasteAddonLoaded = false; +function getFastPasteAddon(): FastPasteAddon | null { + if (_fastPasteAddonLoaded) return _fastPasteAddon; + _fastPasteAddonLoaded = true; + try { + _fastPasteAddon = require(path.join(__dirname, '..', 'native', 'fast_paste.node')); + } catch { + _fastPasteAddon = null; + } + return _fastPasteAddon; +} + /** * Write a GIF file to macOS pasteboard with file URL + GIF data + TIFF * fallback via NSPasteboard so apps like Twitter/Slack treat it as a GIF @@ -79,6 +96,10 @@ let lastClipboardImageHash = ''; let lastClipboardFilePath = ''; let pollInterval: NodeJS.Timeout | null = null; let isEnabled = true; +// Last-seen NSPasteboard changeCount. -1 = not yet read. +// When changeCount hasn't changed, the pasteboard is identical to the last poll +// and we can skip all reads entirely (O(1) check via native addon). +let lastPasteboardChangeCount = -1; // Bundle IDs (lower-cased) whose copies should be skipped. Kept lowercase // so membership checks are case-insensitive against whatever macOS reports. let blacklistedAppBundleIds: Set = new Set(); @@ -538,14 +559,17 @@ function isImageFilePath(filePath: string): boolean { /** * Compute a cheap fingerprint for the image currently on the clipboard WITHOUT - * calling toPNG() / decoding pixels. Strategy (in priority order): + * calling toPNG() / decoding pixels where possible. Strategy (in priority order): * * GIF → full GIF buffer hash (GIFs are small) - * PNG → size + hash of first 4 KB of the PNG buffer (compressed, no decode) - * TIFF → size + hash of first 4 KB of the TIFF buffer (raw IPC read, no encode) - * - * Reading raw format bytes is an OS IPC copy (memory-bandwidth-bound, fast). - * toPNG() on a large TIFF is a pixel decode + PNG encode (CPU-bound, very slow). + * PNG → size + hash of sampled PNG buffer (compressed, no decode) + * TIFF → size + hash of sampled TIFF buffer (raw IPC read, no encode) + * img → readImage() fallback for browsers that use non-standard UTIs + * (e.g. Chrome puts com.google.chrome.image-htm; Electron's readImage() + * decodes it via NSImage but none of our UTI checks match). + * This path calls toPNG() once, but it is only reached when the + * clipboard actually changed (changeCount guard in pollClipboard), so + * it is a one-time cost per copy, not a per-poll cost. * * Returns the fingerprint string and any pre-read rawGifData. */ @@ -556,8 +580,6 @@ function getClipboardImageFingerprint(): { fingerprint: string; rawGifData?: Buf const hasPng = formats.includes('public.png'); const hasTiff = formats.includes('public.tiff'); - if (!hasGif && !hasPng && !hasTiff) return { fingerprint: '' }; - if (hasGif) { try { const gifBuf = clipboard.readBuffer('com.compuserve.gif'); @@ -588,6 +610,20 @@ function getClipboardImageFingerprint(): { fingerprint: string; rawGifData?: Buf } } catch {} } + + // Fallback: browser or app used a non-standard UTI (e.g. Chrome's + // com.google.chrome.image-htm). Electron's readImage() can decode many + // image formats via NSImage even when the UTI isn't one of the three above. + // This only runs when changeCount changed, so toPNG() is a one-time cost. + try { + const img = clipboard.readImage(); + if (!img.isEmpty()) { + const png = img.toPNG(); + if (png.length > 8) { + return { fingerprint: buildImageFingerprint('img', png) }; + } + } + } catch {} } catch {} return { fingerprint: '' }; } @@ -596,8 +632,20 @@ function pollClipboard(): void { if (!isEnabled) return; try { - // Compute image fingerprint using raw format bytes — no toPNG() / pixel decode. - // toPNG() is only called inside addImageItem when we actually save a new image. + // Cheap pre-check: NSPasteboard.changeCount increments on every write. + // If it hasn't changed since the last poll, nothing is on the clipboard that + // we haven't already seen — skip all IPC reads entirely. + const addon = getFastPasteAddon(); + if (addon?.getPasteboardChangeCount) { + const currentChangeCount = addon.getPasteboardChangeCount(); + if (currentChangeCount === lastPasteboardChangeCount) return; + lastPasteboardChangeCount = currentChangeCount; + } + + // Compute image fingerprint. For known UTIs (gif/png/tiff) this reads raw + // format bytes without decoding pixels. For non-standard UTIs (e.g. Chrome) + // it falls back to readImage()+toPNG() once — acceptable because changeCount + // already confirmed the clipboard changed, so this is not a steady-state cost. const { fingerprint: imageFingerprint, rawGifData } = getClipboardImageFingerprint(); // A file URL on the pasteboard (Finder copy) takes priority over @@ -749,13 +797,20 @@ function handleClipboardFileCopy(filePath: string): boolean { export function startClipboardMonitor(): void { loadHistory(); - - // Initial read — use the same cheap fingerprint approach as pollClipboard. + + // Seed state from whatever is on the clipboard right now so the first poll + // doesn't create spurious entries for pre-existing clipboard content. + // NOTE: do NOT seed lastClipboardImageHash here. The changeCount guard below + // prevents reprocessing of startup clipboard state. Seeding the hash would + // permanently block the first user copy of an image that was already on the + // clipboard when the app launched (fingerprint === seeded hash → skip forever). try { lastClipboardText = clipboard.readText(); - const { fingerprint } = getClipboardImageFingerprint(); - if (fingerprint) lastClipboardImageHash = fingerprint; lastClipboardFilePath = readClipboardFilePath() || ''; + const addon = getFastPasteAddon(); + if (addon?.getPasteboardChangeCount) { + lastPasteboardChangeCount = addon.getPasteboardChangeCount(); + } } catch {} // Start polling @@ -944,11 +999,21 @@ export function copyItemToClipboard(id: string): boolean { sortClipboardHistory(); saveHistory(); + // Seed the changeCount so the next poll recognises our write as "already seen" + // and doesn't create a duplicate entry. This works even if the poll fires + // before the isEnabled timeout below expires. + try { + const addon = getFastPasteAddon(); + if (addon?.getPasteboardChangeCount) { + lastPasteboardChangeCount = addon.getPasteboardChangeCount(); + } + } catch {} + // Re-enable monitoring after a short delay setTimeout(() => { isEnabled = true; }, 500); - + return true; } catch (e) { isEnabled = true; diff --git a/src/native/fast-paste-addon/fast_paste.mm b/src/native/fast-paste-addon/fast_paste.mm index 468fe01c..5e9bd8ec 100644 --- a/src/native/fast-paste-addon/fast_paste.mm +++ b/src/native/fast-paste-addon/fast_paste.mm @@ -104,10 +104,21 @@ return PostPaste(info); } +// Returns NSPasteboard.general.changeCount — an integer that increments +// whenever any application writes to the system pasteboard. Checking this +// value is O(1) and avoids reading pasteboard data entirely when nothing has +// changed. +Napi::Value GetPasteboardChangeCount(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + NSInteger count = [[NSPasteboard generalPasteboard] changeCount]; + return Napi::Number::New(env, (double)count); +} + Napi::Object Init(Napi::Env env, Napi::Object exports) { exports.Set("activateApp", Napi::Function::New(env, ActivateApp)); exports.Set("postPaste", Napi::Function::New(env, PostPaste)); exports.Set("activateAndPaste", Napi::Function::New(env, ActivateAndPaste)); + exports.Set("getPasteboardChangeCount", Napi::Function::New(env, GetPasteboardChangeCount)); return exports; } From 951382516b2a252538b9c2e7cfa142123852709d Mon Sep 17 00:00:00 2001 From: shobhit99 Date: Fri, 15 May 2026 21:01:39 +0530 Subject: [PATCH 2/2] fix: selected text not captured on first inline AI prompt open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prompt window's 'window-shown' IPC message was sent immediately after createPromptWindow() — but on the first open the BrowserWindow had just been created and the React app hadn't mounted yet, so PromptApp's onWindowShown listener wasn't registered. The message was silently dropped. Subsequent opens reused the already-loaded window, so the listener was live and the text arrived correctly. Fix mirrors the pattern used for the main launcher window: - PromptApp now calls rendererReady() on mount, emitting the same 'renderer-ready' IPC signal the main App.tsx sends. - createPromptWindow registers an ipcMain.once('renderer-ready') handler (with re-register-on-wrong-sender semantics) that sets promptRendererReady=true and flushes any pending payload. - showPromptWindow checks promptRendererReady: if true, sends immediately (all subsequent opens); if false, stores the payload in pendingPromptWindowShown so the handler delivers it once the app mounts. Also rewrites get-selected-text.swift with BFS traversal across focused element, focused window, and system-wide element roots; adds text marker range support for Chromium/Electron apps; skips secure text fields; and adds AXEnhancedUserInterface opt-in for richer attribute access. Adds AppKit framework to the native build command to support NSWorkspace. Co-Authored-By: Claude Sonnet 4.6 --- package.json | 2 +- src/main/main.ts | 54 ++++-- src/native/get-selected-text.swift | 254 ++++++++++++++++++++++++----- src/renderer/src/PromptApp.tsx | 4 + 4 files changed, 260 insertions(+), 54 deletions(-) diff --git a/package.json b/package.json index d0827de7..f201a6c8 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "build:main": "tsc -p tsconfig.main.json && cp src/main/emoji-data.json dist/main/emoji-data.json", "build:renderer": "vite build", "check:i18n": "node scripts/check-i18n.mjs", - "build:native": "mkdir -p dist/native && swiftc -O -o dist/native/get-selected-text src/native/get-selected-text.swift -framework Foundation -framework ApplicationServices && swiftc -O -o dist/native/color-picker src/native/color-picker.swift -framework AppKit && swiftc -O -o dist/native/keyboard-lock src/native/keyboard-lock.swift -framework CoreGraphics -framework Foundation && swiftc -O -o dist/native/screen-ocr src/native/screen-ocr.swift -framework AppKit -framework CoreGraphics -framework Foundation -framework Vision && swiftc -O -o dist/native/snippet-expander src/native/snippet-expander.swift -framework AppKit && swiftc -O -o dist/native/emoji-trigger-monitor src/native/emoji-trigger-monitor.swift src/native/ax-caret-query.swift -framework AppKit -framework ApplicationServices && swiftc -O -o dist/native/hotkey-hold-monitor src/native/hotkey-hold-monitor.swift -framework CoreGraphics -framework AppKit -framework Carbon && swiftc -O -o dist/native/speech-recognizer src/native/speech-recognizer.swift -framework Speech -framework AVFoundation && swiftc -O -o dist/native/microphone-access src/native/microphone-access.swift -framework AVFoundation && swiftc -O -o dist/native/input-monitoring-request src/native/input-monitoring-request.swift -framework CoreGraphics && swiftc -O -o dist/native/window-adjust src/native/window-adjust.swift -framework ApplicationServices -framework AppKit && swiftc -O -o dist/native/calendar-events src/native/calendar-events.swift -framework EventKit && swiftc -O -o dist/native/settings-coordinator src/native/settings-coordinator.swift -framework Foundation && cd src/native/fast-paste-addon && HOME=~/.electron-gyp npx node-gyp rebuild --target=$(node -e \"console.log(require('../../../node_modules/electron/package.json').version)\") --arch=$(node -e \"console.log(process.arch)\") --dist-url=https://electronjs.org/headers && cp build/Release/fast_paste.node ../../../dist/native/fast_paste.node && cd ../../.. && node scripts/build-whispercpp.mjs && node scripts/build-parakeet.mjs && node scripts/build-soulver-calculator.mjs", + "build:native": "mkdir -p dist/native && swiftc -O -o dist/native/get-selected-text src/native/get-selected-text.swift -framework Foundation -framework ApplicationServices -framework AppKit && swiftc -O -o dist/native/color-picker src/native/color-picker.swift -framework AppKit && swiftc -O -o dist/native/keyboard-lock src/native/keyboard-lock.swift -framework CoreGraphics -framework Foundation && swiftc -O -o dist/native/screen-ocr src/native/screen-ocr.swift -framework AppKit -framework CoreGraphics -framework Foundation -framework Vision && swiftc -O -o dist/native/snippet-expander src/native/snippet-expander.swift -framework AppKit && swiftc -O -o dist/native/emoji-trigger-monitor src/native/emoji-trigger-monitor.swift src/native/ax-caret-query.swift -framework AppKit -framework ApplicationServices && swiftc -O -o dist/native/hotkey-hold-monitor src/native/hotkey-hold-monitor.swift -framework CoreGraphics -framework AppKit -framework Carbon && swiftc -O -o dist/native/speech-recognizer src/native/speech-recognizer.swift -framework Speech -framework AVFoundation && swiftc -O -o dist/native/microphone-access src/native/microphone-access.swift -framework AVFoundation && swiftc -O -o dist/native/input-monitoring-request src/native/input-monitoring-request.swift -framework CoreGraphics && swiftc -O -o dist/native/window-adjust src/native/window-adjust.swift -framework ApplicationServices -framework AppKit && swiftc -O -o dist/native/calendar-events src/native/calendar-events.swift -framework EventKit && swiftc -O -o dist/native/settings-coordinator src/native/settings-coordinator.swift -framework Foundation && cd src/native/fast-paste-addon && HOME=~/.electron-gyp npx node-gyp rebuild --target=$(node -e \"console.log(require('../../../node_modules/electron/package.json').version)\") --arch=$(node -e \"console.log(process.arch)\") --dist-url=https://electronjs.org/headers && cp build/Release/fast_paste.node ../../../dist/native/fast_paste.node && cd ../../.. && node scripts/build-whispercpp.mjs && node scripts/build-parakeet.mjs && node scripts/build-soulver-calculator.mjs", "postinstall": "electron-builder install-app-deps", "start": "electron .", "package": "cross-env NODE_ENV=production npm run build && electron-builder", diff --git a/src/main/main.ts b/src/main/main.ts index ae3be443..e2c616d7 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -2037,6 +2037,8 @@ function computeDetachedPopupPosition( let mainWindow: InstanceType | null = null; let promptWindow: InstanceType | null = null; let promptWindowPrewarmScheduled = false; +let promptRendererReady = false; +let pendingPromptWindowShown: { mode: string; selectedTextSnapshot: string } | null = null; let memoryStatusWindow: InstanceType | null = null; let memoryStatusHideTimer: NodeJS.Timeout | null = null; let memoryStatusRenderSeq = 0; @@ -5570,13 +5572,13 @@ async function captureSelectionSnapshotBeforeShow(options?: { allowClipboardFall rememberSelectionSnapshot(''); return ''; } - // Skip System Events during window-show if permission hasn't been confirmed - // yet, to avoid triggering the macOS Automation dialog unexpectedly. - if (!systemEventsPermissionConfirmed) { + const allowClipboardFallback = options?.allowClipboardFallback === true; + // Skip only the System Events fallback during window-show if permission + // has not been confirmed. AX selection reads do not require Automation. + if (allowClipboardFallback && !systemEventsPermissionConfirmed) { rememberSelectionSnapshot(''); return ''; } - const allowClipboardFallback = options?.allowClipboardFallback === true; try { const selected = String( await getSelectedTextForSpeak({ allowClipboardFallback, clipboardWaitMs: 90 }) || '' @@ -7659,6 +7661,7 @@ function getDefaultPromptWindowBounds(): { x: number; y: number; width: number; function createPromptWindow(initialBounds?: { x: number; y: number; width: number; height: number }): void { if (promptWindow && !promptWindow.isDestroyed()) return; + promptRendererReady = false; const useNativeLiquidGlass = shouldUseNativeLiquidGlass(); const bounds = initialBounds || getDefaultPromptWindowBounds(); promptWindow = new BrowserWindow({ @@ -7695,7 +7698,27 @@ function createPromptWindow(initialBounds?: { x: number; y: number; width: numbe loadWindowUrl(promptWindow, '/prompt'); promptWindow.on('closed', () => { promptWindow = null; - }); + promptRendererReady = false; + pendingPromptWindowShown = null; + }); + + // Defer any queued window-shown until the React app has mounted. + // 'did-finish-load' fires before React mounts (dynamic import chunks), so we + // wait for the explicit 'renderer-ready' signal from PromptApp instead. + const capturedWindow = promptWindow; + const onPromptRendererReady = (event: Electron.IpcMainEvent) => { + if (!capturedWindow || capturedWindow.isDestroyed()) return; + if (event.sender !== capturedWindow.webContents) { + ipcMain.once('renderer-ready', onPromptRendererReady); + return; + } + promptRendererReady = true; + if (pendingPromptWindowShown) { + capturedWindow.webContents.send('window-shown', pendingPromptWindowShown); + pendingPromptWindowShown = null; + } + }; + ipcMain.once('renderer-ready', onPromptRendererReady); } function schedulePromptWindowPrewarm(): void { @@ -7723,10 +7746,14 @@ function showPromptWindow( promptWindow.moveTop(); promptWindow.webContents.focus(); const selectedTextSnapshot = String(getRecentSelectionSnapshot() || lastCursorPromptSelection || '').trim(); - promptWindow.webContents.send('window-shown', { - mode: 'prompt', - selectedTextSnapshot, - }); + const payload = { mode: 'prompt', selectedTextSnapshot }; + if (promptRendererReady) { + promptWindow.webContents.send('window-shown', payload); + } else { + // Renderer hasn't mounted yet (first open) — the createPromptWindow + // ipcMain.once('renderer-ready') handler will deliver this once PromptApp mounts. + pendingPromptWindowShown = payload; + } } function hidePromptWindow(): void { @@ -9712,7 +9739,7 @@ async function runCommandById(commandId: string, source: 'launcher' | 'hotkey' | const isLauncherPath = source === 'launcher'; const selectionPromise = isLauncherPath ? Promise.resolve('') - : getSelectedTextForSpeak({ allowClipboardFallback: true }); + : getSelectedTextForSpeak({ allowClipboardFallback: false }); // Caret/input captures must happen synchronously before focus shifts. const earlyCaretRect = isLauncherPath ? null : getTypingCaretRect(); @@ -9728,10 +9755,9 @@ async function runCommandById(commandId: string, source: 'launcher' | 'hotkey' | return true; } - // Await the selection. For the hotkey path the AX query has typically - // already resolved during the ~150 ms caret capture above, so this adds - // no measurable delay. Cmd+C clipboard fallback also reaches the right - // target because the original app is still frontmost here. + // Await the selection. For the hotkey path the native AX query has + // typically resolved during the caret capture above, so this adds no + // measurable delay and does not touch the user's clipboard. const selectedBeforeOpenRaw = String( (await selectionPromise) || getRecentSelectionSnapshot() || lastCursorPromptSelection || '' ); diff --git a/src/native/get-selected-text.swift b/src/native/get-selected-text.swift index 0351b521..7e4f4112 100644 --- a/src/native/get-selected-text.swift +++ b/src/native/get-selected-text.swift @@ -1,54 +1,230 @@ import Foundation import ApplicationServices +import AppKit -// Use AXUIElementCreateSystemWide to get the focused application directly — -// no NSWorkspace / AppKit needed, keeps startup overhead minimal (~10 ms). -let systemElement = AXUIElementCreateSystemWide() +private let debugEnabled = ProcessInfo.processInfo.environment["GET_SELECTED_TEXT_DEBUG"] == "1" -var focusedAppRaw: AnyObject? -guard AXUIElementCopyAttributeValue(systemElement, kAXFocusedApplicationAttribute as CFString, &focusedAppRaw) == .success, - let focusedApp = focusedAppRaw else { - exit(0) +private func dbg(_ message: @autoclosure () -> String) { + if debugEnabled { + FileHandle.standardError.write(Data(("[get-selected-text] " + message() + "\n").utf8)) + } } -let appElement = focusedApp as! AXUIElement -var focusedRaw: AnyObject? -guard AXUIElementCopyAttributeValue(appElement, kAXFocusedUIElementAttribute as CFString, &focusedRaw) == .success, - let focused = focusedRaw else { - exit(0) +private func writeAndExit(_ text: String) -> Never { + if !text.isEmpty { + FileHandle.standardOutput.write(Data(text.utf8)) + } + exit(0) } -let focusedElement = focused as! AXUIElement -// 1. kAXSelectedTextAttribute — supported by most native text controls. -var selectedRaw: AnyObject? -if AXUIElementCopyAttributeValue(focusedElement, kAXSelectedTextAttribute as CFString, &selectedRaw) == .success, - let text = selectedRaw as? String, !text.isEmpty { - FileHandle.standardOutput.write(text.data(using: .utf8)!) - exit(0) +private func copyAttribute(_ element: AXUIElement, _ attribute: CFString) -> AnyObject? { + var raw: AnyObject? + let err = AXUIElementCopyAttributeValue(element, attribute, &raw) + if err != .success { + dbg("attribute \(attribute) err=\(err.rawValue)") + return nil + } + return raw } -// 2. Fall back: derive selection from kAXSelectedTextRangeAttribute + kAXValueAttribute. -// Works for controls that expose a range but not the text slice directly. -var rangeRaw: AnyObject? -var valueRaw: AnyObject? -guard AXUIElementCopyAttributeValue(focusedElement, kAXSelectedTextRangeAttribute as CFString, &rangeRaw) == .success, - let rangeVal = rangeRaw, - AXUIElementCopyAttributeValue(focusedElement, kAXValueAttribute as CFString, &valueRaw) == .success, - let fullText = valueRaw as? String else { - exit(0) +private func copyParameterizedAttribute(_ element: AXUIElement, _ attribute: CFString, _ parameter: AnyObject) -> AnyObject? { + var raw: AnyObject? + let err = AXUIElementCopyParameterizedAttributeValue(element, attribute, parameter, &raw) + if err != .success { + dbg("parameterized \(attribute) err=\(err.rawValue)") + return nil + } + return raw } -var cfRange = CFRange(location: 0, length: 0) -AXValueGetValue(rangeVal as! AXValue, .cfRange, &cfRange) -guard cfRange.length > 0 else { exit(0) } +private func stringFromAXResult(_ raw: AnyObject?) -> String? { + guard let raw else { return nil } + if let text = raw as? String { return text.isEmpty ? nil : text } + if let attributed = raw as? NSAttributedString { + let text = attributed.string + return text.isEmpty ? nil : text + } + if CFGetTypeID(raw) == CFAttributedStringGetTypeID() { + let attributed = raw as! NSAttributedString + let text = attributed.string + return text.isEmpty ? nil : text + } + return nil +} + +private func selectedRangeValue(_ element: AXUIElement) -> AnyObject? { + guard let rangeValue = copyAttribute(element, kAXSelectedTextRangeAttribute as CFString) else { + return nil + } + var range = CFRange(location: 0, length: 0) + guard AXValueGetValue(rangeValue as! AXValue, .cfRange, &range), range.length > 0 else { + return nil + } + return rangeValue +} + +private func selectedTextViaValueRange(_ element: AXUIElement, _ rangeValue: AnyObject) -> String? { + guard let fullText = copyAttribute(element, kAXValueAttribute as CFString) as? String else { + return nil + } + var range = CFRange(location: 0, length: 0) + guard AXValueGetValue(rangeValue as! AXValue, .cfRange, &range), range.length > 0 else { + return nil + } + + // CFRange from AX text controls is expressed in UTF-16 offsets. + let utf16 = fullText.utf16 + guard let startIdx = utf16.index(utf16.startIndex, offsetBy: range.location, limitedBy: utf16.endIndex), + let endIdx = utf16.index(startIdx, offsetBy: range.length, limitedBy: utf16.endIndex), + let slice = String(utf16[startIdx.. String? { + let attributes: [CFString] = [ + kAXStringForRangeParameterizedAttribute as CFString, + kAXAttributedStringForRangeParameterizedAttribute as CFString, + "AXStringForRange" as CFString, + "AXAttributedStringForRange" as CFString, + ] + for attribute in attributes { + if let text = stringFromAXResult(copyParameterizedAttribute(element, attribute, rangeValue)) { + return text + } + } + return nil +} + +private func selectedTextViaTextMarkerRange(_ element: AXUIElement) -> String? { + guard let markerRange = copyAttribute(element, "AXSelectedTextMarkerRange" as CFString) else { + return nil + } + + let attributes: [CFString] = [ + "AXStringForTextMarkerRange" as CFString, + "AXAttributedStringForTextMarkerRange" as CFString, + ] + for attribute in attributes { + if let text = stringFromAXResult(copyParameterizedAttribute(element, attribute, markerRange)) { + return text + } + } + return nil +} + +private func selectedTextFromElement(_ element: AXUIElement) -> String? { + let role = copyAttribute(element, kAXRoleAttribute as CFString) as? String ?? "" + let subrole = copyAttribute(element, kAXSubroleAttribute as CFString) as? String ?? "" + if role == "AXSecureTextField" || subrole == (kAXSecureTextFieldSubrole as String) { + return nil + } + + if let text = stringFromAXResult(copyAttribute(element, kAXSelectedTextAttribute as CFString)) { + return text + } + if let text = selectedTextViaTextMarkerRange(element) { + return text + } + if let rangeValue = selectedRangeValue(element) { + if let text = selectedTextViaRangeParameterizedAttribute(element, rangeValue) { + return text + } + if let text = selectedTextViaValueRange(element, rangeValue) { + return text + } + } + return nil +} + +private func axElementFromRaw(_ raw: AnyObject?) -> AXUIElement? { + guard let raw, CFGetTypeID(raw) == AXUIElementGetTypeID() else { + return nil + } + return (raw as! AXUIElement) +} + +private func enqueueFocusedChild(of element: AXUIElement, depth: Int, into queue: inout [(AXUIElement, Int)]) { + if let focused = axElementFromRaw(copyAttribute(element, kAXFocusedUIElementAttribute as CFString)) { + queue.append((focused, depth + 1)) + } +} -// CFRange uses UTF-16 offsets. -let utf16 = fullText.utf16 -guard let startIdx = utf16.index(utf16.startIndex, offsetBy: cfRange.location, limitedBy: utf16.endIndex), - let endIdx = utf16.index(startIdx, offsetBy: cfRange.length, limitedBy: utf16.endIndex) else { - exit(0) +private func enqueueChildren(of element: AXUIElement, depth: Int, into queue: inout [(AXUIElement, Int)]) { + guard let children = copyAttribute(element, kAXChildrenAttribute as CFString) as? [AXUIElement] else { + return + } + for child in children { + queue.append((child, depth + 1)) + } } -if let slice = String(utf16[startIdx.. String? { + var queue = roots.map { ($0, 0) } + var inspected = 0 + let maxDepth = 8 + let maxElements = 240 + + while let (element, depth) = queue.first { + queue.removeFirst() + inspected += 1 + if inspected > maxElements { break } + + if let text = selectedTextFromElement(element) { + dbg("selected text found at depth \(depth)") + return text + } + if depth >= maxDepth { continue } + + enqueueFocusedChild(of: element, depth: depth, into: &queue) + enqueueChildren(of: element, depth: depth, into: &queue) + } + return nil +} + +private func frontmostApplicationElement() -> AXUIElement? { + guard let frontApp = NSWorkspace.shared.frontmostApplication else { + return nil + } + let appElement = AXUIElementCreateApplication(frontApp.processIdentifier) + + // Chromium/Electron apps often expose richer text-marker attributes only + // after these AX opt-in flags have been set. They are idempotent. + AXUIElementSetAttributeValue(appElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) + AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue) + + return appElement } -exit(0) + +private func focusedElementRoots() -> [AXUIElement] { + var roots: [AXUIElement] = [] + + if let appElement = frontmostApplicationElement() { + var focused = axElementFromRaw(copyAttribute(appElement, kAXFocusedUIElementAttribute as CFString)) + if focused == nil { + Thread.sleep(forTimeInterval: 0.06) + focused = axElementFromRaw(copyAttribute(appElement, kAXFocusedUIElementAttribute as CFString)) + } + if let focused { + roots.append(focused) + } + if let focusedWindow = axElementFromRaw(copyAttribute(appElement, kAXFocusedWindowAttribute as CFString)) { + roots.append(focusedWindow) + } + } + + let systemElement = AXUIElementCreateSystemWide() + if let focused = axElementFromRaw(copyAttribute(systemElement, kAXFocusedUIElementAttribute as CFString)) { + roots.append(focused) + } + + return roots +} + +if let text = findSelectedText(from: focusedElementRoots()) { + writeAndExit(text) +} + +writeAndExit("") diff --git a/src/renderer/src/PromptApp.tsx b/src/renderer/src/PromptApp.tsx index 85a950d5..6c998f66 100644 --- a/src/renderer/src/PromptApp.tsx +++ b/src/renderer/src/PromptApp.tsx @@ -137,6 +137,10 @@ const PromptApp: React.FC = () => { return () => clearTimeout(timer); }, []); + useEffect(() => { + window.electron.rendererReady(); + }, []); + useEffect(() => { const cleanupWindowShown = window.electron.onWindowShown((payload) => { if (payload?.mode !== 'prompt') return;