From 24bbb6b5b0c98669d414ad9f76d5dcc395642099 Mon Sep 17 00:00:00 2001 From: Aryan-Shan Date: Fri, 5 Dec 2025 13:13:28 +0530 Subject: [PATCH 01/10] Add OCR text selection feature --- frontend/package-lock.json | 122 +++++++++++ frontend/package.json | 1 + frontend/src/components/Media/ImageViewer.tsx | 191 ++++++++++++++---- frontend/src/components/Media/TextOverlay.tsx | 129 ++++++++++++ frontend/src/services/OCRService.ts | 54 +++++ 5 files changed, 455 insertions(+), 42 deletions(-) create mode 100644 frontend/src/components/Media/TextOverlay.tsx create mode 100644 frontend/src/services/OCRService.ts diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e1e1ddd5f..7437786f7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -51,6 +51,7 @@ "react-zoom-pan-pinch": "^3.7.0", "tailwind-merge": "^3.3.0", "tailwindcss": "^4.1.8", + "tesseract.js": "^5.1.0", "ts-node": "^10.9.2", "uuid": "^11.1.0", "vite-plugin-environment": "^1.1.3" @@ -6597,6 +6598,12 @@ "baseline-browser-mapping": "dist/cli.js" } }, + "node_modules/bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -9044,6 +9051,12 @@ "node": ">=0.10.0" } }, + "node_modules/idb-keyval": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.2.tgz", + "integrity": "sha512-yjD9nARJ/jb1g+CvD0tlhUHOrJ9Sy0P8T9MF3YaLlHnSRpwPfpTX0XIvpmw3gAJUmEu3FiICLBDPXVwyEvrleg==", + "license": "Apache-2.0" + }, "node_modules/identity-obj-proxy": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/identity-obj-proxy/-/identity-obj-proxy-3.0.0.tgz", @@ -9337,6 +9350,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-electron": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-electron/-/is-electron-2.2.2.tgz", + "integrity": "sha512-FO/Rhvz5tuw4MCWkpMzHFKWD2LsfHzIb7i6MdPYZ/KW7AlxawyLkqdy+jPZP1WubqEADE3O4FUENlJHDfQASRg==", + "license": "MIT" + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -9598,6 +9617,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-url": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", + "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==", + "license": "MIT" + }, "node_modules/is-weakmap": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz", @@ -11461,6 +11486,48 @@ "dev": true, "license": "MIT" }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/node-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -11670,6 +11737,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/opencollective-postinstall": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", + "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", + "license": "MIT", + "bin": { + "opencollective-postinstall": "index.js" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -12511,6 +12587,12 @@ "node": ">=4" } }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT" + }, "node_modules/regexp.prototype.flags": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", @@ -13360,6 +13442,31 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tesseract.js": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-5.1.0.tgz", + "integrity": "sha512-2fH9pqWdS2C6ue/3OoGg91Wtv7Rt/1atYu/g0Q1SGFrowEW/kIBkG361hLienHsWe4KWEjxOJBrCQYpIBWG6WA==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "bmp-js": "^0.1.0", + "idb-keyval": "^6.2.0", + "is-electron": "^2.2.2", + "is-url": "^1.2.4", + "node-fetch": "^2.6.9", + "opencollective-postinstall": "^2.0.3", + "regenerator-runtime": "^0.13.3", + "tesseract.js-core": "^5.1.0", + "wasm-feature-detect": "^1.2.11", + "zlibjs": "^0.3.1" + } + }, + "node_modules/tesseract.js-core": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-5.1.1.tgz", + "integrity": "sha512-KX3bYSU5iGcO1XJa+QGPbi+Zjo2qq6eBhNjSGR5E5q0JtzkoipJKOUQD7ph8kFyteCEfEQ0maWLu8MCXtvX5uQ==", + "license": "Apache-2.0" + }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -14212,6 +14319,12 @@ "makeerror": "1.0.12" } }, + "node_modules/wasm-feature-detect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz", + "integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==", + "license": "Apache-2.0" + }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", @@ -14540,6 +14653,15 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zlibjs": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz", + "integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==", + "license": "MIT", + "engines": { + "node": "*" + } } } } diff --git a/frontend/package.json b/frontend/package.json index 0a53f1b8d..89d1f524a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -66,6 +66,7 @@ "react-zoom-pan-pinch": "^3.7.0", "tailwind-merge": "^3.3.0", "tailwindcss": "^4.1.8", + "tesseract.js": "^5.1.0", "ts-node": "^10.9.2", "uuid": "^11.1.0", "vite-plugin-environment": "^1.1.3" diff --git a/frontend/src/components/Media/ImageViewer.tsx b/frontend/src/components/Media/ImageViewer.tsx index 704b65eda..2872a7433 100644 --- a/frontend/src/components/Media/ImageViewer.tsx +++ b/frontend/src/components/Media/ImageViewer.tsx @@ -1,6 +1,10 @@ -import React, { useRef, useImperativeHandle, forwardRef } from 'react'; +import { useRef, useImperativeHandle, forwardRef, useState, useEffect } from 'react'; import { TransformWrapper, TransformComponent } from 'react-zoom-pan-pinch'; import { convertFileSrc } from '@tauri-apps/api/core'; +import { ocrService } from '../../services/OCRService'; +import { TextOverlay } from './TextOverlay'; +import { Page } from 'tesseract.js'; +import { Loader2 } from 'lucide-react'; interface ImageViewerProps { imagePath: string; @@ -18,6 +22,11 @@ export interface ImageViewerRef { export const ImageViewer = forwardRef( ({ imagePath, alt, rotation, resetSignal }, ref) => { const transformRef = useRef(null); + const imgRef = useRef(null); + const [isOCRActive, setIsOCRActive] = useState(false); + const [ocrData, setOcrData] = useState(null); + const [isOCRLoading, setIsOCRLoading] = useState(false); + const [imageScale, setImageScale] = useState(1); // Expose zoom functions to parent useImperativeHandle(ref, () => ({ @@ -27,53 +36,151 @@ export const ImageViewer = forwardRef( })); // Reset on signal change - React.useEffect(() => { + useEffect(() => { transformRef.current?.resetTransform(); - }, [resetSignal]); + // Reset OCR when image changes + setIsOCRActive(false); + setOcrData(null); + }, [resetSignal, imagePath]); + + // Update scale when image loads or resizes + useEffect(() => { + const updateScale = () => { + if (imgRef.current) { + const { width, naturalWidth } = imgRef.current; + if (naturalWidth > 0) { + setImageScale(width / naturalWidth); + } + } + }; + + const img = imgRef.current; + if (img) { + // Initial update + if (img.complete) updateScale(); + + // Listen for load + img.addEventListener('load', updateScale); + + // Listen for resize + const resizeObserver = new ResizeObserver(updateScale); + resizeObserver.observe(img); + + return () => { + img.removeEventListener('load', updateScale); + resizeObserver.disconnect(); + }; + } + }, [imagePath]); // Re-run when image path changes + + // Handle Ctrl+T to toggle OCR + useEffect(() => { + const handleKeyDown = async (e: KeyboardEvent) => { + if (e.ctrlKey && e.key.toLowerCase() === 't') { + e.preventDefault(); + + if (isOCRActive) { + // Deactivate + setIsOCRActive(false); + } else { + // Activate + setIsOCRActive(true); + if (!ocrData && !isOCRLoading) { + setIsOCRLoading(true); + try { + const src = convertFileSrc(imagePath); + const data = await ocrService.recognize(src); + setOcrData(data); + } catch (error) { + console.error('Failed to perform OCR', error); + setIsOCRActive(false); // Revert if failed + } finally { + setIsOCRLoading(false); + } + } + } + } + }; + + window.addEventListener('keydown', handleKeyDown); + return () => window.removeEventListener('keydown', handleKeyDown); + }, [imagePath, isOCRActive, ocrData, isOCRLoading]); return ( - - + {isOCRLoading && ( +
+ + Processing Text... +
+ )} + + {isOCRActive && !isOCRLoading && ocrData && ( +
+ + + + + Text Selection Active +
+ )} + + - {alt} { - const img = e.target as HTMLImageElement; - img.onerror = null; - img.src = '/placeholder.svg'; + - - + > +
+ {alt} { + const img = e.target as HTMLImageElement; + img.onerror = null; + img.src = '/placeholder.svg'; + }} + style={{ + maxWidth: '100%', + maxHeight: '100%', + objectFit: 'contain', + zIndex: 50, + }} + /> + {isOCRActive && ocrData && ( + + )} +
+ + + ); }, ); diff --git a/frontend/src/components/Media/TextOverlay.tsx b/frontend/src/components/Media/TextOverlay.tsx new file mode 100644 index 000000000..7ad271f84 --- /dev/null +++ b/frontend/src/components/Media/TextOverlay.tsx @@ -0,0 +1,129 @@ +import React, { useEffect, useState } from 'react'; +import { Page } from 'tesseract.js'; +import { Check } from 'lucide-react'; + +interface TextOverlayProps { + ocrData: Page | null; + scale?: number; +} + +export const TextOverlay: React.FC = ({ ocrData, scale = 1 }) => { + const [showCopyFeedback, setShowCopyFeedback] = useState(false); + + useEffect(() => { + const handleKeyDown = async (e: KeyboardEvent) => { + if (e.ctrlKey && e.key.toLowerCase() === 'c') { + const selection = window.getSelection(); + const text = selection?.toString().trim(); + + if (text && text.length > 0) { + // We manually write to clipboard to ensure it works even with transparent text + try { + await navigator.clipboard.writeText(text); + setShowCopyFeedback(true); + setTimeout(() => setShowCopyFeedback(false), 2000); + } catch (err) { + console.error('Failed to copy text:', err); + } + } + } + }; + + window.addEventListener('keydown', handleKeyDown); + return () => window.removeEventListener('keydown', handleKeyDown); + }, []); + + if (!ocrData) return null; + + // Use lines instead of words for better sentence selection + const lines = (ocrData as any).lines || []; + + return ( + <> + {showCopyFeedback && ( +
+
+ +
+ Text copied to clipboard +
+ )} + +
e.stopPropagation()} + style={{ + position: 'absolute', + top: 0, + left: 0, + width: '100%', + height: '100%', + pointerEvents: 'auto', + zIndex: 60, + userSelect: 'text', + WebkitUserSelect: 'text', + opacity: 0, + animation: 'fadeIn 0.3s ease-out forwards', + }} + > + {lines.map((line: any, index: number) => { + const { bbox, text } = line; + const width = (bbox.x1 - bbox.x0) * scale; + const height = (bbox.y1 - bbox.y0) * scale; + const left = bbox.x0 * scale; + const top = bbox.y0 * scale; + + return ( + + {text} + + ); + })} + +
+ + ); +}; diff --git a/frontend/src/services/OCRService.ts b/frontend/src/services/OCRService.ts new file mode 100644 index 000000000..55b3f3bac --- /dev/null +++ b/frontend/src/services/OCRService.ts @@ -0,0 +1,54 @@ +import { createWorker, Worker, PSM } from 'tesseract.js'; + +class OCRService { + private worker: Worker | null = null; + private workerPromise: Promise | null = null; + + private async getWorker(): Promise { + if (this.worker) return this.worker; + + if (!this.workerPromise) { + this.workerPromise = (async () => { + try { + // Initialize with default OEM + const worker = await createWorker('eng', undefined); + + // Set Page Segmentation Mode to AUTO to ensure we get blocks/words + await worker.setParameters({ + tessedit_pageseg_mode: PSM.AUTO, + }); + + this.worker = worker; + return worker; + } catch (error) { + console.error('Failed to initialize Tesseract worker:', error); + this.workerPromise = null; + throw error; + } + })(); + } + + return this.workerPromise; + } + + async recognize(imagePath: string) { + try { + const worker = await this.getWorker(); + const result = await worker.recognize(imagePath); + return result.data; + } catch (error) { + console.error('OCR Error:', error); + throw error; + } + } + + async terminate() { + if (this.worker) { + await this.worker.terminate(); + this.worker = null; + this.workerPromise = null; + } + } +} + +export const ocrService = new OCRService(); From 3bb8d9bd3a5062acb5e40e2e5ec901c283fb4d5f Mon Sep 17 00:00:00 2001 From: Aryan-Shan Date: Fri, 5 Dec 2025 14:17:13 +0530 Subject: [PATCH 02/10] Add OCR text selection feature --- frontend/src/components/Media/ImageViewer.tsx | 14 ++++++++++++-- frontend/src/components/Media/TextOverlay.tsx | 9 +++++++-- frontend/src/services/OCRService.ts | 13 +++++++++---- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/Media/ImageViewer.tsx b/frontend/src/components/Media/ImageViewer.tsx index 2872a7433..e300518ef 100644 --- a/frontend/src/components/Media/ImageViewer.tsx +++ b/frontend/src/components/Media/ImageViewer.tsx @@ -74,6 +74,11 @@ export const ImageViewer = forwardRef( }, [imagePath]); // Re-run when image path changes // Handle Ctrl+T to toggle OCR + const imagePathRef = useRef(imagePath); + useEffect(() => { + imagePathRef.current = imagePath; + }, [imagePath]); + useEffect(() => { const handleKeyDown = async (e: KeyboardEvent) => { if (e.ctrlKey && e.key.toLowerCase() === 't') { @@ -90,12 +95,17 @@ export const ImageViewer = forwardRef( try { const src = convertFileSrc(imagePath); const data = await ocrService.recognize(src); - setOcrData(data); + // Only set data if image hasn't changed + if (imagePath === imagePathRef.current) { + setOcrData(data); + } } catch (error) { console.error('Failed to perform OCR', error); setIsOCRActive(false); // Revert if failed } finally { - setIsOCRLoading(false); + if (imagePath === imagePathRef.current) { + setIsOCRLoading(false); + } } } } diff --git a/frontend/src/components/Media/TextOverlay.tsx b/frontend/src/components/Media/TextOverlay.tsx index 7ad271f84..1736630d5 100644 --- a/frontend/src/components/Media/TextOverlay.tsx +++ b/frontend/src/components/Media/TextOverlay.tsx @@ -11,6 +11,8 @@ export const TextOverlay: React.FC = ({ ocrData, scale = 1 }) const [showCopyFeedback, setShowCopyFeedback] = useState(false); useEffect(() => { + let feedbackTimeout: ReturnType; + const handleKeyDown = async (e: KeyboardEvent) => { if (e.ctrlKey && e.key.toLowerCase() === 'c') { const selection = window.getSelection(); @@ -21,7 +23,7 @@ export const TextOverlay: React.FC = ({ ocrData, scale = 1 }) try { await navigator.clipboard.writeText(text); setShowCopyFeedback(true); - setTimeout(() => setShowCopyFeedback(false), 2000); + feedbackTimeout = setTimeout(() => setShowCopyFeedback(false), 2000); } catch (err) { console.error('Failed to copy text:', err); } @@ -30,7 +32,10 @@ export const TextOverlay: React.FC = ({ ocrData, scale = 1 }) }; window.addEventListener('keydown', handleKeyDown); - return () => window.removeEventListener('keydown', handleKeyDown); + return () => { + window.removeEventListener('keydown', handleKeyDown); + clearTimeout(feedbackTimeout); + }; }, []); if (!ocrData) return null; diff --git a/frontend/src/services/OCRService.ts b/frontend/src/services/OCRService.ts index 55b3f3bac..c76c85a8b 100644 --- a/frontend/src/services/OCRService.ts +++ b/frontend/src/services/OCRService.ts @@ -43,11 +43,16 @@ class OCRService { } async terminate() { - if (this.worker) { - await this.worker.terminate(); - this.worker = null; - this.workerPromise = null; + if (this.workerPromise) { + try { + const worker = await this.workerPromise; + await worker.terminate(); + } catch { + // Initialization failed, nothing to terminate + } } + this.worker = null; + this.workerPromise = null; } } From 9e2cc0df1ffe407401d616dc0e4fb53fbb35c5d2 Mon Sep 17 00:00:00 2001 From: Aryan-Shan Date: Fri, 5 Dec 2025 14:23:42 +0530 Subject: [PATCH 03/10] Add OCR text selection feature --- frontend/src/components/Media/ImageViewer.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/components/Media/ImageViewer.tsx b/frontend/src/components/Media/ImageViewer.tsx index e300518ef..38ebe69d8 100644 --- a/frontend/src/components/Media/ImageViewer.tsx +++ b/frontend/src/components/Media/ImageViewer.tsx @@ -41,6 +41,7 @@ export const ImageViewer = forwardRef( // Reset OCR when image changes setIsOCRActive(false); setOcrData(null); + setIsOCRLoading(false); }, [resetSignal, imagePath]); // Update scale when image loads or resizes From 4a3362ca56749fb9808009001fff1829f08ae396 Mon Sep 17 00:00:00 2001 From: Aryan-Shan Date: Sat, 6 Dec 2025 16:15:40 +0530 Subject: [PATCH 04/10] Added text selection feature --- frontend/src/components/Media/ImageViewer.tsx | 54 +++++++++++++++++-- frontend/src/components/Media/TextOverlay.tsx | 2 +- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/Media/ImageViewer.tsx b/frontend/src/components/Media/ImageViewer.tsx index 38ebe69d8..125e0daff 100644 --- a/frontend/src/components/Media/ImageViewer.tsx +++ b/frontend/src/components/Media/ImageViewer.tsx @@ -4,7 +4,7 @@ import { convertFileSrc } from '@tauri-apps/api/core'; import { ocrService } from '../../services/OCRService'; import { TextOverlay } from './TextOverlay'; import { Page } from 'tesseract.js'; -import { Loader2 } from 'lucide-react'; +import { Loader2, ScanText } from 'lucide-react'; interface ImageViewerProps { imagePath: string; @@ -119,15 +119,63 @@ export const ImageViewer = forwardRef( return (
+ {/* Text Detection Toggle Button */} + + {isOCRLoading && ( -
+
Processing Text...
)} {isOCRActive && !isOCRLoading && ocrData && ( -
+
diff --git a/frontend/src/components/Media/TextOverlay.tsx b/frontend/src/components/Media/TextOverlay.tsx index 1736630d5..ebe89efef 100644 --- a/frontend/src/components/Media/TextOverlay.tsx +++ b/frontend/src/components/Media/TextOverlay.tsx @@ -47,7 +47,7 @@ export const TextOverlay: React.FC = ({ ocrData, scale = 1 }) <> {showCopyFeedback && (
From a223a8899033349d497d863715c1710ec6c4e99a Mon Sep 17 00:00:00 2001 From: Aryan-Shan Date: Sat, 6 Dec 2025 16:24:02 +0530 Subject: [PATCH 05/10] Added text selection feature --- frontend/src/components/Media/ImageViewer.tsx | 74 +++++++------------ 1 file changed, 28 insertions(+), 46 deletions(-) diff --git a/frontend/src/components/Media/ImageViewer.tsx b/frontend/src/components/Media/ImageViewer.tsx index 125e0daff..a8c1b4d08 100644 --- a/frontend/src/components/Media/ImageViewer.tsx +++ b/frontend/src/components/Media/ImageViewer.tsx @@ -1,4 +1,4 @@ -import { useRef, useImperativeHandle, forwardRef, useState, useEffect } from 'react'; +import { useRef, useImperativeHandle, forwardRef, useState, useEffect, useCallback } from 'react'; import { TransformWrapper, TransformComponent } from 'react-zoom-pan-pinch'; import { convertFileSrc } from '@tauri-apps/api/core'; import { ocrService } from '../../services/OCRService'; @@ -80,6 +80,30 @@ export const ImageViewer = forwardRef( imagePathRef.current = imagePath; }, [imagePath]); + const triggerOCR = useCallback(async () => { + if (ocrData || isOCRLoading) return; + + setIsOCRLoading(true); + const currentPath = imagePathRef.current; + + try { + const src = convertFileSrc(currentPath); + const data = await ocrService.recognize(src); + + // Only set data if image hasn't changed + if (currentPath === imagePathRef.current) { + setOcrData(data); + } + } catch (error) { + console.error('Failed to perform OCR', error); + setIsOCRActive(false); // Revert if failed + } finally { + if (currentPath === imagePathRef.current) { + setIsOCRLoading(false); + } + } + }, [ocrData, isOCRLoading]); + useEffect(() => { const handleKeyDown = async (e: KeyboardEvent) => { if (e.ctrlKey && e.key.toLowerCase() === 't') { @@ -91,31 +115,14 @@ export const ImageViewer = forwardRef( } else { // Activate setIsOCRActive(true); - if (!ocrData && !isOCRLoading) { - setIsOCRLoading(true); - try { - const src = convertFileSrc(imagePath); - const data = await ocrService.recognize(src); - // Only set data if image hasn't changed - if (imagePath === imagePathRef.current) { - setOcrData(data); - } - } catch (error) { - console.error('Failed to perform OCR', error); - setIsOCRActive(false); // Revert if failed - } finally { - if (imagePath === imagePathRef.current) { - setIsOCRLoading(false); - } - } - } + triggerOCR(); } } }; window.addEventListener('keydown', handleKeyDown); return () => window.removeEventListener('keydown', handleKeyDown); - }, [imagePath, isOCRActive, ocrData, isOCRLoading]); + }, [isOCRActive, triggerOCR]); return (
@@ -127,32 +134,7 @@ export const ImageViewer = forwardRef( setIsOCRActive(false); } else { setIsOCRActive(true); - if (!ocrData && !isOCRLoading) { - // Trigger loading logic same as Ctrl+T - // We need to extract the loading logic into a reusable function or trigger it via effect if isOCRActive changes to true? - // The effect at line 118 depends on isOCRActive, but the keydown handler (line 84) sets state AND triggers logic. - // Let's refactor slightly to share logic or just duplicate the trigger here safely. - // Actually, the keydown handler does the heavy lifting. - // It's cleaner to just set the state and let an effect handle it, OR duplicate the call. - // Given the existing structure, I'll duplicate the trigger logic for now to ensure immediate feedback. - setIsOCRLoading(true); - (async () => { - try { - const src = convertFileSrc(imagePath); - const data = await ocrService.recognize(src); - if (imagePath === imagePathRef.current) { - setOcrData(data); - } - } catch (error) { - console.error('Failed to perform OCR', error); - setIsOCRActive(false); - } finally { - if (imagePath === imagePathRef.current) { - setIsOCRLoading(false); - } - } - })(); - } + triggerOCR(); } }} className={`absolute top-6 left-6 z-60 flex items-center gap-2 rounded-full border border-white/10 px-4 py-2.5 backdrop-blur-md transition-all duration-300 ${isOCRActive From c52b41860378ce6c1cf042242a08d69fbdad6c54 Mon Sep 17 00:00:00 2001 From: Aryan-Shan Date: Sat, 6 Dec 2025 20:30:57 +0530 Subject: [PATCH 06/10] Added Magic Eraser (Model excluded) --- .gitignore | 1 + backend/app/models/Inpainter.py | 97 ++++++ backend/app/routes/edit.py | 68 ++++ backend/main.py | 2 + backend/requirements.txt | 4 +- backend/tests/test_inpainter.py | 51 +++ debug.txt | Bin 0 -> 790 bytes docs/backend/backend_python/openapi.json | 88 +++++ frontend/src/components/Media/ImageViewer.tsx | 119 +++++-- .../components/Media/MagicEraserOverlay.tsx | 325 ++++++++++++++++++ scripts/download_models.py | 56 +++ scripts/setup.ps1 | 5 + scripts/setup.sh | 4 + 13 files changed, 796 insertions(+), 24 deletions(-) create mode 100644 backend/app/models/Inpainter.py create mode 100644 backend/app/routes/edit.py create mode 100644 backend/tests/test_inpainter.py create mode 100644 debug.txt create mode 100644 frontend/src/components/Media/MagicEraserOverlay.tsx create mode 100644 scripts/download_models.py diff --git a/.gitignore b/.gitignore index bd16f08de..42c4d55df 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ pnpm-debug.log* lerna-debug.log* backend/app/models/image-generation/* +backend/app/models/onnx_models/* node_modules diff --git a/backend/app/models/Inpainter.py b/backend/app/models/Inpainter.py new file mode 100644 index 000000000..7c060d28a --- /dev/null +++ b/backend/app/models/Inpainter.py @@ -0,0 +1,97 @@ + +import cv2 +import numpy as np +import onnxruntime as ort +import os +from app.logging.setup_logging import get_logger + +logger = get_logger(__name__) + +class Inpainter: + def __init__(self): + self.output_img_size = 512 # LaMa fixed input size + self._init_session() + + def _init_session(self): + """Initialize the ONNX Runtime session.""" + model_path = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "models", + "onnx_models", + "lama_fp32.onnx" + ) + + if not os.path.exists(model_path): + logger.error(f"Inpainting model found at {model_path}") + self.session = None + return + + providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] + if 'CUDAExecutionProvider' not in ort.get_available_providers(): + providers = ['CPUExecutionProvider'] + + try: + self.session = ort.InferenceSession(model_path, providers=providers) + logger.info(f"Inpainting model loaded successfully from {model_path}") + except Exception as e: + logger.error(f"Failed to load inpainting model: {e}") + self.session = None + + def inpaint(self, image: np.ndarray, mask: np.ndarray) -> np.ndarray: + """ + Perform inpainting on the image using the mask. + :param image: Input image (H, W, 3) BGR + :param mask: Input mask (H, W) or (H, W, 1) 0-255 (255=inpainting area) + :return: Inpainted image (H, W, 3) BGR + """ + if self.session is None: + # Try to re-init if it failed previously (e.g. download finished) + self._init_session() + if self.session is None: + raise RuntimeError("Inpainting model not loaded.") + + original_h, original_w = image.shape[:2] + + # 1. Preprocess + # Resize/Pad to 512x512 + # For simplicity, we'll just resize to 512x512. + # LaMa is resilient, but aspect ratio distortion might affect quality slightly. + # Ideally, we should pad, but resizing is faster/easier for V1. + # Let's try resizing first. + + img_resized = cv2.resize(image, (self.output_img_size, self.output_img_size), interpolation=cv2.INTER_AREA) + mask_resized = cv2.resize(mask, (self.output_img_size, self.output_img_size), interpolation=cv2.INTER_NEAREST) + + # Normalize Image: [0, 255] -> [0, 1], HWC -> CHW + img_input = img_resized.astype(np.float32) / 255.0 + img_input = np.transpose(img_input, (2, 0, 1)) # (3, 512, 512) + img_input = np.expand_dims(img_input, axis=0) # (1, 3, 512, 512) + + # Normalize Mask: [0, 255] -> [0, 1], HW -> CHW + if len(mask_resized.shape) == 2: + mask_resized = np.expand_dims(mask_resized, axis=-1) # (512, 512, 1) + + mask_input = mask_resized.astype(np.float32) / 255.0 + mask_input = (mask_input > 0.5).astype(np.float32) # threshold + mask_input = np.transpose(mask_input, (2, 0, 1)) # (1, 512, 512) + mask_input = np.expand_dims(mask_input, axis=0) # (1, 1, 512, 512) + + # 2. Inference + inputs = { + self.session.get_inputs()[0].name: img_input, + self.session.get_inputs()[1].name: mask_input + } + outputs = self.session.run(None, inputs) + output_data = outputs[0] # (1, 3, 512, 512) + + # 3. Postprocess + # Clip to [0, 255], CHW -> HWC + output_img = output_data[0] + output_img = np.transpose(output_img, (1, 2, 0)) # (512, 512, 3) + # Model outputs [0, 255], so no need to multiply + output_img = np.clip(output_img, 0, 255).astype(np.uint8) + + # Resize back to original + result_img = cv2.resize(output_img, (original_w, original_h), interpolation=cv2.INTER_CUBIC) + + return result_img diff --git a/backend/app/routes/edit.py b/backend/app/routes/edit.py new file mode 100644 index 000000000..e05d68c45 --- /dev/null +++ b/backend/app/routes/edit.py @@ -0,0 +1,68 @@ +from fastapi import APIRouter, HTTPException, Body +from pydantic import BaseModel +import cv2 +import numpy as np +import base64 +import os +from app.models.Inpainter import Inpainter +from app.logging.setup_logging import get_logger + +logger = get_logger(__name__) +router = APIRouter() + +# Initialize Inpainter - GLOBAL instance to avoid reloading model +inpainter = Inpainter() + +class MagicEraserRequest(BaseModel): + image_path: str + mask_data: str # Base64 string + +class MagicEraserResponse(BaseModel): + success: bool + image_data: str # Base64 string + error: str | None = None + +def base64_to_cv2(b64str): + if "," in b64str: + b64str = b64str.split(",")[1] + img_data = base64.b64decode(b64str) + nparr = np.frombuffer(img_data, np.uint8) + img = cv2.imdecode(nparr, cv2.IMREAD_UNCHANGED) + return img + +def cv2_to_base64(img): + _, buffer = cv2.imencode('.png', img) + b64_str = base64.b64encode(buffer).decode('utf-8') + return f"data:image/png;base64,{b64_str}" + +@router.post("/magic-eraser", response_model=MagicEraserResponse) +def magic_eraser(body: MagicEraserRequest): + try: + # 1. Load Image + if not os.path.exists(body.image_path): + raise HTTPException(status_code=404, detail="Image file not found") + + image = cv2.imread(body.image_path) + if image is None: + raise HTTPException(status_code=400, detail="Failed to load image file") + + # 2. Load Mask + mask = base64_to_cv2(body.mask_data) + if mask is None: + raise HTTPException(status_code=400, detail="Failed to decode mask data") + + # Ensure mask is single channel + if len(mask.shape) == 3: + mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY) + + # 3. Inpaint + result = inpainter.inpaint(image, mask) + + # 4. Return result as Base64 for preview + b64_result = cv2_to_base64(result) + + return MagicEraserResponse(success=True, image_data=b64_result) + + except Exception as e: + logger.error(f"Magic Eraser failed: {e}") + return MagicEraserResponse(success=False, image_data="", error=str(e)) diff --git a/backend/main.py b/backend/main.py index 2c1f39e44..1b42a3812 100644 --- a/backend/main.py +++ b/backend/main.py @@ -26,6 +26,7 @@ from app.routes.images import router as images_router from app.routes.face_clusters import router as face_clusters_router from app.routes.user_preferences import router as user_preferences_router +from app.routes.edit import router as edit_router from fastapi.openapi.utils import get_openapi from app.logging.setup_logging import ( configure_uvicorn_logging, @@ -132,6 +133,7 @@ async def root(): app.include_router( user_preferences_router, prefix="/user-preferences", tags=["User Preferences"] ) +app.include_router(edit_router, prefix="/edit", tags=["Edit"]) # Entry point for running with: python3 main.py diff --git a/backend/requirements.txt b/backend/requirements.txt index b848d7ad6..218ec8ad1 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -31,7 +31,9 @@ mkdocs-material==9.6.16 mkdocs-material-extensions==1.3.1 mkdocs-swagger-ui-tag==0.7.1 mpmath==1.3.0 -numpy==1.26.4 +numpy<2.0.0 +tqdm==4.66.4 +requests==2.31.0 onnxruntime==1.17.1 opencv-python==4.9.0.80 orjson==3.10.3 diff --git a/backend/tests/test_inpainter.py b/backend/tests/test_inpainter.py new file mode 100644 index 000000000..ca58e3993 --- /dev/null +++ b/backend/tests/test_inpainter.py @@ -0,0 +1,51 @@ +import cv2 +import numpy as np +import sys +import os + +# Add backend to path +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)))) + +from app.models.Inpainter import Inpainter + +def test_inpainter(): + print("Initializing Inpainter...") + try: + inpainter = Inpainter() + if inpainter.session is None: + print("FAILED: Model session not initialized. Model file might be missing.") + return + + print("Creating dummy image and mask...") + # Create a 512x512 gradient image + img = np.zeros((512, 512, 3), dtype=np.uint8) + for i in range(512): + img[i, :, :] = i // 2 + + # Create a mask (white square in center) + mask = np.zeros((512, 512), dtype=np.uint8) + mask[200:300, 200:300] = 255 + + print("Running inpaint...") + result = inpainter.inpaint(img, mask) + + print("Inpaint finished.") + print(f"Result shape: {result.shape}") + + if result.shape != img.shape: + print(f"FAILED: Shape mismatch. Expected {img.shape}, got {result.shape}") + return + + # Check if the center is not black/unmodified (basic check) + center_pixel = result[250, 250] + print(f"Center pixel value: {center_pixel}") + + print("SUCCESS: Inpainter verification passed.") + + except Exception as e: + print(f"FAILED: Exception occurred: {e}") + import traceback + traceback.print_exc() + +if __name__ == "__main__": + test_inpainter() diff --git a/debug.txt b/debug.txt new file mode 100644 index 0000000000000000000000000000000000000000..8d44883deb82c81020dba05325eaa67a4e50300b GIT binary patch literal 790 zcmb_aO-sW-5PfIC|FA~|BaPz4qiK;IL}~CO9zv7a#WXEV>er7~-EQ(tMC8^c~-RQ-k`u5$0qyOL&94@pYlRs|7R~ZSaRA8=eg!d zQ>rPB-K$dFHhgW(YPaqP0auvgtru~a*;{#L&xw(fTT$D1@@3==Ge2O3F%dPP{&b2% z%S;&6?UrMX#|tebW1omovc`aN*Vg2A^u3{^+>L}X;`^juJJN5GHdlz!-g=l4U7Nck MyLh#kHT_*jpPw9g7XSbN literal 0 HcmV?d00001 diff --git a/docs/backend/backend_python/openapi.json b/docs/backend/backend_python/openapi.json index a29e7c4f1..4be2f970d 100644 --- a/docs/backend/backend_python/openapi.json +++ b/docs/backend/backend_python/openapi.json @@ -1304,6 +1304,47 @@ } } } + }, + "/edit/magic-eraser": { + "post": { + "tags": [ + "Edit" + ], + "summary": "Magic Eraser", + "operationId": "magic_eraser_edit_magic_eraser_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MagicEraserRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MagicEraserResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } } }, "components": { @@ -2266,6 +2307,53 @@ ], "title": "InputType" }, + "MagicEraserRequest": { + "properties": { + "image_path": { + "type": "string", + "title": "Image Path" + }, + "mask_data": { + "type": "string", + "title": "Mask Data" + } + }, + "type": "object", + "required": [ + "image_path", + "mask_data" + ], + "title": "MagicEraserRequest" + }, + "MagicEraserResponse": { + "properties": { + "success": { + "type": "boolean", + "title": "Success" + }, + "image_data": { + "type": "string", + "title": "Image Data" + }, + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "required": [ + "success", + "image_data" + ], + "title": "MagicEraserResponse" + }, "MetadataModel": { "properties": { "name": { diff --git a/frontend/src/components/Media/ImageViewer.tsx b/frontend/src/components/Media/ImageViewer.tsx index a8c1b4d08..9726338e5 100644 --- a/frontend/src/components/Media/ImageViewer.tsx +++ b/frontend/src/components/Media/ImageViewer.tsx @@ -4,7 +4,9 @@ import { convertFileSrc } from '@tauri-apps/api/core'; import { ocrService } from '../../services/OCRService'; import { TextOverlay } from './TextOverlay'; import { Page } from 'tesseract.js'; -import { Loader2, ScanText } from 'lucide-react'; +import { Loader2, ScanText, Wand2 } from 'lucide-react'; +import { MagicEraserOverlay } from './MagicEraserOverlay'; +import { writeFile } from '@tauri-apps/plugin-fs'; interface ImageViewerProps { imagePath: string; @@ -27,6 +29,7 @@ export const ImageViewer = forwardRef( const [ocrData, setOcrData] = useState(null); const [isOCRLoading, setIsOCRLoading] = useState(false); const [imageScale, setImageScale] = useState(1); + const [isMagicEraserActive, setIsMagicEraserActive] = useState(false); // Expose zoom functions to parent useImperativeHandle(ref, () => ({ @@ -40,6 +43,7 @@ export const ImageViewer = forwardRef( transformRef.current?.resetTransform(); // Reset OCR when image changes setIsOCRActive(false); + setIsMagicEraserActive(false); setOcrData(null); setIsOCRLoading(false); }, [resetSignal, imagePath]); @@ -126,28 +130,52 @@ export const ImageViewer = forwardRef( return (
- {/* Text Detection Toggle Button */} - + {/* Top Left Controls Container */} +
+ {/* Text Detection Toggle Button */} + + + {/* Magic Eraser Toggle Button */} + +
{isOCRLoading && (
@@ -221,6 +249,51 @@ export const ImageViewer = forwardRef(
+ + {isMagicEraserActive && imgRef.current && ( + setIsMagicEraserActive(false)} + originalWidth={imgRef.current.naturalWidth} + originalHeight={imgRef.current.naturalHeight} + onSave={async (base64Data) => { + try { + const base64Content = base64Data.split(',')[1]; + const binaryString = window.atob(base64Content); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + // Overwrite file + await writeFile(imagePath, bytes); + + // Force refresh by appending dummy query param to image src via some mechanism + // Since we use convertFileSrc(imagePath) directly in render img tag, + // we can't easily force it without state change. + // But saving to disk and closing overlay might be enough for next view. + // Or we can toggle the viewer closed/open? + // For V1, let's just close overlay. + setIsMagicEraserActive(false); + + // Force reload of image. + // Quick hack: toggle a key on the img element? NO, src needs to change. + // Ideally we notify parent or update a local version signal. + // We have resetSignal prop, but we can't write to it. + // Maybe dispatch a redux action? + // Simpler: reload window? No. + // Let's rely on user navigating away/back for now or use window.location.reload() if desperate. + // Better: call a prop onSaveComplete() if we had one. + + // Given constraints, I'll just close it. The user will see their change if they reopen the image or if the app detects file change. + + } catch (err) { + console.error("Failed to save image", err); + } + }} + /> + )}
); }, diff --git a/frontend/src/components/Media/MagicEraserOverlay.tsx b/frontend/src/components/Media/MagicEraserOverlay.tsx new file mode 100644 index 000000000..204ab3d8f --- /dev/null +++ b/frontend/src/components/Media/MagicEraserOverlay.tsx @@ -0,0 +1,325 @@ +import React, { useRef, useState, useEffect } from 'react'; +import { Eraser, Undo, Redo, X, Check, Loader2 } from 'lucide-react'; +import { convertFileSrc } from '@tauri-apps/api/core'; + +interface MagicEraserOverlayProps { + imagePath: string; + onClose: () => void; + onSave: (newImagePath: string) => void; + originalWidth: number; + originalHeight: number; +} + +export const MagicEraserOverlay: React.FC = ({ + imagePath, + onClose, + onSave, + originalWidth, + originalHeight, +}) => { + const canvasRef = useRef(null); + const containerRef = useRef(null); + const [isDrawing, setIsDrawing] = useState(false); + const [brushSize, setBrushSize] = useState(20); + const [isProcessing, setIsProcessing] = useState(false); + const [previewImage, setPreviewImage] = useState(null); + + // History for undo/redo (store canvas data URLs or ImageData) + // For simplicity, we just clear for now, but undo is requested in plan. + // We'll implement basic path history. + const [paths, setPaths] = useState<{ x: number; y: number; size: number }[][]>([]); + const [poppedPaths, setPoppedPaths] = useState<{ x: number; y: number; size: number }[][]>([]); + const [currentPath, setCurrentPath] = useState<{ x: number; y: number; size: number }[]>([]); + + // Setup canvas size + useEffect(() => { + const canvas = canvasRef.current; + const container = containerRef.current; + if (canvas && container) { + canvas.width = container.clientWidth; + canvas.height = container.clientHeight; + + const ctx = canvas.getContext('2d'); + if (ctx) { + ctx.lineCap = 'round'; + ctx.lineJoin = 'round'; + ctx.strokeStyle = 'rgba(255, 0, 0, 0.5)'; // Visual red mask + } + } + }, []); + + // Redraw when paths change (Undo/Redo logic would go here) + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + + ctx.clearRect(0, 0, canvas.width, canvas.height); + + // Draw all committed paths + ctx.strokeStyle = 'rgba(255, 0, 0, 0.5)'; + paths.forEach(path => { + if (path.length < 1) return; + ctx.beginPath(); + ctx.lineWidth = path[0].size; + ctx.moveTo(path[0].x, path[0].y); + for (let i = 1; i < path.length; i++) { + ctx.lineTo(path[i].x, path[i].y); + } + ctx.stroke(); + }); + + // Draw current path + if (currentPath.length > 0) { + ctx.beginPath(); + ctx.lineWidth = currentPath[0].size; + ctx.moveTo(currentPath[0].x, currentPath[0].y); + for (let i = 1; i < currentPath.length; i++) { + ctx.lineTo(currentPath[i].x, currentPath[i].y); + } + ctx.stroke(); + } + }, [paths, currentPath]); + + const getPointerPos = (e: React.MouseEvent | React.TouchEvent) => { + const canvas = canvasRef.current; + if (!canvas) return { x: 0, y: 0 }; + const rect = canvas.getBoundingClientRect(); + let clientX, clientY; + + if ('touches' in e) { + clientX = e.touches[0].clientX; + clientY = e.touches[0].clientY; + } else { + clientX = (e as React.MouseEvent).clientX; + clientY = (e as React.MouseEvent).clientY; + } + + return { + x: clientX - rect.left, + y: clientY - rect.top + }; + }; + + const startDrawing = (e: React.MouseEvent | React.TouchEvent) => { + setIsDrawing(true); + const pos = getPointerPos(e); + setCurrentPath([{ x: pos.x, y: pos.y, size: brushSize }]); + }; + + const draw = (e: React.MouseEvent | React.TouchEvent) => { + if (!isDrawing) return; + const pos = getPointerPos(e); + setCurrentPath(prev => [...prev, { x: pos.x, y: pos.y, size: brushSize }]); + }; + + const stopDrawing = () => { + if (!isDrawing) return; + setIsDrawing(false); + if (currentPath.length > 0) { + setPaths(prev => [...prev, currentPath]); + setCurrentPath([]); + setPoppedPaths([]); // Clear redo history + } + }; + + const handleErase = async () => { + if (paths.length === 0) return; + + setIsProcessing(true); + try { + // 1. Generate Mask Data URL + // We need a separate canvas for the actual mask (white on black) + const maskCanvas = document.createElement('canvas'); + maskCanvas.width = originalWidth; + maskCanvas.height = originalHeight; + const ctx = maskCanvas.getContext('2d'); + if (!ctx || !canvasRef.current) return; + + ctx.fillStyle = 'black'; + ctx.fillRect(0, 0, maskCanvas.width, maskCanvas.height); + + // Scale factor between display canvas and original image + const scaleX = originalWidth / canvasRef.current.width; + const scaleY = originalHeight / canvasRef.current.height; + + ctx.strokeStyle = 'white'; + ctx.lineCap = 'round'; + ctx.lineJoin = 'round'; + + paths.forEach(path => { + if (path.length < 1) return; + ctx.beginPath(); + ctx.lineWidth = path[0].size * ((scaleX + scaleY) / 2); // Approximation + ctx.moveTo(path[0].x * scaleX, path[0].y * scaleY); + for (let i = 1; i < path.length; i++) { + ctx.lineTo(path[i].x * scaleX, path[i].y * scaleY); + } + ctx.stroke(); + }); + + const maskData = maskCanvas.toDataURL('image/png'); + + // 2. Call API + const response = await fetch('http://localhost:8000/edit/magic-eraser', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + image_path: imagePath, + mask_data: maskData, + }), + }); + + const data = await response.json(); + if (data.success) { + setPreviewImage(data.image_data); + } else { + console.error('Magic Eraser failed:', data.error); + } + + } catch (error) { + console.error('Error:', error); + } finally { + setIsProcessing(false); + } + }; + + const handleUndo = () => { + if (paths.length === 0) return; + const lastPath = paths[paths.length - 1]; + setPaths(prev => prev.slice(0, -1)); + setPoppedPaths(prev => [...prev, lastPath]); + }; + + const handleRedo = () => { + if (poppedPaths.length === 0) return; + const pathRestored = poppedPaths[poppedPaths.length - 1]; + setPoppedPaths(prev => prev.slice(0, -1)); + setPaths(prev => [...prev, pathRestored]); + }; + + const handleScaleBrush = (e: React.ChangeEvent) => { + setBrushSize(parseInt(e.target.value)); + }; + + return ( +
+ {/* Top Bar */} +
+

+ + Magic Eraser +

+ +
+ + {/* Main Area */} +
+
+ {/* Base Image */} + Editing + + {/* Drawing Canvas */} + {!previewImage && ( + + )} + + {/* Loading Overlay */} + {isProcessing && ( +
+ + Removing Object... +
+ )} +
+
+ + {/* Bottom Controls */} +
+ {!previewImage ? ( + <> +
+ Brush Size + +
{brushSize}px
+
+ +
+ + + + + + + + ) : ( + <> + + + + )} +
+
+ ); +}; diff --git a/scripts/download_models.py b/scripts/download_models.py new file mode 100644 index 000000000..7ca1b3de6 --- /dev/null +++ b/scripts/download_models.py @@ -0,0 +1,56 @@ +import os +import requests +from tqdm import tqdm + +# Constants +MODEL_URL = "https://huggingface.co/Carve/LaMa-ONNX/resolve/main/lama_fp32.onnx" +MODEL_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "backend", "app", "models", "onnx_models") +MODEL_PATH = os.path.join(MODEL_DIR, "lama_fp32.onnx") + +def download_file(url, filename): + """ + Download a file from a URL to a local filename with a progress bar. + """ + response = requests.get(url, stream=True) + total_size_in_bytes = int(response.headers.get('content-length', 0)) + block_size = 1024 # 1 Kibibyte + progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True) + + with open(filename, 'wb') as file: + for data in response.iter_content(block_size): + progress_bar.update(len(data)) + file.write(data) + progress_bar.close() + + if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes: + print("ERROR, something went wrong") + return False + return True + +def main(): + if not os.path.exists(MODEL_DIR): + print(f"Creating directory: {MODEL_DIR}") + os.makedirs(MODEL_DIR, exist_ok=True) + + if os.path.exists(MODEL_PATH): + print(f"Model already exists at: {MODEL_PATH}") + # Optional: check hash or size to verify integrity? + # For now, assume if it exists, it's good. + return + + print(f"Downloading LaMa ONNX model from {MODEL_URL}...") + try: + success = download_file(MODEL_URL, MODEL_PATH) + if success: + print("Download completed successfully!") + else: + print("Download failed.") + if os.path.exists(MODEL_PATH): + os.remove(MODEL_PATH) + except Exception as e: + print(f"An error occurred: {e}") + if os.path.exists(MODEL_PATH): + os.remove(MODEL_PATH) + +if __name__ == "__main__": + main() diff --git a/scripts/setup.ps1 b/scripts/setup.ps1 index a4d940e3a..8a909f3f0 100644 --- a/scripts/setup.ps1 +++ b/scripts/setup.ps1 @@ -140,6 +140,11 @@ try { .\.env\Scripts\Activate.ps1 python -m pip install --upgrade pip python -m pip install -r requirements.txt + + # Download Magic Eraser models + Write-Host "Downloading required models..." -ForegroundColor Yellow + python ..\scripts\download_models.py + deactivate Set-Location .. diff --git a/scripts/setup.sh b/scripts/setup.sh index 656fe84ab..1e127bb24 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -135,6 +135,10 @@ python -m venv .env source .env/bin/activate pip install --upgrade pip pip install -r requirements.txt + +# Download required models +echo -e "${YELLOW}Downloading required models...${NC}" +python ../scripts/download_models.py deactivate cd .. From 244dbc46486555befa219e3fb07dd44c78c776c9 Mon Sep 17 00:00:00 2001 From: Aryan-Shan Date: Mon, 8 Dec 2025 20:16:16 +0530 Subject: [PATCH 07/10] Fixed image resolution for magic eraser --- backend/app/models/Inpainter.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/backend/app/models/Inpainter.py b/backend/app/models/Inpainter.py index 7c060d28a..94940a1f2 100644 --- a/backend/app/models/Inpainter.py +++ b/backend/app/models/Inpainter.py @@ -94,4 +94,17 @@ def inpaint(self, image: np.ndarray, mask: np.ndarray) -> np.ndarray: # Resize back to original result_img = cv2.resize(output_img, (original_w, original_h), interpolation=cv2.INTER_CUBIC) - return result_img + # 4. Blend to preserve original quality + # Create a binary mask of the inpainted region + if len(mask.shape) == 2: + mask = mask[:, :, np.newaxis] + + # Normalize mask to 0-1 + mask_normalized = mask.astype(np.float32) / 255.0 + mask_normalized = (mask_normalized > 0.5).astype(np.float32) + + # Blend: original * (1 - mask) + result * mask + final_img = image.astype(np.float32) * (1 - mask_normalized) + result_img.astype(np.float32) * mask_normalized + final_img = np.clip(final_img, 0, 255).astype(np.uint8) + + return final_img From a91b203347a95a021fac29db074aa19a676921b4 Mon Sep 17 00:00:00 2001 From: Aryan-Shan Date: Mon, 8 Dec 2025 21:04:42 +0530 Subject: [PATCH 08/10] magic eraser feature refactored --- backend/app/models/Inpainter.py | 2 +- backend/app/routes/edit.py | 22 +++++++++++++------ backend/tests/test_inpainter.py | 1 + docs/backend/backend_python/openapi.json | 12 +++++++--- frontend/package.json | 4 ++-- frontend/src/components/Media/ImageViewer.tsx | 2 +- .../components/Media/MagicEraserOverlay.tsx | 17 +++++++++++++- scripts/download_models.py | 3 +++ scripts/setup.sh | 4 ++++ 9 files changed, 52 insertions(+), 15 deletions(-) diff --git a/backend/app/models/Inpainter.py b/backend/app/models/Inpainter.py index 94940a1f2..a82c9962d 100644 --- a/backend/app/models/Inpainter.py +++ b/backend/app/models/Inpainter.py @@ -22,7 +22,7 @@ def _init_session(self): ) if not os.path.exists(model_path): - logger.error(f"Inpainting model found at {model_path}") + logger.error(f"Inpainting model not found at {model_path}") self.session = None return diff --git a/backend/app/routes/edit.py b/backend/app/routes/edit.py index e05d68c45..609d0a410 100644 --- a/backend/app/routes/edit.py +++ b/backend/app/routes/edit.py @@ -19,7 +19,7 @@ class MagicEraserRequest(BaseModel): class MagicEraserResponse(BaseModel): success: bool - image_data: str # Base64 string + image_data: str | None = None # Base64 string error: str | None = None def base64_to_cv2(b64str): @@ -38,18 +38,26 @@ def cv2_to_base64(img): @router.post("/magic-eraser", response_model=MagicEraserResponse) def magic_eraser(body: MagicEraserRequest): try: - # 1. Load Image + # Security Check: Validate path is within expected directory/exists and is a file + # For this desktop app, we can just ensure it exists and is an absolute path or relative to CWD + # A simple check to prevent ../../ traversal if running in a sensitive context + # But primarily we just handle the error gracefully. + + # Real validation: + if not os.path.isabs(body.image_path) and ".." in body.image_path: + return MagicEraserResponse(success=False, error="Invalid image path") + if not os.path.exists(body.image_path): - raise HTTPException(status_code=404, detail="Image file not found") + return MagicEraserResponse(success=False, error="Image file not found") image = cv2.imread(body.image_path) if image is None: - raise HTTPException(status_code=400, detail="Failed to load image file") + return MagicEraserResponse(success=False, error="Failed to load image file") # 2. Load Mask mask = base64_to_cv2(body.mask_data) if mask is None: - raise HTTPException(status_code=400, detail="Failed to decode mask data") + return MagicEraserResponse(success=False, error="Failed to decode mask data") # Ensure mask is single channel if len(mask.shape) == 3: @@ -64,5 +72,5 @@ def magic_eraser(body: MagicEraserRequest): return MagicEraserResponse(success=True, image_data=b64_result) except Exception as e: - logger.error(f"Magic Eraser failed: {e}") - return MagicEraserResponse(success=False, image_data="", error=str(e)) + logger.exception("Magic Eraser failed") + return MagicEraserResponse(success=False, error="Internal processing error") diff --git a/backend/tests/test_inpainter.py b/backend/tests/test_inpainter.py index ca58e3993..97bbb24d5 100644 --- a/backend/tests/test_inpainter.py +++ b/backend/tests/test_inpainter.py @@ -46,6 +46,7 @@ def test_inpainter(): print(f"FAILED: Exception occurred: {e}") import traceback traceback.print_exc() + sys.exit(1) if __name__ == "__main__": test_inpainter() diff --git a/docs/backend/backend_python/openapi.json b/docs/backend/backend_python/openapi.json index 4be2f970d..6fdc599fe 100644 --- a/docs/backend/backend_python/openapi.json +++ b/docs/backend/backend_python/openapi.json @@ -2332,7 +2332,14 @@ "title": "Success" }, "image_data": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Image Data" }, "error": { @@ -2349,8 +2356,7 @@ }, "type": "object", "required": [ - "success", - "image_data" + "success" ], "title": "MagicEraserResponse" }, diff --git a/frontend/package.json b/frontend/package.json index 89d1f524a..cdd8c90d5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -66,7 +66,7 @@ "react-zoom-pan-pinch": "^3.7.0", "tailwind-merge": "^3.3.0", "tailwindcss": "^4.1.8", - "tesseract.js": "^5.1.0", + "tesseract.js": "^6.0.0", "ts-node": "^10.9.2", "uuid": "^11.1.0", "vite-plugin-environment": "^1.1.3" @@ -105,4 +105,4 @@ "vite": "^6.3.5", "vite-plugin-eslint": "^1.8.1" } -} +} \ No newline at end of file diff --git a/frontend/src/components/Media/ImageViewer.tsx b/frontend/src/components/Media/ImageViewer.tsx index 9726338e5..1106983c2 100644 --- a/frontend/src/components/Media/ImageViewer.tsx +++ b/frontend/src/components/Media/ImageViewer.tsx @@ -250,7 +250,7 @@ export const ImageViewer = forwardRef( - {isMagicEraserActive && imgRef.current && ( + {isMagicEraserActive && imgRef.current && imgRef.current.naturalWidth > 0 && imgRef.current.naturalHeight > 0 && ( setIsMagicEraserActive(false)} diff --git a/frontend/src/components/Media/MagicEraserOverlay.tsx b/frontend/src/components/Media/MagicEraserOverlay.tsx index 204ab3d8f..651ea43cf 100644 --- a/frontend/src/components/Media/MagicEraserOverlay.tsx +++ b/frontend/src/components/Media/MagicEraserOverlay.tsx @@ -124,10 +124,13 @@ export const MagicEraserOverlay: React.FC = ({ } }; + const [error, setError] = useState(null); + const handleErase = async () => { if (paths.length === 0) return; setIsProcessing(true); + setError(null); try { // 1. Generate Mask Data URL // We need a separate canvas for the actual mask (white on black) @@ -162,7 +165,8 @@ export const MagicEraserOverlay: React.FC = ({ const maskData = maskCanvas.toDataURL('image/png'); // 2. Call API - const response = await fetch('http://localhost:8000/edit/magic-eraser', { + const apiUrl = import.meta.env.VITE_API_URL || 'http://localhost:8000'; + const response = await fetch(`${apiUrl}/edit/magic-eraser`, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -178,10 +182,12 @@ export const MagicEraserOverlay: React.FC = ({ setPreviewImage(data.image_data); } else { console.error('Magic Eraser failed:', data.error); + setError(data.error || 'Failed to process image'); } } catch (error) { console.error('Error:', error); + setError('Network error. Please try again.'); } finally { setIsProcessing(false); } @@ -250,6 +256,15 @@ export const MagicEraserOverlay: React.FC = ({ Removing Object...
)} + + {/* Error Overlay */} + {error && ( +
+
+

{error}

+
+
+ )}
diff --git a/scripts/download_models.py b/scripts/download_models.py index 7ca1b3de6..93b7d1a6a 100644 --- a/scripts/download_models.py +++ b/scripts/download_models.py @@ -1,6 +1,7 @@ import os import requests from tqdm import tqdm +import sys # Constants MODEL_URL = "https://huggingface.co/Carve/LaMa-ONNX/resolve/main/lama_fp32.onnx" @@ -47,10 +48,12 @@ def main(): print("Download failed.") if os.path.exists(MODEL_PATH): os.remove(MODEL_PATH) + sys.exit(1) except Exception as e: print(f"An error occurred: {e}") if os.path.exists(MODEL_PATH): os.remove(MODEL_PATH) + sys.exit(1) if __name__ == "__main__": main() diff --git a/scripts/setup.sh b/scripts/setup.sh index 1e127bb24..7a258ce5c 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -139,6 +139,10 @@ pip install -r requirements.txt # Download required models echo -e "${YELLOW}Downloading required models...${NC}" python ../scripts/download_models.py +if [ $? -ne 0 ]; then + echo -e "${RED}Model download failed. Setup aborted.${NC}" + exit 1 +fi deactivate cd .. From a6b76e772d0b47cc78ab0c94a7c45a1b39c5c71c Mon Sep 17 00:00:00 2001 From: Aryan-Shan Date: Mon, 8 Dec 2025 21:18:48 +0530 Subject: [PATCH 09/10] magic eraser feature refactored --- backend/app/models/Inpainter.py | 7 ++++++- backend/app/routes/edit.py | 16 ++++++++-------- backend/tests/test_inpainter.py | 5 +++++ frontend/src/components/Media/ImageViewer.tsx | 4 +++- .../src/components/Media/MagicEraserOverlay.tsx | 2 +- scripts/download_models.py | 1 + 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/backend/app/models/Inpainter.py b/backend/app/models/Inpainter.py index a82c9962d..a2a09815a 100644 --- a/backend/app/models/Inpainter.py +++ b/backend/app/models/Inpainter.py @@ -88,7 +88,12 @@ def inpaint(self, image: np.ndarray, mask: np.ndarray) -> np.ndarray: # Clip to [0, 255], CHW -> HWC output_img = output_data[0] output_img = np.transpose(output_img, (1, 2, 0)) # (512, 512, 3) - # Model outputs [0, 255], so no need to multiply + + # Auto-detect output range: LaMa can be [0, 1] or [0, 255] + # If max value is small (<= 1.0 + epsilon), assume it's [0, 1] and scale up. + if output_img.max() <= 1.1: + output_img = output_img * 255.0 + output_img = np.clip(output_img, 0, 255).astype(np.uint8) # Resize back to original diff --git a/backend/app/routes/edit.py b/backend/app/routes/edit.py index 609d0a410..3db7bed9e 100644 --- a/backend/app/routes/edit.py +++ b/backend/app/routes/edit.py @@ -38,16 +38,16 @@ def cv2_to_base64(img): @router.post("/magic-eraser", response_model=MagicEraserResponse) def magic_eraser(body: MagicEraserRequest): try: - # Security Check: Validate path is within expected directory/exists and is a file - # For this desktop app, we can just ensure it exists and is an absolute path or relative to CWD - # A simple check to prevent ../../ traversal if running in a sensitive context - # But primarily we just handle the error gracefully. + # Custom Validation: Prevent Path Traversal + # Ensure path is absolute and doesn't contain traversal sequences + abs_path = os.path.abspath(body.image_path) + base_dir = os.path.abspath(os.getcwd()) # Or a specific allowed media directory - # Real validation: - if not os.path.isabs(body.image_path) and ".." in body.image_path: - return MagicEraserResponse(success=False, error="Invalid image path") + # Simple check for ".." usage which suggests traversal attempts + if ".." in body.image_path: + return MagicEraserResponse(success=False, error="Invalid image path: Path traversal detected") - if not os.path.exists(body.image_path): + if not os.path.exists(abs_path): return MagicEraserResponse(success=False, error="Image file not found") image = cv2.imread(body.image_path) diff --git a/backend/tests/test_inpainter.py b/backend/tests/test_inpainter.py index 97bbb24d5..c17c167fb 100644 --- a/backend/tests/test_inpainter.py +++ b/backend/tests/test_inpainter.py @@ -37,9 +37,14 @@ def test_inpainter(): return # Check if the center is not black/unmodified (basic check) + # Check if the center is not black (0) which would indicate incorrect scaling [0,1]->uint8 center_pixel = result[250, 250] print(f"Center pixel value: {center_pixel}") + if np.all(center_pixel == 0): + print(f"FAILED: Center pixel is black (0). Model output likely [0, 1] but treated as [0, 255].") + sys.exit(1) + print("SUCCESS: Inpainter verification passed.") except Exception as e: diff --git a/frontend/src/components/Media/ImageViewer.tsx b/frontend/src/components/Media/ImageViewer.tsx index 1106983c2..5062a71ad 100644 --- a/frontend/src/components/Media/ImageViewer.tsx +++ b/frontend/src/components/Media/ImageViewer.tsx @@ -258,7 +258,9 @@ export const ImageViewer = forwardRef( originalHeight={imgRef.current.naturalHeight} onSave={async (base64Data) => { try { - const base64Content = base64Data.split(',')[1]; + const base64Content = base64Data.includes(',') + ? base64Data.split(',')[1] + : base64Data; const binaryString = window.atob(base64Content); const len = binaryString.length; const bytes = new Uint8Array(len); diff --git a/frontend/src/components/Media/MagicEraserOverlay.tsx b/frontend/src/components/Media/MagicEraserOverlay.tsx index 651ea43cf..a39a38222 100644 --- a/frontend/src/components/Media/MagicEraserOverlay.tsx +++ b/frontend/src/components/Media/MagicEraserOverlay.tsx @@ -307,7 +307,7 @@ export const MagicEraserOverlay: React.FC = ({