diff --git a/client/bundle.css b/client/bundle.css new file mode 100644 index 000000000..b2f0abd65 --- /dev/null +++ b/client/bundle.css @@ -0,0 +1,477 @@ +.event-chart { + position: relative; + height: calc(100% - 10px); + margin: 5px 0; + overflow-y: auto; + overflow-x: hidden; +} +.event-chart .tooltip { + position: absolute; + background: black; + border: 1px solid white; + padding: 0px 5px; + font-size: 14px; + z-index: 2; +} + +.controls { + position: absolute; +} + +.video-annotator { + position: relative; + left: 0; + right: 0; + top: 0; + bottom: 0; + z-index: 0; + display: flex; + flex-direction: column; +} +.video-annotator .geojs-map { + margin: 2px; +} +.video-annotator .geojs-map.geojs-map:focus { + outline: none; +} +.video-annotator .playback-container { + flex: 1; +} +.video-annotator .loadingSpinnerContainer { + z-index: 20; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); +} +.video-annotator .geojs-map.annotation-input { + cursor: inherit; +} + +.selected-camera { + box-sizing: content-box; +} +.selected-camera .geojs-map { + outline: 3px cyan dashed; +} +.selected-camera .geojs-map.geojs-map:focus { + outline: 3px cyan dashed; +} + +.imageCursor { + z-index: 10; + position: fixed; + backface-visibility: hidden; + top: 0; + left: 0; + pointer-events: none; +} + +.controls { + bottom: 0; +} + +.controls { + position: absolute; +} + +.video-annotator { + position: relative; + left: 0; + right: 0; + top: 0; + bottom: 0; + z-index: 0; + display: flex; + flex-direction: column; +} +.video-annotator .geojs-map { + margin: 2px; +} +.video-annotator .geojs-map.geojs-map:focus { + outline: none; +} +.video-annotator .playback-container { + flex: 1; +} +.video-annotator .loadingSpinnerContainer { + z-index: 20; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); +} +.video-annotator .geojs-map.annotation-input { + cursor: inherit; +} + +.selected-camera { + box-sizing: content-box; +} +.selected-camera .geojs-map { + outline: 3px cyan dashed; +} +.selected-camera .geojs-map.geojs-map:focus { + outline: 3px cyan dashed; +} + +.imageCursor { + z-index: 10; + position: fixed; + backface-visibility: hidden; + top: 0; + left: 0; + pointer-events: none; +} + +.controls { + bottom: 0; +} + +.border-radius[data-v-77dee125] { + border: 1px solid #888888; + padding: 2px 5px; + border-radius: 5px; +} + +.line-chart { + height: 100%; +} +.line-chart .line { + fill: none; + stroke-width: 1.5px; +} +.line-chart .axis-y { + font-size: 12px; +} +.line-chart .axis-y g:first-of-type, +.line-chart .axis-y g:last-of-type { + display: none; +} +.line-chart .tooltip { + position: absolute; + background: black; + border: 1px solid white; + padding: 0px 5px; + font-size: 14px; +} + +.timeline .tick { + shape-rendering: crispEdges; + font-size: 12px; + stroke-opacity: 0.5; + stroke-dasharray: 2, 2; +} + +.timeline[data-v-0d0fe2ba] { + min-height: 175px; + position: relative; + display: flex; + flex-direction: column; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] { + flex: 1; + position: relative; + overflow: hidden; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .hand[data-v-0d0fe2ba] { + position: absolute; + top: 0; + width: 0; + height: 100%; + border-left: 1px solid #299be3; + z-index: 10; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-line[data-v-0d0fe2ba] { + position: absolute; + top: 0; + width: 0; + height: 100%; + z-index: 2; + cursor: col-resize; + pointer-events: auto; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-tooltip[data-v-0d0fe2ba] { + position: absolute; + top: 30px; + transform: translateX(-50%); + background-color: rgba(0, 0, 0, 0.8); + color: white; + padding: 4px 8px; + border-radius: 4px; + font-size: 12px; + white-space: nowrap; + pointer-events: none; + z-index: 20; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-start-line[data-v-0d0fe2ba] { + border-left: 3px solid #4caf50; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-end-line[data-v-0d0fe2ba] { + border-left: 3px solid #f44336; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .time-filter-dimming[data-v-0d0fe2ba] { + position: absolute; + top: 0; + height: 100%; + background-color: rgba(0, 0, 0, 0.3); + pointer-events: none; + z-index: 1; +} +.timeline[data-v-0d0fe2ba] .work-area[data-v-0d0fe2ba] .child[data-v-0d0fe2ba] { + position: absolute; + top: 0; + bottom: 17px; + left: 0; + right: 0; + z-index: 0; +} +.timeline[data-v-0d0fe2ba] .minimap[data-v-0d0fe2ba] { + height: 10px; +} +.timeline[data-v-0d0fe2ba] .minimap[data-v-0d0fe2ba] .fill[data-v-0d0fe2ba] { + position: relative; + height: 100%; + background-color: #80c6e8; +} + +.controls { + position: absolute; +} + +.video-annotator { + position: relative; + left: 0; + right: 0; + top: 0; + bottom: 0; + z-index: 0; + display: flex; + flex-direction: column; +} +.video-annotator .geojs-map { + margin: 2px; +} +.video-annotator .geojs-map.geojs-map:focus { + outline: none; +} +.video-annotator .playback-container { + flex: 1; +} +.video-annotator .loadingSpinnerContainer { + z-index: 20; + margin: 0; + position: absolute; + top: 50%; + left: 50%; + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); +} +.video-annotator .geojs-map.annotation-input { + cursor: inherit; +} + +.selected-camera { + box-sizing: content-box; +} +.selected-camera .geojs-map { + outline: 3px cyan dashed; +} +.selected-camera .geojs-map.geojs-map:focus { + outline: 3px cyan dashed; +} + +.imageCursor { + z-index: 10; + position: fixed; + backface-visibility: hidden; + top: 0; + left: 0; + pointer-events: none; +} + +.controls { + bottom: 0; +} + +.input-box { + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + padding: 0 6px; + color: white; +} + +.trackNumber { + font-family: monospace; + max-width: 80px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} +.trackNumber:hover { + cursor: pointer; + font-weight: bolder; + text-decoration: underline; +} + +.border-highlight[data-v-0d46f934] { + border-bottom: 1px solid gray; +} + +.type-checkbox[data-v-0d46f934] { + max-width: 80%; + overflow-wrap: anywhere; +} + +.hover-show-parent[data-v-0d46f934] .hover-show-child[data-v-0d46f934] { + display: none; +} +.hover-show-parent[data-v-0d46f934][data-v-0d46f934]:hover .hover-show-child[data-v-0d46f934] { + display: inherit; +} + +.outlined[data-v-0d46f934] { + background-color: gray; + color: #222; + font-weight: 600; + border-radius: 6px; + padding: 0 5px; + font-size: 12px; +} + +.input-box { + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + padding: 0 6px; + color: white; +} + +.trackNumber { + font-family: monospace; + max-width: 80px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} +.trackNumber:hover { + cursor: pointer; + font-weight: bolder; + text-decoration: underline; +} + +.freeform-input[data-v-d679c59c] { + width: 150px; +} + +.groups[data-v-c26ed586] { + overflow-y: auto; + overflow-x: hidden; +} + +.input-box { + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + padding: 0 6px; + color: white; +} + +.trackNumber { + font-family: monospace; + max-width: 80px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} +.trackNumber:hover { + cursor: pointer; + font-weight: bolder; + text-decoration: underline; +} + +.track-item[data-v-7a688bfe] { + border-radius: inherit; +} +.track-item[data-v-7a688bfe] .item-row[data-v-7a688bfe] { + width: 100%; +} +.track-item[data-v-7a688bfe] .type-color-box[data-v-7a688bfe] { + margin: 7px; + margin-top: 4px; + min-width: 15px; + max-width: 15px; + min-height: 15px; + max-height: 15px; +} + +.strcoller { + height: 100%; +} + +.trackHeader { + height: auto; +} + +.tracks { + overflow-y: auto; + overflow-x: hidden; +} +.tracks .v-input--checkbox label { + white-space: pre-wrap; +} + +.nowrap[data-v-a4da19c6] { + white-space: nowrap; + overflow: hidden; + max-width: var(--content-width); + text-overflow: ellipsis; +} + +.hover-show-parent[data-v-a4da19c6] .hover-show-child[data-v-a4da19c6] { + display: none; +} +.hover-show-parent[data-v-a4da19c6][data-v-a4da19c6]:hover .hover-show-child[data-v-a4da19c6] { + display: inherit; +} + +.outlined[data-v-a4da19c6] { + background-color: gray; + color: #222; + font-weight: 600; + border-radius: 6px; + padding: 0 5px; + font-size: 12px; +} + +.input-box { + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 4px; + padding: 0 6px; + color: white; +} + +.trackNumber { + font-family: monospace; + max-width: 80px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} +.trackNumber:hover { + cursor: pointer; + font-weight: bolder; + text-decoration: underline; +} + +.freeform-input[data-v-07a75698] { + width: 135px; +} + +.select-input[data-v-07a75698] { + width: 120px; + background-color: #1e1e1e; + appearance: menulist; +} \ No newline at end of file diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 6aa73861d..2ecdc8279 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -207,6 +207,122 @@ function useApi() { return use>(ApiSymbol); } +/** + * Interactive Segmentation Types + */ +export interface SegmentationPredictRequest { + /** Path to the image file */ + imagePath: string; + /** Point coordinates as [x, y] pairs */ + points: [number, number][]; + /** Point labels: 1 for foreground, 0 for background */ + pointLabels: number[]; + /** Optional low-res mask from previous prediction for refinement */ + maskInput?: number[][]; + /** Whether to return multiple mask options */ + multimaskOutput?: boolean; +} + +export interface SegmentationPredictResponse { + /** Whether the prediction succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** Polygon coordinates as [x, y] pairs */ + polygon?: [number, number][]; + /** Bounding box [x_min, y_min, x_max, y_max] */ + bounds?: [number, number, number, number]; + /** Quality score from segmentation model */ + score?: number; + /** Low-res mask for subsequent refinement */ + lowResMask?: number[][]; + /** Mask dimensions [height, width] */ + maskShape?: [number, number]; + /** RLE-encoded full-resolution mask for display: [[value, count], ...] */ + rleMask?: [number, number][]; +} + +export interface SegmentationStatusResponse { + /** Whether segmentation is available */ + available: boolean; + /** Whether the model is currently loaded */ + loaded?: boolean; + /** Whether the service is ready for predictions */ + ready?: boolean; +} + +/** + * Text Query Types for open-vocabulary detection/segmentation + */ + +/** A single detection returned from a text query */ +export interface TextQueryDetection { + /** Bounding box [x1, y1, x2, y2] */ + box: [number, number, number, number]; + /** Polygon coordinates as [x, y] pairs */ + polygon?: [number, number][]; + /** Confidence score */ + score: number; + /** Label/class name (often the query text) */ + label: string; + /** Low-res mask for refinement (optional) */ + lowResMask?: number[][]; +} + +export interface TextQueryRequest { + /** Path to the image file */ + imagePath: string; + /** Text query describing what to find (e.g., "fish", "person swimming") */ + text: string; + /** Confidence threshold for detections (default: 0.3) */ + boxThreshold?: number; + /** Maximum number of detections to return (default: 10) */ + maxDetections?: number; + /** Optional boxes to refine [x1, y1, x2, y2][] */ + boxes?: [number, number, number, number][]; + /** Optional keypoints for refinement [x, y][] */ + points?: [number, number][]; + /** Labels for points: 1 for foreground, 0 for background */ + pointLabels?: number[]; + /** Optional masks to refine */ + masks?: number[][][]; +} + +export interface TextQueryResponse { + /** Whether the query succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** List of detections found */ + detections?: TextQueryDetection[]; + /** The original query text */ + query?: string; + /** Whether fallback method was used (no native text support) */ + fallback?: boolean; +} + +export interface RefineDetectionsRequest { + /** Path to the image file */ + imagePath: string; + /** Detections to refine */ + detections: TextQueryDetection[]; + /** Optional additional keypoints for refinement [x, y][] */ + points?: [number, number][]; + /** Labels for additional points: 1 for foreground, 0 for background */ + pointLabels?: number[]; + /** Whether to include refined masks in response */ + refineMasks?: boolean; +} + +export interface RefineDetectionsResponse { + /** Whether the refinement succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; + /** Refined detections */ + detections?: TextQueryDetection[]; +} + export { provideApi, useApi, diff --git a/client/dive-common/components/DeleteControls.vue b/client/dive-common/components/DeleteControls.vue index 50ecdd08a..f6022ac8a 100644 --- a/client/dive-common/components/DeleteControls.vue +++ b/client/dive-common/components/DeleteControls.vue @@ -24,6 +24,9 @@ export default Vue.extend({ if (this.editingMode === 'rectangle') { return true; // deleting rectangle is unsupported } + if (this.editingMode === 'Point') { + return true; // Point mode uses reset instead of delete + } return false; }, }, diff --git a/client/dive-common/components/EditorMenu.vue b/client/dive-common/components/EditorMenu.vue index 13b74443a..1f7642332 100644 --- a/client/dive-common/components/EditorMenu.vue +++ b/client/dive-common/components/EditorMenu.vue @@ -11,6 +11,7 @@ import { flatten } from 'lodash'; import { Mousetrap } from 'vue-media-annotator/types'; import { EditAnnotationTypes, VisibleAnnotationTypes } from 'vue-media-annotator/layers'; import Recipe from 'vue-media-annotator/recipe'; +import SegmentationPointClick from 'dive-common/recipes/segmentationpointclick'; import AnnotationVisibilityMenu from './AnnotationVisibilityMenu.vue'; @@ -19,6 +20,7 @@ interface ButtonData { icon: string; type?: VisibleAnnotationTypes; active: boolean; + loading?: boolean; mousetrap?: Mousetrap[]; description: string; click: () => void; @@ -63,7 +65,7 @@ export default defineComponent({ default: () => ({ before: 20, after: 10 }), }, }, - emits: ['set-annotation-state', 'update:tail-settings'], + emits: ['set-annotation-state', 'update:tail-settings', 'text-query-init', 'text-query', 'text-query-all-frames'], setup(props, { emit }) { const toolTimeTimeout = ref(null); const STORAGE_KEY = 'editorMenu.editButtonsExpanded'; @@ -81,6 +83,59 @@ export default defineComponent({ localStorage.setItem(STORAGE_KEY, String(value)); }); + // Text query state + const textQueryDialogOpen = ref(false); + const textQueryInput = ref(''); + const textQueryLoading = ref(false); + const textQueryThreshold = ref(0.3); + const textQueryInitializing = ref(false); + const textQueryServiceError = ref(''); + const textQueryAllFrames = ref(false); + + const openTextQueryDialog = () => { + textQueryDialogOpen.value = true; + textQueryInput.value = ''; + textQueryServiceError.value = ''; + textQueryAllFrames.value = false; + textQueryInitializing.value = true; + emit('text-query-init'); + }; + + const closeTextQueryDialog = () => { + textQueryDialogOpen.value = false; + textQueryInput.value = ''; + textQueryServiceError.value = ''; + textQueryInitializing.value = false; + textQueryAllFrames.value = false; + }; + + const onTextQueryServiceReady = (success: boolean, error?: string) => { + textQueryInitializing.value = false; + if (!success) { + textQueryServiceError.value = error || 'Text query service is not available'; + } + }; + + const submitTextQuery = () => { + if (!textQueryInput.value.trim()) { + return; + } + textQueryLoading.value = true; + if (textQueryAllFrames.value) { + emit('text-query-all-frames', { + text: textQueryInput.value.trim(), + boxThreshold: textQueryThreshold.value, + }); + } else { + emit('text-query', { + text: textQueryInput.value.trim(), + boxThreshold: textQueryThreshold.value, + }); + } + closeTextQueryDialog(); + textQueryLoading.value = false; + }; + const modeToolTips = { Creating: { rectangle: 'Drag to draw rectangle. Press ESC to exit.', @@ -117,6 +172,7 @@ export default defineComponent({ id: r.name, icon: r.icon.value || 'mdi-pencil', active: props.editingTrack && r.active.value, + loading: r.loading?.value ?? false, description: r.name, click: () => r.activate(), mousetrap: [ @@ -130,7 +186,13 @@ export default defineComponent({ ]; }); - const mousetrap = computed((): Mousetrap[] => flatten(editButtons.value.map((b) => b.mousetrap || []))); + const mousetrap = computed((): Mousetrap[] => [ + ...flatten(editButtons.value.map((b) => b.mousetrap || [])), + { + bind: 't', + handler: () => openTextQueryDialog(), + }, + ]); const activeEditButton = computed(() => editButtons.value.find((b) => b.active) || editButtons.value[0]); @@ -157,6 +219,13 @@ export default defineComponent({ return { text: 'Not editing', icon: 'mdi-pencil-off-outline', color: '' }; }); + const activeSegmentationRecipe = computed((): SegmentationPointClick | null => { + const segRecipe = props.recipes.find( + (r) => r instanceof SegmentationPointClick && r.active.value, + ) as SegmentationPointClick | undefined; + return segRecipe || null; + }); + const editingTooltip = computed(() => { if (props.editingDetails === 'disabled' || !props.editingMode || typeof props.editingMode !== 'string') { return ''; @@ -190,6 +259,19 @@ export default defineComponent({ toggleEditButtonsExpanded, activeEditButton, editButtonsMenuKey, + activeSegmentationRecipe, + // Text query + textQueryDialogOpen, + textQueryInput, + textQueryLoading, + textQueryThreshold, + textQueryInitializing, + textQueryServiceError, + textQueryAllFrames, + openTextQueryDialog, + closeTextQueryDialog, + onTextQueryServiceReady, + submitTextQuery, }; }, }); @@ -240,7 +322,7 @@ export default defineComponent({ + + +
T:
+ mdi-text-search +
+ + - + + @@ -329,6 +450,103 @@ export default defineComponent({ @update:tail-settings="$emit('update:tail-settings', $event)" /> + + + + + + + mdi-text-search + + Text Query + + + +
+ +

+ Loading text query model... +

+
+ +
+ + mdi-alert-circle + +

+ {{ textQueryServiceError }} +

+
+ + +
+ + + + {{ textQueryServiceError ? 'Close' : 'Cancel' }} + + + Search + + +
+
diff --git a/client/dive-common/components/Sidebar.vue b/client/dive-common/components/Sidebar.vue index d7b277dc9..7642c1fdd 100644 --- a/client/dive-common/components/Sidebar.vue +++ b/client/dive-common/components/Sidebar.vue @@ -18,6 +18,7 @@ import { } from 'vue-media-annotator/provides'; import { clientSettings } from 'dive-common/store/settings'; +import ConfidenceFilter from 'dive-common/components/ConfidenceFilter.vue'; import TrackDetailsPanel from 'dive-common/components/TrackDetailsPanel.vue'; import TrackSettingsPanel from 'dive-common/components/TrackSettingsPanel.vue'; import TypeSettingsPanel from 'dive-common/components/TypeSettingsPanel.vue'; @@ -27,6 +28,7 @@ import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; export default defineComponent({ components: { + ConfidenceFilter, StackedVirtualSidebarContainer, TrackDetailsPanel, TrackSettingsPanel, @@ -43,6 +45,14 @@ export default defineComponent({ type: Boolean, default: true, }, + horizontal: { + type: Boolean, + default: false, + }, + isStereoDataset: { + type: Boolean, + default: false, + }, }, setup() { @@ -63,7 +73,9 @@ export default defineComponent({ const styleManager = useTrackStyleManager(); const data = reactive({ - currentTab: 'tracks' as 'tracks' | 'attributes', + currentTab: 'tracks' as 'tracks' | 'attributes' | 'types', + // For horizontal mode, cycle through 3 tabs + horizontalTab: 'tracks' as 'tracks' | 'attributes' | 'types', }); function swapTabs() { @@ -74,6 +86,28 @@ export default defineComponent({ } } + function cycleHorizontalTabs() { + if (data.horizontalTab === 'tracks') { + data.horizontalTab = 'attributes'; + } else if (data.horizontalTab === 'attributes') { + data.horizontalTab = 'types'; + } else { + data.horizontalTab = 'tracks'; + } + } + + const horizontalTabIcon = computed(() => { + if (data.horizontalTab === 'tracks') return 'mdi-format-list-bulleted'; + if (data.horizontalTab === 'attributes') return 'mdi-card-text'; + return 'mdi-filter-variant'; + }); + + const horizontalTabTooltip = computed(() => { + if (data.horizontalTab === 'tracks') return 'Detection List (click to cycle)'; + if (data.horizontalTab === 'attributes') return 'Detection Details (click to cycle)'; + return 'Type Filters (click to cycle)'; + }); + function doToggleMerge() { if (toggleMerge().length) { data.currentTab = 'attributes'; @@ -121,17 +155,23 @@ export default defineComponent({ readOnlyMode, styleManager, disableAnnotationFilters: trackFilterControls.disableAnnotationFilters, + confidenceFilters: trackFilterControls.confidenceFilters, visible, + horizontalTabIcon, + horizontalTabTooltip, /* methods */ doToggleMerge, swapTabs, + cycleHorizontalTabs, }; }, }); + + +
+ + + {{ horizontalTabTooltip }} + + +
+ +
+ +
+ +
+ + + +
+
+ +
+ +
+ +
+ + + +
+
diff --git a/client/dive-common/components/TrackSettingsPanel.vue b/client/dive-common/components/TrackSettingsPanel.vue index 71b1ec1b2..ba8672680 100644 --- a/client/dive-common/components/TrackSettingsPanel.vue +++ b/client/dive-common/components/TrackSettingsPanel.vue @@ -16,6 +16,10 @@ export default defineComponent({ type: Array as PropType>, required: true, }, + isStereoDataset: { + type: Boolean, + default: false, + }, }, setup(props) { @@ -33,6 +37,7 @@ export default defineComponent({ filterTracksByFrame: 'Filter the track list by those with detections in the current frame', autoZoom: 'Automatically zoom to the track when selected', showMultiCamToolbar: 'Show multi-camera tools in the top toolbar when a track is selected', + stereoInteractiveMode: 'When enabled, annotations created on one camera are automatically warped to the other camera using stereo disparity', }); const modes = ref(['Track', 'Detection']); // Add unknown as the default type to the typeList @@ -362,6 +367,47 @@ export default defineComponent({ + diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index b3da641be..f2a2896b5 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -1,6 +1,6 @@ + + diff --git a/client/platform/web-girder/api/rpc.service.ts b/client/platform/web-girder/api/rpc.service.ts index cda1aed53..435a89742 100644 --- a/client/platform/web-girder/api/rpc.service.ts +++ b/client/platform/web-girder/api/rpc.service.ts @@ -1,6 +1,9 @@ import girderRest from 'platform/web-girder/plugins/girder'; import type { GirderModel } from '@girder/components/src'; -import { Pipe } from 'dive-common/apispec'; +import { + Pipe, SegmentationPredictRequest, SegmentationPredictResponse, SegmentationStatusResponse, + TextQueryRequest, TextQueryResponse, +} from 'dive-common/apispec'; function postProcess(folderId: string, skipJobs = false, skipTranscoding = false, additive = false, additivePrepend = '', set: string | undefined = undefined) { return girderRest.post<{folder: GirderModel, warnings: string[], job_ids: string[]}>(`dive_rpc/postprocess/${folderId}`, null, { @@ -56,6 +59,75 @@ function convertLargeImage(folderId: string) { return girderRest.post(`dive_rpc/convert_large_image/${folderId}`, null, {}); } +/** + * Interactive Segmentation API + */ + +async function segmentationPredict( + folderId: string, + frameNumber: number, + request: SegmentationPredictRequest, +): Promise { + const { data } = await girderRest.post('dive_rpc/segmentation_predict', { + points: request.points, + pointLabels: request.pointLabels, + maskInput: request.maskInput, + multimaskOutput: request.multimaskOutput, + }, { + params: { + folderId, + frameNumber, + }, + }); + return data; +} + +async function segmentationStatus(): Promise { + const { data } = await girderRest.get('dive_rpc/segmentation_status'); + return data; +} + +/** + * Initialize segmentation service by checking availability. + * Throws an error if segmentation is not available on the server. + */ +async function segmentationInitialize(): Promise { + const status = await segmentationStatus(); + if (!status.available) { + throw new Error('Unable to load segmentation module'); + } +} + +/** + * Text Query API for open-vocabulary detection/segmentation + */ + +async function textQuery( + folderId: string, + frameNumber: number, + request: Omit, +): Promise { + const { data } = await girderRest.post('dive_rpc/text_query', { + text: request.text, + boxThreshold: request.boxThreshold, + maxDetections: request.maxDetections, + }, { + params: { + folderId, + frameNumber, + }, + }); + return data; +} + +async function textQueryStatus(): Promise<{ available: boolean; grounding_available: boolean }> { + const { data } = await girderRest.get<{ available: boolean; loaded: boolean; text_query_available: boolean }>('dive_rpc/segmentation_status'); + return { + available: data.available, + grounding_available: data.text_query_available, + }; +} + export { convertLargeImage, postProcess, @@ -63,4 +135,9 @@ export { runTraining, deleteTrainedPipeline, exportTrainedPipeline, + segmentationPredict, + segmentationStatus, + segmentationInitialize, + textQuery, + textQueryStatus, }; diff --git a/client/platform/web-girder/views/ViewerLoader.vue b/client/platform/web-girder/views/ViewerLoader.vue index 91544ae1b..84a23054a 100644 --- a/client/platform/web-girder/views/ViewerLoader.vue +++ b/client/platform/web-girder/views/ViewerLoader.vue @@ -1,6 +1,6 @@