diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 646db429e..be7b1c100 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -85,6 +85,8 @@ interface SaveAttributeTrackFilterArgs { interface FrameImage { url: string; filename: string; + /** Required for large-image (tiled) datasets; used as itemId for getTiles/getTileURL */ + id?: string; } export interface MultiCamImportFolderArgs { diff --git a/client/dive-common/constants.ts b/client/dive-common/constants.ts index 8cc7401b7..20c002fc0 100644 --- a/client/dive-common/constants.ts +++ b/client/dive-common/constants.ts @@ -76,11 +76,18 @@ const fileVideoTypes = [ const largeImageTypes = [ 'image/geotiff', 'image/tiff', + 'image/tif', 'image/x-tiff', 'image/nitf', 'image/ntf', ]; +const largeImageDesktopTypes = [ + 'geotiff', + 'tiff', + 'tif', +]; + const websafeImageTypes = [ // 'image/apng', // 'image/bmp', @@ -149,6 +156,7 @@ export { websafeVideoTypes, inputAnnotationTypes, largeImageTypes, + largeImageDesktopTypes, inputAnnotationFileTypes, listFileTypes, zipFileTypes, diff --git a/client/package.json b/client/package.json index f5c18e2d3..3db048dc0 100644 --- a/client/package.json +++ b/client/package.json @@ -46,6 +46,8 @@ "csv-stringify": "^5.6.0", "d3": "^5.12.0", "geojs": "~1.6.2", + "geotiff": "^3.0.0", + "pngjs": "^7.0.0", "glob-to-regexp": "^0.4.1", "lodash": "^4.17.19", "moment": "^2.29.1", diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index ea8bfbc0f..d12e127f1 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -32,7 +32,8 @@ import kpf from 'platform/desktop/backend/serializers/kpf'; // eslint-disable-next-line import/no-cycle import { checkMedia } from 'platform/desktop/backend/native/mediaJobs'; import { - websafeImageTypes, websafeVideoTypes, otherImageTypes, otherVideoTypes, MultiType, JsonMetaRegEx, + websafeImageTypes, websafeVideoTypes, otherImageTypes, otherVideoTypes, + MultiType, JsonMetaRegEx, largeImageTypes, } from 'dive-common/constants'; import { JsonMeta, Settings, JsonMetaCurrentVersion, DesktopMetadata, @@ -300,6 +301,13 @@ async function loadMetadata( }; }); } + } else if (projectMetaData.type === 'large-image' && projectMetaData.originalLargeImageFile) { + const tiffPath = npath.join(projectMetaData.originalBasePath, projectMetaData.originalLargeImageFile); + imageData = [{ + url: makeMediaUrl(tiffPath), + id: datasetId, + filename: npath.basename(tiffPath), + }]; } else { throw new Error(`unexpected project type for id="${datasetId}" type="${projectMetaData.type}"`); } @@ -943,6 +951,8 @@ async function beginMediaImport(path: string): Promise { + try { + const projectDirData = await getValidatedProjectDir(settings, datasetId); + const meta = await loadJsonMetadata(projectDirData.metaFileAbsPath); + if (meta.type !== 'large-image' || !meta.originalLargeImageFile) { + console.warn( + `[tiles] getLargeImagePath: no path for dataset "${datasetId}" (meta.type=${meta.type}, hasOriginalLargeImageFile=${!!meta.originalLargeImageFile})`, + ); + return null; + } + const path = npath.join(meta.originalBasePath, meta.originalLargeImageFile); + return path; + } catch (err) { + console.warn(`[tiles] getLargeImagePath: error for dataset "${datasetId}":`, err); + return null; + } +} + async function openLink(url: string) { shell.openExternal(url); } @@ -1333,6 +1371,7 @@ export { getTrainingConfigs, getProjectDir, getValidatedProjectDir, + getLargeImagePath, loadMetadata, loadJsonMetadata, loadAnnotationFile, diff --git a/client/platform/desktop/backend/server.ts b/client/platform/desktop/backend/server.ts index 81822bb98..8ec867de4 100644 --- a/client/platform/desktop/backend/server.ts +++ b/client/platform/desktop/backend/server.ts @@ -12,6 +12,7 @@ import { SaveAttributeArgs, SaveAttributeTrackFilterArgs, SaveDetectionsArgs } f import settings from './state/settings'; import * as common from './native/common'; +import * as geotiffTiles from './tiles/geotiffTiles'; const app = express(); app.use(express.json({ limit: '250MB' })); @@ -123,6 +124,56 @@ apirouter.post('/dataset/:id/:camera?/detections', async (req, res, next) => { return null; }); +/* Large image (GeoTIFF) tiles - compatible with LargeImageAnnotator getTiles/getTileURL */ +apirouter.get('/dataset/:id/:camera?/tiles/:level/:x/:y', async (req, res, next) => { + try { + const datasetId = req.params.camera + ? `${req.params.id}/${req.params.camera}` + : req.params.id; + const level = parseInt(req.params.level, 10); + const x = parseInt(req.params.x, 10); + const y = parseInt(req.params.y, 10); + if (Number.isNaN(level) || Number.isNaN(x) || Number.isNaN(y)) { + return next({ status: 400, statusMessage: 'Invalid level, x, or y' }); + } + const png = await geotiffTiles.getTilePng(settings.get(), datasetId, level, x, y); + if (!png) { + console.warn(`[tiles] GET tile 404: datasetId=${datasetId} level=${level} x=${x} y=${y} (see tile layer logs for reason)`); + return next({ status: 404, statusMessage: 'Tile not found or dataset is not a large image' }); + } + res.setHeader('Content-Type', 'image/png'); + res.send(png); + } catch (err) { + console.error('[tiles] GET tile error:', err); + (err as { status?: number }).status = 500; + next(err); + } + return null; +}); + +apirouter.get('/dataset/:id/:camera?/tiles', async (req, res, next) => { + try { + const datasetId = req.params.camera + ? `${req.params.id}/${req.params.camera}` + : req.params.id; + const meta = await geotiffTiles.getTilesMetadata(settings.get(), datasetId); + if (!meta) { + console.warn(`[tiles] GET tiles metadata 404: datasetId=${datasetId} (see tile layer logs for reason)`); + return next({ status: 404, statusMessage: 'Dataset not found or is not a large image' }); + } + if (meta.preconversionRequired && meta.error) { + console.warn(`[tiles] GET tiles metadata 422: datasetId=${datasetId} requires pre-conversion`); + return next({ status: 422, statusMessage: meta.error }); + } + res.json(meta); + } catch (err) { + console.error('[tiles] GET tiles metadata error:', err); + (err as { status?: number }).status = 500; + next(err); + } + return null; +}); + /* STREAM media */ apirouter.get('/media', (req, res, next) => { let { path } = req.query; diff --git a/client/platform/desktop/backend/tiles/geotiffTiles.ts b/client/platform/desktop/backend/tiles/geotiffTiles.ts new file mode 100644 index 000000000..67057afcc --- /dev/null +++ b/client/platform/desktop/backend/tiles/geotiffTiles.ts @@ -0,0 +1,796 @@ +/** + * Serve large image (GeoTIFF) tiles using geotiff.js. + * Compatible with LargeImageAnnotator's getTiles/getTileURL expectations. + * + * Inspired by https://github.com/rowanwins/geotiff-server: + * - Uses image-space (pixel) window for readRasters, not bbox. + * - Normalizes 16-bit/float data to 0-255 by clamping (no per-dataset min/max stretch). + * - Handles single-band (grayscale) by replicating to R=G=B. + * + * Geospatial metadata in the file is not used for tiling. + */ + +import fs from 'fs-extra'; +import { fromArrayBuffer, fromFile } from 'geotiff'; +import PNG from 'pngjs'; +import type { Settings } from 'platform/desktop/constants'; +import { getLargeImagePath } from '../native/common'; + +const TILE_SIZE = 256; + +export interface TileRange { + level: number; + maxX: number; + maxY: number; +} + +export interface TilesMetadata { + sizeX: number; + sizeY: number; + sourceSizeX?: number; + sourceSizeY?: number; + tileWidth: number; + tileHeight: number; + levels: number; + sourceLevels?: number; + preconversionRequired?: boolean; + error?: string; + /** Valid tile indices per level: for each level, x in [0, maxX], y in [0, maxY]. */ + tileRanges: TileRange[]; +} + +/** + * Compute valid tile index ranges from image dimensions and level count. + * Uses geoJS convention: level 0 = overview (scale 2^maxLevel), level maxLevel = full res (scale 1). + * At geoJS level L, scale = 2^(maxLevel - L); valid x in [0, maxX], y in [0, maxY]. + */ +export function getValidTileRanges( + sizeX: number, + sizeY: number, + levels: number, +): TileRange[] { + const ranges: TileRange[] = []; + const maxLevel = Math.max(0, levels); + for (let level = 0; level <= maxLevel; level += 1) { + const scale = 2 ** (maxLevel - level); + const maxX = Math.max(0, Math.ceil(sizeX / (TILE_SIZE * scale)) - 1); + const maxY = Math.max(0, Math.ceil(sizeY / (TILE_SIZE * scale)) - 1); + ranges.push({ level, maxX, maxY }); + } + return ranges; +} + +const LOG_PREFIX = '[tiles]'; +const MAX_TILE_CACHE_ENTRIES = 256; +const TIFF_STAT_REFRESH_MS = 1000; +const DEFAULT_MAX_IN_MEMORY_TIFF_MB = 5120; +const PATH_CACHE_TTL_MS = 5000; +type RasterArray = Uint8Array | Uint16Array | Float32Array | Float64Array; + +interface GeoTiffReadableImage { + getWidth(): number; + getHeight(): number; + getSamplesPerPixel(): number; + readRasters(options: Record): Promise; + readRGB(options: Record): Promise; + getFileDirectory?: () => unknown; + fileDirectory?: unknown; +} + +interface GeoTiffReadable { + getImage(index: number): Promise; + getImageCount?: () => Promise | number; +} + +interface TiffImageSource { + index: number; + image: GeoTiffReadableImage; + width: number; + height: number; + overviewLevel: number; + scaleX: number; + scaleY: number; + scale: number; +} + +interface RawTiffImageEntry { + index: number; + image: GeoTiffReadableImage; + width: number; + height: number; +} + +interface TileWindow { + levelClamped: number; + scale: number; + left: number; + top: number; + right: number; + bottom: number; + outputWidth: number; + outputHeight: number; +} + +interface TiffContext { + path: string; + size: number; + mtimeMs: number; + inMemory: boolean; + tiff: unknown; + image: GeoTiffReadableImage; + imageSources: TiffImageSource[]; + width: number; + height: number; + sourceMaxLevel: number; + maxLevel: number; + preconversionRequired: boolean; + preconversionError: string | null; +} + +let tiffContext: TiffContext | null = null; +let tiffContextPromise: Promise | null = null; +let tiffContextPromisePath: string | null = null; +let tiffContextStatCheckedAtMs = 0; +const datasetPathCache = new Map(); +const datasetPathPromiseCache = new Map>(); + +const tilePngCache = new Map(); + +function touchTileCacheEntry(key: string, value: Buffer): void { + if (tilePngCache.has(key)) { + tilePngCache.delete(key); + } + tilePngCache.set(key, value); + if (tilePngCache.size > MAX_TILE_CACHE_ENTRIES) { + const oldestKey = tilePngCache.keys().next().value as string | undefined; + if (oldestKey) { + tilePngCache.delete(oldestKey); + } + } +} + +function getTileCacheKey(path: string, level: number, x: number, y: number): string { + return `${path}::${level}/${x}/${y}`; +} + +function resetTileCacheForPath(path: string): void { + const prefix = `${path}::`; + const keys = Array.from(tilePngCache.keys()); + keys.forEach((key) => { + if (key.startsWith(prefix)) { + tilePngCache.delete(key); + } + }); +} + +function getCachedDatasetPath(datasetId: string): string | null | undefined { + const now = Date.now(); + const hit = datasetPathCache.get(datasetId); + if (!hit) return undefined; + if (hit.expiresAtMs <= now) { + datasetPathCache.delete(datasetId); + return undefined; + } + return hit.path; +} + +function setCachedDatasetPath(datasetId: string, path: string | null): void { + datasetPathCache.set(datasetId, { + path, + expiresAtMs: Date.now() + PATH_CACHE_TTL_MS, + }); +} + +function getMaxInMemoryTiffBytes(): number { + const raw = process.env.MAX_IN_MEMORY_TIFF_MB; + if (!raw || raw.trim() === '') { + return DEFAULT_MAX_IN_MEMORY_TIFF_MB * 1024 * 1024; + } + const mb = Number(raw); + if (!Number.isFinite(mb) || mb <= 0) { + return 0; + } + return Math.floor(mb * 1024 * 1024); +} + +function shouldLoadTiffInMemory(sizeBytes: number): boolean { + if (process.env.TILES_FORCE_FILE_BACKED?.trim() === '1') { + return false; + } + if (process.env.TILES_FORCE_IN_MEMORY?.trim() === '1') { + return true; + } + return sizeBytes <= getMaxInMemoryTiffBytes(); +} + +async function loadTiffContext(path: string): Promise { + const stat = await fs.stat(path); + const inMemory = shouldLoadTiffInMemory(stat.size); + const tiff = inMemory + ? await fs.readFile(path).then((buffer) => { + const arrayBuffer = buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength, + ); + return fromArrayBuffer(arrayBuffer); + }) + // File-backed mode fetches bytes lazily and keeps process memory lower. + : await fromFile(path); + const tiffReadable = tiff as unknown as GeoTiffReadable; + const imageCountRaw = tiffReadable.getImageCount ? await tiffReadable.getImageCount() : 1; + const imageCount = Number.isFinite(imageCountRaw) ? Math.max(1, Math.floor(Number(imageCountRaw))) : 1; + const rawImages: RawTiffImageEntry[] = []; + for (let i = 0; i < imageCount; i += 1) { + // eslint-disable-next-line no-await-in-loop + const image = await tiffReadable.getImage(i); + rawImages.push({ + index: i, + image, + width: image.getWidth(), + height: image.getHeight(), + }); + } + // Use the largest IFD as full resolution; overviews are typically smaller. + let fullRes = rawImages[0]; + for (let i = 1; i < rawImages.length; i += 1) { + const candidate = rawImages[i]; + if ((candidate.width * candidate.height) > (fullRes.width * fullRes.height)) { + fullRes = candidate; + } + } + const { width, height } = fullRes; + const theoreticalMaxLevel = Math.max(0, Math.ceil(Math.log2(Math.max(width / TILE_SIZE, height / TILE_SIZE)))); + const imageSources = buildAlignedImageSources(rawImages, fullRes); + const maxAvailableScale = imageSources.length > 0 + ? imageSources[imageSources.length - 1].scale + : 1; + const overviewCount = imageSources.reduce((count, source) => ( + source.scale > 1 ? count + 1 : count + ), 0); + const availableMaxLevel = Math.max(0, Math.floor(Math.log2(Math.max(1, maxAvailableScale)))); + const maxLevel = Math.min(theoreticalMaxLevel, availableMaxLevel); + const preconversionRequired = theoreticalMaxLevel > 0 && availableMaxLevel === 0; + const preconversionError = preconversionRequired + ? `Detected ${overviewCount} internal overview levels (need at least 1). Need to use GDAL to pre-convert the image to a tiled pyramidal COG.` + : null; + return { + path, + size: stat.size, + mtimeMs: stat.mtimeMs, + inMemory, + tiff, + image: fullRes.image, + imageSources, + width, + height, + sourceMaxLevel: theoreticalMaxLevel, + maxLevel, + preconversionRequired, + preconversionError, + }; +} + +function buildAlignedImageSources(rawImages: RawTiffImageEntry[], fullRes: RawTiffImageEntry): TiffImageSource[] { + /** + * Select an IFD pyramid that behaves predictably for tile rendering: + * 1) keep only dimensions/samples compatible with full resolution, + * 2) snap each source to the nearest power-of-two overview level, + * 3) keep the best candidate per level and always keep full-res fallback. + * + * This avoids stitching seams from non-pyramidal sidecar IFDs while still + * making use of valid internal overviews when present. + */ + const fullResSamples = fullRes.image.getSamplesPerPixel(); + const aligned = rawImages.filter((entry) => { + if (entry.width <= 0 || entry.height <= 0) return false; + if (entry.width > fullRes.width || entry.height > fullRes.height) return false; + if (entry.image.getSamplesPerPixel() !== fullResSamples) return false; + const scaleX = fullRes.width / entry.width; + const scaleY = fullRes.height / entry.height; + const aspectDelta = Math.abs(scaleX - scaleY) / Math.max(scaleX, scaleY, 1); + return aspectDelta <= 0.03; + }); + // Always include full-res source as a safe fallback. + const withFallback = aligned.some((entry) => entry.index === fullRes.index) + ? aligned + : [...aligned, fullRes]; + const byOverviewLevel = new Map(); + withFallback.forEach((entry) => { + const rawScaleX = Math.max(1, fullRes.width / Math.max(1, entry.width)); + const rawScaleY = Math.max(1, fullRes.height / Math.max(1, entry.height)); + const rawScale = Math.max(rawScaleX, rawScaleY); + const overviewLevel = Math.max(0, Math.round(Math.log2(rawScale))); + const expectedScale = 2 ** overviewLevel; + const scaleError = Math.max( + Math.abs(rawScaleX - expectedScale) / expectedScale, + Math.abs(rawScaleY - expectedScale) / expectedScale, + ); + // Keep only pyramid-like levels; odd non-pyramid IFDs can cause visible tile seams. + if (entry.index !== fullRes.index && scaleError > 0.12) { + return; + } + const candidate: TiffImageSource = { + index: entry.index, + image: entry.image, + width: entry.width, + height: entry.height, + overviewLevel, + scaleX: rawScaleX, + scaleY: rawScaleY, + scale: expectedScale, + }; + const existing = byOverviewLevel.get(overviewLevel); + if (!existing) { + byOverviewLevel.set(overviewLevel, candidate); + return; + } + const existingError = Math.max( + Math.abs(existing.scaleX - expectedScale) / expectedScale, + Math.abs(existing.scaleY - expectedScale) / expectedScale, + ); + // Prefer the source whose dimensions are closer to the expected pyramid level. + if (scaleError < existingError) { + byOverviewLevel.set(overviewLevel, candidate); + } + }); + if (!byOverviewLevel.has(0)) { + byOverviewLevel.set(0, { + index: fullRes.index, + image: fullRes.image, + width: fullRes.width, + height: fullRes.height, + overviewLevel: 0, + scaleX: 1, + scaleY: 1, + scale: 1, + }); + } + return Array.from(byOverviewLevel.values()).sort((a, b) => a.overviewLevel - b.overviewLevel); +} + +function pickImageSourceForScale(imageSources: TiffImageSource[], requestedScale: number): TiffImageSource { + if (process.env.TILES_DISABLE_OVERVIEW_SELECTION?.trim() === '1') { + return imageSources[0]; + } + const target = Math.max(1, requestedScale); + const targetOverviewLevel = Math.max(0, Math.round(Math.log2(target))); + const exact = imageSources.find((source) => source.overviewLevel === targetOverviewLevel); + if (exact) { + return exact; + } + // Prefer finer-than-requested sources (avoid upscaling when no exact level exists). + let best = imageSources[0]; + let bestLevel = -1; + for (let i = 0; i < imageSources.length; i += 1) { + const source = imageSources[i]; + if (source.overviewLevel <= targetOverviewLevel && source.overviewLevel > bestLevel) { + best = source; + bestLevel = source.overviewLevel; + } + } + if (bestLevel >= 0) { + return best; + } + // Fallback: no finer level exists, use the nearest coarser one. + for (let i = 0; i < imageSources.length; i += 1) { + const source = imageSources[i]; + if (source.overviewLevel > targetOverviewLevel) { + return source; + } + } + return imageSources[imageSources.length - 1]; +} + +async function getCachedTiffContext(path: string): Promise { + /** + * Cache policy: + * - Return hot context quickly within a short stat refresh interval. + * - Re-stat and reuse context when file size/mtime match. + * - Coalesce concurrent reloads through a single in-flight promise per path. + */ + const now = Date.now(); + if ( + tiffContext + && tiffContext.path === path + && now - tiffContextStatCheckedAtMs < TIFF_STAT_REFRESH_MS + ) { + return tiffContext; + } + const stat = await fs.stat(path); + tiffContextStatCheckedAtMs = now; + const cacheValid = tiffContext + && tiffContext.path === path + && tiffContext.size === stat.size + && tiffContext.mtimeMs === stat.mtimeMs; + + if (cacheValid && tiffContext) { + return tiffContext; + } + + if (!tiffContextPromise || tiffContextPromisePath !== path) { + tiffContextPromisePath = path; + tiffContextPromise = loadTiffContext(path).then((ctx) => { + const pathChanged = tiffContext && tiffContext.path !== ctx.path; + const fileChanged = tiffContext + && tiffContext.path === ctx.path + && (tiffContext.size !== ctx.size || tiffContext.mtimeMs !== ctx.mtimeMs); + if (pathChanged || fileChanged) { + resetTileCacheForPath(ctx.path); + } + tiffContext = ctx; + return ctx; + }).finally(() => { + if (tiffContextPromisePath === path) { + tiffContextPromise = null; + tiffContextPromisePath = null; + } + }); + } + + return tiffContextPromise; +} + +function clampToByte(value: number): number { + return Math.max(0, Math.min(255, Math.round(value))); +} + +function createOpaqueRgba(pixelCount: number): Uint8Array { + const rgba = new Uint8Array(pixelCount * 4); + for (let i = 3; i < rgba.length; i += 4) { + rgba[i] = 255; + } + return rgba; +} + +function computeTileWindow( + width: number, + height: number, + maxLevel: number, + level: number, + x: number, + y: number, +): TileWindow | null { + const levelClamped = Math.max(0, Math.min(level, maxLevel)); + const scale = 2 ** (maxLevel - levelClamped); + const left = Math.min(x * TILE_SIZE * scale, width); + const top = Math.min(y * TILE_SIZE * scale, height); + const right = Math.min((x + 1) * TILE_SIZE * scale, width); + const bottom = Math.min((y + 1) * TILE_SIZE * scale, height); + if (left >= right || top >= bottom) return null; + return { + levelClamped, + scale, + left, + top, + right, + bottom, + outputWidth: Math.max(1, Math.min(TILE_SIZE, Math.ceil((right - left) / scale))), + outputHeight: Math.max(1, Math.min(TILE_SIZE, Math.ceil((bottom - top) / scale))), + }; +} + +function computeSourceWindow( + source: TiffImageSource, + tileWindow: TileWindow, +): [number, number, number, number] { + const sourceLeft = Math.min(source.width, Math.max(0, Math.floor(tileWindow.left / source.scaleX))); + const sourceTop = Math.min(source.height, Math.max(0, Math.floor(tileWindow.top / source.scaleY))); + const sourceRight = Math.min( + source.width, + Math.max(sourceLeft + 1, Math.ceil(tileWindow.right / source.scaleX)), + ); + const sourceBottom = Math.min( + source.height, + Math.max(sourceTop + 1, Math.ceil(tileWindow.bottom / source.scaleY)), + ); + return [sourceLeft, sourceTop, sourceRight, sourceBottom]; +} + +/** If set (e.g. DEBUG_TILE_PATH=/path/to/lake.tiff), use this path for all tile requests (for debugging). */ +function getEffectiveTiffPath( + settings: Settings, + datasetId: string, +): Promise { + const debugPath = process.env.DEBUG_TILE_PATH?.trim(); + if (debugPath) { + return fs.pathExists(debugPath).then((exists) => { + if (exists) { + return debugPath; + } + return null; + }); + } + const cachedPath = getCachedDatasetPath(datasetId); + if (cachedPath !== undefined) { + return Promise.resolve(cachedPath); + } + const inFlight = datasetPathPromiseCache.get(datasetId); + if (inFlight) { + return inFlight; + } + const lookupPromise = getLargeImagePath(settings, datasetId) + .then((path) => { + setCachedDatasetPath(datasetId, path); + return path; + }) + .finally(() => { + datasetPathPromiseCache.delete(datasetId); + }); + datasetPathPromiseCache.set(datasetId, lookupPromise); + return lookupPromise; +} + +/** + * Get tile metadata for a large-image dataset (compatible with Girder large_image response). + * Returns null if the file is missing or invalid (e.g. LZW decode issues). + */ +export async function getTilesMetadata( + settings: Settings, + datasetId: string, +): Promise { + const tiffPath = await getEffectiveTiffPath(settings, datasetId); + if (!tiffPath) { + console.warn(`${LOG_PREFIX} getTilesMetadata: no large-image path for dataset "${datasetId}" (not large-image or missing originalLargeImageFile in meta)`); + return null; + } + try { + const ctx = await getCachedTiffContext(tiffPath); + const { + width, + height, + maxLevel, + sourceMaxLevel, + preconversionRequired, + preconversionError, + } = ctx; + const levelCount = maxLevel + 1; + const sourceLevelCount = sourceMaxLevel + 1; + // geoJS pixelCoordinateParams uses maxLevel = ceil(log2(max(w/256, h/256))); level 0 = overview, maxLevel = full res. + const tileRanges = getValidTileRanges(width, height, maxLevel); + return { + sizeX: width, + sizeY: height, + sourceSizeX: width, + sourceSizeY: height, + tileWidth: TILE_SIZE, + tileHeight: TILE_SIZE, + levels: levelCount, + sourceLevels: sourceLevelCount, + preconversionRequired, + error: preconversionError ?? undefined, + tileRanges, + }; + } catch (err) { + if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') { + console.warn(`${LOG_PREFIX} getTilesMetadata: file does not exist for dataset "${datasetId}": ${tiffPath}`); + return null; + } + console.error(`${LOG_PREFIX} getTilesMetadata: failed to read/parse GeoTIFF for dataset "${datasetId}" (${tiffPath}):`, err); + return null; + } +} + +/** + * Get a single tile as PNG buffer for level, x, y. + * Returns null on decode errors (e.g. LZW "ran off the end of the buffer" in + * geotiff.js for truncated or malformed GeoTIFF strips). + */ +export async function getTilePng( + settings: Settings, + datasetId: string, + level: number, + x: number, + y: number, +): Promise { + const tiffPath = await getEffectiveTiffPath(settings, datasetId); + if (!tiffPath) { + console.warn(`${LOG_PREFIX} getTilePng: no large-image path for dataset "${datasetId}" (level=${level}, x=${x}, y=${y})`); + return null; + } + try { + const ctx = await getCachedTiffContext(tiffPath); + const { + width, + height, + maxLevel, + imageSources, + } = ctx; + const tileCacheKey = getTileCacheKey(tiffPath, level, x, y); + const cached = tilePngCache.get(tileCacheKey); + if (cached) { + touchTileCacheEntry(tileCacheKey, cached); + return cached; + } + // geoJS uses level 0 = overview (fewest tiles), maxLevel = full res (most tiles). + // tilesAtZoom(level): scale = 2^(maxLevel - level). Match that here. + const tileWindow = computeTileWindow(width, height, maxLevel, level, x, y); + if (!tileWindow) { + const attemptedScale = 2 ** (maxLevel - Math.max(0, Math.min(level, maxLevel))); + const attemptedLeft = Math.min(x * TILE_SIZE * attemptedScale, width); + const attemptedTop = Math.min(y * TILE_SIZE * attemptedScale, height); + const attemptedRight = Math.min((x + 1) * TILE_SIZE * attemptedScale, width); + const attemptedBottom = Math.min((y + 1) * TILE_SIZE * attemptedScale, height); + console.warn(`${LOG_PREFIX} getTilePng: tile window out of bounds for dataset "${datasetId}" level=${level} x=${x} y=${y} (image ${width}x${height}, window [${attemptedLeft},${attemptedTop},${attemptedRight},${attemptedBottom}])`); + return null; + } + // Read window is in image-space pixels. We then remap the same window to the + // selected overview IFD so geotiff.js can resample to the requested tile size. + const source = pickImageSourceForScale(imageSources, tileWindow.scale); + const sourceWindow = computeSourceWindow(source, tileWindow); + const sourceImage = source.image; + const opts = { + window: sourceWindow, + width: tileWindow.outputWidth, + height: tileWindow.outputHeight, + resampleMethod: 'bilinear' as const, + }; + const expectedPixels = tileWindow.outputWidth * tileWindow.outputHeight; + const sampledRgba = createOpaqueRgba(expectedPixels); + + const sampleCount = sourceImage.getSamplesPerPixel(); + try { + if (sampleCount >= 3) { + const rgbInterleaved = await sourceImage.readRasters({ + ...opts, + samples: [0, 1, 2], + interleave: true, + } as Record) as RasterArray; + normalizeInterleavedRgbToRgba(rgbInterleaved, sampledRgba, expectedPixels); + } else { + const rasters = await sourceImage.readRasters({ + ...opts, + samples: [0], + interleave: false, + } as Record) as RasterArray[]; + const gray = (rasters[0] ?? new Uint8Array(0)) as RasterArray; + normalizeGrayToRgba(gray, sampledRgba, expectedPixels); + } + } catch (readErr) { + if (process.env.DEBUG_TILE_PATH) { + console.warn(`${LOG_PREFIX} getTilePng: readRasters failed, trying readRGB:`, readErr); + } + try { + const rgb = await sourceImage.readRGB(opts as Record); + fillRgbaFromReadRgbResult(rgb, sampledRgba, expectedPixels); + } catch { + throw readErr; + } + } + const rgba = new Uint8Array(TILE_SIZE * TILE_SIZE * 4); + blitRgba(sampledRgba, tileWindow.outputWidth, tileWindow.outputHeight, rgba, TILE_SIZE, TILE_SIZE); + const pngBuffer = encodePngRgba(TILE_SIZE, TILE_SIZE, rgba); + touchTileCacheEntry(tileCacheKey, pngBuffer); + return pngBuffer; + } catch (err) { + // LZW/geotiff decode errors (e.g. "ran off the end of the buffer before + // finding EOI_CODE") or other tile read failures: treat as missing tile. + if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') { + console.warn(`${LOG_PREFIX} getTilePng: file does not exist for dataset "${datasetId}" (level=${level}, x=${x}, y=${y}): ${tiffPath}`); + return null; + } + console.error(`${LOG_PREFIX} getTilePng: failed to read tile for dataset "${datasetId}" level=${level} x=${x} y=${y} (${tiffPath}):`, err); + return null; + } +} + +/** Map raster samples to display bytes by clamping to [0, 255] (no dynamic range stretch). */ +function normalizeToU8( + raw: Uint8Array | Uint16Array | Float32Array | Float64Array, + out: Uint8Array, + offset = 0, + stride = 1, +): void { + const n = Math.min(raw.length, Math.floor((out.length - offset) / stride)); + if (n === 0) return; + const outView = out; + for (let i = 0; i < n; i += 1) { + outView[offset + i * stride] = clampToByte(Number(raw[i])); + } +} + +function normalizeInterleavedRgbToRgba( + raw: Uint8Array | Uint16Array | Float32Array | Float64Array, + outRgba: Uint8Array, + pixelCount: number, +): void { + if (pixelCount <= 0 || raw.length < 3) return; + const out = outRgba; + const n = Math.min(pixelCount, Math.floor(raw.length / 3)); + for (let i = 0; i < n; i += 1) { + const src = i * 3; + const dst = i * 4; + const r = Number(raw[src]); + const g = Number(raw[src + 1] ?? raw[src]); + const b = Number(raw[src + 2] ?? raw[src]); + out[dst] = clampToByte(r); + out[dst + 1] = clampToByte(g); + out[dst + 2] = clampToByte(b); + } +} + +function normalizeGrayToRgba( + raw: Uint8Array | Uint16Array | Float32Array | Float64Array, + outRgba: Uint8Array, + pixelCount: number, +): void { + if (pixelCount <= 0 || raw.length === 0) return; + const out = outRgba; + const n = Math.min(pixelCount, raw.length); + if (raw instanceof Uint8Array) { + for (let i = 0; i < n; i += 1) { + const dst = i * 4; + const v = raw[i]; + out[dst] = v; + out[dst + 1] = v; + out[dst + 2] = v; + } + return; + } + normalizeToU8(raw, outRgba, 0, 4); + for (let i = 0; i < pixelCount; i += 1) { + const dst = i * 4; + const v = out[dst]; + out[dst + 1] = v; + out[dst + 2] = v; + } +} + +function fillRgbaFromReadRgbResult( + rgb: unknown, + outRgba: Uint8Array, + pixelCount: number, +): void { + const rgbBands = rgb as { [0]?: RasterArray; [1]?: RasterArray; [2]?: RasterArray }; + if (rgb && rgbBands[0] !== undefined && rgbBands[1] !== undefined && rgbBands[2] !== undefined) { + normalizeToU8(rgbBands[0], outRgba, 0, 4); + normalizeToU8(rgbBands[1], outRgba, 1, 4); + normalizeToU8(rgbBands[2], outRgba, 2, 4); + return; + } + const raw = rgb as RasterArray | undefined; + if (!raw || raw.length === 0) return; + if (raw.length >= pixelCount * 3) { + normalizeInterleavedRgbToRgba(raw, outRgba, pixelCount); + } else { + normalizeGrayToRgba(raw, outRgba, pixelCount); + } +} + +function blitRgba( + sourceRgba: Uint8Array, + sourceWidth: number, + sourceHeight: number, + targetRgba: Uint8Array, + targetWidth: number, + targetHeight: number, +): void { + if ( + sourceWidth <= 0 + || sourceHeight <= 0 + || targetWidth <= 0 + || targetHeight <= 0 + ) { + return; + } + const copyWidth = Math.min(sourceWidth, targetWidth); + const copyHeight = Math.min(sourceHeight, targetHeight); + for (let row = 0; row < copyHeight; row += 1) { + const srcOffset = row * sourceWidth * 4; + const dstOffset = row * targetWidth * 4; + const rowLength = copyWidth * 4; + targetRgba.set(sourceRgba.subarray(srcOffset, srcOffset + rowLength), dstOffset); + } +} + +function encodePngRgba(width: number, height: number, rgba: Uint8Array): Buffer { + type PngInstance = { data: Uint8Array }; + type PngConstructor = { + new(options: { width: number; height: number }): PngInstance; + sync: { write(png: PngInstance): Buffer }; + }; + const pngModule = PNG as unknown as { PNG?: PngConstructor }; + const PngCtor = (pngModule.PNG ?? (PNG as unknown as PngConstructor)); + const png = new PngCtor({ width, height }); + png.data.set(rgba); + return PngCtor.sync.write(png); +} diff --git a/client/platform/desktop/constants.ts b/client/platform/desktop/constants.ts index 2513cb15d..6f7e6808b 100644 --- a/client/platform/desktop/constants.ts +++ b/client/platform/desktop/constants.ts @@ -89,6 +89,9 @@ export interface JsonMeta extends DatasetMetaMutable { // relative to originalBasePath originalVideoFile: string; + // large image (e.g. GeoTIFF) file path, relative to originalBasePath + originalLargeImageFile?: string; + // output of web safe transcoding // relative to project path transcodedVideoFile: string; diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index dcf31ef7b..460f96f43 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -29,6 +29,8 @@ import { gpuJobQueue, cpuJobQueue } from './store/jobs'; * Native functions that run entirely in the renderer */ +const largeImageFileExtensions = ['tif', 'tiff', 'nitf', 'ntf']; + async function openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | 'annotation' | 'text', directory = false) { let filters: FileFilter[] = []; const allFiles = { name: 'All Files', extensions: ['*'] }; @@ -38,6 +40,12 @@ async function openFromDisk(datasetType: DatasetType | 'bulk' | 'calibration' | allFiles, ]; } + if (datasetType === 'large-image') { + filters = [ + { name: 'GeoTIFF / TIFF', extensions: largeImageFileExtensions }, + allFiles, + ]; + } if (datasetType === 'calibration') { filters = [ { name: 'calibration', extensions: calibrationFileTypes }, @@ -215,15 +223,34 @@ async function cancelJob(job: DesktopJob): Promise { * address details fetched from backend over ipc */ let _axiosClient: AxiosInstance; // do not use elsewhere +let _baseURL: string | null = null; + async function getClient(): Promise { if (_axiosClient === undefined) { const addr = await ipcRenderer.invoke('server-info'); - const baseURL = `http://${addr.address}:${addr.port}/api`; - _axiosClient = axios.create({ baseURL }); + _baseURL = `http://${addr.address}:${addr.port}/api`; + _axiosClient = axios.create({ baseURL: _baseURL }); } return _axiosClient; } +// eslint-disable-next-line @typescript-eslint/no-unused-vars -- projection kept for API compatibility +async function getTiles(itemId: string, _projection?: string) { + const client = await getClient(); // ensures _baseURL is set for getTileURL + const { data } = await client.get(`dataset/${itemId}/tiles`); + return data; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function getTileURL(itemId: string, x: number, y: number, level: number, query: Record): string { + if (!_baseURL) { + throw new Error('API not initialized: getTileURL called before any REST request'); + } + const params = new URLSearchParams(query || {}).toString(); + const suffix = params ? `?${params}` : ''; + return `${_baseURL}/dataset/${itemId}/tiles/${level}/${x}/${y}${suffix}`; +} + async function loadMetadata(id: string) { const client = await getClient(); const { data } = await client.get(`dataset/${id}/meta`); @@ -272,6 +299,8 @@ export { saveAttributes, saveAttributeTrackFilters, openFromDisk, + getTiles, + getTileURL, /* Nonstandard APIs */ exportDataset, exportConfiguration, diff --git a/client/platform/desktop/frontend/components/Recent.vue b/client/platform/desktop/frontend/components/Recent.vue index 04d355811..fb43e91fd 100644 --- a/client/platform/desktop/frontend/components/Recent.vue +++ b/client/platform/desktop/frontend/components/Recent.vue @@ -174,6 +174,9 @@ export default defineComponent({ if (recent.type === 'video') { return 'mdi-file-video'; } + if (recent.type === 'large-image') { + return 'mdi-map'; + } if (recent.imageListPath) { return 'mdi-view-list-outline'; } @@ -413,6 +416,13 @@ export default defineComponent({ @open="open($event)" @multi-cam="openMultiCamDialog" /> + diff --git a/client/platform/web-girder/views/Upload.vue b/client/platform/web-girder/views/Upload.vue index 91e6dc47a..0d90e4637 100644 --- a/client/platform/web-girder/views/Upload.vue +++ b/client/platform/web-girder/views/Upload.vue @@ -6,7 +6,7 @@ import { import { ImageSequenceType, VideoType, DefaultVideoFPS, FPSOptions, inputAnnotationFileTypes, websafeVideoTypes, otherVideoTypes, - websafeImageTypes, otherImageTypes, JsonMetaRegEx, largeImageTypes, LargeImageType, + websafeImageTypes, otherImageTypes, JsonMetaRegEx, largeImageTypes, largeImageDesktopTypes, LargeImageType, } from 'dive-common/constants'; import { @@ -70,6 +70,7 @@ export default defineComponent({ const multiCamOpenType = ref('image-sequence'); const importMultiCamDialog = ref(false); const girderUpload: Ref = ref(null); + const isDesktopMode = navigator.userAgent.includes('Electron'); const { prompt } = usePrompt(); const addPendingZipUpload = (name: string, allFiles: File[]) => { @@ -233,6 +234,9 @@ export default defineComponent({ } if (type === 'video') { return websafeVideoTypes.concat(otherVideoTypes); } if (type === 'large-image') { + if (isDesktopMode) { + return largeImageDesktopTypes.map((item) => `.${item}`).join(','); + } return largeImageTypes; } return websafeImageTypes.concat(otherImageTypes); diff --git a/client/src/@types/pngjs.d.ts b/client/src/@types/pngjs.d.ts new file mode 100644 index 000000000..9281ded7c --- /dev/null +++ b/client/src/@types/pngjs.d.ts @@ -0,0 +1,15 @@ +declare module 'pngjs' { + export default class PNG { + width: number; + + height: number; + + data: Buffer; + + constructor(options?: { width?: number; height?: number }); + + static sync: { + write(png: PNG, options?: object): Buffer; + }; + } +} diff --git a/client/src/components/annotators/LargeImageAnnotator.vue b/client/src/components/annotators/LargeImageAnnotator.vue index ff1d96013..4f0b06ca9 100644 --- a/client/src/components/annotators/LargeImageAnnotator.vue +++ b/client/src/components/annotators/LargeImageAnnotator.vue @@ -1,6 +1,6 @@