From a096f7645b3643e065a7c60fda619379f153dabf Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sat, 19 Aug 2023 10:28:07 +0100 Subject: [PATCH 1/5] getMetadata should take appended filepath --- SlideServer.py | 8 +++++--- dev_utils.py | 7 +++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index b3f7d59..594ea2d 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -203,7 +203,7 @@ def testRoute(): @app.route("/data/one/", methods=['GET']) def singleSlide(filepath): extended = request.args.get('extended') - res = dev_utils.getMetadata(filepath, app.config['UPLOAD_FOLDER'], extended) + res = dev_utils.getMetadata(join(app.config['UPLOAD_FOLDER'], filepath), extended) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: @@ -223,8 +223,10 @@ def singleThumb(filepath): @app.route("/data/many/", methods=['GET']) def multiSlide(filepathlist): - request.args.get('extended') - res = dev_utils.getMetadataList(json.loads(filepathlist), app.config['UPLOAD_FOLDER'], extended) + extended = request.args.get('extended') + filenames = json.loads(filepathlist) + paths = [join(app.config['UPLOAD_FOLDER'], filename) for filename in filenames] + res = dev_utils.getMetadataList(paths, extended) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: diff --git a/dev_utils.py b/dev_utils.py index 9180e64..47ca303 100644 --- a/dev_utils.py +++ b/dev_utils.py @@ -10,10 +10,9 @@ # given a path, get metadata -def getMetadata(filename, upload_folder, extended): +def getMetadata(filepath, extended): # TODO consider restricting filepath metadata = {} - filepath = os.path.join(upload_folder, filename) if not os.path.isfile(filepath): msg = {"error": "No such file"} print(msg) @@ -60,10 +59,10 @@ def postslide(img, url, token=''): # given a list of path, get metadata for each -def getMetadataList(filenames, upload_folder, extended): +def getMetadataList(filenames, extended): allData = [] for filename in filenames: - allData.append(getMetadata(filename, upload_folder, extended)) + allData.append(getMetadata(filename, extended)) return allData From 3a3d0a6a82729ae2e0b6c886cbfa5af922e83ddd Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sat, 19 Aug 2023 10:50:21 +0100 Subject: [PATCH 2/5] typo --- NCISlideUtil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NCISlideUtil.py b/NCISlideUtil.py index 78fc370..055ee72 100644 --- a/NCISlideUtil.py +++ b/NCISlideUtil.py @@ -72,7 +72,7 @@ def openslidedata(metadata): slide = openslide.OpenSlide(metadata['location']) slideData = slide.properties metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None) - metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) + metadata['mpp-y'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) metadata['mpp'] = metadata['mpp-x'] or metadata['mpp-x'] or None metadata['height'] = slideData.get( openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) From ddd0a140413d546ea63972b235faa88938fb1283 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sat, 19 Aug 2023 11:17:04 +0100 Subject: [PATCH 3/5] reuse same metadata routes --- OmniLoad.py | 20 +++++--------------- SlideServer.py | 4 ++-- SlideUtil.py | 20 ++++++-------------- dev_utils.py | 10 +++++++--- 4 files changed, 20 insertions(+), 34 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index 32a46ac..0ce0d5d 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -8,6 +8,7 @@ import json # for json in and out import requests # for api and pathdb in and out import hashlib +import dev_utils # for large csv fields, especially segmentations csv.field_size_limit(sys.maxsize) @@ -51,21 +52,10 @@ def file_md5(fileName): def openslidedata(manifest): for img in manifest: img['location'] = img.get("path", "") or img.get("location", "") or img.get("filename", "") or img.get("file", "") - slide = openslide.OpenSlide(img['location']) - slideData = slide.properties - img['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None) - img['mpp-y'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) - img['mpp'] = img['mpp-x'] or img['mpp-y'] - img['height'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) or slideData.get( - "openslide.level[0].height", None) - img['width'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_WIDTH, None) or slideData.get( - "openslide.level[0].width", None) - img['vendor'] = slideData.get(openslide.PROPERTY_NAME_VENDOR, None) - img['level_count'] = int(slideData.get('level_count', 1)) - img['objective'] = float(slideData.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER, 0) or - slideData.get("aperio.AppMag", -1.0)) - img['md5sum'] = file_md5(img['location']) - img['comment'] = slideData.get(openslide.PROPERTY_NAME_COMMENT, None) + metadata = dev_utils.getMetadata(img['location'], False, True) + for k, v in metadata: + if k not in img: + img[k] = v # required values which are often unused img['study'] = img.get('study', "") img['specimen'] = img.get('specimen', "") diff --git a/SlideServer.py b/SlideServer.py index 594ea2d..04576fd 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -203,7 +203,7 @@ def testRoute(): @app.route("/data/one/", methods=['GET']) def singleSlide(filepath): extended = request.args.get('extended') - res = dev_utils.getMetadata(join(app.config['UPLOAD_FOLDER'], filepath), extended) + res = dev_utils.getMetadata(join(app.config['UPLOAD_FOLDER'], filepath), extended, False) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: @@ -226,7 +226,7 @@ def multiSlide(filepathlist): extended = request.args.get('extended') filenames = json.loads(filepathlist) paths = [join(app.config['UPLOAD_FOLDER'], filename) for filename in filenames] - res = dev_utils.getMetadataList(paths, extended) + res = dev_utils.getMetadataList(paths, extended, False) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: diff --git a/SlideUtil.py b/SlideUtil.py index 2017f8b..63b79d1 100644 --- a/SlideUtil.py +++ b/SlideUtil.py @@ -5,7 +5,7 @@ import openslide -from dev_utils import file_md5 +from dev_utils import getMetadata from dev_utils import postslide from dev_utils import post_url @@ -34,22 +34,14 @@ def gen_thumbnail(filename, slide, size, imgtype="png"): def openslidedata(metadata): - slide = openslide.OpenSlide(metadata['location']) - slideData = slide.properties - metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None) - metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) - metadata['mpp'] = metadata['mpp-x'] or metadata['mpp-x'] or None - # metadata['height'] = slideData.get("openslide.level[0].height", None) - # metadata['width'] = slideData.get("openslide.level[0].width", None) - metadata['height'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) - metadata['width'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_WIDTH, None) - metadata['vendor'] = slideData.get(openslide.PROPERTY_NAME_VENDOR, None) - metadata['level_count'] = int(slideData.get('level_count', 1)) - metadata['objective'] = float(slideData.get("aperio.AppMag", 0.0)) - metadata['md5sum'] = file_md5(metadata['location']) + metadata_retrieved = getMetadata(metadata['location'], False, True) + for k, v in metadata_retrieved: + if k not in metadata: + metadata[k] = v metadata['timestamp'] = time.time() thumbnail_size = config.get('thumbnail_size', None) if thumbnail_size: + slide = openslide.OpenSlide(metadata['location']) gen_thumbnail(metadata['location'], slide, thumbnail_size) return metadata diff --git a/dev_utils.py b/dev_utils.py index 47ca303..81d6f3b 100644 --- a/dev_utils.py +++ b/dev_utils.py @@ -10,10 +10,12 @@ # given a path, get metadata -def getMetadata(filepath, extended): +def getMetadata(filepath, extended, raise_exception): # TODO consider restricting filepath metadata = {} if not os.path.isfile(filepath): + if raise_exception: + raise ValueError("No such file") msg = {"error": "No such file"} print(msg) return msg @@ -21,6 +23,8 @@ def getMetadata(filepath, extended): try: slide = openslide.OpenSlide(filepath) except BaseException as e: + if raise_exception: + raise e msg = {"type": "Openslide", "error": str(e)} print(msg) return msg @@ -59,10 +63,10 @@ def postslide(img, url, token=''): # given a list of path, get metadata for each -def getMetadataList(filenames, extended): +def getMetadataList(filenames, extended, raise_exception): allData = [] for filename in filenames: - allData.append(getMetadata(filename, extended)) + allData.append(getMetadata(filename, extended, raise_exception)) return allData From 690283600edc84f32d47972cb765bf5659a4817a Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sat, 19 Aug 2023 11:35:50 +0100 Subject: [PATCH 4/5] fix my typo --- OmniLoad.py | 2 +- SlideUtil.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index 0ce0d5d..d5e3a69 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -53,7 +53,7 @@ def openslidedata(manifest): for img in manifest: img['location'] = img.get("path", "") or img.get("location", "") or img.get("filename", "") or img.get("file", "") metadata = dev_utils.getMetadata(img['location'], False, True) - for k, v in metadata: + for k, v in metadata.items(): if k not in img: img[k] = v # required values which are often unused diff --git a/SlideUtil.py b/SlideUtil.py index 63b79d1..08d4e9f 100644 --- a/SlideUtil.py +++ b/SlideUtil.py @@ -35,7 +35,7 @@ def gen_thumbnail(filename, slide, size, imgtype="png"): def openslidedata(metadata): metadata_retrieved = getMetadata(metadata['location'], False, True) - for k, v in metadata_retrieved: + for k, v in metadata_retrieved.items(): if k not in metadata: metadata[k] = v metadata['timestamp'] = time.time() From d6eac56b839c2daf082d6e72855062e6e14661c7 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Tue, 22 Aug 2023 07:57:34 +0100 Subject: [PATCH 5/5] join -> os.path.join --- SlideServer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index 04576fd..8a2fe5c 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -6,8 +6,7 @@ import string import sys import pyvips -from os import listdir -from os.path import isfile, join +import os from spritemaker import createSpritesheet from PIL import Image import urllib @@ -203,7 +202,7 @@ def testRoute(): @app.route("/data/one/", methods=['GET']) def singleSlide(filepath): extended = request.args.get('extended') - res = dev_utils.getMetadata(join(app.config['UPLOAD_FOLDER'], filepath), extended, False) + res = dev_utils.getMetadata(os.path.join(app.config['UPLOAD_FOLDER'], filepath), extended, False) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: @@ -225,7 +224,7 @@ def singleThumb(filepath): def multiSlide(filepathlist): extended = request.args.get('extended') filenames = json.loads(filepathlist) - paths = [join(app.config['UPLOAD_FOLDER'], filename) for filename in filenames] + paths = [os.path.join(app.config['UPLOAD_FOLDER'], filename) for filename in filenames] res = dev_utils.getMetadataList(paths, extended, False) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500)