diff --git a/medcat-trainer/webapp/api/api/admin/models.py b/medcat-trainer/webapp/api/api/admin/models.py
index 5d44a5a09..50cae265d 100644
--- a/medcat-trainer/webapp/api/api/admin/models.py
+++ b/medcat-trainer/webapp/api/api/admin/models.py
@@ -9,7 +9,7 @@
_PROJECT_ANNO_ENTS_SETTINGS_FIELD_ORDER = (
'concept_db', 'vocab', 'model_pack', 'cdb_search_filter', 'deid_model_annotation', 'require_entity_validation', 'train_model_on_submit',
- 'add_new_entities', 'restrict_concept_lookup', 'terminate_available', 'irrelevant_available',
+ 'use_model_service', 'model_service_url', 'add_new_entities', 'restrict_concept_lookup', 'terminate_available', 'irrelevant_available',
'enable_entity_annotation_comments', 'tasks', 'relations'
)
@@ -177,7 +177,7 @@ class ModelPackAdmin(admin.ModelAdmin):
def metacats(self, obj):
return ", ".join(str(m_c) for m_c in obj.meta_cats.all())
-
+
def save_model(self, request, obj, form, change):
obj.last_modified_by = request.user
super().save_model(request, obj, form, change)
diff --git a/medcat-trainer/webapp/api/api/migrations/0093_add_remote_model_service_fields.py b/medcat-trainer/webapp/api/api/migrations/0093_add_remote_model_service_fields.py
new file mode 100644
index 000000000..bf3a8b760
--- /dev/null
+++ b/medcat-trainer/webapp/api/api/migrations/0093_add_remote_model_service_fields.py
@@ -0,0 +1,33 @@
+# Generated by Django 5.1.11 on 2025-12-11 11:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('api', '0092_exportedproject_cdb_search_filter_id_and_more'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='projectannotateentities',
+ name='model_service_url',
+ field=models.CharField(blank=True, help_text='URL of the remote MedCAT service API (e.g., http://medcat-service:8000)', max_length=500, null=True),
+ ),
+ migrations.AddField(
+ model_name='projectannotateentities',
+ name='use_model_service',
+ field=models.BooleanField(default=False, help_text='Use a remote MedCAT service API for document processing instead of local models'),
+ ),
+ migrations.AddField(
+ model_name='projectgroup',
+ name='model_service_url',
+ field=models.CharField(blank=True, help_text='URL of the remote MedCAT service API (e.g., http://medcat-service:8000)', max_length=500, null=True),
+ ),
+ migrations.AddField(
+ model_name='projectgroup',
+ name='use_model_service',
+ field=models.BooleanField(default=False, help_text='Use a remote MedCAT service API for document processing instead of local models'),
+ ),
+ ]
diff --git a/medcat-trainer/webapp/api/api/models.py b/medcat-trainer/webapp/api/api/models.py
index 7e62cc33b..d8df36223 100644
--- a/medcat-trainer/webapp/api/api/models.py
+++ b/medcat-trainer/webapp/api/api/models.py
@@ -458,15 +458,24 @@ class Meta:
'if a model pack is used for the project')
relations = models.ManyToManyField('Relation', blank=True, default=None,
help_text='Relations that will be available for this project')
+ use_model_service = models.BooleanField(default=False,
+ help_text='Use a remote MedCAT service API for document processing instead of local models'\
+ '(note: interim model training is not supported for remote model service projects)')
+ model_service_url = models.CharField(max_length=500, blank=True, null=True,
+ help_text='URL of the remote MedCAT service API (e.g., http://medcat-service:8003)')
def save(self, *args, **kwargs):
- if self.model_pack is None and (self.concept_db is None or self.vocab is None):
- raise ValidationError('Must set at least the ModelPack or a Concept Database and Vocab Pair')
- if self.model_pack and (self.concept_db is not None or self.vocab is not None):
- raise ValidationError('Cannot set model pack and ConceptDB or a Vocab. You must use one or the other.')
- if self.deid_model_annotation and self.model_pack is None:
- raise ValidationError('Must set a DeID ModelPack for De-ID Model Annotation, cannot only set a cdb / vocab pair as'
- ' not be a DeId model')
+ # If using remote model service, skip local model validation
+ if not self.use_model_service:
+ if self.model_pack is None and (self.concept_db is None or self.vocab is None):
+ raise ValidationError('Must set at least the ModelPack or a Concept Database and Vocab Pair')
+ if self.model_pack and (self.concept_db is not None or self.vocab is not None):
+ raise ValidationError('Cannot set model pack and ConceptDB or a Vocab. You must use one or the other.')
+ if self.deid_model_annotation and self.model_pack is None:
+ raise ValidationError('Must set a DeID ModelPack for De-ID Model Annotation, cannot only set a cdb / vocab pair as'
+ ' not be a DeId model')
+ elif self.use_model_service and not self.model_service_url:
+ raise ValidationError('When using model service, model_service_url must be set')
super().save(*args, **kwargs)
diff --git a/medcat-trainer/webapp/api/api/utils.py b/medcat-trainer/webapp/api/api/utils.py
index f88cb5a9c..dde42cc4f 100644
--- a/medcat-trainer/webapp/api/api/utils.py
+++ b/medcat-trainer/webapp/api/api/utils.py
@@ -3,6 +3,7 @@
import os
from typing import List
+import requests
from background_task import background
from django.contrib.auth.models import User
from django.db import transaction
@@ -20,6 +21,70 @@
logger = logging.getLogger('trainer')
+class RemoteEntity:
+ """A simple class to mimic spaCy entity structure for remote API responses."""
+ def __init__(self, entity_data, text):
+ self.cui = entity_data.get('cui', '')
+ self.start_char_index = entity_data.get('start', 0)
+ self.end_char_index = entity_data.get('end', 0)
+ self.text = entity_data.get('detected_name') or entity_data.get('source_value', '')
+ self.context_similarity = entity_data.get('context_similarity', entity_data.get('acc', 0.0))
+ self._meta_anns = entity_data.get('meta_anns', {})
+ self._text = text
+
+ def get_addon_data(self, key):
+ """Mimic get_addon_data for meta_cat_meta_anns."""
+ if key == 'meta_cat_meta_anns':
+ return self._meta_anns
+ return {}
+
+
+class RemoteSpacyDoc:
+ """A simple class to mimic spaCy document structure for remote API responses."""
+ def __init__(self, linked_ents):
+ self.linked_ents = linked_ents
+
+
+def call_remote_model_service(service_url, text):
+ """
+ Call the remote MedCAT service API to process text.
+
+ Args:
+ service_url: Base URL of the remote service (e.g., http://medcat-service:8000)
+ text: Text to process
+
+ Returns:
+ RemoteSpacyDoc object with linked_ents
+ """
+ # Ensure service_url doesn't end with /
+ service_url = service_url.rstrip('/')
+ api_url = f"{service_url}/api/process"
+
+ payload = {
+ "text": text
+ }
+
+ try:
+ response = requests.post(api_url, json=payload, timeout=60)
+ response.raise_for_status()
+ result = response.json()
+
+ # Extract entities from the response
+ entities_data = result.get('entities', {})
+ linked_ents = []
+
+ for _, entity_data in entities_data.items():
+ linked_ents.append(RemoteEntity(entity_data, text))
+
+ return RemoteSpacyDoc(linked_ents)
+ except requests.exceptions.RequestException as e:
+ logger.error(f"Error calling remote model service at {api_url}: {e}")
+ raise Exception(f"Failed to call remote model service: {str(e)}") from e
+ except Exception as e:
+ logger.error(f"Error processing remote model service response: {e}")
+ raise Exception(f"Failed to process remote model service response: {str(e)}") from e
+
+
def remove_annotations(document, project, partial=False):
try:
if partial:
@@ -36,7 +101,27 @@ def remove_annotations(document, project, partial=False):
logger.debug(f"Something went wrong: {e}")
-def add_annotations(spacy_doc, user, project, document, existing_annotations, cat):
+class SimpleFilters:
+ """Simple filter object for remote service when cat is not available."""
+ def __init__(self, cuis=None, cuis_exclude=None):
+ self.cuis = cuis or set()
+ self.cuis_exclude = cuis_exclude or set()
+
+
+def add_annotations(spacy_doc, user, project, document, existing_annotations, cat=None, filters=None, similarity_threshold=0.3):
+ """
+ Add annotations from spacy_doc to the database.
+
+ Args:
+ spacy_doc: spaCy document with linked_ents or RemoteSpacyDoc
+ user: User object
+ project: ProjectAnnotateEntities object
+ document: Document object
+ existing_annotations: List of existing AnnotatedEntity objects
+ cat: CAT object (optional, required if filters not provided)
+ filters: SimpleFilters object (optional, used when cat is None)
+ similarity_threshold: float (optional, default 0.3, used when cat is None)
+ """
spacy_doc.linked_ents.sort(key=lambda x: len(x.text), reverse=True)
tkns_in = []
@@ -57,9 +142,13 @@ def add_annotations(spacy_doc, user, project, document, existing_annotations, ca
metataskvals2obj = {}
pass
- def check_ents(ent):
- return any((ea[0] < ent.start_char_index < ea[1]) or
- (ea[0] < ent.end_char_index < ea[1]) for ea in existing_annos_intervals)
+ # Get filters and similarity threshold
+ if cat is not None:
+ filters_obj = cat.config.components.linking.filters
+ MIN_ACC = cat.config.components.linking.similarity_threshold
+ else:
+ filters_obj = filters or SimpleFilters()
+ MIN_ACC = similarity_threshold
def check_filters(cui, filters):
if cui in filters.cuis or not filters.cuis:
@@ -68,15 +157,8 @@ def check_filters(cui, filters):
return False
for ent in spacy_doc.linked_ents:
- if not check_ents(ent) and check_filters(ent.cui, cat.config.components.linking.filters):
- to_add = True
- for tkn in ent:
- if tkn in tkns_in:
- to_add = False
- if to_add:
- for tkn in ent:
- tkns_in.append(tkn)
- ents.append(ent)
+ if check_filters(ent.cui, filters_obj):
+ ents.append(ent)
logger.debug('Found %s annotations to store', len(ents))
for ent in ents:
@@ -93,6 +175,7 @@ def check_filters(cui, filters):
ann_ent = AnnotatedEntity.objects.filter(project=project,
document=document,
+ entity=entity,
start_ind=ent.start_char_index,
end_ind=ent.end_char_index).first()
if ann_ent is None:
@@ -107,7 +190,6 @@ def check_filters(cui, filters):
ann_ent.end_ind = ent.end_char_index
ann_ent.acc = ent.context_similarity
- MIN_ACC = cat.config.components.linking.similarity_threshold
if ent.context_similarity < MIN_ACC:
ann_ent.deleted = True
ann_ent.validated = True
@@ -157,7 +239,7 @@ def get_create_cdb_infos(cdb, concept, cui, cui_info_prop, code_prop, desc_prop,
def create_annotation(source_val: str, selection_occurrence_index: int, cui: str, user: User,
- project: ProjectAnnotateEntities, document, cat: CAT):
+ project: ProjectAnnotateEntities, document: Document):
text = document.text
id = None
@@ -251,29 +333,58 @@ def prep_docs(project_id: List[int], doc_ids: List[int], user_id: int):
project = ProjectAnnotateEntities.objects.get(id=project_id)
docs = Document.objects.filter(id__in=doc_ids)
- logger.info('Loading CAT object in bg process for project: %s', project.id)
- cat = get_medcat(project=project)
-
- # Set CAT filters
- cat.config.components.linking.filters.cuis = project.cuis
-
- for doc in docs:
- logger.info(f'Running MedCAT model for project {project.id}:{project.name} over doc: {doc.id}')
- if not project.deid_model_annotation:
- spacy_doc = cat(doc.text)
- else:
- deid = DeIdModel(cat)
- spacy_doc = deid(doc.text)
- anns = AnnotatedEntity.objects.filter(document=doc).filter(project=project)
- with transaction.atomic():
- add_annotations(spacy_doc=spacy_doc,
- user=user,
- project=project,
- document=doc,
- cat=cat,
- existing_annotations=anns)
- # add doc to prepared_documents
- project.prepared_documents.add(doc)
+ # Get CUI filters
+ cuis = set()
+ if project.cuis is not None and project.cuis:
+ cuis = set([str(cui).strip() for cui in project.cuis.split(",")])
+ if project.cuis_file is not None and project.cuis_file:
+ try:
+ cuis.update(json.load(open(project.cuis_file.path)))
+ except FileNotFoundError:
+ logger.warning('Missing CUI filter file for project %s', project.id)
+
+ if project.use_model_service:
+ # Use remote model service
+ logger.info('Using remote model service in bg process for project: %s', project.id)
+ filters = SimpleFilters(cuis=cuis)
+ for doc in docs:
+ logger.info('Running remote MedCAT service for project %s:%s over doc: %s', project.id, project.name, doc.id)
+ spacy_doc = call_remote_model_service(project.model_service_url, doc.text)
+ anns = AnnotatedEntity.objects.filter(document=doc).filter(project=project)
+ with transaction.atomic():
+ add_annotations(spacy_doc=spacy_doc,
+ user=user,
+ project=project,
+ document=doc,
+ cat=None,
+ filters=filters,
+ similarity_threshold=0.3,
+ existing_annotations=anns)
+ project.prepared_documents.add(doc)
+ else:
+ # Use local medcat model
+ logger.info('Loading CAT object in bg process for project: %s', project.id)
+ cat = get_medcat(project=project)
+
+ # Set CAT filters
+ cat.config.components.linking.filters.cuis = cuis
+
+ for doc in docs:
+ logger.info('Running MedCAT model for project %s:%s over doc: %s', project.id, project.name, doc.id)
+ if not project.deid_model_annotation:
+ spacy_doc = cat(doc.text)
+ else:
+ deid = DeIdModel(cat)
+ spacy_doc = deid(doc.text)
+ anns = AnnotatedEntity.objects.filter(document=doc).filter(project=project)
+ with transaction.atomic():
+ add_annotations(spacy_doc=spacy_doc,
+ user=user,
+ project=project,
+ document=doc,
+ cat=cat,
+ existing_annotations=anns)
+ project.prepared_documents.add(doc)
project.save()
logger.info('Prepared all docs for project: %s, docs processed: %s',
project.id, project.prepared_documents)
diff --git a/medcat-trainer/webapp/api/api/views.py b/medcat-trainer/webapp/api/api/views.py
index c079e7547..f9d9e0d36 100644
--- a/medcat-trainer/webapp/api/api/views.py
+++ b/medcat-trainer/webapp/api/api/views.py
@@ -278,27 +278,40 @@ def prepare_documents(request):
with transaction.atomic():
# If the document is not already annotated, annotate it
if (len(anns) == 0 and not is_validated) or update:
- # Based on the project id get the right medcat
- cat = get_medcat(project=project)
- logger.info('loaded medcat model for project: %s', project.id)
-
- # Set CAT filters
- cat.config.components.linking.filters.cuis = cuis
-
- if not project.deid_model_annotation:
- spacy_doc = cat(document.text)
+ if project.use_model_service:
+ # Use remote model service
+ logger.info('Using remote model service for project: %s', project.id)
+ from .utils import call_remote_model_service, SimpleFilters
+ spacy_doc = call_remote_model_service(project.model_service_url, document.text)
+ filters = SimpleFilters(cuis=cuis)
+ add_annotations(spacy_doc=spacy_doc,
+ user=user,
+ project=project,
+ document=document,
+ cat=None,
+ filters=filters,
+ similarity_threshold=0.3,
+ existing_annotations=anns)
else:
- deid = DeIdModel(cat)
- spacy_doc = deid(document.text)
-
- spacy_doc = cat(document.text)
-
- add_annotations(spacy_doc=spacy_doc,
- user=user,
- project=project,
- document=document,
- cat=cat,
- existing_annotations=anns)
+ # Use local medcat model
+ cat = get_medcat(project=project)
+ logger.info('loaded medcat model for project: %s', project.id)
+
+ # Set CAT filters
+ cat.config.components.linking.filters.cuis = cuis
+
+ if not project.deid_model_annotation:
+ spacy_doc = cat(document.text)
+ else:
+ deid = DeIdModel(cat)
+ spacy_doc = deid(document.text)
+
+ add_annotations(spacy_doc=spacy_doc,
+ user=user,
+ project=project,
+ document=document,
+ cat=cat,
+ existing_annotations=anns)
# add doc to prepared_documents
project.prepared_documents.add(document)
@@ -382,15 +395,12 @@ def add_annotation(request):
user = request.user
project = ProjectAnnotateEntities.objects.get(id=p_id)
document = Document.objects.get(id=d_id)
-
- cat = get_medcat(project=project)
id = create_annotation(source_val=source_val,
selection_occurrence_index=sel_occur_idx,
cui=cui,
user=user,
project=project,
- document=document,
- cat=cat)
+ document=document)
logger.debug('Annotation added.')
return Response({'message': 'Annotation added successfully', 'id': id})
@@ -414,6 +424,14 @@ def add_concept(request):
user = request.user
project = ProjectAnnotateEntities.objects.get(id=p_id)
document = Document.objects.get(id=d_id)
+
+ if project.use_model_service:
+ # Use remote model service
+ logger.error('Adding concepts is not supported for remote model service'\
+ 'projects, you likely want to use a local model')
+ raise NotImplementedError('Adding concepts is not supported for remote model service projects')
+
+
cat = get_medcat(project=project)
if cui in cat.cdb.cui2names:
@@ -463,8 +481,13 @@ def import_cdb_concepts(request):
def _submit_document(project: ProjectAnnotateEntities, document: Document):
if project.train_model_on_submit:
- cat = get_medcat(project=project)
- train_medcat(cat, project, document)
+ if project.use_model_service:
+ # TODO: Implement this, already available in CMS / gateway instances.
+ # interim model training not supported for remote model service projects
+ logger.warning('Interim model training is not supported for remote model service projects')
+ else:
+ cat = get_medcat(project=project)
+ train_medcat(cat, project, document)
# Add cuis to filter if they did not exist
cuis = []
diff --git a/medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue b/medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue
index 667c80c86..61872d4d1 100644
--- a/medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue
+++ b/medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue
@@ -66,7 +66,8 @@ export default {
name: 'Add Term'
}
],
- selection: null
+ selection: null,
+ openPopoverId: null // Track which popover is open
}
},
computed: {
@@ -84,7 +85,12 @@ export default {
if (a.ent.start_ind !== b.ent.start_ind) {
return a.ent.start_ind - b.ent.start_ind
}
- return b.ent.end_ind - a.ent.end_ind // Longer spans first when same start
+ if (a.ent.end_ind !== b.ent.end_ind) {
+ return b.ent.end_ind - a.ent.end_ind // Longer spans first when same start
+ }
+ // For exactly overlapping annotations (same start and end), use original index
+ // as tiebreaker to ensure stable, consistent ordering
+ return a.origIdx - b.origIdx
})
const taskHighlightDefault = 'highlight-task-default'
@@ -106,12 +112,45 @@ export default {
if (a.type !== b.type) {
return a.type === 'start' ? -1 : 1
}
- return 0
+ // For events at same position and same type (exact overlaps),
+ // use entIndex as tiebreaker to ensure stable, consistent ordering
+ // For starts: open in order (lower index first)
+ // For ends: close in reverse order (higher index first) to maintain proper nesting
+ if (a.type === 'start') {
+ return a.entIndex - b.entIndex
+ } else {
+ return b.entIndex - a.entIndex
+ }
+ })
+
+ // Pre-compute overlapping groups: for each annotation, find all annotations that overlap with it
+ const overlappingGroups = new Map() // Map from origIndex to set of all overlapping origIndices
+ sortedEnts.forEach((entData1, i1) => {
+ const origIdx1 = entData1.origIdx
+ const start1 = entData1.ent.start_ind
+ const end1 = entData1.ent.end_ind
+ const group = new Set([origIdx1])
+
+ sortedEnts.forEach((entData2, i2) => {
+ if (i1 === i2) return
+ const start2 = entData2.ent.start_ind
+ const end2 = entData2.ent.end_ind
+ // Check if annotations overlap (they overlap if one starts before the other ends)
+ if (!(end1 <= start2 || end2 <= start1)) {
+ group.add(entData2.origIdx)
+ }
+ })
+
+ // Store sorted array of indices for consistent ID generation
+ const sortedGroup = Array.from(group).sort((a, b) => a - b)
+ overlappingGroups.set(origIdx1, sortedGroup)
})
let formattedText = ''
let currentPos = 0
const activeEnts = [] // Stack of active entities (ordered by when they were opened)
+ const createdPopovers = new Set() // Track which popover IDs have been created to avoid duplicates
+ const createdBadges = new Set() // Track which popover IDs have had badges created to avoid multiple badges
// Helper function to get style class for an entity
const getStyleClass = (ent, origIndex) => {
@@ -140,13 +179,48 @@ export default {
return ``
}
- // Helper function to build closing span tag with optional remove button
- const buildCloseSpan = (ent, origIndex, isInnermost) => {
+ // Helper function to build closing span tag with optional remove button and overlap indicator
+ const buildCloseSpan = (ent, origIndex, isInnermost, overlappingEnts = []) => {
let removeButtonEl = ''
if (isInnermost && ent.manually_created) {
removeButtonEl = `