From e18eabee0d6f3123689a9122a976de4457f2c789 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 9 Feb 2024 13:57:22 -0800 Subject: [PATCH 01/35] Default to including animals with status = Unknown in MCC ETL --- mcc/resources/queries/mcc/aggregatedDemographics.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcc/resources/queries/mcc/aggregatedDemographics.sql b/mcc/resources/queries/mcc/aggregatedDemographics.sql index 1a059ca63..cb0de39a9 100644 --- a/mcc/resources/queries/mcc/aggregatedDemographics.sql +++ b/mcc/resources/queries/mcc/aggregatedDemographics.sql @@ -39,7 +39,7 @@ LEFT JOIN (SELECT o."medical_history::observation" as medical_history, FROM "/data/Colonies/SNPRC/".study.mostRecentObservationsPivoted o ) o ON (o.Id = d.Id) -WHERE (d.excludeFromCensus IS NULL or d.excludeFromCensus = false) and d.calculated_status NOT IN ('Unknown', 'Other') +WHERE (d.excludeFromCensus IS NULL or d.excludeFromCensus = false) and d.calculated_status NOT IN ('Other') UNION ALL From b150665dd12b8f80ed3c8b8e388a7cd13fe1d842 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 12 Feb 2024 14:02:12 -0800 Subject: [PATCH 02/35] mGAP demographics table should use mGAP-specific table --- mGAP/resources/views/demographics.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mGAP/resources/views/demographics.html b/mGAP/resources/views/demographics.html index d1f38e4d2..8ed56996f 100644 --- a/mGAP/resources/views/demographics.html +++ b/mGAP/resources/views/demographics.html @@ -8,7 +8,7 @@ LDK.Utils.getReadOnlyQWP({ frame: 'none', title: 'mGAP Demographics', - schemaName: 'study', + schemaName: 'mgap', queryName: 'demographics', showRecordSelectors: false, showDetailsColumn: false From d17ddf980024012f520e6d6838c79df3fe7b8eed Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 14 Feb 2024 13:05:52 -0800 Subject: [PATCH 03/35] Enhance MCC census import --- mcc/resources/queries/study/deaths.query.xml | 4 +- .../queries/study/mostRecentObservations.sql | 4 +- mcc/resources/views/mccColony.html | 3 + mcc/resources/web/mcc/panel/MccImportPanel.js | 166 ++++++++++++++++-- 4 files changed, 161 insertions(+), 16 deletions(-) diff --git a/mcc/resources/queries/study/deaths.query.xml b/mcc/resources/queries/study/deaths.query.xml index 25e309e6e..06a89d4ce 100644 --- a/mcc/resources/queries/study/deaths.query.xml +++ b/mcc/resources/queries/study/deaths.query.xml @@ -15,11 +15,12 @@ Time of Death - Type of Death + Cause of Death ehr_lookups death_cause value + @@ -28,6 +29,7 @@ ehr_lookups death_manner value + diff --git a/mcc/resources/queries/study/mostRecentObservations.sql b/mcc/resources/queries/study/mostRecentObservations.sql index ce43020e1..5aaa53c41 100644 --- a/mcc/resources/queries/study/mostRecentObservations.sql +++ b/mcc/resources/queries/study/mostRecentObservations.sql @@ -16,12 +16,12 @@ FROM study.clinical_observations co WHERE co.qcstate.publicdata = true and co.observation is not null AND - co.category IN ('Medical History', 'Fertility Status', 'Infant History', 'Current Housing Status', 'Availability') AND + co.category IN ('Medical History', 'Fertility Status', 'Infant History', 'Current Housing Status', 'Availability', 'Usage_Current', 'Usage_Future') AND co.date = (SELECT max(o.date) asMaxDate FROM study.clinical_observations o WHERE o.Id = co.Id AND o.qcstate.publicdata = true AND - o.category IN ('Medical History', 'Fertility Status', 'Infant History', 'Current Housing Status', 'Availability') AND + o.category IN ('Medical History', 'Fertility Status', 'Infant History', 'Current Housing Status', 'Availability', 'Usage_Current', 'Usage_Future') AND o.observation is not null ) GROUP BY co.id, co.category, co.date diff --git a/mcc/resources/views/mccColony.html b/mcc/resources/views/mccColony.html index 8006ae21b..03617f758 100644 --- a/mcc/resources/views/mccColony.html +++ b/mcc/resources/views/mccColony.html @@ -71,6 +71,9 @@ },{ name: 'Parent IDs Assigned to Multiple MCC IDs', url: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'mcc', queryName: 'duplicatedAggregatedDemographicsParents'}) + },{ + name: 'Aggregated Demographics (All Centers)', + url: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'mcc', queryName: 'aggregatedDemographics'}) },{ name: 'Pedigree Export', url: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'mcc', queryName: 'mccPedigree'}) diff --git a/mcc/resources/web/mcc/panel/MccImportPanel.js b/mcc/resources/web/mcc/panel/MccImportPanel.js index 5b4fa157c..f58dddccc 100644 --- a/mcc/resources/web/mcc/panel/MccImportPanel.js +++ b/mcc/resources/web/mcc/panel/MccImportPanel.js @@ -1,5 +1,6 @@ Ext4.define('MCC.panel.MccImportPanel', { extend: 'Ext.panel.Panel', + alias: 'widget.mcc-mccimportpanel', COLUMNS: [{ name: 'existingRecord', @@ -14,6 +15,18 @@ Ext4.define('MCC.panel.MccImportPanel', { alwaysShow: true, allowBlank: false, transform: 'center' + },{ + name: 'death', + labels: ['Death', 'dod', 'DOD (mm/dd/yyyy)'], + allowRowSpan: false, + alwaysShow: true, + allowBlank: true + },{ + name: 'causeOfDeath', + labels: ['Cause of Death'], + allowRowSpan: false, + alwaysShow: true, + allowBlank: true },{ name: 'Id', labels: ['Id', 'animal ID', 'AnimalId', 'MarmId', 'Marm Id'], @@ -34,6 +47,12 @@ Ext4.define('MCC.panel.MccImportPanel', { allowRowSpan: false, allowBlank: true, alwaysShow: true + },{ + name: 'shippingDestination', + labels: ['Shipping Destination'], + allowRowSpan: false, + allowBlank: true, + alwaysShow: true },{ name: 'species', labels: ['Species'], @@ -58,16 +77,17 @@ Ext4.define('MCC.panel.MccImportPanel', { labels: ['Status'], allowRowSpan: false, allowBlank: true, - alwaysShow: true + alwaysShow: true, + transform: 'status' },{ name: 'dam', - labels: ['Dam', 'maternal ID'], + labels: ['Dam', 'maternal ID', 'maternal ID (previous)'], allowRowSpan: false, allowBlank: true, transform: 'damOrSire' },{ name: 'sire', - labels: ['Sire', 'paternal ID'], + labels: ['Sire', 'paternal ID', 'paternal ID (previous)'], allowRowSpan: false, allowBlank: true, transform: 'damOrSire' @@ -79,7 +99,7 @@ Ext4.define('MCC.panel.MccImportPanel', { transform: 'weight' },{ name: 'weightDate', - labels: ['Date of Weight', 'date of weight'], + labels: ['Date of Weight', 'date of weight', 'date of weight (mm/dd/yyyy)'], alwaysShow: true, allowRowSpan: false, transform: 'genericDate', @@ -100,10 +120,15 @@ Ext4.define('MCC.panel.MccImportPanel', { transform: 'u24' },{ name: 'availability', - labels: ['Available to Transfer', 'availalble to transfer'], + labels: ['Available to Transfer', 'available to transfer'], allowRowSpan: false, allowBlank: true, transform: 'available' + },{ + name: 'breedingPartnerId', + labels: ['Breeding Partner Id'], + allowRowSpan: false, + allowBlank: true },{ name: 'housingStatus', labels: ['Current Housing Status'], @@ -128,6 +153,23 @@ Ext4.define('MCC.panel.MccImportPanel', { allowRowSpan: false, allowBlank: true, transform: 'medicalHistory' + },{ + name: 'currentUsage', + labels: ['Usage (Current)', 'usage (current)'], + allowRowSpan: false, + allowBlank: true, + transform: 'usage' + },{ + name: 'futureUsage', + labels: ['Usage (Future)', 'usage (future)'], + allowRowSpan: false, + allowBlank: true, + transform: 'usage' + },{ + name: 'studyNames', + labels: ['Study Names'], + allowRowSpan: false, + allowBlank: true },{ name: 'errors', labels: ['Warnings/Errors'], @@ -140,7 +182,7 @@ Ext4.define('MCC.panel.MccImportPanel', { stripLeadingNumbers: function(val) { if (val) { - val = val.replace(/^[0-9]( )+-( )+/, ''); + val = val.replace(/^[0-9]( )*[=-]( )*/, ''); } return val; @@ -163,6 +205,14 @@ Ext4.define('MCC.panel.MccImportPanel', { }, transforms: { + animalId: function(val) { + if (val) { + val = val.replace(/^\s+|\s+$/gm,''); + } + + return val; + }, + species: function(val, panel, row) { return val || 'CJ'; }, @@ -206,6 +256,13 @@ Ext4.define('MCC.panel.MccImportPanel', { return(val); }, + status: function(val, panel, row) { + val = panel.stripLeadingNumbers(val); + val = panel.enforceAllowableValues(val, ['Alive', 'Dead', 'Shipped', 'Unknown'], row); + + return(val); + }, + available: function(val, panel, row) { val = panel.stripLeadingNumbers(val); val = panel.enforceAllowableValues(val, ['not available for transfer', 'available for transfer'], row); @@ -215,7 +272,10 @@ Ext4.define('MCC.panel.MccImportPanel', { housingStatus: function(val, panel, row) { val = panel.stripLeadingNumbers(val); - val = panel.enforceAllowableValues(val, ['singly housed', 'natal family group', 'active breeding', 'social non breeding'], row); + if (val === 'social non breeding') { + val = 'social non-breeding'; + } + val = panel.enforceAllowableValues(val, ['singly housed', 'natal family unit', 'active breeding', 'social non-breeding'], row); return(val); }, @@ -229,7 +289,17 @@ Ext4.define('MCC.panel.MccImportPanel', { medicalHistory: function(val, panel, row) { val = panel.stripLeadingNumbers(val); - val = panel.enforceAllowableValues(val, ['naive animal', 'animal assigned to invasive study'], row); + if (val) { + val = val.replaceAll('naïve', 'naive', 'i'); + } + val = panel.enforceAllowableValues(val, ['naive animal', 'assigned to invasive study'], row); + + return(val); + }, + + usage: function(val, panel, row) { + val = panel.stripLeadingNumbers(val); + val = panel.enforceAllowableValues(val, ['breeding', 'experiment', 'TBD', 'other'], row); return(val); }, @@ -371,7 +441,7 @@ Ext4.define('MCC.panel.MccImportPanel', { LABKEY.Query.selectRows({ schemaName: 'study', queryName: 'demographics', - columns: 'Id,alternateIds,dam,sire,birth,death,colony,objectid,lsid,mccAlias/externalId', + columns: 'Id,alternateIds,dam,sire,birth,death,colony,objectid,lsid,mccAlias/externalId,Id/death/date,Id/MostRecentDeparture/MostRecentDeparture', scope: this, failure: LDK.Utils.getErrorCallback(), success: function(results) { @@ -434,6 +504,14 @@ Ext4.define('MCC.panel.MccImportPanel', { } } + console.log(existingRecord) + if (existingRecord['Id/death/date']) { + row.existingDeathRecord = true; + } + + if (existingRecord['Id/MostRecentDeparture/MostRecentDeparture']) { + row.existingDepartureRecord = true; + } } //TODO: look for alternateIds? @@ -661,7 +739,11 @@ Ext4.define('MCC.panel.MccImportPanel', { panel: this } }], - columns: columns + columns: columns, + initComplete: function(){ + var width = jQuery(id).width(); + Ext4.ComponentQuery.query('mcc-mccimportpanel')[0].setWidth(width + 100); + } }); previewArea.doLayout(); @@ -976,8 +1058,12 @@ Ext4.define('MCC.panel.MccImportPanel', { var observationInserts = []; var weightInserts = []; + var deathInserts = []; + var departureInserts = [] Ext4.Array.forEach(rawData, function(row){ + // TODO: check status and update death/departure + if (row.existingRecord) { demographicsUpdates.push({ Id: row.Id, @@ -987,7 +1073,7 @@ Ext4.define('MCC.panel.MccImportPanel', { dam: row.dam, sire: row.sire, alternateIds: row.alternateIds, - colony: row.colony, + colony: row.shippingDestination || row.colony, source: row.source, gender: row.gender, u24_status: row.u24_status, @@ -1005,7 +1091,7 @@ Ext4.define('MCC.panel.MccImportPanel', { dam: row.dam, sire: row.sire, alternateIds: row.alternateIds, - colony: row.colony, + colony: row.shippingDestination || row.colony, source: row.source, gender: row.gender, u24_status: row.u24_status, @@ -1026,12 +1112,46 @@ Ext4.define('MCC.panel.MccImportPanel', { }); } + if (row.death) { + if (row.existingDeathRecord) { + console.log('Death record already exists: ' + row.Id); + } + else { + deathInserts.push({ + Id: row.Id, + date: row.death, + cause: row.causeOfDeath, + objectId: LABKEY.Utils.generateUUID().toUpperCase(), + QCStateLabel: 'Completed' + }); + } + } + + if (row.shippingDestination) { + if (row.existingDepartureRecord) { + console.log('Departure record already exists: ' + row.Id); + } + else { + departureInserts.push({ + Id: row.Id, + date: row.date, + destination: row.shippingDestination, + objectId: LABKEY.Utils.generateUUID().toUpperCase(), + QCStateLabel: 'Completed' + }); + } + } + var obsFieldMap = { availability: 'Availability', housingStatus: 'Current Housing Status', fertilityStatus: 'Fertility Status', infantHistory: 'Infant History', - medicalHistory: 'Medical History' + medicalHistory: 'Medical History', + currentUsage: 'Usage_Current', + futureUsage: 'Usage_Future', + breedingPartnerId: 'Breeding Partner Id', + studyNames: 'Study Names' } Ext4.Array.forEach(Ext4.Object.getKeys(obsFieldMap), function(fn){ @@ -1085,6 +1205,24 @@ Ext4.define('MCC.panel.MccImportPanel', { }); } + if (deathInserts.length) { + commands.push({ + command: 'insert', + schemaName: 'study', + queryName: 'deaths', + rows: deathInserts + }); + } + + if (departureInserts.length) { + commands.push({ + command: 'insert', + schemaName: 'study', + queryName: 'departures', + rows: departureInserts + }); + } + LABKEY.Query.saveRows({ commands: commands, scope: this, @@ -1094,6 +1232,8 @@ Ext4.define('MCC.panel.MccImportPanel', { '
Demographic Records Created: ' + demographicsInserts.length + '
Demographic Records Updated: ' + demographicsUpdates.length + '
Weight Records Created: ' + weightInserts.length + + '
Death Records Created: ' + deathInserts.length + + '
Departure Records Created: ' + departureInserts.length + '
Observation Records Created: ' + observationInserts.length, function(){ window.location = LABKEY.ActionURL.buildURL('project', 'begin.view'); }, this); From 68f0a0f9bd7de22f9dc75fd502262d567c1be393 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 14 Feb 2024 13:06:29 -0800 Subject: [PATCH 04/35] Enhance MCC census import --- mcc/resources/web/mcc/panel/MccImportPanel.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/mcc/resources/web/mcc/panel/MccImportPanel.js b/mcc/resources/web/mcc/panel/MccImportPanel.js index f58dddccc..faee7cdcb 100644 --- a/mcc/resources/web/mcc/panel/MccImportPanel.js +++ b/mcc/resources/web/mcc/panel/MccImportPanel.js @@ -1062,8 +1062,6 @@ Ext4.define('MCC.panel.MccImportPanel', { var departureInserts = [] Ext4.Array.forEach(rawData, function(row){ - // TODO: check status and update death/departure - if (row.existingRecord) { demographicsUpdates.push({ Id: row.Id, From 88bba9aaa5e7b346fcd0ba8482fd8ae897af9862 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 15 Feb 2024 13:02:10 -0800 Subject: [PATCH 05/35] Update MccTest data --- mcc/test/src/org/labkey/test/tests/mcc/MccTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java index fdf94d2aa..b1da3960d 100644 --- a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java +++ b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java @@ -68,9 +68,9 @@ public void testMccModule() throws Exception testAnimalImportAndTransfer(); } - private static final String ANIMAL_DATA_HEADER = "animal ID\tprevious IDs\tsource\t\"DOB\n(MM/DD/YYYY)\"\tsex\tmaternal ID\tpaternal ID\t\"weight(grams)\"\t\"date of weight\n(MM/DD/YY)\"\tU24 status\tavailalble to transfer\tcurrent housing status\tinfant history\tfertility status\tmedical history\n"; + private static final String ANIMAL_DATA_HEADER = "animal ID\tprevious IDs\tsource\t\"DOB\n(MM/DD/YYYY)\"\tsex\tmaternal ID\tpaternal ID\t\"weight(grams)\"\t\"date of weight\n(MM/DD/YY)\"\tU24 status\tavailable to transfer\tcurrent housing status\tinfant history\tfertility status\tmedical history\n"; - private static final String ANIMAL_DATA1 = "12345\t\t\t7/10/2011\t0 - male\t23456\t23453\t382.8\t5/19/2021\t0 - not assigned to U24 breeding colony\t0 - not available for transfer\t1 - natal family group\t3 - successful rearing of offspring\t2 - successful offspring produced\t0 - naive animal\n"; + private static final String ANIMAL_DATA1 = "12345\t\t\t7/10/2011\t0 - male\t23456\t23453\t382.8\t5/19/2021\t0 - not assigned to U24 breeding colony\t0 - not available for transfer\t1 - natal family unit\t3 - successful rearing of offspring\t2 - successful offspring produced\t0 - naive animal\n"; private static final String ANIMAL_DATA2 = "Animal2\t\t\t6/3/2015\t1 - female\tDam2\tSire2\t361.2\t1/28/2021\t0 - not assigned to U24 breeding colony\t0 - not available for transfer\t2 - active breeding\t3 - successful rearing of offspring\t2 - successful offspring produced\t0 - naive animal\n"; From 37e8a6edea1176fd3d2f35e9e0af349730a4509d Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 16 Feb 2024 10:43:56 -0800 Subject: [PATCH 06/35] Allow sex=unknown in MCC --- mcc/resources/web/mcc/panel/MccImportPanel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcc/resources/web/mcc/panel/MccImportPanel.js b/mcc/resources/web/mcc/panel/MccImportPanel.js index faee7cdcb..972dbb7bc 100644 --- a/mcc/resources/web/mcc/panel/MccImportPanel.js +++ b/mcc/resources/web/mcc/panel/MccImportPanel.js @@ -251,7 +251,7 @@ Ext4.define('MCC.panel.MccImportPanel', { sex: function(val, panel, row) { val = panel.stripLeadingNumbers(val); - val = panel.enforceAllowableValues(val, ['male', 'female'], row); + val = panel.enforceAllowableValues(val, ['male', 'female', 'unknown'], row); return(val); }, From 832950e45c06b2420f1ab5cac3ecffc5873bb321 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 22 Feb 2024 10:14:33 -0800 Subject: [PATCH 07/35] Add more validation checks to MccMaintenanceTask --- .../labkey/mgap/jbrowse/mGAPFieldCustomizer.java | 2 +- mcc/src/org/labkey/mcc/MccMaintenanceTask.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/mGAP/src/org/labkey/mgap/jbrowse/mGAPFieldCustomizer.java b/mGAP/src/org/labkey/mgap/jbrowse/mGAPFieldCustomizer.java index b1ab44524..679d967ac 100644 --- a/mGAP/src/org/labkey/mgap/jbrowse/mGAPFieldCustomizer.java +++ b/mGAP/src/org/labkey/mgap/jbrowse/mGAPFieldCustomizer.java @@ -92,7 +92,7 @@ public List getPromotedFilters(Collection indexedFields, Contain List ret = new ArrayList<>(); if (indexedFields.contains("IMPACT")) { - ret.add("Protein Coding Variants|IMPACT,does not equal,MODIFIER"); + ret.add("Protein Coding Variants|IMPACT,does not equal,MODIFIER&IMPACT,is not empty,"); } if (indexedFields.contains("OG")) diff --git a/mcc/src/org/labkey/mcc/MccMaintenanceTask.java b/mcc/src/org/labkey/mcc/MccMaintenanceTask.java index 1d000685c..4db130712 100644 --- a/mcc/src/org/labkey/mcc/MccMaintenanceTask.java +++ b/mcc/src/org/labkey/mcc/MccMaintenanceTask.java @@ -1,14 +1,18 @@ package org.labkey.mcc; import org.apache.logging.log4j.Logger; +import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.ldk.LDKService; import org.labkey.api.module.ModuleLoader; +import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryService; import org.labkey.api.security.User; +import org.labkey.api.study.Dataset; import org.labkey.api.study.Study; import org.labkey.api.study.StudyService; import org.labkey.api.util.SystemMaintenance; @@ -81,6 +85,16 @@ private void checkForDuplicateAliases(Logger log) { log.error("Duplicate MCC aliases found in the folder: " + s.getContainer().getPath() + ". Please load the query mcc/duplicateAliases for more detail"); } + + for (Dataset ds : s.getDatasets()) + { + TableInfo dsTableInfo = ds.getTableInfo(u); + long missingRecords = new TableSelector(dsTableInfo, new SimpleFilter(FieldKey.fromString("Id/Demographics/Id"), CompareType.ISBLANK, null), null).getRowCount(); + if (missingRecords > 0) + { + log.error("Found " + missingRecords + " dataset records with an ID not found in demographics for: " + ds.getLabel() + " / " + ds.getContainer().getPath()); + } + } } } } From 92d94e17503dcf94a654d775a86335b58c7e6476 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 22 Feb 2024 13:33:03 -0800 Subject: [PATCH 08/35] Misc improvements to the MCC --- mcc/resources/data/reports.tsv | 13 ++ .../folderTypes/MCC Colony.folderType.xml | 30 +++++ .../folderTypes/MCC Data.folderType.xml | 83 +++++++++++++ .../queries/study/genomicDatasets.query.xml | 25 ++++ .../queries/study/pedigree.query.xml | 12 ++ mcc/resources/queries/study/pedigree.sql | 29 +++++ .../study/datasets/datasets_manifest.xml | 3 + .../study/datasets/datasets_metadata.xml | 28 +++++ .../postgresql/mcc-20.015-20.016.sql | 19 +++ .../dbscripts/sqlserver/mcc-20.015-20.016.sql | 19 +++ mcc/resources/schemas/mcc.xml | 85 +++++++++++++ mcc/resources/views/animalHistory.html | 32 +++++ mcc/resources/views/animalHistory.view.xml | 13 ++ mcc/resources/views/animalHistory.webpart.xml | 3 + mcc/resources/views/mccCensus.html | 13 ++ mcc/resources/views/mccCensus.view.xml | 5 + mcc/resources/views/mccCensus.webpart.xml | 6 + mcc/resources/views/mccDemographics.html | 13 ++ mcc/resources/views/mccDemographics.view.xml | 5 + .../views/mccDemographics.webpart.xml | 6 + mcc/resources/views/populateData.html | 13 +- .../web/mcc/window/RenameIdWindow.js | 95 +++++++++++++++ mcc/src/org/labkey/mcc/MccController.java | 112 ++++++++++++++++++ .../org/labkey/mcc/MccMaintenanceTask.java | 2 +- mcc/src/org/labkey/mcc/MccModule.java | 4 +- .../labkey/mcc/query/MccEhrCustomizer.java | 2 +- .../org/labkey/mcc/query/RenameIdButton.java | 18 +++ 27 files changed, 679 insertions(+), 9 deletions(-) create mode 100644 mcc/resources/data/reports.tsv create mode 100644 mcc/resources/folderTypes/MCC Data.folderType.xml create mode 100644 mcc/resources/queries/study/genomicDatasets.query.xml create mode 100644 mcc/resources/queries/study/pedigree.query.xml create mode 100644 mcc/resources/queries/study/pedigree.sql create mode 100644 mcc/resources/schemas/dbscripts/postgresql/mcc-20.015-20.016.sql create mode 100644 mcc/resources/schemas/dbscripts/sqlserver/mcc-20.015-20.016.sql create mode 100644 mcc/resources/views/animalHistory.html create mode 100644 mcc/resources/views/animalHistory.view.xml create mode 100644 mcc/resources/views/animalHistory.webpart.xml create mode 100644 mcc/resources/views/mccCensus.html create mode 100644 mcc/resources/views/mccCensus.view.xml create mode 100644 mcc/resources/views/mccCensus.webpart.xml create mode 100644 mcc/resources/views/mccDemographics.html create mode 100644 mcc/resources/views/mccDemographics.view.xml create mode 100644 mcc/resources/views/mccDemographics.webpart.xml create mode 100644 mcc/resources/web/mcc/window/RenameIdWindow.js create mode 100644 mcc/src/org/labkey/mcc/query/RenameIdButton.java diff --git a/mcc/resources/data/reports.tsv b/mcc/resources/data/reports.tsv new file mode 100644 index 000000000..731f7be9b --- /dev/null +++ b/mcc/resources/data/reports.tsv @@ -0,0 +1,13 @@ +reportname category reporttype reporttitle visible containerpath schemaname queryname viewname report datefieldname todayonly queryhaslocation sort_order QCStateLabelFieldName description +birth General query Birth Records TRUE study birth date FALSE FALSE qcstate/publicdata Birth records +weight General js Weights TRUE study weightGraph date FALSE FALSE qcstate/publicdata This report contains a summary of the animal\'s weight, including a graph +Flags General query Flags TRUE study flags StartDate FALSE FALSE qcstate/publicdata Animal attribute flags +demographics General query Demographics TRUE study demographics FALSE FALSE qcstate/publicdata This report displays the demographics data about each animal including species, sex and birth +death General query Death Records TRUE study deaths date FALSE FALSE qcstate/publicdata Death records +departure General query Shipments TRUE study departure date FALSE FALSE qcstate/publicdata Displays departure dates +obs General query Observations TRUE study clinical_observations date FALSE FALSE qcstate/publicdata This report displays observations for the selected animal +pedigreePlot Genetics report Pedigree Plot TRUE study pedigree module:EHR/schemas/study/Pedigree/Pedigree.r FALSE FALSE qcstate/publicdata This report will generate a pedigree plot for the selected animal +offspring General query Offspring TRUE study demographicsOffspring FALSE FALSE qcstate/publicdata This report displays pedigree data for animals, including parents, grandparents, siblings and offspring +kinship Genetics js Kinship TRUE ehr kinshipSummary FALSE FALSE qcstate/publicdata This report shows the kinship coefficient between every animal in the colony. The kinship coefficient is a measure of relatedness between two individuals. It represents the probability that two genes, sampled at random from each individual are identical (e.g. the kinship coefficient between a parent and an offspring is 0.25). +parentage Genetics query Parentage TRUE study demographicsParents FALSE FALSE qcstate/publicdata This report shows information about the parentage of each animal, drawing from genetic data and observations +geneticData Genetics query Genomic Datasets TRUE study geneticData FALSE FALSE qcstate/publicdata This report shows summarizes available genomic datasets diff --git a/mcc/resources/folderTypes/MCC Colony.folderType.xml b/mcc/resources/folderTypes/MCC Colony.folderType.xml index 6c88300ce..741997139 100644 --- a/mcc/resources/folderTypes/MCC Colony.folderType.xml +++ b/mcc/resources/folderTypes/MCC Colony.folderType.xml @@ -26,6 +26,36 @@ + + animalHistory + Animal History + + + animalHistory + + + + + MCC Animal History + body + + + + + census + Census + + + census + + + + + MCC Census + body + + + MCC diff --git a/mcc/resources/folderTypes/MCC Data.folderType.xml b/mcc/resources/folderTypes/MCC Data.folderType.xml new file mode 100644 index 000000000..7d409cf2b --- /dev/null +++ b/mcc/resources/folderTypes/MCC Data.folderType.xml @@ -0,0 +1,83 @@ + + MCC Data + The default folder layout for the MCC data aggregation folder + + + MCC Menu + menu + + + + + mcc + Overview + + + overview + + + + + + + + MCC Dashboard Webpart + body + + + + + animalData + All Animals + + + animalData + + + + + MCC Demographics + body + + + + + animalHistory + Animal Detail + + + animalHistory + + + + + MCC Animal History + body + + + + + census + Census + + + census + + + + + MCC Census + body + + + + + + MCC + Study + Query + Pipeline + Core + + Core + \ No newline at end of file diff --git a/mcc/resources/queries/study/genomicDatasets.query.xml b/mcc/resources/queries/study/genomicDatasets.query.xml new file mode 100644 index 000000000..8eaed6ad2 --- /dev/null +++ b/mcc/resources/queries/study/genomicDatasets.query.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + Data Type + + + SRA Accession + https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=${sraAccession} + + +
+ + + diff --git a/mcc/resources/queries/study/pedigree.query.xml b/mcc/resources/queries/study/pedigree.query.xml new file mode 100644 index 000000000..6e1620b87 --- /dev/null +++ b/mcc/resources/queries/study/pedigree.query.xml @@ -0,0 +1,12 @@ + + + + + Pedigree + + + +
+ + + diff --git a/mcc/resources/queries/study/pedigree.sql b/mcc/resources/queries/study/pedigree.sql new file mode 100644 index 000000000..c18ce19e6 --- /dev/null +++ b/mcc/resources/queries/study/pedigree.sql @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2013-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ +SELECT + +d.id as Id, +d.dam as Dam, +d.sire as Sire, + +CASE (d.id.demographics.gender.origGender) + WHEN 'm' THEN 1 + WHEN 'f' THEN 2 + ELSE 3 +END AS gender, +d.id.demographics.gender as gender_code, +CASE (d.id.demographics.calculated_status) + WHEN 'Alive' THEN 0 + ELSE 1 +END +AS status, +d.id.demographics.calculated_status as status_code, +d.id.demographics.species, +'' as Display, +'Demographics' as source + +FROM study.demographicsParents d +WHERE d.numParents > 0 and d.id.demographics.species IS NOT NULL \ No newline at end of file diff --git a/mcc/resources/referenceStudy/study/datasets/datasets_manifest.xml b/mcc/resources/referenceStudy/study/datasets/datasets_manifest.xml index bab0de73a..4040aaf0b 100644 --- a/mcc/resources/referenceStudy/study/datasets/datasets_manifest.xml +++ b/mcc/resources/referenceStudy/study/datasets/datasets_manifest.xml @@ -61,5 +61,8 @@ + + + diff --git a/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml index e5ae4f48e..0ee7322e1 100644 --- a/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -1035,4 +1035,32 @@ Kinship + + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + Genetic Datasets +
diff --git a/mcc/resources/schemas/dbscripts/postgresql/mcc-20.015-20.016.sql b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.015-20.016.sql new file mode 100644 index 000000000..86128d4a0 --- /dev/null +++ b/mcc/resources/schemas/dbscripts/postgresql/mcc-20.015-20.016.sql @@ -0,0 +1,19 @@ +CREATE TABLE mcc.census ( + rowid serial, + yearNo int, + startdate timestamp, + enddate timestamp, + centerName varchar(1000), + totalBreedingPairs int, + totalLivingOffspring int, + survivalRates int, + marmosetsShipped int, + + container entityid, + created timestamp, + createdby userid, + modified timestamp, + modifiedby userid, + + CONSTRAINT PK_census PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.015-20.016.sql b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.015-20.016.sql new file mode 100644 index 000000000..502ea2a5b --- /dev/null +++ b/mcc/resources/schemas/dbscripts/sqlserver/mcc-20.015-20.016.sql @@ -0,0 +1,19 @@ +CREATE TABLE mcc.census ( + rowid int identity(1,1), + yearNo int, + startdate datetime, + enddate datetime, + centerName varchar(1000), + totalBreedingPairs int, + totalLivingOffspring int, + survivalRates int, + marmosetsShipped int, + + container entityid, + created datetime, + createdby userid, + modified datetime, + modifiedby userid, + + CONSTRAINT PK_census PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/mcc/resources/schemas/mcc.xml b/mcc/resources/schemas/mcc.xml index d95f6542c..154d002e7 100644 --- a/mcc/resources/schemas/mcc.xml +++ b/mcc/resources/schemas/mcc.xml @@ -776,4 +776,89 @@ + + + + + + MCC Census + DETAILED + rowid + + + Row Id + false + true + + + Year # + false + + + Period Start + false + + + Period End + false + + + Center Name + false + + + # Breeding Pairs + + + # Living Offspring + + + Survival Rates + + + # Animals Shipped + + + true + + + true + + + false + false + false + true + 29 + true + + + true + + + false + false + false + true + 29 + true + + + Actions + 200 + + org.labkey.mcc.query.RequestScoreActionsDisplayColumnFactory + + + + + ldk.context + + + +
diff --git a/mcc/resources/views/animalHistory.html b/mcc/resources/views/animalHistory.html new file mode 100644 index 000000000..9eb133c7e --- /dev/null +++ b/mcc/resources/views/animalHistory.html @@ -0,0 +1,32 @@ + \ No newline at end of file diff --git a/mcc/resources/views/animalHistory.view.xml b/mcc/resources/views/animalHistory.view.xml new file mode 100644 index 000000000..885cf5f88 --- /dev/null +++ b/mcc/resources/views/animalHistory.view.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/animalHistory.webpart.xml b/mcc/resources/views/animalHistory.webpart.xml new file mode 100644 index 000000000..c26a73a35 --- /dev/null +++ b/mcc/resources/views/animalHistory.webpart.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/mcc/resources/views/mccCensus.html b/mcc/resources/views/mccCensus.html new file mode 100644 index 000000000..46c161473 --- /dev/null +++ b/mcc/resources/views/mccCensus.html @@ -0,0 +1,13 @@ + \ No newline at end of file diff --git a/mcc/resources/views/mccCensus.view.xml b/mcc/resources/views/mccCensus.view.xml new file mode 100644 index 000000000..22104272d --- /dev/null +++ b/mcc/resources/views/mccCensus.view.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/mcc/resources/views/mccCensus.webpart.xml b/mcc/resources/views/mccCensus.webpart.xml new file mode 100644 index 000000000..eed712c4e --- /dev/null +++ b/mcc/resources/views/mccCensus.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/mccDemographics.html b/mcc/resources/views/mccDemographics.html new file mode 100644 index 000000000..8b5bda189 --- /dev/null +++ b/mcc/resources/views/mccDemographics.html @@ -0,0 +1,13 @@ + \ No newline at end of file diff --git a/mcc/resources/views/mccDemographics.view.xml b/mcc/resources/views/mccDemographics.view.xml new file mode 100644 index 000000000..c058132f0 --- /dev/null +++ b/mcc/resources/views/mccDemographics.view.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/mcc/resources/views/mccDemographics.webpart.xml b/mcc/resources/views/mccDemographics.webpart.xml new file mode 100644 index 000000000..2e1aaa1d4 --- /dev/null +++ b/mcc/resources/views/mccDemographics.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/populateData.html b/mcc/resources/views/populateData.html index edba7ff24..ee4e32e04 100644 --- a/mcc/resources/views/populateData.html +++ b/mcc/resources/views/populateData.html @@ -145,12 +145,13 @@ schemaName: 'ehr_lookups', queryName: 'lookup_sets' },{ - // label: 'Reports', - // populateFn: 'populateFromFile', - // schemaName: 'ehr', - // queryName: 'reports', - // pk: 'rowid' - // },{ + label: 'Reports', + populateFn: 'populateFromFile', + moduleName: 'mcc', + schemaName: 'ehr', + queryName: 'reports', + pk: 'rowid' + },{ label: 'Species', populateFn: 'populateFromFile', moduleName: 'mcc', diff --git a/mcc/resources/web/mcc/window/RenameIdWindow.js b/mcc/resources/web/mcc/window/RenameIdWindow.js new file mode 100644 index 000000000..26f9ba43e --- /dev/null +++ b/mcc/resources/web/mcc/window/RenameIdWindow.js @@ -0,0 +1,95 @@ +Ext4.define('MCC.window.RenameIdWindow', { + extend: 'Ext.window.Window', + + statics: { + buttonHandler: function (dataRegionName) { + Ext4.create('MCC.window.RenameIdWindow', { + dataRegionName: dataRegionName + }).show(); + } + }, + + initComponent: function() { + Ext4.apply(this, { + title: 'Rename ID(s)', + bodyStyle: 'padding: 5px;', + defaults: { + bodyStyle: 'padding: 5px;' + }, + maxWidth: 550, + items: [{ + html: 'This will globally change all records associated with one animal ID to reference a new ID. This should not be used often, and is primarily intended for the situation where an incorrect ID was entered, or the source center provided a different ID after the fact. If an animal was shipped, it is generally fine to allow existing records to reference the original colony\'s ID, since both old and new IDs should point to the same MCC ID.', + style: 'padding-bottom: 10px;', + width: 500, + border: false + }, { + xtype: 'textarea', + width: 500, + height: 250, + boxLabel: 'Enter Old/New IDs, one pair per line:' + }], + buttons: [{ + text: 'Submit', + scope: this, + handler: this.doSubmit + }, { + text: 'Cancel', + handler: function (btn) { + btn.up('window').close(); + } + }] + }); + + this.callParent(arguments); + }, + + doSubmit: function(btn){ + var text = btn.up('window').down('textarea').getValue(); + if (!text) { + Ext4.Msg.alert('Error', 'No rows entered!'); + return; + } + + var rows = text.split(/[\n\r]/) + if (!rows.length) { + Ext4.Msg.alert('Error', 'No rows entered!'); + return; + } + + var oldToNew = {}; + var hadError = false; + Ext4.Array.forEach(rows, function(r){ + r = r.split(/[\t ;,]/) + if (r.length !== 2) { + Ext4.Msg.alert('Error', 'Row was not two elements long: ' + r.join(',')); + hadError = true; + return false; + } + + oldToNew[r[0]] = r[1]; + }, this); + + if (hadError) { + return; + } + + Ext4.Msg.wait('Saving...'); + LABKEY.Ajax.request({ + method: 'POST', + url: LABKEY.ActionURL.buildURL('mcc', 'renameIds'), + scope: this, + params: { + originalIds: Ext4.Object.getKeys(oldToNew), + newIds: Ext4.Object.getValues(oldToNew) + }, + success: LABKEY.Utils.getCallbackWrapper(function (response) { + Ext4.Msg.hide(); + this.close(); + console.log(response); + + Ext4.Msg.alert('Success', 'Total IDs Updated: ' + response.totalIdsUpdated + '
Total Records Updated: ' + response.totalRecordsUpdated); + }), + failure: LDK.Utils.getErrorCallback() + }); + } +}); diff --git a/mcc/src/org/labkey/mcc/MccController.java b/mcc/src/org/labkey/mcc/MccController.java index 38c6e76d2..67aaedd2a 100644 --- a/mcc/src/org/labkey/mcc/MccController.java +++ b/mcc/src/org/labkey/mcc/MccController.java @@ -24,6 +24,8 @@ import org.labkey.api.action.ConfirmAction; import org.labkey.api.action.MutatingApiAction; import org.labkey.api.action.SpringActionController; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; @@ -38,8 +40,11 @@ import org.labkey.api.module.ModuleLoader; import org.labkey.api.module.ModuleProperty; import org.labkey.api.pipeline.PipelineUrls; +import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.DetailsURL; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.QueryUpdateServiceException; import org.labkey.api.security.AuthenticationManager; import org.labkey.api.security.Group; import org.labkey.api.security.GroupManager; @@ -57,8 +62,12 @@ import org.labkey.api.settings.AppProps; import org.labkey.api.settings.LookAndFeelProperties; import org.labkey.api.studies.StudiesService; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.GUID; import org.labkey.api.util.HtmlString; import org.labkey.api.util.MailHelper; import org.labkey.api.util.PageFlowUtil; @@ -66,6 +75,7 @@ import org.labkey.api.util.URLHelper; import org.labkey.api.view.HtmlView; import org.labkey.mcc.etl.ZimsImportTask; +import org.labkey.mcc.security.MccDataAdminPermission; import org.labkey.mcc.security.MccDataAdminRole; import org.labkey.mcc.security.MccFinalReviewerRole; import org.labkey.mcc.security.MccRabReviewerRole; @@ -78,6 +88,7 @@ import javax.mail.Address; import javax.mail.Message; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -85,7 +96,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import java.util.stream.IntStream; public class MccController extends SpringActionController { @@ -690,6 +703,9 @@ public boolean handlePost(Object o, BindException errors) throws Exception ModuleProperty mp = ehr.getModuleProperties().get("EHRStudyContainer"); mp.saveValue(getUser(), getContainer(), getContainer().getPath()); + ModuleProperty mp2 = ehr.getModuleProperties().get("DefaultAnimalHistoryReport"); + mp2.saveValue(getUser(), getContainer(), "demographics"); + StudiesService.get().importFolderDefinition(getContainer(), getUser(), ModuleLoader.getInstance().getModule(MccModule.NAME), new Path("referenceStudy")); return true; @@ -781,4 +797,100 @@ public void setRowIds(Integer[] rowIds) this.rowIds = rowIds; } } + + @RequiresPermission(MccDataAdminPermission.class) + public class RenameIdsAction extends MutatingApiAction + { + @Override + public Object execute(RenameIdsForm form, BindException errors) throws Exception + { + if (form.getOriginalIds() == null || form.getOriginalIds().length == 0) + { + errors.reject(ERROR_MSG, "Must provide a list of IDs to change"); + return null; + } + + if (form.getNewIds() == null || form.getOriginalIds().length != form.getNewIds().length) + { + errors.reject(ERROR_MSG, "Differing number of Original/New IDs were provided"); + return null; + } + + Study s = StudyService.get().getStudy(getContainer()); + if (s == null) + { + errors.reject(ERROR_MSG, "There is no study in this container"); + return null; + } + + final Map oldToNew = new CaseInsensitiveHashMap<>(); + IntStream.range(0, form.getOriginalIds().length).forEach(i -> oldToNew.put(form.getOriginalIds()[i], form.getNewIds()[i])); + + final Set idsUpdated = new CaseInsensitiveHashSet(); + final AtomicInteger totalRecordsUpdated = new AtomicInteger(); + + // NOTE: include this value so it will get added to the audit trail. This is a loose way to connect changes made in this transaction + final String batchId = "MCC.Rename." + new GUID(); + for (Dataset ds : s.getDatasets()) + { + TableInfo ti = ds.getTableInfo(getUser()); + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("Id", "lsid"), new SimpleFilter(FieldKey.fromString("Id"), oldToNew.keySet(), CompareType.IN), null); + if (!ts.exists()) + { + continue; + } + + List> toUpdate = new ArrayList<>(); + List> oldKeys = new ArrayList<>(); + ts.forEachResults(rs -> { + toUpdate.add(Map.of("lsid", rs.getString(FieldKey.fromString("lsid")), "Id", oldToNew.get(rs.getString(FieldKey.fromString("Id"))), "_batchId_", batchId)); + oldKeys.add(Map.of("lsid", rs.getString(FieldKey.fromString("lsid")))); + idsUpdated.add(rs.getString(FieldKey.fromString("Id"))); + totalRecordsUpdated.getAndIncrement(); + }); + + if (!toUpdate.isEmpty()) + { + try + { + ti.getUpdateService().updateRows(getUser(), getContainer(), toUpdate, oldKeys, null, null); + } + catch (InvalidKeyException | BatchValidationException | QueryUpdateServiceException | SQLException e) + { + _log.error("Error updating MCC dataset rows", e); + errors.reject(ERROR_MSG, "Error updating MCC dataset rows: " + e.getMessage()); + return null; + } + } + } + + return new ApiSimpleResponse(Map.of("success", true, "totalIdsUpdated", idsUpdated.size(), "totalRecordsUpdated", totalRecordsUpdated.get())); + } + } + + public static class RenameIdsForm + { + private String[] _originalIds; + private String[] _newIds; + + public String[] getOriginalIds() + { + return _originalIds; + } + + public void setOriginalIds(String[] originalIds) + { + _originalIds = originalIds; + } + + public String[] getNewIds() + { + return _newIds; + } + + public void setNewIds(String[] newIds) + { + _newIds = newIds; + } + } } diff --git a/mcc/src/org/labkey/mcc/MccMaintenanceTask.java b/mcc/src/org/labkey/mcc/MccMaintenanceTask.java index 4db130712..7182ee73b 100644 --- a/mcc/src/org/labkey/mcc/MccMaintenanceTask.java +++ b/mcc/src/org/labkey/mcc/MccMaintenanceTask.java @@ -89,7 +89,7 @@ private void checkForDuplicateAliases(Logger log) for (Dataset ds : s.getDatasets()) { TableInfo dsTableInfo = ds.getTableInfo(u); - long missingRecords = new TableSelector(dsTableInfo, new SimpleFilter(FieldKey.fromString("Id/Demographics/Id"), CompareType.ISBLANK, null), null).getRowCount(); + long missingRecords = new TableSelector(dsTableInfo, new SimpleFilter(FieldKey.fromString("Id/Demographics/Id"), null, CompareType.ISBLANK), null).getRowCount(); if (missingRecords > 0) { log.error("Found " + missingRecords + " dataset records with an ID not found in demographics for: " + ds.getLabel() + " / " + ds.getContainer().getPath()); diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index b2b969cdb..92f086233 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -33,6 +33,7 @@ import org.labkey.api.writer.ContainerUser; import org.labkey.mcc.query.MarkShippedButton; import org.labkey.mcc.query.MccEhrCustomizer; +import org.labkey.mcc.query.RenameIdButton; import org.labkey.mcc.query.ReviewerNotifyButton; import org.labkey.mcc.security.MccDataAdminRole; import org.labkey.mcc.security.MccFinalReviewPermission; @@ -59,7 +60,7 @@ public String getName() @Override public @Nullable Double getSchemaVersion() { - return 20.015; + return 20.016; } @Override @@ -121,6 +122,7 @@ private void registerEHRResources() EHRService.get().registerModule(this); EHRService.get().registerTableCustomizer(this, MccEhrCustomizer.class); EHRService.get().registerMoreActionsButton(new MarkShippedButton(), "study", "demographics"); + EHRService.get().registerMoreActionsButton(new RenameIdButton(), "study", "demographics"); } @Override diff --git a/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java b/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java index 817d86a39..792f0669f 100644 --- a/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java +++ b/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java @@ -74,7 +74,7 @@ private void customizeWeight(AbstractTableInfo ti) private void customizeAnimalTable(AbstractTableInfo ti) { - for (String colName : new String[]{"MostRecentArrival", "numRoommates", "curLocation", "lastHousing", "weightChange", "CageClass", "MhcStatus"}) + for (String colName : new String[]{"MostRecentArrival", "numRoommates", "curLocation", "lastHousing", "weightChange", "CageClass", "MhcStatus", "history"}) { ColumnInfo ci = ti.getColumn(colName); if (ci != null) diff --git a/mcc/src/org/labkey/mcc/query/RenameIdButton.java b/mcc/src/org/labkey/mcc/query/RenameIdButton.java new file mode 100644 index 000000000..a5fc2ea9a --- /dev/null +++ b/mcc/src/org/labkey/mcc/query/RenameIdButton.java @@ -0,0 +1,18 @@ +package org.labkey.mcc.query; + +import org.labkey.api.ldk.table.SimpleButtonConfigFactory; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.view.template.ClientDependency; +import org.labkey.mcc.MccModule; +import org.labkey.mcc.security.MccDataAdminPermission; + +import java.util.List; + +public class RenameIdButton extends SimpleButtonConfigFactory +{ + public RenameIdButton() + { + super(ModuleLoader.getInstance().getModule(MccModule.class), "Rename ID", "MCC.window.RenameIdWindow.buttonHandler(dataRegionName);", List.of(ClientDependency.supplierFromPath("mcc/window/RenameIdWindow.js"))); + setPermission(MccDataAdminPermission.class); + } +} From b37240836a1ed704d44a6522321aa190b6d73472 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 22 Feb 2024 14:15:31 -0800 Subject: [PATCH 09/35] Fix typo --- mcc/resources/data/reports.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcc/resources/data/reports.tsv b/mcc/resources/data/reports.tsv index 731f7be9b..d203ed2c1 100644 --- a/mcc/resources/data/reports.tsv +++ b/mcc/resources/data/reports.tsv @@ -10,4 +10,4 @@ pedigreePlot Genetics report Pedigree Plot TRUE study pedigree module:EHR/sche offspring General query Offspring TRUE study demographicsOffspring FALSE FALSE qcstate/publicdata This report displays pedigree data for animals, including parents, grandparents, siblings and offspring kinship Genetics js Kinship TRUE ehr kinshipSummary FALSE FALSE qcstate/publicdata This report shows the kinship coefficient between every animal in the colony. The kinship coefficient is a measure of relatedness between two individuals. It represents the probability that two genes, sampled at random from each individual are identical (e.g. the kinship coefficient between a parent and an offspring is 0.25). parentage Genetics query Parentage TRUE study demographicsParents FALSE FALSE qcstate/publicdata This report shows information about the parentage of each animal, drawing from genetic data and observations -geneticData Genetics query Genomic Datasets TRUE study geneticData FALSE FALSE qcstate/publicdata This report shows summarizes available genomic datasets +geneticData Genetics query Genomic Datasets TRUE study genomicDatasets FALSE FALSE qcstate/publicdata This report shows summarizes available genomic datasets From 473980b0b2cccdfef5fe8b81f5119c439a577add Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 22 Feb 2024 17:08:38 -0800 Subject: [PATCH 10/35] Create automatic MCC queries to aggregate data across containers for ETL --- mcc/resources/etls/mcc.xml | 69 ++++++ .../queries/mcc/aggregatedKinship.sql | 57 ----- mcc/src/org/labkey/mcc/MccSchema.java | 1 + mcc/src/org/labkey/mcc/MccUserSchema.java | 226 +++++++++++++++++- 4 files changed, 295 insertions(+), 58 deletions(-) delete mode 100644 mcc/resources/queries/mcc/aggregatedKinship.sql diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml index b54a3327b..c1462bb7e 100644 --- a/mcc/resources/etls/mcc.xml +++ b/mcc/resources/etls/mcc.xml @@ -60,6 +60,75 @@ + + Copy to target + + + Id + date + weight + objectid + + + + + + + + + + Copy to target + + + Id + date + datatype + sra_accession + objectid + + + + + + + + + + Copy to target + + + Id + date + category + observation + objectid + + + + + + + + + + Copy to target + + + Id + yearNo + startdate + enddate + centerName + totalBreedingPairs + totalLivingOffspring + survivalRates + marmosetsShipped + + + + + + diff --git a/mcc/resources/queries/mcc/aggregatedKinship.sql b/mcc/resources/queries/mcc/aggregatedKinship.sql deleted file mode 100644 index c5e82fba2..000000000 --- a/mcc/resources/queries/mcc/aggregatedKinship.sql +++ /dev/null @@ -1,57 +0,0 @@ -SELECT - d.Id.mccAlias.externalAlias as Id, - d.Id as originalId, - d.date, - d.Id2MccAlias.externalAlias as Id2, - d.Id2 as originalId2, - d.kinship, - d.relationship, - d.objectid, - d.container - -FROM "/data/Colonies/SNPRC/".study.kinship d - -UNION ALL - -SELECT - d.Id.mccAlias.externalAlias as Id, - d.Id as originalId, - d.date, - d.Id2MccAlias.externalAlias as Id2, - d.Id2 as originalId2, - d.kinship, - d.relationship, - d.objectid, - d.container - -FROM "/data/Colonies/WNPRC/".study.kinship d - -UNION ALL - -SELECT - d.Id.mccAlias.externalAlias as Id, - d.Id as originalId, - d.date, - d.Id2MccAlias.externalAlias as Id2, - d.Id2 as originalId2, - d.kinship, - d.relationship, - d.objectid, - d.container - -FROM "/data/Colonies/UCSD/".study.kinship d - -UNION ALL - -SELECT - d.Id.mccAlias.externalAlias as Id, - d.Id as originalId, - d.date, - d.Id2MccAlias.externalAlias as Id2, - d.Id2 as originalId2, - d.kinship, - d.relationship, - d.objectid, - d.container - -FROM "/data/Colonies/Other/".study.kinship d \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/MccSchema.java b/mcc/src/org/labkey/mcc/MccSchema.java index 1745b67bb..03e4cd3ed 100644 --- a/mcc/src/org/labkey/mcc/MccSchema.java +++ b/mcc/src/org/labkey/mcc/MccSchema.java @@ -31,6 +31,7 @@ public class MccSchema public static final String TABLE_ANIMAL_REQUESTS = "animalRequests"; public static final String TABLE_REQUEST_REVIEWS = "requestReviews"; public static final String TABLE_REQUEST_SCORE = "requestScores"; + public static final String TABLE_CENSUS = "census"; public static MccSchema getInstance() { diff --git a/mcc/src/org/labkey/mcc/MccUserSchema.java b/mcc/src/org/labkey/mcc/MccUserSchema.java index 6b0a6d35a..50c0e4100 100644 --- a/mcc/src/org/labkey/mcc/MccUserSchema.java +++ b/mcc/src/org/labkey/mcc/MccUserSchema.java @@ -1,7 +1,9 @@ package org.labkey.mcc; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.collections.CaseInsensitiveTreeSet; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.DataColumn; @@ -13,26 +15,66 @@ import org.labkey.api.ldk.table.CustomPermissionsTable; import org.labkey.api.query.ExprColumn; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryDefinition; +import org.labkey.api.query.QueryException; +import org.labkey.api.query.QueryService; import org.labkey.api.query.SimpleUserSchema; import org.labkey.api.security.User; import org.labkey.api.security.permissions.DeletePermission; import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.mcc.security.MccDataAdminPermission; import org.labkey.mcc.security.MccRabReviewPermission; import org.labkey.mcc.security.MccRequestAdminPermission; import org.labkey.mcc.security.MccRequestorPermission; import org.labkey.mcc.security.MccViewRequestsPermission; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; +import java.util.Set; public class MccUserSchema extends SimpleUserSchema { + private static final Logger _log = LogHelper.getLogger(MccUserSchema.class, "MCC UserSchema messages"); + public MccUserSchema(User user, Container container, DbSchema dbschema) { super(MccSchema.NAME, "MCC-specific tables, such as requests", user, container, dbschema); } + @Override + public TableInfo createTable(String name, ContainerFilter cf) + { + if (TABLE_AGGREGATED_WEIGHT.equalsIgnoreCase(name)) + { + return getWeightQuery(); + } + else if (TABLE_AGGREGATED_KINSHIP.equalsIgnoreCase(name)) + { + return getKinshipQuery(); + } + else if (TABLE_AGGREGATED_DEPARTURES.equalsIgnoreCase(name)) + { + return getDepartureQuery(); + } + else if (TABLE_AGGREGATED_GENOMICS.equalsIgnoreCase(name)) + { + return getGenomicsQuery(); + } + else if (TABLE_AGGREGATED_OBS.equalsIgnoreCase(name)) + { + return getObsQuery(); + } + else if (TABLE_AGGREGATED_CENSUS.equalsIgnoreCase(name)) + { + return getCensusQuery(); + } + return super.createTable(name, cf); + } + @Override @Nullable protected TableInfo createWrappedTable(String name, @NotNull TableInfo schemaTable, ContainerFilter cf) @@ -74,11 +116,193 @@ else if (MccSchema.TABLE_REQUEST_SCORE.equalsIgnoreCase(name)) return addScoreColumns(ret); } + else if (MccSchema.TABLE_CENSUS.equalsIgnoreCase(name)) + { + CustomPermissionsTable ret = new CustomPermissionsTable<>(this, schemaTable, cf); + ret.addPermissionMapping(InsertPermission.class, MccDataAdminPermission.class); + ret.addPermissionMapping(UpdatePermission.class, MccDataAdminPermission.class); + ret.addPermissionMapping(DeletePermission.class, MccDataAdminPermission.class); + + return ret.init(); + } return super.createWrappedTable(name, schemaTable, cf); } - // TODO: make automatic aggregation queries for ETLs + @Override + public Set getTableNames() + { + Set available = new CaseInsensitiveTreeSet(); + available.addAll(super.getTableNames()); + addAggregatedTableNames(available); + + return available; + } + + private static final String TABLE_AGGREGATED_WEIGHT = "aggregatedWeight"; + private static final String TABLE_AGGREGATED_KINSHIP = "aggregatedKinship"; + private static final String TABLE_AGGREGATED_DEPARTURES = "aggregatedDepartures"; + private static final String TABLE_AGGREGATED_OBS = "aggregatedObservations"; + private static final String TABLE_AGGREGATED_GENOMICS = "aggregatedGenomicDatasets"; + private static final String TABLE_AGGREGATED_CENSUS = "aggregatedCensus"; + + @Override + public Set getVisibleTableNames() + { + Set available = new CaseInsensitiveTreeSet(); + available.addAll(super.getVisibleTableNames()); + addAggregatedTableNames(available); + + return Collections.unmodifiableSet(available); + } + + private void addAggregatedTableNames(Set available) + { + if (getContainer().hasPermission(getUser(), MccDataAdminPermission.class)) + { + available.add(TABLE_AGGREGATED_KINSHIP); + available.add(TABLE_AGGREGATED_WEIGHT); + available.add(TABLE_AGGREGATED_DEPARTURES); + available.add(TABLE_AGGREGATED_OBS); + available.add(TABLE_AGGREGATED_GENOMICS); + available.add(TABLE_AGGREGATED_CENSUS); + } + } + + private TableInfo getKinshipQuery() + { + String template = "SELECT\n" + + " d.Id.mccAlias.externalAlias as Id,\n" + + " d.Id as originalId,\n" + + " d.date,\n" + + " d.Id2MccAlias.externalAlias as Id2,\n" + + " d.Id2 as originalId2,\n" + + " d.kinship,\n" + + " d.relationship,\n" + + " d.objectid,\n" + + " d.container\n" + + "\n" + + "FROM \"\".study.kinship d WHERE d.qcstate.publicdata = true\n"; + + return makeAggregatedQuery(TABLE_AGGREGATED_KINSHIP, template); + } + + private TableInfo getWeightQuery() + { + String template = "SELECT\n" + + " d.Id.mccAlias.externalAlias as Id,\n" + + " d.Id as originalId,\n" + + " d.date,\n" + + " d.weight,\n" + + " d.objectid,\n" + + " d.container\n" + + "\n" + + "FROM \"\".study.weight d WHERE d.qcstate.publicdata = true\n"; + + return makeAggregatedQuery(TABLE_AGGREGATED_WEIGHT, template); + } + + private TableInfo getCensusQuery() + { + String template = "SELECT\n" + + " d.Id.mccAlias.externalAlias as Id,\n" + + " d.Id as originalId,\n" + + " d.yearNo,\n" + + " d.startdate,\n" + + " d.enddate,\n" + + " d.centerName,\n" + + " d.totalBreedingPairs,\n" + + " d.totalLivingOffspring,\n" + + " d.survivalRates,\n" + + " d.marmosetsShipped,\n" + + " d.container\n" + + "\n" + + "FROM \"\".mcc.census d\n"; + + return makeAggregatedQuery(TABLE_AGGREGATED_CENSUS, template); + } + + private TableInfo getDepartureQuery() + { + String template = "SELECT\n" + + " d.Id.mccAlias.externalAlias as Id,\n" + + " d.Id as originalId,\n" + + " d.date,\n" + + " d.destination,\n" + + " d.objectid,\n" + + " d.container\n" + + "\n" + + "FROM \"\".study.departure d WHERE d.qcstate.publicdata = true\n"; + + return makeAggregatedQuery(TABLE_AGGREGATED_DEPARTURES, template); + } + + private TableInfo getObsQuery() + { + String template = "SELECT\n" + + " d.Id.mccAlias.externalAlias as Id,\n" + + " d.Id as originalId,\n" + + " d.date,\n" + + " d.category,\n" + + " d.observation,\n" + + " d.objectid,\n" + + " d.container\n" + + "\n" + + "FROM \"\".study.clinical_observations d WHERE d.qcstate.publicdata = true\n"; + + return makeAggregatedQuery(TABLE_AGGREGATED_OBS, template); + } + + private TableInfo getGenomicsQuery() + { + String template = "SELECT\n" + + " d.Id.mccAlias.externalAlias as Id,\n" + + " d.Id as originalId,\n" + + " d.date,\n" + + " d.datatype,\n" + + " d.sra_accession,\n" + + " d.objectid,\n" + + " d.container\n" + + "\n" + + "FROM \"\".study.genomicDatasets d WHERE d.qcstate.publicdata = true\n"; + + return makeAggregatedQuery(TABLE_AGGREGATED_GENOMICS, template); + } + + private TableInfo makeAggregatedQuery(String queryName, String sqlTemplate) + { + if (!getContainer().hasPermission(getUser(), MccDataAdminPermission.class)) + { + return null; + } + + QueryDefinition qd = QueryService.get().createQueryDef(getUser(), getContainer(), this, queryName); + StringBuilder sql = new StringBuilder(); + + Container parent = MccManager.get().getMCCInternalDataContainer(getContainer()); + String unionClause = ""; + for (Container c : parent.getChildren()) + { + sql.append(unionClause); + unionClause = " UNION ALL "; + sql.append(sqlTemplate.replaceAll("", c.getPath())); + } + + qd.setSql(sql.toString()); + + List errors = new ArrayList(); + TableInfo ti = qd.getTable(errors, true); + if (!errors.isEmpty()) + { + _log.error("Problem with aggregated query: " + queryName); + for (QueryException e : errors) + { + _log.error(e.getMessage()); + } + } + + return ti; + } private CustomPermissionsTable addScoreColumns(CustomPermissionsTable ti) { From d067203e250b3a1562f7ee344e4d0b914c52572c Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 22 Feb 2024 20:16:22 -0800 Subject: [PATCH 11/35] Set FK for MCC demographicsParents --- .../study/demographicsParents.query.xml | 19 +++++++++++++++++++ mcc/src/org/labkey/mcc/MccUserSchema.java | 2 -- 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 mcc/resources/queries/study/demographicsParents.query.xml diff --git a/mcc/resources/queries/study/demographicsParents.query.xml b/mcc/resources/queries/study/demographicsParents.query.xml new file mode 100644 index 000000000..80993d8cc --- /dev/null +++ b/mcc/resources/queries/study/demographicsParents.query.xml @@ -0,0 +1,19 @@ + + + + + Parents + + + true + + study + animal + id + + + +
+ + + diff --git a/mcc/src/org/labkey/mcc/MccUserSchema.java b/mcc/src/org/labkey/mcc/MccUserSchema.java index 50c0e4100..7ad0fec29 100644 --- a/mcc/src/org/labkey/mcc/MccUserSchema.java +++ b/mcc/src/org/labkey/mcc/MccUserSchema.java @@ -205,8 +205,6 @@ private TableInfo getWeightQuery() private TableInfo getCensusQuery() { String template = "SELECT\n" + - " d.Id.mccAlias.externalAlias as Id,\n" + - " d.Id as originalId,\n" + " d.yearNo,\n" + " d.startdate,\n" + " d.enddate,\n" + From a2c3fc559b23198201c0cbe0721e94768e05b875 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 22 Feb 2024 20:31:35 -0800 Subject: [PATCH 12/35] Allow GeographicOriginStep to save results --- .../dbscripts/postgresql/mgap-16.71-16.72.sql | 1 + .../dbscripts/sqlserver/mgap-16.71-16.72.sql | 1 + mGAP/resources/schemas/mgap.xml | 11 +++ mGAP/src/org/labkey/mgap/mGAPModule.java | 4 +- mGAP/src/org/labkey/mgap/mGAPSchema.java | 1 + .../mgap}/pipeline/GeographicOriginStep.java | 76 +++++++++++++++++-- .../org/labkey/primeseq/PrimeseqModule.java | 3 - 7 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 mGAP/resources/schemas/dbscripts/postgresql/mgap-16.71-16.72.sql create mode 100644 mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.71-16.72.sql rename {primeseq/src/org/labkey/primeseq => mGAP/src/org/labkey/mgap}/pipeline/GeographicOriginStep.java (64%) diff --git a/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.71-16.72.sql b/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.71-16.72.sql new file mode 100644 index 000000000..c8d436f98 --- /dev/null +++ b/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.71-16.72.sql @@ -0,0 +1 @@ +ALTER TABLE mGAP.geneticMeasurements ADD dataid int; \ No newline at end of file diff --git a/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.71-16.72.sql b/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.71-16.72.sql new file mode 100644 index 000000000..c8d436f98 --- /dev/null +++ b/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.71-16.72.sql @@ -0,0 +1 @@ +ALTER TABLE mGAP.geneticMeasurements ADD dataid int; \ No newline at end of file diff --git a/mGAP/resources/schemas/mgap.xml b/mGAP/resources/schemas/mgap.xml index 08f52c43e..56c761a40 100644 --- a/mGAP/resources/schemas/mgap.xml +++ b/mGAP/resources/schemas/mgap.xml @@ -1132,6 +1132,17 @@ Comment + + File + false + false + false + + exp + data + RowId + + true diff --git a/mGAP/src/org/labkey/mgap/mGAPModule.java b/mGAP/src/org/labkey/mgap/mGAPModule.java index 6bdc5af3c..6eff0b5b0 100644 --- a/mGAP/src/org/labkey/mgap/mGAPModule.java +++ b/mGAP/src/org/labkey/mgap/mGAPModule.java @@ -46,6 +46,7 @@ import org.labkey.mgap.jbrowse.mGAPLuceneDetector; import org.labkey.mgap.pipeline.AnnotationStep; import org.labkey.mgap.pipeline.GenerateMgapTracksStep; +import org.labkey.mgap.pipeline.GeographicOriginStep; import org.labkey.mgap.pipeline.GroupCompareStep; import org.labkey.mgap.pipeline.IndexVariantsForMgapStep; import org.labkey.mgap.pipeline.RemoveAnnotationsForMgapStep; @@ -74,7 +75,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 16.71; + return 16.72; } @Override @@ -138,6 +139,7 @@ public PipelineStartup() SequencePipelineService.get().registerPipelineStep(new GenerateMgapTracksStep.Provider()); SequencePipelineService.get().registerPipelineStep(new IndexVariantsForMgapStep.Provider()); SequencePipelineService.get().registerPipelineStep(new mGapReleaseAlleleFreqStep.Provider()); + SequencePipelineService.get().registerPipelineStep(new GeographicOriginStep.Provider()); _hasRegistered = true; } diff --git a/mGAP/src/org/labkey/mgap/mGAPSchema.java b/mGAP/src/org/labkey/mgap/mGAPSchema.java index 9a3ed5aa8..b9a06a5fe 100644 --- a/mGAP/src/org/labkey/mgap/mGAPSchema.java +++ b/mGAP/src/org/labkey/mgap/mGAPSchema.java @@ -38,6 +38,7 @@ public class mGAPSchema public static final String TABLE_DEMOGRAPHICS = "demographics"; public static final String TABLE_SUBJECT_SOURCE = "subjectsSource"; public static final String TABLE_SEQUENCE_DATASETS = "sequenceDatasets"; + public static final String TABLE_GENETIC_MEASUREMENTS = "geneticMeasurements"; public static mGAPSchema getInstance() diff --git a/primeseq/src/org/labkey/primeseq/pipeline/GeographicOriginStep.java b/mGAP/src/org/labkey/mgap/pipeline/GeographicOriginStep.java similarity index 64% rename from primeseq/src/org/labkey/primeseq/pipeline/GeographicOriginStep.java rename to mGAP/src/org/labkey/mgap/pipeline/GeographicOriginStep.java index 8a14cd729..d98410cbd 100644 --- a/primeseq/src/org/labkey/primeseq/pipeline/GeographicOriginStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/GeographicOriginStep.java @@ -1,29 +1,46 @@ -package org.labkey.primeseq.pipeline; +package org.labkey.mgap.pipeline; +import au.com.bytecode.opencsv.CSVReader; import htsjdk.samtools.util.Interval; import org.apache.logging.log4j.Logger; import org.json.JSONObject; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.TableInfo; +import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.reader.Readers; +import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStep; import org.labkey.api.sequenceanalysis.pipeline.AbstractVariantProcessingStepProvider; import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; +import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStep; import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStepOutputImpl; import org.labkey.api.sequenceanalysis.run.AbstractDiscvrSeqWrapper; +import org.labkey.mgap.mGAPSchema; import javax.annotation.Nullable; import java.io.File; +import java.io.IOException; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; public class GeographicOriginStep extends AbstractPipelineStep implements VariantProcessingStep { - public GeographicOriginStep(PipelineStepProvider provider, PipelineContext ctx) + public static final String CATEGORY = "Macaque Geographic Origin Scores"; + + public GeographicOriginStep(PipelineStepProvider provider, PipelineContext ctx) { super(provider, ctx); } @@ -31,7 +48,7 @@ public GeographicOriginStep(PipelineStepProvider provider, PipelineContext ctx) private static final String INDIAN_MARKERS = "indianMarkers"; private static final String CHINESE_MARKERS = "chineseMarkers"; - public static class Provider extends AbstractVariantProcessingStepProvider implements VariantProcessingStep.RequiresPedigree + public static class Provider extends AbstractVariantProcessingStepProvider implements RequiresPedigree { public Provider() { @@ -43,7 +60,8 @@ public Provider() ToolParameterDescriptor.createExpDataParam(CHINESE_MARKERS, "Chinese-origin Markers", "This is the ID of a VCF file with markers of chinese-origin.", "ldk-expdatafield", new JSONObject() {{ put("allowBlank", false); - }}, null) + }}, null), + ToolParameterDescriptor.create("storeResults", "Store Results", "If checked, the results will be stored in the database", "checkbox", null, null) ), null, "https://bimberlab.github.io/DISCVRSeq/"); } @@ -54,6 +72,54 @@ public GeographicOriginStep create(PipelineContext ctx) } } + @Override + public void complete(PipelineJob job, List inputs, List outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException + { + // TODO: store in DB + for (SequenceOutputFile so : outputsCreated) + { + if (!CATEGORY.equals(so.getCategory())) + { + continue; + } + + List> toInsert = new ArrayList<>(); + try (CSVReader reader = new CSVReader(Readers.getReader(so.getFile()), '\t')) + { + String[] line; + while ((line = reader.readNext()) != null) + { + // SampleName ReferenceName MarkersWithData MarkersNoData FractionWithData UniqueContigs CumulativeAF MinPossible MaxPossible Score + Map row = new CaseInsensitiveHashMap<>(); + row.put("sampleName", line[0]); + row.put("measurementName", line[1]); + row.put("measurementValue", line[9]); + row.put("dataId", so.getDataId()); + + toInsert.add(row); + } + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + + if (!toInsert.isEmpty()) + { + TableInfo ti = QueryService.get().getUserSchema(job.getUser(), job.getContainer(), mGAPSchema.NAME).getTable(mGAPSchema.TABLE_GENETIC_MEASUREMENTS); + BatchValidationException bve = new BatchValidationException(); + try + { + ti.getUpdateService().insertRows(job.getUser(), job.getContainer(), toInsert, bve, null, null); + } + catch (DuplicateKeyException | BatchValidationException | QueryUpdateServiceException | SQLException e) + { + throw new PipelineJobException(e); + } + } + } + } + @Override public Output processVariants(File inputVCF, File outputDirectory, ReferenceGenome genome, @Nullable List intervals) throws PipelineJobException { @@ -97,7 +163,7 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno output.addInput(genome.getWorkingFastaFile(), "Reference Genome"); output.addOutput(outputTable, "Macaque Geographic Origin Scores"); - output.addSequenceOutput(outputTable, "Macaque Geographic Origin Scores for: " + inputVCF.getName(), "Macaque Geographic Origin Scores", null, null, genome.getGenomeId(), null); + output.addSequenceOutput(outputTable, "Macaque Geographic Origin Scores for: " + inputVCF.getName(), CATEGORY, null, null, genome.getGenomeId(), null); return output; } diff --git a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java index a924e18fd..c8c3539ae 100644 --- a/primeseq/src/org/labkey/primeseq/PrimeseqModule.java +++ b/primeseq/src/org/labkey/primeseq/PrimeseqModule.java @@ -39,7 +39,6 @@ import org.labkey.primeseq.pipeline.ClusterMaintenanceTask; import org.labkey.primeseq.pipeline.DeleteJobCheckpointButton; import org.labkey.primeseq.pipeline.ExacloudResourceSettings; -import org.labkey.primeseq.pipeline.GeographicOriginStep; import org.labkey.primeseq.pipeline.MhcCleanupPipelineJob; import org.labkey.primeseq.pipeline.SequenceJobResourceAllocator; import org.labkey.primeseq.query.PerformMhcCleanupButton; @@ -121,8 +120,6 @@ public PipelineStartup() SequenceAnalysisService.get().registerFileHandler(new MethylationRateComparisonHandler()); SequenceAnalysisService.get().registerFileHandler(new CombineMethylationRatesHandler()); - SequencePipelineService.get().registerPipelineStep(new GeographicOriginStep.Provider()); - _hasRegistered = true; } } From d00232ef425143e11d048769c28f44e6942641a6 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 23 Feb 2024 05:55:25 -0800 Subject: [PATCH 13/35] Add MCC reference data --- mcc/resources/data/parentageRelationship.tsv | 5 +++++ mcc/resources/etls/mcc.xml | 2 +- mcc/resources/views/populateData.html | 7 +++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 mcc/resources/data/parentageRelationship.tsv diff --git a/mcc/resources/data/parentageRelationship.tsv b/mcc/resources/data/parentageRelationship.tsv new file mode 100644 index 000000000..23efed649 --- /dev/null +++ b/mcc/resources/data/parentageRelationship.tsv @@ -0,0 +1,5 @@ +value +Dam +Sire +Surrogate Dam +Foster Dam \ No newline at end of file diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml index c1462bb7e..d87fe6104 100644 --- a/mcc/resources/etls/mcc.xml +++ b/mcc/resources/etls/mcc.xml @@ -76,7 +76,7 @@ - + Copy to target diff --git a/mcc/resources/views/populateData.html b/mcc/resources/views/populateData.html index ee4e32e04..4cbae6282 100644 --- a/mcc/resources/views/populateData.html +++ b/mcc/resources/views/populateData.html @@ -221,6 +221,13 @@ schemaName: 'ehr_lookups', queryName: 'geographic_origins', pk: 'rowid' + },{ + label: 'Parentage Relationships', + populateFn: 'populateFromFile', + moduleName: 'mcc', + schemaName: 'ehr_lookups', + queryName: 'parentageRelationship', + pk: 'rowid' },{ label: 'Gender Codes', populateFn: 'populateFromFile', From 2024f719eeca8f9dc3616d4ce00bc7893527b85c Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 23 Feb 2024 10:54:22 -0800 Subject: [PATCH 14/35] Initial work on HIVRC module --- hivrc/build.gradle | 15 +++ hivrc/module.properties | 6 + .../folderTypes/HIVRC Analysis.folderType.xml | 34 ++++++ .../folderTypes/HIVRC.folderType.xml | 59 +++++++++ .../queries/laboratory/workbooks.query.xml | 21 ++++ .../postgresql/hivrc-0.00-23.000.sql | 1 + .../dbscripts/sqlserver/hivrc-0.00-23.000.sql | 2 + hivrc/resources/schemas/hivrc.xml | 3 + hivrc/resources/views/analysisHeader.html | 1 + hivrc/resources/views/analysisHeader.view.xml | 6 + .../views/analysisHeader.webpart.xml | 3 + hivrc/resources/views/begin.html | 1 + hivrc/resources/views/begin.view.xml | 5 + hivrc/resources/views/begin.webpart.xml | 3 + hivrc/resources/views/hivrcAnalyses.html | 115 ++++++++++++++++++ hivrc/resources/views/hivrcAnalyses.view.xml | 5 + .../resources/views/hivrcAnalyses.webpart.xml | 6 + hivrc/resources/views/hivrcDatasets.html | 1 + hivrc/resources/views/hivrcDatasets.view.xml | 5 + .../resources/views/hivrcDatasets.webpart.xml | 6 + .../src/org/labkey/hivrc/HivrcController.java | 64 ++++++++++ hivrc/src/org/labkey/hivrc/HivrcManager.java | 16 +++ hivrc/src/org/labkey/hivrc/HivrcModule.java | 69 +++++++++++ hivrc/src/org/labkey/hivrc/HivrcSchema.java | 31 +++++ .../labkey/mgap/buttons/ReleaseButton.java | 3 +- mcc/resources/etls/mcc.xml | 2 +- 26 files changed, 481 insertions(+), 2 deletions(-) create mode 100644 hivrc/build.gradle create mode 100644 hivrc/module.properties create mode 100644 hivrc/resources/folderTypes/HIVRC Analysis.folderType.xml create mode 100644 hivrc/resources/folderTypes/HIVRC.folderType.xml create mode 100644 hivrc/resources/queries/laboratory/workbooks.query.xml create mode 100644 hivrc/resources/schemas/dbscripts/postgresql/hivrc-0.00-23.000.sql create mode 100644 hivrc/resources/schemas/dbscripts/sqlserver/hivrc-0.00-23.000.sql create mode 100644 hivrc/resources/schemas/hivrc.xml create mode 100644 hivrc/resources/views/analysisHeader.html create mode 100644 hivrc/resources/views/analysisHeader.view.xml create mode 100644 hivrc/resources/views/analysisHeader.webpart.xml create mode 100644 hivrc/resources/views/begin.html create mode 100644 hivrc/resources/views/begin.view.xml create mode 100644 hivrc/resources/views/begin.webpart.xml create mode 100644 hivrc/resources/views/hivrcAnalyses.html create mode 100644 hivrc/resources/views/hivrcAnalyses.view.xml create mode 100644 hivrc/resources/views/hivrcAnalyses.webpart.xml create mode 100644 hivrc/resources/views/hivrcDatasets.html create mode 100644 hivrc/resources/views/hivrcDatasets.view.xml create mode 100644 hivrc/resources/views/hivrcDatasets.webpart.xml create mode 100644 hivrc/src/org/labkey/hivrc/HivrcController.java create mode 100644 hivrc/src/org/labkey/hivrc/HivrcManager.java create mode 100644 hivrc/src/org/labkey/hivrc/HivrcModule.java create mode 100644 hivrc/src/org/labkey/hivrc/HivrcSchema.java diff --git a/hivrc/build.gradle b/hivrc/build.gradle new file mode 100644 index 000000000..b065efb11 --- /dev/null +++ b/hivrc/build.gradle @@ -0,0 +1,15 @@ +import org.labkey.gradle.util.BuildUtils +import org.labkey.gradle.util.ExternalDependency + +repositories { + mavenCentral() +} + +dependencies { + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiJarFile") + + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module") +} diff --git a/hivrc/module.properties b/hivrc/module.properties new file mode 100644 index 000000000..0136192fd --- /dev/null +++ b/hivrc/module.properties @@ -0,0 +1,6 @@ +ModuleClass: org.labkey.hivrc.HivrcModule +Label: BMGF HIVRC +Description: This is designed to manage analyses and data collected as part of the BMGF HIVRC +License: Apache 2.0 +LicenseURL: http://www.apache.org/licenses/LICENSE-2.0 +ManageVersion: false diff --git a/hivrc/resources/folderTypes/HIVRC Analysis.folderType.xml b/hivrc/resources/folderTypes/HIVRC Analysis.folderType.xml new file mode 100644 index 000000000..634c02d58 --- /dev/null +++ b/hivrc/resources/folderTypes/HIVRC Analysis.folderType.xml @@ -0,0 +1,34 @@ + + HIVRC Analyses + Workbook template for HIVRC analyses + + + HIVRC Header + body + + + + + Pipeline Files + body + + + + Messages + body + + + + + + HIVRC + laboratory + Pipeline + Experiment + Query + Core + + Core + true + true + \ No newline at end of file diff --git a/hivrc/resources/folderTypes/HIVRC.folderType.xml b/hivrc/resources/folderTypes/HIVRC.folderType.xml new file mode 100644 index 000000000..a57250222 --- /dev/null +++ b/hivrc/resources/folderTypes/HIVRC.folderType.xml @@ -0,0 +1,59 @@ + + HIVRC Overview + The default folder layout for an HIVRC project + + + overview + Overview + + + + + + + HIVRC Data and Analyses + body + + + + + datasets + Datasets + + + datasets + + + + + + + HIVRC Datasets + body + + + + + analyses + Analyses + + + analyses + + + + + + + HIVRC Analyses + body + + + + + + hivrc + + hivrc + true + \ No newline at end of file diff --git a/hivrc/resources/queries/laboratory/workbooks.query.xml b/hivrc/resources/queries/laboratory/workbooks.query.xml new file mode 100644 index 000000000..dab7a4529 --- /dev/null +++ b/hivrc/resources/queries/laboratory/workbooks.query.xml @@ -0,0 +1,21 @@ + + + + + HIVRC Analyses + workbookId + + + Analysis # + + laboratory + workbooks + container + workbookId + + + +
+ + + \ No newline at end of file diff --git a/hivrc/resources/schemas/dbscripts/postgresql/hivrc-0.00-23.000.sql b/hivrc/resources/schemas/dbscripts/postgresql/hivrc-0.00-23.000.sql new file mode 100644 index 000000000..5230cdf8c --- /dev/null +++ b/hivrc/resources/schemas/dbscripts/postgresql/hivrc-0.00-23.000.sql @@ -0,0 +1 @@ +CREATE SCHEMA hivrc; \ No newline at end of file diff --git a/hivrc/resources/schemas/dbscripts/sqlserver/hivrc-0.00-23.000.sql b/hivrc/resources/schemas/dbscripts/sqlserver/hivrc-0.00-23.000.sql new file mode 100644 index 000000000..ff0f8ccc9 --- /dev/null +++ b/hivrc/resources/schemas/dbscripts/sqlserver/hivrc-0.00-23.000.sql @@ -0,0 +1,2 @@ +CREATE SCHEMA hivrc; +GO \ No newline at end of file diff --git a/hivrc/resources/schemas/hivrc.xml b/hivrc/resources/schemas/hivrc.xml new file mode 100644 index 000000000..7918b339b --- /dev/null +++ b/hivrc/resources/schemas/hivrc.xml @@ -0,0 +1,3 @@ + + \ No newline at end of file diff --git a/hivrc/resources/views/analysisHeader.html b/hivrc/resources/views/analysisHeader.html new file mode 100644 index 000000000..05a682bd4 --- /dev/null +++ b/hivrc/resources/views/analysisHeader.html @@ -0,0 +1 @@ +Hello! \ No newline at end of file diff --git a/hivrc/resources/views/analysisHeader.view.xml b/hivrc/resources/views/analysisHeader.view.xml new file mode 100644 index 000000000..9b6d2cc3a --- /dev/null +++ b/hivrc/resources/views/analysisHeader.view.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/hivrc/resources/views/analysisHeader.webpart.xml b/hivrc/resources/views/analysisHeader.webpart.xml new file mode 100644 index 000000000..f4020cc8b --- /dev/null +++ b/hivrc/resources/views/analysisHeader.webpart.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/hivrc/resources/views/begin.html b/hivrc/resources/views/begin.html new file mode 100644 index 000000000..3b188623f --- /dev/null +++ b/hivrc/resources/views/begin.html @@ -0,0 +1 @@ +This is a prototype for a web-based system to manage BMGF datasets and analyses. \ No newline at end of file diff --git a/hivrc/resources/views/begin.view.xml b/hivrc/resources/views/begin.view.xml new file mode 100644 index 000000000..3ea006296 --- /dev/null +++ b/hivrc/resources/views/begin.view.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/hivrc/resources/views/begin.webpart.xml b/hivrc/resources/views/begin.webpart.xml new file mode 100644 index 000000000..5c585de57 --- /dev/null +++ b/hivrc/resources/views/begin.webpart.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/hivrc/resources/views/hivrcAnalyses.html b/hivrc/resources/views/hivrcAnalyses.html new file mode 100644 index 000000000..fab245df6 --- /dev/null +++ b/hivrc/resources/views/hivrcAnalyses.html @@ -0,0 +1,115 @@ + diff --git a/hivrc/resources/views/hivrcAnalyses.view.xml b/hivrc/resources/views/hivrcAnalyses.view.xml new file mode 100644 index 000000000..3dcb2a095 --- /dev/null +++ b/hivrc/resources/views/hivrcAnalyses.view.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/hivrc/resources/views/hivrcAnalyses.webpart.xml b/hivrc/resources/views/hivrcAnalyses.webpart.xml new file mode 100644 index 000000000..06b52f4d0 --- /dev/null +++ b/hivrc/resources/views/hivrcAnalyses.webpart.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/hivrc/resources/views/hivrcDatasets.html b/hivrc/resources/views/hivrcDatasets.html new file mode 100644 index 000000000..e2126d9fd --- /dev/null +++ b/hivrc/resources/views/hivrcDatasets.html @@ -0,0 +1 @@ +This page wil eventually hold a listing of HIVRC datasets, and allow specific files to be identified/queried by ID. \ No newline at end of file diff --git a/hivrc/resources/views/hivrcDatasets.view.xml b/hivrc/resources/views/hivrcDatasets.view.xml new file mode 100644 index 000000000..007700474 --- /dev/null +++ b/hivrc/resources/views/hivrcDatasets.view.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/hivrc/resources/views/hivrcDatasets.webpart.xml b/hivrc/resources/views/hivrcDatasets.webpart.xml new file mode 100644 index 000000000..7c8c99040 --- /dev/null +++ b/hivrc/resources/views/hivrcDatasets.webpart.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/hivrc/src/org/labkey/hivrc/HivrcController.java b/hivrc/src/org/labkey/hivrc/HivrcController.java new file mode 100644 index 000000000..9edd08d94 --- /dev/null +++ b/hivrc/src/org/labkey/hivrc/HivrcController.java @@ -0,0 +1,64 @@ +package org.labkey.hivrc; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.action.ConfirmAction; +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.ModuleProperty; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.URLHelper; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.servlet.ModelAndView; + +public class HivrcController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(HivrcController.class); + + public static final String NAME = "hivrc"; + + public HivrcController() + { + setActionResolver(_actionResolver); + } + + public static class SetupDefaultsAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) throws Exception + { + setTitle("Set up HIVRC"); + + return new HtmlView(HtmlString.unsafe("This will ensure various settings required by HIVRC are configured in this folder. Do you want to continue?")); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + Module m = ModuleLoader.getInstance().getModule("laboratory"); + ModuleProperty mp = m.getModuleProperties().get("DefaultWorkbookFolderType"); + mp.saveValue(getUser(), getContainer(), "HIVRC Analyses"); + + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + + } + + @Override + public @NotNull URLHelper getSuccessURL(Object o) + { + return getContainer().getStartURL(getUser()); + } + } +} diff --git a/hivrc/src/org/labkey/hivrc/HivrcManager.java b/hivrc/src/org/labkey/hivrc/HivrcManager.java new file mode 100644 index 000000000..3f6d95f30 --- /dev/null +++ b/hivrc/src/org/labkey/hivrc/HivrcManager.java @@ -0,0 +1,16 @@ +package org.labkey.hivrc; + +public class HivrcManager +{ + private static final HivrcManager _instance = new HivrcManager(); + + private HivrcManager() + { + + } + + public static HivrcManager get() + { + return _instance; + } +} \ No newline at end of file diff --git a/hivrc/src/org/labkey/hivrc/HivrcModule.java b/hivrc/src/org/labkey/hivrc/HivrcModule.java new file mode 100644 index 000000000..587f6a72f --- /dev/null +++ b/hivrc/src/org/labkey/hivrc/HivrcModule.java @@ -0,0 +1,69 @@ +package org.labkey.hivrc; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.view.WebPartFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +public class HivrcModule extends DefaultModule +{ + public static final String NAME = "HIVRC"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public @Nullable Double getSchemaVersion() + { + return 23.000; + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + return Collections.emptyList(); + } + + @Override + protected void init() + { + addController(HivrcController.NAME, HivrcController.class); + } + + @Override + public void doStartup(ModuleContext moduleContext) + { + + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + return Collections.emptyList(); + } + + @Override + @NotNull + public Set getSchemaNames() + { + return Collections.singleton(HivrcSchema.NAME); + } +} \ No newline at end of file diff --git a/hivrc/src/org/labkey/hivrc/HivrcSchema.java b/hivrc/src/org/labkey/hivrc/HivrcSchema.java new file mode 100644 index 000000000..712835f67 --- /dev/null +++ b/hivrc/src/org/labkey/hivrc/HivrcSchema.java @@ -0,0 +1,31 @@ +package org.labkey.hivrc; + +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.dialect.SqlDialect; + +public class HivrcSchema +{ + private static final HivrcSchema _instance = new HivrcSchema(); + public static final String NAME = "hivrc"; + + public static HivrcSchema getInstance() + { + return _instance; + } + + private HivrcSchema() + { + + } + + public DbSchema getSchema() + { + return DbSchema.get(NAME, DbSchemaType.Module); + } + + public SqlDialect getSqlDialect() + { + return getSchema().getSqlDialect(); + } +} diff --git a/mGAP/src/org/labkey/mgap/buttons/ReleaseButton.java b/mGAP/src/org/labkey/mgap/buttons/ReleaseButton.java index 47a3c437b..8f9dfbead 100644 --- a/mGAP/src/org/labkey/mgap/buttons/ReleaseButton.java +++ b/mGAP/src/org/labkey/mgap/buttons/ReleaseButton.java @@ -14,6 +14,7 @@ public ReleaseButton(Module owner) ClientDependency.supplierFromPath("laboratory.context"), ClientDependency.supplierFromPath("/SequenceAnalysis/window/OutputHandlerWindow.js"), ClientDependency.supplierFromPath("/mGAP/window/ReleaseWindow.js"), - ClientDependency.supplierFromPath("sequenceanalysis/field/GenomeFileSelectorField.js"))); + ClientDependency.supplierFromPath("sequenceanalysis/field/GenomeFileSelectorField.js"), + ClientDependency.supplierFromPath("sequenceanalysis/field/SequenceOutputFileSelectorField.js"))); } } diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml index d87fe6104..104d0bfc5 100644 --- a/mcc/resources/etls/mcc.xml +++ b/mcc/resources/etls/mcc.xml @@ -87,7 +87,7 @@ objectid - + From 03bce5810cb02e6ada32b5ef4374b6a847279912 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 23 Feb 2024 13:35:29 -0800 Subject: [PATCH 15/35] Correct webpart name in hivrc --- hivrc/resources/folderTypes/HIVRC Analysis.folderType.xml | 4 ++-- hivrc/resources/views/analysisHeader.webpart.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hivrc/resources/folderTypes/HIVRC Analysis.folderType.xml b/hivrc/resources/folderTypes/HIVRC Analysis.folderType.xml index 634c02d58..8e4f41dc4 100644 --- a/hivrc/resources/folderTypes/HIVRC Analysis.folderType.xml +++ b/hivrc/resources/folderTypes/HIVRC Analysis.folderType.xml @@ -3,7 +3,7 @@ Workbook template for HIVRC analyses - HIVRC Header + HIVRC Analysis Header body @@ -16,7 +16,7 @@ Messages body - + diff --git a/hivrc/resources/views/analysisHeader.webpart.xml b/hivrc/resources/views/analysisHeader.webpart.xml index f4020cc8b..2e162dfcf 100644 --- a/hivrc/resources/views/analysisHeader.webpart.xml +++ b/hivrc/resources/views/analysisHeader.webpart.xml @@ -1,3 +1,3 @@ - + \ No newline at end of file From c76a32823e53141feaa1db09f55f6445f7d47862 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 26 Feb 2024 14:22:37 -0800 Subject: [PATCH 16/35] Bugfixes to MCC import page --- mcc/resources/web/mcc/panel/MccImportPanel.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/mcc/resources/web/mcc/panel/MccImportPanel.js b/mcc/resources/web/mcc/panel/MccImportPanel.js index 972dbb7bc..8e3b0fd79 100644 --- a/mcc/resources/web/mcc/panel/MccImportPanel.js +++ b/mcc/resources/web/mcc/panel/MccImportPanel.js @@ -62,7 +62,7 @@ Ext4.define('MCC.panel.MccImportPanel', { transform: 'species' },{ name: 'birth', - labels: ['Birth', 'DOB'], + labels: ['Birth', 'DOB', 'DOB (mm/dd/yyyy)'], allowRowSpan: false, allowBlank: true, transform: 'genericDate' @@ -81,13 +81,13 @@ Ext4.define('MCC.panel.MccImportPanel', { transform: 'status' },{ name: 'dam', - labels: ['Dam', 'maternal ID', 'maternal ID (previous)'], + labels: ['Dam', 'maternal ID'], allowRowSpan: false, allowBlank: true, transform: 'damOrSire' },{ name: 'sire', - labels: ['Sire', 'paternal ID', 'paternal ID (previous)'], + labels: ['Sire', 'paternal ID'], allowRowSpan: false, allowBlank: true, transform: 'damOrSire' @@ -481,9 +481,9 @@ Ext4.define('MCC.panel.MccImportPanel', { mergeWithDemographics: function(rows, demographicsRecords, errorMsgs) { Ext4.Array.forEach(rows, function(row){ - row.existingRecord = row.Id && demographicsRecords.allIds.indexOf(row.Id) > -1; + row.existingRecord = row.Id && demographicsRecords.allIds.indexOf(row.Id.toLowerCase()) > -1; if (row.existingRecord) { - var existingRecord = demographicsRecords.rowMap[row.Id]; + var existingRecord = demographicsRecords.rowMap[row.Id.toLowerCase()]; if (existingRecord.colony !== row.colony) { row.errors.push('Colony does not match existing row: ' + existingRecord.colony); } @@ -504,7 +504,6 @@ Ext4.define('MCC.panel.MccImportPanel', { } } - console.log(existingRecord) if (existingRecord['Id/death/date']) { row.existingDeathRecord = true; } @@ -537,8 +536,8 @@ Ext4.define('MCC.panel.MccImportPanel', { ret.idsByCenter[center] = ret.idsByCenter[center] || []; ret.idsByCenter[center].push(row.Id); - ret.allIds.push(row.Id); - ret.rowMap[row.Id] = row; + ret.allIds.push(row.Id.toLowerCase()); + ret.rowMap[row.Id.toLowerCase()] = row; if (row.alternateIds) { var alterateIds = LDK.Utils.textToArray(row.alternateIds); From 52d87a3b78cbfaf482c58bc47dbfe78e6b573907 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 27 Feb 2024 19:22:24 -0800 Subject: [PATCH 17/35] Updates to MCC --- .../queries/study/demographics/.qview.xml | 40 ++++++ hivrc/build.gradle | 2 + hivrc/resources/views/analysisHeader.view.xml | 6 - .../views/analysisHeader.webpart.xml | 3 - mcc/resources/data/reports.tsv | 3 +- mcc/resources/etls/mcc.xml | 2 + .../queries/mcc/aggregatedDemographics.sql | 12 +- .../queries/study/demographics.query.xml | 6 + .../queries/study/demographics/.qview.xml | 2 + .../queries/study/genomicDatasets.js | 2 + .../queries/study/mostRecentObservations.sql | 4 +- .../mostRecentObservationsPivoted.query.xml | 3 + .../study/datasets/datasets_metadata.xml | 6 + .../web/mcc/window/RenameIdWindow.js | 8 +- mcc/src/org/labkey/mcc/MccController.java | 118 ++++++++++++++---- .../labkey/mcc/query/MccEhrCustomizer.java | 27 ++++ 16 files changed, 205 insertions(+), 39 deletions(-) delete mode 100644 hivrc/resources/views/analysisHeader.view.xml delete mode 100644 hivrc/resources/views/analysisHeader.webpart.xml create mode 100644 mcc/resources/queries/study/genomicDatasets.js diff --git a/MccColony/resources/queries/study/demographics/.qview.xml b/MccColony/resources/queries/study/demographics/.qview.xml index 7cc677499..a35d07baa 100644 --- a/MccColony/resources/queries/study/demographics/.qview.xml +++ b/MccColony/resources/queries/study/demographics/.qview.xml @@ -27,6 +27,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hivrc/build.gradle b/hivrc/build.gradle index b065efb11..1c2afe179 100644 --- a/hivrc/build.gradle +++ b/hivrc/build.gradle @@ -9,6 +9,8 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiJarFile") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module") diff --git a/hivrc/resources/views/analysisHeader.view.xml b/hivrc/resources/views/analysisHeader.view.xml deleted file mode 100644 index 9b6d2cc3a..000000000 --- a/hivrc/resources/views/analysisHeader.view.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/hivrc/resources/views/analysisHeader.webpart.xml b/hivrc/resources/views/analysisHeader.webpart.xml deleted file mode 100644 index 2e162dfcf..000000000 --- a/hivrc/resources/views/analysisHeader.webpart.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/mcc/resources/data/reports.tsv b/mcc/resources/data/reports.tsv index d203ed2c1..21a547cd9 100644 --- a/mcc/resources/data/reports.tsv +++ b/mcc/resources/data/reports.tsv @@ -10,4 +10,5 @@ pedigreePlot Genetics report Pedigree Plot TRUE study pedigree module:EHR/sche offspring General query Offspring TRUE study demographicsOffspring FALSE FALSE qcstate/publicdata This report displays pedigree data for animals, including parents, grandparents, siblings and offspring kinship Genetics js Kinship TRUE ehr kinshipSummary FALSE FALSE qcstate/publicdata This report shows the kinship coefficient between every animal in the colony. The kinship coefficient is a measure of relatedness between two individuals. It represents the probability that two genes, sampled at random from each individual are identical (e.g. the kinship coefficient between a parent and an offspring is 0.25). parentage Genetics query Parentage TRUE study demographicsParents FALSE FALSE qcstate/publicdata This report shows information about the parentage of each animal, drawing from genetic data and observations -geneticData Genetics query Genomic Datasets TRUE study genomicDatasets FALSE FALSE qcstate/publicdata This report shows summarizes available genomic datasets +geneticData Genetics query Genomic Datasets TRUE study genomicDatasets FALSE FALSE qcstate/publicdata This report summarizes available genomic datasets +mostRecentObservations General query Most Recent Observations TRUE study mostRecentObservations FALSE FALSE This report summarizes the most recent observations per animal diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml index 104d0bfc5..b4f41a9c4 100644 --- a/mcc/resources/etls/mcc.xml +++ b/mcc/resources/etls/mcc.xml @@ -29,6 +29,8 @@ infant_history fertility_status medical_history + usage_current + usage_future date_of_observations colony dam diff --git a/mcc/resources/queries/mcc/aggregatedDemographics.sql b/mcc/resources/queries/mcc/aggregatedDemographics.sql index cb0de39a9..6434b2f36 100644 --- a/mcc/resources/queries/mcc/aggregatedDemographics.sql +++ b/mcc/resources/queries/mcc/aggregatedDemographics.sql @@ -25,6 +25,8 @@ SELECT o.infant_history, o.fertility_status, o.medical_history, + null as usage_current, + null as usage_future, o.date_of_observations, d.container @@ -36,7 +38,7 @@ LEFT JOIN (SELECT o."current_housing_status::observation" as current_housing_status, o."infant_history::observation" as infant_history, o."fertility_status::observation" as fertility_status, - o."medical_history::observation" as medical_history, + o."medical_history::observation" as medical_history FROM "/data/Colonies/SNPRC/".study.mostRecentObservationsPivoted o ) o ON (o.Id = d.Id) WHERE (d.excludeFromCensus IS NULL or d.excludeFromCensus = false) and d.calculated_status NOT IN ('Other') @@ -70,6 +72,8 @@ SELECT o.infant_history, o.fertility_status, o.medical_history, + null as usage_current, + null as usage_future, o.date_of_observations, d.container @@ -112,6 +116,8 @@ SELECT o.infant_history, o.fertility_status, o.medical_history, + null as usage_current, + null as usage_future, o.date_of_observations, d.container @@ -154,6 +160,8 @@ SELECT o.infant_history, o.fertility_status, o.medical_history, + o.usage_current, + o.usage_future, o.date_of_observations, d.container @@ -166,6 +174,8 @@ FROM "/data/Colonies/Other/".study.demographics d o."infant_history::observation" as infant_history, o."fertility_status::observation" as fertility_status, o."medical_history::observation" as medical_history, + o."usage_current::observation" as usage_current, + o."usage_current::observation" as usage_future, FROM "/data/Colonies/Other/".study.mostRecentObservationsPivoted o ) o ON (o.Id = d.Id) WHERE (d.excludeFromCensus IS NULL or d.excludeFromCensus = false) \ No newline at end of file diff --git a/mcc/resources/queries/study/demographics.query.xml b/mcc/resources/queries/study/demographics.query.xml index 7e2e2457b..3de9b0e1a 100644 --- a/mcc/resources/queries/study/demographics.query.xml +++ b/mcc/resources/queries/study/demographics.query.xml @@ -32,6 +32,12 @@ Medical History + + Usage (Current) + + + Usage (Future) + Date of Observations diff --git a/mcc/resources/queries/study/demographics/.qview.xml b/mcc/resources/queries/study/demographics/.qview.xml index 8c208d409..8950e4cd5 100644 --- a/mcc/resources/queries/study/demographics/.qview.xml +++ b/mcc/resources/queries/study/demographics/.qview.xml @@ -33,6 +33,8 @@ + + diff --git a/mcc/resources/queries/study/genomicDatasets.js b/mcc/resources/queries/study/genomicDatasets.js new file mode 100644 index 000000000..1121c8378 --- /dev/null +++ b/mcc/resources/queries/study/genomicDatasets.js @@ -0,0 +1,2 @@ +require("ehr/triggers").initScript(this); + diff --git a/mcc/resources/queries/study/mostRecentObservations.sql b/mcc/resources/queries/study/mostRecentObservations.sql index 5aaa53c41..f44329c76 100644 --- a/mcc/resources/queries/study/mostRecentObservations.sql +++ b/mcc/resources/queries/study/mostRecentObservations.sql @@ -16,12 +16,12 @@ FROM study.clinical_observations co WHERE co.qcstate.publicdata = true and co.observation is not null AND - co.category IN ('Medical History', 'Fertility Status', 'Infant History', 'Current Housing Status', 'Availability', 'Usage_Current', 'Usage_Future') AND + co.category IN ('Medical History', 'Fertility Status', 'Infant History', 'Current Housing Status', 'Availability', 'Usage_Current', 'Usage_Future', 'Breeding Partner Id') AND co.date = (SELECT max(o.date) asMaxDate FROM study.clinical_observations o WHERE o.Id = co.Id AND o.qcstate.publicdata = true AND - o.category IN ('Medical History', 'Fertility Status', 'Infant History', 'Current Housing Status', 'Availability', 'Usage_Current', 'Usage_Future') AND + o.category IN ('Medical History', 'Fertility Status', 'Infant History', 'Current Housing Status', 'Availability', 'Usage_Current', 'Usage_Future', 'Breeding Partner Id') AND o.observation is not null ) GROUP BY co.id, co.category, co.date diff --git a/mcc/resources/queries/study/mostRecentObservationsPivoted.query.xml b/mcc/resources/queries/study/mostRecentObservationsPivoted.query.xml index dac4e59cb..c5e7098c6 100644 --- a/mcc/resources/queries/study/mostRecentObservationsPivoted.query.xml +++ b/mcc/resources/queries/study/mostRecentObservationsPivoted.query.xml @@ -4,7 +4,10 @@ Most Recent Observations + + true +
diff --git a/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml b/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml index 0ee7322e1..912def432 100644 --- a/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -661,6 +661,12 @@ varchar + + varchar + + + varchar + timestamp diff --git a/mcc/resources/web/mcc/window/RenameIdWindow.js b/mcc/resources/web/mcc/window/RenameIdWindow.js index 26f9ba43e..0288e6fa7 100644 --- a/mcc/resources/web/mcc/window/RenameIdWindow.js +++ b/mcc/resources/web/mcc/window/RenameIdWindow.js @@ -85,9 +85,13 @@ Ext4.define('MCC.window.RenameIdWindow', { success: LABKEY.Utils.getCallbackWrapper(function (response) { Ext4.Msg.hide(); this.close(); - console.log(response); - Ext4.Msg.alert('Success', 'Total IDs Updated: ' + response.totalIdsUpdated + '
Total Records Updated: ' + response.totalRecordsUpdated); + if (response.success) { + Ext4.Msg.alert('Success', 'Total IDs Updated: ' + response.totalIdsUpdated + '
Total Records Updated: ' + response.totalRecordsUpdated + (response.messages ? '
' + response.messages : '')); + } + else { + Ext4.Msg.alert('Error', 'Error renaming IDs: ' + response.error); + } }), failure: LDK.Utils.getErrorCallback() }); diff --git a/mcc/src/org/labkey/mcc/MccController.java b/mcc/src/org/labkey/mcc/MccController.java index 67aaedd2a..ceb3b3ab4 100644 --- a/mcc/src/org/labkey/mcc/MccController.java +++ b/mcc/src/org/labkey/mcc/MccController.java @@ -44,6 +44,7 @@ import org.labkey.api.query.DetailsURL; import org.labkey.api.query.FieldKey; import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.QueryService; import org.labkey.api.query.QueryUpdateServiceException; import org.labkey.api.security.AuthenticationManager; import org.labkey.api.security.Group; @@ -828,43 +829,112 @@ public Object execute(RenameIdsForm form, BindException errors) throws Exception final Set idsUpdated = new CaseInsensitiveHashSet(); final AtomicInteger totalRecordsUpdated = new AtomicInteger(); + final TableInfo mccAliases = QueryService.get().getUserSchema(getUser(), getContainer(), MccSchema.NAME).getTable(MccSchema.TABLE_ANIMAL_MAPPING); - // NOTE: include this value so it will get added to the audit trail. This is a loose way to connect changes made in this transaction - final String batchId = "MCC.Rename." + new GUID(); - for (Dataset ds : s.getDatasets()) + try (DbScope.Transaction transaction = DbScope.getLabKeyScope().ensureTransaction()) { - TableInfo ti = ds.getTableInfo(getUser()); - TableSelector ts = new TableSelector(ti, PageFlowUtil.set("Id", "lsid"), new SimpleFilter(FieldKey.fromString("Id"), oldToNew.keySet(), CompareType.IN), null); - if (!ts.exists()) + // NOTE: include this value so it will get added to the audit trail. This is a loose way to connect changes made in this transaction + final String batchId = "MCC.Rename." + new GUID(); + Set messages = new HashSet<>(); + for (Dataset ds : s.getDatasets()) { - continue; - } + TableInfo ti = ds.getTableInfo(getUser()); + TableSelector ts = new TableSelector(ti, PageFlowUtil.set("Id", "lsid"), new SimpleFilter(FieldKey.fromString("Id"), oldToNew.keySet(), CompareType.IN), null); + if (!ts.exists()) + { + continue; + } + + List> toUpdate = new ArrayList<>(); + List> oldKeys = new ArrayList<>(); + ts.forEachResults(rs -> { + if (ds.isDemographicData()) + { + // test if a record exists with the new ID + if (new TableSelector(ti, new SimpleFilter(FieldKey.fromString("Id"), oldToNew.get(rs.getString(FieldKey.fromString("Id")))), null).exists()) + { + messages.add("Existing record for ID: " + oldToNew.get(rs.getString(FieldKey.fromString("Id"))) + " for dataset: " + ds.getLabel() + " in container: " + getContainer().getPath() + ", skipping rename"); + return; + } + } - List> toUpdate = new ArrayList<>(); - List> oldKeys = new ArrayList<>(); - ts.forEachResults(rs -> { - toUpdate.add(Map.of("lsid", rs.getString(FieldKey.fromString("lsid")), "Id", oldToNew.get(rs.getString(FieldKey.fromString("Id"))), "_batchId_", batchId)); - oldKeys.add(Map.of("lsid", rs.getString(FieldKey.fromString("lsid")))); - idsUpdated.add(rs.getString(FieldKey.fromString("Id"))); - totalRecordsUpdated.getAndIncrement(); - }); + toUpdate.add(Map.of("lsid", rs.getString(FieldKey.fromString("lsid")), "Id", oldToNew.get(rs.getString(FieldKey.fromString("Id"))), "_batchId_", batchId)); + oldKeys.add(Map.of("lsid", rs.getString(FieldKey.fromString("lsid")))); + idsUpdated.add(rs.getString(FieldKey.fromString("Id"))); + totalRecordsUpdated.getAndIncrement(); + }); + + if (!toUpdate.isEmpty()) + { + try + { + ti.getUpdateService().updateRows(getUser(), getContainer(), toUpdate, oldKeys, null, null); + } + catch (InvalidKeyException | BatchValidationException | QueryUpdateServiceException | SQLException e) + { + _log.error("Error updating MCC dataset rows", e); + errors.reject(ERROR_MSG, "Error updating MCC dataset rows: " + e.getMessage()); + return null; + } + } + } - if (!toUpdate.isEmpty()) + for (String oldId : oldToNew.keySet()) { - try + // Find the MCC ID of the new ID: + String mccIdForOldId = new TableSelector(mccAliases, PageFlowUtil.set("externalAlias"), new SimpleFilter(FieldKey.fromString("subjectname"), oldId), null).getObject(String.class); + String mccIdForNewId = new TableSelector(mccAliases, PageFlowUtil.set("externalAlias"), new SimpleFilter(FieldKey.fromString("subjectname"), oldToNew.get(oldId)), null).getObject(String.class); + + if (mccIdForOldId == null) + { + // This should not really happen... + _log.error("An MCC rename was performed where the original ID lacked an MCC alias: " + oldId); + messages.add("Missing MCC alias: " + oldId); + } + + if (mccIdForNewId == null) { - ti.getUpdateService().updateRows(getUser(), getContainer(), toUpdate, oldKeys, null, null); + if (mccIdForOldId != null) + { + // Create record for the new ID pointing to the MCC ID of the original + List> toInsert = Arrays.asList(Map.of( + "subjectname", mccIdForOldId, + "_batchId_", batchId + )); + BatchValidationException bve = new BatchValidationException(); + mccAliases.getUpdateService().insertRows(getUser(), getContainer(), toInsert, bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } + } } - catch (InvalidKeyException | BatchValidationException | QueryUpdateServiceException | SQLException e) + else if (mccIdForOldId != null) { - _log.error("Error updating MCC dataset rows", e); - errors.reject(ERROR_MSG, "Error updating MCC dataset rows: " + e.getMessage()); - return null; + messages.add("Both IDs have existing MCC aliases, no changes were made: " + oldId + " / " + oldToNew.get(oldId)); } } + + transaction.commit(); + + return new ApiSimpleResponse(Map.of( + "success", true, + "totalIdsUpdated", idsUpdated.size(), + "totalRecordsUpdated", totalRecordsUpdated.get(), + "messages", StringUtils.join(messages, "
") + )); + } + catch (Exception e) + { + _log.error("Error renaming MCC IDs", e); + + return new ApiSimpleResponse(Map.of( + "success", false, + "error", e.getMessage() + )); } - return new ApiSimpleResponse(Map.of("success", true, "totalIdsUpdated", idsUpdated.size(), "totalRecordsUpdated", totalRecordsUpdated.get())); + } } diff --git a/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java b/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java index 792f0669f..ff6a0d31c 100644 --- a/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java +++ b/mcc/src/org/labkey/mcc/query/MccEhrCustomizer.java @@ -3,8 +3,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.BaseColumnInfo; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MutableColumnInfo; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.TableInfo; import org.labkey.api.data.WrappedColumn; @@ -12,6 +14,7 @@ import org.labkey.api.query.ExprColumn; import org.labkey.api.query.FieldKey; import org.labkey.api.query.LookupForeignKey; +import org.labkey.api.query.QueryForeignKey; import org.labkey.api.query.UserSchema; import org.labkey.mcc.MccSchema; @@ -84,6 +87,30 @@ private void customizeAnimalTable(AbstractTableInfo ti) } addMccAlias(ti, "Id", "mccAlias", "MCC Alias"); + + if (ti.getColumn("mostRecentObservations") == null) + { + MutableColumnInfo col = getWrappedIdCol(ti.getUserSchema(), ti, "mostRecentObservations", "mostRecentObservationsPivoted"); + col.setLabel("Most Recent Observations"); + col.setDescription("Displays the most recent observation of each category"); + ti.addColumn(col); + } + + } + + private BaseColumnInfo getWrappedIdCol(UserSchema us, AbstractTableInfo ds, String name, String queryName) + { + + String colName = "Id"; + String targetCol = "Id"; + + WrappedColumn col = new WrappedColumn(ds.getColumn(colName), name); + col.setReadOnly(true); + col.setIsUnselectable(true); + col.setUserEditable(false); + col.setFk(new QueryForeignKey(us, null, queryName, targetCol, targetCol)); + + return col; } private void addMccAlias(AbstractTableInfo ti, String sourceCol, String name, String label) From 79d3ef36dbab76c0333130b10ed50edc98ad3cf1 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 28 Feb 2024 11:15:17 -0800 Subject: [PATCH 18/35] Improve HIVRC header panel --- .../queries/laboratory/workbooks.query.xml | 8 +- .../web/hivrc/panel/AnalysisHeaderPanel.js | 46 ++++++ .../src/org/labkey/hivrc/HivrcController.java | 2 + hivrc/src/org/labkey/hivrc/HivrcManager.java | 45 ++++++ hivrc/src/org/labkey/hivrc/HivrcModule.java | 42 +++++- hivrc/src/org/labkey/hivrc/HivrcSchema.java | 9 ++ .../org/labkey/hivrc/query/AnalysisModel.java | 140 ++++++++++++++++++ .../org/labkey/hivrc/view/analysisHeader.jsp | 75 ++++++++++ mcc/resources/views/mccColonies.html | 52 +++++++ mcc/resources/views/mccColonies.view.xml | 8 + mcc/resources/views/mccColonies.webpart.xml | 6 + 11 files changed, 424 insertions(+), 9 deletions(-) create mode 100644 hivrc/resources/web/hivrc/panel/AnalysisHeaderPanel.js create mode 100644 hivrc/src/org/labkey/hivrc/query/AnalysisModel.java create mode 100644 hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp create mode 100644 mcc/resources/views/mccColonies.html create mode 100644 mcc/resources/views/mccColonies.view.xml create mode 100644 mcc/resources/views/mccColonies.webpart.xml diff --git a/hivrc/resources/queries/laboratory/workbooks.query.xml b/hivrc/resources/queries/laboratory/workbooks.query.xml index dab7a4529..fbee6d1d8 100644 --- a/hivrc/resources/queries/laboratory/workbooks.query.xml +++ b/hivrc/resources/queries/laboratory/workbooks.query.xml @@ -5,14 +5,8 @@ HIVRC Analyses workbookId - + Analysis # - - laboratory - workbooks - container - workbookId - diff --git a/hivrc/resources/web/hivrc/panel/AnalysisHeaderPanel.js b/hivrc/resources/web/hivrc/panel/AnalysisHeaderPanel.js new file mode 100644 index 000000000..4ca67d407 --- /dev/null +++ b/hivrc/resources/web/hivrc/panel/AnalysisHeaderPanel.js @@ -0,0 +1,46 @@ +/** + * @param description + * @param materials + * @param methods + * @param results + */ +Ext4.define('HIVRC.panel.AnalysisHeaderPanel', { + extend: 'Laboratory.panel.WorkbookHeaderPanel', + panelTitle: 'Analysis Details', + + getPanelItems: function() { + return [{ + html: 'Analysis Description:', + style: 'font-weight: bold;' + }, + this.getFieldCfg('description', this.description) + ,{ + html: 'Results:', + style: 'font-weight: bold;padding-top: 10px;' + }, + this.getFieldCfg('results', this.results) + ,{ + html: 'Tags:', + style: 'font-weight: bold;padding-top: 10px;' + }, + this.getTagFieldCfg() + ] + }, + + onUpdate: function(){ + var values = {}; + Ext4.each(['description', 'results'], function(text){ + values[text] = this.down('#' + text).getValue(); + }, this); + + values.tags = this.tags; + values.forceTagUpdate = true; + + LABKEY.Ajax.request({ + url: LABKEY.ActionURL.buildURL('laboratory', 'updateWorkbook'), + method: 'POST', + params: values, + failure: LDK.Utils.getErrorCallback() + }); + } +}); \ No newline at end of file diff --git a/hivrc/src/org/labkey/hivrc/HivrcController.java b/hivrc/src/org/labkey/hivrc/HivrcController.java index 9edd08d94..624163ff2 100644 --- a/hivrc/src/org/labkey/hivrc/HivrcController.java +++ b/hivrc/src/org/labkey/hivrc/HivrcController.java @@ -8,6 +8,7 @@ import org.labkey.api.module.ModuleLoader; import org.labkey.api.module.ModuleProperty; import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.permissions.AdminPermission; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.util.HtmlString; import org.labkey.api.util.URLHelper; @@ -29,6 +30,7 @@ public HivrcController() setActionResolver(_actionResolver); } + @RequiresPermission(AdminPermission.class) public static class SetupDefaultsAction extends ConfirmAction { @Override diff --git a/hivrc/src/org/labkey/hivrc/HivrcManager.java b/hivrc/src/org/labkey/hivrc/HivrcManager.java index 3f6d95f30..0407b5be5 100644 --- a/hivrc/src/org/labkey/hivrc/HivrcManager.java +++ b/hivrc/src/org/labkey/hivrc/HivrcManager.java @@ -1,5 +1,14 @@ package org.labkey.hivrc; +import org.labkey.api.data.Container; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.query.FieldKey; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.hivrc.query.AnalysisModel; + public class HivrcManager { private static final HivrcManager _instance = new HivrcManager(); @@ -13,4 +22,40 @@ public static HivrcManager get() { return _instance; } + + public AnalysisModel getAnalysisModel(Container c, boolean createIfNotPresent) + { + if (!c.isWorkbook()) + { + return null; + } + + TableInfo ti = HivrcSchema.getInstance().getLaboratorySchema().getTable(HivrcSchema.TABLE_WORKBOOKS); + TableSelector ts = new TableSelector(ti, new SimpleFilter(FieldKey.fromString("container"), c.getId()), null); + AnalysisModel[] arr = ts.getArray(AnalysisModel.class); + AnalysisModel m = arr.length == 0 ? null : arr[0]; + if (m == null) + { + if (createIfNotPresent) + { + m = AnalysisModel.createNew(c); + } + else + { + return null; + } + } + + if (m.getContainer() == null) + { + m.setContainer(c.getId()); + } + + TableInfo ti2 = HivrcSchema.getInstance().getLaboratorySchema().getTable(HivrcSchema.TABLE_WORKBOOK_TAGS); + TableSelector ts2 = new TableSelector(ti2, PageFlowUtil.set("tag"), new SimpleFilter(FieldKey.fromString("container"), c.getId()), null); + String[] arr2 = ts2.getArray(String.class); + m.setTags(arr2); + + return m; + } } \ No newline at end of file diff --git a/hivrc/src/org/labkey/hivrc/HivrcModule.java b/hivrc/src/org/labkey/hivrc/HivrcModule.java index 587f6a72f..673f1c9fb 100644 --- a/hivrc/src/org/labkey/hivrc/HivrcModule.java +++ b/hivrc/src/org/labkey/hivrc/HivrcModule.java @@ -3,11 +3,19 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; import org.labkey.api.module.DefaultModule; import org.labkey.api.module.ModuleContext; +import org.labkey.api.util.HtmlString; +import org.labkey.api.view.BaseWebPartFactory; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.Portal; +import org.labkey.api.view.ViewContext; import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; +import org.labkey.hivrc.query.AnalysisModel; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Set; @@ -38,7 +46,37 @@ public boolean hasScripts() @NotNull protected Collection createWebPartFactories() { - return Collections.emptyList(); + return Arrays.asList( + new BaseWebPartFactory("HIVRC Analysis Header") + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + if (!portalCtx.getContainer().isWorkbook()) + { + return new HtmlView(HtmlString.of("This container is not a workbook")); + } + + AnalysisModel model = HivrcManager.get().getAnalysisModel(portalCtx.getContainer(), true); + if (model == null) + { + model = AnalysisModel.createNew(portalCtx.getContainer()); + } + + JspView view = new JspView<>("/org/labkey/hivrc/view/analysisHeader.jsp", model); + view.setTitle("HIVRC Analysis Summary"); + view.setFrame(WebPartView.FrameType.NONE); + + return view; + } + + @Override + public boolean isAvailable(Container c, String scope, String location) + { + return false; + } + } + ); } @Override diff --git a/hivrc/src/org/labkey/hivrc/HivrcSchema.java b/hivrc/src/org/labkey/hivrc/HivrcSchema.java index 712835f67..ee11da04e 100644 --- a/hivrc/src/org/labkey/hivrc/HivrcSchema.java +++ b/hivrc/src/org/labkey/hivrc/HivrcSchema.java @@ -6,6 +6,10 @@ public class HivrcSchema { + public static final String LABORATORY = "laboratory"; + public static final String TABLE_WORKBOOKS = "workbooks"; + public static final String TABLE_WORKBOOK_TAGS = "workbook_tags"; + private static final HivrcSchema _instance = new HivrcSchema(); public static final String NAME = "hivrc"; @@ -24,6 +28,11 @@ public DbSchema getSchema() return DbSchema.get(NAME, DbSchemaType.Module); } + public DbSchema getLaboratorySchema() + { + return DbSchema.get(LABORATORY, DbSchemaType.Module); + } + public SqlDialect getSqlDialect() { return getSchema().getSqlDialect(); diff --git a/hivrc/src/org/labkey/hivrc/query/AnalysisModel.java b/hivrc/src/org/labkey/hivrc/query/AnalysisModel.java new file mode 100644 index 000000000..bc6dfe788 --- /dev/null +++ b/hivrc/src/org/labkey/hivrc/query/AnalysisModel.java @@ -0,0 +1,140 @@ +package org.labkey.hivrc.query; + +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; + +/** + * User: bimber + * Date: 3/30/13 + * Time: 11:58 AM + */ +public class AnalysisModel +{ + private Integer _workbookId; + private String _containerId; + private String _materials; + private String _methods; + private String _results; + private String _exptGroup; + private String[] _tags; + + public AnalysisModel() + { + + } + + public Integer getWorkbookId() + { + return _workbookId; + } + + public void setWorkbookId(Integer workbookId) + { + _workbookId = workbookId; + } + + public String getContainer() + { + return _containerId; + } + + public void setContainer(String container) + { + _containerId = container; + } + + public String getMaterials() + { + return _materials; + } + + public void setMaterials(String materials) + { + _materials = materials; + } + + public String getMethods() + { + return _methods; + } + + public void setMethods(String methods) + { + _methods = methods; + } + + public String getResults() + { + return _results; + } + + public void setResults(String results) + { + _results = results; + } + + public String getExptGroup() + { + return _exptGroup; + } + + public void setExptGroup(String exptGroup) + { + _exptGroup = exptGroup; + } + + private Container _getContainer() + { + if (_containerId == null) + throw new IllegalArgumentException("The containerId has not been set"); + + Container c = ContainerManager.getForId(_containerId); + if (c == null) + throw new IllegalArgumentException("Unknown container: " + _containerId); + + return c; + } + + public void setDescription(String description, User u) + { + try + { + ContainerManager.updateDescription(_getContainer(), description, u); + } + catch (ValidationException e) + { + throw new IllegalArgumentException(e.getMessage()); + } + } + + public String getDescription() + { + return _getContainer().getDescription(); + } + + public String[] getTags() + { + return _tags; + } + + public void setTags(String[] tags) + { + _tags = tags; + } + + public static AnalysisModel createNew(Container c) + { + if (!c.isWorkbook()) + { + throw new IllegalArgumentException("Container is not a workbook: " + c.getPath()); + } + + AnalysisModel model = new AnalysisModel(); + model.setWorkbookId(Integer.parseInt(c.getName())); + model.setContainer(c.getId()); + + return model; + } +} diff --git a/hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp b/hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp new file mode 100644 index 000000000..7ae9e255a --- /dev/null +++ b/hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp @@ -0,0 +1,75 @@ +<%@ page import="org.apache.commons.lang3.StringUtils" %> +<%@ page import="org.labkey.api.view.HttpView" %> +<%@ page import="org.labkey.api.view.JspView" %> +<%@ page import="org.labkey.api.view.template.ClientDependencies" %> +<%@ page import="org.labkey.hivrc.query.AnalysisModel" %> +<%@ page import="java.util.Arrays" %> +<%@ page extends="org.labkey.api.jsp.JspBase" %> +<%! + @Override + public void addClientDependencies(ClientDependencies dependencies) + { + dependencies.add("Ext4"); + dependencies.add("laboratory.context"); + dependencies.add("laboratory/panel/WorkbookHeaderPanel.js"); + dependencies.add("hivrc/panel/AnalysisHeaderPanel.js"); + dependencies.add("editInPlaceElement.css"); + dependencies.add("editInPlaceElement.js"); + } +%> +<% + JspView me = (JspView) HttpView.currentView(); + AnalysisModel model = me.getModelBean(); + Integer workbookId = model.getWorkbookId(); + String wpId = "wp_" + me.getWebPartRowId(); +%> + + + +
>
+ + diff --git a/mcc/resources/views/mccColonies.html b/mcc/resources/views/mccColonies.html new file mode 100644 index 000000000..f34ee393f --- /dev/null +++ b/mcc/resources/views/mccColonies.html @@ -0,0 +1,52 @@ + \ No newline at end of file diff --git a/mcc/resources/views/mccColonies.view.xml b/mcc/resources/views/mccColonies.view.xml new file mode 100644 index 000000000..e55aaf566 --- /dev/null +++ b/mcc/resources/views/mccColonies.view.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mcc/resources/views/mccColonies.webpart.xml b/mcc/resources/views/mccColonies.webpart.xml new file mode 100644 index 000000000..df91e7d75 --- /dev/null +++ b/mcc/resources/views/mccColonies.webpart.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file From c11705d535671a3eb445f296652f4f9434622d91 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 29 Feb 2024 05:24:30 -0800 Subject: [PATCH 19/35] Update MCC reports.tsv --- mcc/resources/data/reports.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcc/resources/data/reports.tsv b/mcc/resources/data/reports.tsv index 21a547cd9..ceb98134d 100644 --- a/mcc/resources/data/reports.tsv +++ b/mcc/resources/data/reports.tsv @@ -1,6 +1,6 @@ reportname category reporttype reporttitle visible containerpath schemaname queryname viewname report datefieldname todayonly queryhaslocation sort_order QCStateLabelFieldName description birth General query Birth Records TRUE study birth date FALSE FALSE qcstate/publicdata Birth records -weight General js Weights TRUE study weightGraph date FALSE FALSE qcstate/publicdata This report contains a summary of the animal\'s weight, including a graph +weight General query Weights TRUE study weight date FALSE FALSE qcstate/publicdata This report contains a summary of the animal\'s weight Flags General query Flags TRUE study flags StartDate FALSE FALSE qcstate/publicdata Animal attribute flags demographics General query Demographics TRUE study demographics FALSE FALSE qcstate/publicdata This report displays the demographics data about each animal including species, sex and birth death General query Death Records TRUE study deaths date FALSE FALSE qcstate/publicdata Death records From 2ec74231564dcf14f4f9168aa0fcbaaf473a83ac Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 29 Feb 2024 08:46:56 -0800 Subject: [PATCH 20/35] Correct typo --- mcc/src/client/AnimalRequest/animal-request.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcc/src/client/AnimalRequest/animal-request.tsx b/mcc/src/client/AnimalRequest/animal-request.tsx index ccacef6f6..f95c74c8d 100644 --- a/mcc/src/client/AnimalRequest/animal-request.tsx +++ b/mcc/src/client/AnimalRequest/animal-request.tsx @@ -704,7 +704,7 @@ export function AnimalRequest() { - + <Title text={"10. " + commentsPlaceholder}/> <div className="tw-w-full tw-px-3 tw-mb-6"> <ErrorMessageHandler isSubmitting={isSubmitting}> <div className="tw-w-full tw-px-3 tw-mb-6"> From 8513f1af9cb1cb7deb8b0074522ed74658553a47 Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Thu, 29 Feb 2024 12:55:52 -0800 Subject: [PATCH 21/35] Refactor LaboratorySerivce to allow registration of DemographicsProvider and perform minor refactors related to generics --- .../queries/mGAP/combinedPedigree.query.xml | 14 ++++++++ .../queries/mGAP/combinedPedigree.sql | 22 ++++++++++++ .../dbscripts/postgresql/mgap-16.72-16.73.sql | 2 ++ .../dbscripts/sqlserver/mgap-16.72-16.73.sql | 2 ++ mGAP/resources/schemas/mgap.xml | 6 ++++ .../labkey/mgap/mGAPDemographicsProvider.java | 34 +++++++++++++++++++ .../labkey/mgap/mGAPDemographicsSource.java | 1 - mGAP/src/org/labkey/mgap/mGAPModule.java | 5 ++- .../labkey/mgap/pipeline/AnnotationStep.java | 2 +- .../mgap/pipeline/GenerateMgapTracksStep.java | 2 +- .../mgap/pipeline/GeographicOriginStep.java | 7 ++-- .../RemoveAnnotationsForMgapStep.java | 2 +- .../mgap/pipeline/RemoveAnnotationsStep.java | 2 +- .../pipeline/RenameSamplesForMgapStep.java | 2 +- 14 files changed, 94 insertions(+), 9 deletions(-) create mode 100644 mGAP/resources/queries/mGAP/combinedPedigree.query.xml create mode 100644 mGAP/resources/queries/mGAP/combinedPedigree.sql create mode 100644 mGAP/resources/schemas/dbscripts/postgresql/mgap-16.72-16.73.sql create mode 100644 mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.72-16.73.sql create mode 100644 mGAP/src/org/labkey/mgap/mGAPDemographicsProvider.java diff --git a/mGAP/resources/queries/mGAP/combinedPedigree.query.xml b/mGAP/resources/queries/mGAP/combinedPedigree.query.xml new file mode 100644 index 000000000..c5171b024 --- /dev/null +++ b/mGAP/resources/queries/mGAP/combinedPedigree.query.xml @@ -0,0 +1,14 @@ +<query xmlns="http://labkey.org/data/xml/query"> + <metadata> + <tables xmlns="http://labkey.org/data/xml"> + <table tableName="" tableDbType="TABLE"> + <tableTitle>mGAP Combined Pedigree</tableTitle> + <columns> + <column columnName="subjectname"> + <isKeyField>true</isKeyField> + </column> + </columns> + </table> + </tables> + </metadata> +</query> \ No newline at end of file diff --git a/mGAP/resources/queries/mGAP/combinedPedigree.sql b/mGAP/resources/queries/mGAP/combinedPedigree.sql new file mode 100644 index 000000000..55512965a --- /dev/null +++ b/mGAP/resources/queries/mGAP/combinedPedigree.sql @@ -0,0 +1,22 @@ +SELECT + s.subjectname, + s.gender, + s.mother as dam, + s.father as sire, + s.species, + s.geographic_origin, + +FROM laboratory.subjects s + +UNION ALL + +SELECT + d.subjectname, + d.gender, + d.dam, + d.sire, + d.species, + null as geographic_origin + +FROM mgap.demographics d +WHERE d.subjectname NOT IN (SELECT DISTINCT s.subjectname FROM laboratory.subjects s) \ No newline at end of file diff --git a/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.72-16.73.sql b/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.72-16.73.sql new file mode 100644 index 000000000..52699de10 --- /dev/null +++ b/mGAP/resources/schemas/dbscripts/postgresql/mgap-16.72-16.73.sql @@ -0,0 +1,2 @@ +ALTER TABLE mGAP.demographics ADD sire varchar(1000); +ALTER TABLE mGAP.demographics ADD dam varchar(1000); \ No newline at end of file diff --git a/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.72-16.73.sql b/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.72-16.73.sql new file mode 100644 index 000000000..52699de10 --- /dev/null +++ b/mGAP/resources/schemas/dbscripts/sqlserver/mgap-16.72-16.73.sql @@ -0,0 +1,2 @@ +ALTER TABLE mGAP.demographics ADD sire varchar(1000); +ALTER TABLE mGAP.demographics ADD dam varchar(1000); \ No newline at end of file diff --git a/mGAP/resources/schemas/mgap.xml b/mGAP/resources/schemas/mgap.xml index 56c761a40..24bfd6332 100644 --- a/mGAP/resources/schemas/mgap.xml +++ b/mGAP/resources/schemas/mgap.xml @@ -1060,6 +1060,12 @@ <fkDisplayColumnName useRawValue="true"/> </fk> </column> + <column columnName="sire"> + <columnTitle>Sire</columnTitle> + </column> + <column columnName="dam"> + <columnTitle>Dam</columnTitle> + </column> <column columnName="center"> <columnTitle>Center</columnTitle> </column> diff --git a/mGAP/src/org/labkey/mgap/mGAPDemographicsProvider.java b/mGAP/src/org/labkey/mgap/mGAPDemographicsProvider.java new file mode 100644 index 000000000..b59a6ec3d --- /dev/null +++ b/mGAP/src/org/labkey/mgap/mGAPDemographicsProvider.java @@ -0,0 +1,34 @@ +package org.labkey.mgap; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.laboratory.DemographicsProvider; +import org.labkey.api.module.ModuleLoader; + +public class mGAPDemographicsProvider extends DemographicsProvider +{ + public mGAPDemographicsProvider() + { + super(ModuleLoader.getInstance().getModule(mGAPModule.class), mGAPSchema.NAME, "combinedPedigree", "subjectname"); + } + + @Nullable + @Override + public String getMotherField() + { + return "dam"; + } + + @Nullable + @Override + public String getFatherField() + { + return "sire"; + } + + @Nullable + @Override + public String getSexField() + { + return "gender"; + } +} diff --git a/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java b/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java index 97047cd49..584db3e88 100644 --- a/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java +++ b/mGAP/src/org/labkey/mgap/mGAPDemographicsSource.java @@ -24,7 +24,6 @@ public class mGAPDemographicsSource implements DemographicsSource { - @Override public Map<String, Map<String, Object>> resolveSubjects(List<String> subjects, Container c, User u) { diff --git a/mGAP/src/org/labkey/mgap/mGAPModule.java b/mGAP/src/org/labkey/mgap/mGAPModule.java index 6eff0b5b0..affb5cbf7 100644 --- a/mGAP/src/org/labkey/mgap/mGAPModule.java +++ b/mGAP/src/org/labkey/mgap/mGAPModule.java @@ -27,6 +27,7 @@ import org.labkey.api.data.TableSelector; import org.labkey.api.data.UpgradeCode; import org.labkey.api.jbrowse.JBrowseService; +import org.labkey.api.laboratory.LaboratoryService; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.ldk.LDKService; import org.labkey.api.ldk.buttons.ShowBulkEditButton; @@ -75,7 +76,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 16.72; + return 16.73; } @Override @@ -102,6 +103,8 @@ public void doStartupAfterSpringConfig(ModuleContext moduleContext) JBrowseService.get().registerGroupsProvider(new mGAPGroupsProvider()); JBrowseService.get().registerLuceneIndexDetector(new mGAPLuceneDetector()); + LaboratoryService.get().registerDemographicsProvider(new mGAPDemographicsProvider()); + SystemMaintenance.addTask(new mGapMaintenanceTask()); new PipelineStartup(); diff --git a/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java b/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java index b5edd6da6..96dd56b56 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/AnnotationStep.java @@ -112,7 +112,7 @@ public Provider() } @Override - public PipelineStep create(PipelineContext context) + public AnnotationStep create(PipelineContext context) { return new AnnotationStep(this, context); } diff --git a/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java b/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java index 5b28df51a..df573c6ec 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/GenerateMgapTracksStep.java @@ -86,7 +86,7 @@ public Provider() } @Override - public PipelineStep create(PipelineContext context) + public GenerateMgapTracksStep create(PipelineContext context) { return new GenerateMgapTracksStep(this, context); } diff --git a/mGAP/src/org/labkey/mgap/pipeline/GeographicOriginStep.java b/mGAP/src/org/labkey/mgap/pipeline/GeographicOriginStep.java index d98410cbd..2fa3825be 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/GeographicOriginStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/GeographicOriginStep.java @@ -16,6 +16,7 @@ import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStep; import org.labkey.api.sequenceanalysis.pipeline.AbstractVariantProcessingStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.PedigreeToolParameterDescriptor; import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; @@ -25,6 +26,7 @@ import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStep; import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStepOutputImpl; import org.labkey.api.sequenceanalysis.run.AbstractDiscvrSeqWrapper; +import org.labkey.api.util.PageFlowUtil; import org.labkey.mgap.mGAPSchema; import javax.annotation.Nullable; @@ -61,8 +63,9 @@ public Provider() {{ put("allowBlank", false); }}, null), - ToolParameterDescriptor.create("storeResults", "Store Results", "If checked, the results will be stored in the database", "checkbox", null, null) - ), null, "https://bimberlab.github.io/DISCVRSeq/"); + ToolParameterDescriptor.create("storeResults", "Store Results", "If checked, the results will be stored in the database", "checkbox", null, null), + new PedigreeToolParameterDescriptor() + ), PageFlowUtil.set(PedigreeToolParameterDescriptor.getClientDependencyPath()), "https://bimberlab.github.io/DISCVRSeq/"); } @Override diff --git a/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsForMgapStep.java b/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsForMgapStep.java index 4b83e59ac..54f58e519 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsForMgapStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsForMgapStep.java @@ -45,7 +45,7 @@ public Provider() } @Override - public PipelineStep create(PipelineContext context) + public RemoveAnnotationsForMgapStep create(PipelineContext context) { return new RemoveAnnotationsForMgapStep(this, context); } diff --git a/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsStep.java b/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsStep.java index 378a2ea3d..71a87dead 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/RemoveAnnotationsStep.java @@ -60,7 +60,7 @@ public Provider() } @Override - public PipelineStep create(PipelineContext context) + public RemoveAnnotationsStep create(PipelineContext context) { return new RemoveAnnotationsStep(this, context); } diff --git a/mGAP/src/org/labkey/mgap/pipeline/RenameSamplesForMgapStep.java b/mGAP/src/org/labkey/mgap/pipeline/RenameSamplesForMgapStep.java index 527817b36..5751ae9a1 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/RenameSamplesForMgapStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/RenameSamplesForMgapStep.java @@ -67,7 +67,7 @@ public Provider() } @Override - public PipelineStep create(PipelineContext context) + public RenameSamplesForMgapStep create(PipelineContext context) { return new RenameSamplesForMgapStep(this, context); } From 6d5121d556613e2386b9a41fd4cc814a0b4286bb Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Fri, 1 Mar 2024 12:45:33 -0800 Subject: [PATCH 22/35] Augment the HIVRC module --- .../postgresql/hivrc-23.000-23.001.sql | 17 ++++++ .../sqlserver/hivrc-23.000-23.001.sql | 17 ++++++ hivrc/resources/schemas/hivrc.xml | 60 ++++++++++++++++++- hivrc/resources/views/analysisHeader.html | 1 - hivrc/resources/views/begin.html | 10 +++- hivrc/resources/views/hivrcDatasets.html | 23 ++++++- hivrc/src/org/labkey/hivrc/HivrcModule.java | 15 ++--- 7 files changed, 130 insertions(+), 13 deletions(-) create mode 100644 hivrc/resources/schemas/dbscripts/postgresql/hivrc-23.000-23.001.sql create mode 100644 hivrc/resources/schemas/dbscripts/sqlserver/hivrc-23.000-23.001.sql delete mode 100644 hivrc/resources/views/analysisHeader.html diff --git a/hivrc/resources/schemas/dbscripts/postgresql/hivrc-23.000-23.001.sql b/hivrc/resources/schemas/dbscripts/postgresql/hivrc-23.000-23.001.sql new file mode 100644 index 000000000..6b94a880a --- /dev/null +++ b/hivrc/resources/schemas/dbscripts/postgresql/hivrc-23.000-23.001.sql @@ -0,0 +1,17 @@ +CREATE TABLE hivrc.datasets ( + rowid SERIAL, + label varchar(1000), + description varchar(4000), + + datatype varchar(1000), + dataowner varchar(1000), + fileid int, + + container entityid, + created timestamp, + createdby int, + modified timestamp, + modifiedby int, + + constraint PK_datasets PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/hivrc/resources/schemas/dbscripts/sqlserver/hivrc-23.000-23.001.sql b/hivrc/resources/schemas/dbscripts/sqlserver/hivrc-23.000-23.001.sql new file mode 100644 index 000000000..42b28c1eb --- /dev/null +++ b/hivrc/resources/schemas/dbscripts/sqlserver/hivrc-23.000-23.001.sql @@ -0,0 +1,17 @@ +CREATE TABLE hivrc.datasets ( + rowid int IDENTITY(1,1), + label varchar(1000), + description varchar(4000), + + datatype varchar(1000), + dataowner varchar(1000), + fileid int, + + container entityid, + created datetime, + createdby int, + modified datetime, + modifiedby int, + + constraint PK_datasets PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/hivrc/resources/schemas/hivrc.xml b/hivrc/resources/schemas/hivrc.xml index 7918b339b..04f6daa40 100644 --- a/hivrc/resources/schemas/hivrc.xml +++ b/hivrc/resources/schemas/hivrc.xml @@ -1,3 +1,61 @@ <?xml version="1.0" encoding="UTF-8"?> <tables xsi:schemaLocation="http://labkey.org/data/xml ..\..\..\..\schemas\tableInfo.xsd" - xmlns="http://labkey.org/data/xml" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"/> \ No newline at end of file + xmlns="http://labkey.org/data/xml" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <table tableName="datasets" tableDbType="TABLE" useColumnOrder="true"> + <tableTitle>Subjects</tableTitle> + <javaCustomizer class="org.labkey.ldk.query.DefaultTableCustomizer" /> + <auditLogging>DETAILED</auditLogging> + <columns> + <column columnName="rowid"> + <isAutoInc>true</isAutoInc> + <shownInInsertView>false</shownInInsertView> + <shownInUpdateView>false</shownInUpdateView> + <isUserEditable>false</isUserEditable> + <columnTitle>Row Id</columnTitle> + </column> + <column columnName="label"> + <columnTitle>Label</columnTitle> + </column> + <column columnName="description"> + <columnTitle>Description</columnTitle> + </column> + <column columnName="datatype"> + <columnTitle>Data Type</columnTitle> + </column> + <column columnName="dataowner"> + <columnTitle>Data Owner</columnTitle> + </column> + <column columnName="fileid"> + <columnTitle>File ID</columnTitle> + <fk> + <fkDbSchema>exp</fkDbSchema> + <fkTable>data</fkTable> + <fkColumnName>RowId</fkColumnName> + </fk> + </column> + <column columnName="container"> + <isHidden>true</isHidden> + </column> + <column columnName="createdby"> + <isHidden>true</isHidden> + </column> + <column columnName="created"> + <isUserEditable>false</isUserEditable> + <shownInInsertView>false</shownInInsertView> + <shownInUpdateView>false</shownInUpdateView> + <nullable>true</nullable> + <isHidden>true</isHidden> + </column> + <column columnName="modifiedby"> + <isHidden>true</isHidden> + </column> + <column columnName="modified"> + <isUserEditable>false</isUserEditable> + <shownInInsertView>false</shownInInsertView> + <shownInUpdateView>false</shownInUpdateView> + <nullable>true</nullable> + <isHidden>true</isHidden> + </column> + </columns> + </table> +</tables> \ No newline at end of file diff --git a/hivrc/resources/views/analysisHeader.html b/hivrc/resources/views/analysisHeader.html deleted file mode 100644 index 05a682bd4..000000000 --- a/hivrc/resources/views/analysisHeader.html +++ /dev/null @@ -1 +0,0 @@ -Hello! \ No newline at end of file diff --git a/hivrc/resources/views/begin.html b/hivrc/resources/views/begin.html index 3b188623f..4f908505e 100644 --- a/hivrc/resources/views/begin.html +++ b/hivrc/resources/views/begin.html @@ -1 +1,9 @@ -This is a prototype for a web-based system to manage BMGF datasets and analyses. \ No newline at end of file +This is a prototype for a web-based system to manage BMGF datasets and analyses. Currently, the system is primarily designed to facilitate the work of informaticians/analysts, +aiding in discoverability of data, access to data, and documentation/sharing of analyses. There are two components to this system: +<br> +<br> +<ol> + <li>This system will contain a registry for input datasets. This provides a central table of inputs, each of which can be identified by ID.</li> + + <li>The system also tracks analyses. Each analysis is loosely defined as one unit of work, such as validation of a given biomarker candidate in one cohort. Creating an analysis provides a folder to save any relevant files. The minimum requirement for an analysis is to upload a document (such as a markdown file) that serves to document the details of what was performed.</li> +</ol> \ No newline at end of file diff --git a/hivrc/resources/views/hivrcDatasets.html b/hivrc/resources/views/hivrcDatasets.html index e2126d9fd..61f3ded6f 100644 --- a/hivrc/resources/views/hivrcDatasets.html +++ b/hivrc/resources/views/hivrcDatasets.html @@ -1 +1,22 @@ -This page wil eventually hold a listing of HIVRC datasets, and allow specific files to be identified/queried by ID. \ No newline at end of file +<script type="text/javascript"> + + Ext4.onReady(function (){ + var webpart = <%=webpartContext%>; + + Ext4.get(webpart.wrapperDivId).update( + '<div>Below are the list of datasets registered with the HIVRC. These datasets can be referenced by ID, and queried using the HIVRC R Package.' + + '</div>' + + '<br>' + + '<br>' + + '<div id="qwp_'+webpart.wrapperDivId+'"></div>' + ); + + LDK.Utils.getReadOnlyQWP({ + name: 'query', + schemaName: 'hivrc', + queryName: 'datasets' + }).render('qwp_' + webpart.wrapperDivId); + }); + +</script> + diff --git a/hivrc/src/org/labkey/hivrc/HivrcModule.java b/hivrc/src/org/labkey/hivrc/HivrcModule.java index 673f1c9fb..d5a945e4b 100644 --- a/hivrc/src/org/labkey/hivrc/HivrcModule.java +++ b/hivrc/src/org/labkey/hivrc/HivrcModule.java @@ -3,8 +3,11 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Container; -import org.labkey.api.module.DefaultModule; +import org.labkey.api.ldk.ExtendedSimpleModule; +import org.labkey.api.module.Module; import org.labkey.api.module.ModuleContext; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.QuerySchema; import org.labkey.api.util.HtmlString; import org.labkey.api.view.BaseWebPartFactory; import org.labkey.api.view.HtmlView; @@ -20,7 +23,7 @@ import java.util.Collections; import java.util.Set; -public class HivrcModule extends DefaultModule +public class HivrcModule extends ExtendedSimpleModule { public static final String NAME = "HIVRC"; @@ -33,7 +36,7 @@ public String getName() @Override public @Nullable Double getSchemaVersion() { - return 23.000; + return 23.001; } @Override @@ -85,12 +88,6 @@ protected void init() addController(HivrcController.NAME, HivrcController.class); } - @Override - public void doStartup(ModuleContext moduleContext) - { - - } - @Override @NotNull public Collection<String> getSummary(Container c) From 2be12633eb6b6264b31986713d112235d58369e9 Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Fri, 1 Mar 2024 14:19:57 -0800 Subject: [PATCH 23/35] Add warning about mismatches alternateIds --- mcc/resources/web/mcc/panel/MccImportPanel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcc/resources/web/mcc/panel/MccImportPanel.js b/mcc/resources/web/mcc/panel/MccImportPanel.js index 8e3b0fd79..43c0bc4d1 100644 --- a/mcc/resources/web/mcc/panel/MccImportPanel.js +++ b/mcc/resources/web/mcc/panel/MccImportPanel.js @@ -490,7 +490,7 @@ Ext4.define('MCC.panel.MccImportPanel', { else { row.objectId = existingRecord.objectid; - var fields = ['birth', 'dam', 'sire', 'source']; + var fields = ['birth', 'dam', 'sire', 'source', 'alternateIds']; for (var idx in fields) { var fn = fields[idx]; From 7fb5a39b27a0979096f544df9bda5d8d6f268455 Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Mon, 4 Mar 2024 11:51:58 -0800 Subject: [PATCH 24/35] Null checks for MccUserSchema --- mcc/src/org/labkey/mcc/MccUserSchema.java | 60 ++++++++++++++--------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/mcc/src/org/labkey/mcc/MccUserSchema.java b/mcc/src/org/labkey/mcc/MccUserSchema.java index 7ad0fec29..8a503f291 100644 --- a/mcc/src/org/labkey/mcc/MccUserSchema.java +++ b/mcc/src/org/labkey/mcc/MccUserSchema.java @@ -48,30 +48,34 @@ public MccUserSchema(User user, Container container, DbSchema dbschema) @Override public TableInfo createTable(String name, ContainerFilter cf) { - if (TABLE_AGGREGATED_WEIGHT.equalsIgnoreCase(name)) + if (supportsAggregatedTables()) { - return getWeightQuery(); - } - else if (TABLE_AGGREGATED_KINSHIP.equalsIgnoreCase(name)) - { - return getKinshipQuery(); - } - else if (TABLE_AGGREGATED_DEPARTURES.equalsIgnoreCase(name)) - { - return getDepartureQuery(); - } - else if (TABLE_AGGREGATED_GENOMICS.equalsIgnoreCase(name)) - { - return getGenomicsQuery(); - } - else if (TABLE_AGGREGATED_OBS.equalsIgnoreCase(name)) - { - return getObsQuery(); - } - else if (TABLE_AGGREGATED_CENSUS.equalsIgnoreCase(name)) - { - return getCensusQuery(); + if (TABLE_AGGREGATED_WEIGHT.equalsIgnoreCase(name)) + { + return getWeightQuery(); + } + else if (TABLE_AGGREGATED_KINSHIP.equalsIgnoreCase(name)) + { + return getKinshipQuery(); + } + else if (TABLE_AGGREGATED_DEPARTURES.equalsIgnoreCase(name)) + { + return getDepartureQuery(); + } + else if (TABLE_AGGREGATED_GENOMICS.equalsIgnoreCase(name)) + { + return getGenomicsQuery(); + } + else if (TABLE_AGGREGATED_OBS.equalsIgnoreCase(name)) + { + return getObsQuery(); + } + else if (TABLE_AGGREGATED_CENSUS.equalsIgnoreCase(name)) + { + return getCensusQuery(); + } } + return super.createTable(name, cf); } @@ -156,9 +160,14 @@ public Set<String> getVisibleTableNames() return Collections.unmodifiableSet(available); } + private boolean supportsAggregatedTables() + { + return getContainer().hasPermission(getUser(), MccDataAdminPermission.class) && MccManager.get().getMCCInternalDataContainer(getContainer()) != null; + } + private void addAggregatedTableNames(Set<String> available) { - if (getContainer().hasPermission(getUser(), MccDataAdminPermission.class)) + if (supportsAggregatedTables()) { available.add(TABLE_AGGREGATED_KINSHIP); available.add(TABLE_AGGREGATED_WEIGHT); @@ -278,6 +287,11 @@ private TableInfo makeAggregatedQuery(String queryName, String sqlTemplate) StringBuilder sql = new StringBuilder(); Container parent = MccManager.get().getMCCInternalDataContainer(getContainer()); + if (parent == null) + { + return null; + } + String unionClause = ""; for (Container c : parent.getChildren()) { From f3d3b2dc669a324cc44e660c1b8b1f4acff1e3ca Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Tue, 5 Mar 2024 21:01:40 -0800 Subject: [PATCH 25/35] Bugfix to mcc census --- mcc/resources/schemas/mcc.xml | 8 +------- mcc/src/org/labkey/mcc/MccModule.java | 2 ++ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/mcc/resources/schemas/mcc.xml b/mcc/resources/schemas/mcc.xml index 154d002e7..412deaf93 100644 --- a/mcc/resources/schemas/mcc.xml +++ b/mcc/resources/schemas/mcc.xml @@ -781,6 +781,7 @@ <importUrl /> <updateUrl /> <deleteUrl /> + <javaCustomizer class="org.labkey.ldk.query.DefaultTableCustomizer" /> <tableTitle>MCC Census</tableTitle> <auditLogging>DETAILED</auditLogging> <pkColumnName>rowid</pkColumnName> @@ -843,13 +844,6 @@ <scale>29</scale> <isHidden>true</isHidden> </column> - <column columnName="actions" wrappedColumnName="rowid"> - <columnTitle>Actions</columnTitle> - <displayWidth>200</displayWidth> - <displayColumnFactory> - <className>org.labkey.mcc.query.RequestScoreActionsDisplayColumnFactory</className> - </displayColumnFactory> - </column> </columns> <buttonBarOptions includeStandardButtons="true"> <includeScript>ldk.context</includeScript> diff --git a/mcc/src/org/labkey/mcc/MccModule.java b/mcc/src/org/labkey/mcc/MccModule.java index 92f086233..c7c4fe169 100644 --- a/mcc/src/org/labkey/mcc/MccModule.java +++ b/mcc/src/org/labkey/mcc/MccModule.java @@ -23,6 +23,7 @@ import org.labkey.api.ehr.EHRService; import org.labkey.api.ldk.ExtendedSimpleModule; import org.labkey.api.ldk.LDKService; +import org.labkey.api.ldk.buttons.ShowEditUIButton; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleContext; import org.labkey.api.query.DefaultSchema; @@ -123,6 +124,7 @@ private void registerEHRResources() EHRService.get().registerTableCustomizer(this, MccEhrCustomizer.class); EHRService.get().registerMoreActionsButton(new MarkShippedButton(), "study", "demographics"); EHRService.get().registerMoreActionsButton(new RenameIdButton(), "study", "demographics"); + LDKService.get().registerQueryButton(new ShowEditUIButton(this, MccSchema.NAME, MccSchema.TABLE_CENSUS), MccSchema.NAME, MccSchema.TABLE_CENSUS); } @Override From 7f150b31d7343c617b82329744e271cb97efb18a Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Tue, 5 Mar 2024 21:37:22 -0800 Subject: [PATCH 26/35] Add step to populate MCC genomic datasets --- mcc/resources/etls/mcc.xml | 7 + .../mcc/genomicDatasetsSource.query.xml | 14 ++ .../queries/mcc/genomicDatasetsSource.sql | 10 ++ .../mcc/etl/PopulateGeneticDataStep.java | 161 ++++++++++++++++++ 4 files changed, 192 insertions(+) create mode 100644 mcc/resources/queries/mcc/genomicDatasetsSource.query.xml create mode 100644 mcc/resources/queries/mcc/genomicDatasetsSource.sql create mode 100644 mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml index b4f41a9c4..c697b8303 100644 --- a/mcc/resources/etls/mcc.xml +++ b/mcc/resources/etls/mcc.xml @@ -10,6 +10,13 @@ </settings> </taskref> </transform> + <transform id="step0" type="TaskRefTransformStep"> + <taskref ref="org.labkey.mcc.etl.PopulateGeneticDataStep"> + <settings> + <setting name="remoteSource" value="PRIMESEQ"/> + </settings> + </taskref> + </transform> <transform type="SimpleQueryTransformStep" id="demographics"> <description>Copy to target</description> <source schemaName="mcc" queryName="aggregatedDemographics" > diff --git a/mcc/resources/queries/mcc/genomicDatasetsSource.query.xml b/mcc/resources/queries/mcc/genomicDatasetsSource.query.xml new file mode 100644 index 000000000..a2ebcc6c6 --- /dev/null +++ b/mcc/resources/queries/mcc/genomicDatasetsSource.query.xml @@ -0,0 +1,14 @@ +<query xmlns="http://labkey.org/data/xml/query"> + <metadata> + <tables xmlns="http://labkey.org/data/xml"> + <table tableName="" tableDbType="TABLE"> + <tableTitle>MCC Genomic Datasets Source</tableTitle> + <columns> + <column columnName="Id"> + <columnTitle>MCC ID</columnTitle> + </column> + </columns> + </table> + </tables> + </metadata> +</query> \ No newline at end of file diff --git a/mcc/resources/queries/mcc/genomicDatasetsSource.sql b/mcc/resources/queries/mcc/genomicDatasetsSource.sql new file mode 100644 index 000000000..1036baafd --- /dev/null +++ b/mcc/resources/queries/mcc/genomicDatasetsSource.sql @@ -0,0 +1,10 @@ +SELECT + + r.subjectid as Id, + r.created as date, + r.application as datatype, + r.sraRuns as sra_accession + +FROM sequenceanalysis.sequence_readsets r + +WHERE r.subjectid LIKE 'MCC%' \ No newline at end of file diff --git a/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java new file mode 100644 index 000000000..c9aa0f41e --- /dev/null +++ b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java @@ -0,0 +1,161 @@ +package org.labkey.mcc.etl; + +import org.apache.xmlbeans.XmlException; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.writer.ContainerUser; +import org.labkey.mcc.MccManager; +import org.labkey.mcc.MccSchema; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.SelectRowsResponse; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PopulateGeneticDataStep implements TaskRefTask +{ + protected final Map<String, String> _settings = new CaseInsensitiveHashMap<>(); + + protected ContainerUser _containerUser; + + private enum Settings + { + remoteSource() + } + + @Override + public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException + { + populateGeneticData(job); + + return new RecordedActionSet(); + } + + private void populateGeneticData(PipelineJob job) throws PipelineJobException + { + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(_settings.get(Settings.remoteSource.name()), _containerUser.getContainer(), job.getLogger()); + if (rc == null) + { + throw new PipelineJobException("Unable to find remote connection: " + _settings.get(Settings.remoteSource.name())); + } + + try + { + //first select all rows from remote table + SelectRowsCommand sr = new SelectRowsCommand(_settings.get(MccSchema.NAME), _settings.get("genomicDatasetsSource")); + sr.setColumns(Arrays.asList("Id", "date", "datatype", "sra_accession")); + + TableInfo aggregatedDemographics = QueryService.get().getUserSchema(job.getUser(), job.getContainer(), MccSchema.NAME).getTable("aggregatedDemographics"); + + //and select all rows from our source table + SelectRowsResponse srr = sr.execute(rc.connection, rc.remoteContainer); + Map<Container, List<Map<String, Object>>> toInsert = new HashMap<>(); + srr.getRows().forEach(x -> { + String mccId = (String)x.get("Id"); + + Collection<Map<String, Object>> rows = new TableSelector(aggregatedDemographics, PageFlowUtil.set("originalId", "container"), new SimpleFilter(FieldKey.fromString("Id"), mccId), null).getMapCollection(); + if (rows.isEmpty()) + { + job.getLogger().error("Unable to find ID: " + mccId); + return; + } + + if (rows.size() > 1) + { + job.getLogger().error("MCC ID linked to multiple containers: " + mccId); + return; + } + + Map<String, Object> row = rows.iterator().next(); + + Container target = ContainerManager.getForId(String.valueOf(row.get("container"))); + if (!toInsert.containsKey(target)) + { + toInsert.put(target, new ArrayList<>()); + } + + toInsert.get(target).add(Map.of( + "Id", row.get("originalId"), + "date", x.get("date"), + "datatype", x.get("datatype"), + "sra_accession", x.get("sra_accession") + )); + }); + + Container c = MccManager.get().getMCCInternalDataContainer(job.getContainer()); + if (c == null) + { + throw new IllegalStateException("MCCInternalDataContainer not set"); + } + + for (Container child : c.getChildren()) + { + TableInfo ti = QueryService.get().getUserSchema(job.getUser(), child, MccSchema.NAME).getTable(MccSchema.TABLE_CENSUS); + try + { + ti.getUpdateService().truncateRows(job.getUser(), child, null, null); + + if (toInsert.containsKey(child)) + { + BatchValidationException bve = new BatchValidationException(); + ti.getUpdateService().insertRows(job.getUser(), child, toInsert.get(child), bve, null, null); + if (bve.hasErrors()) + { + throw bve; + } + } + } + catch (BatchValidationException | SQLException | DuplicateKeyException | QueryUpdateServiceException e) + { + throw new PipelineJobException(e); + } + } + } + catch (CommandException | IOException e) + { + throw new PipelineJobException(e); + } + } + + @Override + public List<String> getRequiredSettings() + { + return Collections.unmodifiableList(Arrays.asList(Settings.remoteSource.name())); + } + + @Override + public void setSettings(Map<String, String> settings) throws XmlException + { + _settings.putAll(settings); + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } +} \ No newline at end of file From a93d88de74537eaee417ecf8e73976f66893647c Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Tue, 5 Mar 2024 22:25:34 -0800 Subject: [PATCH 27/35] Bugfix to MCC ETL --- mcc/resources/etls/mcc.xml | 2 +- mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml index c697b8303..c76f2a078 100644 --- a/mcc/resources/etls/mcc.xml +++ b/mcc/resources/etls/mcc.xml @@ -10,7 +10,7 @@ </settings> </taskref> </transform> - <transform id="step0" type="TaskRefTransformStep"> + <transform id="step1" type="TaskRefTransformStep"> <taskref ref="org.labkey.mcc.etl.PopulateGeneticDataStep"> <settings> <setting name="remoteSource" value="PRIMESEQ"/> diff --git a/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java index c9aa0f41e..d6f1c89a1 100644 --- a/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java +++ b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java @@ -66,7 +66,7 @@ private void populateGeneticData(PipelineJob job) throws PipelineJobException try { //first select all rows from remote table - SelectRowsCommand sr = new SelectRowsCommand(_settings.get(MccSchema.NAME), _settings.get("genomicDatasetsSource")); + SelectRowsCommand sr = new SelectRowsCommand(MccSchema.NAME, "genomicDatasetsSource"); sr.setColumns(Arrays.asList("Id", "date", "datatype", "sra_accession")); TableInfo aggregatedDemographics = QueryService.get().getUserSchema(job.getUser(), job.getContainer(), MccSchema.NAME).getTable("aggregatedDemographics"); From 4016bc159f63b6c9a33f25ea2c23c48d4deeac67 Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Tue, 5 Mar 2024 23:13:43 -0800 Subject: [PATCH 28/35] Bugfix to MCC ETL --- .../org/labkey/mcc/etl/PopulateGeneticDataStep.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java index d6f1c89a1..6d2c61651 100644 --- a/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java +++ b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java @@ -98,12 +98,13 @@ private void populateGeneticData(PipelineJob job) throws PipelineJobException toInsert.put(target, new ArrayList<>()); } - toInsert.get(target).add(Map.of( - "Id", row.get("originalId"), - "date", x.get("date"), - "datatype", x.get("datatype"), - "sra_accession", x.get("sra_accession") - )); + Map<String, Object> newRow = new CaseInsensitiveHashMap<>(); + newRow.put("Id", row.get("originalId")); + newRow.put("date", x.get("date")); + newRow.put("datatype", x.get("datatype")); + newRow.put("sra_accession", x.get("sra_accession")); + + toInsert.get(target).add(newRow); }); Container c = MccManager.get().getMCCInternalDataContainer(job.getContainer()); From 23d38c901c3aa0ba04ecca90f870bd6e3ff62c54 Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Wed, 6 Mar 2024 09:35:33 -0800 Subject: [PATCH 29/35] Add columns to MCC view --- mcc/resources/queries/mcc/requestScores/.qview.xml | 2 ++ mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mcc/resources/queries/mcc/requestScores/.qview.xml b/mcc/resources/queries/mcc/requestScores/.qview.xml index 81956a4f5..3899ec233 100644 --- a/mcc/resources/queries/mcc/requestScores/.qview.xml +++ b/mcc/resources/queries/mcc/requestScores/.qview.xml @@ -5,6 +5,8 @@ <column name="requestId/firstname"/> <column name="requestId/title"/> <column name="requestId/institutionname"/> + <column name="requestId/fundingsource"/> + <column name="requestId/grantnumber"/> <column name="requestId/status"/> <column name="preliminaryScore"/> <column name="resourceAvailabilityAssessment"/> diff --git a/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java index 6d2c61651..a71c552d0 100644 --- a/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java +++ b/mcc/src/org/labkey/mcc/etl/PopulateGeneticDataStep.java @@ -115,15 +115,15 @@ private void populateGeneticData(PipelineJob job) throws PipelineJobException for (Container child : c.getChildren()) { - TableInfo ti = QueryService.get().getUserSchema(job.getUser(), child, MccSchema.NAME).getTable(MccSchema.TABLE_CENSUS); + TableInfo gd = QueryService.get().getUserSchema(job.getUser(), child, "study").getTable("genomicDatasets"); try { - ti.getUpdateService().truncateRows(job.getUser(), child, null, null); + gd.getUpdateService().truncateRows(job.getUser(), child, null, null); if (toInsert.containsKey(child)) { BatchValidationException bve = new BatchValidationException(); - ti.getUpdateService().insertRows(job.getUser(), child, toInsert.get(child), bve, null, null); + gd.getUpdateService().insertRows(job.getUser(), child, toInsert.get(child), bve, null, null); if (bve.hasErrors()) { throw bve; From 07a73bd8e20398346d54d68d7c07ef27c7264e5f Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Wed, 6 Mar 2024 11:32:39 -0800 Subject: [PATCH 30/35] Separate MCC genomic ETL --- mcc/resources/etls/mcc-genomics.xml | 18 ++++++++++++++++++ mcc/resources/etls/mcc.xml | 7 ------- 2 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 mcc/resources/etls/mcc-genomics.xml diff --git a/mcc/resources/etls/mcc-genomics.xml b/mcc/resources/etls/mcc-genomics.xml new file mode 100644 index 000000000..33977d7b9 --- /dev/null +++ b/mcc/resources/etls/mcc-genomics.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8"?> +<etl xmlns="http://labkey.org/etl/xml"> + <name>MCC_Genomic</name> + <description>MCC Genomic Data</description> + <transforms> + <transform id="step1" type="TaskRefTransformStep"> + <taskref ref="org.labkey.mcc.etl.PopulateGeneticDataStep"> + <settings> + <setting name="remoteSource" value="PRIMESEQ"/> + </settings> + </taskref> + </transform> + </transforms> + <schedule> + <!--11PM--> + <cron expression="0 30 23 * * ?"/> + </schedule> +</etl> diff --git a/mcc/resources/etls/mcc.xml b/mcc/resources/etls/mcc.xml index c76f2a078..b4f41a9c4 100644 --- a/mcc/resources/etls/mcc.xml +++ b/mcc/resources/etls/mcc.xml @@ -10,13 +10,6 @@ </settings> </taskref> </transform> - <transform id="step1" type="TaskRefTransformStep"> - <taskref ref="org.labkey.mcc.etl.PopulateGeneticDataStep"> - <settings> - <setting name="remoteSource" value="PRIMESEQ"/> - </settings> - </taskref> - </transform> <transform type="SimpleQueryTransformStep" id="demographics"> <description>Copy to target</description> <source schemaName="mcc" queryName="aggregatedDemographics" > From ef372032e6d8779b045374b032405e4a38d46f3e Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Wed, 6 Mar 2024 13:11:07 -0800 Subject: [PATCH 31/35] Run mcc-genomics ahead of general mcc ETL --- mcc/resources/etls/mcc-genomics.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcc/resources/etls/mcc-genomics.xml b/mcc/resources/etls/mcc-genomics.xml index 33977d7b9..4e567a553 100644 --- a/mcc/resources/etls/mcc-genomics.xml +++ b/mcc/resources/etls/mcc-genomics.xml @@ -13,6 +13,6 @@ </transforms> <schedule> <!--11PM--> - <cron expression="0 30 23 * * ?"/> + <cron expression="0 30 22 * * ?"/> </schedule> </etl> From a9d909ddc5b53107ebebbd84c49d980867f2bbb3 Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Wed, 6 Mar 2024 14:14:36 -0800 Subject: [PATCH 32/35] Update several MCC views --- mcc/resources/queries/study/assignment/.qview.xml | 2 +- mcc/resources/queries/study/birth/.qview.xml | 2 +- mcc/resources/queries/study/clinremarks/.qview.xml | 2 +- mcc/resources/queries/study/deaths/.qview.xml | 2 +- mcc/resources/queries/study/encounters/.qview.xml | 5 +---- mcc/resources/queries/study/flags/.qview.xml | 2 +- .../queries/study/genomicDatasets/.qview.xml | 12 ++++++++++++ mcc/resources/queries/study/weight/.qview.xml | 2 +- 8 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 mcc/resources/queries/study/genomicDatasets/.qview.xml diff --git a/mcc/resources/queries/study/assignment/.qview.xml b/mcc/resources/queries/study/assignment/.qview.xml index 4c0c2fdb7..59c0fd9f0 100644 --- a/mcc/resources/queries/study/assignment/.qview.xml +++ b/mcc/resources/queries/study/assignment/.qview.xml @@ -10,7 +10,7 @@ <column name="date"/> <column name="enddate"/> <column name="qcstate" /> - <column name="taskid" /> + <column name="history" /> </columns> <sorts> <sort column="date" descending="true"/> diff --git a/mcc/resources/queries/study/birth/.qview.xml b/mcc/resources/queries/study/birth/.qview.xml index f1449329f..cbf21f1dc 100644 --- a/mcc/resources/queries/study/birth/.qview.xml +++ b/mcc/resources/queries/study/birth/.qview.xml @@ -11,7 +11,7 @@ <column name="type" /> <column name="remark" /> <column name="qcstate" /> - <column name="taskid" /> + <column name="history" /> </columns> <sorts> <sort column="date" descending="true"/> diff --git a/mcc/resources/queries/study/clinremarks/.qview.xml b/mcc/resources/queries/study/clinremarks/.qview.xml index 78d39cdfc..b76188c72 100644 --- a/mcc/resources/queries/study/clinremarks/.qview.xml +++ b/mcc/resources/queries/study/clinremarks/.qview.xml @@ -6,8 +6,8 @@ <column name="description" /> <column name="performedby"/> <column name="category"/> - <column name="taskid" /> <column name="qcstate" /> + <column name="history" /> </columns> <sorts> <sort column="date" descending="true"/> diff --git a/mcc/resources/queries/study/deaths/.qview.xml b/mcc/resources/queries/study/deaths/.qview.xml index fa7420819..a66d41d01 100644 --- a/mcc/resources/queries/study/deaths/.qview.xml +++ b/mcc/resources/queries/study/deaths/.qview.xml @@ -5,7 +5,7 @@ <column name="cause" /> <column name="remark" /> <column name="qcstate" /> - <column name="taskid" /> + <column name="history" /> </columns> <sorts> <sort column="date" descending="true"/> diff --git a/mcc/resources/queries/study/encounters/.qview.xml b/mcc/resources/queries/study/encounters/.qview.xml index f1da9f3a4..05e3ae27d 100644 --- a/mcc/resources/queries/study/encounters/.qview.xml +++ b/mcc/resources/queries/study/encounters/.qview.xml @@ -9,11 +9,8 @@ <column name="performedby"/> <column name="remark"/> - <column name="history" /> <column name="qcstate" /> - <column name="taskid" /> - <column name="requestid" /> - <column name="chargetype" /> + <column name="history" /> </columns> <sorts> <sort column="date" descending="true"/> diff --git a/mcc/resources/queries/study/flags/.qview.xml b/mcc/resources/queries/study/flags/.qview.xml index 05349f54b..35d818f7c 100644 --- a/mcc/resources/queries/study/flags/.qview.xml +++ b/mcc/resources/queries/study/flags/.qview.xml @@ -7,8 +7,8 @@ <column name="flag/value"/> <column name="remark"/> <column name="performedby"/> - <column name="taskid"/> <column name="qcstate"/> + <column name="history" /> </columns> <sorts> <sort column="date" descending="true"/> diff --git a/mcc/resources/queries/study/genomicDatasets/.qview.xml b/mcc/resources/queries/study/genomicDatasets/.qview.xml new file mode 100644 index 000000000..e583f4f41 --- /dev/null +++ b/mcc/resources/queries/study/genomicDatasets/.qview.xml @@ -0,0 +1,12 @@ +<customView xmlns="http://labkey.org/data/xml/queryCustomView"> + <columns> + <column name="Id" /> + <column name="date" /> + <column name="datatype" /> + <column name="sra_accession" /> + <column name="history" /> + </columns> + <sorts> + <sort column="date" descending="true"/> + </sorts> +</customView> \ No newline at end of file diff --git a/mcc/resources/queries/study/weight/.qview.xml b/mcc/resources/queries/study/weight/.qview.xml index 3a0c5182c..5ea528af7 100644 --- a/mcc/resources/queries/study/weight/.qview.xml +++ b/mcc/resources/queries/study/weight/.qview.xml @@ -5,7 +5,7 @@ <column name="weightGrams" /> <column name="remark" /> <column name="qcstate" /> - <column name="taskid" /> + <column name="history" /> </columns> <sorts> <sort column="date" descending="true"/> From 11b605e2111fe7b0036822de4d5282069aa2dc75 Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Thu, 7 Mar 2024 08:23:26 -0800 Subject: [PATCH 33/35] Update several MCC views --- .../resources/queries/study/kinship/.qview.xml | 15 +++++++++++++++ mcc/resources/data/reports.tsv | 2 +- .../queries/study/genomicDatasets.query.xml | 2 +- mcc/resources/queries/study/kinship/.qview.xml | 6 ------ 4 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 MccColony/resources/queries/study/kinship/.qview.xml diff --git a/MccColony/resources/queries/study/kinship/.qview.xml b/MccColony/resources/queries/study/kinship/.qview.xml new file mode 100644 index 000000000..71e0ae624 --- /dev/null +++ b/MccColony/resources/queries/study/kinship/.qview.xml @@ -0,0 +1,15 @@ +<customView xmlns="http://labkey.org/data/xml/queryCustomView" label="All Flags"> + <columns> + <column name="Id"/> + <column name="Id2"/> + <column name="kinship"/> + <column name="relationship"/> + <column name="Id/mccAlias/externalAlias"/> + <column name="Id2MccAlias/externalAlias"> + <properties> + <property name="columnTitle" value="Id2 MCC Alias"/> + </properties> + </column> + </columns> + +</customView> \ No newline at end of file diff --git a/mcc/resources/data/reports.tsv b/mcc/resources/data/reports.tsv index ceb98134d..5e3587aa8 100644 --- a/mcc/resources/data/reports.tsv +++ b/mcc/resources/data/reports.tsv @@ -8,7 +8,7 @@ departure General query Shipments TRUE study departure date FALSE FALSE qcst obs General query Observations TRUE study clinical_observations date FALSE FALSE qcstate/publicdata This report displays observations for the selected animal pedigreePlot Genetics report Pedigree Plot TRUE study pedigree module:EHR/schemas/study/Pedigree/Pedigree.r FALSE FALSE qcstate/publicdata This report will generate a pedigree plot for the selected animal offspring General query Offspring TRUE study demographicsOffspring FALSE FALSE qcstate/publicdata This report displays pedigree data for animals, including parents, grandparents, siblings and offspring -kinship Genetics js Kinship TRUE ehr kinshipSummary FALSE FALSE qcstate/publicdata This report shows the kinship coefficient between every animal in the colony. The kinship coefficient is a measure of relatedness between two individuals. It represents the probability that two genes, sampled at random from each individual are identical (e.g. the kinship coefficient between a parent and an offspring is 0.25). +kinship Genetics query Kinship TRUE study kinship FALSE FALSE qcstate/publicdata This report shows the kinship coefficient between every animal in the colony. The kinship coefficient is a measure of relatedness between two individuals. It represents the probability that two genes, sampled at random from each individual are identical (e.g. the kinship coefficient between a parent and an offspring is 0.25). parentage Genetics query Parentage TRUE study demographicsParents FALSE FALSE qcstate/publicdata This report shows information about the parentage of each animal, drawing from genetic data and observations geneticData Genetics query Genomic Datasets TRUE study genomicDatasets FALSE FALSE qcstate/publicdata This report summarizes available genomic datasets mostRecentObservations General query Most Recent Observations TRUE study mostRecentObservations FALSE FALSE This report summarizes the most recent observations per animal diff --git a/mcc/resources/queries/study/genomicDatasets.query.xml b/mcc/resources/queries/study/genomicDatasets.query.xml index 8eaed6ad2..c78b85463 100644 --- a/mcc/resources/queries/study/genomicDatasets.query.xml +++ b/mcc/resources/queries/study/genomicDatasets.query.xml @@ -16,7 +16,7 @@ </column> <column columnName="sra_accession"> <columnTitle>SRA Accession</columnTitle> - <url>https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=${sraAccession}</url> + <url>https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=${sra_accession}</url> </column> </columns> </table> diff --git a/mcc/resources/queries/study/kinship/.qview.xml b/mcc/resources/queries/study/kinship/.qview.xml index 71e0ae624..b238340c8 100644 --- a/mcc/resources/queries/study/kinship/.qview.xml +++ b/mcc/resources/queries/study/kinship/.qview.xml @@ -4,12 +4,6 @@ <column name="Id2"/> <column name="kinship"/> <column name="relationship"/> - <column name="Id/mccAlias/externalAlias"/> - <column name="Id2MccAlias/externalAlias"> - <properties> - <property name="columnTitle" value="Id2 MCC Alias"/> - </properties> - </column> </columns> </customView> \ No newline at end of file From a1cf6f6ac36e0b488faf50d50835d3d515075ceb Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Thu, 7 Mar 2024 10:07:48 -0800 Subject: [PATCH 34/35] Update MCC query --- mcc/resources/queries/mcc/aggregatedDemographics.sql | 12 ++++++++++-- .../queries/mcc/aggregatedDemographics/.qview.xml | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mcc/resources/queries/mcc/aggregatedDemographics.sql b/mcc/resources/queries/mcc/aggregatedDemographics.sql index 6434b2f36..4f481572f 100644 --- a/mcc/resources/queries/mcc/aggregatedDemographics.sql +++ b/mcc/resources/queries/mcc/aggregatedDemographics.sql @@ -27,6 +27,7 @@ SELECT o.medical_history, null as usage_current, null as usage_future, + null as breeding_partner_id, o.date_of_observations, d.container @@ -74,6 +75,7 @@ SELECT o.medical_history, null as usage_current, null as usage_future, + null as breeding_partner_id, o.date_of_observations, d.container @@ -116,8 +118,9 @@ SELECT o.infant_history, o.fertility_status, o.medical_history, - null as usage_current, - null as usage_future, + o.usage_current, + o.usage_future, + o.breeding_partner_id, o.date_of_observations, d.container @@ -130,6 +133,9 @@ FROM "/data/Colonies/UCSD/".study.demographics d o."infant_history::observation" as infant_history, o."fertility_status::observation" as fertility_status, o."medical_history::observation" as medical_history, + o."usage_current::observation" as usage_current, + o."usage_current::observation" as usage_future, + o."breeding_partner_id::observation" as breeding_partner_id FROM "/data/Colonies/UCSD/".study.mostRecentObservationsPivoted o ) o ON (o.Id = d.Id) WHERE (d.excludeFromCensus IS NULL or d.excludeFromCensus = false) @@ -162,6 +168,7 @@ SELECT o.medical_history, o.usage_current, o.usage_future, + o.breeding_partner_id, o.date_of_observations, d.container @@ -176,6 +183,7 @@ FROM "/data/Colonies/Other/".study.demographics d o."medical_history::observation" as medical_history, o."usage_current::observation" as usage_current, o."usage_current::observation" as usage_future, + o."breeding_partner_id::observation" as breeding_partner_id FROM "/data/Colonies/Other/".study.mostRecentObservationsPivoted o ) o ON (o.Id = d.Id) WHERE (d.excludeFromCensus IS NULL or d.excludeFromCensus = false) \ No newline at end of file diff --git a/mcc/resources/queries/mcc/aggregatedDemographics/.qview.xml b/mcc/resources/queries/mcc/aggregatedDemographics/.qview.xml index a5b17c161..834f038ee 100644 --- a/mcc/resources/queries/mcc/aggregatedDemographics/.qview.xml +++ b/mcc/resources/queries/mcc/aggregatedDemographics/.qview.xml @@ -21,6 +21,9 @@ <column name="infant_history"/> <column name="fertility_status"/> <column name="medical_history"/> + <column name="usage_current" /> + <column name="usage_future" /> + <column name="breeding_partner_id" /> <column name="date_of_observations"/> </columns> <sorts> From b72102afd732efcecf8e593194ae84e2afe49ec3 Mon Sep 17 00:00:00 2001 From: bbimber <bbimber@gmail.com> Date: Sat, 9 Mar 2024 11:09:04 -0800 Subject: [PATCH 35/35] Convert text() to unsafe() and add scriptNonce --- hivrc/resources/views/hivrcAnalyses.html | 2 +- hivrc/resources/views/hivrcDatasets.html | 2 +- hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp | 4 ++-- mcc/resources/views/animalHistory.html | 2 +- mcc/resources/views/mccCensus.html | 2 +- mcc/resources/views/mccColonies.html | 2 +- mcc/resources/views/mccDemographics.html | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hivrc/resources/views/hivrcAnalyses.html b/hivrc/resources/views/hivrcAnalyses.html index fab245df6..eda36eaef 100644 --- a/hivrc/resources/views/hivrcAnalyses.html +++ b/hivrc/resources/views/hivrcAnalyses.html @@ -1,4 +1,4 @@ -<script type="text/javascript"> +<script type="text/javascript" nonce="<%=scriptNonce%>"> Ext4.onReady(function(){ var webpart = <%=webpartContext%>; diff --git a/hivrc/resources/views/hivrcDatasets.html b/hivrc/resources/views/hivrcDatasets.html index 61f3ded6f..ef9472881 100644 --- a/hivrc/resources/views/hivrcDatasets.html +++ b/hivrc/resources/views/hivrcDatasets.html @@ -1,4 +1,4 @@ -<script type="text/javascript"> +<script type="text/javascript" nonce="<%=scriptNonce%>"> Ext4.onReady(function (){ var webpart = <%=webpartContext%>; diff --git a/hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp b/hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp index 7ae9e255a..c4641d294 100644 --- a/hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp +++ b/hivrc/src/org/labkey/hivrc/view/analysisHeader.jsp @@ -33,7 +33,7 @@ <div id=<%=q(wpId)%>></div> -<script type="text/javascript"> +<script type="text/javascript" nonce="<%=getScriptNonce()%>"> Ext4.onReady(function(){ var webpartId = <%=q(wpId)%>; var workbookId = <%=h(workbookId)%>; @@ -53,7 +53,7 @@ materials: <%=q(model.getMaterials())%>, methods: <%=q(model.getMethods())%>, results: <%=q(model.getResults())%>, - tags: <%=text(model.getTags() == null || model.getTags().length == 0 ? "null" : "['" + text(StringUtils.join(Arrays.asList(model.getTags()), "','")) + "']")%> + tags: <%=unsafe(model.getTags() == null || model.getTags().length == 0 ? "null" : "['" + unsafe(StringUtils.join(Arrays.asList(model.getTags()), "','")) + "']")%> }).render(webpartId); if (LABKEY.Security.currentUser.canInsert) { diff --git a/mcc/resources/views/animalHistory.html b/mcc/resources/views/animalHistory.html index 9eb133c7e..512aea1da 100644 --- a/mcc/resources/views/animalHistory.html +++ b/mcc/resources/views/animalHistory.html @@ -1,4 +1,4 @@ -<script type="text/javascript"> +<script type="text/javascript" nonce="<%=scriptNonce%>"> Ext4.onReady(function (){ var webpart = <%=webpartContext%>; diff --git a/mcc/resources/views/mccCensus.html b/mcc/resources/views/mccCensus.html index 46c161473..9e2ea9418 100644 --- a/mcc/resources/views/mccCensus.html +++ b/mcc/resources/views/mccCensus.html @@ -1,4 +1,4 @@ -<script type="text/javascript"> +<script type="text/javascript" nonce="<%=scriptNonce%>"> Ext4.onReady(function (){ var webpart = <%=webpartContext%>; diff --git a/mcc/resources/views/mccColonies.html b/mcc/resources/views/mccColonies.html index f34ee393f..5df06528c 100644 --- a/mcc/resources/views/mccColonies.html +++ b/mcc/resources/views/mccColonies.html @@ -1,4 +1,4 @@ -<script type="text/javascript"> +<script type="text/javascript" nonce="<%=scriptNonce%>"> Ext4.onReady(function (){ var webpart = <%=webpartContext%>; diff --git a/mcc/resources/views/mccDemographics.html b/mcc/resources/views/mccDemographics.html index 8b5bda189..f51dcf21b 100644 --- a/mcc/resources/views/mccDemographics.html +++ b/mcc/resources/views/mccDemographics.html @@ -1,4 +1,4 @@ -<script type="text/javascript"> +<script type="text/javascript" nonce="<%=scriptNonce%>"> Ext4.onReady(function (){ var webpart = <%=webpartContext%>;