diff --git a/LabPurchasing/resources/queries/labpurchasing/purchases.js b/LabPurchasing/resources/queries/labpurchasing/purchases.js index dfb59f997..602ce37ff 100644 --- a/LabPurchasing/resources/queries/labpurchasing/purchases.js +++ b/LabPurchasing/resources/queries/labpurchasing/purchases.js @@ -3,6 +3,30 @@ var console = require("console"); var triggerHelper = new org.labkey.labpurchasing.LabPurchasingTriggerHelper(LABKEY.Security.currentUser.id, LABKEY.Security.currentContainer.id); +function beforeInsert(row, errors){ + // The purpose of this is to allow the user to provide a string value for + // vendorId or vendorName, and attempt to resolve this against known vendors: + if (!row.vendorId || isNaN(row.vendorId)) { + var vendorName = row.vendorName || row.vendorId; + if (vendorName) { + LABKEY.Query.selectRows({ + schemaName: 'labpurchasing', + queryName: 'vendors', + columns: 'rowid', + filterArray: [LABKEY.Filter.create('vendorName', vendorName)], + sort: '-rowid', + maxRows: 1, + scope: this, + success: function(results) { + if (results.rows.length) { + row.vendorId = results.rows[0].rowId; + } + } + }) + } + } +} + function afterInsert(row, errors){ afterUpsert(row, null, errors); } diff --git a/LabPurchasing/resources/queries/labpurchasing/purchases/.qview.xml b/LabPurchasing/resources/queries/labpurchasing/purchases/.qview.xml index 34b409cfc..963d35ba4 100644 --- a/LabPurchasing/resources/queries/labpurchasing/purchases/.qview.xml +++ b/LabPurchasing/resources/queries/labpurchasing/purchases/.qview.xml @@ -1,4 +1,4 @@ - + diff --git a/LabPurchasing/resources/queries/labpurchasing/purchases/Items To Order.qview.xml b/LabPurchasing/resources/queries/labpurchasing/purchases/Items To Order.qview.xml index 9d7deee43..31c56032a 100644 --- a/LabPurchasing/resources/queries/labpurchasing/purchases/Items To Order.qview.xml +++ b/LabPurchasing/resources/queries/labpurchasing/purchases/Items To Order.qview.xml @@ -1,4 +1,4 @@ - + diff --git a/LabPurchasing/resources/queries/labpurchasing/purchases/Waiting for Item.qview.xml b/LabPurchasing/resources/queries/labpurchasing/purchases/Waiting for Item.qview.xml index f4736de93..68e15235e 100644 --- a/LabPurchasing/resources/queries/labpurchasing/purchases/Waiting for Item.qview.xml +++ b/LabPurchasing/resources/queries/labpurchasing/purchases/Waiting for Item.qview.xml @@ -1,4 +1,4 @@ - + diff --git a/LabPurchasing/resources/queries/labpurchasing/referenceItems.js b/LabPurchasing/resources/queries/labpurchasing/referenceItems.js index 4848cdbcc..8a6d70e73 100644 --- a/LabPurchasing/resources/queries/labpurchasing/referenceItems.js +++ b/LabPurchasing/resources/queries/labpurchasing/referenceItems.js @@ -19,9 +19,6 @@ function beforeInsert(row, errors){ if (results.rows.length) { row.vendorId = results.rows[0].rowId; } - else { - console.error("Unable to resolve vendor: " + vendorName); - } } }) } diff --git a/LabPurchasing/resources/queries/labpurchasing/referenceItems/.qview.xml b/LabPurchasing/resources/queries/labpurchasing/referenceItems/.qview.xml index 62c4822f9..17c6c0344 100644 --- a/LabPurchasing/resources/queries/labpurchasing/referenceItems/.qview.xml +++ b/LabPurchasing/resources/queries/labpurchasing/referenceItems/.qview.xml @@ -1,4 +1,4 @@ - + diff --git a/PMR/build.gradle b/PMR/build.gradle index 40493795c..14abecf43 100644 --- a/PMR/build.gradle +++ b/PMR/build.gradle @@ -8,6 +8,7 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:Studies", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:onprcEHRModules:GeneticsCore", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:Studies", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "apiJarFile") diff --git a/PMR/resources/etls/KinshipDataStaging.xml b/PMR/resources/etls/KinshipDataStaging.xml new file mode 100644 index 000000000..d0b0b1e47 --- /dev/null +++ b/PMR/resources/etls/KinshipDataStaging.xml @@ -0,0 +1,14 @@ + + + KinshipDataStaging + Prepare PRIMe-seq Kinship Data for Import into PRIMe + + + + + + + + + + diff --git a/PMR/resources/folderTypes/PMR.folderType.xml b/PMR/resources/folderTypes/PMR.folderType.xml index 791e44bd1..95eb08a07 100644 --- a/PMR/resources/folderTypes/PMR.folderType.xml +++ b/PMR/resources/folderTypes/PMR.folderType.xml @@ -64,6 +64,7 @@ PMR EHR + GeneticsCore PMR \ No newline at end of file diff --git a/PMR/src/org/labkey/pmr/etl/TriggerRemoteGeneticsImportStep.java b/PMR/src/org/labkey/pmr/etl/TriggerRemoteGeneticsImportStep.java new file mode 100644 index 000000000..305d57dab --- /dev/null +++ b/PMR/src/org/labkey/pmr/etl/TriggerRemoteGeneticsImportStep.java @@ -0,0 +1,169 @@ +package org.labkey.pmr.etl; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.xmlbeans.XmlException; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.ModuleProperty; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.writer.ContainerUser; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.CommandResponse; +import org.labkey.remoteapi.PostCommand; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class TriggerRemoteGeneticsImportStep implements TaskRefTask +{ + protected final Map _settings = new CaseInsensitiveHashMap<>(); + protected ContainerUser _containerUser; + + private enum Settings + { + remoteSource() + } + + @Override + public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException + { + // First find the last successful pipeline iteration: + Module ehr = ModuleLoader.getInstance().getModule("ehr"); + Module geneticsCore = ModuleLoader.getInstance().getModule("GeneticsCore"); + + ModuleProperty mp = ehr.getModuleProperties().get("EHRStudyContainer"); + String ehrContainerPath = StringUtils.trimToNull(mp.getEffectiveValue(_containerUser.getContainer())); + if (ehrContainerPath == null) + { + throw new PipelineJobException("EHRStudyContainer has not been set"); + } + + Container ehrContainer = ContainerManager.getForPath(ehrContainerPath); + if (ehrContainer == null) + { + throw new PipelineJobException("Invalid container: " + ehrContainerPath); + } + + if (!_containerUser.getContainer().equals(ehrContainer)) + { + throw new PipelineJobException("This ETL can only be run from the EHRStudyContainer"); + } + + ModuleProperty mp2 = geneticsCore.getModuleProperties().get("KinshipDataPath"); + String pipeDirPath = StringUtils.trimToNull(mp2.getEffectiveValue(ehrContainer)); + if (pipeDirPath == null) + { + throw new PipelineJobException("Must provide the filepath to import data using the KinshipDataPath module property"); + } + + File targetPipelineDir = new File(pipeDirPath); + if (!targetPipelineDir.exists()) + { + targetPipelineDir.mkdirs(); + } + + // Then copy the file to the expected folder: + PipeRoot pr = PipelineService.get().getPipelineRootSetting(ehrContainer); + if (pr == null) + { + throw new PipelineJobException("Unable to find pipeline root for: " + ehrContainer); + } + + File sourceDir = new File(pr.getRootPath(), "/kinship/EHR Kinship Calculation"); + if (!sourceDir.exists()) + { + throw new PipelineJobException("Unable to find source pipeline dir: " + sourceDir.getPath()); + } + + copyReplaceFile(sourceDir, targetPipelineDir, "kinship.txt"); + copyReplaceFile(sourceDir, targetPipelineDir, "inbreeding.txt"); + + // Then ping the main server to import this file: + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(_settings.get(Settings.remoteSource.name()), _containerUser.getContainer(), job.getLogger()); + if (rc == null) + { + throw new PipelineJobException("Unable to find remote connection: " + _settings.get(Settings.remoteSource.name())); + } + + try + { + KinshipCommand command = new KinshipCommand(); + command.execute(rc.connection, rc.remoteContainer); + } + catch (CommandException | IOException e) + { + throw new PipelineJobException(e); + } + + return new RecordedActionSet(); + } + + private static class KinshipCommand extends PostCommand + { + public KinshipCommand() + { + super("geneticscore", "importGeneticsData"); + } + } + + private void copyReplaceFile(File sourceDir, File targetDir, String filename) throws PipelineJobException + { + File sourceFile = new File(sourceDir, filename); + if (!sourceFile.exists()) + { + throw new PipelineJobException("File does not exist: " + sourceFile.getPath()); + } + + File destFile = new File(targetDir, filename); + if (destFile.exists()) + { + destFile.delete(); + } + + try + { + FileUtils.copyFile(sourceFile, destFile); + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + } + + @Override + public List getRequiredSettings() + { + return Collections.unmodifiableList(Arrays.asList(Settings.remoteSource.name())); + } + + @Override + public void setSettings(Map settings) throws XmlException + { + _settings.putAll(settings); + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } +} diff --git a/PMR/test/sampledata/PMR/testPedigree.txt b/PMR/test/sampledata/PMR/testPedigree.txt new file mode 100644 index 000000000..866dbab16 --- /dev/null +++ b/PMR/test/sampledata/PMR/testPedigree.txt @@ -0,0 +1,10 @@ +99991 999911 2 Cynomolgus +999910 99993 99998 2 Cynomolgus +99992 999912 1 Cynomolgus +99993 999912 2 Rhesus +99994 999913 999914 1 Cynomolgus +99995 99991 99992 2 Cynomolgus +99996 99991 99992 1 Cynomolgus +99997 99993 99992 1 Cynomolgus +99998 99995 99992 1 Cynomolgus +99999 99995 99994 2 Cynomolgus diff --git a/PMR/test/src/org/labkey/test/tests/pmr/PMRTest.java b/PMR/test/src/org/labkey/test/tests/pmr/PMRTest.java index 028d7b755..650b7ffd8 100644 --- a/PMR/test/src/org/labkey/test/tests/pmr/PMRTest.java +++ b/PMR/test/src/org/labkey/test/tests/pmr/PMRTest.java @@ -16,25 +16,44 @@ package org.labkey.test.tests.pmr; +import au.com.bytecode.opencsv.CSVReader; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.TruncateTableCommand; +import org.labkey.serverapi.reader.Readers; import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.Locator; import org.labkey.test.ModulePropertyValue; +import org.labkey.test.TestFileUtils; import org.labkey.test.TestTimeoutException; import org.labkey.test.WebTestHelper; import org.labkey.test.categories.External; import org.labkey.test.categories.LabModule; +import org.labkey.test.util.Ext4Helper; +import org.labkey.test.util.RReportHelper; +import org.labkey.test.util.RemoteConnectionHelper; import org.labkey.test.util.SqlserverOnlyTest; +import org.labkey.test.util.di.DataIntegrationHelper; +import org.labkey.test.util.ehr.EHRClientAPIHelper; +import java.io.File; import java.util.Arrays; import java.util.Collections; +import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; @Category({External.class, LabModule.class}) public class PMRTest extends BaseWebDriverTest implements SqlserverOnlyTest { + private final DataIntegrationHelper _etlHelper = new DataIntegrationHelper(getProjectName()); + @Override protected void doCleanup(boolean afterTest) throws TestTimeoutException { @@ -48,6 +67,10 @@ public static void setupProject() throws Exception init.doSetup(); } + private File getKinshipPath() + { + return new File(TestFileUtils.getDefaultFileRoot(getProjectName()), "kinshipEtlDir"); + } private void doSetup() { @@ -55,10 +78,18 @@ private void doSetup() setModuleProperties(Arrays.asList( new ModulePropertyValue("EHR", "/" + getProjectName(), "EHRStudyContainer", "/" + getProjectName()), - new ModulePropertyValue("EHR", "/" + getProjectName(), "EHRAdminUser", getCurrentUser()) + new ModulePropertyValue("EHR", "/" + getProjectName(), "EHRAdminUser", getCurrentUser()), + new ModulePropertyValue("GeneticsCore", "/" + getProjectName(), "KinshipDataPath", getKinshipPath().getPath()) )); importStudy(getProjectName()); + populateLookups(); + + RemoteConnectionHelper rconnHelper = new RemoteConnectionHelper(this); + rconnHelper.goToManageRemoteConnections(); + rconnHelper.createConnection("EHR_ClinicalSource", WebTestHelper.getBaseURL(), getProjectName()); + + new RReportHelper(this).ensureRConfig(); goToHome(); } @@ -70,6 +101,16 @@ private void importStudy(String containerPath) waitForPipelineJobsToComplete(1, "Study import", false, MAX_WAIT_SECONDS * 2500); } + private void populateLookups() + { + beginAt(getProjectName() + "/pmr-populateData.view"); + waitAndClick(Ext4Helper.Locators.ext4Button("Populate Lookup Sets")); + waitForElement(Locator.tagWithText("div", "Populating lookup_sets...")); + waitForElement(Locator.tagWithText("div", "Populate Complete")); + + waitAndClick(Ext4Helper.Locators.ext4Button("Populate All")); + waitForElement(Locator.tagWithText("div", "Populate Complete")); + } @Before public void preTest() @@ -78,9 +119,92 @@ public void preTest() } @Test - public void testPMRModule() + public void testPMRModule() throws Exception + { + testKinshipEtl(); + } + + private void testKinshipEtl() throws Exception + { + createTestPedigreeData(); + + // Calculate genetics, which will output results in the projects file root: + beginAt(getProjectName() + "/ehr-ehrAdmin.view"); + waitAndClickAndWait(Locator.tagContainingText("a", "Genetics Calculations")); + _ext4Helper.checkCheckbox(Ext4Helper.Locators.checkbox(this, "Kinship validation?:")); + _ext4Helper.checkCheckbox(Ext4Helper.Locators.checkbox(this, "Allow Import During Business Hours?:")); + Locator loc = Locator.inputByIdContaining("numberfield"); + waitForElement(loc); + setFormElement(loc, "23"); + click(Ext4Helper.Locators.ext4Button("Save Settings")); + waitAndClick(Ext4Helper.Locators.ext4Button("OK")); + waitAndClickAndWait(Ext4Helper.Locators.ext4Button("Run Now")); + waitAndClickAndWait(Locator.lkButton("OK")); + waitForPipelineJobsToComplete(2, "EHR Kinship Calculation", false); + + // Verify data imported, and then delete from the DB + SelectRowsCommand select1 = new SelectRowsCommand("ehr", "kinship"); + Assert.assertEquals("Incorrect number of kinship rows", 136, select1.execute(getApiHelper().getConnection(), getProjectName()).getRowCount().intValue()); + + new TruncateTableCommand("ehr", "kinship").execute(getApiHelper().getConnection(), getProjectName()); + Assert.assertEquals("Incorrect number of kinship rows", 0, select1.execute(getApiHelper().getConnection(), getProjectName()).getRowCount().intValue()); + + // Kick off ETL to stage data. This should also kick off a separate pipeline job to import, using geneticscore-importGeneticsData.view + _etlHelper.runTransform("{PMR}/KinshipDataStaging"); + goToDataPipeline(); + waitForPipelineJobsToComplete(4, "ETL Job: Import PRIMe-seq Kinship Data", false); + + Assert.assertEquals("Incorrect number of kinship rows after ETL", 136, select1.execute(getApiHelper().getConnection(), getProjectName()).getRowCount().intValue()); + } + + private void createTestPedigreeData() throws Exception + { + // Create dummy data: + Set demographicsAdded = new HashSet<>(); + final Date d = new Date(); + + File testPedigree = TestFileUtils.getSampleData("PMR/testPedigree.txt"); + try (CSVReader reader = new CSVReader(Readers.getReader(testPedigree), '\t')) + { + String[] line; + while ((line = reader.readNext()) != null) + { + if (!demographicsAdded.contains(line[0])) + { + getApiHelper().insertRow("study", "demographics", Map.of("Id", line[0], "species", line[4], "gender", ("1".equals(line[3]) ? "m" : "f"), "date", d, "QCStateLabel", "Completed"), false); + demographicsAdded.add(line[0]); + } + + // dam + if (!line[1].isEmpty()) + { + if (!demographicsAdded.contains(line[1])) + { + getApiHelper().insertRow("study", "demographics", Map.of("Id", line[1], "species", line[4], "gender", "f", "date", d, "QCStateLabel", "Completed"), false); + demographicsAdded.add(line[1]); + } + + getApiHelper().insertRow("study", "parentage", Map.of("Id", line[0], "parent", line[1], "relationship", "Dam", "method", "Genetic", "date", d, "QCStateLabel", "Completed"), false); + } + + // sire + if (!line[2].isEmpty()) + { + if (!demographicsAdded.contains(line[2])) + { + getApiHelper().insertRow("study", "demographics", Map.of("Id", line[2], "species", line[4], "gender", "m", "date", d, "QCStateLabel", "Completed"), false); + demographicsAdded.add(line[2]); + } + + getApiHelper().insertRow("study", "parentage", Map.of("Id", line[0], "parent", line[2], "relationship", "Sire", "method", "Genetic", "date", d, "QCStateLabel", "Completed"), false); + } + } + } + } + + private EHRClientAPIHelper getApiHelper() { - _containerHelper.enableModule("PMR"); + return new EHRClientAPIHelper(this, getProjectName()); } @Override diff --git a/flowassays/resources/queries/flowassays/populations.js b/flowassays/resources/queries/flowassays/populations.js index 2902908d0..cf24cdf21 100644 --- a/flowassays/resources/queries/flowassays/populations.js +++ b/flowassays/resources/queries/flowassays/populations.js @@ -16,7 +16,7 @@ function beforeInsert(row, errors){ validateName(row, errors); } -function beforeUpdate(row, errors){ +function beforeUpdate(row, oldRow, errors){ validateName(row, errors); } diff --git a/flowassays/resources/views/initModule.html b/flowassays/resources/views/initModule.html deleted file mode 100644 index 0eb8fd42d..000000000 --- a/flowassays/resources/views/initModule.html +++ /dev/null @@ -1,135 +0,0 @@ - - - -
-
diff --git a/mGAP/resources/views/quickLinks.html b/mGAP/resources/views/quickLinks.html index 38e32072b..c486c8bbf 100644 --- a/mGAP/resources/views/quickLinks.html +++ b/mGAP/resources/views/quickLinks.html @@ -2,7 +2,7 @@ ul.mgapList { padding-left: 20px; padding-top: 5px; - }​ + }