From aea2c297da5c8a3d3c927c2e83d27878da82daf7 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 14 Dec 2022 17:50:10 -0800 Subject: [PATCH 1/7] Improve test timing --- mcc/test/src/org/labkey/test/tests/mcc/MccTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java index af03bdab2..9af9747a4 100644 --- a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java +++ b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java @@ -733,6 +733,7 @@ private void doSetup() throws Exception beginAt("/mcc/" + getProjectName() + "/configureMcc.view"); clickButton("OK"); + waitForElement(Locator.tagWithText("a", "Populate Lookups")); ApiPermissionsHelper helper = new ApiPermissionsHelper(this); if (!helper.isUserInGroup(getCurrentUser(), "MCC RAB Members", "/", PermissionsHelper.PrincipalType.USER)) From a67a64f5604337974bcb1d9fc7d59c3ee53f624d Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 14 Dec 2022 19:33:44 -0800 Subject: [PATCH 2/7] Improve test timing --- .../org/labkey/test/tests/mcc/MccTest.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java index 9af9747a4..718a4b4ad 100644 --- a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java +++ b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java @@ -65,7 +65,7 @@ public void testMccModule() throws Exception testAnimalImportAndTransfer(); } - private void testAnimalImportAndTransfer() + private void testAnimalImportAndTransfer() throws Exception { beginAt(getProjectName() + "/Colonies/SNPRC/project-begin.view"); waitAndClickAndWait(Locator.tagWithText("a", "Import Excel-Based Data")); @@ -130,6 +130,14 @@ private void testAnimalImportAndTransfer() Assert.assertEquals("Incorrect Status", "", dr.getDataAsText(0, "Status")); Assert.assertEquals("Incorrect Colony", "TargetColony", dr.getDataAsText(0, "colony")); Assert.assertEquals("Incorrect Source", "SNPRC", dr.getDataAsText(0, "source")); + + // These were inserted using a cross-folder SaveRows, and this check ensures the trigger script containerPath and serverContex works as expected: +// SelectRowsCommand sr = new SelectRowsCommand("study", "demographics"); +// sr.setColumns(Arrays.asList("Id", "QCState/Label")); +// SelectRowsResponse srr = sr.execute(createDefaultConnection(), getProjectName() + "/Colonies/Other"); +// srr.getRows().forEach(row -> { +// Assert.assertEquals("Incorrect QCState", "Completed", row.get("QCState/Label")); +// }); } private static class FormElement @@ -292,12 +300,16 @@ private Locator getButton(String text) return Locator.tagWithText("button", text); } + private void waitForCensusToLoad() + { + waitForElement(Locator.tagWithText("div", "Age (Living Animals)")); //proxy for data loading + } + private void goToAnimalRequests() { goToProjectHome(); - waitForElement(Locator.tagWithText("div", "Age (Living Animals)")); //proxy for data loading + waitForCensusToLoad(); waitAndClickAndWait(Locator.tagContainingText("div", "Animal Requests")); - waitForElement(Locator.tagWithText("a", "Submit New Animal Request")); } @@ -733,7 +745,7 @@ private void doSetup() throws Exception beginAt("/mcc/" + getProjectName() + "/configureMcc.view"); clickButton("OK"); - waitForElement(Locator.tagWithText("a", "Populate Lookups")); + waitForCensusToLoad(); ApiPermissionsHelper helper = new ApiPermissionsHelper(this); if (!helper.isUserInGroup(getCurrentUser(), "MCC RAB Members", "/", PermissionsHelper.PrincipalType.USER)) @@ -751,6 +763,7 @@ private void doSetup() throws Exception { _containerHelper.createSubfolder(getProjectName() + "/Colonies", name, "MCC Colony"); importStudy(getProjectName() + "/Colonies/" + name); + waitForElement(Locator.tagWithText("a", "Populate Lookups")); } } @@ -770,7 +783,6 @@ private void importStudy(String containerPath) beginAt(WebTestHelper.getBaseURL() + "/ehr/" + containerPath + "/ensureQcStates.view"); clickButton("OK"); - } @Before From ffd440b0db9b998c0f54f1fcaee705a9904d906f Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 27 Dec 2022 11:18:08 -0800 Subject: [PATCH 3/7] Expand MCC test now that cross-folder saveRows works --- .../src/org/labkey/test/tests/mcc/MccTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java index 718a4b4ad..cf1163020 100644 --- a/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java +++ b/mcc/test/src/org/labkey/test/tests/mcc/MccTest.java @@ -102,7 +102,7 @@ private void testAnimalImportAndTransfer() throws Exception combo.clickTrigger(); waitAndClick(Locator.tagContainingText("li", "Other")); - Window dialog = new Window.WindowFinder(getDriver()).withTitle("Enter Value").waitFor(); + Window dialog = new Window.WindowFinder(getDriver()).withTitle("Enter Value").waitFor(); dialog.findElement(Locator.tag("input")).sendKeys("TargetColony"); waitAndClick(Ext4Helper.Locators.ext4Button("OK")); sleep(100); @@ -132,12 +132,12 @@ private void testAnimalImportAndTransfer() throws Exception Assert.assertEquals("Incorrect Source", "SNPRC", dr.getDataAsText(0, "source")); // These were inserted using a cross-folder SaveRows, and this check ensures the trigger script containerPath and serverContex works as expected: -// SelectRowsCommand sr = new SelectRowsCommand("study", "demographics"); -// sr.setColumns(Arrays.asList("Id", "QCState/Label")); -// SelectRowsResponse srr = sr.execute(createDefaultConnection(), getProjectName() + "/Colonies/Other"); -// srr.getRows().forEach(row -> { -// Assert.assertEquals("Incorrect QCState", "Completed", row.get("QCState/Label")); -// }); + SelectRowsCommand sr = new SelectRowsCommand("study", "demographics"); + sr.setColumns(Arrays.asList("Id", "QCState/Label")); + SelectRowsResponse srr = sr.execute(createDefaultConnection(), getProjectName() + "/Colonies/Other"); + srr.getRows().forEach(row -> { + Assert.assertEquals("Incorrect QCState", "Completed", row.get("QCState/Label")); + }); } private static class FormElement From a35d85d9dc74d944109253b78cbd8c9165a353b0 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 5 Jan 2023 15:43:01 -0800 Subject: [PATCH 4/7] - Expand SDA plotting - Allow TCR cell hashing to use demuxEM --- .../pipeline/CellRangerVDJCellHashingHandler.java | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 770ef058a..48a574353 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -62,12 +62,9 @@ private static List getDefaultParams() ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook. This is primarily useful if submitting a large batch of files to process separately. This only applies if 'Run Separately' is selected.", "checkbox", new JSONObject(){{ put("checked", true); }}, false) -// ToolParameterDescriptor.create(USE_GEX_BARCODES, "Use GEX and TCR Cell Barcodes", "If checked, the cell barcode whitelist used for cell hashing will be the union of TCR and GEX cell barcodes. If T-cells are a rare component of total cells, this might enhance the effectiveness of the callers by providing more positive signal.", "checkbox", new JSONObject(){{ -// put("checked", true); -// }}, false) )); - ret.addAll(CellHashingService.get().getHashingCallingParams(false)); + ret.addAll(CellHashingService.get().getHashingCallingParams(true)); return ret; } @@ -114,14 +111,7 @@ public class Processor implements SequenceOutputHandler.SequenceOutputProcessor public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException { //NOTE: this is the pathway to import assay data, whether hashing is used or not - CellHashingService.get().prepareHashingForVdjIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", false); - - if (ctx.getParams().optBoolean(USE_GEX_BARCODES, false)) - { - ctx.getJob().getLogger().info("The union of TCR and GEX cell barcodes will be used for calling"); - Map vLoupeIdToGexBarcodeDir = new HashMap<>(); - - } + CellHashingService.get().prepareHashingForVdjIfNeeded(ctx, false); } @Override From e685ce5cc42fabe274d2bb12c58c8d061aeb24fd Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 6 Jan 2023 09:32:40 -0800 Subject: [PATCH 5/7] Bugfix to demuxEM with TCR data --- .../CellRangerVDJCellHashingHandler.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java index 48a574353..d43cfbc7d 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java @@ -45,7 +45,6 @@ public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutput public static final String TARGET_ASSAY = "targetAssay"; public static final String DELETE_EXISTING_ASSAY_DATA = "deleteExistingAssayData"; - public static final String USE_GEX_BARCODES = "useGexBarcodes"; public CellRangerVDJCellHashingHandler() { @@ -205,6 +204,27 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco parameters.basename = FileUtil.makeLegalName(rs.getName()); parameters.allowableHtoBarcodes = htosPerReadset; + // If demuxEM used: + if (parameters.methods.contains(CellHashingService.CALLING_METHOD.demuxem) || parameters.consensusMethods.contains(CellHashingService.CALLING_METHOD.demuxem)) + { + ctx.getLogger().debug("demuxEM is used, adding H5 file"); + if (genomeId == null) + { + genomeId = ctx.getSequenceSupport().getCachedGenomes().iterator().next().getGenomeId(); + ctx.getLogger().debug("Unable to infer genome ID from output, defaulting to the first cached genome: " + genomeId); + } + + parameters.h5File = CellHashingService.get().getH5FileForGexReadset(ctx.getSequenceSupport(), rs.getReadsetId(), genomeId); + if (parameters.h5File == null) + { + throw new PipelineJobException("Unable to find h5 file for: " + rs.getRowId()); + } + else if (!parameters.h5File.exists()) + { + throw new PipelineJobException("h5 file does not exist: " + parameters.h5File.getPath()); + } + } + parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true); File existingCountMatrixUmiDir = CellHashingService.get().getExistingFeatureBarcodeCountDir(rs, CellHashingService.BARCODE_TYPE.hashing, ctx.getSequenceSupport()); From 37e7f0a3ea7554cbd60d647359dc1814ff28206f Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 7 Jan 2023 14:19:03 -0800 Subject: [PATCH 6/7] Allow CreateSequenceDictionary to work even if picard.jar isnt found --- tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java b/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java index 56ad37e5b..75abc6d8c 100644 --- a/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java +++ b/tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java @@ -231,7 +231,7 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc wrapper.addToEnvironment("JAVA", SequencePipelineService.get().getJavaFilepath()); wrapper.addToEnvironment("SAMTOOLS", SequencePipelineService.get().getExeForPackage("SAMTOOLSPATH", "samtools").getPath()); - wrapper.addToEnvironment("PICARD", PicardWrapper.getPicardJar().getPath()); + wrapper.addToEnvironment("PICARD", PicardWrapper.getPicardJar(true).getPath()); wrapper.setWorkingDir(outputDir); wrapper.execute(Arrays.asList("bash", bamScript.getPath(), inputBam.getPath(), forwardFq.getPath(), reverseFq.getPath())); From 8063c2226601a82e32e32a73eb9018e05b2b8d04 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 10 Jan 2023 09:16:59 -0800 Subject: [PATCH 7/7] Allow both light and deep WGS samples --- .../mgap/pipeline/SampleSpecificGenotypeFiltrationStep.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mGAP/src/org/labkey/mgap/pipeline/SampleSpecificGenotypeFiltrationStep.java b/mGAP/src/org/labkey/mgap/pipeline/SampleSpecificGenotypeFiltrationStep.java index 50b9cbbb9..f2e202d3d 100644 --- a/mGAP/src/org/labkey/mgap/pipeline/SampleSpecificGenotypeFiltrationStep.java +++ b/mGAP/src/org/labkey/mgap/pipeline/SampleSpecificGenotypeFiltrationStep.java @@ -112,6 +112,9 @@ else if (header.getSampleNamesInOrder().size() != 1) case "Whole Genome: Deep Coverage": setName = "WGS"; break; + case "Whole Genome: Light Coverage": + setName = "WGS"; + break; case "Whole Exome": setName = "WXS"; break;