Merge discvr-22.11 to develop

mGAP/src/org/labkey/mgap/pipeline/SampleSpecificGenotypeFiltrationStep.java

@@ -112,6 +112,9 @@ else if (header.getSampleNamesInOrder().size() != 1)
                         case "Whole Genome: Deep Coverage":
                             setName = "WGS";
                             break;
+                        case "Whole Genome: Light Coverage":
+                            setName = "WGS";
+                            break;
                         case "Whole Exome":
                             setName = "WXS";
                             break;

mcc/test/src/org/labkey/test/tests/mcc/MccTest.java

@@ -65,7 +65,7 @@ public void testMccModule() throws Exception
         testAnimalImportAndTransfer();
     }
 
-    private void testAnimalImportAndTransfer()
+    private void testAnimalImportAndTransfer() throws Exception
     {
         beginAt(getProjectName() + "/Colonies/SNPRC/project-begin.view");
         waitAndClickAndWait(Locator.tagWithText("a", "Import Excel-Based Data"));
@@ -102,7 +102,7 @@ private void testAnimalImportAndTransfer()
         combo.clickTrigger();
         waitAndClick(Locator.tagContainingText("li", "Other"));
 
-        Window dialog = new Window.WindowFinder(getDriver()).withTitle("Enter Value").waitFor();
+        Window<?> dialog = new Window.WindowFinder(getDriver()).withTitle("Enter Value").waitFor();
         dialog.findElement(Locator.tag("input")).sendKeys("TargetColony");
         waitAndClick(Ext4Helper.Locators.ext4Button("OK"));
         sleep(100);
@@ -130,6 +130,14 @@ private void testAnimalImportAndTransfer()
         Assert.assertEquals("Incorrect Status", "<Alive>", dr.getDataAsText(0, "Status"));
         Assert.assertEquals("Incorrect Colony", "TargetColony", dr.getDataAsText(0, "colony"));
         Assert.assertEquals("Incorrect Source", "SNPRC", dr.getDataAsText(0, "source"));
+
+        // These were inserted using a cross-folder SaveRows, and this check ensures the trigger script containerPath and serverContex works as expected:
+        SelectRowsCommand sr = new SelectRowsCommand("study", "demographics");
+        sr.setColumns(Arrays.asList("Id", "QCState/Label"));
+        SelectRowsResponse srr = sr.execute(createDefaultConnection(), getProjectName() + "/Colonies/Other");
+        srr.getRows().forEach(row -> {
+            Assert.assertEquals("Incorrect QCState", "Completed", row.get("QCState/Label"));
+        });
     }
 
     private static class FormElement
@@ -292,12 +300,16 @@ private Locator getButton(String text)
         return Locator.tagWithText("button", text);
     }
 
+    private void waitForCensusToLoad()
+    {
+        waitForElement(Locator.tagWithText("div", "Age (Living Animals)")); //proxy for data loading
+    }
+
     private void goToAnimalRequests()
     {
         goToProjectHome();
-        waitForElement(Locator.tagWithText("div", "Age (Living Animals)")); //proxy for data loading
+        waitForCensusToLoad();
         waitAndClickAndWait(Locator.tagContainingText("div", "Animal Requests"));
-
         waitForElement(Locator.tagWithText("a", "Submit New Animal Request"));
     }
 
@@ -733,6 +745,7 @@ private void doSetup() throws Exception
 
         beginAt("/mcc/" + getProjectName() + "/configureMcc.view");
         clickButton("OK");
+        waitForCensusToLoad();
 
         ApiPermissionsHelper helper = new ApiPermissionsHelper(this);
         if (!helper.isUserInGroup(getCurrentUser(), "MCC RAB Members", "/", PermissionsHelper.PrincipalType.USER))
@@ -750,6 +763,7 @@ private void doSetup() throws Exception
         {
             _containerHelper.createSubfolder(getProjectName() + "/Colonies", name, "MCC Colony");
             importStudy(getProjectName() + "/Colonies/" + name);
+            waitForElement(Locator.tagWithText("a", "Populate Lookups"));
         }
     }
 
@@ -769,7 +783,6 @@ private void importStudy(String containerPath)
 
         beginAt(WebTestHelper.getBaseURL() + "/ehr/" + containerPath + "/ensureQcStates.view");
         clickButton("OK");
-
     }
 
     @Before

tcrdb/src/org/labkey/tcrdb/pipeline/CellRangerVDJCellHashingHandler.java

@@ -45,7 +45,6 @@ public class CellRangerVDJCellHashingHandler extends AbstractParameterizedOutput
 
     public static final String TARGET_ASSAY = "targetAssay";
     public static final String DELETE_EXISTING_ASSAY_DATA = "deleteExistingAssayData";
-    public static final String USE_GEX_BARCODES = "useGexBarcodes";
 
     public CellRangerVDJCellHashingHandler()
     {
@@ -62,12 +61,9 @@ private static List<ToolParameterDescriptor> getDefaultParams()
                 ToolParameterDescriptor.create("useOutputFileContainer", "Submit to Source File Workbook", "If checked, each job will be submitted to the same workbook as the input file, as opposed to submitting all jobs to the same workbook.  This is primarily useful if submitting a large batch of files to process separately. This only applies if 'Run Separately' is selected.", "checkbox", new JSONObject(){{
                     put("checked", true);
                 }}, false)
-//                ToolParameterDescriptor.create(USE_GEX_BARCODES, "Use GEX and TCR Cell Barcodes", "If checked, the cell barcode whitelist used for cell hashing will be the union of TCR and GEX cell barcodes. If T-cells are a rare component of total cells, this might enhance the effectiveness of the callers by providing more positive signal.", "checkbox", new JSONObject(){{
-//                    put("checked", true);
-//                }}, false)
         ));
 
-        ret.addAll(CellHashingService.get().getHashingCallingParams(false));
+        ret.addAll(CellHashingService.get().getHashingCallingParams(true));
 
         return ret;
     }
@@ -114,14 +110,7 @@ public class Processor implements SequenceOutputHandler.SequenceOutputProcessor
         public void init(JobContext ctx, List<SequenceOutputFile> inputFiles, List<RecordedAction> actions, List<SequenceOutputFile> outputsToCreate) throws UnsupportedOperationException, PipelineJobException
         {
             //NOTE: this is the pathway to import assay data, whether hashing is used or not
-            CellHashingService.get().prepareHashingForVdjIfNeeded(ctx.getOutputDir(), ctx.getJob(), ctx.getSequenceSupport(), "tcrReadsetId", false);
-
-            if (ctx.getParams().optBoolean(USE_GEX_BARCODES, false))
-            {
-                ctx.getJob().getLogger().info("The union of TCR and GEX cell barcodes will be used for calling");
-                Map<Integer, File> vLoupeIdToGexBarcodeDir = new HashMap<>();
-
-            }
+            CellHashingService.get().prepareHashingForVdjIfNeeded(ctx, false);
         }
 
         @Override
@@ -215,6 +204,27 @@ private void processVloupeFile(JobContext ctx, File perCellTsv, Readset rs, Reco
                 parameters.basename = FileUtil.makeLegalName(rs.getName());
                 parameters.allowableHtoBarcodes = htosPerReadset;
 
+                // If demuxEM used:
+                if (parameters.methods.contains(CellHashingService.CALLING_METHOD.demuxem) || parameters.consensusMethods.contains(CellHashingService.CALLING_METHOD.demuxem))
+                {
+                    ctx.getLogger().debug("demuxEM is used, adding H5 file");
+                    if (genomeId == null)
+                    {
+                        genomeId = ctx.getSequenceSupport().getCachedGenomes().iterator().next().getGenomeId();
+                        ctx.getLogger().debug("Unable to infer genome ID from output, defaulting to the first cached genome: " + genomeId);
+                    }
+
+                    parameters.h5File = CellHashingService.get().getH5FileForGexReadset(ctx.getSequenceSupport(), rs.getReadsetId(), genomeId);
+                    if (parameters.h5File == null)
+                    {
+                        throw new PipelineJobException("Unable to find h5 file for: " + rs.getRowId());
+                    }
+                    else if (!parameters.h5File.exists())
+                    {
+                        throw new PipelineJobException("h5 file does not exist: " + parameters.h5File.getPath());
+                    }
+                }
+
                 parameters.cellBarcodeWhitelistFile = createCellbarcodeWhitelist(ctx, perCellTsv, true);
                 File existingCountMatrixUmiDir = CellHashingService.get().getExistingFeatureBarcodeCountDir(rs, CellHashingService.BARCODE_TYPE.hashing, ctx.getSequenceSupport());
 

tcrdb/src/org/labkey/tcrdb/pipeline/MiXCRAnalysis.java

@@ -231,7 +231,7 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc
 
             wrapper.addToEnvironment("JAVA", SequencePipelineService.get().getJavaFilepath());
             wrapper.addToEnvironment("SAMTOOLS", SequencePipelineService.get().getExeForPackage("SAMTOOLSPATH", "samtools").getPath());
-            wrapper.addToEnvironment("PICARD", PicardWrapper.getPicardJar().getPath());
+            wrapper.addToEnvironment("PICARD", PicardWrapper.getPicardJar(true).getPath());
             wrapper.setWorkingDir(outputDir);
             wrapper.execute(Arrays.asList("bash", bamScript.getPath(), inputBam.getPath(), forwardFq.getPath(), reverseFq.getPath()));