From 1653f87394303416b01d371135cac0094f7dbe31 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 20 Mar 2024 15:26:07 -0700
Subject: [PATCH 01/31] Read pipeline params from the URL in more places

---
 .../panel/AlignmentImportPanel.js                |  3 ++-
 .../panel/AnalysisSectionPanel.js                |  6 +++++-
 .../SequenceAnalysis/panel/BaseSequencePanel.js  | 16 +++++++++++++---
 .../panel/SequenceAnalysisPanel.js               |  3 ++-
 .../panel/VariantProcessingPanel.js              |  3 ++-
 .../panel/SingleCellProcessingPanel.js           |  3 ++-
 6 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js
index 73a786c9d..bb484a17c 100644
--- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js
+++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js
@@ -86,7 +86,8 @@ Ext4.define('SequenceAnalysis.panel.AlignmentImportPanel', {
                 helpPopup: 'Description for this run, such as detail about the source of the alignments (optional)',
                 itemId: 'jobDescription',
                 name: 'jobDescription',
-                allowBlank: true
+                allowBlank: true,
+                value: LABKEY.ActionURL.getParameter('jobDescription')
             },{
                 fieldLabel: 'Delete Intermediate Files',
                 helpPopup: 'Check to delete the intermediate files created by this pipeline.  In general these are not needed and it will save disk space.  These files might be useful for debugging though.',
diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AnalysisSectionPanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AnalysisSectionPanel.js
index 56c7f02b4..8522f184c 100644
--- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AnalysisSectionPanel.js
+++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AnalysisSectionPanel.js
@@ -570,7 +570,7 @@ Ext4.define('SequenceAnalysis.panel.AnalysisSectionPanel', {
         return [];
     },
 
-    applySavedValues: function(values){
+    applySavedValues: function(values, allowUrlOverride){
         if (this.stepType){
             var tools = values[this.stepType] ? values[this.stepType].split(';') : [];
             this.setActiveTools(tools);
@@ -585,6 +585,10 @@ Ext4.define('SequenceAnalysis.panel.AnalysisSectionPanel', {
                 if (Ext4.isDefined(values[name])){
                     p.setValue(values[name]);
                 }
+
+                if (allowUrlOverride && LABKEY.ActionURL.getParameter(name)) {
+                    p.setValue(LABKEY.ActionURL.getParameter(name));
+                }
             }, this);
         }
     },
diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/BaseSequencePanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/BaseSequencePanel.js
index 2c393ab36..81ea54519 100644
--- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/BaseSequencePanel.js
+++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/BaseSequencePanel.js
@@ -269,6 +269,9 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
                                                     return;
                                                 }
 
+                                                // If auto-loading, assume we want to read the URL
+                                                thePanel.down('#readUrlParams').setValue(true);
+
                                                 var recIdx = store.find('name', LABKEY.ActionURL.getParameter('template'));
                                                 if (recIdx > -1) {
                                                     thePanel.down('labkey-combo').setValue(store.getAt(recIdx));
@@ -297,6 +300,12 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
                                 helpPopup: 'By default, the pipelines jobs and their outputs will be created in the workbook you selected. However, in certain cases, such as bulk submission of many jobs, it might be preferable to submit each job to the source folder/workbook for each input. Checking this box will enable this.',
                                 fieldLabel: 'Submit Jobs to Same Folder/Workbook as Readset',
                                 labelWidth: 200
+                            },{
+                                xtype: 'checkbox',
+                                itemId: 'readUrlParams',
+                                helpPopup: 'If true, any parameters provided on the URL with the same name as a parameter in the JSON will be read and override the template.',
+                                fieldLabel: 'Read Parameters From URL',
+                                labelWidth: 200
                             }]
                         }],
                         buttons: [{
@@ -353,7 +362,8 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
                                 delete json.submitJobToReadsetContainer;
                             }
 
-                            win.sequencePanel.applySavedValues(json);
+                            var readUrlParams = win.down('#readUrlParams').getValue();
+                            win.sequencePanel.applySavedValues(json, readUrlParams);
 
                             var submitJobToReadsetContainer = win.sequencePanel.down('[name="submitJobToReadsetContainer"]');
                             if (submitJobToReadsetContainer) {
@@ -386,7 +396,7 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
         }
     },
 
-    applySavedValues: function(values){
+    applySavedValues: function(values, allowUrlOverride){
         //allows for subclasses to exclude this panel
         var alignPanel = this.down('sequenceanalysis-alignmentpanel');
         if (alignPanel) {
@@ -395,7 +405,7 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
 
         var sections = this.query('sequenceanalysis-analysissectionpanel');
         Ext4.Array.forEach(sections, function(s){
-            s.applySavedValues(values);
+            s.applySavedValues(values, allowUrlOverride);
         }, this);
 
         // For top-level properties:
diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js
index bd3a422dd..dfaa1e07d 100644
--- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js
+++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js
@@ -300,7 +300,8 @@ Ext4.define('SequenceAnalysis.panel.SequenceAnalysisPanel', {
                 height: 100,
                 helpPopup: 'Description for this analysis (optional)',
                 name: 'jobDescription',
-                allowBlank:true
+                allowBlank: true,
+                value: LABKEY.ActionURL.getParameter('jobDescription')
             },{
                 fieldLabel: 'Delete Intermediate Files',
                 helpPopup: 'Check to delete the intermediate files created by this pipeline.  In general these are not needed and it will save disk space.  These files might be useful for debugging though.',
diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js
index 9cf64ff68..b67b01477 100644
--- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js
+++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js
@@ -61,7 +61,8 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
 				height: 100,
 				helpPopup: 'Description for this analysis (optional)',
 				name: 'jobDescription',
-				allowBlank:true
+				allowBlank: true,
+				value: LABKEY.ActionURL.getParameter('jobDescription')
 			},{
 				xtype: 'sequenceanalysis-variantscattergatherpanel',
 				width: 620,
diff --git a/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js b/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js
index 242b90c1f..da0d0a192 100644
--- a/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js
+++ b/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js
@@ -58,7 +58,8 @@ Ext4.define('SingleCell.panel.SingleCellProcessingPanel', {
                 height: 100,
                 helpPopup: 'Description for this analysis (optional)',
                 name: 'jobDescription',
-                allowBlank:true
+                allowBlank:true,
+                value: LABKEY.ActionURL.getParameter('jobDescription')
             },{
                 xtype: 'combo',
                 name: 'submissionType',

From 5a55965b44c15a411a430a5b63e44a7524567962 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 21 Mar 2024 09:54:52 -0700
Subject: [PATCH 02/31] Support maxDEPTH for cellsnp-lite

---
 .../singlecell/pipeline/singlecell/VireoHandler.java | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
index 9dbc0a864..f54dd5bc7 100644
--- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
+++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
@@ -36,6 +36,9 @@ public VireoHandler()
                 ToolParameterDescriptor.create("nDonors", "# Donors", "The number of donors to demultiplex", "ldk-integerfield", new JSONObject(){{
                     put("allowBlank", false);
                 }}, null),
+                ToolParameterDescriptor.create("maxDepth", "Max Depth", "At a position, read maximally INT reads per input file, to avoid excessive memory usage", "ldk-integerfield", new JSONObject(){{
+                    put("minValue", 0);
+                }}, null),
                 ToolParameterDescriptor.create("contigs", "Allowable Contigs", "A comma-separated list of contig names to use", "textfield", new JSONObject(){{
 
                 }}, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20")
@@ -72,7 +75,7 @@ public SequenceOutputProcessor getProcessor()
         return new Processor();
     }
 
-    public class Processor implements SequenceOutputProcessor
+    public static class Processor implements SequenceOutputProcessor
     {
         @Override
         public void init(JobContext ctx, List<SequenceOutputFile> inputFiles, List<RecordedAction> actions, List<SequenceOutputFile> outputsToCreate) throws UnsupportedOperationException, PipelineJobException
@@ -178,6 +181,13 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
             cellsnp.add("--minCOUNT");
             cellsnp.add("100");
 
+            String maxDepth = StringUtils.trimToNull(ctx.getParams().optString("maxDepth"));
+            if (maxDepth != null)
+            {
+                cellsnp.add("--maxDEPTH");
+                cellsnp.add(maxDepth);
+            }
+
             cellsnp.add("--gzip");
 
             ReferenceGenome genome = ctx.getSequenceSupport().getCachedGenome(inputFiles.get(0).getLibrary_id());

From d0dd7f55ba35452ab48e6f76eedbf32a06939aae Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 22 Mar 2024 14:31:22 -0700
Subject: [PATCH 03/31] Support reference-guided mode for cellsnp-lite

---
 .../pipeline/singlecell/VireoHandler.java     | 39 +++++++++++++++----
 1 file changed, 32 insertions(+), 7 deletions(-)

diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
index f54dd5bc7..6dab957b1 100644
--- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
+++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
@@ -16,6 +16,7 @@
 import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
 import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
+import org.labkey.api.util.PageFlowUtil;
 import org.labkey.api.writer.PrintWriters;
 import org.labkey.singlecell.SingleCellModule;
 import org.labkey.singlecell.run.CellRangerGexCountStep;
@@ -30,18 +31,27 @@
 
 public class VireoHandler  extends AbstractParameterizedOutputHandler<SequenceOutputHandler.SequenceOutputProcessor>
 {
+    private static final String REF_VCF =  "refVCF";
+
     public VireoHandler()
     {
-        super(ModuleLoader.getInstance().getModule(SingleCellModule.class), "Run Vireo", "This will run cellsnp-lite and vireo to infer cell-to-sample based on genotype.", null, Arrays.asList(
+        super(ModuleLoader.getInstance().getModule(SingleCellModule.class), "Run Vireo", "This will run cellsnp-lite and vireo to infer cell-to-sample based on genotype.", PageFlowUtil.set("sequenceanalysis/field/SequenceOutputFileSelectorField.js"), Arrays.asList(
                 ToolParameterDescriptor.create("nDonors", "# Donors", "The number of donors to demultiplex", "ldk-integerfield", new JSONObject(){{
                     put("allowBlank", false);
                 }}, null),
                 ToolParameterDescriptor.create("maxDepth", "Max Depth", "At a position, read maximally INT reads per input file, to avoid excessive memory usage", "ldk-integerfield", new JSONObject(){{
                     put("minValue", 0);
-                }}, null),
+                }}, 50000),
                 ToolParameterDescriptor.create("contigs", "Allowable Contigs", "A comma-separated list of contig names to use", "textfield", new JSONObject(){{
 
-                }}, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20")
+                }}, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20"),
+                ToolParameterDescriptor.createExpDataParam(REF_VCF, "Reference SNV Sites", "If provided, these sites will be used to screen for SNPs, instead of discovering them. If provided, the contig list will be ignored", "sequenceanalysis-sequenceoutputfileselectorfield", new JSONObject()
+                {{
+                    put("allowBlank", true);
+                    put("category", "VCF File");
+                    put("performGenomeFilter", false);
+                    put("doNotIncludeInTemplates", true);
+                }}, null)
         ));
     }
 
@@ -195,11 +205,26 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
             cellsnp.add("--refseq");
             cellsnp.add(genome.getWorkingFastaFile().getPath());
 
-            String contigs = ctx.getParams().optString("contigs", "");
-            if (!StringUtils.isEmpty(contigs))
+            int vcfFile = ctx.getParams().optInt(REF_VCF, -1);
+            if (vcfFile > -1)
+            {
+                File vcf = ctx.getSequenceSupport().getCachedData(vcfFile);
+                if (vcf == null || ! vcf.exists())
+                {
+                    throw new PipelineJobException("Unable to find file with ID: " + vcfFile);
+                }
+
+                cellsnp.add("-R");
+                cellsnp.add(vcf.getPath());
+            }
+            else
             {
-                cellsnp.add("--chrom");
-                cellsnp.add(contigs);
+                String contigs = ctx.getParams().optString("contigs", "");
+                if (!StringUtils.isEmpty(contigs))
+                {
+                    cellsnp.add("--chrom");
+                    cellsnp.add(contigs);
+                }
             }
 
             new SimpleScriptWrapper(ctx.getLogger()).execute(cellsnp);

From cde3d392224dad97ad6c6082e5cc7b5aa7670b46 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 22 Mar 2024 14:43:30 -0700
Subject: [PATCH 04/31] Improve mGapMaintenanceTask

---
 .../labkey/singlecell/pipeline/singlecell/VireoHandler.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
index 6dab957b1..4c460425f 100644
--- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
+++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
@@ -27,6 +27,7 @@
 import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.LinkedHashSet;
 import java.util.List;
 
 public class VireoHandler  extends AbstractParameterizedOutputHandler<SequenceOutputHandler.SequenceOutputProcessor>
@@ -35,7 +36,7 @@ public class VireoHandler  extends AbstractParameterizedOutputHandler<SequenceOu
 
     public VireoHandler()
     {
-        super(ModuleLoader.getInstance().getModule(SingleCellModule.class), "Run Vireo", "This will run cellsnp-lite and vireo to infer cell-to-sample based on genotype.", PageFlowUtil.set("sequenceanalysis/field/SequenceOutputFileSelectorField.js"), Arrays.asList(
+        super(ModuleLoader.getInstance().getModule(SingleCellModule.class), "Run Vireo", "This will run cellsnp-lite and vireo to infer cell-to-sample based on genotype.", new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/SequenceOutputFileSelectorField.js")), Arrays.asList(
                 ToolParameterDescriptor.create("nDonors", "# Donors", "The number of donors to demultiplex", "ldk-integerfield", new JSONObject(){{
                     put("allowBlank", false);
                 }}, null),

From 94d4e91c5c03532a50004931c308155e238aa389 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 23 Mar 2024 19:54:21 -0700
Subject: [PATCH 05/31] Add logging

---
 singlecell/resources/chunks/CalculateGeneComponentScores.R | 1 +
 1 file changed, 1 insertion(+)

diff --git a/singlecell/resources/chunks/CalculateGeneComponentScores.R b/singlecell/resources/chunks/CalculateGeneComponentScores.R
index f49d12043..0b47da4bc 100644
--- a/singlecell/resources/chunks/CalculateGeneComponentScores.R
+++ b/singlecell/resources/chunks/CalculateGeneComponentScores.R
@@ -3,6 +3,7 @@ for (datasetId in names(seuratObjects)) {
     seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
 
     for (sc in savedComponent) {
+        logger::log_info(paste0('Processing ', datasetId, ' for ', sc))
         seuratObj <- RIRA::ScoreUsingSavedComponent(seuratObj, componentOrName = sc, fieldName = sc)
     }
 

From fc22aa53bb5225dc90e9d39e5f435e6ae56cbd3d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 23 Mar 2024 20:10:35 -0700
Subject: [PATCH 06/31] Bump webpack-dev-middleware from 5.3.3 to 5.3.4 in
 /jbrowse (#268)

Bumps [webpack-dev-middleware](https://github.com/webpack/webpack-dev-middleware) from 5.3.3 to 5.3.4.
- [Release notes](https://github.com/webpack/webpack-dev-middleware/releases)
- [Changelog](https://github.com/webpack/webpack-dev-middleware/blob/v5.3.4/CHANGELOG.md)
- [Commits](https://github.com/webpack/webpack-dev-middleware/compare/v5.3.3...v5.3.4)

---
updated-dependencies:
- dependency-name: webpack-dev-middleware
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: bbimber <bbimber@gmail.com>
---
 jbrowse/package-lock.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/jbrowse/package-lock.json b/jbrowse/package-lock.json
index b32c1324c..8c0ece145 100644
--- a/jbrowse/package-lock.json
+++ b/jbrowse/package-lock.json
@@ -18335,9 +18335,9 @@
       }
     },
     "node_modules/webpack-dev-middleware": {
-      "version": "5.3.3",
-      "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.3.tgz",
-      "integrity": "sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA==",
+      "version": "5.3.4",
+      "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz",
+      "integrity": "sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q==",
       "dev": true,
       "dependencies": {
         "colorette": "^2.0.10",

From 058d732f0e2d0e9bde112648bf27608cc7bb617b Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 23 Mar 2024 20:40:22 -0700
Subject: [PATCH 07/31] When subsetting, skip objects with just one cell

---
 singlecell/resources/chunks/SubsetSeurat.R | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/singlecell/resources/chunks/SubsetSeurat.R b/singlecell/resources/chunks/SubsetSeurat.R
index 028bbe4d6..fddd46294 100644
--- a/singlecell/resources/chunks/SubsetSeurat.R
+++ b/singlecell/resources/chunks/SubsetSeurat.R
@@ -12,8 +12,13 @@ for (datasetId in names(seuratObjects)) {
 	<SUBSET_CODE>
 
 	if (!is.null(seuratObj)) {
-		saveData(seuratObj, datasetId)
-		totalPassed <- totalPassed + 1
+		if (ncol(seuratObj) <= 1) {
+			# NOTE: Seurat v5 assays do not perform well with one cell
+			print(paste0('There is only one cell remaining, skipping'))
+		} else {
+			saveData(seuratObj, datasetId)
+			totalPassed <- totalPassed + 1
+		}
 	}
 
 	# Cleanup

From 8902551381243477b31080c16e53e5de93e7770c Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 23 Mar 2024 20:45:36 -0700
Subject: [PATCH 08/31] Skip objects with just one cell

---
 singlecell/resources/chunks/CalculateGeneComponentScores.R | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/singlecell/resources/chunks/CalculateGeneComponentScores.R b/singlecell/resources/chunks/CalculateGeneComponentScores.R
index 0b47da4bc..5471d17f0 100644
--- a/singlecell/resources/chunks/CalculateGeneComponentScores.R
+++ b/singlecell/resources/chunks/CalculateGeneComponentScores.R
@@ -2,6 +2,12 @@ for (datasetId in names(seuratObjects)) {
     printName(datasetId)
     seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
 
+    if (ncol(seuratObj) == 1) {
+        print('Object has a single cell, skipping')
+        rm(seuratObj)
+        next
+    }
+
     for (sc in savedComponent) {
         logger::log_info(paste0('Processing ', datasetId, ' for ', sc))
         seuratObj <- RIRA::ScoreUsingSavedComponent(seuratObj, componentOrName = sc, fieldName = sc)

From 5421f7ec0fb150fa234ecfd1e1cce7e8167f2762 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sun, 24 Mar 2024 03:11:16 -0700
Subject: [PATCH 09/31] Bugfix to vireo

---
 .../org/labkey/singlecell/pipeline/singlecell/VireoHandler.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
index 4c460425f..05ef4d24f 100644
--- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
+++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
@@ -255,7 +255,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
 
             new SimpleScriptWrapper(ctx.getLogger()).execute(vireo);
 
-            File[] outFiles = ctx.getWorkingDirectory().listFiles(f -> f.getName().endsWith("_donor_ids.tsv"));
+            File[] outFiles = ctx.getWorkingDirectory().listFiles(f -> f.getName().endsWith("donor_ids.tsv"));
             if (outFiles == null || outFiles.length == 0)
             {
                 throw new PipelineJobException("Missing vireo output file");

From bf1760adbb5b3e5e06c9b3ded9cadad42dbbda20 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 27 Mar 2024 13:14:56 -0700
Subject: [PATCH 10/31] improve CheckExpectations criteria

---
 .../resources/chunks/CheckExpectations.R      | 26 ++++++++++++++-----
 .../singlecell/CheckExpectations.java         |  3 ++-
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/singlecell/resources/chunks/CheckExpectations.R b/singlecell/resources/chunks/CheckExpectations.R
index e95d91063..e74e0761a 100644
--- a/singlecell/resources/chunks/CheckExpectations.R
+++ b/singlecell/resources/chunks/CheckExpectations.R
@@ -1,3 +1,13 @@
+CheckField <- function(seuratObj, datasetId, fieldName) {
+  if (!fieldName %in% names(seuratObj@meta.data)) {
+    addErrorMessage(paste0(paste0('Missing ', fieldName, ' for dataset: ', datasetId)))
+  }
+
+  if (any(is.na(seuratObj@meta.data[[fieldName]]))) {
+    addErrorMessage(paste0(paste0('NA values found for ', fieldName, ' for dataset: ', datasetId)))
+  }
+}
+
 for (datasetId in names(seuratObjects)) {
   printName(datasetId)
   seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
@@ -28,16 +38,20 @@ for (datasetId in names(seuratObjects)) {
     }
   }
 
-  if (requireSaturation && !'Saturation.RNA' %in% names(seuratObj@meta.data)) {
-    addErrorMessage(paste0('Missing per-cell RNA saturation data for dataset: ', datasetId))
+  if (requireSaturation) {
+    CheckField(seuratObj, datasetId, 'Saturation.RNA')
+  }
+
+  if (requireSingleR) {
+    CheckField(seuratObj, datasetId, 'SingleRConsensus')
   }
 
-  if (requireSingleR && !'SingleRConsensus' %in% names(seuratObj@meta.data)) {
-    addErrorMessage(paste0('Missing SingleRConsensus label for dataset: ', datasetId))
+  if (requireScGate) {
+    CheckField(seuratObj, datasetId, 'scGateConsensus')
   }
 
-  if (requireScGate && !'scGateConsensus' %in% names(seuratObj@meta.data)) {
-    addErrorMessage(paste0('Missing scGateConsensus label for dataset: ', datasetId))
+  if (requireRiraImmune) {
+    CheckField(seuratObj, datasetId, 'RIRA_Immune_v2.cellclass')
   }
 
   if (length(errorMessages) > 0) {
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java
index 9a6327f18..fcffcac5a 100644
--- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java
+++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java
@@ -33,7 +33,8 @@ public Provider()
                     SeuratToolParameter.create("requireCiteSeq", "Require Cite-Seq, If Used", "If this dataset uses CITE-seq, cite-seq data are required", "checkbox", null, true),
                     SeuratToolParameter.create("requireSaturation", "Require Per-Cell Saturation", "If this dataset uses TCR sequencing, these data are required", "checkbox", null, true),
                     SeuratToolParameter.create("requireSingleR", "Require SingleR", "If checked, SingleR calls, including singleRConsensus are required to pass", "checkbox", null, true),
-                    SeuratToolParameter.create("requireScGate", "Require scGate", "If checked, scGateConsensus calls are required to pass", "checkbox", null, true)
+                    SeuratToolParameter.create("requireScGate", "Require scGate", "If checked, scGateConsensus calls are required to pass", "checkbox", null, true),
+                    SeuratToolParameter.create("requireRiraImmune", "Require RIRA Immune V2", "If checked, RIRA_Immune_v2 calls (field RIRA_Immune_v2.cellclass) are required to pass", "checkbox", null, true)
             ), null, null);
         }
 

From bb1221bde772a6c18c45e7c8ed86f09eb4b5275c Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 28 Mar 2024 05:11:01 -0700
Subject: [PATCH 11/31] Ensure SEURAT_MAX_THREADS is an integer

---
 .../resources/queries/sequenceanalysis/outputfiles.js      | 2 +-
 singlecell/resources/chunks/Functions.R                    | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/SequenceAnalysis/resources/queries/sequenceanalysis/outputfiles.js b/SequenceAnalysis/resources/queries/sequenceanalysis/outputfiles.js
index 1b0657f51..ba82cc75b 100644
--- a/SequenceAnalysis/resources/queries/sequenceanalysis/outputfiles.js
+++ b/SequenceAnalysis/resources/queries/sequenceanalysis/outputfiles.js
@@ -19,7 +19,7 @@ function beforeInsert(row, errors){
     beforeUpsert(row, errors);
 }
 
-function beforeUpdate(row, errors){
+function beforeUpdate(row, oldRow, errors){
     beforeUpsert(row, errors);
 }
 
diff --git a/singlecell/resources/chunks/Functions.R b/singlecell/resources/chunks/Functions.R
index 86ee68ed8..bf77508e8 100644
--- a/singlecell/resources/chunks/Functions.R
+++ b/singlecell/resources/chunks/Functions.R
@@ -180,5 +180,10 @@ options('Seurat.memsafe' = TRUE)
 
 if (Sys.getenv('SEURAT_MAX_THREADS') != '') {
     print(paste0('Setting future::plan workers to: ', Sys.getenv('SEURAT_MAX_THREADS')))
-    future::plan(strategy='multisession', workers=Sys.getenv('SEURAT_MAX_THREADS'))
+    mt <- as.integer(Sys.getenv('SEURAT_MAX_THREADS'))
+    if (is.na(mt)) {
+        stop(paste0('SEURAT_MAX_THREAD is not an integer: ', mt <- Sys.getenv('SEURAT_MAX_THREADS')))
+    }
+
+    future::plan(strategy='multisession', workers=mt)
 }

From f2dd3170cebb88b63eb82414475944bb6bb07ab8 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 21 Mar 2024 09:54:52 -0700
Subject: [PATCH 12/31] Expand cellsnp-lite to export cell-level genotypes

---
 .../pipeline/singlecell/VireoHandler.java     | 24 +++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
index 05ef4d24f..4d652ae4c 100644
--- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
+++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
@@ -42,7 +42,7 @@ public VireoHandler()
                 }}, null),
                 ToolParameterDescriptor.create("maxDepth", "Max Depth", "At a position, read maximally INT reads per input file, to avoid excessive memory usage", "ldk-integerfield", new JSONObject(){{
                     put("minValue", 0);
-                }}, 50000),
+                }}, null),
                 ToolParameterDescriptor.create("contigs", "Allowable Contigs", "A comma-separated list of contig names to use", "textfield", new JSONObject(){{
 
                 }}, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20"),
@@ -52,7 +52,8 @@ public VireoHandler()
                     put("category", "VCF File");
                     put("performGenomeFilter", false);
                     put("doNotIncludeInTemplates", true);
-                }}, null)
+                }}, null),
+                ToolParameterDescriptor.create("storeCellSnpVcf", "Store CellSnp-Lite VCF", "If checked, the cellsnp donor calls VCF will be stored as an output file", "checkbox", null, false)
         ));
     }
 
@@ -162,6 +163,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
             cellsnp.add(bam.getPath());
             cellsnp.add("-b");
             cellsnp.add(barcodes.getPath());
+            cellsnp.add("--genotype");
 
             File cellsnpDir = new File(ctx.getWorkingDirectory(), "cellsnp");
             if (cellsnpDir.exists())
@@ -279,6 +281,24 @@ else if (outFiles.length > 1)
             }
             so.setCategory("Vireo Demultiplexing");
             ctx.addSequenceOutput(so);
+
+            if (ctx.getParams().optBoolean("storeCellSnpVcf", false))
+            {
+                so = new SequenceOutputFile();
+                so.setReadset(inputFiles.get(0).getReadset());
+                so.setLibrary_id(inputFiles.get(0).getLibrary_id());
+                so.setFile(outFiles[0]);
+                if (so.getReadset() != null)
+                {
+                    so.setName(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName() + ": Cellsnp-lite VCF");
+                }
+                else
+                {
+                    so.setName(inputFiles.get(0).getName() + ": Cellsnp-lite VCF");
+                }
+                so.setCategory("VCF File");
+                ctx.addSequenceOutput(so);
+            }
         }
     }
 }

From 2325f2b39ffe10ad39fdc4ff0edca320e0564282 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 28 Mar 2024 09:50:58 -0700
Subject: [PATCH 13/31] Ensure cellsnp-lite VCFs are sorted and indexed

---
 .../pipeline/singlecell/VireoHandler.java     | 76 ++++++++++++++-----
 1 file changed, 56 insertions(+), 20 deletions(-)

diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
index 4d652ae4c..a29f22ece 100644
--- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
+++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
@@ -8,6 +8,7 @@
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.pipeline.RecordedAction;
+import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
 import org.labkey.api.sequenceanalysis.SequenceOutputFile;
 import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler;
 import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
@@ -36,7 +37,7 @@ public class VireoHandler  extends AbstractParameterizedOutputHandler<SequenceOu
 
     public VireoHandler()
     {
-        super(ModuleLoader.getInstance().getModule(SingleCellModule.class), "Run Vireo", "This will run cellsnp-lite and vireo to infer cell-to-sample based on genotype.", new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/SequenceOutputFileSelectorField.js")), Arrays.asList(
+        super(ModuleLoader.getInstance().getModule(SingleCellModule.class), "Run CellSnp-Lite/Vireo", "This will run cellsnp-lite and vireo to infer cell-to-sample based on genotype.", new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/SequenceOutputFileSelectorField.js")), Arrays.asList(
                 ToolParameterDescriptor.create("nDonors", "# Donors", "The number of donors to demultiplex", "ldk-integerfield", new JSONObject(){{
                     put("allowBlank", false);
                 }}, null),
@@ -247,6 +248,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
             vireo.add(ctx.getWorkingDirectory().getPath());
 
             int nDonors = ctx.getParams().optInt("nDonors", 0);
+            boolean storeCellSnpVcf = ctx.getParams().optBoolean("storeCellSnpVcf", false);
             if (nDonors == 0)
             {
                 throw new PipelineJobException("Must provide nDonors");
@@ -255,39 +257,73 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
             vireo.add("-N");
             vireo.add(String.valueOf(nDonors));
 
-            new SimpleScriptWrapper(ctx.getLogger()).execute(vireo);
+            if (nDonors == 1)
+            {
+                storeCellSnpVcf = true;
+                ctx.getLogger().info("nDonor was 1, skipping vireo");
+            }
+            else
+            {
+                new SimpleScriptWrapper(ctx.getLogger()).execute(vireo);
+
+                File[] outFiles = ctx.getWorkingDirectory().listFiles(f -> f.getName().endsWith("donor_ids.tsv"));
+                if (outFiles == null || outFiles.length == 0)
+                {
+                    throw new PipelineJobException("Missing vireo output file");
+                }
+                else if (outFiles.length > 1)
+                {
+                    throw new PipelineJobException("More than one possible vireo output file found");
+                }
+
+                SequenceOutputFile so = new SequenceOutputFile();
+                so.setReadset(inputFiles.get(0).getReadset());
+                so.setLibrary_id(inputFiles.get(0).getLibrary_id());
+                so.setFile(outFiles[0]);
+                if (so.getReadset() != null)
+                {
+                    so.setName(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName() + ": Vireo Demultiplexing");
+                }
+                else
+                {
+                    so.setName(inputFiles.get(0).getName() + ": Vireo Demultiplexing");
+                }
+                so.setCategory("Vireo Demultiplexing");
+                ctx.addSequenceOutput(so);
+            }
 
-            File[] outFiles = ctx.getWorkingDirectory().listFiles(f -> f.getName().endsWith("donor_ids.tsv"));
-            if (outFiles == null || outFiles.length == 0)
+            File cellSnpBaseVcf = new File(cellsnpDir, "cellSNP.base.vcf.gz");
+            if (!cellSnpBaseVcf.exists())
             {
-                throw new PipelineJobException("Missing vireo output file");
+                throw new PipelineJobException("Unable to find cellsnp base VCF");
             }
-            else if (outFiles.length > 1)
+
+
+            File cellSnpCellsVcf = new File(cellsnpDir, "cellSNP.cells.vcf.gz");
+            if (!cellSnpCellsVcf.exists())
             {
-                throw new PipelineJobException("More than one possible vireo output file found");
+                throw new PipelineJobException("Unable to find cellsnp calls VCF");
             }
 
-            SequenceOutputFile so = new SequenceOutputFile();
-            so.setReadset(inputFiles.get(0).getReadset());
-            so.setLibrary_id(inputFiles.get(0).getLibrary_id());
-            so.setFile(outFiles[0]);
-            if (so.getReadset() != null)
+            try
             {
-                so.setName(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName() + ": Vireo Demultiplexing");
+                SequencePipelineService.get().sortVcf(cellSnpBaseVcf, null, genome.getSequenceDictionary(), ctx.getLogger());
+                SequenceAnalysisService.get().ensureVcfIndex(cellSnpBaseVcf, ctx.getLogger());
+
+                SequencePipelineService.get().sortVcf(cellSnpCellsVcf, null, genome.getSequenceDictionary(), ctx.getLogger());
+                SequenceAnalysisService.get().ensureVcfIndex(cellSnpCellsVcf, ctx.getLogger());
             }
-            else
+            catch (IOException e)
             {
-                so.setName(inputFiles.get(0).getName() + ": Vireo Demultiplexing");
+                throw new PipelineJobException(e);
             }
-            so.setCategory("Vireo Demultiplexing");
-            ctx.addSequenceOutput(so);
 
-            if (ctx.getParams().optBoolean("storeCellSnpVcf", false))
+            if (storeCellSnpVcf)
             {
-                so = new SequenceOutputFile();
+                SequenceOutputFile so = new SequenceOutputFile();
                 so.setReadset(inputFiles.get(0).getReadset());
                 so.setLibrary_id(inputFiles.get(0).getLibrary_id());
-                so.setFile(outFiles[0]);
+                so.setFile(cellSnpCellsVcf);
                 if (so.getReadset() != null)
                 {
                     so.setName(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName() + ": Cellsnp-lite VCF");

From 68bd133409c6b77cb95b684998c9fb4ed5179f59 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 28 Mar 2024 11:12:26 -0700
Subject: [PATCH 14/31] Bump express from 4.18.2 to 4.19.2 in /jbrowse (#269)

Bumps [express](https://github.com/expressjs/express) from 4.18.2 to 4.19.2.
- [Release notes](https://github.com/expressjs/express/releases)
- [Changelog](https://github.com/expressjs/express/blob/master/History.md)
- [Commits](https://github.com/expressjs/express/compare/4.18.2...4.19.2)

---
updated-dependencies:
- dependency-name: express
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 jbrowse/package-lock.json | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/jbrowse/package-lock.json b/jbrowse/package-lock.json
index 8c0ece145..dc3e491f6 100644
--- a/jbrowse/package-lock.json
+++ b/jbrowse/package-lock.json
@@ -6689,13 +6689,13 @@
       "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ=="
     },
     "node_modules/body-parser": {
-      "version": "1.20.1",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
-      "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
+      "version": "1.20.2",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz",
+      "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==",
       "dev": true,
       "dependencies": {
         "bytes": "3.1.2",
-        "content-type": "~1.0.4",
+        "content-type": "~1.0.5",
         "debug": "2.6.9",
         "depd": "2.0.0",
         "destroy": "1.2.0",
@@ -6703,7 +6703,7 @@
         "iconv-lite": "0.4.24",
         "on-finished": "2.4.1",
         "qs": "6.11.0",
-        "raw-body": "2.5.1",
+        "raw-body": "2.5.2",
         "type-is": "~1.6.18",
         "unpipe": "1.0.0"
       },
@@ -7525,9 +7525,9 @@
       "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A=="
     },
     "node_modules/cookie": {
-      "version": "0.5.0",
-      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
-      "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
+      "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==",
       "dev": true,
       "engines": {
         "node": ">= 0.6"
@@ -8945,17 +8945,17 @@
       }
     },
     "node_modules/express": {
-      "version": "4.18.2",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
-      "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
+      "version": "4.19.2",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz",
+      "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==",
       "dev": true,
       "dependencies": {
         "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
-        "body-parser": "1.20.1",
+        "body-parser": "1.20.2",
         "content-disposition": "0.5.4",
         "content-type": "~1.0.4",
-        "cookie": "0.5.0",
+        "cookie": "0.6.0",
         "cookie-signature": "1.0.6",
         "debug": "2.6.9",
         "depd": "2.0.0",
@@ -15160,9 +15160,9 @@
       }
     },
     "node_modules/raw-body": {
-      "version": "2.5.1",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz",
-      "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
+      "version": "2.5.2",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz",
+      "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==",
       "dev": true,
       "dependencies": {
         "bytes": "3.1.2",

From 0b0f1d0868514709dd749db5ec2974b3b067ed47 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 28 Mar 2024 12:12:58 -0700
Subject: [PATCH 15/31] Ensure cellsnp-lite VCFs are reheadered, sorted and
 indexed

---
 .../pipeline/singlecell/VireoHandler.java     | 76 ++++++++++++++++---
 1 file changed, 64 insertions(+), 12 deletions(-)

diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
index a29f22ece..a0afd3e84 100644
--- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
+++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
@@ -3,6 +3,7 @@
 import htsjdk.samtools.util.IOUtil;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.Logger;
 import org.json.JSONObject;
 import org.labkey.api.module.ModuleLoader;
 import org.labkey.api.pipeline.PipelineJob;
@@ -16,6 +17,7 @@
 import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
 import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor;
+import org.labkey.api.sequenceanalysis.run.AbstractGatk4Wrapper;
 import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
 import org.labkey.api.util.PageFlowUtil;
 import org.labkey.api.writer.PrintWriters;
@@ -305,18 +307,8 @@ else if (outFiles.length > 1)
                 throw new PipelineJobException("Unable to find cellsnp calls VCF");
             }
 
-            try
-            {
-                SequencePipelineService.get().sortVcf(cellSnpBaseVcf, null, genome.getSequenceDictionary(), ctx.getLogger());
-                SequenceAnalysisService.get().ensureVcfIndex(cellSnpBaseVcf, ctx.getLogger());
-
-                SequencePipelineService.get().sortVcf(cellSnpCellsVcf, null, genome.getSequenceDictionary(), ctx.getLogger());
-                SequenceAnalysisService.get().ensureVcfIndex(cellSnpCellsVcf, ctx.getLogger());
-            }
-            catch (IOException e)
-            {
-                throw new PipelineJobException(e);
-            }
+            sortAndFixVcf(cellSnpBaseVcf, genome, ctx.getLogger());
+            sortAndFixVcf(cellSnpCellsVcf, genome, ctx.getLogger());
 
             if (storeCellSnpVcf)
             {
@@ -336,5 +328,65 @@ else if (outFiles.length > 1)
                 ctx.addSequenceOutput(so);
             }
         }
+
+        private void sortAndFixVcf(File vcf, ReferenceGenome genome, Logger log) throws PipelineJobException
+        {
+            // NOTE: this is required since cellsnp-lite creates a non-compliant header dictionary
+            new UpdateVCFSequenceDictionary(log).execute(vcf, genome.getSequenceDictionary());
+
+            try
+            {
+                SequencePipelineService.get().sortVcf(vcf, null, genome.getSequenceDictionary(), log);
+                SequenceAnalysisService.get().ensureVcfIndex(vcf, log);
+            }
+            catch (IOException e)
+            {
+                throw new PipelineJobException(e);
+            }
+        }
+
+        public static class UpdateVCFSequenceDictionary extends AbstractGatk4Wrapper
+        {
+            public UpdateVCFSequenceDictionary(Logger log)
+            {
+                super(log);
+            }
+
+            public void execute(File vcf, File dict) throws PipelineJobException
+            {
+                List<String> args = new ArrayList<>(getBaseArgs("UpdateVCFSequenceDictionary"));
+                args.add("-V");
+                args.add(vcf.getPath());
+
+                args.add("--source-dictionary");
+                args.add(dict.getPath());
+
+                args.add("--replace");
+                args.add("true");
+
+                File output = new File(vcf.getParentFile(), "tmp.vcf.gz");
+                args.add("-O");
+                args.add(output.getPath());
+
+                execute(args);
+
+                if (!output.exists())
+                {
+                    throw new PipelineJobException("Unable to find file: " + output.getPath());
+                }
+
+                vcf.delete();
+
+                try
+                {
+                    SequenceAnalysisService.get().ensureVcfIndex(vcf, getLogger(), true);
+                }
+                catch (IOException e)
+                {
+                    throw new PipelineJobException(e);
+                }
+
+            }
+        }
     }
 }

From a20819325a9192d8bfddff40f13ab26b2eab28ca Mon Sep 17 00:00:00 2001
From: hextraza <hex@hextraza.moe>
Date: Thu, 28 Mar 2024 12:14:23 -0700
Subject: [PATCH 16/31] Custom server-side sorting (#264)

* Support custom sorting fields for JBrowse

---------

Co-authored-by: Sebastian Benjamin <sebastiancbenjamin@gmail.com>
Co-authored-by: bbimber <bbimber@gmail.com>
---
 .../components/VariantTableWidget.tsx         | 53 +++++++++--
 jbrowse/src/client/JBrowse/utils.ts           | 19 +++-
 .../org/labkey/jbrowse/JBrowseController.java | 14 ++-
 .../labkey/jbrowse/JBrowseLuceneSearch.java   | 44 ++++++----
 .../external/labModules/JBrowseTest.java      | 87 ++++++++++++++++++-
 5 files changed, 190 insertions(+), 27 deletions(-)

diff --git a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx
index afe99496c..ac75ef4de 100644
--- a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx
+++ b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx
@@ -5,13 +5,16 @@ import {
     GridColumnVisibilityModel,
     GridPaginationModel,
     GridRenderCellParams,
+    GridSortDirection,
+    GridSortModel,
     GridToolbarColumnsButton,
     GridToolbarContainer,
     GridToolbarDensitySelector,
     GridToolbarExport
 } from '@mui/x-data-grid';
 import SearchIcon from '@mui/icons-material/Search';
-import React, { useEffect, useState, useMemo } from 'react';
+import LinkIcon from '@mui/icons-material/Link';
+import React, { useEffect, useState } from 'react';
 import { getConf } from '@jbrowse/core/configuration';
 import { AppBar, Box, Button, Dialog, Paper, Popover, Toolbar, Tooltip, Typography } from '@mui/material';
 import { FilterFormModal } from './FilterFormModal';
@@ -67,14 +70,17 @@ const VariantTableWidget = observer(props => {
         session.hideWidget(widget)
     }
 
-    function handleQuery(passedFilters, pushToHistory, pageQueryModel = pageSizeModel) {
+    function handleQuery(passedFilters, pushToHistory, pageQueryModel = pageSizeModel, sortQueryModel = sortModel) {
         const { page = pageSizeModel.page, pageSize = pageSizeModel.pageSize } = pageQueryModel;
+        const { field = "genomicPosition", sort = false } = sortQueryModel[0] ?? {};
 
         const encodedSearchString = createEncodedFilterString(passedFilters, false);
         const currentUrl = new URL(window.location.href);
         currentUrl.searchParams.set("searchString", encodedSearchString);
         currentUrl.searchParams.set("page", page.toString());
         currentUrl.searchParams.set("pageSize", pageSize.toString());
+        currentUrl.searchParams.set("sortField", field.toString());
+        currentUrl.searchParams.set("sortDirection", sort.toString());
 
         if (pushToHistory) {
           window.history.pushState(null, "", currentUrl.toString());
@@ -82,7 +88,7 @@ const VariantTableWidget = observer(props => {
 
         setFilters(passedFilters);
         setDataLoaded(false)
-        fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)});
+        fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, field, sort, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)});
     }
 
     const TableCellWithPopover = (props: { value: any }) => {
@@ -190,7 +196,29 @@ const VariantTableWidget = observer(props => {
                     Search
                 </Button>
                 <GridToolbarDensitySelector />
-                <GridToolbarExport />
+                <GridToolbarExport csvOptions={{
+                    delimiter: ';',
+                }} />
+
+                <Button
+                    startIcon={<LinkIcon />}
+                    size="small"
+                    color="primary"
+                    onClick={() => {
+                        navigator.clipboard.writeText(window.location.href)
+                        .then(() => {
+                            // Popup message for successful copy
+                            alert('URL copied to clipboard.');
+                        })
+                        .catch(err => {
+                            // Error handling
+                            console.error('Failed to copy the URL: ', err);
+                            alert('Failed to copy the URL.');
+                        });
+                    }}
+                >
+                Share
+                </Button>
             </GridToolbarContainer>
         );
     }
@@ -234,11 +262,15 @@ const VariantTableWidget = observer(props => {
     // False until initial data load or an error:
     const [dataLoaded, setDataLoaded] = useState(false)
 
-    const urlParams = new URLSearchParams(window.location.search);
-    const page = parseInt(urlParams.get('page') || '0');
-    const pageSize = parseInt(urlParams.get('pageSize') || '50');
+    const urlParams = new URLSearchParams(window.location.search)
+    const page = parseInt(urlParams.get('page') || '0')
+    const pageSize = parseInt(urlParams.get('pageSize') || '50')
     const [pageSizeModel, setPageSizeModel] = React.useState<GridPaginationModel>({ page, pageSize });
 
+    const sortField = urlParams.get('sortField') || 'genomicPosition'
+    const sortDirection = urlParams.get('sortDirection') || 'desc'
+    const [sortModel, setSortModel] = React.useState<GridSortModel>([{ field: sortField, sort: sortDirection as GridSortDirection }])
+
     const colVisURLComponent = urlParams.get("colVisModel") || "{}"
     const colVisModel = JSON.parse(decodeURIComponent(colVisURLComponent))
     const [columnVisibilityModel, setColumnVisibilityModel] = useState<GridColumnVisibilityModel>(colVisModel);
@@ -419,6 +451,11 @@ const VariantTableWidget = observer(props => {
                 currentUrl.searchParams.set("colVisModel", encodeURIComponent(JSON.stringify(trueValuesModel)));
                 window.history.pushState(null, "", currentUrl.toString());
             }}
+            sortingMode="server"
+            onSortModelChange={(newModel) => {
+                setSortModel(newModel)
+                handleQuery(filters, true, { page: 0, pageSize: pageSizeModel.pageSize }, newModel);
+            }}
         />
     )
 
@@ -440,7 +477,7 @@ const VariantTableWidget = observer(props => {
                 fieldTypeInfo: fieldTypeInfo,
                 allowedGroupNames: allowedGroupNames,
                 promotedFilters: promotedFilters,
-                handleQuery: (filters) => handleQuery(filters, true)
+                handleQuery: (filters) => handleQuery(filters, true, { page: 0, pageSize: pageSizeModel.pageSize}, sortModel)
             }}
         />
     );
diff --git a/jbrowse/src/client/JBrowse/utils.ts b/jbrowse/src/client/JBrowse/utils.ts
index 1401ef791..32d514cab 100644
--- a/jbrowse/src/client/JBrowse/utils.ts
+++ b/jbrowse/src/client/JBrowse/utils.ts
@@ -419,7 +419,7 @@ function generateLuceneString(field, operator, value) {
   return luceneQueryString;
 }
 
-export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, successCallback, failureCallback) {
+export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, sortField, sortReverseString, successCallback, failureCallback) {
     if (!offset) {
         offset = 0
     }
@@ -439,6 +439,13 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa
         return
     }
 
+    let sortReverse;
+    if(sortReverseString == "asc") {
+        sortReverse = true
+    } else {
+        sortReverse = false
+    }
+
     return Ajax.request({
         url: ActionURL.buildURL('jbrowse', 'luceneQuery.api'),
         method: 'GET',
@@ -449,7 +456,15 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa
         failure: function(res) {
             failureCallback("There was an error: " + res.status + "\n Status Body: " + res.responseText + "\n Session ID:" + sessionId)
         },
-        params: {"searchString": createEncodedFilterString(filters, true), "sessionId": sessionId, "trackId": trackGUID, "offset": offset, "pageSize": pageSize},
+        params: {
+            "searchString": createEncodedFilterString(filters, true),
+            "sessionId": sessionId,
+            "trackId": trackGUID,
+            "offset": offset,
+            "pageSize": pageSize,
+            "sortField": sortField ?? "genomicPosition",
+            "sortReverse": sortReverse 
+        },
     });
 }
 
diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java
index ada6cbbd5..11a01c283 100644
--- a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java
+++ b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java
@@ -910,7 +910,7 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors)
 
             try
             {
-                return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset()));
+                return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset(), form.getSortField(), form.getSortReverse()));
             }
             catch (Exception e)
             {
@@ -947,6 +947,10 @@ public static class LuceneQueryForm
 
         private int _offset = 0;
 
+        private String _sortField = "genomicPosition";
+
+        private boolean _sortReverse = false;
+
         public String getSearchString()
         {
             return _searchString;
@@ -987,6 +991,14 @@ public void setOffset(int offset)
             _offset = offset;
         }
 
+        public String getSortField() { return _sortField; }
+
+        public void setSortField(String sortField) { _sortField = sortField; }
+
+        public boolean getSortReverse() { return _sortReverse; }
+
+        public void setSortReverse(boolean sortReverse) { _sortReverse = sortReverse; }
+
         public String getTrackId()
         {
             return _trackId;
diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java
index c5e1889a9..14d0aa031 100644
--- a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java
+++ b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.search.TopFieldDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.NumericUtils;
 import org.jetbrains.annotations.Nullable;
 import org.json.JSONObject;
 import org.labkey.api.data.Container;
@@ -50,6 +51,7 @@
 import java.util.StringTokenizer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static org.labkey.jbrowse.JBrowseFieldUtils.VARIABLE_SAMPLES;
 import static org.labkey.jbrowse.JBrowseFieldUtils.getSession;
@@ -61,6 +63,8 @@ public class JBrowseLuceneSearch
     private final JsonFile _jsonFile;
     private final User _user;
     private final String[] specialStartPatterns = {"*:* -", "+", "-"};
+    private static final String ALL_DOCS = "all";
+    private static final String GENOMIC_POSITION = "genomicPosition";
 
     private JBrowseLuceneSearch(final JBrowseSession session, final JsonFile jsonFile, User u)
     {
@@ -130,7 +134,7 @@ public String extractFieldName(String queryString) {
         return parts.length > 0 ? parts[0].trim() : null;
     }
 
-    public JSONObject doSearch(User u, String searchString, final int pageSize, final int offset) throws IOException, ParseException
+    public JSONObject doSearch(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException
     {
         searchString = tryUrlDecode(searchString);
         File indexPath = _jsonFile.getExpectedLocationOfLuceneIndex(true);
@@ -146,7 +150,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
             IndexSearcher indexSearcher = new IndexSearcher(indexReader);
 
             List<String> stringQueryParserFields = new ArrayList<>();
-            List<String> numericQueryParserFields = new ArrayList<>();
+            Map<String, SortField.Type> numericQueryParserFields = new HashMap<>();
             PointsConfig intPointsConfig = new PointsConfig(new DecimalFormat(), Integer.class);
             PointsConfig doublePointsConfig = new PointsConfig(new DecimalFormat(), Double.class);
             Map<String, PointsConfig> pointsConfigMap = new HashMap<>();
@@ -161,11 +165,11 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
                 {
                     case Flag, String, Character -> stringQueryParserFields.add(field);
                     case Float -> {
-                        numericQueryParserFields.add(field);
+                        numericQueryParserFields.put(field, SortField.Type.DOUBLE);
                         pointsConfigMap.put(field, doublePointsConfig);
                     }
                     case Integer -> {
-                        numericQueryParserFields.add(field);
+                        numericQueryParserFields.put(field, SortField.Type.INT);
                         pointsConfigMap.put(field, intPointsConfig);
                     }
                 }
@@ -182,14 +186,14 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
 
             BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder();
 
-            if (searchString.equals("all")) {
+            if (searchString.equals(ALL_DOCS)) {
                 booleanQueryBuilder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
             }
 
             // Split input into tokens, 1 token per query separated by &
             StringTokenizer tokenizer = new StringTokenizer(searchString, "&");
 
-            while (tokenizer.hasMoreTokens() && !searchString.equals("all"))
+            while (tokenizer.hasMoreTokens() && !searchString.equals(ALL_DOCS))
             {
                 String queryString = tokenizer.nextToken();
                 Query query = null;
@@ -205,7 +209,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina
                 {
                     query = queryParser.parse(queryString);
                 }
-                else if (numericQueryParserFields.contains(fieldName))
+                else if (numericQueryParserFields.containsKey(fieldName))
                 {
                     try
                     {
@@ -226,16 +230,28 @@ else if (numericQueryParserFields.contains(fieldName))
 
             BooleanQuery query = booleanQueryBuilder.build();
 
+            // By default, sort in INDEXORDER, which is by genomicPosition
+            Sort sort = Sort.INDEXORDER;
+
+            // If the sort field is not genomicPosition, use the provided sorting data
+            if (!sortField.equals(GENOMIC_POSITION)) {
+                SortField.Type fieldType;
+
+                if (stringQueryParserFields.contains(sortField)) {
+                    fieldType = SortField.Type.STRING;
+                } else if (numericQueryParserFields.containsKey(sortField)) {
+                    fieldType = numericQueryParserFields.get(sortField);
+                } else {
+                    throw new IllegalArgumentException("Could not find type for sort field: " + sortField);
+                }
+
+                sort = new Sort(new SortField(sortField, fieldType, sortReverse));
+            }
+
             // Get chunks of size {pageSize}. Default to 1 chunk -- add to the offset to get more.
             // We then iterate over the range of documents we want based on the offset. This does grow in memory
             // linearly with the number of documents, but my understanding is that these are just score,id pairs
             // rather than full documents, so mem usage *should* still be pretty low.
-            //TopDocs topDocs = indexSearcher.search(query, pageSize * (offset + 1));
-
-            // Define sort field
-            SortField sortField = new SortField("pos", SortField.Type.INT, false);
-            Sort sort = new Sort(sortField);
-
             // Perform the search with sorting
             TopFieldDocs topDocs = indexSearcher.search(query, pageSize * (offset + 1), sort);
 
@@ -253,10 +269,8 @@ else if (numericQueryParserFields.contains(fieldName))
                     String fieldName = field.name();
                     String[] fieldValues = doc.getValues(fieldName);
                     if (fieldValues.length > 1) {
-                        // If there is more than one value, put the array of values into the JSON object.
                         elem.put(fieldName, fieldValues);
                     } else {
-                        // If there is only one value, just put this single value into the JSON object.
                         elem.put(fieldName, fieldValues[0]);
                     }
                 }
diff --git a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java
index 57006b138..ddbc580ac 100644
--- a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java
+++ b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java
@@ -569,6 +569,7 @@ private void testFullTextSearch() throws Exception
         String sessionId = info.getKey();
         String trackId = info.getValue();
 
+
         // all
         // this should return 143 results. We can't make any other assumptions about the content
         String url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=143";
@@ -580,7 +581,6 @@ private void testFullTextSearch() throws Exception
         JSONArray jsonArray = mainJsonObject.getJSONArray("data");
         Assert.assertEquals(143, jsonArray.length());
 
-
         // stringType:
         // ref equals A
         url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=ref%3AA";
@@ -1150,6 +1150,91 @@ private void testFullTextSearch() throws Exception
             Assert.assertEquals("A", jsonObject.getString("ref"));
         }
 
+        // Default genomic position sort (ascending)
+        url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100";
+        beginAt(url);
+        waitForText("data");
+        waitAndClick(Locator.tagWithId("a", "rawdata-tab"));
+        jsonString = getText(Locator.tagWithClass("pre", "data"));
+        mainJsonObject = new JSONObject(jsonString);
+        jsonArray = mainJsonObject.getJSONArray("data");
+
+        long previousGenomicPosition = Long.MIN_VALUE;
+        for (int i = 0; i < jsonArray.length(); i++) {
+            JSONObject jsonObject = jsonArray.getJSONObject(i);
+            long currentGenomicPosition = jsonObject.getLong("genomicPosition");
+            Assert.assertTrue(currentGenomicPosition >= previousGenomicPosition);
+            previousGenomicPosition = currentGenomicPosition;
+        }
+
+        // Sort by alt, ascending
+        url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=alt";
+        beginAt(url);
+        waitForText("data");
+        waitAndClick(Locator.tagWithId("a", "rawdata-tab"));
+        jsonString = getText(Locator.tagWithClass("pre", "data"));
+        mainJsonObject = new JSONObject(jsonString);
+        jsonArray = mainJsonObject.getJSONArray("data");
+
+        String previousAlt = "";
+        for (int i = 0; i < jsonArray.length(); i++) {
+            JSONObject jsonObject = jsonArray.getJSONObject(i);
+            String currentAlt = jsonObject.getString("alt");
+            Assert.assertTrue(currentAlt.compareTo(previousAlt) >= 0);
+            previousAlt = currentAlt;
+        }
+
+        // Sort by alt, descending
+        url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=alt&sortReverse=true";
+        beginAt(url);
+        waitForText("data");
+        waitAndClick(Locator.tagWithId("a", "rawdata-tab"));
+        jsonString = getText(Locator.tagWithClass("pre", "data"));
+        mainJsonObject = new JSONObject(jsonString);
+        jsonArray = mainJsonObject.getJSONArray("data");
+
+        previousAlt = "ZZZZ"; // Assuming 'Z' is higher than any character in your data
+        for (int i = 0; i < jsonArray.length(); i++) {
+            JSONObject jsonObject = jsonArray.getJSONObject(i);
+            String currentAlt = jsonObject.getString("alt");
+            Assert.assertTrue(currentAlt.compareTo(previousAlt) <= 0);
+            previousAlt = currentAlt;
+        }
+
+        // Sort by af, ascending
+        url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=AF";
+        beginAt(url);
+        waitForText("data");
+        waitAndClick(Locator.tagWithId("a", "rawdata-tab"));
+        jsonString = getText(Locator.tagWithClass("pre", "data"));
+        mainJsonObject = new JSONObject(jsonString);
+        jsonArray = mainJsonObject.getJSONArray("data");
+
+        double previousAf = -1.0;
+        for (int i = 0; i < jsonArray.length(); i++) {
+            JSONObject jsonObject = jsonArray.getJSONObject(i);
+            double currentAf = jsonObject.getDouble("AF");
+            Assert.assertTrue(currentAf >= previousAf);
+            previousAf = currentAf;
+        }
+
+        // Sort by af, descending
+        url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=AF&sortReverse=true";
+        beginAt(url);
+        waitForText("data");
+        waitAndClick(Locator.tagWithId("a", "rawdata-tab"));
+        jsonString = getText(Locator.tagWithClass("pre", "data"));
+        mainJsonObject = new JSONObject(jsonString);
+        jsonArray = mainJsonObject.getJSONArray("data");
+
+        previousAf = 2.0; // Assuming 'af' is <= 1.0
+        for (int i = 0; i < jsonArray.length(); i++) {
+            JSONObject jsonObject = jsonArray.getJSONObject(i);
+            double currentAf = jsonObject.getDouble("AF");
+            Assert.assertTrue(currentAf <= previousAf);
+            previousAf = currentAf;
+        }
+
         testLuceneSearchUI(sessionId);
     }
 

From cb4ed8158ac54a2e72b0d75980c5907f6941b92d Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 28 Mar 2024 14:04:41 -0700
Subject: [PATCH 17/31] Add option to keep VCF calls for DeepVariant

---
 .../analysis/DeepVariantHandler.java          | 17 +++++++++++-
 .../run/analysis/DeepVariantAnalysis.java     | 27 +++++++++++++++----
 2 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/DeepVariantHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/DeepVariantHandler.java
index 56d18fb7f..249e53344 100644
--- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/DeepVariantHandler.java
+++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/DeepVariantHandler.java
@@ -97,6 +97,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
             action.addInput(so.getFile(), "Input BAM File");
 
             File outputFile = new File(ctx.getOutputDir(), FileUtil.getBaseName(so.getFile()) + ".g.vcf.gz");
+            File outputFileVcf = new File(ctx.getOutputDir(), FileUtil.getBaseName(so.getFile()) + ".vcf.gz");
 
             DeepVariantAnalysis.DeepVariantWrapper wrapper = new DeepVariantAnalysis.DeepVariantWrapper(job.getLogger());
             wrapper.setOutputDir(ctx.getOutputDir());
@@ -123,7 +124,8 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
                 throw new PipelineJobException("Missing binVersion");
             }
 
-            wrapper.execute(so.getFile(), referenceGenome.getWorkingFastaFile(), outputFile, ctx.getFileManager(), binVersion, args);
+            boolean retainVcf = ctx.getParams().optBoolean("retainVcf", false);
+            wrapper.execute(so.getFile(), referenceGenome.getWorkingFastaFile(), outputFile, retainVcf, ctx.getFileManager(), binVersion, args);
 
             action.addOutput(outputFile, "gVCF File", false);
 
@@ -137,6 +139,19 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
 
             ctx.addSequenceOutput(o);
 
+            if (retainVcf)
+            {
+                SequenceOutputFile vcf = new SequenceOutputFile();
+                vcf.setName(outputFileVcf.getName());
+                vcf.setFile(outputFileVcf);
+                vcf.setLibrary_id(so.getLibrary_id());
+                vcf.setCategory("DeepVariant VCF File");
+                vcf.setReadset(so.getReadset());
+                vcf.setDescription("DeepVariant Version: " + binVersion);
+
+                ctx.addSequenceOutput(vcf);
+            }
+
             ctx.addActions(action);
         }
 
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/DeepVariantAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/DeepVariantAnalysis.java
index 51c6bf296..b49680702 100644
--- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/DeepVariantAnalysis.java
+++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/DeepVariantAnalysis.java
@@ -70,7 +70,8 @@ public static List<ToolParameterDescriptor> getToolDescriptors()
                 }}, "X,Y"),
                 ToolParameterDescriptor.create("binVersion", "DeepVariant Version", "The version of DeepVariant to run, which is passed to their docker container", "textfield", new JSONObject(){{
                     put("allowBlank", false);
-                }}, "1.6.0")
+                }}, "1.6.0"),
+                ToolParameterDescriptor.create("retainVcf", "Retain VCF", "If selected, the VCF with called genotypes will be retained", "checkbox", null, false)
         );
     }
 
@@ -153,9 +154,11 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc
             throw new PipelineJobException("Missing binVersion");
         }
 
+        boolean retainVcf = getProvider().getParameterByName("retainVcf").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false);
+
         getWrapper().setOutputDir(outputDir);
         getWrapper().setWorkingDir(outputDir);
-        getWrapper().execute(inputBam, referenceGenome.getWorkingFastaFile(), outputFile, output, binVersion, args);
+        getWrapper().execute(inputBam, referenceGenome.getWorkingFastaFile(), outputFile, retainVcf, output, binVersion, args);
 
         output.addOutput(outputFile, "gVCF File");
         output.addSequenceOutput(outputFile, outputFile.getName(), "DeepVariant gVCF File", rs.getReadsetId(), null, referenceGenome.getGenomeId(), "DeepVariant Version: " + binVersion);
@@ -164,6 +167,17 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc
             output.addOutput(idxFile, "VCF Index");
         }
 
+        if (retainVcf)
+        {
+            File outputFileVcf = new File(outputDir, FileUtil.getBaseName(inputBam) + ".vcf.gz");
+            if (!outputFileVcf.exists())
+            {
+                throw new PipelineJobException("Missing expected file: " + outputFileVcf.getPath());
+            }
+
+            output.addSequenceOutput(outputFile, outputFileVcf.getName(), "DeepVariant VCF File", rs.getReadsetId(), null, referenceGenome.getGenomeId(), "DeepVariant Version: " + binVersion);
+        }
+
         return output;
     }
 
@@ -206,12 +220,15 @@ private File ensureLocalCopy(File input, File workingDirectory, PipelineOutputTr
             }
         }
 
-        public void execute(File inputBam, File refFasta, File outputGvcf, PipelineOutputTracker tracker, String binVersion, List<String> extraArgs) throws PipelineJobException
+        public void execute(File inputBam, File refFasta, File outputGvcf, boolean retainVcf, PipelineOutputTracker tracker, String binVersion, List<String> extraArgs) throws PipelineJobException
         {
             File workDir = outputGvcf.getParentFile();
             File outputVcf = new File(outputGvcf.getPath().replaceAll(".g.vcf", ".vcf"));
-            tracker.addIntermediateFile(outputVcf);
-            tracker.addIntermediateFile(new File(outputVcf.getPath() + ".tbi"));
+            if (!retainVcf)
+            {
+                tracker.addIntermediateFile(outputVcf);
+                tracker.addIntermediateFile(new File(outputVcf.getPath() + ".tbi"));
+            }
 
             File inputBamLocal = ensureLocalCopy(inputBam, workDir, tracker);
             ensureLocalCopy(SequenceUtil.getExpectedIndex(inputBam), workDir, tracker);

From 2c5ee66008f53572f206e16cebca68e145035a0f Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 28 Mar 2024 14:34:38 -0700
Subject: [PATCH 18/31] Adjust location of VCF index tasks to better allow
 resume

---
 .../pipeline/ProcessVariantsHandler.java      | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java
index 32b082fda..ff6d4bdbb 100644
--- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java
+++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java
@@ -369,15 +369,6 @@ public static List<Interval> getIntervals(JobContext ctx)
 
     public static File processVCF(File input, Integer libraryId, JobContext ctx, Resumer resumer, boolean subsetToIntervals) throws PipelineJobException
     {
-        try
-        {
-            SequenceAnalysisService.get().ensureVcfIndex(input, ctx.getLogger());
-        }
-        catch (IOException e)
-        {
-            throw new PipelineJobException(e);
-        }
-
         File currentVCF = input;
 
         ctx.getJob().getLogger().info("***Starting processing of file: " + input.getName());
@@ -409,6 +400,15 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res
             }
             else
             {
+                try
+                {
+                    SequenceAnalysisService.get().ensureVcfIndex(input, ctx.getLogger());
+                }
+                catch (IOException e)
+                {
+                    throw new PipelineJobException(e);
+                }
+
                 OutputVariantsStartingInIntervalsStep.Wrapper wrapper = new OutputVariantsStartingInIntervalsStep.Wrapper(ctx.getLogger());
                 wrapper.execute(input, outputFile, getIntervals(ctx));
             }
@@ -432,6 +432,15 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res
                 continue;
             }
 
+            try
+            {
+                SequenceAnalysisService.get().ensureVcfIndex(input, ctx.getLogger());
+            }
+            catch (IOException e)
+            {
+                throw new PipelineJobException(e);
+            }
+
             RecordedAction action = new RecordedAction(stepCtx.getProvider().getLabel());
             Date start = new Date();
             action.setStartTime(start);

From d340e39d36be9a0c605a4bf8556dc06f793ab3c0 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 29 Mar 2024 07:08:36 -0700
Subject: [PATCH 19/31] Additional validation over cellsnp-lite VCFs

---
 .../singlecell/pipeline/singlecell/VireoHandler.java     | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
index a0afd3e84..e2a051661 100644
--- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
+++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
@@ -331,12 +331,13 @@ else if (outFiles.length > 1)
 
         private void sortAndFixVcf(File vcf, ReferenceGenome genome, Logger log) throws PipelineJobException
         {
-            // NOTE: this is required since cellsnp-lite creates a non-compliant header dictionary
-            new UpdateVCFSequenceDictionary(log).execute(vcf, genome.getSequenceDictionary());
-
+            // This original VCF is generally not properly sorted, and has an invalid index. This is redundant, the VCF is not that large:
             try
             {
-                SequencePipelineService.get().sortVcf(vcf, null, genome.getSequenceDictionary(), log);
+                SequencePipelineService.get().sortROD(vcf, log, 2);
+                SequenceAnalysisService.get().ensureVcfIndex(vcf, log, true);
+
+                new UpdateVCFSequenceDictionary(log).execute(vcf, genome.getSequenceDictionary());
                 SequenceAnalysisService.get().ensureVcfIndex(vcf, log);
             }
             catch (IOException e)

From c46b9a35d600e58a2bc2242ac4e07055901e8e0f Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 29 Mar 2024 07:18:28 -0700
Subject: [PATCH 20/31] Improve parallelization for seurat steps

---
 singlecell/resources/chunks/Functions.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singlecell/resources/chunks/Functions.R b/singlecell/resources/chunks/Functions.R
index bf77508e8..246bd9359 100644
--- a/singlecell/resources/chunks/Functions.R
+++ b/singlecell/resources/chunks/Functions.R
@@ -176,7 +176,7 @@ readSeuratRDS <- function(filePath) {
 print('Updating future.globals.maxSize')
 options(future.globals.maxSize = Inf)
 
-options('Seurat.memsafe' = TRUE)
+options('Seurat.memsafe' = TRUE, future.globals.onReference = "error")
 
 if (Sys.getenv('SEURAT_MAX_THREADS') != '') {
     print(paste0('Setting future::plan workers to: ', Sys.getenv('SEURAT_MAX_THREADS')))

From 9bc6d4d44aa401fac3142c42a17868fab9fb36ec Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 29 Mar 2024 07:34:58 -0700
Subject: [PATCH 21/31] Improve validation for seurat prototypes

---
 singlecell/resources/chunks/SeuratPrototype.R | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/singlecell/resources/chunks/SeuratPrototype.R b/singlecell/resources/chunks/SeuratPrototype.R
index f03ab5403..19e783387 100644
--- a/singlecell/resources/chunks/SeuratPrototype.R
+++ b/singlecell/resources/chunks/SeuratPrototype.R
@@ -1,3 +1,13 @@
+CheckField <- function(seuratObj, datasetId, fieldName) {
+  if (!fieldName %in% names(seuratObj@meta.data)) {
+    addErrorMessage(paste0(paste0('Missing ', fieldName, ' for dataset: ', datasetId)))
+  }
+
+  if (any(is.na(seuratObj@meta.data[[fieldName]]))) {
+    addErrorMessage(paste0(paste0('NA values found for ', fieldName, ' for dataset: ', datasetId)))
+  }
+}
+
 metricData <- data.frame(dataId = integer(), readsetId = integer(), metricname = character(), metricvalue = numeric())
 
 for (datasetId in names(seuratObjects)) {
@@ -72,12 +82,12 @@ for (datasetId in names(seuratObjects)) {
     metricData <- rbind(metricData, data.frame(dataId = datasetId, readsetId = datasetIdToReadset[[datasetId]], metricname = 'MeanSaturation.RNA', metricvalue = meanSaturation.RNA))
   }
 
-  if (requireSingleR && !'SingleRConsensus' %in% names(seuratObj@meta.data)) {
-    addErrorMessage(paste0('Missing SingleRConsensus label for dataset: ', datasetId))
+  if (requireSingleR) {
+    CheckField(seuratObj, datasetId, 'SingleRConsensus')
   }
 
-  if (requireScGate && !'scGateConsensus' %in% names(seuratObj@meta.data)) {
-    addErrorMessage(paste0('Missing scGateConsensus label for dataset: ', datasetId))
+  if (requireScGate) {
+    CheckField(seuratObj, datasetId, 'scGateConsensus')
   }
 
   if (length(errorMessages) > 0) {

From 4494130e60c9fa4eef8a26e2ad401ec718c2312d Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Fri, 29 Mar 2024 08:08:59 -0700
Subject: [PATCH 22/31] Bugfix to UpdateVCFSequenceDictionary

---
 .../pipeline/singlecell/VireoHandler.java         | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
index e2a051661..c3e432cf8 100644
--- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
+++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java
@@ -376,11 +376,20 @@ public void execute(File vcf, File dict) throws PipelineJobException
                     throw new PipelineJobException("Unable to find file: " + output.getPath());
                 }
 
-                vcf.delete();
-
                 try
                 {
-                    SequenceAnalysisService.get().ensureVcfIndex(vcf, getLogger(), true);
+                    // replace original:
+                    vcf.delete();
+                    FileUtils.moveFile(output, vcf);
+
+                    File outputIdx = new File(output.getPath() + ".tbi");
+                    File vcfIdx = new File(vcf.getPath() + ".tbi");
+                    if (vcfIdx.exists())
+                    {
+                        vcfIdx.delete();
+                    }
+
+                    FileUtils.moveFile(outputIdx, vcfIdx);
                 }
                 catch (IOException e)
                 {

From 470e249fcb0e93d6f88c60a1505e61572bd0f51a Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 30 Mar 2024 14:51:05 -0700
Subject: [PATCH 23/31] Switch future.globals.onReference to warning

---
 singlecell/resources/chunks/Functions.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singlecell/resources/chunks/Functions.R b/singlecell/resources/chunks/Functions.R
index 246bd9359..a5cea2da4 100644
--- a/singlecell/resources/chunks/Functions.R
+++ b/singlecell/resources/chunks/Functions.R
@@ -176,7 +176,7 @@ readSeuratRDS <- function(filePath) {
 print('Updating future.globals.maxSize')
 options(future.globals.maxSize = Inf)
 
-options('Seurat.memsafe' = TRUE, future.globals.onReference = "error")
+options('Seurat.memsafe' = TRUE, future.globals.onReference = "warning")
 
 if (Sys.getenv('SEURAT_MAX_THREADS') != '') {
     print(paste0('Setting future::plan workers to: ', Sys.getenv('SEURAT_MAX_THREADS')))

From 48c4caff128107613e9d9020dc9be4a614d43428 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 30 Mar 2024 19:34:52 -0700
Subject: [PATCH 24/31] Allow NA values for scGateConsensus

---
 singlecell/resources/chunks/CheckExpectations.R | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/singlecell/resources/chunks/CheckExpectations.R b/singlecell/resources/chunks/CheckExpectations.R
index e74e0761a..9437c6d2c 100644
--- a/singlecell/resources/chunks/CheckExpectations.R
+++ b/singlecell/resources/chunks/CheckExpectations.R
@@ -1,4 +1,4 @@
-CheckField <- function(seuratObj, datasetId, fieldName) {
+CheckField <- function(seuratObj, datasetId, fieldName, errorOnNA= TRUE) {
   if (!fieldName %in% names(seuratObj@meta.data)) {
     addErrorMessage(paste0(paste0('Missing ', fieldName, ' for dataset: ', datasetId)))
   }
@@ -47,7 +47,7 @@ for (datasetId in names(seuratObjects)) {
   }
 
   if (requireScGate) {
-    CheckField(seuratObj, datasetId, 'scGateConsensus')
+    CheckField(seuratObj, datasetId, 'scGateConsensus', errorOnNA = FALSE)
   }
 
   if (requireRiraImmune) {

From 38f84353d65f8007fe11b93b5fcb6db62586bc26 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Sat, 30 Mar 2024 22:24:45 -0700
Subject: [PATCH 25/31] Bugfix to NA check in CheckExpectations

---
 singlecell/resources/chunks/CheckExpectations.R | 4 ++--
 singlecell/resources/chunks/SeuratPrototype.R   | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/singlecell/resources/chunks/CheckExpectations.R b/singlecell/resources/chunks/CheckExpectations.R
index 9437c6d2c..a82c2ac31 100644
--- a/singlecell/resources/chunks/CheckExpectations.R
+++ b/singlecell/resources/chunks/CheckExpectations.R
@@ -1,9 +1,9 @@
-CheckField <- function(seuratObj, datasetId, fieldName, errorOnNA= TRUE) {
+CheckField <- function(seuratObj, datasetId, fieldName, errorOnNA = TRUE) {
   if (!fieldName %in% names(seuratObj@meta.data)) {
     addErrorMessage(paste0(paste0('Missing ', fieldName, ' for dataset: ', datasetId)))
   }
 
-  if (any(is.na(seuratObj@meta.data[[fieldName]]))) {
+  if (errorOnNA && any(is.na(seuratObj@meta.data[[fieldName]]))) {
     addErrorMessage(paste0(paste0('NA values found for ', fieldName, ' for dataset: ', datasetId)))
   }
 }
diff --git a/singlecell/resources/chunks/SeuratPrototype.R b/singlecell/resources/chunks/SeuratPrototype.R
index 19e783387..47ed6621e 100644
--- a/singlecell/resources/chunks/SeuratPrototype.R
+++ b/singlecell/resources/chunks/SeuratPrototype.R
@@ -1,9 +1,9 @@
-CheckField <- function(seuratObj, datasetId, fieldName) {
+CheckField <- function(seuratObj, datasetId, fieldName, errorOnNA = TRUE) {
   if (!fieldName %in% names(seuratObj@meta.data)) {
     addErrorMessage(paste0(paste0('Missing ', fieldName, ' for dataset: ', datasetId)))
   }
 
-  if (any(is.na(seuratObj@meta.data[[fieldName]]))) {
+  if (errorOnNA && any(is.na(seuratObj@meta.data[[fieldName]]))) {
     addErrorMessage(paste0(paste0('NA values found for ', fieldName, ' for dataset: ', datasetId)))
   }
 }
@@ -87,7 +87,7 @@ for (datasetId in names(seuratObjects)) {
   }
 
   if (requireScGate) {
-    CheckField(seuratObj, datasetId, 'scGateConsensus')
+    CheckField(seuratObj, datasetId, 'scGateConsensus', errorOnNA = FALSE)
   }
 
   if (length(errorMessages) > 0) {

From 1a64e10fde0cb351660e8a801533bbb02364fe66 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 1 Apr 2024 06:04:28 -0700
Subject: [PATCH 26/31] Improve logging

---
 .../sequenceanalysis/pipeline/ProcessVariantsHandler.java       | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java
index ff6d4bdbb..bf372e7cd 100644
--- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java
+++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java
@@ -422,6 +422,7 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res
         for (PipelineStepCtx<VariantProcessingStep> stepCtx : providers)
         {
             ctx.getLogger().info("Starting to run: " + stepCtx.getProvider().getLabel());
+            ctx.getLogger().debug("VCF: " + currentVCF);
             ctx.getJob().setStatus(PipelineJob.TaskStatus.running, "Running: " + stepCtx.getProvider().getLabel());
             stepIdx++;
 
@@ -476,6 +477,7 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res
             {
                 currentVCF = output.getVCF();
 
+                ctx.getJob().getLogger().info("output VCF: " + currentVCF.getPath());
                 ctx.getJob().getLogger().info("total variants: " + getVCFLineCount(currentVCF, ctx.getJob().getLogger(), false, true));
                 ctx.getJob().getLogger().info("passing variants: " + getVCFLineCount(currentVCF, ctx.getJob().getLogger(), true, false));
                 ctx.getJob().getLogger().debug("index exists: " + (new File(currentVCF.getPath() + ".tbi")).exists());

From ef08c067f8113c0e7eac59a022b577b9f64c0fc8 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 1 Apr 2024 06:54:59 -0700
Subject: [PATCH 27/31] Improve logging

---
 .../labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java
index bf372e7cd..db3beb297 100644
--- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java
+++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java
@@ -778,6 +778,7 @@ protected String getJsonName()
 
         public void setStepComplete(int stepIdx, String inputFilePath, RecordedAction action, File scatterOutput) throws PipelineJobException
         {
+            getLogger().debug("Marking step complete with VCF: " + inputFilePath);
             _scatterOutputs.put(getKey(stepIdx, inputFilePath), scatterOutput);
             _recordedActions.add(action);
             saveState();

From 5fb77dc6436e7451045adb2cddc87fbf91ae602e Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Mon, 1 Apr 2024 20:56:53 -0700
Subject: [PATCH 28/31] Support more studies

---
 singlecell/resources/chunks/StudyMetadata.R                     | 2 ++
 .../labkey/singlecell/pipeline/singlecell/StudyMetadata.java    | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/singlecell/resources/chunks/StudyMetadata.R b/singlecell/resources/chunks/StudyMetadata.R
index 688478840..ee0726402 100644
--- a/singlecell/resources/chunks/StudyMetadata.R
+++ b/singlecell/resources/chunks/StudyMetadata.R
@@ -18,6 +18,8 @@ for (datasetId in names(seuratObjects)) {
         seuratObj <- Rdiscvr::ApplyMalariaMetadata(seuratObj, errorIfUnknownIdsFound = errorIfUnknownIdsFound)
     } else if (studyName == 'PC531') {
         seuratObj <- Rdiscvr::ApplyPC531Metadata(seuratObj, errorIfUnknownIdsFound = errorIfUnknownIdsFound)
+    } else if (studyName == 'AcuteNx') {
+        seuratObj <- Rdiscvr::ApplyAcuteNxMetadata(seuratObj, errorIfUnknownIdsFound = errorIfUnknownIdsFound)
     } else {
         stop(paste0('Unknown study: ', studyName))
     }
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/StudyMetadata.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/StudyMetadata.java
index f9061672e..666250a2a 100644
--- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/StudyMetadata.java
+++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/StudyMetadata.java
@@ -24,7 +24,7 @@ public Provider()
                     {{
                         put("multiSelect", false);
                         put("allowBlank", false);
-                        put("storeValues", "PC475;PC531;TB;Malaria");
+                        put("storeValues", "PC475;PC531;TB;Malaria;AcuteNx");
                         put("delimiter", ";");
                     }}, null, null, false, false),
                     SeuratToolParameter.create("errorIfUnknownIdsFound", "Error If Unknown Ids Found", "If true, the job will fail if the seurat object contains ID not present in the metadata", "checkbox", null, true)

From 63474a31a74debb4a7dc8ab97745ffad01dc5bd5 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Tue, 2 Apr 2024 21:28:02 -0700
Subject: [PATCH 29/31] Bugfix to ProcessVariantsHandler resume

---
 .../sequenceanalysis/pipeline/ProcessVariantsHandler.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java
index db3beb297..26f2de20f 100644
--- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java
+++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java
@@ -435,7 +435,7 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res
 
             try
             {
-                SequenceAnalysisService.get().ensureVcfIndex(input, ctx.getLogger());
+                SequenceAnalysisService.get().ensureVcfIndex(currentVCF, ctx.getLogger());
             }
             catch (IOException e)
             {

From 685f5db52d7366288acb5b8a191b25ed6bae7a47 Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Wed, 3 Apr 2024 13:44:26 -0700
Subject: [PATCH 30/31] Allow parameterization of R future strategy

---
 singlecell/resources/chunks/Functions.R | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/singlecell/resources/chunks/Functions.R b/singlecell/resources/chunks/Functions.R
index a5cea2da4..7b00292fc 100644
--- a/singlecell/resources/chunks/Functions.R
+++ b/singlecell/resources/chunks/Functions.R
@@ -173,17 +173,21 @@ readSeuratRDS <- function(filePath) {
     return(seuratObj)
 }
 
-print('Updating future.globals.maxSize')
-options(future.globals.maxSize = Inf)
-
-options('Seurat.memsafe' = TRUE, future.globals.onReference = "warning")
+options(future.globals.maxSize = Inf, future.globals.onReference = "warning")
+options('Seurat.memsafe' = TRUE)
 
 if (Sys.getenv('SEURAT_MAX_THREADS') != '') {
-    print(paste0('Setting future::plan workers to: ', Sys.getenv('SEURAT_MAX_THREADS')))
+    logger::log_info(paste0('Setting future::plan workers to: ', Sys.getenv('SEURAT_MAX_THREADS')))
     mt <- as.integer(Sys.getenv('SEURAT_MAX_THREADS'))
     if (is.na(mt)) {
         stop(paste0('SEURAT_MAX_THREAD is not an integer: ', mt <- Sys.getenv('SEURAT_MAX_THREADS')))
     }
 
-    future::plan(strategy='multisession', workers=mt)
+    fs <- ifelse(exists('futureStrategy'), yes = get('futureStrategy'), no = 'multicore')
+    if (mt == 1) {
+        fs <- 'sequential'
+    }
+    logger::log_info(paste0('Setting future::strategy to: ', fs))
+
+    future::plan(strategy = fs, workers = mt)
 }

From b22a68546c1dd410f7dd3f571573786cabdda65b Mon Sep 17 00:00:00 2001
From: bbimber <bbimber@gmail.com>
Date: Thu, 4 Apr 2024 20:56:32 -0700
Subject: [PATCH 31/31] Remove onclick handlers

---
 .../DownloadSequenceDisplayColumnFactory.java | 20 ++++++++++++-------
 .../query/SequenceAnalysisCustomizer.java     | 12 +++++++++--
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/DownloadSequenceDisplayColumnFactory.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/DownloadSequenceDisplayColumnFactory.java
index 8878a465d..127f58fec 100644
--- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/DownloadSequenceDisplayColumnFactory.java
+++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/DownloadSequenceDisplayColumnFactory.java
@@ -6,20 +6,16 @@
 import org.labkey.api.data.DisplayColumn;
 import org.labkey.api.data.DisplayColumnFactory;
 import org.labkey.api.data.RenderContext;
-import org.labkey.api.data.Sort;
-import org.labkey.api.data.UrlColumn;
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.util.PageFlowUtil;
-import org.labkey.api.util.StringExpression;
-import org.labkey.api.util.StringExpressionFactory;
-import org.labkey.api.util.URLHelper;
+import org.labkey.api.view.HttpView;
 import org.labkey.api.view.template.ClientDependency;
 
 import java.io.IOException;
 import java.io.Writer;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -39,11 +35,21 @@ public DisplayColumn createRenderer(ColumnInfo colInfo)
     {
         DataColumn ret = new DataColumn(colInfo)
         {
+            private boolean _handlerRegistered = false;
+
             @Override
             public void renderGridCellContents(RenderContext ctx, Writer out) throws IOException
             {
                 Object val = ctx.get(FieldKey.fromString(getBoundColumn().getFieldKey().getParent(), "rowId"));
-                out.write(PageFlowUtil.link("Download Sequence").onClick("SequenceAnalysis.window.DownloadSequencesWindow.downloadSingle(" + val + ")").toString());
+                out.write(PageFlowUtil.link("Download Sequence").attributes(Map.of(
+                    "data-rowid", val.toString()
+                )).addClass("sdc-row").toString());
+
+                if (!_handlerRegistered)
+                {
+                    HttpView.currentPageConfig().addHandlerForQuerySelector("a.sdc-row", "click", "SequenceAnalysis.window.DownloadSequencesWindow.downloadSingle(this.attributes.getNamedItem('data-rowid').value); return false;");
+                    _handlerRegistered = true;
+                }
             }
 
             @Override
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceAnalysisCustomizer.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceAnalysisCustomizer.java
index 4e82fab0f..f23070d67 100644
--- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceAnalysisCustomizer.java
+++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceAnalysisCustomizer.java
@@ -28,6 +28,7 @@
 import org.labkey.api.security.User;
 import org.labkey.api.util.PageFlowUtil;
 import org.labkey.api.view.ActionURL;
+import org.labkey.api.view.HttpView;
 import org.labkey.api.view.template.ClientDependency;
 import org.labkey.sequenceanalysis.SequenceAnalysisSchema;
 
@@ -232,6 +233,8 @@ public DisplayColumn createRenderer(ColumnInfo colInfo)
         {
             return new DataColumn(colInfo)
             {
+                private boolean _handlerRegistered = false;
+
                 @Override
                 public @NotNull Set<ClientDependency> getClientDependencies()
                 {
@@ -275,8 +278,13 @@ public void renderGridCellContents(RenderContext ctx, Writer out) throws IOExcep
                             }
                         }
 
-                        String onclick = "onclick=\"SequenceAnalysis.window.ManageFileSetsWindow.buttonHandlerForOutputFiles(" + PageFlowUtil.jsString(rowId.toString()) + ", " + PageFlowUtil.jsString(ctx.getCurrentRegion().getName()) + ");\"";
-                        out.write("<a class=\"fa fa-pencil lk-dr-action-icon\" data-tt=\"tooltip\" data-original-title=\"add/edit\" " + onclick + "></a>");
+                        out.write("<a class=\"fa fa-pencil lk-dr-action-icon sfs-row\" data-tt=\"tooltip\" data-rowid=\"" + rowId +"\" data-original-title=\"add/edit\"></a>");
+
+                        if (!_handlerRegistered)
+                        {
+                            HttpView.currentPageConfig().addHandlerForQuerySelector("a.sfs-row", "click", "SequenceAnalysis.window.ManageFileSetsWindow.buttonHandlerForOutputFiles(this.attributes.getNamedItem('data-rowid').value, " + PageFlowUtil.jsString(ctx.getCurrentRegion().getName()) + "); return false;");
+                            _handlerRegistered = true;
+                        }
                     }
                 }
             };