Merge discvr-24.3 to develop

SequenceAnalysis/resources/queries/sequenceanalysis/outputfiles.js

@@ -19,7 +19,7 @@ function beforeInsert(row, errors){
     beforeUpsert(row, errors);
 }
 
-function beforeUpdate(row, errors){
+function beforeUpdate(row, oldRow, errors){
     beforeUpsert(row, errors);
 }
 

SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js

@@ -86,7 +86,8 @@ Ext4.define('SequenceAnalysis.panel.AlignmentImportPanel', {
                 helpPopup: 'Description for this run, such as detail about the source of the alignments (optional)',
                 itemId: 'jobDescription',
                 name: 'jobDescription',
-                allowBlank: true
+                allowBlank: true,
+                value: LABKEY.ActionURL.getParameter('jobDescription')
             },{
                 fieldLabel: 'Delete Intermediate Files',
                 helpPopup: 'Check to delete the intermediate files created by this pipeline.  In general these are not needed and it will save disk space.  These files might be useful for debugging though.',

SequenceAnalysis/resources/web/SequenceAnalysis/panel/AnalysisSectionPanel.js

@@ -570,7 +570,7 @@ Ext4.define('SequenceAnalysis.panel.AnalysisSectionPanel', {
         return [];
     },
 
-    applySavedValues: function(values){
+    applySavedValues: function(values, allowUrlOverride){
         if (this.stepType){
             var tools = values[this.stepType] ? values[this.stepType].split(';') : [];
             this.setActiveTools(tools);
@@ -585,6 +585,10 @@ Ext4.define('SequenceAnalysis.panel.AnalysisSectionPanel', {
                 if (Ext4.isDefined(values[name])){
                     p.setValue(values[name]);
                 }
+
+                if (allowUrlOverride && LABKEY.ActionURL.getParameter(name)) {
+                    p.setValue(LABKEY.ActionURL.getParameter(name));
+                }
             }, this);
         }
     },

SequenceAnalysis/resources/web/SequenceAnalysis/panel/BaseSequencePanel.js

@@ -269,6 +269,9 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
                                                     return;
                                                 }
 
+                                                // If auto-loading, assume we want to read the URL
+                                                thePanel.down('#readUrlParams').setValue(true);
+
                                                 var recIdx = store.find('name', LABKEY.ActionURL.getParameter('template'));
                                                 if (recIdx > -1) {
                                                     thePanel.down('labkey-combo').setValue(store.getAt(recIdx));
@@ -297,6 +300,12 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
                                 helpPopup: 'By default, the pipelines jobs and their outputs will be created in the workbook you selected. However, in certain cases, such as bulk submission of many jobs, it might be preferable to submit each job to the source folder/workbook for each input. Checking this box will enable this.',
                                 fieldLabel: 'Submit Jobs to Same Folder/Workbook as Readset',
                                 labelWidth: 200
+                            },{
+                                xtype: 'checkbox',
+                                itemId: 'readUrlParams',
+                                helpPopup: 'If true, any parameters provided on the URL with the same name as a parameter in the JSON will be read and override the template.',
+                                fieldLabel: 'Read Parameters From URL',
+                                labelWidth: 200
                             }]
                         }],
                         buttons: [{
@@ -353,7 +362,8 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
                                 delete json.submitJobToReadsetContainer;
                             }
 
-                            win.sequencePanel.applySavedValues(json);
+                            var readUrlParams = win.down('#readUrlParams').getValue();
+                            win.sequencePanel.applySavedValues(json, readUrlParams);
 
                             var submitJobToReadsetContainer = win.sequencePanel.down('[name="submitJobToReadsetContainer"]');
                             if (submitJobToReadsetContainer) {
@@ -386,7 +396,7 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
         }
     },
 
-    applySavedValues: function(values){
+    applySavedValues: function(values, allowUrlOverride){
         //allows for subclasses to exclude this panel
         var alignPanel = this.down('sequenceanalysis-alignmentpanel');
         if (alignPanel) {
@@ -395,7 +405,7 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
 
         var sections = this.query('sequenceanalysis-analysissectionpanel');
         Ext4.Array.forEach(sections, function(s){
-            s.applySavedValues(values);
+            s.applySavedValues(values, allowUrlOverride);
         }, this);
 
         // For top-level properties:

SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js

@@ -300,7 +300,8 @@ Ext4.define('SequenceAnalysis.panel.SequenceAnalysisPanel', {
                 height: 100,
                 helpPopup: 'Description for this analysis (optional)',
                 name: 'jobDescription',
-                allowBlank:true
+                allowBlank: true,
+                value: LABKEY.ActionURL.getParameter('jobDescription')
             },{
                 fieldLabel: 'Delete Intermediate Files',
                 helpPopup: 'Check to delete the intermediate files created by this pipeline.  In general these are not needed and it will save disk space.  These files might be useful for debugging though.',

SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js

@@ -61,7 +61,8 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
 				height: 100,
 				helpPopup: 'Description for this analysis (optional)',
 				name: 'jobDescription',
-				allowBlank:true
+				allowBlank: true,
+				value: LABKEY.ActionURL.getParameter('jobDescription')
 			},{
 				xtype: 'sequenceanalysis-variantscattergatherpanel',
 				width: 620,

SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/DeepVariantHandler.java

@@ -97,6 +97,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
             action.addInput(so.getFile(), "Input BAM File");
 
             File outputFile = new File(ctx.getOutputDir(), FileUtil.getBaseName(so.getFile()) + ".g.vcf.gz");
+            File outputFileVcf = new File(ctx.getOutputDir(), FileUtil.getBaseName(so.getFile()) + ".vcf.gz");
 
             DeepVariantAnalysis.DeepVariantWrapper wrapper = new DeepVariantAnalysis.DeepVariantWrapper(job.getLogger());
             wrapper.setOutputDir(ctx.getOutputDir());
@@ -123,7 +124,8 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
                 throw new PipelineJobException("Missing binVersion");
             }
 
-            wrapper.execute(so.getFile(), referenceGenome.getWorkingFastaFile(), outputFile, ctx.getFileManager(), binVersion, args);
+            boolean retainVcf = ctx.getParams().optBoolean("retainVcf", false);
+            wrapper.execute(so.getFile(), referenceGenome.getWorkingFastaFile(), outputFile, retainVcf, ctx.getFileManager(), binVersion, args);
 
             action.addOutput(outputFile, "gVCF File", false);
 
@@ -137,6 +139,19 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
 
             ctx.addSequenceOutput(o);
 
+            if (retainVcf)
+            {
+                SequenceOutputFile vcf = new SequenceOutputFile();
+                vcf.setName(outputFileVcf.getName());
+                vcf.setFile(outputFileVcf);
+                vcf.setLibrary_id(so.getLibrary_id());
+                vcf.setCategory("DeepVariant VCF File");
+                vcf.setReadset(so.getReadset());
+                vcf.setDescription("DeepVariant Version: " + binVersion);
+
+                ctx.addSequenceOutput(vcf);
+            }
+
             ctx.addActions(action);
         }
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java

@@ -369,15 +369,6 @@ public static List<Interval> getIntervals(JobContext ctx)
 
     public static File processVCF(File input, Integer libraryId, JobContext ctx, Resumer resumer, boolean subsetToIntervals) throws PipelineJobException
     {
-        try
-        {
-            SequenceAnalysisService.get().ensureVcfIndex(input, ctx.getLogger());
-        }
-        catch (IOException e)
-        {
-            throw new PipelineJobException(e);
-        }
-
         File currentVCF = input;
 
         ctx.getJob().getLogger().info("***Starting processing of file: " + input.getName());
@@ -409,6 +400,15 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res
             }
             else
             {
+                try
+                {
+                    SequenceAnalysisService.get().ensureVcfIndex(input, ctx.getLogger());
+                }
+                catch (IOException e)
+                {
+                    throw new PipelineJobException(e);
+                }
+
                 OutputVariantsStartingInIntervalsStep.Wrapper wrapper = new OutputVariantsStartingInIntervalsStep.Wrapper(ctx.getLogger());
                 wrapper.execute(input, outputFile, getIntervals(ctx));
             }
@@ -422,6 +422,7 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res
         for (PipelineStepCtx<VariantProcessingStep> stepCtx : providers)
         {
             ctx.getLogger().info("Starting to run: " + stepCtx.getProvider().getLabel());
+            ctx.getLogger().debug("VCF: " + currentVCF);
             ctx.getJob().setStatus(PipelineJob.TaskStatus.running, "Running: " + stepCtx.getProvider().getLabel());
             stepIdx++;
 
@@ -432,6 +433,15 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res
                 continue;
             }
 
+            try
+            {
+                SequenceAnalysisService.get().ensureVcfIndex(currentVCF, ctx.getLogger());
+            }
+            catch (IOException e)
+            {
+                throw new PipelineJobException(e);
+            }
+
             RecordedAction action = new RecordedAction(stepCtx.getProvider().getLabel());
             Date start = new Date();
             action.setStartTime(start);
@@ -467,6 +477,7 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res
             {
                 currentVCF = output.getVCF();
 
+                ctx.getJob().getLogger().info("output VCF: " + currentVCF.getPath());
                 ctx.getJob().getLogger().info("total variants: " + getVCFLineCount(currentVCF, ctx.getJob().getLogger(), false, true));
                 ctx.getJob().getLogger().info("passing variants: " + getVCFLineCount(currentVCF, ctx.getJob().getLogger(), true, false));
                 ctx.getJob().getLogger().debug("index exists: " + (new File(currentVCF.getPath() + ".tbi")).exists());
@@ -767,6 +778,7 @@ protected String getJsonName()
 
         public void setStepComplete(int stepIdx, String inputFilePath, RecordedAction action, File scatterOutput) throws PipelineJobException
         {
+            getLogger().debug("Marking step complete with VCF: " + inputFilePath);
             _scatterOutputs.put(getKey(stepIdx, inputFilePath), scatterOutput);
             _recordedActions.add(action);
             saveState();

SequenceAnalysis/src/org/labkey/sequenceanalysis/query/DownloadSequenceDisplayColumnFactory.java

@@ -6,20 +6,16 @@
 import org.labkey.api.data.DisplayColumn;
 import org.labkey.api.data.DisplayColumnFactory;
 import org.labkey.api.data.RenderContext;
-import org.labkey.api.data.Sort;
-import org.labkey.api.data.UrlColumn;
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.util.PageFlowUtil;
-import org.labkey.api.util.StringExpression;
-import org.labkey.api.util.StringExpressionFactory;
-import org.labkey.api.util.URLHelper;
+import org.labkey.api.view.HttpView;
 import org.labkey.api.view.template.ClientDependency;
 
 import java.io.IOException;
 import java.io.Writer;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -39,11 +35,21 @@ public DisplayColumn createRenderer(ColumnInfo colInfo)
     {
         DataColumn ret = new DataColumn(colInfo)
         {
+            private boolean _handlerRegistered = false;
+
             @Override
             public void renderGridCellContents(RenderContext ctx, Writer out) throws IOException
             {
                 Object val = ctx.get(FieldKey.fromString(getBoundColumn().getFieldKey().getParent(), "rowId"));
-                out.write(PageFlowUtil.link("Download Sequence").onClick("SequenceAnalysis.window.DownloadSequencesWindow.downloadSingle(" + val + ")").toString());
+                out.write(PageFlowUtil.link("Download Sequence").attributes(Map.of(
+                    "data-rowid", val.toString()
+                )).addClass("sdc-row").toString());
+
+                if (!_handlerRegistered)
+                {
+                    HttpView.currentPageConfig().addHandlerForQuerySelector("a.sdc-row", "click", "SequenceAnalysis.window.DownloadSequencesWindow.downloadSingle(this.attributes.getNamedItem('data-rowid').value); return false;");
+                    _handlerRegistered = true;
+                }
             }
 
             @Override

SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceAnalysisCustomizer.java

@@ -28,6 +28,7 @@
 import org.labkey.api.security.User;
 import org.labkey.api.util.PageFlowUtil;
 import org.labkey.api.view.ActionURL;
+import org.labkey.api.view.HttpView;
 import org.labkey.api.view.template.ClientDependency;
 import org.labkey.sequenceanalysis.SequenceAnalysisSchema;
 
@@ -232,6 +233,8 @@ public DisplayColumn createRenderer(ColumnInfo colInfo)
         {
             return new DataColumn(colInfo)
             {
+                private boolean _handlerRegistered = false;
+
                 @Override
                 public @NotNull Set<ClientDependency> getClientDependencies()
                 {
@@ -275,8 +278,13 @@ public void renderGridCellContents(RenderContext ctx, Writer out) throws IOExcep
                             }
                         }
 
-                        String onclick = "onclick=\"SequenceAnalysis.window.ManageFileSetsWindow.buttonHandlerForOutputFiles(" + PageFlowUtil.jsString(rowId.toString()) + ", " + PageFlowUtil.jsString(ctx.getCurrentRegion().getName()) + ");\"";
-                        out.write("<a class=\"fa fa-pencil lk-dr-action-icon\" data-tt=\"tooltip\" data-original-title=\"add/edit\" " + onclick + "></a>");
+                        out.write("<a class=\"fa fa-pencil lk-dr-action-icon sfs-row\" data-tt=\"tooltip\" data-rowid=\"" + rowId +"\" data-original-title=\"add/edit\"></a>");
+
+                        if (!_handlerRegistered)
+                        {
+                            HttpView.currentPageConfig().addHandlerForQuerySelector("a.sfs-row", "click", "SequenceAnalysis.window.ManageFileSetsWindow.buttonHandlerForOutputFiles(this.attributes.getNamedItem('data-rowid').value, " + PageFlowUtil.jsString(ctx.getCurrentRegion().getName()) + "); return false;");
+                            _handlerRegistered = true;
+                        }
                     }
                 }
             };

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/DeepVariantAnalysis.java

@@ -70,7 +70,8 @@ public static List<ToolParameterDescriptor> getToolDescriptors()
                 }}, "X,Y"),
                 ToolParameterDescriptor.create("binVersion", "DeepVariant Version", "The version of DeepVariant to run, which is passed to their docker container", "textfield", new JSONObject(){{
                     put("allowBlank", false);
-                }}, "1.6.0")
+                }}, "1.6.0"),
+                ToolParameterDescriptor.create("retainVcf", "Retain VCF", "If selected, the VCF with called genotypes will be retained", "checkbox", null, false)
         );
     }
 
@@ -153,9 +154,11 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc
             throw new PipelineJobException("Missing binVersion");
         }
 
+        boolean retainVcf = getProvider().getParameterByName("retainVcf").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false);
+
         getWrapper().setOutputDir(outputDir);
         getWrapper().setWorkingDir(outputDir);
-        getWrapper().execute(inputBam, referenceGenome.getWorkingFastaFile(), outputFile, output, binVersion, args);
+        getWrapper().execute(inputBam, referenceGenome.getWorkingFastaFile(), outputFile, retainVcf, output, binVersion, args);
 
         output.addOutput(outputFile, "gVCF File");
         output.addSequenceOutput(outputFile, outputFile.getName(), "DeepVariant gVCF File", rs.getReadsetId(), null, referenceGenome.getGenomeId(), "DeepVariant Version: " + binVersion);
@@ -164,6 +167,17 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc
             output.addOutput(idxFile, "VCF Index");
         }
 
+        if (retainVcf)
+        {
+            File outputFileVcf = new File(outputDir, FileUtil.getBaseName(inputBam) + ".vcf.gz");
+            if (!outputFileVcf.exists())
+            {
+                throw new PipelineJobException("Missing expected file: " + outputFileVcf.getPath());
+            }
+
+            output.addSequenceOutput(outputFile, outputFileVcf.getName(), "DeepVariant VCF File", rs.getReadsetId(), null, referenceGenome.getGenomeId(), "DeepVariant Version: " + binVersion);
+        }
+
         return output;
     }
 
@@ -206,12 +220,15 @@ private File ensureLocalCopy(File input, File workingDirectory, PipelineOutputTr
             }
         }
 
-        public void execute(File inputBam, File refFasta, File outputGvcf, PipelineOutputTracker tracker, String binVersion, List<String> extraArgs) throws PipelineJobException
+        public void execute(File inputBam, File refFasta, File outputGvcf, boolean retainVcf, PipelineOutputTracker tracker, String binVersion, List<String> extraArgs) throws PipelineJobException
         {
             File workDir = outputGvcf.getParentFile();
             File outputVcf = new File(outputGvcf.getPath().replaceAll(".g.vcf", ".vcf"));
-            tracker.addIntermediateFile(outputVcf);
-            tracker.addIntermediateFile(new File(outputVcf.getPath() + ".tbi"));
+            if (!retainVcf)
+            {
+                tracker.addIntermediateFile(outputVcf);
+                tracker.addIntermediateFile(new File(outputVcf.getPath() + ".tbi"));
+            }
 
             File inputBamLocal = ensureLocalCopy(inputBam, workDir, tracker);
             ensureLocalCopy(SequenceUtil.getExpectedIndex(inputBam), workDir, tracker);