Merge 24.3 to develop

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java

@@ -289,7 +289,7 @@ protected boolean isThrowNonZeroExits()
         return _throwNonZeroExits;
     }
 
-    protected static File resolveFileInPath(String exe, @Nullable String packageName, boolean throwIfNotFound)
+    public static File resolveFileInPath(String exe, @Nullable String packageName, boolean throwIfNotFound)
     {
         File fn;
         String path;

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java

@@ -90,6 +90,7 @@ public List<String> getBaseArgs(@Nullable String toolName)
         List<String> args = new ArrayList<>();
         args.add(SequencePipelineService.get().getJavaFilepath());
         args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride));
+        args.add("-DGATK_STACKTRACE_ON_USER_EXCEPTION=true");
         args.add("-jar");
         args.add(getJAR().getPath());
 
@@ -98,6 +99,8 @@ public List<String> getBaseArgs(@Nullable String toolName)
             args.add(toolName);
         }
 
+
+
         return args;
     }
 

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/DockerWrapper.java

@@ -0,0 +1,110 @@
+package org.labkey.api.sequenceanalysis.run;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.Logger;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.sequenceanalysis.pipeline.PipelineOutputTracker;
+import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
+import org.labkey.api.writer.PrintWriters;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.List;
+
+public class DockerWrapper extends AbstractCommandWrapper
+{
+    private final String _containerName;
+    private File _tmpDir = null;
+
+    public DockerWrapper(String containerName, Logger log)
+    {
+        super(log);
+        _containerName = containerName;
+    }
+
+    public void setTmpDir(File tmpDir)
+    {
+        _tmpDir = tmpDir;
+    }
+
+    public void executeWithDocker(List<String> containerArgs, File workDir, PipelineOutputTracker tracker) throws PipelineJobException
+    {
+        File localBashScript = new File(workDir, "docker.sh");
+        File dockerBashScript = new File(workDir, "dockerRun.sh");
+        tracker.addIntermediateFile(localBashScript);
+        tracker.addIntermediateFile(dockerBashScript);
+
+        setWorkingDir(workDir);
+        try (PrintWriter writer = PrintWriters.getPrintWriter(localBashScript); PrintWriter dockerWriter = PrintWriters.getPrintWriter(dockerBashScript))
+        {
+            writer.println("#!/bin/bash");
+            writer.println("set -x");
+            writer.println("WD=`pwd`");
+            writer.println("HOME=`echo ~/`");
+            writer.println("DOCKER='" + SequencePipelineService.get().getDockerCommand() + "'");
+            writer.println("sudo $DOCKER pull " + _containerName);
+            writer.println("sudo $DOCKER run --rm=true \\");
+            writer.println("\t-v \"${WD}:/work\" \\");
+            writer.println("\t-v \"${HOME}:/homeDir\" \\");
+            if (_tmpDir != null)
+            {
+                writer.println("\t-v \"" + _tmpDir.getPath() + ":/tmp\" \\");
+            }
+            writer.println("\t--entrypoint /bin/bash \\");
+            writer.println("\t-w /work \\");
+            Integer maxRam = SequencePipelineService.get().getMaxRam();
+            if (maxRam != null)
+            {
+                writer.println("\t-e SEQUENCEANALYSIS_MAX_RAM=" + maxRam + " \\");
+                writer.println("\t--memory='" + maxRam + "g' \\");
+            }
+            writer.println("\t" + _containerName + " \\");
+            writer.println("\t/work/" + dockerBashScript.getName());
+            writer.println("EXIT_CODE=$?");
+            writer.println("echo 'Docker run exit code: '$EXIT_CODE");
+            writer.println("exit $EXIT_CODE");
+
+            dockerWriter.println("#!/bin/bash");
+            dockerWriter.println("set -x");
+            dockerWriter.println(StringUtils.join(containerArgs, " "));
+            dockerWriter.println("EXIT_CODE=$?");
+            dockerWriter.println("echo 'Exit code: '$?");
+            dockerWriter.println("exit $EXIT_CODE");
+        }
+        catch (IOException e)
+        {
+            throw new PipelineJobException(e);
+        }
+
+        execute(Arrays.asList("/bin/bash", localBashScript.getPath()));
+    }
+
+    public File ensureLocalCopy(File input, File workingDirectory, PipelineOutputTracker output) throws PipelineJobException
+    {
+        try
+        {
+            if (workingDirectory.equals(input.getParentFile()))
+            {
+                return input;
+            }
+
+            File local = new File(workingDirectory, input.getName());
+            if (!local.exists())
+            {
+                getLogger().debug("Copying file locally: " + input.getPath());
+                FileUtils.copyFile(input, local);
+            }
+
+            output.addIntermediateFile(local);
+
+            return local;
+        }
+        catch (IOException e)
+        {
+            throw new PipelineJobException(e);
+        }
+    }
+}

SequenceAnalysis/pipeline_code/sequence_tools_install.sh

@@ -335,30 +335,6 @@ else
 fi
 
 
-#
-# BisSNP
-#
-echo ""
-echo ""
-echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"
-echo "Install BisSNP"
-echo ""
-cd $LKSRC_DIR
-
-if [[ ! -e ${LKTOOLS_DIR}/BisSNP.jar || ! -z $FORCE_REINSTALL ]];
-then
-    echo "Cleaning up previous installs"
-    rm -Rf BisSNP*
-    rm -Rf $LKTOOLS_DIR/BisSNP.jar
-
-    wget $WGET_OPTS https://downloads.sourceforge.net/project/bissnp/BisSNP-0.82.2/BisSNP-0.82.2.jar
-
-    install ./BisSNP-0.82.2.jar $LKTOOLS_DIR/BisSNP.jar
-else
-    echo "Already installed"
-fi
-
-
 #
 #mosaik
 #
@@ -510,10 +486,10 @@ then
     rm -Rf bcftools*
     rm -Rf $LKTOOLS_DIR/bcftools
 
-    wget $WGET_OPTS https://github.com/samtools/bcftools/releases/download/1.18/bcftools-1.18.tar.bz2
-    tar xjvf bcftools-1.18.tar.bz2
-    chmod 755 bcftools-1.18
-    cd bcftools-1.18
+    wget $WGET_OPTS https://github.com/samtools/bcftools/releases/download/1.20/bcftools-1.20.tar.bz2
+    tar xjvf bcftools-1.20.tar.bz2
+    chmod 755 bcftools-1.20
+    cd bcftools-1.20
     rm -f plugins/liftover.c
     wget $WGET_OPTS -P plugins https://raw.githubusercontent.com/freeseek/score/master/liftover.c
 

SequenceAnalysis/resources/queries/sequenceanalysis/chain_files/.qview.xml

@@ -0,0 +1,16 @@
+<customView xmlns="http://labkey.org/data/xml/queryCustomView">
+   <columns>
+       <!--<column name="rowid" />-->
+       <column name="genomeId1" />
+       <column name="genomeId2" />
+       <column name="source" />
+       <column name="version" />
+       <column name="datedisabled" />
+       <column name="chainFile" />
+   </columns>
+   <sorts>
+       <sort column="genomeId1/name"/>
+       <sort column="genomeId2/name"/>
+       <sort column="version"/>
+   </sorts>
+</customView>

SequenceAnalysis/resources/queries/sequenceanalysis/chain_files/With Filepath.qview.xml

@@ -0,0 +1,17 @@
+<customView xmlns="http://labkey.org/data/xml/queryCustomView">
+   <columns>
+       <!--<column name="rowid" />-->
+       <column name="genomeId1" />
+       <column name="genomeId2" />
+       <column name="source" />
+       <column name="version" />
+       <column name="dateDisabled" />
+       <column name="chainFile" />
+       <column name="chainFile/DataFileUrl" />
+   </columns>
+   <sorts>
+       <sort column="genomeId1/name"/>
+       <sort column="genomeId2/name"/>
+       <sort column="version"/>
+   </sorts>
+</customView>

SequenceAnalysis/resources/web/SequenceAnalysis/window/LiftoverWindow.js

@@ -104,14 +104,24 @@ Ext4.define('SequenceAnalysis.window.LiftoverWindow', {
                 maxValue: 1.0,
                 value: 0.95,
                 fieldLabel: 'Min Percent Match',
-                helpPopup: 'In order to lift to the target genome, the feature must have at least this percent match.  Lower this value to be more permissive; however, this risks incorrect liftovers',
+                helpPopup: 'In order to lift to the target genome, the feature must have at least this percent match.  Lower this value to be more permissive; however, this risks incorrect liftovers. This is ignored if using bcftools.',
                 itemId: 'pctField'
             },{
                 xtype: 'checkbox',
                 itemId: 'dropGenotypes',
                 checked: false,
                 helpPopup: 'If checked, no genotypes will be written to the output file (applies to VCFs only).  This can be useful (and necessary) when lifting VCFs with extremely high sample number.',
                 fieldLabel: 'Drop Genotypes'
+            },{
+                xtype: 'checkbox',
+                itemId: 'useBcfTools',
+                checked: false,
+                fieldLabel: 'Use bcftools'
+            },{
+                xtype: 'checkbox',
+                itemId: 'doNotRetainUnmapped',
+                checked: false,
+                fieldLabel: 'Do Not Retain Unmapped'
             }].concat(SequenceAnalysis.window.OutputHandlerWindow.getCfgForToolParameters(this.toolParameters)),
             buttons: [{
                 text: 'Submit',
@@ -152,6 +162,14 @@ Ext4.define('SequenceAnalysis.window.LiftoverWindow', {
             params.dropGenotypes = this.down('#dropGenotypes').getValue();
         }
 
+        if (this.down('#useBcfTools').getValue()){
+            params.useBcfTools = this.down('#useBcfTools').getValue();
+        }
+
+        if (this.down('#doNotRetainUnmapped').getValue()){
+            params.doNotRetainUnmapped = this.down('#doNotRetainUnmapped').getValue();
+        }
+
         Ext4.Msg.wait('Saving...');
         LABKEY.Ajax.request({
             url: LABKEY.ActionURL.buildURL('sequenceanalysis', 'runSequenceHandler', this.containerPath),

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java

@@ -59,6 +59,7 @@
 import org.labkey.sequenceanalysis.analysis.RnaSeqcHandler;
 import org.labkey.sequenceanalysis.analysis.SbtGeneCountHandler;
 import org.labkey.sequenceanalysis.analysis.UnmappedSequenceBasedGenotypeHandler;
+import org.labkey.sequenceanalysis.analysis.UpdateReadsetFilesHandler;
 import org.labkey.sequenceanalysis.button.AddSraRunButton;
 import org.labkey.sequenceanalysis.button.ArchiveReadsetsButton;
 import org.labkey.sequenceanalysis.button.ChangeReadsetStatusButton;
@@ -77,6 +78,7 @@
 import org.labkey.sequenceanalysis.run.alignment.BowtieWrapper;
 import org.labkey.sequenceanalysis.run.alignment.GSnapWrapper;
 import org.labkey.sequenceanalysis.run.alignment.MosaikWrapper;
+import org.labkey.sequenceanalysis.run.alignment.ParagraphStep;
 import org.labkey.sequenceanalysis.run.alignment.Pbmm2Wrapper;
 import org.labkey.sequenceanalysis.run.alignment.StarWrapper;
 import org.labkey.sequenceanalysis.run.alignment.VulcanWrapper;
@@ -113,6 +115,7 @@
 import org.labkey.sequenceanalysis.run.util.FastqcRunner;
 import org.labkey.sequenceanalysis.run.util.GenomicsDBAppendHandler;
 import org.labkey.sequenceanalysis.run.util.GenomicsDBImportHandler;
+import org.labkey.sequenceanalysis.run.util.SVAnnotateStep;
 import org.labkey.sequenceanalysis.run.variant.*;
 import org.labkey.sequenceanalysis.util.Barcoder;
 import org.labkey.sequenceanalysis.util.ChainFileValidator;
@@ -299,6 +302,7 @@ public static void registerPipelineSteps()
         SequencePipelineService.get().registerPipelineStep(new MendelianViolationReportStep.Provider());
         SequencePipelineService.get().registerPipelineStep(new SummarizeGenotypeQualityStep.Provider());
         SequencePipelineService.get().registerPipelineStep(new BcftoolsFillTagsStep.Provider());
+        SequencePipelineService.get().registerPipelineStep(new SVAnnotateStep.Provider());
 
         //handlers
         SequenceAnalysisService.get().registerFileHandler(new LiftoverHandler());
@@ -333,6 +337,8 @@ public static void registerPipelineSteps()
         SequenceAnalysisService.get().registerFileHandler(new PbsvJointCallingHandler());
         SequenceAnalysisService.get().registerFileHandler(new DeepVariantHandler());
         SequenceAnalysisService.get().registerFileHandler(new GLNexusHandler());
+        SequenceAnalysisService.get().registerFileHandler(new ParagraphStep());
+        SequenceAnalysisService.get().registerFileHandler(new UpdateReadsetFilesHandler());
 
         SequenceAnalysisService.get().registerReadsetHandler(new MultiQCHandler());
         SequenceAnalysisService.get().registerReadsetHandler(new RestoreSraDataHandler());