diff --git a/SequenceAnalysis/resources/queries/sequenceanalysis/outputfiles.js b/SequenceAnalysis/resources/queries/sequenceanalysis/outputfiles.js index 1b0657f51..ba82cc75b 100644 --- a/SequenceAnalysis/resources/queries/sequenceanalysis/outputfiles.js +++ b/SequenceAnalysis/resources/queries/sequenceanalysis/outputfiles.js @@ -19,7 +19,7 @@ function beforeInsert(row, errors){ beforeUpsert(row, errors); } -function beforeUpdate(row, errors){ +function beforeUpdate(row, oldRow, errors){ beforeUpsert(row, errors); } diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js index 73a786c9d..bb484a17c 100644 --- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js +++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js @@ -86,7 +86,8 @@ Ext4.define('SequenceAnalysis.panel.AlignmentImportPanel', { helpPopup: 'Description for this run, such as detail about the source of the alignments (optional)', itemId: 'jobDescription', name: 'jobDescription', - allowBlank: true + allowBlank: true, + value: LABKEY.ActionURL.getParameter('jobDescription') },{ fieldLabel: 'Delete Intermediate Files', helpPopup: 'Check to delete the intermediate files created by this pipeline. In general these are not needed and it will save disk space. These files might be useful for debugging though.', diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AnalysisSectionPanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AnalysisSectionPanel.js index 56c7f02b4..8522f184c 100644 --- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AnalysisSectionPanel.js +++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AnalysisSectionPanel.js @@ -570,7 +570,7 @@ Ext4.define('SequenceAnalysis.panel.AnalysisSectionPanel', { return []; }, - applySavedValues: function(values){ + applySavedValues: function(values, allowUrlOverride){ if (this.stepType){ var tools = values[this.stepType] ? values[this.stepType].split(';') : []; this.setActiveTools(tools); @@ -585,6 +585,10 @@ Ext4.define('SequenceAnalysis.panel.AnalysisSectionPanel', { if (Ext4.isDefined(values[name])){ p.setValue(values[name]); } + + if (allowUrlOverride && LABKEY.ActionURL.getParameter(name)) { + p.setValue(LABKEY.ActionURL.getParameter(name)); + } }, this); } }, diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/BaseSequencePanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/BaseSequencePanel.js index 2c393ab36..81ea54519 100644 --- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/BaseSequencePanel.js +++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/BaseSequencePanel.js @@ -269,6 +269,9 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', { return; } + // If auto-loading, assume we want to read the URL + thePanel.down('#readUrlParams').setValue(true); + var recIdx = store.find('name', LABKEY.ActionURL.getParameter('template')); if (recIdx > -1) { thePanel.down('labkey-combo').setValue(store.getAt(recIdx)); @@ -297,6 +300,12 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', { helpPopup: 'By default, the pipelines jobs and their outputs will be created in the workbook you selected. However, in certain cases, such as bulk submission of many jobs, it might be preferable to submit each job to the source folder/workbook for each input. Checking this box will enable this.', fieldLabel: 'Submit Jobs to Same Folder/Workbook as Readset', labelWidth: 200 + },{ + xtype: 'checkbox', + itemId: 'readUrlParams', + helpPopup: 'If true, any parameters provided on the URL with the same name as a parameter in the JSON will be read and override the template.', + fieldLabel: 'Read Parameters From URL', + labelWidth: 200 }] }], buttons: [{ @@ -353,7 +362,8 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', { delete json.submitJobToReadsetContainer; } - win.sequencePanel.applySavedValues(json); + var readUrlParams = win.down('#readUrlParams').getValue(); + win.sequencePanel.applySavedValues(json, readUrlParams); var submitJobToReadsetContainer = win.sequencePanel.down('[name="submitJobToReadsetContainer"]'); if (submitJobToReadsetContainer) { @@ -386,7 +396,7 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', { } }, - applySavedValues: function(values){ + applySavedValues: function(values, allowUrlOverride){ //allows for subclasses to exclude this panel var alignPanel = this.down('sequenceanalysis-alignmentpanel'); if (alignPanel) { @@ -395,7 +405,7 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', { var sections = this.query('sequenceanalysis-analysissectionpanel'); Ext4.Array.forEach(sections, function(s){ - s.applySavedValues(values); + s.applySavedValues(values, allowUrlOverride); }, this); // For top-level properties: diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js index bd3a422dd..dfaa1e07d 100644 --- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js +++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js @@ -300,7 +300,8 @@ Ext4.define('SequenceAnalysis.panel.SequenceAnalysisPanel', { height: 100, helpPopup: 'Description for this analysis (optional)', name: 'jobDescription', - allowBlank:true + allowBlank: true, + value: LABKEY.ActionURL.getParameter('jobDescription') },{ fieldLabel: 'Delete Intermediate Files', helpPopup: 'Check to delete the intermediate files created by this pipeline. In general these are not needed and it will save disk space. These files might be useful for debugging though.', diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js index 9cf64ff68..b67b01477 100644 --- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js +++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js @@ -61,7 +61,8 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', { height: 100, helpPopup: 'Description for this analysis (optional)', name: 'jobDescription', - allowBlank:true + allowBlank: true, + value: LABKEY.ActionURL.getParameter('jobDescription') },{ xtype: 'sequenceanalysis-variantscattergatherpanel', width: 620, diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/DeepVariantHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/DeepVariantHandler.java index 56d18fb7f..249e53344 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/DeepVariantHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/DeepVariantHandler.java @@ -97,6 +97,7 @@ public void processFilesRemote(List inputFiles, JobContext c action.addInput(so.getFile(), "Input BAM File"); File outputFile = new File(ctx.getOutputDir(), FileUtil.getBaseName(so.getFile()) + ".g.vcf.gz"); + File outputFileVcf = new File(ctx.getOutputDir(), FileUtil.getBaseName(so.getFile()) + ".vcf.gz"); DeepVariantAnalysis.DeepVariantWrapper wrapper = new DeepVariantAnalysis.DeepVariantWrapper(job.getLogger()); wrapper.setOutputDir(ctx.getOutputDir()); @@ -123,7 +124,8 @@ public void processFilesRemote(List inputFiles, JobContext c throw new PipelineJobException("Missing binVersion"); } - wrapper.execute(so.getFile(), referenceGenome.getWorkingFastaFile(), outputFile, ctx.getFileManager(), binVersion, args); + boolean retainVcf = ctx.getParams().optBoolean("retainVcf", false); + wrapper.execute(so.getFile(), referenceGenome.getWorkingFastaFile(), outputFile, retainVcf, ctx.getFileManager(), binVersion, args); action.addOutput(outputFile, "gVCF File", false); @@ -137,6 +139,19 @@ public void processFilesRemote(List inputFiles, JobContext c ctx.addSequenceOutput(o); + if (retainVcf) + { + SequenceOutputFile vcf = new SequenceOutputFile(); + vcf.setName(outputFileVcf.getName()); + vcf.setFile(outputFileVcf); + vcf.setLibrary_id(so.getLibrary_id()); + vcf.setCategory("DeepVariant VCF File"); + vcf.setReadset(so.getReadset()); + vcf.setDescription("DeepVariant Version: " + binVersion); + + ctx.addSequenceOutput(vcf); + } + ctx.addActions(action); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java index 32b082fda..26f2de20f 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java @@ -369,15 +369,6 @@ public static List getIntervals(JobContext ctx) public static File processVCF(File input, Integer libraryId, JobContext ctx, Resumer resumer, boolean subsetToIntervals) throws PipelineJobException { - try - { - SequenceAnalysisService.get().ensureVcfIndex(input, ctx.getLogger()); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - File currentVCF = input; ctx.getJob().getLogger().info("***Starting processing of file: " + input.getName()); @@ -409,6 +400,15 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res } else { + try + { + SequenceAnalysisService.get().ensureVcfIndex(input, ctx.getLogger()); + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + OutputVariantsStartingInIntervalsStep.Wrapper wrapper = new OutputVariantsStartingInIntervalsStep.Wrapper(ctx.getLogger()); wrapper.execute(input, outputFile, getIntervals(ctx)); } @@ -422,6 +422,7 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res for (PipelineStepCtx stepCtx : providers) { ctx.getLogger().info("Starting to run: " + stepCtx.getProvider().getLabel()); + ctx.getLogger().debug("VCF: " + currentVCF); ctx.getJob().setStatus(PipelineJob.TaskStatus.running, "Running: " + stepCtx.getProvider().getLabel()); stepIdx++; @@ -432,6 +433,15 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res continue; } + try + { + SequenceAnalysisService.get().ensureVcfIndex(currentVCF, ctx.getLogger()); + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + RecordedAction action = new RecordedAction(stepCtx.getProvider().getLabel()); Date start = new Date(); action.setStartTime(start); @@ -467,6 +477,7 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res { currentVCF = output.getVCF(); + ctx.getJob().getLogger().info("output VCF: " + currentVCF.getPath()); ctx.getJob().getLogger().info("total variants: " + getVCFLineCount(currentVCF, ctx.getJob().getLogger(), false, true)); ctx.getJob().getLogger().info("passing variants: " + getVCFLineCount(currentVCF, ctx.getJob().getLogger(), true, false)); ctx.getJob().getLogger().debug("index exists: " + (new File(currentVCF.getPath() + ".tbi")).exists()); @@ -767,6 +778,7 @@ protected String getJsonName() public void setStepComplete(int stepIdx, String inputFilePath, RecordedAction action, File scatterOutput) throws PipelineJobException { + getLogger().debug("Marking step complete with VCF: " + inputFilePath); _scatterOutputs.put(getKey(stepIdx, inputFilePath), scatterOutput); _recordedActions.add(action); saveState(); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/DownloadSequenceDisplayColumnFactory.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/DownloadSequenceDisplayColumnFactory.java index 8878a465d..127f58fec 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/DownloadSequenceDisplayColumnFactory.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/DownloadSequenceDisplayColumnFactory.java @@ -6,20 +6,16 @@ import org.labkey.api.data.DisplayColumn; import org.labkey.api.data.DisplayColumnFactory; import org.labkey.api.data.RenderContext; -import org.labkey.api.data.Sort; -import org.labkey.api.data.UrlColumn; import org.labkey.api.query.FieldKey; import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.StringExpression; -import org.labkey.api.util.StringExpressionFactory; -import org.labkey.api.util.URLHelper; +import org.labkey.api.view.HttpView; import org.labkey.api.view.template.ClientDependency; import java.io.IOException; import java.io.Writer; -import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; +import java.util.Map; import java.util.Set; /** @@ -39,11 +35,21 @@ public DisplayColumn createRenderer(ColumnInfo colInfo) { DataColumn ret = new DataColumn(colInfo) { + private boolean _handlerRegistered = false; + @Override public void renderGridCellContents(RenderContext ctx, Writer out) throws IOException { Object val = ctx.get(FieldKey.fromString(getBoundColumn().getFieldKey().getParent(), "rowId")); - out.write(PageFlowUtil.link("Download Sequence").onClick("SequenceAnalysis.window.DownloadSequencesWindow.downloadSingle(" + val + ")").toString()); + out.write(PageFlowUtil.link("Download Sequence").attributes(Map.of( + "data-rowid", val.toString() + )).addClass("sdc-row").toString()); + + if (!_handlerRegistered) + { + HttpView.currentPageConfig().addHandlerForQuerySelector("a.sdc-row", "click", "SequenceAnalysis.window.DownloadSequencesWindow.downloadSingle(this.attributes.getNamedItem('data-rowid').value); return false;"); + _handlerRegistered = true; + } } @Override diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceAnalysisCustomizer.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceAnalysisCustomizer.java index 4e82fab0f..f23070d67 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceAnalysisCustomizer.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/query/SequenceAnalysisCustomizer.java @@ -28,6 +28,7 @@ import org.labkey.api.security.User; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ActionURL; +import org.labkey.api.view.HttpView; import org.labkey.api.view.template.ClientDependency; import org.labkey.sequenceanalysis.SequenceAnalysisSchema; @@ -232,6 +233,8 @@ public DisplayColumn createRenderer(ColumnInfo colInfo) { return new DataColumn(colInfo) { + private boolean _handlerRegistered = false; + @Override public @NotNull Set getClientDependencies() { @@ -275,8 +278,13 @@ public void renderGridCellContents(RenderContext ctx, Writer out) throws IOExcep } } - String onclick = "onclick=\"SequenceAnalysis.window.ManageFileSetsWindow.buttonHandlerForOutputFiles(" + PageFlowUtil.jsString(rowId.toString()) + ", " + PageFlowUtil.jsString(ctx.getCurrentRegion().getName()) + ");\""; - out.write(""); + out.write(""); + + if (!_handlerRegistered) + { + HttpView.currentPageConfig().addHandlerForQuerySelector("a.sfs-row", "click", "SequenceAnalysis.window.ManageFileSetsWindow.buttonHandlerForOutputFiles(this.attributes.getNamedItem('data-rowid').value, " + PageFlowUtil.jsString(ctx.getCurrentRegion().getName()) + "); return false;"); + _handlerRegistered = true; + } } } }; diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/DeepVariantAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/DeepVariantAnalysis.java index 51c6bf296..b49680702 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/DeepVariantAnalysis.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/DeepVariantAnalysis.java @@ -70,7 +70,8 @@ public static List getToolDescriptors() }}, "X,Y"), ToolParameterDescriptor.create("binVersion", "DeepVariant Version", "The version of DeepVariant to run, which is passed to their docker container", "textfield", new JSONObject(){{ put("allowBlank", false); - }}, "1.6.0") + }}, "1.6.0"), + ToolParameterDescriptor.create("retainVcf", "Retain VCF", "If selected, the VCF with called genotypes will be retained", "checkbox", null, false) ); } @@ -153,9 +154,11 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc throw new PipelineJobException("Missing binVersion"); } + boolean retainVcf = getProvider().getParameterByName("retainVcf").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false); + getWrapper().setOutputDir(outputDir); getWrapper().setWorkingDir(outputDir); - getWrapper().execute(inputBam, referenceGenome.getWorkingFastaFile(), outputFile, output, binVersion, args); + getWrapper().execute(inputBam, referenceGenome.getWorkingFastaFile(), outputFile, retainVcf, output, binVersion, args); output.addOutput(outputFile, "gVCF File"); output.addSequenceOutput(outputFile, outputFile.getName(), "DeepVariant gVCF File", rs.getReadsetId(), null, referenceGenome.getGenomeId(), "DeepVariant Version: " + binVersion); @@ -164,6 +167,17 @@ public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, Referenc output.addOutput(idxFile, "VCF Index"); } + if (retainVcf) + { + File outputFileVcf = new File(outputDir, FileUtil.getBaseName(inputBam) + ".vcf.gz"); + if (!outputFileVcf.exists()) + { + throw new PipelineJobException("Missing expected file: " + outputFileVcf.getPath()); + } + + output.addSequenceOutput(outputFile, outputFileVcf.getName(), "DeepVariant VCF File", rs.getReadsetId(), null, referenceGenome.getGenomeId(), "DeepVariant Version: " + binVersion); + } + return output; } @@ -206,12 +220,15 @@ private File ensureLocalCopy(File input, File workingDirectory, PipelineOutputTr } } - public void execute(File inputBam, File refFasta, File outputGvcf, PipelineOutputTracker tracker, String binVersion, List extraArgs) throws PipelineJobException + public void execute(File inputBam, File refFasta, File outputGvcf, boolean retainVcf, PipelineOutputTracker tracker, String binVersion, List extraArgs) throws PipelineJobException { File workDir = outputGvcf.getParentFile(); File outputVcf = new File(outputGvcf.getPath().replaceAll(".g.vcf", ".vcf")); - tracker.addIntermediateFile(outputVcf); - tracker.addIntermediateFile(new File(outputVcf.getPath() + ".tbi")); + if (!retainVcf) + { + tracker.addIntermediateFile(outputVcf); + tracker.addIntermediateFile(new File(outputVcf.getPath() + ".tbi")); + } File inputBamLocal = ensureLocalCopy(inputBam, workDir, tracker); ensureLocalCopy(SequenceUtil.getExpectedIndex(inputBam), workDir, tracker); diff --git a/jbrowse/package-lock.json b/jbrowse/package-lock.json index b32c1324c..dc3e491f6 100644 --- a/jbrowse/package-lock.json +++ b/jbrowse/package-lock.json @@ -6689,13 +6689,13 @@ "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==" }, "node_modules/body-parser": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", - "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "dev": true, "dependencies": { "bytes": "3.1.2", - "content-type": "~1.0.4", + "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", @@ -6703,7 +6703,7 @@ "iconv-lite": "0.4.24", "on-finished": "2.4.1", "qs": "6.11.0", - "raw-body": "2.5.1", + "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" }, @@ -7525,9 +7525,9 @@ "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==" }, "node_modules/cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", "dev": true, "engines": { "node": ">= 0.6" @@ -8945,17 +8945,17 @@ } }, "node_modules/express": { - "version": "4.18.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", - "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "dev": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.1", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -15160,9 +15160,9 @@ } }, "node_modules/raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "dev": true, "dependencies": { "bytes": "3.1.2", @@ -18335,9 +18335,9 @@ } }, "node_modules/webpack-dev-middleware": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.3.tgz", - "integrity": "sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA==", + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz", + "integrity": "sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q==", "dev": true, "dependencies": { "colorette": "^2.0.10", diff --git a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx index afe99496c..ac75ef4de 100644 --- a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx +++ b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx @@ -5,13 +5,16 @@ import { GridColumnVisibilityModel, GridPaginationModel, GridRenderCellParams, + GridSortDirection, + GridSortModel, GridToolbarColumnsButton, GridToolbarContainer, GridToolbarDensitySelector, GridToolbarExport } from '@mui/x-data-grid'; import SearchIcon from '@mui/icons-material/Search'; -import React, { useEffect, useState, useMemo } from 'react'; +import LinkIcon from '@mui/icons-material/Link'; +import React, { useEffect, useState } from 'react'; import { getConf } from '@jbrowse/core/configuration'; import { AppBar, Box, Button, Dialog, Paper, Popover, Toolbar, Tooltip, Typography } from '@mui/material'; import { FilterFormModal } from './FilterFormModal'; @@ -67,14 +70,17 @@ const VariantTableWidget = observer(props => { session.hideWidget(widget) } - function handleQuery(passedFilters, pushToHistory, pageQueryModel = pageSizeModel) { + function handleQuery(passedFilters, pushToHistory, pageQueryModel = pageSizeModel, sortQueryModel = sortModel) { const { page = pageSizeModel.page, pageSize = pageSizeModel.pageSize } = pageQueryModel; + const { field = "genomicPosition", sort = false } = sortQueryModel[0] ?? {}; const encodedSearchString = createEncodedFilterString(passedFilters, false); const currentUrl = new URL(window.location.href); currentUrl.searchParams.set("searchString", encodedSearchString); currentUrl.searchParams.set("page", page.toString()); currentUrl.searchParams.set("pageSize", pageSize.toString()); + currentUrl.searchParams.set("sortField", field.toString()); + currentUrl.searchParams.set("sortDirection", sort.toString()); if (pushToHistory) { window.history.pushState(null, "", currentUrl.toString()); @@ -82,7 +88,7 @@ const VariantTableWidget = observer(props => { setFilters(passedFilters); setDataLoaded(false) - fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)}); + fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, field, sort, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)}); } const TableCellWithPopover = (props: { value: any }) => { @@ -190,7 +196,29 @@ const VariantTableWidget = observer(props => { Search - + + + ); } @@ -234,11 +262,15 @@ const VariantTableWidget = observer(props => { // False until initial data load or an error: const [dataLoaded, setDataLoaded] = useState(false) - const urlParams = new URLSearchParams(window.location.search); - const page = parseInt(urlParams.get('page') || '0'); - const pageSize = parseInt(urlParams.get('pageSize') || '50'); + const urlParams = new URLSearchParams(window.location.search) + const page = parseInt(urlParams.get('page') || '0') + const pageSize = parseInt(urlParams.get('pageSize') || '50') const [pageSizeModel, setPageSizeModel] = React.useState({ page, pageSize }); + const sortField = urlParams.get('sortField') || 'genomicPosition' + const sortDirection = urlParams.get('sortDirection') || 'desc' + const [sortModel, setSortModel] = React.useState([{ field: sortField, sort: sortDirection as GridSortDirection }]) + const colVisURLComponent = urlParams.get("colVisModel") || "{}" const colVisModel = JSON.parse(decodeURIComponent(colVisURLComponent)) const [columnVisibilityModel, setColumnVisibilityModel] = useState(colVisModel); @@ -419,6 +451,11 @@ const VariantTableWidget = observer(props => { currentUrl.searchParams.set("colVisModel", encodeURIComponent(JSON.stringify(trueValuesModel))); window.history.pushState(null, "", currentUrl.toString()); }} + sortingMode="server" + onSortModelChange={(newModel) => { + setSortModel(newModel) + handleQuery(filters, true, { page: 0, pageSize: pageSizeModel.pageSize }, newModel); + }} /> ) @@ -440,7 +477,7 @@ const VariantTableWidget = observer(props => { fieldTypeInfo: fieldTypeInfo, allowedGroupNames: allowedGroupNames, promotedFilters: promotedFilters, - handleQuery: (filters) => handleQuery(filters, true) + handleQuery: (filters) => handleQuery(filters, true, { page: 0, pageSize: pageSizeModel.pageSize}, sortModel) }} /> ); diff --git a/jbrowse/src/client/JBrowse/utils.ts b/jbrowse/src/client/JBrowse/utils.ts index 1401ef791..32d514cab 100644 --- a/jbrowse/src/client/JBrowse/utils.ts +++ b/jbrowse/src/client/JBrowse/utils.ts @@ -419,7 +419,7 @@ function generateLuceneString(field, operator, value) { return luceneQueryString; } -export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, successCallback, failureCallback) { +export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, sortField, sortReverseString, successCallback, failureCallback) { if (!offset) { offset = 0 } @@ -439,6 +439,13 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa return } + let sortReverse; + if(sortReverseString == "asc") { + sortReverse = true + } else { + sortReverse = false + } + return Ajax.request({ url: ActionURL.buildURL('jbrowse', 'luceneQuery.api'), method: 'GET', @@ -449,7 +456,15 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa failure: function(res) { failureCallback("There was an error: " + res.status + "\n Status Body: " + res.responseText + "\n Session ID:" + sessionId) }, - params: {"searchString": createEncodedFilterString(filters, true), "sessionId": sessionId, "trackId": trackGUID, "offset": offset, "pageSize": pageSize}, + params: { + "searchString": createEncodedFilterString(filters, true), + "sessionId": sessionId, + "trackId": trackGUID, + "offset": offset, + "pageSize": pageSize, + "sortField": sortField ?? "genomicPosition", + "sortReverse": sortReverse + }, }); } diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java index ada6cbbd5..11a01c283 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseController.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseController.java @@ -910,7 +910,7 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors) try { - return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset())); + return new ApiSimpleResponse(searcher.doSearch(getUser(), PageFlowUtil.decode(form.getSearchString()), form.getPageSize(), form.getOffset(), form.getSortField(), form.getSortReverse())); } catch (Exception e) { @@ -947,6 +947,10 @@ public static class LuceneQueryForm private int _offset = 0; + private String _sortField = "genomicPosition"; + + private boolean _sortReverse = false; + public String getSearchString() { return _searchString; @@ -987,6 +991,14 @@ public void setOffset(int offset) _offset = offset; } + public String getSortField() { return _sortField; } + + public void setSortField(String sortField) { _sortField = sortField; } + + public boolean getSortReverse() { return _sortReverse; } + + public void setSortReverse(boolean sortReverse) { _sortReverse = sortReverse; } + public String getTrackId() { return _trackId; diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java index c5e1889a9..14d0aa031 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java @@ -22,6 +22,7 @@ import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.NumericUtils; import org.jetbrains.annotations.Nullable; import org.json.JSONObject; import org.labkey.api.data.Container; @@ -50,6 +51,7 @@ import java.util.StringTokenizer; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.labkey.jbrowse.JBrowseFieldUtils.VARIABLE_SAMPLES; import static org.labkey.jbrowse.JBrowseFieldUtils.getSession; @@ -61,6 +63,8 @@ public class JBrowseLuceneSearch private final JsonFile _jsonFile; private final User _user; private final String[] specialStartPatterns = {"*:* -", "+", "-"}; + private static final String ALL_DOCS = "all"; + private static final String GENOMIC_POSITION = "genomicPosition"; private JBrowseLuceneSearch(final JBrowseSession session, final JsonFile jsonFile, User u) { @@ -130,7 +134,7 @@ public String extractFieldName(String queryString) { return parts.length > 0 ? parts[0].trim() : null; } - public JSONObject doSearch(User u, String searchString, final int pageSize, final int offset) throws IOException, ParseException + public JSONObject doSearch(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException { searchString = tryUrlDecode(searchString); File indexPath = _jsonFile.getExpectedLocationOfLuceneIndex(true); @@ -146,7 +150,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina IndexSearcher indexSearcher = new IndexSearcher(indexReader); List stringQueryParserFields = new ArrayList<>(); - List numericQueryParserFields = new ArrayList<>(); + Map numericQueryParserFields = new HashMap<>(); PointsConfig intPointsConfig = new PointsConfig(new DecimalFormat(), Integer.class); PointsConfig doublePointsConfig = new PointsConfig(new DecimalFormat(), Double.class); Map pointsConfigMap = new HashMap<>(); @@ -161,11 +165,11 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina { case Flag, String, Character -> stringQueryParserFields.add(field); case Float -> { - numericQueryParserFields.add(field); + numericQueryParserFields.put(field, SortField.Type.DOUBLE); pointsConfigMap.put(field, doublePointsConfig); } case Integer -> { - numericQueryParserFields.add(field); + numericQueryParserFields.put(field, SortField.Type.INT); pointsConfigMap.put(field, intPointsConfig); } } @@ -182,14 +186,14 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); - if (searchString.equals("all")) { + if (searchString.equals(ALL_DOCS)) { booleanQueryBuilder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST); } // Split input into tokens, 1 token per query separated by & StringTokenizer tokenizer = new StringTokenizer(searchString, "&"); - while (tokenizer.hasMoreTokens() && !searchString.equals("all")) + while (tokenizer.hasMoreTokens() && !searchString.equals(ALL_DOCS)) { String queryString = tokenizer.nextToken(); Query query = null; @@ -205,7 +209,7 @@ public JSONObject doSearch(User u, String searchString, final int pageSize, fina { query = queryParser.parse(queryString); } - else if (numericQueryParserFields.contains(fieldName)) + else if (numericQueryParserFields.containsKey(fieldName)) { try { @@ -226,16 +230,28 @@ else if (numericQueryParserFields.contains(fieldName)) BooleanQuery query = booleanQueryBuilder.build(); + // By default, sort in INDEXORDER, which is by genomicPosition + Sort sort = Sort.INDEXORDER; + + // If the sort field is not genomicPosition, use the provided sorting data + if (!sortField.equals(GENOMIC_POSITION)) { + SortField.Type fieldType; + + if (stringQueryParserFields.contains(sortField)) { + fieldType = SortField.Type.STRING; + } else if (numericQueryParserFields.containsKey(sortField)) { + fieldType = numericQueryParserFields.get(sortField); + } else { + throw new IllegalArgumentException("Could not find type for sort field: " + sortField); + } + + sort = new Sort(new SortField(sortField, fieldType, sortReverse)); + } + // Get chunks of size {pageSize}. Default to 1 chunk -- add to the offset to get more. // We then iterate over the range of documents we want based on the offset. This does grow in memory // linearly with the number of documents, but my understanding is that these are just score,id pairs // rather than full documents, so mem usage *should* still be pretty low. - //TopDocs topDocs = indexSearcher.search(query, pageSize * (offset + 1)); - - // Define sort field - SortField sortField = new SortField("pos", SortField.Type.INT, false); - Sort sort = new Sort(sortField); - // Perform the search with sorting TopFieldDocs topDocs = indexSearcher.search(query, pageSize * (offset + 1), sort); @@ -253,10 +269,8 @@ else if (numericQueryParserFields.contains(fieldName)) String fieldName = field.name(); String[] fieldValues = doc.getValues(fieldName); if (fieldValues.length > 1) { - // If there is more than one value, put the array of values into the JSON object. elem.put(fieldName, fieldValues); } else { - // If there is only one value, just put this single value into the JSON object. elem.put(fieldName, fieldValues[0]); } } diff --git a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java index 82a8098c4..948810736 100644 --- a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java +++ b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java @@ -564,6 +564,7 @@ private void testFullTextSearch() throws Exception String sessionId = info.getKey(); String trackId = info.getValue(); + // all // this should return 143 results. We can't make any other assumptions about the content String url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=143"; @@ -575,7 +576,6 @@ private void testFullTextSearch() throws Exception JSONArray jsonArray = mainJsonObject.getJSONArray("data"); Assert.assertEquals(143, jsonArray.length()); - // stringType: // ref equals A url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=ref%3AA"; @@ -1145,6 +1145,91 @@ private void testFullTextSearch() throws Exception Assert.assertEquals("A", jsonObject.getString("ref")); } + // Default genomic position sort (ascending) + url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100"; + beginAt(url); + waitForText("data"); + waitAndClick(Locator.tagWithId("a", "rawdata-tab")); + jsonString = getText(Locator.tagWithClass("pre", "data")); + mainJsonObject = new JSONObject(jsonString); + jsonArray = mainJsonObject.getJSONArray("data"); + + long previousGenomicPosition = Long.MIN_VALUE; + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + long currentGenomicPosition = jsonObject.getLong("genomicPosition"); + Assert.assertTrue(currentGenomicPosition >= previousGenomicPosition); + previousGenomicPosition = currentGenomicPosition; + } + + // Sort by alt, ascending + url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=alt"; + beginAt(url); + waitForText("data"); + waitAndClick(Locator.tagWithId("a", "rawdata-tab")); + jsonString = getText(Locator.tagWithClass("pre", "data")); + mainJsonObject = new JSONObject(jsonString); + jsonArray = mainJsonObject.getJSONArray("data"); + + String previousAlt = ""; + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + String currentAlt = jsonObject.getString("alt"); + Assert.assertTrue(currentAlt.compareTo(previousAlt) >= 0); + previousAlt = currentAlt; + } + + // Sort by alt, descending + url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=alt&sortReverse=true"; + beginAt(url); + waitForText("data"); + waitAndClick(Locator.tagWithId("a", "rawdata-tab")); + jsonString = getText(Locator.tagWithClass("pre", "data")); + mainJsonObject = new JSONObject(jsonString); + jsonArray = mainJsonObject.getJSONArray("data"); + + previousAlt = "ZZZZ"; // Assuming 'Z' is higher than any character in your data + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + String currentAlt = jsonObject.getString("alt"); + Assert.assertTrue(currentAlt.compareTo(previousAlt) <= 0); + previousAlt = currentAlt; + } + + // Sort by af, ascending + url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=AF"; + beginAt(url); + waitForText("data"); + waitAndClick(Locator.tagWithId("a", "rawdata-tab")); + jsonString = getText(Locator.tagWithClass("pre", "data")); + mainJsonObject = new JSONObject(jsonString); + jsonArray = mainJsonObject.getJSONArray("data"); + + double previousAf = -1.0; + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + double currentAf = jsonObject.getDouble("AF"); + Assert.assertTrue(currentAf >= previousAf); + previousAf = currentAf; + } + + // Sort by af, descending + url = "/jbrowse/" + getProjectName() + "/luceneQuery.view?sessionId=" + sessionId + "&trackId=" + trackId + "&searchString=all&pageSize=100&sortField=AF&sortReverse=true"; + beginAt(url); + waitForText("data"); + waitAndClick(Locator.tagWithId("a", "rawdata-tab")); + jsonString = getText(Locator.tagWithClass("pre", "data")); + mainJsonObject = new JSONObject(jsonString); + jsonArray = mainJsonObject.getJSONArray("data"); + + previousAf = 2.0; // Assuming 'af' is <= 1.0 + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + double currentAf = jsonObject.getDouble("AF"); + Assert.assertTrue(currentAf <= previousAf); + previousAf = currentAf; + } + testLuceneSearchUI(sessionId); } diff --git a/singlecell/resources/chunks/CalculateGeneComponentScores.R b/singlecell/resources/chunks/CalculateGeneComponentScores.R index f49d12043..5471d17f0 100644 --- a/singlecell/resources/chunks/CalculateGeneComponentScores.R +++ b/singlecell/resources/chunks/CalculateGeneComponentScores.R @@ -2,7 +2,14 @@ for (datasetId in names(seuratObjects)) { printName(datasetId) seuratObj <- readSeuratRDS(seuratObjects[[datasetId]]) + if (ncol(seuratObj) == 1) { + print('Object has a single cell, skipping') + rm(seuratObj) + next + } + for (sc in savedComponent) { + logger::log_info(paste0('Processing ', datasetId, ' for ', sc)) seuratObj <- RIRA::ScoreUsingSavedComponent(seuratObj, componentOrName = sc, fieldName = sc) } diff --git a/singlecell/resources/chunks/CheckExpectations.R b/singlecell/resources/chunks/CheckExpectations.R index e95d91063..a82c2ac31 100644 --- a/singlecell/resources/chunks/CheckExpectations.R +++ b/singlecell/resources/chunks/CheckExpectations.R @@ -1,3 +1,13 @@ +CheckField <- function(seuratObj, datasetId, fieldName, errorOnNA = TRUE) { + if (!fieldName %in% names(seuratObj@meta.data)) { + addErrorMessage(paste0(paste0('Missing ', fieldName, ' for dataset: ', datasetId))) + } + + if (errorOnNA && any(is.na(seuratObj@meta.data[[fieldName]]))) { + addErrorMessage(paste0(paste0('NA values found for ', fieldName, ' for dataset: ', datasetId))) + } +} + for (datasetId in names(seuratObjects)) { printName(datasetId) seuratObj <- readSeuratRDS(seuratObjects[[datasetId]]) @@ -28,16 +38,20 @@ for (datasetId in names(seuratObjects)) { } } - if (requireSaturation && !'Saturation.RNA' %in% names(seuratObj@meta.data)) { - addErrorMessage(paste0('Missing per-cell RNA saturation data for dataset: ', datasetId)) + if (requireSaturation) { + CheckField(seuratObj, datasetId, 'Saturation.RNA') + } + + if (requireSingleR) { + CheckField(seuratObj, datasetId, 'SingleRConsensus') } - if (requireSingleR && !'SingleRConsensus' %in% names(seuratObj@meta.data)) { - addErrorMessage(paste0('Missing SingleRConsensus label for dataset: ', datasetId)) + if (requireScGate) { + CheckField(seuratObj, datasetId, 'scGateConsensus', errorOnNA = FALSE) } - if (requireScGate && !'scGateConsensus' %in% names(seuratObj@meta.data)) { - addErrorMessage(paste0('Missing scGateConsensus label for dataset: ', datasetId)) + if (requireRiraImmune) { + CheckField(seuratObj, datasetId, 'RIRA_Immune_v2.cellclass') } if (length(errorMessages) > 0) { diff --git a/singlecell/resources/chunks/Functions.R b/singlecell/resources/chunks/Functions.R index 86ee68ed8..7b00292fc 100644 --- a/singlecell/resources/chunks/Functions.R +++ b/singlecell/resources/chunks/Functions.R @@ -173,12 +173,21 @@ readSeuratRDS <- function(filePath) { return(seuratObj) } -print('Updating future.globals.maxSize') -options(future.globals.maxSize = Inf) - +options(future.globals.maxSize = Inf, future.globals.onReference = "warning") options('Seurat.memsafe' = TRUE) if (Sys.getenv('SEURAT_MAX_THREADS') != '') { - print(paste0('Setting future::plan workers to: ', Sys.getenv('SEURAT_MAX_THREADS'))) - future::plan(strategy='multisession', workers=Sys.getenv('SEURAT_MAX_THREADS')) + logger::log_info(paste0('Setting future::plan workers to: ', Sys.getenv('SEURAT_MAX_THREADS'))) + mt <- as.integer(Sys.getenv('SEURAT_MAX_THREADS')) + if (is.na(mt)) { + stop(paste0('SEURAT_MAX_THREAD is not an integer: ', mt <- Sys.getenv('SEURAT_MAX_THREADS'))) + } + + fs <- ifelse(exists('futureStrategy'), yes = get('futureStrategy'), no = 'multicore') + if (mt == 1) { + fs <- 'sequential' + } + logger::log_info(paste0('Setting future::strategy to: ', fs)) + + future::plan(strategy = fs, workers = mt) } diff --git a/singlecell/resources/chunks/SeuratPrototype.R b/singlecell/resources/chunks/SeuratPrototype.R index f03ab5403..47ed6621e 100644 --- a/singlecell/resources/chunks/SeuratPrototype.R +++ b/singlecell/resources/chunks/SeuratPrototype.R @@ -1,3 +1,13 @@ +CheckField <- function(seuratObj, datasetId, fieldName, errorOnNA = TRUE) { + if (!fieldName %in% names(seuratObj@meta.data)) { + addErrorMessage(paste0(paste0('Missing ', fieldName, ' for dataset: ', datasetId))) + } + + if (errorOnNA && any(is.na(seuratObj@meta.data[[fieldName]]))) { + addErrorMessage(paste0(paste0('NA values found for ', fieldName, ' for dataset: ', datasetId))) + } +} + metricData <- data.frame(dataId = integer(), readsetId = integer(), metricname = character(), metricvalue = numeric()) for (datasetId in names(seuratObjects)) { @@ -72,12 +82,12 @@ for (datasetId in names(seuratObjects)) { metricData <- rbind(metricData, data.frame(dataId = datasetId, readsetId = datasetIdToReadset[[datasetId]], metricname = 'MeanSaturation.RNA', metricvalue = meanSaturation.RNA)) } - if (requireSingleR && !'SingleRConsensus' %in% names(seuratObj@meta.data)) { - addErrorMessage(paste0('Missing SingleRConsensus label for dataset: ', datasetId)) + if (requireSingleR) { + CheckField(seuratObj, datasetId, 'SingleRConsensus') } - if (requireScGate && !'scGateConsensus' %in% names(seuratObj@meta.data)) { - addErrorMessage(paste0('Missing scGateConsensus label for dataset: ', datasetId)) + if (requireScGate) { + CheckField(seuratObj, datasetId, 'scGateConsensus', errorOnNA = FALSE) } if (length(errorMessages) > 0) { diff --git a/singlecell/resources/chunks/StudyMetadata.R b/singlecell/resources/chunks/StudyMetadata.R index 688478840..ee0726402 100644 --- a/singlecell/resources/chunks/StudyMetadata.R +++ b/singlecell/resources/chunks/StudyMetadata.R @@ -18,6 +18,8 @@ for (datasetId in names(seuratObjects)) { seuratObj <- Rdiscvr::ApplyMalariaMetadata(seuratObj, errorIfUnknownIdsFound = errorIfUnknownIdsFound) } else if (studyName == 'PC531') { seuratObj <- Rdiscvr::ApplyPC531Metadata(seuratObj, errorIfUnknownIdsFound = errorIfUnknownIdsFound) + } else if (studyName == 'AcuteNx') { + seuratObj <- Rdiscvr::ApplyAcuteNxMetadata(seuratObj, errorIfUnknownIdsFound = errorIfUnknownIdsFound) } else { stop(paste0('Unknown study: ', studyName)) } diff --git a/singlecell/resources/chunks/SubsetSeurat.R b/singlecell/resources/chunks/SubsetSeurat.R index 028bbe4d6..fddd46294 100644 --- a/singlecell/resources/chunks/SubsetSeurat.R +++ b/singlecell/resources/chunks/SubsetSeurat.R @@ -12,8 +12,13 @@ for (datasetId in names(seuratObjects)) { if (!is.null(seuratObj)) { - saveData(seuratObj, datasetId) - totalPassed <- totalPassed + 1 + if (ncol(seuratObj) <= 1) { + # NOTE: Seurat v5 assays do not perform well with one cell + print(paste0('There is only one cell remaining, skipping')) + } else { + saveData(seuratObj, datasetId) + totalPassed <- totalPassed + 1 + } } # Cleanup diff --git a/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js b/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js index 242b90c1f..da0d0a192 100644 --- a/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js +++ b/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js @@ -58,7 +58,8 @@ Ext4.define('SingleCell.panel.SingleCellProcessingPanel', { height: 100, helpPopup: 'Description for this analysis (optional)', name: 'jobDescription', - allowBlank:true + allowBlank:true, + value: LABKEY.ActionURL.getParameter('jobDescription') },{ xtype: 'combo', name: 'submissionType', diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java index 9a6327f18..fcffcac5a 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CheckExpectations.java @@ -33,7 +33,8 @@ public Provider() SeuratToolParameter.create("requireCiteSeq", "Require Cite-Seq, If Used", "If this dataset uses CITE-seq, cite-seq data are required", "checkbox", null, true), SeuratToolParameter.create("requireSaturation", "Require Per-Cell Saturation", "If this dataset uses TCR sequencing, these data are required", "checkbox", null, true), SeuratToolParameter.create("requireSingleR", "Require SingleR", "If checked, SingleR calls, including singleRConsensus are required to pass", "checkbox", null, true), - SeuratToolParameter.create("requireScGate", "Require scGate", "If checked, scGateConsensus calls are required to pass", "checkbox", null, true) + SeuratToolParameter.create("requireScGate", "Require scGate", "If checked, scGateConsensus calls are required to pass", "checkbox", null, true), + SeuratToolParameter.create("requireRiraImmune", "Require RIRA Immune V2", "If checked, RIRA_Immune_v2 calls (field RIRA_Immune_v2.cellclass) are required to pass", "checkbox", null, true) ), null, null); } diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/StudyMetadata.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/StudyMetadata.java index f9061672e..666250a2a 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/StudyMetadata.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/StudyMetadata.java @@ -24,7 +24,7 @@ public Provider() {{ put("multiSelect", false); put("allowBlank", false); - put("storeValues", "PC475;PC531;TB;Malaria"); + put("storeValues", "PC475;PC531;TB;Malaria;AcuteNx"); put("delimiter", ";"); }}, null, null, false, false), SeuratToolParameter.create("errorIfUnknownIdsFound", "Error If Unknown Ids Found", "If true, the job will fail if the seurat object contains ID not present in the metadata", "checkbox", null, true) diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java index 9dbc0a864..c3e432cf8 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/VireoHandler.java @@ -3,11 +3,13 @@ import htsjdk.samtools.util.IOUtil; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; import org.json.JSONObject; import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedAction; +import org.labkey.api.sequenceanalysis.SequenceAnalysisService; import org.labkey.api.sequenceanalysis.SequenceOutputFile; import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; @@ -15,7 +17,9 @@ import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.api.sequenceanalysis.run.AbstractGatk4Wrapper; import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper; +import org.labkey.api.util.PageFlowUtil; import org.labkey.api.writer.PrintWriters; import org.labkey.singlecell.SingleCellModule; import org.labkey.singlecell.run.CellRangerGexCountStep; @@ -26,19 +30,33 @@ import java.io.PrintWriter; import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedHashSet; import java.util.List; public class VireoHandler extends AbstractParameterizedOutputHandler { + private static final String REF_VCF = "refVCF"; + public VireoHandler() { - super(ModuleLoader.getInstance().getModule(SingleCellModule.class), "Run Vireo", "This will run cellsnp-lite and vireo to infer cell-to-sample based on genotype.", null, Arrays.asList( + super(ModuleLoader.getInstance().getModule(SingleCellModule.class), "Run CellSnp-Lite/Vireo", "This will run cellsnp-lite and vireo to infer cell-to-sample based on genotype.", new LinkedHashSet<>(PageFlowUtil.set("sequenceanalysis/field/SequenceOutputFileSelectorField.js")), Arrays.asList( ToolParameterDescriptor.create("nDonors", "# Donors", "The number of donors to demultiplex", "ldk-integerfield", new JSONObject(){{ put("allowBlank", false); }}, null), + ToolParameterDescriptor.create("maxDepth", "Max Depth", "At a position, read maximally INT reads per input file, to avoid excessive memory usage", "ldk-integerfield", new JSONObject(){{ + put("minValue", 0); + }}, null), ToolParameterDescriptor.create("contigs", "Allowable Contigs", "A comma-separated list of contig names to use", "textfield", new JSONObject(){{ - }}, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20") + }}, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20"), + ToolParameterDescriptor.createExpDataParam(REF_VCF, "Reference SNV Sites", "If provided, these sites will be used to screen for SNPs, instead of discovering them. If provided, the contig list will be ignored", "sequenceanalysis-sequenceoutputfileselectorfield", new JSONObject() + {{ + put("allowBlank", true); + put("category", "VCF File"); + put("performGenomeFilter", false); + put("doNotIncludeInTemplates", true); + }}, null), + ToolParameterDescriptor.create("storeCellSnpVcf", "Store CellSnp-Lite VCF", "If checked, the cellsnp donor calls VCF will be stored as an output file", "checkbox", null, false) )); } @@ -72,7 +90,7 @@ public SequenceOutputProcessor getProcessor() return new Processor(); } - public class Processor implements SequenceOutputProcessor + public static class Processor implements SequenceOutputProcessor { @Override public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException @@ -148,6 +166,7 @@ public void processFilesRemote(List inputFiles, JobContext c cellsnp.add(bam.getPath()); cellsnp.add("-b"); cellsnp.add(barcodes.getPath()); + cellsnp.add("--genotype"); File cellsnpDir = new File(ctx.getWorkingDirectory(), "cellsnp"); if (cellsnpDir.exists()) @@ -178,6 +197,13 @@ public void processFilesRemote(List inputFiles, JobContext c cellsnp.add("--minCOUNT"); cellsnp.add("100"); + String maxDepth = StringUtils.trimToNull(ctx.getParams().optString("maxDepth")); + if (maxDepth != null) + { + cellsnp.add("--maxDEPTH"); + cellsnp.add(maxDepth); + } + cellsnp.add("--gzip"); ReferenceGenome genome = ctx.getSequenceSupport().getCachedGenome(inputFiles.get(0).getLibrary_id()); @@ -185,11 +211,26 @@ public void processFilesRemote(List inputFiles, JobContext c cellsnp.add("--refseq"); cellsnp.add(genome.getWorkingFastaFile().getPath()); - String contigs = ctx.getParams().optString("contigs", ""); - if (!StringUtils.isEmpty(contigs)) + int vcfFile = ctx.getParams().optInt(REF_VCF, -1); + if (vcfFile > -1) + { + File vcf = ctx.getSequenceSupport().getCachedData(vcfFile); + if (vcf == null || ! vcf.exists()) + { + throw new PipelineJobException("Unable to find file with ID: " + vcfFile); + } + + cellsnp.add("-R"); + cellsnp.add(vcf.getPath()); + } + else { - cellsnp.add("--chrom"); - cellsnp.add(contigs); + String contigs = ctx.getParams().optString("contigs", ""); + if (!StringUtils.isEmpty(contigs)) + { + cellsnp.add("--chrom"); + cellsnp.add(contigs); + } } new SimpleScriptWrapper(ctx.getLogger()).execute(cellsnp); @@ -209,6 +250,7 @@ public void processFilesRemote(List inputFiles, JobContext c vireo.add(ctx.getWorkingDirectory().getPath()); int nDonors = ctx.getParams().optInt("nDonors", 0); + boolean storeCellSnpVcf = ctx.getParams().optBoolean("storeCellSnpVcf", false); if (nDonors == 0) { throw new PipelineJobException("Must provide nDonors"); @@ -217,32 +259,144 @@ public void processFilesRemote(List inputFiles, JobContext c vireo.add("-N"); vireo.add(String.valueOf(nDonors)); - new SimpleScriptWrapper(ctx.getLogger()).execute(vireo); + if (nDonors == 1) + { + storeCellSnpVcf = true; + ctx.getLogger().info("nDonor was 1, skipping vireo"); + } + else + { + new SimpleScriptWrapper(ctx.getLogger()).execute(vireo); - File[] outFiles = ctx.getWorkingDirectory().listFiles(f -> f.getName().endsWith("_donor_ids.tsv")); - if (outFiles == null || outFiles.length == 0) + File[] outFiles = ctx.getWorkingDirectory().listFiles(f -> f.getName().endsWith("donor_ids.tsv")); + if (outFiles == null || outFiles.length == 0) + { + throw new PipelineJobException("Missing vireo output file"); + } + else if (outFiles.length > 1) + { + throw new PipelineJobException("More than one possible vireo output file found"); + } + + SequenceOutputFile so = new SequenceOutputFile(); + so.setReadset(inputFiles.get(0).getReadset()); + so.setLibrary_id(inputFiles.get(0).getLibrary_id()); + so.setFile(outFiles[0]); + if (so.getReadset() != null) + { + so.setName(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName() + ": Vireo Demultiplexing"); + } + else + { + so.setName(inputFiles.get(0).getName() + ": Vireo Demultiplexing"); + } + so.setCategory("Vireo Demultiplexing"); + ctx.addSequenceOutput(so); + } + + File cellSnpBaseVcf = new File(cellsnpDir, "cellSNP.base.vcf.gz"); + if (!cellSnpBaseVcf.exists()) { - throw new PipelineJobException("Missing vireo output file"); + throw new PipelineJobException("Unable to find cellsnp base VCF"); } - else if (outFiles.length > 1) + + + File cellSnpCellsVcf = new File(cellsnpDir, "cellSNP.cells.vcf.gz"); + if (!cellSnpCellsVcf.exists()) { - throw new PipelineJobException("More than one possible vireo output file found"); + throw new PipelineJobException("Unable to find cellsnp calls VCF"); } - SequenceOutputFile so = new SequenceOutputFile(); - so.setReadset(inputFiles.get(0).getReadset()); - so.setLibrary_id(inputFiles.get(0).getLibrary_id()); - so.setFile(outFiles[0]); - if (so.getReadset() != null) + sortAndFixVcf(cellSnpBaseVcf, genome, ctx.getLogger()); + sortAndFixVcf(cellSnpCellsVcf, genome, ctx.getLogger()); + + if (storeCellSnpVcf) { - so.setName(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName() + ": Vireo Demultiplexing"); + SequenceOutputFile so = new SequenceOutputFile(); + so.setReadset(inputFiles.get(0).getReadset()); + so.setLibrary_id(inputFiles.get(0).getLibrary_id()); + so.setFile(cellSnpCellsVcf); + if (so.getReadset() != null) + { + so.setName(ctx.getSequenceSupport().getCachedReadset(so.getReadset()).getName() + ": Cellsnp-lite VCF"); + } + else + { + so.setName(inputFiles.get(0).getName() + ": Cellsnp-lite VCF"); + } + so.setCategory("VCF File"); + ctx.addSequenceOutput(so); } - else + } + + private void sortAndFixVcf(File vcf, ReferenceGenome genome, Logger log) throws PipelineJobException + { + // This original VCF is generally not properly sorted, and has an invalid index. This is redundant, the VCF is not that large: + try + { + SequencePipelineService.get().sortROD(vcf, log, 2); + SequenceAnalysisService.get().ensureVcfIndex(vcf, log, true); + + new UpdateVCFSequenceDictionary(log).execute(vcf, genome.getSequenceDictionary()); + SequenceAnalysisService.get().ensureVcfIndex(vcf, log); + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + } + + public static class UpdateVCFSequenceDictionary extends AbstractGatk4Wrapper + { + public UpdateVCFSequenceDictionary(Logger log) + { + super(log); + } + + public void execute(File vcf, File dict) throws PipelineJobException { - so.setName(inputFiles.get(0).getName() + ": Vireo Demultiplexing"); + List args = new ArrayList<>(getBaseArgs("UpdateVCFSequenceDictionary")); + args.add("-V"); + args.add(vcf.getPath()); + + args.add("--source-dictionary"); + args.add(dict.getPath()); + + args.add("--replace"); + args.add("true"); + + File output = new File(vcf.getParentFile(), "tmp.vcf.gz"); + args.add("-O"); + args.add(output.getPath()); + + execute(args); + + if (!output.exists()) + { + throw new PipelineJobException("Unable to find file: " + output.getPath()); + } + + try + { + // replace original: + vcf.delete(); + FileUtils.moveFile(output, vcf); + + File outputIdx = new File(output.getPath() + ".tbi"); + File vcfIdx = new File(vcf.getPath() + ".tbi"); + if (vcfIdx.exists()) + { + vcfIdx.delete(); + } + + FileUtils.moveFile(outputIdx, vcfIdx); + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + } - so.setCategory("Vireo Demultiplexing"); - ctx.addSequenceOutput(so); } } }