diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/BcftoolsRunner.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/BcftoolsRunner.java index 91a6d3937..0767a7471 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/BcftoolsRunner.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/BcftoolsRunner.java @@ -18,7 +18,7 @@ public BcftoolsRunner(@Nullable Logger logger) super(logger); } - public File getBcfToolsPath() + public static File getBcfToolsPath() { return SequencePipelineService.get().getExeForPackage("BCFTOOLSPATH", "bcftools"); } diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/TaskFileManager.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/TaskFileManager.java index 6263c7994..beecdcb51 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/TaskFileManager.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/TaskFileManager.java @@ -56,6 +56,8 @@ public interface TaskFileManager extends PipelineOutputTracker boolean isDeleteIntermediateFiles(); + public boolean performCleanupAfterEachStep(); + boolean isCopyInputsLocally(); void addPicardMetricsFiles(List files) throws PipelineJobException; diff --git a/SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets/SRA Info.qview.xml b/SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets/SRA Info.qview.xml index dd3328383..10a6f3edf 100644 --- a/SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets/SRA Info.qview.xml +++ b/SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets/SRA Info.qview.xml @@ -17,6 +17,8 @@ + + diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js index 53132b175..73a786c9d 100644 --- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js +++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js @@ -94,6 +94,14 @@ Ext4.define('SequenceAnalysis.panel.AlignmentImportPanel', { inputValue: true, checked: true, xtype: 'checkbox' + },{ + fieldLabel: 'Perform Cleanup After Each Step', + helpPopup: 'Is selected, intermediate files from this job will be deleted after each step, instead of once at the end of the job. This can reduce the working directory size. Note: this will only apply if deleteIntermediateFiles is selected, and this is not supported across every possible pipeline type.', + name: 'performCleanupAfterEachStep', + inputValue: true, + uncheckedValue: false, + checked: true, + xtype: 'checkbox' },{ fieldLabel: 'Treatment of Input Files', xtype: 'combo', diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js index 42e7bec99..bd3a422dd 100644 --- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js +++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js @@ -309,6 +309,14 @@ Ext4.define('SequenceAnalysis.panel.SequenceAnalysisPanel', { uncheckedValue: false, checked: true, xtype: 'checkbox' + },{ + fieldLabel: 'Perform Cleanup After Each Step', + helpPopup: 'Is selected, intermediate files from this job will be deleted after each step, instead of once at the end of the job. This can reduce the working directory size. Note: this will only apply if deleteIntermediateFiles is selected, and this is not supported across every possible pipeline type.', + name: 'performCleanupAfterEachStep', + inputValue: true, + uncheckedValue: false, + checked: true, + xtype: 'checkbox' },{ fieldLabel: 'Copy Inputs To Working Directory?', helpPopup: 'Check to copy the input files to the working directory. Depending on your environment, this may or may not help performance.', diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js index c76c39f89..9cf64ff68 100644 --- a/SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js +++ b/SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js @@ -74,6 +74,14 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', { inputValue: true, checked: true, xtype: 'checkbox' + },{ + fieldLabel: 'Perform Cleanup After Each Step', + helpPopup: 'Is selected, intermediate files from this job will be deleted after each step, instead of once at the end of the job. This can reduce the working directory size. Note: this will only apply if deleteIntermediateFiles is selected, and this is not supported across every possible pipeline type.', + name: 'performCleanupAfterEachStep', + inputValue: true, + uncheckedValue: false, + checked: true, + xtype: 'checkbox' }, this.getSaveTemplateCfg()] }; }, diff --git a/SequenceAnalysis/resources/web/SequenceAnalysis/window/ArchiveReadsetsWindow.js b/SequenceAnalysis/resources/web/SequenceAnalysis/window/ArchiveReadsetsWindow.js new file mode 100644 index 000000000..ca1514d1a --- /dev/null +++ b/SequenceAnalysis/resources/web/SequenceAnalysis/window/ArchiveReadsetsWindow.js @@ -0,0 +1,69 @@ +Ext4.define('SequenceAnalysis.window.ArchiveReadsetsWindow', { + extend: 'Ext.window.Window', + + statics: { + buttonHandler: function(dataRegionName){ + Ext4.create('SequenceAnalysis.window.ArchiveReadsetsWindow', { + dataRegionName: dataRegionName, + readsetIds: LABKEY.DataRegions[dataRegionName].getChecked() + }).show(); + } + }, + + initComponent: function() { + Ext4.apply(this, { + modal: true, + title: 'Archive Readsets', + width: 600, + bodyStyle: 'padding: 5px;', + defaults: { + border: false + }, + items: [{ + html: 'This helper will delete the actual FASTQ files associated with the selected readsets. It will error unless each readdata row has an SRA accession listed. You selected ' + this.readsetIds.length + ' readsets.', + style: 'padding-bottom: 10px;' + }], + buttons: [{ + text: 'Submit', + scope: this, + handler: this.onSubmit + },{ + text: 'Cancel', + handler: function(btn){ + btn.up('window').close(); + } + }] + }); + + this.callParent(arguments); + }, + + onSubmit: function(btn){ + if (!this.readsetIds.length) { + Ext4.Msg.alert('Error', 'No readsets selected!'); + return; + } + + Ext4.Msg.wait('Saving...'); + LABKEY.Ajax.request({ + url: LABKEY.ActionURL.buildURL('sequenceanalysis', 'archiveReadsets', null), + method: 'POST', + jsonData: { + readsetIds: this.readsetIds + }, + scope: this, + success: function(){ + Ext4.Msg.hide(); + this.close(); + Ext4.Msg.alert('Success', 'Readsets archived!', function(){ + if (this.dataRegionName){ + LABKEY.DataRegions[this.dataRegionName].clearSelected(); + } + + LABKEY.DataRegions[this.dataRegionName].refresh(); + }, this); + }, + failure: LABKEY.Utils.getCallbackWrapper(LDK.Utils.getErrorCallback()) + }); + } +}); \ No newline at end of file diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java index d29e0a4b6..f40f73ba0 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java @@ -77,6 +77,7 @@ import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.files.FileContentService; import org.labkey.api.laboratory.NavItem; +import org.labkey.api.laboratory.security.LaboratoryAdminPermission; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleHtmlView; import org.labkey.api.module.ModuleLoader; @@ -3297,6 +3298,15 @@ public ApiResponse execute(CheckFileStatusForm form, BindException errors) toolArr.put(intermediateFiles); + JSONObject performCleanupAfterEachStep = new JSONObject(); + performCleanupAfterEachStep.put("name", "performCleanupAfterEachStep"); + performCleanupAfterEachStep.put("defaultValue", true); + performCleanupAfterEachStep.put("label", "Perform Cleanup After Each Step"); + performCleanupAfterEachStep.put("description", "Is selected, intermediate files from this job will be deleted after each step, instead of once at the end of the job. This can reduce the working directory size. Note: this will only apply if deleteIntermediateFiles is selected, and this is not supported across every possible pipeline type."); + performCleanupAfterEachStep.put("fieldXtype", "checkbox"); + + toolArr.put(performCleanupAfterEachStep); + ret.put("toolParameters", toolArr); ret.put("description", handler.getDescription()); @@ -5128,4 +5138,157 @@ public void setDataFileUrl(String dataFileUrl) _dataFileUrl = dataFileUrl; } } + + @RequiresPermission(UpdatePermission.class) + public static class ArchiveReadsetsAction extends MutatingApiAction + { + @Override + public ApiResponse execute(ArchiveReadsetsForm form, BindException errors) throws Exception + { + if (form.getReadsetIds() == null || form.getReadsetIds().length == 0) + { + errors.reject(ERROR_MSG, "No readset Ids provided"); + return null; + } + + TableInfo readData = QueryService.get().getUserSchema(getUser(), getContainer(), SequenceAnalysisSchema.SCHEMA_NAME).getTable(SequenceAnalysisSchema.TABLE_READ_DATA); + for (int readsetId : form.getReadsetIds()) + { + Readset rs = SequenceAnalysisService.get().getReadset(readsetId, getUser()); + Container c = ContainerManager.getForId(rs.getContainer()); + if (!getContainer().equals(c)) + { + Container toTest = c.isWorkbook() ? c.getParent() : c; + if (!getContainer().equals(toTest)) + { + errors.reject(ERROR_MSG, "Readset is not from this container: " + readsetId); + return null; + } + } + + if (!c.hasPermission(getUser(), LaboratoryAdminPermission.class)) + { + errors.reject(ERROR_MSG, "Insufficient permissions to archive readsets in the folder: " + c.getPath()); + return null; + } + + Set toDelete = new HashSet<>(); + List> toUpdate = new ArrayList<>(); + for (ReadData rd : rs.getReadData()) + { + if (rd.getSra_accession() == null) + { + errors.reject(ERROR_MSG, "Cannot mark a readdata as archived that does not have an SRA accession: " + readsetId + " / " + rd.getRowid()); + return null; + } + + toUpdate.add(new CaseInsensitiveHashMap<>(Map.of("rowid", rd.getRowid(), "archived", true, "container", rd.getContainer()))); + + // File 1: + ExpData d1 = ExperimentService.get().getExpData(rd.getFileId1()); + if (d1 != null) + { + File file1 = d1.getFile(); + if (file1 != null && file1.exists()) + { + toDelete.add(file1); + } + + // find matching readdata: + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("fileid1/dataFileUrl"), d1.getDataFileUrl()).addCondition(FieldKey.fromString("rowid"), rd.getRowid(), CompareType.NEQ); + TableSelector ts = new TableSelector(readData, PageFlowUtil.set("rowid", "container"), filter, null); + if (ts.exists()) + { + ts.forEachResults(r -> { + toUpdate.add(new CaseInsensitiveHashMap<>(Map.of("rowid", r.getInt(FieldKey.fromString("rowid")), "archived", true, "container", r.getString(FieldKey.fromString("container"))))); + }); + } + } + + if (rd.getFileId2() != null) + { + ExpData d2 = ExperimentService.get().getExpData(rd.getFileId2()); + if (d2 != null) + { + File file2 = d2.getFile(); + if (file2 != null) + { + if (file2.exists()) + { + toDelete.add(file2); + } + + // find matching readdata: + SimpleFilter filter = new SimpleFilter(FieldKey.fromString("fileid2/dataFileUrl"), d2.getDataFileUrl()).addCondition(FieldKey.fromString("rowid"), rd.getRowid(), CompareType.NEQ); + TableSelector ts = new TableSelector(readData, PageFlowUtil.set("rowid", "container"), filter, null); + if (ts.exists()) + { + ts.forEachResults(r -> { + toUpdate.add(new CaseInsensitiveHashMap<>(Map.of("rowid", r.getInt(FieldKey.fromString("rowid")), "archived", true, "container", r.getString(FieldKey.fromString("container"))))); + }); + } + } + } + } + } + + if (!toUpdate.isEmpty()) + { + List> keys = new ArrayList<>(); + toUpdate.forEach(row -> { + + keys.add(new CaseInsensitiveHashMap<>(Map.of("rowid", row.get("rowid")))); + }); + + try + { + readData.getUpdateService().updateRows(getUser(), getContainer(), toUpdate, keys, null, null); + } + catch (Exception e) + { + _log.error(e); + errors.reject(ERROR_MSG, "Error archiving readset: " + readsetId + ", " + e.getMessage()); + return null; + } + } + + if (!toDelete.isEmpty()) + { + for (File f : toDelete) + { + _log.info("Deleting archived file: " + f.getPath()); + f.delete(); + } + } + } + + return new ApiSimpleResponse("Success", true); + } + } + + public static class ArchiveReadsetsForm + { + private int[] _readsetIds; + private boolean _doNotRequireSra; + + public int[] getReadsetIds() + { + return _readsetIds; + } + + public void setReadsetIds(int... readsetIds) + { + _readsetIds = readsetIds; + } + + public boolean isDoNotRequireSra() + { + return _doNotRequireSra; + } + + public void setDoNotRequireSra(boolean doNotRequireSra) + { + _doNotRequireSra = doNotRequireSra; + } + } } \ No newline at end of file diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java index f392f46b5..950cae97b 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java @@ -45,6 +45,8 @@ import org.labkey.sequenceanalysis.analysis.BamHaplotypeHandler; import org.labkey.sequenceanalysis.analysis.CombineStarGeneCountsHandler; import org.labkey.sequenceanalysis.analysis.CombineSubreadGeneCountsHandler; +import org.labkey.sequenceanalysis.analysis.DeepVariantHandler; +import org.labkey.sequenceanalysis.analysis.GLNexusHandler; import org.labkey.sequenceanalysis.analysis.GenotypeGVCFHandler; import org.labkey.sequenceanalysis.analysis.HaplotypeCallerHandler; import org.labkey.sequenceanalysis.analysis.LiftoverHandler; @@ -58,6 +60,7 @@ import org.labkey.sequenceanalysis.analysis.SbtGeneCountHandler; import org.labkey.sequenceanalysis.analysis.UnmappedSequenceBasedGenotypeHandler; import org.labkey.sequenceanalysis.button.AddSraRunButton; +import org.labkey.sequenceanalysis.button.ArchiveReadsetsButton; import org.labkey.sequenceanalysis.button.ChangeReadsetStatusButton; import org.labkey.sequenceanalysis.button.ChangeReadsetStatusForAnalysesButton; import org.labkey.sequenceanalysis.button.DownloadSraButton; @@ -77,25 +80,7 @@ import org.labkey.sequenceanalysis.run.alignment.Pbmm2Wrapper; import org.labkey.sequenceanalysis.run.alignment.StarWrapper; import org.labkey.sequenceanalysis.run.alignment.VulcanWrapper; -import org.labkey.sequenceanalysis.run.analysis.BamIterator; -import org.labkey.sequenceanalysis.run.analysis.BcftoolsFillTagsStep; -import org.labkey.sequenceanalysis.run.analysis.ExportOverlappingReadsAnalysis; -import org.labkey.sequenceanalysis.run.analysis.GenrichStep; -import org.labkey.sequenceanalysis.run.analysis.HaplotypeCallerAnalysis; -import org.labkey.sequenceanalysis.run.analysis.ImmunoGenotypingAnalysis; -import org.labkey.sequenceanalysis.run.analysis.LofreqAnalysis; -import org.labkey.sequenceanalysis.run.analysis.MergeLoFreqVcfHandler; -import org.labkey.sequenceanalysis.run.analysis.NextCladeHandler; -import org.labkey.sequenceanalysis.run.analysis.PARalyzerAnalysis; -import org.labkey.sequenceanalysis.run.analysis.PangolinHandler; -import org.labkey.sequenceanalysis.run.analysis.PbsvAnalysis; -import org.labkey.sequenceanalysis.run.analysis.PbsvJointCallingHandler; -import org.labkey.sequenceanalysis.run.analysis.PindelAnalysis; -import org.labkey.sequenceanalysis.run.analysis.SequenceBasedTypingAnalysis; -import org.labkey.sequenceanalysis.run.analysis.SnpCountAnalysis; -import org.labkey.sequenceanalysis.run.analysis.SubreadAnalysis; -import org.labkey.sequenceanalysis.run.analysis.UnmappedReadExportHandler; -import org.labkey.sequenceanalysis.run.analysis.ViralAnalysis; +import org.labkey.sequenceanalysis.run.analysis.*; import org.labkey.sequenceanalysis.run.assembly.TrinityRunner; import org.labkey.sequenceanalysis.run.bampostprocessing.AddOrReplaceReadGroupsStep; import org.labkey.sequenceanalysis.run.bampostprocessing.BaseQualityScoreRecalibrator; @@ -280,6 +265,7 @@ public static void registerPipelineSteps() SequencePipelineService.get().registerPipelineStep(new ImmunoGenotypingAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new ViralAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new HaplotypeCallerAnalysis.Provider()); + SequencePipelineService.get().registerPipelineStep(new DeepVariantAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new SnpCountAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new ExportOverlappingReadsAnalysis.Provider()); SequencePipelineService.get().registerPipelineStep(new SubreadAnalysis.Provider()); @@ -346,6 +332,8 @@ public static void registerPipelineSteps() SequenceAnalysisService.get().registerFileHandler(new NextCladeHandler()); SequenceAnalysisService.get().registerFileHandler(new ConvertToCramHandler()); SequenceAnalysisService.get().registerFileHandler(new PbsvJointCallingHandler()); + SequenceAnalysisService.get().registerFileHandler(new DeepVariantHandler()); + SequenceAnalysisService.get().registerFileHandler(new GLNexusHandler()); SequenceAnalysisService.get().registerReadsetHandler(new MultiQCHandler()); SequenceAnalysisService.get().registerReadsetHandler(new RestoreSraDataHandler()); @@ -396,9 +384,10 @@ public void doStartupAfterSpringConfig(ModuleContext moduleContext) LDKService.get().registerQueryButton(new AddSraRunButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_READSETS); LDKService.get().registerQueryButton(new RunMultiQCButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_READSETS); LDKService.get().registerQueryButton(new DownloadSraButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_READSETS); + LDKService.get().registerQueryButton(new ArchiveReadsetsButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_READSETS); - LDKService.get().registerQueryButton(new ChangeReadsetStatusForAnalysesButton(), "sequenceanalysis", "sequence_analyses"); - LDKService.get().registerQueryButton(new ChangeReadsetStatusButton(), "sequenceanalysis", "sequence_readsets"); + LDKService.get().registerQueryButton(new ChangeReadsetStatusForAnalysesButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_ANALYSES); + LDKService.get().registerQueryButton(new ChangeReadsetStatusButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_READSETS); ExperimentService.get().registerExperimentRunTypeSource(new ExperimentRunTypeSource() { diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/DeepVariantHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/DeepVariantHandler.java new file mode 100644 index 000000000..6b3c3d6b8 --- /dev/null +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/DeepVariantHandler.java @@ -0,0 +1,162 @@ +package org.labkey.sequenceanalysis.analysis; + +import org.apache.commons.lang3.StringUtils; +import org.json.JSONObject; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedAction; +import org.labkey.api.sequenceanalysis.SequenceOutputFile; +import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; +import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; +import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; +import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; +import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.api.util.FileType; +import org.labkey.api.util.FileUtil; +import org.labkey.sequenceanalysis.SequenceAnalysisModule; +import org.labkey.sequenceanalysis.run.analysis.DeepVariantAnalysis; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +/** + * Created by bimber on 2/3/2016. + */ +public class DeepVariantHandler extends AbstractParameterizedOutputHandler +{ + private final FileType _bamOrCramFileType = new FileType(Arrays.asList("bam", "cram"), "bam"); + + public DeepVariantHandler() + { + super(ModuleLoader.getInstance().getModule(SequenceAnalysisModule.class), "Run DeepVariant", "This will run DeepVariant on the selected BAMs to generate gVCF files.", null, DeepVariantAnalysis.getToolDescriptors()); + } + + @Override + public boolean canProcess(SequenceOutputFile o) + { + return o.getFile() != null && _bamOrCramFileType.isType(o.getFile()); + } + + @Override + public boolean doRunRemote() + { + return true; + } + + @Override + public boolean doRunLocal() + { + return false; + } + + @Override + public SequenceOutputProcessor getProcessor() + { + return new Processor(); + } + + @Override + public boolean doSplitJobs() + { + return true; + } + + public class Processor implements SequenceOutputProcessor + { + @Override + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + { + String modelType = ctx.getParams().optString("modelType"); + DeepVariantAnalysis.inferModelType(modelType, ctx); + } + + @Override + public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException + { + PipelineJob job = ctx.getJob(); + if (inputFiles.size() != 1) + { + throw new PipelineJobException("Expected a single input file"); + } + + SequenceOutputFile so = inputFiles.get(0); + + RecordedAction action = new RecordedAction(getName()); + action.setStartTime(new Date()); + + action.addInput(so.getFile(), "Input BAM File"); + + File outputFile = new File(ctx.getOutputDir(), FileUtil.getBaseName(so.getFile()) + ".g.vcf.gz"); + + DeepVariantAnalysis.DeepVariantWrapper wrapper = new DeepVariantAnalysis.DeepVariantWrapper(job.getLogger()); + wrapper.setOutputDir(ctx.getOutputDir()); + + ReferenceGenome referenceGenome = ctx.getSequenceSupport().getCachedGenome(so.getLibrary_id()); + if (referenceGenome == null) + { + throw new PipelineJobException("No reference genome found for output: " + so.getRowid()); + } + + String inferredModelType = ctx.getSequenceSupport().getCachedObject("modelType", String.class); + String modelType = inferredModelType == null ? ctx.getParams().optString("modelType") : inferredModelType; + if (modelType == null) + { + throw new PipelineJobException("Missing model type"); + } + + List args = new ArrayList<>(getClientCommandArgs(ctx.getParams())); + args.add("--model_type=" + modelType); + + String binVersion = ctx.getParams().optString("binVersion"); + if (binVersion == null) + { + throw new PipelineJobException("Missing binVersion"); + } + + wrapper.execute(so.getFile(), referenceGenome.getWorkingFastaFile(), outputFile, ctx.getFileManager(), binVersion, args); + + action.addOutput(outputFile, "gVCF File", false); + + SequenceOutputFile o = new SequenceOutputFile(); + o.setName(outputFile.getName()); + o.setFile(outputFile); + o.setLibrary_id(so.getLibrary_id()); + o.setCategory("DeepVariant gVCF File"); + o.setReadset(so.getReadset()); + o.setDescription("DeepVariant Version: " + binVersion); + + ctx.addSequenceOutput(o); + + ctx.addActions(action); + } + + private List getClientCommandArgs(JSONObject params) + { + List ret = new ArrayList<>(); + + for (ToolParameterDescriptor desc : getParameters()) + { + if (desc.getCommandLineParam() != null) + { + String val = params.optString(desc.getName(), null); + if (StringUtils.trimToNull(val) != null) + { + ret.addAll(desc.getCommandLineParam().getArguments(" ", val)); + } + } + } + + return ret; + } + + @Override + public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + { + + } + } +} \ No newline at end of file diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/GLNexusHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/GLNexusHandler.java new file mode 100644 index 000000000..3cb14356d --- /dev/null +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/GLNexusHandler.java @@ -0,0 +1,311 @@ +package org.labkey.sequenceanalysis.analysis; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.json.JSONObject; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedAction; +import org.labkey.api.sequenceanalysis.SequenceOutputFile; +import org.labkey.api.sequenceanalysis.pipeline.AbstractParameterizedOutputHandler; +import org.labkey.api.sequenceanalysis.pipeline.BcftoolsRunner; +import org.labkey.api.sequenceanalysis.pipeline.PipelineOutputTracker; +import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; +import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; +import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; +import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; +import org.labkey.api.util.FileType; +import org.labkey.api.writer.PrintWriters; +import org.labkey.sequenceanalysis.SequenceAnalysisModule; +import org.labkey.sequenceanalysis.run.util.BgzipRunner; +import org.labkey.sequenceanalysis.util.SequenceUtil; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.labkey.sequenceanalysis.pipeline.ProcessVariantsHandler.VCF_CATEGORY; + +/** + * Created by bimber on 2/3/2016. + */ +public class GLNexusHandler extends AbstractParameterizedOutputHandler +{ + protected FileType _gvcfFileType = new FileType(List.of(".g.vcf"), ".g.vcf", false, FileType.gzSupportLevel.SUPPORT_GZ); + + public GLNexusHandler() + { + super(ModuleLoader.getInstance().getModule(SequenceAnalysisModule.class), "Run GLNexus", "This will run GLNexus on the selected gVCFs.", null, Arrays.asList( + ToolParameterDescriptor.create("binVersion", "GLNexus Version", "The version of GLNexus to run, which is passed to their docker container", "textfield", new JSONObject(){{ + put("allowBlank", false); + }}, "v1.2.7"), + ToolParameterDescriptor.create("configType", "Config Type", "This is passed to the --config argument of GLNexus.", "ldk-simplecombo", new JSONObject() + {{ + put("multiSelect", false); + put("allowBlank", false); + put("storeValues", "gatk;DeepVariant;DeepVariantWGS;DeepVariantWES"); + put("initialValues", "DeepVariant"); + put("delimiter", ";"); + put("joinReturnValue", true); + }}, null), + ToolParameterDescriptor.create("fileBaseName", "Filename", "This is the basename that will be used for the output gzipped VCF", "textfield", new JSONObject(){{ + put("allowBlank", false); + }}, "CombinedGenotypes") + )); + } + + @Override + public boolean canProcess(SequenceOutputFile o) + { + + return o.getFile() != null && _gvcfFileType.isType(o.getFile()); + } + + @Override + public boolean doRunRemote() + { + return true; + } + + @Override + public boolean doRunLocal() + { + return false; + } + + @Override + public SequenceOutputProcessor getProcessor() + { + return new Processor(); + } + + @Override + public boolean doSplitJobs() + { + return false; + } + + public class Processor implements SequenceOutputProcessor + { + @Override + public void init(JobContext ctx, List inputFiles, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + { + Set genomeIds = new HashSet<>(); + for (SequenceOutputFile so : inputFiles) + { + genomeIds.add(so.getLibrary_id()); + } + + if (genomeIds.size() > 1) + { + throw new PipelineJobException("The selected files use more than one genome"); + } + else if (genomeIds.isEmpty()) + { + throw new PipelineJobException("No genome ID found for inputs"); + } + } + + @Override + public void processFilesRemote(List inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException + { + RecordedAction action = new RecordedAction(getName()); + action.setStartTime(new Date()); + + Set genomeIds = new HashSet<>(); + List inputVcfs = new ArrayList<>(); + for (SequenceOutputFile so : inputFiles) + { + genomeIds.add(so.getLibrary_id()); + inputVcfs.add(so.getFile()); + action.addInput(so.getFile(), "Input gVCF File"); + } + + if (genomeIds.size() > 1) + { + throw new PipelineJobException("The selected files use more than one genome"); + } + else if (genomeIds.isEmpty()) + { + throw new PipelineJobException("No genome ID found for inputs"); + } + + int genomeId = genomeIds.iterator().next(); + + String basename = StringUtils.trimToNull(ctx.getParams().optString("fileBaseName")); + if (basename == null) + { + throw new PipelineJobException("Basename not supplied for output VCF"); + } + + String binVersion = ctx.getParams().optString("binVersion"); + if (binVersion == null) + { + throw new PipelineJobException("Missing binVersion"); + } + + String configType = ctx.getParams().optString("configType", "DeepVariant"); + if (configType == null) + { + throw new PipelineJobException("Missing configType"); + } + + File outputVcf = new File(ctx.getOutputDir(), basename + ".vcf.gz"); + + new GLNexusWrapper(ctx.getLogger()).execute(inputVcfs, outputVcf, ctx.getFileManager(), binVersion, configType); + + ctx.getLogger().debug("adding sequence output: " + outputVcf.getPath()); + SequenceOutputFile so1 = new SequenceOutputFile(); + so1.setName(outputVcf.getName()); + so1.setDescription("GLNexus output. Version: " + binVersion + ". Total samples: " + inputFiles.size()); + so1.setFile(outputVcf); + so1.setLibrary_id(genomeId); + so1.setCategory(VCF_CATEGORY); + so1.setContainer(ctx.getJob().getContainerId()); + so1.setCreated(new Date()); + so1.setModified(new Date()); + + ctx.getFileManager().addSequenceOutput(so1); + } + + @Override + public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException + { + + } + } + + public static class GLNexusWrapper extends AbstractCommandWrapper + { + public GLNexusWrapper(Logger logger) + { + super(logger); + } + + private File ensureLocalCopy(File input, File workingDirectory, PipelineOutputTracker output) throws PipelineJobException + { + try + { + if (workingDirectory.equals(input.getParentFile())) + { + return input; + } + + File local = new File(workingDirectory, input.getName()); + if (!local.exists()) + { + getLogger().debug("Copying file locally: " + input.getPath()); + FileUtils.copyFile(input, local); + } + + output.addIntermediateFile(local); + + return local; + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + } + + public void execute(List inputGvcfs, File outputVcf, PipelineOutputTracker tracker, String binVersion, String configType) throws PipelineJobException + { + File workDir = outputVcf.getParentFile(); + tracker.addIntermediateFile(outputVcf); + tracker.addIntermediateFile(new File(outputVcf.getPath() + ".tbi")); + + List gvcfsLocal = new ArrayList<>(); + for (File f : inputGvcfs) + { + gvcfsLocal.add(ensureLocalCopy(f, workDir, tracker)); + ensureLocalCopy(new File(f.getPath() + ".tbi"), workDir, tracker); + } + + File localBashScript = new File(workDir, "docker.sh"); + tracker.addIntermediateFile(localBashScript); + + try (PrintWriter writer = PrintWriters.getPrintWriter(localBashScript)) + { + writer.println("#!/bin/bash"); + writer.println("set -x"); + writer.println("WD=`pwd`"); + writer.println("HOME=`echo ~/`"); + writer.println("DOCKER='" + SequencePipelineService.get().getDockerCommand() + "'"); + writer.println("sudo $DOCKER pull quay.io/mlin/glnexus:" + binVersion); + writer.println("sudo $DOCKER run --rm=true \\"); + writer.println("\t-v \"${WD}:/work\" \\"); + writer.println("\t-v \"${HOME}:/homeDir\" \\"); + if (!StringUtils.isEmpty(System.getenv("TMPDIR"))) + { + writer.println("\t-v \"${TMPDIR}:/tmp\" \\"); + } + writer.println("\t-u $UID \\"); + writer.println("\t-e USERID=$UID \\"); + + Integer maxRam = SequencePipelineService.get().getMaxRam(); + if (maxRam != null) + { + writer.println("\t--memory='" + maxRam + "g' \\"); + } + writer.println("\tquay.io/mlin/glnexus:" + binVersion + " \\"); + writer.println("\tglnexus_cli \\"); + writer.println("\t--config " + configType + " \\"); + + writer.println("\t--trim-uncalled-alleles \\"); + + if (maxRam != null) + { + writer.println("\t--mem-gbytes " + maxRam + "\\"); + } + + Integer maxThreads = SequencePipelineService.get().getMaxThreads(getLogger()); + if (maxThreads != null) + { + writer.println("\t--threads " + maxThreads + " \\"); + } + + gvcfsLocal.forEach(f -> { + writer.println("\t/work/" + f.getName() + " \\"); + }); + + File bcftools = BcftoolsRunner.getBcfToolsPath(); + File bgzip = BgzipRunner.getExe(); + writer.println("\t| " + bcftools.getPath() + " view | " + bgzip.getPath() + " -c > " + outputVcf.getPath()); + + // Command will fail if this exists: + File dbDir = new File (outputVcf.getParentFile(), "GLnexus.DB"); + if (dbDir.exists()) + { + FileUtils.deleteDirectory(dbDir); + } + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + + setWorkingDir(workDir); + execute(Arrays.asList("/bin/bash", localBashScript.getPath())); + + if (!outputVcf.exists()) + { + throw new PipelineJobException("File not found: " + outputVcf.getPath()); + } + + File idxFile = new File(outputVcf.getPath() + ".tbi"); + if (!idxFile.exists()) + { + throw new PipelineJobException("Missing index: " + idxFile.getPath()); + } + } + } +} \ No newline at end of file diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/button/ArchiveReadsetsButton.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/button/ArchiveReadsetsButton.java new file mode 100644 index 000000000..e19f572c3 --- /dev/null +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/button/ArchiveReadsetsButton.java @@ -0,0 +1,23 @@ +package org.labkey.sequenceanalysis.button; + +import org.labkey.api.laboratory.security.LaboratoryAdminPermission; +import org.labkey.api.ldk.table.SimpleButtonConfigFactory; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.view.template.ClientDependency; +import org.labkey.sequenceanalysis.SequenceAnalysisModule; + +import java.util.Arrays; + +/** + * Created by bimber on 7/12/2017. + */ +public class ArchiveReadsetsButton extends SimpleButtonConfigFactory +{ + public ArchiveReadsetsButton() + { + super(ModuleLoader.getInstance().getModule(SequenceAnalysisModule.class), "Archive Readsets", "SequenceAnalysis.window.ArchiveReadsetsWindow.buttonHandler(dataRegionName);", Arrays.asList(ClientDependency.supplierFromModuleName("ldk"), ClientDependency.supplierFromModuleName("laboratory"), ClientDependency.supplierFromPath("sequenceanalysis/window/ArchiveReadsetsWindow.js"))); + setPermission(LaboratoryAdminPermission.class); + } +} \ No newline at end of file diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java index d1efdc9a2..ac4624b6d 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java @@ -483,6 +483,12 @@ public static File processVCF(File input, Integer libraryId, JobContext ctx, Res action.setEndTime(end); ctx.getJob().getLogger().info(stepCtx.getProvider().getLabel() + " Duration: " + DurationFormatUtils.formatDurationWords(end.getTime() - start.getTime(), true, true)); + if (ctx.getFileManager().performCleanupAfterEachStep()) + { + List toRetain = Arrays.asList(currentVCF, new File(currentVCF.getPath() + ".tbi")); + getTaskFileManagerImpl(ctx).deleteIntermediateFiles(toRetain); + } + resumer.setStepComplete(stepIdx, input.getPath(), action, currentVCF); } @@ -886,4 +892,14 @@ public void performAdditionalMergeTasks(JobContext ctx, PipelineJob job, TaskFil } } } + + private static TaskFileManagerImpl getTaskFileManagerImpl(JobContext ctx) throws PipelineJobException + { + if (!(ctx.getFileManager() instanceof TaskFileManagerImpl tfm)) + { + throw new PipelineJobException("Expected fileManager to be a TaskFileManagerImpl"); + } + + return tfm; + } } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java index aba4524bf..2dd067144 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java @@ -96,6 +96,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -347,11 +348,27 @@ private Map> performFastqPreprocessing(SequenceReadse toAlign.put(d, pair); } + if (getHelper().getFileManager().performCleanupAfterEachStep()) + { + List toRetain = toAlign.values().stream().map(x -> Arrays.asList(x.first, x.second)).flatMap(List::stream).filter(Objects::nonNull).toList(); + getTaskFileManagerImpl().deleteIntermediateFiles(toRetain); + } + _resumer.setFastqPreprocessingDone(toAlign, preprocessingActions, copiedInputs); return toAlign; } + private TaskFileManagerImpl getTaskFileManagerImpl() throws PipelineJobException + { + if (!(getHelper().getFileManager() instanceof TaskFileManagerImpl tfm)) + { + throw new PipelineJobException("Expected fileManager to be a TaskFileManagerImpl"); + } + + return tfm; + } + private SequenceAlignmentJob getPipelineJob() { return (SequenceAlignmentJob)getJob(); @@ -667,6 +684,12 @@ private void alignSet(Readset rs, String basename, Map alignActions = new ArrayList<>(); bam = doAlignment(referenceGenome, rs, files, alignActions); + if (getHelper().getFileManager().performCleanupAfterEachStep()) + { + List toRetain = Arrays.asList(bam, SequenceUtil.getExpectedIndex(bam)); + getTaskFileManagerImpl().deleteIntermediateFiles(toRetain); + } + _resumer.setInitialAlignmentDone(bam, alignActions); } @@ -742,6 +765,12 @@ else if (step.expectToCreateNewBam()) action.setEndTime(end); getJob().getLogger().info(stepCtx.getProvider().getLabel() + " Duration: " + DurationFormatUtils.formatDurationWords(end.getTime() - start.getTime(), true, true)); postProcessActions.add(action); + + if (getHelper().getFileManager().performCleanupAfterEachStep()) + { + List toRetain = Arrays.asList(bam, SequenceUtil.getExpectedIndex(bam)); + getTaskFileManagerImpl().deleteIntermediateFiles(toRetain); + } } } @@ -791,6 +820,12 @@ else if (step.expectToCreateNewBam()) } } + if (getHelper().getFileManager().performCleanupAfterEachStep()) + { + List toRetain = Arrays.asList(bam, SequenceUtil.getExpectedIndex(bam)); + getTaskFileManagerImpl().deleteIntermediateFiles(toRetain); + } + _resumer.setBamSortDone(bam, sortAction); } @@ -841,6 +876,12 @@ else if (step.expectToCreateNewBam()) renameAction.setEndTime(end); getJob().getLogger().info("Rename Bam Duration: " + DurationFormatUtils.formatDurationWords(end.getTime() - start.getTime(), true, true)); + if (getHelper().getFileManager().performCleanupAfterEachStep()) + { + List toRetain = Arrays.asList(renamedBam, SequenceUtil.getExpectedIndex(renamedBam)); + getTaskFileManagerImpl().deleteIntermediateFiles(toRetain); + } + _resumer.setBamRenameDone(renamedBam, List.of(renameAction)); } @@ -888,6 +929,12 @@ else if (step.expectToCreateNewBam()) indexAction.setEndTime(end); getJob().getLogger().info("IndexBam Duration: " + DurationFormatUtils.formatDurationWords(end.getTime() - start.getTime(), true, true)); + if (getHelper().getFileManager().performCleanupAfterEachStep()) + { + List toRetain = Arrays.asList(renamedBam, SequenceUtil.getExpectedIndex(renamedBam)); + getTaskFileManagerImpl().deleteIntermediateFiles(toRetain); + } + _resumer.setIndexBamDone(true, indexAction); } } @@ -1045,8 +1092,15 @@ else if (step.expectToCreateNewBam()) } analysisActions.add(action); - _resumer.setBamAnalysisComplete(analysisActions); + + if (getHelper().getFileManager().performCleanupAfterEachStep()) + { + List toRetain = Arrays.asList(renamedBam, SequenceUtil.getExpectedIndex(renamedBam)); + getTaskFileManagerImpl().deleteIntermediateFiles(toRetain); + } } + + _resumer.setBamAnalysisComplete(analysisActions); } } @@ -1059,7 +1113,14 @@ else if (step.expectToCreateNewBam()) final File cramFile = new File(renamedBam.getParentFile(), FileUtil.getBaseName(renamedBam) + ".cram"); final File cramFileIdx = new File(cramFile.getPath() + ".crai"); Integer threads = SequenceTaskHelper.getMaxThreads(getJob()); - new SamtoolsCramConverter(getJob().getLogger()).convert(renamedBam, cramFile, referenceGenome.getWorkingFastaFileGzipped(), true, threads); + if (!cramFileIdx.exists()) + { + new SamtoolsCramConverter(getJob().getLogger()).convert(renamedBam, cramFile, referenceGenome.getWorkingFastaFileGzipped(), true, threads); + } + else + { + getJob().getLogger().debug("CRAM index already exists, skipping conversion"); + } final File finalBam = renamedBam; final File finalBamIdx = new File(renamedBam.getPath() + ".bai"); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerRemoteTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerRemoteTask.java index c7250b5f1..06cf83837 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerRemoteTask.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerRemoteTask.java @@ -57,7 +57,7 @@ public String getStatusName() public List getProtocolActionNames() { List allowableNames = new ArrayList<>(); - for (SequenceOutputHandler handler : SequenceAnalysisServiceImpl.get().getFileHandlers(SequenceOutputHandler.TYPE.OutputFile)) + for (SequenceOutputHandler handler : SequenceAnalysisServiceImpl.get().getFileHandlers(SequenceOutputHandler.TYPE.OutputFile)) { allowableNames.add(handler.getName()); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java index 670534465..622c2ed74 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java @@ -7,6 +7,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DurationFormatUtils; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; @@ -707,6 +708,12 @@ public boolean isDeleteIntermediateFiles() return "true".equals(_job.getParameters().get("deleteIntermediateFiles")); } + @Override + public boolean performCleanupAfterEachStep() + { + return "true".equals(_job.getParameters().get("performCleanupAfterEachStep")); + } + @Override public boolean isCopyInputsLocally() { @@ -726,19 +733,26 @@ private Set getInputPaths() @Override public void deleteIntermediateFiles() throws PipelineJobException { - _job.getLogger().info("Cleaning up intermediate files"); + deleteIntermediateFiles(Collections.emptySet()); + } - Set inputs = new HashSet<>(); - inputs.addAll(getSupport().getInputFiles()); + public void deleteIntermediateFiles(@NotNull Collection filesToRetain) throws PipelineJobException + { + _job.getLogger().info("Cleaning up intermediate files"); Set inputPaths = getInputPaths(); - if (isDeleteIntermediateFiles()) { _job.getLogger().debug("Intermediate files will be removed, total: " + _intermediateFiles.size()); for (File f : _intermediateFiles) { + if (filesToRetain.contains(f)) + { + _job.getLogger().debug("\tFile marked for deletion, but was part of filesToRetain and will not be deleted: " + f.getPath()); + continue; + } + _job.getLogger().debug("\tDeleting intermediate file: " + f.getPath()); if (inputPaths.contains(f.getPath())) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/BcftoolsFillTagsStep.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/BcftoolsFillTagsStep.java index 3016cf2c0..8feaa42dd 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/BcftoolsFillTagsStep.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/BcftoolsFillTagsStep.java @@ -100,6 +100,7 @@ public VariantProcessingStep.Output processVariants(File inputVCF, File outputDi throw new PipelineJobException("No annotations were selected"); } + options.add("--"); options.add("-t"); options.add(StringUtils.join(annotations, ",")); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/DeepVariantAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/DeepVariantAnalysis.java new file mode 100644 index 000000000..06a5c1b96 --- /dev/null +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/DeepVariantAnalysis.java @@ -0,0 +1,302 @@ +package org.labkey.sequenceanalysis.run.analysis; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.json.JSONObject; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.sequenceanalysis.model.AnalysisModel; +import org.labkey.api.sequenceanalysis.model.Readset; +import org.labkey.api.sequenceanalysis.pipeline.AbstractAnalysisStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.AnalysisOutputImpl; +import org.labkey.api.sequenceanalysis.pipeline.AnalysisStep; +import org.labkey.api.sequenceanalysis.pipeline.CommandLineParam; +import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; +import org.labkey.api.sequenceanalysis.pipeline.PipelineOutputTracker; +import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider; +import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome; +import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; +import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService; +import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.api.sequenceanalysis.run.AbstractCommandPipelineStep; +import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper; +import org.labkey.api.util.FileUtil; +import org.labkey.api.writer.PrintWriters; +import org.labkey.sequenceanalysis.util.SequenceUtil; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * User: bimber + * Date: 7/3/2014 + * Time: 11:29 AM + */ +public class DeepVariantAnalysis extends AbstractCommandPipelineStep implements AnalysisStep +{ + public DeepVariantAnalysis(PipelineStepProvider provider, PipelineContext ctx) + { + super(provider, ctx, new DeepVariantAnalysis.DeepVariantWrapper(ctx.getLogger())); + } + + public static class Provider extends AbstractAnalysisStepProvider + { + public Provider() + { + super("DeepVariantAnalysis", "DeepVariant", "DeepVariant", "This will run DeepVariant on the selected data to generate a gVCF.", getToolDescriptors(), null, null); + } + + @Override + public DeepVariantAnalysis create(PipelineContext ctx) + { + return new DeepVariantAnalysis(this, ctx); + } + } + + public static List getToolDescriptors() + { + return Arrays.asList( + ToolParameterDescriptor.create("modelType", "Model Type", "", "ldk-simplecombo", new JSONObject(){{ + put("storeValues", "AUTO;WGS;WES;PACBIO;ONT_R104;HYBRID_PACBIO_ILLUMINA"); + put("multiSelect", false); + put("allowBlank", false); + }}, "AUTO"), + ToolParameterDescriptor.createCommandLineParam(CommandLineParam.createSwitch("--haploid_contigs"), "haploidContigs", "Haploid Contigs", "", "textfield", new JSONObject(){{ + + }}, "X,Y"), + ToolParameterDescriptor.create("binVersion", "DeepVariant Version", "The version of DeepVariant to run, which is passed to their docker container", "textfield", new JSONObject(){{ + put("allowBlank", false); + }}, "1.6.0") + ); + } + + @Override + public void init(SequenceAnalysisJobSupport support) throws PipelineJobException + { + String modelType = getProvider().getParameterByName("modelType").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class); + if (modelType == null) + { + throw new PipelineJobException("Missing model type"); + } + + inferModelType(modelType, getPipelineCtx()); + } + + public static void inferModelType(String modelType, PipelineContext ctx) throws PipelineJobException + { + if ("AUTO".equals(modelType)) + { + ctx.getLogger().info("Inferring model type by readset type:"); + if (ctx.getSequenceSupport().getCachedReadsets().size() != 1) + { + throw new PipelineJobException("Expected a single cached readset, found: " + ctx.getSequenceSupport().getCachedReadsets().size()); + } + + Readset rs = ctx.getSequenceSupport().getCachedReadsets().get(0); + if ("ILLUMINA".equals(rs.getPlatform())) + { + switch (rs.getApplication()) + { + case "Whole Genome: Deep Coverage": + modelType = "WGS"; + break; + case "Whole Genome: Light Coverage": + modelType = "WGS"; + break; + case "Whole Exome": + modelType = "WXS"; + break; + default: + throw new IllegalArgumentException("Unknown application: " + rs.getApplication()); + } + } + else if ("PACBIO".equals(rs.getPlatform())) + { + modelType = "PACBIO"; + } + + if ("AUTO".equals(modelType)) + { + throw new PipelineJobException("Unable to infer modelType for: " + rs.getName()); + } + + ctx.getSequenceSupport().cacheObject("modelType", modelType); + } + } + + @Override + public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, ReferenceGenome referenceGenome, File outputDir) throws PipelineJobException + { + AnalysisOutputImpl output = new AnalysisOutputImpl(); + output.addInput(inputBam, "Input BAM File"); + + File outputFile = new File(outputDir, FileUtil.getBaseName(inputBam) + ".g.vcf.gz"); + File idxFile = new File(outputDir, FileUtil.getBaseName(inputBam) + ".g.vcf.gz.idx"); + + String inferredModelType = getPipelineCtx().getSequenceSupport().getCachedObject("modelType", String.class); + String modelType = inferredModelType == null ? getProvider().getParameterByName("modelType").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class) : inferredModelType; + if (modelType == null) + { + throw new PipelineJobException("Missing model type"); + } + + List args = new ArrayList<>(getClientCommandArgs()); + args.add("--model_type=" + modelType); + + String binVersion = getProvider().getParameterByName("binVersion").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), String.class); + if (binVersion == null) + { + throw new PipelineJobException("Missing binVersion"); + } + + getWrapper().setOutputDir(outputDir); + getWrapper().setWorkingDir(outputDir); + getWrapper().execute(inputBam, referenceGenome.getWorkingFastaFile(), outputFile, output, binVersion, args); + + output.addOutput(outputFile, "gVCF File"); + output.addSequenceOutput(outputFile, outputFile.getName(), "DeepVariant gVCF File", rs.getReadsetId(), null, referenceGenome.getGenomeId(), "DeepVariant Version: " + binVersion); + if (idxFile.exists()) + { + output.addOutput(idxFile, "VCF Index"); + } + + return output; + } + + @Override + public Output performAnalysisPerSampleLocal(AnalysisModel model, File inputBam, File referenceFasta, File outDir) throws PipelineJobException + { + return null; + } + + public static class DeepVariantWrapper extends AbstractCommandWrapper + { + public DeepVariantWrapper(Logger logger) + { + super(logger); + } + + private File ensureLocalCopy(File input, File workingDirectory, PipelineOutputTracker output) throws PipelineJobException + { + try + { + if (workingDirectory.equals(input.getParentFile())) + { + return input; + } + + File local = new File(workingDirectory, input.getName()); + if (!local.exists()) + { + getLogger().debug("Copying file locally: " + input.getPath()); + FileUtils.copyFile(input, local); + } + + output.addIntermediateFile(local); + + return local; + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + } + + public void execute(File inputBam, File refFasta, File outputGvcf, PipelineOutputTracker tracker, String binVersion, List extraArgs) throws PipelineJobException + { + File workDir = outputGvcf.getParentFile(); + File outputVcf = new File(outputGvcf.getPath().replaceAll(".g.vcf", ".vcf")); + tracker.addIntermediateFile(outputVcf); + tracker.addIntermediateFile(new File(outputVcf.getPath() + ".tbi")); + + File inputBamLocal = ensureLocalCopy(inputBam, workDir, tracker); + ensureLocalCopy(SequenceUtil.getExpectedIndex(inputBam), workDir, tracker); + + File refFastaLocal = ensureLocalCopy(refFasta, workDir, tracker); + ensureLocalCopy(new File(refFasta.getPath() + ".fai"), workDir, tracker); + ensureLocalCopy(new File(FileUtil.getBaseName(refFasta.getPath()) + ".dict"), workDir, tracker); + + File localBashScript = new File(workDir, "docker.sh"); + File dockerBashScript = new File(workDir, "dockerRun.sh"); + tracker.addIntermediateFile(localBashScript); + tracker.addIntermediateFile(dockerBashScript); + + List bashArgs = new ArrayList<>(Arrays.asList("/opt/deepvariant/bin/run_deepvariant")); + bashArgs.add("--ref=/work/" + refFastaLocal.getName()); + bashArgs.add("--reads=/work/" + inputBamLocal.getName()); + bashArgs.add("--output_gvcf=/work/" + outputGvcf.getName()); + bashArgs.add("--output_vcf=/work/" + outputVcf.getName()); + Integer maxThreads = SequencePipelineService.get().getMaxThreads(getLogger()); + if (maxThreads != null) + { + bashArgs.add("--num_shards=" + maxThreads); + } + + if (extraArgs != null) + { + bashArgs.addAll(extraArgs); + } + + try (PrintWriter writer = PrintWriters.getPrintWriter(localBashScript); PrintWriter dockerWriter = PrintWriters.getPrintWriter(dockerBashScript)) + { + writer.println("#!/bin/bash"); + writer.println("set -x"); + writer.println("WD=`pwd`"); + writer.println("HOME=`echo ~/`"); + writer.println("DOCKER='" + SequencePipelineService.get().getDockerCommand() + "'"); + writer.println("sudo $DOCKER pull google/deepvariant:" + binVersion); + writer.println("sudo $DOCKER run --rm=true \\"); + writer.println("\t-v \"${WD}:/work\" \\"); + writer.println("\t-v \"${HOME}:/homeDir\" \\"); + if (!StringUtils.isEmpty(System.getenv("TMPDIR"))) + { + writer.println("\t-v \"${TMPDIR}:/tmp\" \\"); + } + writer.println("\t-u $UID \\"); + writer.println("\t-e USERID=$UID \\"); + writer.println("\t--entrypoint /bin/bash \\"); + writer.println("\t-w /work \\"); + Integer maxRam = SequencePipelineService.get().getMaxRam(); + if (maxRam != null) + { + writer.println("\t-e SEQUENCEANALYSIS_MAX_RAM=" + maxRam + " \\"); + writer.println("\t--memory='" + maxRam + "g' \\"); + } + writer.println("\tgoogle/deepvariant:" + binVersion + " \\"); + writer.println("\t/work/" + dockerBashScript.getName()); + writer.println("EXIT_CODE=$?"); + writer.println("echo 'Docker run exit code: '$EXIT_CODE"); + writer.println("exit $EXIT_CODE"); + + dockerWriter.println("#!/bin/bash"); + dockerWriter.println("set -x"); + dockerWriter.println(StringUtils.join(bashArgs, " ")); + dockerWriter.println("EXIT_CODE=$?"); + dockerWriter.println("echo 'Exit code: '$?"); + dockerWriter.println("exit $EXIT_CODE"); + } + catch (IOException e) + { + throw new PipelineJobException(e); + } + + setWorkingDir(workDir); + execute(Arrays.asList("/bin/bash", localBashScript.getPath())); + + if (!outputGvcf.exists()) + { + throw new PipelineJobException("File not found: " + outputGvcf.getPath()); + } + + File idxFile = new File(outputGvcf.getPath() + ".tbi"); + if (!idxFile.exists()) + { + throw new PipelineJobException("Missing index: " + idxFile.getPath()); + } + } + } +} diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/HaplotypeCallerAnalysis.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/HaplotypeCallerAnalysis.java index 7c3b036ca..fb2eb656a 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/HaplotypeCallerAnalysis.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/HaplotypeCallerAnalysis.java @@ -28,7 +28,7 @@ */ public class HaplotypeCallerAnalysis extends AbstractCommandPipelineStep implements AnalysisStep { - public HaplotypeCallerAnalysis(PipelineStepProvider provider, PipelineContext ctx) + public HaplotypeCallerAnalysis(PipelineStepProvider provider, PipelineContext ctx) { super(provider, ctx, new HaplotypeCallerWrapper(ctx.getLogger())); } @@ -51,7 +51,7 @@ public static List getToolDescriptors() { return Arrays.asList( ToolParameterDescriptor.createCommandLineParam(CommandLineParam.createSwitch("--dont-use-soft-clipped-bases"), "dontUseSoftClippedBases", "Don't Use Soft Clipped Bases", "If specified, we will not analyze soft clipped bases in the reads", "checkbox", null, false), - ToolParameterDescriptor.createCommandLineParam(CommandLineParam.createSwitch("max-alternate-alleles"), "maxAlternateAlleles", "Max Alternate Alleles", "Passed to --max-alternate-alleles", "ldk-integerfield", new JSONObject(){{ + ToolParameterDescriptor.createCommandLineParam(CommandLineParam.createSwitch("--max-alternate-alleles"), "maxAlternateAlleles", "Max Alternate Alleles", "Passed to --max-alternate-alleles", "ldk-integerfield", new JSONObject(){{ put("minValue", 0); }}, 6) ); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/PbsvJointCallingHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/PbsvJointCallingHandler.java index 429405bb5..5453e65be 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/PbsvJointCallingHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/PbsvJointCallingHandler.java @@ -151,14 +151,17 @@ public void processFilesRemote(List inputFiles, JobContext c List outputs = new ArrayList<>(); if (getVariantPipelineJob(ctx.getJob()) != null && getVariantPipelineJob(ctx.getJob()).isScatterJob()) { - for (Interval i : getVariantPipelineJob(ctx.getJob()).getIntervalsForTask()) + int idx = 0; + List intervals = getVariantPipelineJob(ctx.getJob()).getIntervalsForTask(); + for (Interval i : intervals) { + idx++; if (i.getStart() != 1) { throw new PipelineJobException("Expected all intervals to start on the first base: " + i); } - File o = runPbsvCall(ctx, filesToProcess, genome, outputBaseName + (getVariantPipelineJob(ctx.getJob()).getIntervalsForTask().size() == 1 ? "" : "." + i.getContig()), i.getContig(), jobCompleted); + File o = runPbsvCall(ctx, filesToProcess, genome, outputBaseName + (getVariantPipelineJob(ctx.getJob()).getIntervalsForTask().size() == 1 ? "" : "." + i.getContig()), i.getContig(), (" (" + idx + " of " + intervals.size() + ")"), jobCompleted); if (o != null) { outputs.add(o); @@ -167,7 +170,7 @@ public void processFilesRemote(List inputFiles, JobContext c } else { - outputs.add(runPbsvCall(ctx, filesToProcess, genome, outputBaseName, null, jobCompleted)); + outputs.add(runPbsvCall(ctx, filesToProcess, genome, outputBaseName, null, null, jobCompleted)); } try @@ -228,11 +231,11 @@ public void processFilesRemote(List inputFiles, JobContext c } } - private File runPbsvCall(JobContext ctx, List inputs, ReferenceGenome genome, String outputBaseName, @Nullable String contig, boolean jobCompleted) throws PipelineJobException + private File runPbsvCall(JobContext ctx, List inputs, ReferenceGenome genome, String outputBaseName, @Nullable String contig, @Nullable String statusSuffix, boolean jobCompleted) throws PipelineJobException { if (contig != null) { - ctx.getJob().setStatus(PipelineJob.TaskStatus.running, "Processing: " + contig); + ctx.getJob().setStatus(PipelineJob.TaskStatus.running, "Processing: " + contig + (statusSuffix == null ? "" : statusSuffix)); } if (inputs.isEmpty()) diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/BgzipRunner.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/BgzipRunner.java index a0867ae5e..136dba7a3 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/BgzipRunner.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/BgzipRunner.java @@ -95,7 +95,7 @@ private List getParams(File input, boolean preserveInput) return params; } - public File getExe() + public static File getExe() { return SequencePipelineService.get().getExeForPackage("BGZIPPATH", "bgzip"); } diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/MergeVcfsAndGenotypesHandler.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/MergeVcfsAndGenotypesHandler.java index e3609dd9e..4b4957590 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/MergeVcfsAndGenotypesHandler.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/MergeVcfsAndGenotypesHandler.java @@ -11,13 +11,14 @@ import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport; import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.api.util.PageFlowUtil; import org.labkey.sequenceanalysis.SequenceAnalysisModule; import org.labkey.sequenceanalysis.run.util.MergeVcfsAndGenotypesWrapper; import org.labkey.sequenceanalysis.util.SequenceUtil; import java.io.File; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -25,7 +26,7 @@ /** * Created by bimber on 4/4/2017. */ -public class MergeVcfsAndGenotypesHandler extends AbstractParameterizedOutputHandler +public class MergeVcfsAndGenotypesHandler extends AbstractParameterizedOutputHandler implements SequenceOutputHandler.HasActionNames { public MergeVcfsAndGenotypesHandler() { @@ -58,7 +59,14 @@ public SequenceOutputProcessor getProcessor() return new Processor(); } - public static class Processor implements SequenceOutputProcessor + @Override + public Collection getAllowableActionNames() + { + // NOTE: Combine Variants only exists for legacy purposes: + return PageFlowUtil.set(getName(), "Combine Variants"); + } + + public class Processor implements SequenceOutputProcessor { @Override public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport support, List inputFiles, JSONObject params, File outputDir, List actions, List outputsToCreate) throws UnsupportedOperationException, PipelineJobException @@ -71,7 +79,7 @@ public void processFilesRemote(List inputFiles, JobContext c { File outputVcf = new File(ctx.getOutputDir(), ctx.getParams().getString("basename") + ".combined.vcf.gz"); - RecordedAction action = new RecordedAction("Combine Variants"); + RecordedAction action = new RecordedAction(getName()); Set genomeIds = new HashSet<>(); inputFiles.forEach(x -> genomeIds.add(x.getLibrary_id())); diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/ScatterGatherUtils.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/ScatterGatherUtils.java index 1d02a8f41..4affbe7e2 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/ScatterGatherUtils.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/ScatterGatherUtils.java @@ -7,6 +7,8 @@ import org.junit.Test; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -114,7 +116,11 @@ private void addInterval(String refName, int start, int end) public static LinkedHashMap> divideGenome(SAMSequenceDictionary dict, int optimalBasesPerJob, boolean allowSplitChromosomes, int maxContigsPerJob) { ActiveIntervalSet ais = new ActiveIntervalSet(optimalBasesPerJob, allowSplitChromosomes, maxContigsPerJob); - for (SAMSequenceRecord rec : dict.getSequences()) + + // Sort the sequences in descending length, rather than alphabetic on name: + List sortedSeqs = new ArrayList<>(dict.getSequences()); + sortedSeqs.sort(Comparator.comparingInt(SAMSequenceRecord::getSequenceLength).reversed()); + for (SAMSequenceRecord rec : sortedSeqs) { ais.add(rec); } @@ -148,8 +154,8 @@ public void testScatter() SAMSequenceDictionary dict = getDict(); Map> ret = divideGenome(dict, 1000, true, -1); assertEquals("Incorrect number of jobs", 8, ret.size()); - assertEquals("Incorrect interval end", 2000, ret.get("Job3").get(0).getEnd()); - assertEquals("Incorrect start", 1001, ret.get("Job3").get(0).getStart()); + assertEquals("Incorrect interval end", 1000, ret.get("Job3").get(0).getEnd()); + assertEquals("Incorrect start", 1, ret.get("Job3").get(0).getStart()); assertEquals("Incorrect interval end", 4, ret.get("Job8").size()); Map> ret2 = divideGenome(dict, 3000, false, -1); @@ -183,12 +189,12 @@ public void testScatter() } Map> ret5 = divideGenome(dict, 750, true, -1); - assertEquals("Incorrect number of jobs", 10, ret5.size()); - assertEquals("Incorrect interval end", 1000, ret5.get("Job1").get(0).getEnd()); - assertEquals("Incorrect interval end", 4, ret5.get("Job10").size()); + assertEquals("Incorrect number of jobs", 9, ret5.size()); + assertEquals("Incorrect interval end", 750, ret5.get("Job1").get(0).getEnd()); + assertEquals("Incorrect interval end", 4, ret5.get("Job9").size()); - assertEquals("Incorrect interval start", 751, ret5.get("Job3").get(0).getStart()); - assertEquals("Incorrect interval start", 1501, ret5.get("Job8").get(0).getStart()); + assertEquals("Incorrect interval start", 1501, ret5.get("Job3").get(0).getStart()); + assertEquals("Incorrect interval start", 1, ret5.get("Job8").get(0).getStart()); Map> ret6 = divideGenome(dict, 5000, false, 2); assertEquals("Incorrect number of jobs", 5, ret6.size()); diff --git a/cluster/src/org/labkey/cluster/ClusterModule.java b/cluster/src/org/labkey/cluster/ClusterModule.java index 4cf4399ec..e7638ddcc 100644 --- a/cluster/src/org/labkey/cluster/ClusterModule.java +++ b/cluster/src/org/labkey/cluster/ClusterModule.java @@ -45,7 +45,6 @@ import org.labkey.cluster.query.ViewClusterSubmissionsButton; import org.labkey.cluster.query.ViewJavaLogButton; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; diff --git a/jbrowse/package-lock.json b/jbrowse/package-lock.json index 1b9cf0ee5..625de5a5e 100644 --- a/jbrowse/package-lock.json +++ b/jbrowse/package-lock.json @@ -9042,9 +9042,9 @@ } }, "node_modules/follow-redirects": { - "version": "1.15.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", - "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", + "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", "dev": true, "funding": [ { diff --git a/jbrowse/src/client/JBrowse/VariantSearch/VariantTable.tsx b/jbrowse/src/client/JBrowse/VariantSearch/VariantTable.tsx index b1c84e4ee..75dd70709 100644 --- a/jbrowse/src/client/JBrowse/VariantSearch/VariantTable.tsx +++ b/jbrowse/src/client/JBrowse/VariantSearch/VariantTable.tsx @@ -45,7 +45,7 @@ function VariantTable() { const [session, setSession] = useState(null) const [state, setState] = useState(null) - const [theme, setTheme] = useState(null) + const [theme, setTheme] = useState(createTheme()) const [view, setView] = useState(null) const [parsedLocString, setParsedLocString] = useState(null) const [assemblyNames, setAssemblyNames] = useState(null) @@ -86,24 +86,11 @@ function VariantTable() { fetchSession(queryParam, sessionId, nativePlugins, refTheme, setState, true, [trackId], undefined, successCallback, trackId) }, []); - // Error handle and then render the component - if (view === null || theme == null) { - return () - } - else if (view === "invalid" || state == "invalid") { - return (

Error fetching config. See console for more details

) - } - - if (!assemblyNames.length) { - return (

No configured assemblies

) - } - return (
- +
diff --git a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx index 9dd0d31ec..afe99496c 100644 --- a/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx +++ b/jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx @@ -36,21 +36,20 @@ import { BaseFeatureDataAdapter } from '@jbrowse/core/data_adapters/BaseAdapter' import { lastValueFrom } from 'rxjs'; const VariantTableWidget = observer(props => { - const { assembly, trackId, parsedLocString, sessionId, session, pluginManager } = props - const { assemblyNames, assemblyManager } = session - const { view } = session - - // The code expects a proper GUID, yet the trackId is a string containing the GUID + filename - const trackGUID = truncateToValidGUID(props.trackId) - - // NOTE: since the trackId is GUID+filename, allow exact string matching, or a match on the GUID portion alone. - // Upstream code might only have access to the GUID and translating to the trackId isnt always easy - const track = view.tracks.find( - t => t.configuration.trackId === trackId || truncateToValidGUID(t.configuration.trackId).toUpperCase() === trackGUID.toUpperCase() - ) - - if (!track) { - return (

Unknown track: {trackId}

) + const { assembly, trackId, parsedLocString, sessionId, session, pluginManager } = props; + const { assemblyNames = [], assemblyManager } = session ?? {}; + const { view } = session ?? {}; + + var track = undefined; + var trackGUID = undefined; + if(view && trackId) { + // The code expects a proper GUID, yet the trackId is a string containing the GUID + filename + // NOTE: since the trackId is GUID+filename, allow exact string matching, or a match on the GUID portion alone. + // Upstream code might only have access to the GUID and translating to the trackId isnt always easy + trackGUID = truncateToValidGUID(props.trackId) + track = view.tracks.find( + t => t.configuration.trackId === trackId || truncateToValidGUID(t.configuration.trackId).toUpperCase() === trackGUID.toUpperCase() + ) } function handleSearch(data) { @@ -150,7 +149,6 @@ const VariantTableWidget = observer(props => { {displayValue} {renderPopover && - // TODO { const [fieldTypeInfo, setFieldTypeInfo] = useState([]); const [allowedGroupNames, setAllowedGroupNames] = useState([]); const [promotedFilters, setPromotedFilters] = useState>(null); - const [columnVisibilityModel, setColumnVisibilityModel] = useState({}); - const [adapter, setAdapter] = useState(null) // Active widget ID list to force rerender when a JBrowseUIButton is clicked const [activeWidgetList, setActiveWidgetList] = useState([]) // False until initial data load or an error: - const [dataLoaded, setDataLoaded] = useState(!parsedLocString) + const [dataLoaded, setDataLoaded] = useState(false) const urlParams = new URLSearchParams(window.location.search); const page = parseInt(urlParams.get('page') || '0'); const pageSize = parseInt(urlParams.get('pageSize') || '50'); const [pageSizeModel, setPageSizeModel] = React.useState({ page, pageSize }); + const colVisURLComponent = urlParams.get("colVisModel") || "{}" + const colVisModel = JSON.parse(decodeURIComponent(colVisURLComponent)) + const [columnVisibilityModel, setColumnVisibilityModel] = useState(colVisModel); + // API call to retrieve the requested features. useEffect(() => { const handlePopState = () => { @@ -263,9 +263,19 @@ const VariantTableWidget = observer(props => { setColumns(columns) - const columnVisibilityModel = {} - fields.filter((x) => !x.isHidden).forEach((x) => columnVisibilityModel[x.name] = !!x.isInDefaultColumns) - setColumnVisibilityModel(columnVisibilityModel) + if(JSON.stringify(columnVisibilityModel) === '{}') { + const defaultModel = {}; + fields.filter((x) => !x.isHidden).forEach((x) => { + defaultModel[x.name] = !!x.isInDefaultColumns; + }); + setColumnVisibilityModel(defaultModel); + } else { + const updatedModel = fields.reduce((acc, field) => { + acc[field.name] = columnVisibilityModel[field.name] === true; + return acc; + }, {}); + setColumnVisibilityModel(updatedModel); + } setFieldTypeInfo(fields) setAllowedGroupNames(groups) @@ -278,20 +288,15 @@ const VariantTableWidget = observer(props => { }) } - fetch() + if(sessionId && trackGUID) { + fetch() + } + return () => { window.removeEventListener('popstate', handlePopState); }; - }, [pluginManager, parsedLocString, session.visibleWidget]) - - if (!view) { - return - } - - if (!track) { - return(

Unable to find track: {trackId}

) - } + }, [pluginManager, parsedLocString, session?.visibleWidget, sessionId, trackGUID]) if (error) { throw new Error(error) @@ -402,6 +407,17 @@ const VariantTableWidget = observer(props => { }} onColumnVisibilityModelChange={(model) => { setColumnVisibilityModel(model) + + const trueValuesModel = Object.keys(model).reduce((acc, key) => { + if (model[key] === true) { + acc[key] = true; + } + return acc; + }, {}); + + const currentUrl = new URL(window.location.href); + currentUrl.searchParams.set("colVisModel", encodeURIComponent(JSON.stringify(trueValuesModel))); + window.history.pushState(null, "", currentUrl.toString()); }} /> ) @@ -435,7 +451,7 @@ const VariantTableWidget = observer(props => { { - [...session.activeWidgets].map((elem) => { + [...(session?.activeWidgets ?? [])].map((elem) => { const widget = elem[1] const widgetType = pluginManager.getWidgetType(widget.type) const { ReactComponent } = widgetType diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseFieldUtils.java b/jbrowse/src/org/labkey/jbrowse/JBrowseFieldUtils.java index 0633774f2..95951cdfe 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseFieldUtils.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseFieldUtils.java @@ -75,12 +75,10 @@ public static Map getIndexedFields(JsonFile json public static Map getGenotypeDependentFields(@Nullable JsonFile jsonFile) { Map ret = new HashMap<>(); ret.put(VARIABLE_SAMPLES, new JBrowseFieldDescriptor(VARIABLE_SAMPLES, "All samples with this variant", true, true, VCFHeaderLineType.Character, 7).multiValued(true).label("Samples With Variant")); - - // TODO: restore these once existing indexes are regenerated: - // ret.put(N_HET, new JBrowseFieldDescriptor(N_HET, "The number of samples with this allele that are heterozygous", false, true, VCFHeaderLineType.Integer, 9).label("# Heterozygotes")); - // ret.put(N_HOMVAR, new JBrowseFieldDescriptor(N_HOMVAR, "The number of samples with this allele that are homozygous", false, true, VCFHeaderLineType.Integer, 9).label("# Homozygous Variant")); - // ret.put(N_CALLED, new JBrowseFieldDescriptor(N_CALLED, "The number of samples with called genotypes at this position", false, true, VCFHeaderLineType.Integer, 9).label("# Genotypes Called")); - // ret.put(FRACTION_HET, new JBrowseFieldDescriptor(FRACTION_HET, "The fraction of samples with this allele that are heterozygous", false, true, VCFHeaderLineType.Float, 9).label("Fraction Heterozygotes")); + ret.put(N_HET, new JBrowseFieldDescriptor(N_HET, "The number of samples with this allele that are heterozygous", false, true, VCFHeaderLineType.Integer, 9).label("# Heterozygotes")); + ret.put(N_HOMVAR, new JBrowseFieldDescriptor(N_HOMVAR, "The number of samples with this allele that are homozygous", false, true, VCFHeaderLineType.Integer, 9).label("# Homozygous Variant")); + ret.put(N_CALLED, new JBrowseFieldDescriptor(N_CALLED, "The number of samples with called genotypes at this position", false, true, VCFHeaderLineType.Integer, 9).label("# Genotypes Called")); + ret.put(FRACTION_HET, new JBrowseFieldDescriptor(FRACTION_HET, "The fraction of samples with this allele that are heterozygous", false, true, VCFHeaderLineType.Float, 9).label("Fraction Heterozygotes")); if (jsonFile != null) { File vcf = jsonFile.getTrackFile(); @@ -99,6 +97,12 @@ public static Map getGenotypeDependentFields(@Nu } else { + ret.put(VARIABLE_SAMPLES, new JBrowseFieldDescriptor(VARIABLE_SAMPLES, "All samples with this variant", true, true, VCFHeaderLineType.Character, 7).multiValued(true).label("Samples With Variant")); + ret.put(N_HET, new JBrowseFieldDescriptor(N_HET, "The number of samples with this allele that are heterozygous", false, true, VCFHeaderLineType.Integer, 9).label("# Heterozygotes")); + ret.put(N_HOMVAR, new JBrowseFieldDescriptor(N_HOMVAR, "The number of samples with this allele that are homozygous", false, true, VCFHeaderLineType.Integer, 9).label("# Homozygous Variant")); + ret.put(N_CALLED, new JBrowseFieldDescriptor(N_CALLED, "The number of samples with called genotypes at this position", false, true, VCFHeaderLineType.Integer, 9).label("# Genotypes Called")); + ret.put(FRACTION_HET, new JBrowseFieldDescriptor(FRACTION_HET, "The fraction of samples with this allele that are heterozygous", false, true, VCFHeaderLineType.Float, 9).label("Fraction Heterozygotes")); + ret.get(VARIABLE_SAMPLES).allowableValues(header.getSampleNamesInOrder()); } } diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java index 5176ddcdc..c5e1889a9 100644 --- a/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java +++ b/jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java @@ -218,7 +218,7 @@ else if (numericQueryParserFields.contains(fieldName)) } else { - throw new IllegalArgumentException("No such field(s), or malformed query."); + throw new IllegalArgumentException("No such field(s), or malformed query: " + queryString + ", field: " + fieldName); } booleanQueryBuilder.add(query, BooleanClause.Occur.MUST); diff --git a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java index 4b8704d2b..57006b138 100644 --- a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java +++ b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java @@ -1813,5 +1813,83 @@ private void testLuceneSearchUI(String sessionId) waitForElement(Locator.tagWithText("span", "0.029")); clearFilterDialog("IMPACT equals HIGH,MODERATE"); + + testLuceneColumnSerialization(sessionId); + } + + private void testLuceneColumnSerializationFirstRow() { + WebElement locator = TOP_ROW.findElement(getDriver()); + + for (WebElement elem : locator.findElements(By.xpath("./child::*"))) { + String value = elem.getText(); + if (StringUtils.trimToNull(value) == null) + { + value = ""; + } + + if (StringUtils.isEmpty(elem.getText())) { + return; + } + + switch(elem.getAttribute("aria-colindex")) + { + case "1": + Assert.assertEquals(value, "1"); + break; + case "2": + Assert.assertEquals(value, "2"); + break; + case "3": + Assert.assertEquals(value, "A"); + break; + case "4": + Assert.assertEquals(value, "T"); + break; + case "6": + Assert.assertEquals(value, "0.029"); + break; + case "7": + Assert.assertEquals(value, "7.292"); + break; + case "8": + Assert.assertEquals(value, "HIGH"); + break; + } + } + } + + private void testLuceneColumnSerialization(String sessionId) { + beginAt("/" + getProjectName() + "/jbrowse-jbrowse.view?session=" + sessionId); + waitAndClick(Locator.tagContainingText("button", "Show all regions in assembly").withClass("MuiButtonBase-root")); + waitAndClick(Locator.tagWithText("p", "No tracks active.")); + waitAndClick(Locator.tagWithText("button", "Open track selector")); + + Locator l = Locator.tagWithText("span", "TestVCF").withClass("MuiFormControlLabel-label"); + waitAndClick(l); + getDriver().findElement(Locator.tag("body")).sendKeys(Keys.ESCAPE); //close modal + + openTrackMenuItem("Variant Search", true); + waitAndClick(Locator.tagWithAttribute("button", "aria-label", "Select columns")); + + Locator caddScoreToggle = Locator.tagWithAttribute("input", "name", "CADD_PH"); + waitForElement(caddScoreToggle); + WebElement parentOfCaddScoreToggle = caddScoreToggle.findElement(getDriver()).findElement(By.xpath("./..")); + parentOfCaddScoreToggle.click(); + + String colVisModelString = "%257B%2522contig%2522%253Atrue%252C%2522start%2522%253Atrue%252C%2522ref%2522%253Atrue%252C%2522alt%2522%253Atrue%252C%2522variableSamples%2522%253Atrue%252C%2522AF%2522%253Atrue%252C%2522CADD_PH%2522%253Atrue%252C%2522IMPACT%2522%253Atrue%257D"; + Assert.assertEquals(getUrlParam("colVisModel"), colVisModelString); + + getDriver().navigate().refresh(); + + waitForElement(TOP_ROW); + Assert.assertEquals(getUrlParam("colVisModel"), colVisModelString); + testLuceneColumnSerializationFirstRow(); + + waitAndClick(Locator.tagWithText("button", "Search")); + waitAndClick(Locator.tagWithClass("button", "filter-form-select-button")); + + waitForElement(TOP_ROW); + Assert.assertEquals(getUrlParam("colVisModel"), colVisModelString); + testLuceneColumnSerializationFirstRow(); } } \ No newline at end of file diff --git a/singlecell/resources/chunks/AppendNimble.R b/singlecell/resources/chunks/AppendNimble.R index 23874d8d1..2dc826d10 100644 --- a/singlecell/resources/chunks/AppendNimble.R +++ b/singlecell/resources/chunks/AppendNimble.R @@ -11,7 +11,7 @@ for (datasetId in names(seuratObjects)) { seuratObj <- readSeuratRDS(seuratObjects[[datasetId]]) for (genomeId in names(nimbleGenomes)) { - seuratObj <- Rdiscvr::DownloadAndAppendNimble(seuratObject = seuratObj, allowableGenomes = genomeId, targetAssayName = nimbleGenomes[[genomeId]], enforceUniqueFeatureNames = TRUE, dropAmbiguousFeatures = !retainAmbiguousFeatures) + seuratObj <- Rdiscvr::DownloadAndAppendNimble(seuratObject = seuratObj, allowableGenomes = genomeId, ensureSamplesShareAllGenomes = ensureSamplesShareAllGenomes, targetAssayName = nimbleGenomes[[genomeId]], enforceUniqueFeatureNames = TRUE, dropAmbiguousFeatures = !retainAmbiguousFeatures, maxLibrarySizeRatio = maxLibrarySizeRatio) } saveData(seuratObj, datasetId) diff --git a/singlecell/resources/chunks/SeuratPrototype.R b/singlecell/resources/chunks/SeuratPrototype.R index 94ad664fc..0e5a11594 100644 --- a/singlecell/resources/chunks/SeuratPrototype.R +++ b/singlecell/resources/chunks/SeuratPrototype.R @@ -71,7 +71,7 @@ for (datasetId in names(seuratObjects)) { if ('Saturation.RNA' %in% names(seuratObj@meta.data)) { meanSaturation.RNA <- mean(seuratObj$Saturation.RNA) if (!is.null(minSaturation) && meanSaturation.RNA < minSaturation) { - addErrorMessage(paste0('Mean RNA saturation was: ', meanSaturation.RNA, ' for dataset: ', datasetId, ', below threshold of: ', minSaturation)) + addErrorMessage(paste0('Mean RNA saturation was: ', meanSaturation.RNA, ' for dataset: ', datasetId, ', below threshold of: ', minSaturation, ', total cells: ', ncol(seuratObj))) } metricData <- rbind(metricData, data.frame(dataId = datasetId, readsetId = datasetIdToReadset[[datasetId]], metricname = 'MeanSaturation.RNA', metricvalue = meanSaturation.RNA)) diff --git a/singlecell/resources/views/cDNAImport.view.xml b/singlecell/resources/views/cDNAImport.view.xml index 44afd9b9c..14c4dde79 100644 --- a/singlecell/resources/views/cDNAImport.view.xml +++ b/singlecell/resources/views/cDNAImport.view.xml @@ -2,6 +2,7 @@ + diff --git a/singlecell/resources/views/poolImport.view.xml b/singlecell/resources/views/poolImport.view.xml index c4a638ccb..3174e4aff 100644 --- a/singlecell/resources/views/poolImport.view.xml +++ b/singlecell/resources/views/poolImport.view.xml @@ -2,6 +2,7 @@ + diff --git a/singlecell/resources/views/singleCellDataManagement.html b/singlecell/resources/views/singleCellDataManagement.html index 64c42873c..301d70982 100644 --- a/singlecell/resources/views/singleCellDataManagement.html +++ b/singlecell/resources/views/singleCellDataManagement.html @@ -28,8 +28,11 @@ name: '10x VDJ Libraries Needing Alignment', url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {schemaName: 'sequenceanalysis', queryName: 'sequence_readsets', 'query.totalAlignments~eq': 0, 'query.totalForwardReads~isnonblank': null, 'query.application~contains': 'single', 'query.status~isblank': null, 'query.isArchived~eq': 0, 'query.librarytype~doesnotcontain': 'BCR', 'query.librarytype~contains': 'VDJ', 'query.sort': 'name'}) },{ - name: '10x Hashing/Cite-seq Libraries Needing Feature Counts', - url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {schemaName: 'sequenceanalysis', queryName: 'sequence_readsets', 'query.application~containsoneof': 'Cell Hashing;CITE-Seq', 'query.totalForwardReads~isnonblank': null, 'query.totalOutputs~eq': 0, 'query.status~isblank': null, 'query.sort': 'name'}) + name: 'Hashing Libraries Needing Feature Counts', + url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {schemaName: 'sequenceanalysis', queryName: 'sequence_readsets', 'query.application~eq': 'Cell Hashing', 'query.totalForwardReads~isnonblank': null, 'query.outputFileTypes~doesnotcontain': 'Cell Hashing Counts', 'query.status~isblank': null, 'query.sort': 'name'}) + },{ + name: 'Cite-seq Libraries Needing Feature Counts', + url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {schemaName: 'sequenceanalysis', queryName: 'sequence_readsets', 'query.application~eq': 'CITE-Seq', 'query.totalForwardReads~isnonblank': null, 'query.outputFileTypes~doesnotcontain': 'CITE-seq Counts', 'query.status~isblank': null, 'query.sort': 'name'}) },{ name: 'VLoupe Files Needing TCR Import', url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {schemaName: 'sequenceanalysis', queryName: 'outputfiles', 'query.readset/status~isblank': null, 'query.readset/numTcrResults~eq': 0, 'query.category~eq': '10x VLoupe', 'query.sort': 'readset/name'}) @@ -39,6 +42,9 @@ },{ name: 'Readsets Assigned to Run Lacking Data', url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {schemaName: 'sequenceanalysis', queryName: 'sequence_readsets', 'query.viewName': 'Assigned to Run Lacking Data'}) + },{ + name: 'cDNA Libraries with Hashing and Single HTO/Lane', + url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {schemaName: 'singlecell', queryName: 'cdna_libraries', 'query.uniqueHtos~in': '0;1', 'query.hashingReadsetId/name~isnonblank': null, 'query.hashingReadsetId/totalFiles~eq': 0}) },{ name: 'Analyses In Novogene-Related Workbook', url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, {schemaName: 'sequenceanalysis', queryName: 'sequence_analyses', 'query.container/title~containsoneof': 'Novogene;shipment'}) @@ -68,6 +74,37 @@ 'query.isArchived~eq': 0 } ) + },{ + name: 'CITE-seq/Hashing Readsets Possibly Needing Archival', + url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, { + schemaName: 'sequenceanalysis', + queryName: 'sequence_readsets', + 'query.viewName': 'SRA Info', + 'query.totalForwardReads~isnonblank': null, + 'query.application~containsoneof': 'Cell Hashing;CITE-Seq', + 'query.totalOutputs~gt': 0, + 'query.isArchived~eq': 0, + 'query.readdataWithoutSra~eq': 0, + 'query.sort': 'name' + } + ) + },{ + name: 'TCR Readsets Possibly Needing Archival', + url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, { + schemaName: 'sequenceanalysis', + queryName: 'sequence_readsets', + 'query.viewName': 'SRA Info', + 'query.totalForwardReads~isnonblank': null, + 'query.isArchived~eq': 0, + 'query.readdataWithoutSra~eq': 0, + 'query.outputFileTypes~contains': '10x VLoupe', + 'query.application~contains': 'single', + 'query.status~isblank': null, + 'query.librarytype~doesnotcontain': 'BCR', + 'query.librarytype~contains': 'VDJ', + 'query.sort': 'name' + } + ) }, { name: 'Analyses Using Old Readsets', url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, { diff --git a/singlecell/resources/web/singlecell/panel/LibraryExportPanel.js b/singlecell/resources/web/singlecell/panel/LibraryExportPanel.js index e367fe34c..078b40988 100644 --- a/singlecell/resources/web/singlecell/panel/LibraryExportPanel.js +++ b/singlecell/resources/web/singlecell/panel/LibraryExportPanel.js @@ -634,7 +634,7 @@ Ext4.define('SingleCell.panel.LibraryExportPanel', { ',readsetId,readsetId/name,readsetId/application,readsetId/librarytype,readsetId/barcode5,readsetId/barcode5/sequence,readsetId/barcode3,readsetId/barcode3/sequence,readsetId/totalFiles,readsetId/concentration' + ',tcrReadsetId,tcrReadsetId/name,tcrReadsetId/application,tcrReadsetId/librarytype,tcrReadsetId/barcode5,tcrReadsetId/barcode5/sequence,tcrReadsetId/barcode3,tcrReadsetId/barcode3/sequence,tcrReadsetId/totalFiles,tcrReadsetId/concentration' + ',hashingReadsetId,hashingReadsetId/name,hashingReadsetId/application,hashingReadsetId/librarytype,hashingReadsetId/barcode5,hashingReadsetId/barcode5/sequence,hashingReadsetId/barcode3,hashingReadsetId/barcode3/sequence,hashingReadsetId/totalFiles,hashingReadsetId/concentration' + - ',citeseqReadsetId,citeseqReadsetId/name,citeseqReadsetId/application,citeseqReadsetId/librarytype,citeseqReadsetId/barcode5,citeseqReadsetId/barcode5/sequence,citeseqReadsetId/barcode3,citeseqReadsetId/barcode3/sequence,citeseqReadsetId/totalFiles,citeseqReadsetId/concentration', + ',citeseqReadsetId,citeseqReadsetId/name,citeseqReadsetId/application,citeseqReadsetId/librarytype,citeseqReadsetId/barcode5,citeseqReadsetId/barcode5/sequence,citeseqReadsetId/barcode3,citeseqReadsetId/barcode3/sequence,citeseqReadsetId/totalFiles,citeseqReadsetId/concentration,uniqueHtos', scope: this, filterArray: [LABKEY.Filter.create('plateId', plateIds.join(';'), LABKEY.Filter.Types.IN)], failure: LDK.Utils.getErrorCallback(), @@ -661,6 +661,7 @@ Ext4.define('SingleCell.panel.LibraryExportPanel', { if (expectedPairs) { sortedRows = []; var missingRows = []; + var errorMsgs = []; Ext4.Array.forEach(expectedPairs, function(p){ var found = false; Ext4.Array.forEach(results.rows, function(row){ @@ -679,6 +680,11 @@ Ext4.define('SingleCell.panel.LibraryExportPanel', { if (row['hashingReadsetId'] && row['hashingReadsetId/application'] && row['hashingReadsetId/application'] === 'Cell Hashing') { sortedRows.push(Ext4.apply({targetApplication: '10x HTO', laneAssignment: (p.length > 2 ? p[2] : null), plateAlias: (p.length > 3 ? p[3] : null)}, row)); found = true; + + if (row.uniqueHtos <=1) { + errorMsgs.push(row['hashingReadsetId/name'] + ': only ' + row.uniqueHtos + ' present') + } + return false; } } @@ -713,6 +719,11 @@ Ext4.define('SingleCell.panel.LibraryExportPanel', { Ext4.Msg.alert('Error', 'The following plates were not found:
' + missingRows.join('
')); return; } + + if (errorMsgs.length){ + Ext4.Msg.alert('Error', 'The following lanes had HTO libraries, without multiple HTOs:
' + errorMsgs.join('
')); + return; + } } var barcodes = 'Illumina'; diff --git a/singlecell/resources/web/singlecell/panel/PoolImportPanel.js b/singlecell/resources/web/singlecell/panel/PoolImportPanel.js index ef39125eb..8f7855897 100644 --- a/singlecell/resources/web/singlecell/panel/PoolImportPanel.js +++ b/singlecell/resources/web/singlecell/panel/PoolImportPanel.js @@ -56,6 +56,7 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { name: 'assaytype', labels: ['Assay Type', 'Assay Type', 'Assay', 'treatment'], allowRowSpan: false, + alwaysShow: true, allowBlank: false, transform: 'assaytype' },{ @@ -159,11 +160,16 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { }, assaytype: function(val, panel) { + var requireAssayType = panel.down('#requireAssayType').getValue(); if (val && (val === '--' || val === '-')) { - val = 'N/A'; + val = null; + } + + if (!requireAssayType && !val) { + return 'N/A'; } - return val || 'N/A'; + return val; }, subject: function(val, panel) { @@ -387,6 +393,28 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { }); this.callParent(arguments); + + Ext4.Msg.wait('Loading...'); + LABKEY.Ajax.request({ + method: 'POST', + url: LABKEY.ActionURL.buildURL('singlecell', 'getTenXImportDefaults'), + scope: this, + success: function(response){ + LDK.Utils.decodeHttpResponseJson(response); + if (response.responseJSON){ + this.configDefaults = response.responseJSON; + for (var name in this.configDefaults){ + var item = this.down('#' + name); + if (item){ + item.setValue(this.configDefaults[name]); + } + } + + Ext4.Msg.hide(); + } + }, + failure: LDK.Utils.getErrorCallback() + }); }, getPanelItems: function(){ @@ -466,6 +494,31 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { linkCls: 'labkey-text-link', href: LABKEY.ActionURL.buildURL('query', 'executeQuery', Laboratory.Utils.getQueryContainerPath(), {schemaName: 'singlecell', 'query.queryName': 'stim_types'}), style: 'margin-top: 10px;' + }, { + xtype: 'ldk-linkbutton', + width: null, + hidden: !LABKEY.Security.currentUser.isAdmin, + text: 'Set Page Defaults', + itemId: 'copyPrevious', + linkCls: 'labkey-text-link', + scope: this, + handler: function (btn) { + Ext4.create('Ext.window.Window', { + title: 'Set Page Defaults', + items: [{ + xtype: 'singlecell-tenxsettingspanel', + border: false, + hidePageLoadWarning: false, + hideButtons: true + }], + buttons: SingleCell.panel.TenxSettingsPanel.getButtons().concat([{ + text: 'Cancel', + handler: function (btn) { + btn.up('window').close(); + } + }]) + }).show(); + } },{ xtype: 'textfield', style: 'margin-top: 20px;', @@ -506,6 +559,11 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { fieldLabel: 'Require Cite-Seq Library', itemId: 'requireCITE', checked: false + },{ + xtype: 'checkbox', + fieldLabel: 'Require Assay Type', + itemId: 'requireAssayType', + checked: true },{ xtype: 'checkbox', fieldLabel: 'Combine Hashing and Cite-Seq Libraries', @@ -548,6 +606,12 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { fieldLabel: 'Use 10x V2/HT (Dual Index)', itemId: 'useDualIndex', checked: true + },{ + xtype: 'checkbox', + fieldLabel: '# Cells Indicates Totla Per Lane', + helpPopup: '', + itemId: 'cellsReportedAsTotalPerLane', + checked: true },{ xtype: 'checkbox', fieldLabel: 'Use MS (Dual Index)', @@ -764,6 +828,7 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { data[col.name] = cell; + // This indicates that the first row from the plateId has a value for cells, but this does not. if (!cell && col.name === 'cells' && lastValueByCol[colIdx]) { doSplitCellsByPool = true; } @@ -774,7 +839,8 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { }, this); //split cells across rows - if (doSplitCellsByPool) { + var cellsReportedAsTotalPerLane = this.down('#cellsReportedAsTotalPerLane').getValue(); + if (cellsReportedAsTotalPerLane || doSplitCellsByPool) { var cellCountMap = {}; Ext4.Array.forEach(ret, function(data) { if (data.plateId) { @@ -786,8 +852,18 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { Ext4.Array.forEach(Ext4.Object.getKeys(cellCountMap), function(plateId) { var arr = cellCountMap[plateId]; var size = arr.length; + + // Two allowable patterns: + // 1) the first row has a value and rest are blank. Take this as the lane total + // 2) all rows have the same value, so take the first as the lane total arr = Ext4.Array.remove(arr, null); arr = Ext4.Array.remove(arr, ''); + + // Only attempt to collapse if this was selected: + if (cellsReportedAsTotalPerLane) { + arr = Ext4.unique(arr); + } + if (arr.length === 1) { cellCountMap[plateId] = arr[0] / size; } @@ -1049,6 +1125,7 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { var data = []; var missingValues = false; var requireHTO = this.down('#requireHTO').getValue() || (this.down('#requireHashTag') && this.down('#requireHashTag').getValue()); + var requireAssayType = this.down('#requireAssayType').getValue() Ext4.Array.forEach(parsedRows, function(row, rowIdx){ var toAdd = [rowIdx + 1]; Ext4.Array.forEach(colIdxs, function(colIdx){ @@ -1060,6 +1137,10 @@ Ext4.define('SingleCell.panel.PoolImportPanel', { allowBlank = false; } + if (requireAssayType && colDef.name == 'assaytype') { + allowBlank = false; + } + if (allowBlank === false && Ext4.isEmpty(row[propName])){ missingValues = true; toAdd.push('MISSING'); diff --git a/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js b/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js index 52ebeec15..242b90c1f 100644 --- a/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js +++ b/singlecell/resources/web/singlecell/panel/SingleCellProcessingPanel.js @@ -97,6 +97,14 @@ Ext4.define('SingleCell.panel.SingleCellProcessingPanel', { inputValue: true, checked: true, xtype: 'checkbox' + },{ + fieldLabel: 'Perform Cleanup After Each Step', + helpPopup: 'Is selected, intermediate files from this job will be deleted after each step, instead of once at the end of the job. This can reduce the working directory size. Note: this will only apply if deleteIntermediateFiles is selected, and this is not supported across every possible pipeline type.', + name: 'performCleanupAfterEachStep', + inputValue: true, + uncheckedValue: false, + checked: true, + xtype: 'checkbox' }, this.getSaveTemplateCfg()] }; }, diff --git a/singlecell/resources/web/singlecell/panel/TenxSettingsPanel.js b/singlecell/resources/web/singlecell/panel/TenxSettingsPanel.js new file mode 100644 index 000000000..f0a826475 --- /dev/null +++ b/singlecell/resources/web/singlecell/panel/TenxSettingsPanel.js @@ -0,0 +1,94 @@ +Ext4.define('SingleCell.panel.TenxSettingsPanel', { + extend: 'Ext.panel.Panel', + alias: 'widget.singlecell-tenxsettingspanel', + + hidePageLoadWarning: true, + hideButtons: false, + maxWidth: 650, + + initComponent: function(){ + Ext4.applyIf(this, { + bodyStyle: 'padding: 5px;', + items: [{ + html: 'Loading...', + border: false + }], + buttons: this.hideButtons ? null : SingleCell.panel.TenxSettingsPanel.getButtons() + }); + + this.callParent(arguments); + + LABKEY.Ajax.request({ + method: 'POST', + url: LABKEY.ActionURL.buildURL('singlecell', 'getTenXImportDefaults'), + scope: this, + success: this.onDataLoad, + failure: LDK.Utils.getErrorCallback() + }); + }, + + onDataLoad: function(response){ + LDK.Utils.decodeHttpResponseJson(response); + this.removeAll(); + + if (response.responseJSON){ + var configDefaults = response.responseJSON; + var items = [{ + html: 'Note: you must reload this page before any change will be applied.', + border: false, + hidden: !!this.hidePageLoadWarning + },{ + xtype: 'checkbox', + fieldLabel: 'Require Assay Type', + labelWidth: 300, + itemId: 'requireAssayType', + checked: !!JSON.parse(configDefaults.requireAssayType ?? false) + },{ + xtype: 'checkbox', + fieldLabel: 'Combine Hashing and Cite-Seq', + labelWidth: 300, itemId: 'combineHashingCite', + checked: !!JSON.parse(configDefaults.combineHashingCite ?? false) + }]; + + this.add(items); + } + else { + this.add({html: 'Something went wrong loading saved data'}); + } + }, + + statics: { + getButtons: function () { + return [{ + text: 'Submit', + handler: function (btn) { + var win = btn.up('window'); + var panel = win ? win.down('singlecell-tenxsettingspanel') : btn.up('singlecell-tenxsettingspanel'); + + var params = {}; + params['requireAssayType'] = panel.down('#requireAssayType').getValue(); + params['combineHashingCite'] = panel.down('#combineHashingCite').getValue(); + + Ext4.Msg.wait('Saving...'); + LABKEY.Ajax.request({ + method: 'POST', + url: LABKEY.ActionURL.buildURL('singlecell', 'setTenXImportDefaults'), + jsonData: params, + scope: panel, + success: panel.onSuccess, + failure: LDK.Utils.getErrorCallback() + }) + } + }]; + } + }, + + onSuccess: function(){ + Ext4.Msg.hide(); + Ext4.Msg.alert('Success', 'Settings have been saved'); + + if (this.up('window')){ + this.up('window').close(); + } + } +}); \ No newline at end of file diff --git a/singlecell/resources/web/singlecell/panel/cDNAImportPanel.js b/singlecell/resources/web/singlecell/panel/cDNAImportPanel.js index 427484cbd..a02351135 100644 --- a/singlecell/resources/web/singlecell/panel/cDNAImportPanel.js +++ b/singlecell/resources/web/singlecell/panel/cDNAImportPanel.js @@ -96,7 +96,32 @@ Ext4.define('SingleCell.panel.cDNAImportPanel', { scope: this, href: LABKEY.ActionURL.getContextPath() + '/singlecell/exampleData/ImportReadsetTemplate.xlsx' }] - }, { + },{ + xtype: 'ldk-linkbutton', + hidden: !LABKEY.Security.currentUser.isAdmin, + style: 'margin-top: 10px;', + text: 'Set Page Defaults', + itemId: 'copyPrevious', + linkCls: 'labkey-text-link', + scope: this, + handler: function (btn) { + Ext4.create('Ext.window.Window', { + title: 'Set Page Defaults', + items: [{ + xtype: 'singlecell-tenxsettingspanel', + border: false, + hidePageLoadWarning: false, + hideButtons: true + }], + buttons: SingleCell.panel.TenxSettingsPanel.getButtons().concat([{ + text: 'Cancel', + handler: function (btn) { + btn.up('window').close(); + } + }]) + }).show(); + } + },{ xtype: 'textfield', style: 'margin-top: 20px;', fieldLabel: 'Expt Number', diff --git a/singlecell/src/org/labkey/singlecell/CellHashingServiceImpl.java b/singlecell/src/org/labkey/singlecell/CellHashingServiceImpl.java index 4657ddf8f..1df529330 100644 --- a/singlecell/src/org/labkey/singlecell/CellHashingServiceImpl.java +++ b/singlecell/src/org/labkey/singlecell/CellHashingServiceImpl.java @@ -294,7 +294,7 @@ public void prepareHashingAndCiteSeqFilesIfNeeded(File sourceDir, PipelineJob jo } } - support.cacheReadset(hashingReadsetId, job.getUser()); + support.cacheReadset(hashingReadsetId, job.getUser(), true); }); @@ -356,7 +356,7 @@ else if (distinctHTOs.size() == 1) } } - support.cacheReadset(citeseqReadsetId, job.getUser()); + support.cacheReadset(citeseqReadsetId, job.getUser(), true); }); citeToRemove.forEach(readsetToCiteSeqMap::remove); diff --git a/singlecell/src/org/labkey/singlecell/SingleCellController.java b/singlecell/src/org/labkey/singlecell/SingleCellController.java index 13bef2154..a0b32e0d7 100644 --- a/singlecell/src/org/labkey/singlecell/SingleCellController.java +++ b/singlecell/src/org/labkey/singlecell/SingleCellController.java @@ -21,6 +21,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONArray; +import org.labkey.api.action.ApiResponse; import org.labkey.api.action.ApiSimpleResponse; import org.labkey.api.action.ApiUsageException; import org.labkey.api.action.ExportAction; @@ -35,6 +36,7 @@ import org.labkey.api.data.ContainerManager; import org.labkey.api.data.ContainerType; import org.labkey.api.data.DbScope; +import org.labkey.api.data.PropertyManager; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; @@ -47,6 +49,7 @@ import org.labkey.api.query.UserSchema; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminPermission; import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.sequenceanalysis.SequenceOutputFile; @@ -494,4 +497,64 @@ public void addNavTrail(NavTree tree) } } + public final static String CONFIG_PROPERTY_DOMAIN_IMPORT = "org.labkey.singlecell.importsettings"; + + @RequiresPermission(ReadPermission.class) + public static class GetTenXImportDefaultsAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(Object form, BindException errors) throws Exception + { + Container target = getContainer().isWorkbook() ? getContainer().getParent() : getContainer(); + Map resultProperties = new HashMap<>(PropertyManager.getProperties(target, CONFIG_PROPERTY_DOMAIN_IMPORT)); + + return new ApiSimpleResponse(resultProperties); + } + } + + @RequiresPermission(AdminPermission.class) + public static class SetTenXImportDefaultsAction extends MutatingApiAction + { + public static final String REQUIRE_ASSAY_TYPE = "requireAssayType"; + public static final String COMBINE_HASHING_CITE = "combineHashingCite"; + + @Override + public ApiResponse execute(SetSequenceImportDefaultsForm form, BindException errors) throws Exception + { + Container target = getContainer().isWorkbook() ? getContainer().getParent() : getContainer(); + PropertyManager.PropertyMap configMap = PropertyManager.getWritableProperties(target, CONFIG_PROPERTY_DOMAIN_IMPORT, true); + configMap.put(REQUIRE_ASSAY_TYPE, Boolean.valueOf(form.isRequireAssayType()).toString()); + configMap.put(COMBINE_HASHING_CITE, Boolean.valueOf(form.isCombineHashingCite()).toString()); + + configMap.save(); + + return new ApiSimpleResponse("success", true); + } + } + + public static class SetSequenceImportDefaultsForm + { + private boolean _requireAssayType = false; + private boolean _combineHashingCite = false; + + public boolean isRequireAssayType() + { + return _requireAssayType; + } + + public void setRequireAssayType(boolean requireAssayType) + { + _requireAssayType = requireAssayType; + } + + public boolean isCombineHashingCite() + { + return _combineHashingCite; + } + + public void setCombineHashingCite(boolean combineHashingCite) + { + _combineHashingCite = combineHashingCite; + } + } } diff --git a/singlecell/src/org/labkey/singlecell/SingleCellTableCustomizer.java b/singlecell/src/org/labkey/singlecell/SingleCellTableCustomizer.java index 8b578f61a..4d1776473 100644 --- a/singlecell/src/org/labkey/singlecell/SingleCellTableCustomizer.java +++ b/singlecell/src/org/labkey/singlecell/SingleCellTableCustomizer.java @@ -94,6 +94,18 @@ private void customizeCdnas(AbstractTableInfo ti) } LDKService.get().applyNaturalSort(ti, "plateId"); + + String uniqueHTOs = "uniqueHtos"; + if (ti.getColumn(uniqueHTOs) == null) + { + SQLFragment sql = new SQLFragment("(SELECT count(DISTINCT s.hto) as expr FROM " + SingleCellSchema.NAME + "." + SingleCellSchema.TABLE_SORTS + " s WHERE s.rowid IN (SELECT DISTINCT sortId FROM " + SingleCellSchema.NAME + "." + SingleCellSchema.TABLE_CDNAS + " c WHERE c.plateid = " + ExprColumn.STR_TABLE_ALIAS + ".plateid AND c.container = " + ExprColumn.STR_TABLE_ALIAS + ".container))"); + ExprColumn newCol = new ExprColumn(ti, uniqueHTOs, sql, JdbcType.INTEGER, ti.getColumn("plateId"), ti.getColumn("container")); + newCol.setLabel("Distinct HTOs In Lane"); + DetailsURL details = DetailsURL.fromString("/query/executeQuery.view?schemaName=singlecell&query.queryName=cdna_libraries&query.plateId~eq=${plateId}", ti.getUserSchema().getContainer()); + newCol.setURL(details); + + ti.addColumn(newCol); + } } private void customizeSorts(AbstractTableInfo ti) diff --git a/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java b/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java index 7d7c024ba..d2b95a86c 100644 --- a/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java +++ b/singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java @@ -250,7 +250,7 @@ else if (so.getReadset() == null) throw new PipelineJobException("Readset is blank for loupe file: " + loupeId); } - ctx.getSequenceSupport().cacheReadset(so.getReadset(), ctx.getJob().getUser()); + ctx.getSequenceSupport().cacheReadset(so.getReadset(), ctx.getJob().getUser(), true); } } else diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java index bd813c840..bdfe7cb04 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendNimble.java @@ -7,6 +7,7 @@ import org.labkey.api.sequenceanalysis.pipeline.PipelineContext; import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler; import org.labkey.api.sequenceanalysis.pipeline.ToolParameterDescriptor; +import org.labkey.api.singlecell.pipeline.SeuratToolParameter; import org.labkey.api.singlecell.pipeline.SingleCellStep; import java.util.Arrays; @@ -31,14 +32,21 @@ public Provider() {{ put("allowBlank", false); }}, null), - ToolParameterDescriptor.create("retainAmbiguousFeatures", "Retain Ambiguous Features", "If checked, features hitting more than one reference will be retained", "checkbox", new JSONObject() + SeuratToolParameter.create("retainAmbiguousFeatures", "Retain Ambiguous Features", "If checked, features hitting more than one reference will be retained", "checkbox", new JSONObject() {{ put("check", false); - }}, false) + }}, false, null, true), + SeuratToolParameter.create("ensureSamplesShareAllGenomes", "Ensure Samples Share All Genomes", "If checked, the job will fail unless nimble data is found for each requested genome for all samples", "checkbox", new JSONObject() + {{ + put("check", true); + }}, true, null, true), + SeuratToolParameter.create("maxLibrarySizeRatio", "Max Library Size Ratio", "This normalization relies on the assumption that the library size of the assay being normalized in negligible relative to the assayForLibrarySize. To verify this holds true, the method will error if librarySize(assayToNormalize)/librarySize(assayForLibrarySize) exceeds this value", "ldk-numberfield", new JSONObject() + {{ + put("decimalPrecision", 4); + }}, 0.1, null, true) ), Arrays.asList("sequenceanalysis/field/GenomeField.js", "/singlecell/panel/NimbleAppendPanel.js"), null); } - @Override public AppendNimble create(PipelineContext ctx) { diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendTcr.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendTcr.java index 728cbd204..0176d2766 100644 --- a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendTcr.java +++ b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/AppendTcr.java @@ -23,8 +23,8 @@ public Provider() super("AppendTcr", "Append TCR Data", "RDiscvr", "This uses Rdiscvr::DownloadAndAppendTcrClonotypes to append TCR data.", List.of( SeuratToolParameter.create("allowMissing", "Allow Missing Data", "If checked, an error will be thrown if any sample lacks TCR data", "checkbox", new JSONObject() {{ - put("checked", true); - }}, true) + put("checked", false); + }}, false) ), null, null); }