diff --git a/elisa/src/org/labkey/elisa/AbstractElisaImportHelper.java b/elisa/src/org/labkey/elisa/AbstractElisaImportHelper.java index 3df946142..cc913c6bb 100644 --- a/elisa/src/org/labkey/elisa/AbstractElisaImportHelper.java +++ b/elisa/src/org/labkey/elisa/AbstractElisaImportHelper.java @@ -14,8 +14,8 @@ import org.labkey.api.exp.api.ProvenanceService; import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.DomainProperty; +import org.labkey.vfs.FileLike; -import java.io.File; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -26,11 +26,11 @@ public abstract class AbstractElisaImportHelper implements ElisaImportHelper protected AssayUploadXarContext _context; protected PlateBasedAssayProvider _provider; protected ExpProtocol _protocol; - protected File _dataFile; + protected FileLike _dataFile; protected Container _container; Map _specimenGroupMap; - public AbstractElisaImportHelper(AssayUploadXarContext context, PlateBasedAssayProvider provider, ExpProtocol protocol, File dataFile) + public AbstractElisaImportHelper(AssayUploadXarContext context, PlateBasedAssayProvider provider, ExpProtocol protocol, FileLike dataFile) { _context = context; _provider = provider; diff --git a/elisa/src/org/labkey/elisa/ElisaDataHandler.java b/elisa/src/org/labkey/elisa/ElisaDataHandler.java index 67d0512cb..ce09a0bdb 100644 --- a/elisa/src/org/labkey/elisa/ElisaDataHandler.java +++ b/elisa/src/org/labkey/elisa/ElisaDataHandler.java @@ -107,7 +107,7 @@ protected boolean shouldAddInputMaterials() return true; } - private ElisaImportHelper getImportHelper(AssayUploadXarContext context, PlateBasedAssayProvider provider, ExpProtocol protocol, File dataFile) throws ExperimentException + private ElisaImportHelper getImportHelper(AssayUploadXarContext context, PlateBasedAssayProvider provider, ExpProtocol protocol, FileLike dataFile) throws ExperimentException { if (provider.getMetadataInputFormat(protocol).equals(SampleMetadataInputFormat.MANUAL)) { @@ -135,7 +135,7 @@ public Map getValidationDataMap(ExpData data, Fil Map sampleProperties = plateProvider.getSampleWellGroupDomain(protocol) .getProperties().stream() .collect(Collectors.toMap(DomainProperty::getName, dp -> dp)); - ElisaImportHelper importHelper = getImportHelper(xarContext, plateProvider, protocol, dataFile.toNioPathForRead().toFile()); + ElisaImportHelper importHelper = getImportHelper(xarContext, plateProvider, protocol, dataFile); for (String plateName : importHelper.getPlates()) { @@ -148,7 +148,7 @@ public Map getValidationDataMap(ExpData data, Fil SimpleRegression regression = new SimpleRegression(true); Map standardConcentrations = importHelper.getStandardConcentrations(plateName, analytePlateEntry.getKey()); - CurveFit standardCurve = calculateStandardCurve(run, plate, regression, standardConcentrations, runDomain); + CurveFit standardCurve = calculateStandardCurve(run, plate, regression, standardConcentrations, runDomain); if (standardCurve != null && standardCurve.getParameters() == null) throw new ExperimentException("Unable to fit the standard concentrations to a curve, please check the input data and try again"); @@ -321,7 +321,7 @@ private boolean isRowEmptyOrNull(Map row) * data and can be used to generate an R squared value. */ @Nullable - private CurveFit calculateStandardCurve(ExpRun run, Plate plate, @Nullable SimpleRegression regression, Map standardConcentrations, + private CurveFit calculateStandardCurve(ExpRun run, Plate plate, @Nullable SimpleRegression regression, Map standardConcentrations, Domain runDomain) throws ExperimentException { // compute the calibration curve, there could be multiple control groups but one contains the standards @@ -333,7 +333,7 @@ private CurveFit calculateStandardCurve(ExpRun run, Plate plate, @Nullable Simpl for (WellGroup replicate : stdWellGroup.getOverlappingGroups(WellGroup.Type.REPLICATE)) { - maxValue = replicate.getMean() > maxValue ? replicate.getMean() : maxValue; + maxValue = Math.max(replicate.getMean(), maxValue); } for (WellGroup replicate : stdWellGroup.getOverlappingGroups(WellGroup.Type.REPLICATE)) @@ -358,7 +358,7 @@ private CurveFit calculateStandardCurve(ExpRun run, Plate plate, @Nullable Simpl // Compute curve fit parameters based on the selected curve fit (default to linear for legacy assay designs) StatsService.CurveFitType curveFitType = ElisaManager.getRunCurveFitType(runDomain, run); - CurveFit curveFit = StatsService.get().getCurveFit(curveFitType, points.toArray(DoublePoint[]::new)); + CurveFit curveFit = StatsService.get().getCurveFit(curveFitType, points.toArray(DoublePoint[]::new)); curveFit.setLogXScale(false); curveFit.setAssumeCurveDecreasing(false); diff --git a/elisa/src/org/labkey/elisa/ElisaSampleFilePropertyHelper.java b/elisa/src/org/labkey/elisa/ElisaSampleFilePropertyHelper.java index 88e2a8961..08ce8089e 100644 --- a/elisa/src/org/labkey/elisa/ElisaSampleFilePropertyHelper.java +++ b/elisa/src/org/labkey/elisa/ElisaSampleFilePropertyHelper.java @@ -16,8 +16,10 @@ import org.labkey.api.study.assay.SampleMetadataInputFormat; import jakarta.servlet.http.HttpServletRequest; -import java.io.File; +import org.labkey.vfs.FileLike; + import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -41,13 +43,14 @@ public Map> getSampleProperties(HttpServletR if (_sampleProperties != null) return _sampleProperties; - File metadataFile = getSampleMetadata(request); + FileLike metadataFile = getSampleMetadata(request); if (metadataFile == null) throw new ExperimentException("No metadata or data file provided"); Map> allProperties = new HashMap<>(); DataLoaderFactory factory = DataLoaderService.get().findFactory(metadataFile, null); - try (DataLoader loader = factory.createLoader(metadataFile, true)) + try (InputStream in = metadataFile.openInputStream(); + DataLoader loader = factory.createLoader(in, true)) { validateRequiredColumns(loader.getColumns()); diff --git a/elisa/src/org/labkey/elisa/HighThroughputImportHelper.java b/elisa/src/org/labkey/elisa/HighThroughputImportHelper.java index 24daced2b..4e0011056 100644 --- a/elisa/src/org/labkey/elisa/HighThroughputImportHelper.java +++ b/elisa/src/org/labkey/elisa/HighThroughputImportHelper.java @@ -21,9 +21,10 @@ import org.labkey.api.reader.DataLoader; import org.labkey.api.reader.DataLoaderFactory; import org.labkey.api.reader.DataLoaderService; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -37,7 +38,7 @@ public class HighThroughputImportHelper extends AbstractElisaImportHelper private final Map _plateMap = new HashMap<>(); private Plate _plateTemplate; - public HighThroughputImportHelper(AssayUploadXarContext context, PlateBasedAssayProvider provider, ExpProtocol protocol, File dataFile) throws ExperimentException + public HighThroughputImportHelper(AssayUploadXarContext context, PlateBasedAssayProvider provider, ExpProtocol protocol, FileLike dataFile) throws ExperimentException { super(context, provider, protocol, dataFile); ensureData(); @@ -47,7 +48,8 @@ private void ensureData() throws ExperimentException { _plateTemplate = _provider.getPlate(_protocol.getContainer(), _protocol); DataLoaderFactory factory = DataLoaderService.get().findFactory(_dataFile, null); - try (DataLoader loader = factory.createLoader(_dataFile, true)) + try (InputStream in = _dataFile.openInputStream(); + DataLoader loader = factory.createLoader(in, true)) { String signalColumnName = "Signal"; diff --git a/elisa/src/org/labkey/elisa/ManualImportHelper.java b/elisa/src/org/labkey/elisa/ManualImportHelper.java index f0f78ec20..ba093dfe0 100644 --- a/elisa/src/org/labkey/elisa/ManualImportHelper.java +++ b/elisa/src/org/labkey/elisa/ManualImportHelper.java @@ -20,6 +20,7 @@ import org.labkey.api.exp.property.DomainProperty; import org.labkey.elisa.actions.ElisaRunUploadForm; import org.labkey.elisa.plate.BioTekPlateReader; +import org.labkey.vfs.FileLike; import java.io.File; import java.util.HashMap; @@ -32,7 +33,7 @@ */ public class ManualImportHelper extends AbstractElisaImportHelper { - public ManualImportHelper(AssayUploadXarContext context, PlateBasedAssayProvider provider, ExpProtocol protocol, File dataFile) + public ManualImportHelper(AssayUploadXarContext context, PlateBasedAssayProvider provider, ExpProtocol protocol, FileLike dataFile) { super(context, provider, protocol, dataFile); } diff --git a/elispotassay/src/org/labkey/elispot/AbstractElispotDataHandler.java b/elispotassay/src/org/labkey/elispot/AbstractElispotDataHandler.java index 8779272e1..cadabfd5f 100644 --- a/elispotassay/src/org/labkey/elispot/AbstractElispotDataHandler.java +++ b/elispotassay/src/org/labkey/elispot/AbstractElispotDataHandler.java @@ -17,6 +17,7 @@ package org.labkey.elispot; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import org.labkey.api.assay.AssayUrls; import org.labkey.api.assay.plate.Position; import org.labkey.api.data.Container; @@ -39,8 +40,8 @@ import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; -import java.io.File; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -79,10 +80,10 @@ public interface ElispotDataFileParser List> getResults() throws ExperimentException; } - public abstract ElispotDataFileParser getDataFileParser(ExpData data, File dataFile, ViewBackgroundInfo info, Logger log, XarContext context); + public abstract ElispotDataFileParser getDataFileParser(ExpData data, FileLike dataFile, ViewBackgroundInfo info, Logger log, XarContext context); @Override - public void importFile(ExpData data, File dataFile, ViewBackgroundInfo info, Logger log, XarContext context) throws ExperimentException + public void importFile(@NotNull ExpData data, @NotNull FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException { ExpRun run = data.getRun(); diff --git a/elispotassay/src/org/labkey/elispot/ElispotDataHandler.java b/elispotassay/src/org/labkey/elispot/ElispotDataHandler.java index 91bf4aa86..e5a06a3b8 100644 --- a/elispotassay/src/org/labkey/elispot/ElispotDataHandler.java +++ b/elispotassay/src/org/labkey/elispot/ElispotDataHandler.java @@ -89,10 +89,10 @@ public DataType getDataType() static class ElispotFileParser implements ElispotDataFileParser { private final ExpData _data; - private final File _dataFile; + private final FileLike _dataFile; private final XarContext _context; - public ElispotFileParser(ExpData data, File dataFile, XarContext context) + public ElispotFileParser(ExpData data, FileLike dataFile, XarContext context) { _data = data; _dataFile = dataFile; @@ -199,7 +199,7 @@ else if (SPOT_SIZE_PROPERTY_NAME.equalsIgnoreCase(measurement)) } @Override - public ElispotDataFileParser getDataFileParser(ExpData data, File dataFile, ViewBackgroundInfo info, Logger log, XarContext context) + public ElispotDataFileParser getDataFileParser(ExpData data, FileLike dataFile, ViewBackgroundInfo info, Logger log, XarContext context) { return new ElispotFileParser(data, dataFile, context); } @@ -213,7 +213,7 @@ public void importTransformDataMap(ExpData data, AssayRunUploadContext contex @Override public Map getValidationDataMap(ExpData data, FileLike dataFile, ViewBackgroundInfo info, Logger log, XarContext context, DataLoaderSettings settings) throws ExperimentException { - ElispotDataFileParser parser = getDataFileParser(data, dataFile.toNioPathForRead().toFile(), info, log, context); + ElispotDataFileParser parser = getDataFileParser(data, dataFile, info, log, context); Map datas = new HashMap<>(); List> rows = parser.getResults(); @@ -222,7 +222,7 @@ public Map getValidationDataMap(ExpData data, Fil return datas; } - public static Map initializePlates(ExpProtocol protocol, File dataFile, Plate template, PlateReader reader) throws ExperimentException + public static Map initializePlates(ExpProtocol protocol, FileLike dataFile, Plate template, PlateReader reader) throws ExperimentException { AssayProvider provider = AssayService.get().getProvider(protocol); Map plateMap = new HashMap<>(); diff --git a/elispotassay/src/org/labkey/elispot/ElispotUploadWizardAction.java b/elispotassay/src/org/labkey/elispot/ElispotUploadWizardAction.java index 75ff956ce..0ebabad48 100644 --- a/elispotassay/src/org/labkey/elispot/ElispotUploadWizardAction.java +++ b/elispotassay/src/org/labkey/elispot/ElispotUploadWizardAction.java @@ -519,7 +519,7 @@ protected ExpRun finishPost(ElispotRunUploadForm form, BindException errors) if (runPropMap.containsKey(ElispotAssayProvider.READER_PROPERTY_NAME)) { reader = provider.getPlateReader(runPropMap.get(ElispotAssayProvider.READER_PROPERTY_NAME)); - plates = ElispotDataHandler.initializePlates(form.getProtocol(), data.get(0).getFile(), template, reader); + plates = ElispotDataHandler.initializePlates(form.getProtocol(), data.get(0).getFileLike(), template, reader); } boolean subtractBackground = NumberUtils.toInt(runPropMap.get(ElispotAssayProvider.BACKGROUND_WELL_PROPERTY_NAME), 0) > 0; diff --git a/elispotassay/src/org/labkey/elispot/PlateAnalytePropertyHelper.java b/elispotassay/src/org/labkey/elispot/PlateAnalytePropertyHelper.java index 89fd44847..d83a37729 100644 --- a/elispotassay/src/org/labkey/elispot/PlateAnalytePropertyHelper.java +++ b/elispotassay/src/org/labkey/elispot/PlateAnalytePropertyHelper.java @@ -61,7 +61,7 @@ public PlateAnalytePropertyHelper(ElispotRunUploadForm form, List entry : ElispotDataHandler.initializePlates(run.getProtocol(), data.get(0).getFile(), template, reader).entrySet()) + for (Map.Entry entry : ElispotDataHandler.initializePlates(run.getProtocol(), dataFile, template, reader).entrySet()) { if (entry.getKey().getMeasurement().equals(ElispotDataHandler.SFU_PROPERTY_NAME)) { diff --git a/elispotassay/src/org/labkey/elispot/plate/AIDPlateReader.java b/elispotassay/src/org/labkey/elispot/plate/AIDPlateReader.java index a5deaf96b..ad98a0730 100644 --- a/elispotassay/src/org/labkey/elispot/plate/AIDPlateReader.java +++ b/elispotassay/src/org/labkey/elispot/plate/AIDPlateReader.java @@ -22,9 +22,10 @@ import org.labkey.api.query.ValidationException; import org.labkey.api.reader.DataLoader; import org.labkey.api.reader.DataLoaderFactory; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.List; /** @@ -55,13 +56,14 @@ else if ("--".equalsIgnoreCase(token)) } @Override - public double[][] loadFile(Plate template, File dataFile) throws ExperimentException + public double[][] loadFile(Plate template, FileLike dataFile) throws ExperimentException { String fileName = dataFile.getName().toLowerCase(); if (fileName.endsWith(".xls") || fileName.endsWith(".xlsx")) { DataLoaderFactory factory = DataLoader.get().findFactory(dataFile, null); - try (DataLoader loader = factory.createLoader(dataFile, false)) + try (InputStream in = dataFile.openInputStream(); + DataLoader loader = factory.createLoader(in, false)) { return PlateUtils.parseGrid(dataFile, loader.load(), template.getRows(), template.getColumns(), this); } @@ -77,13 +79,14 @@ public double[][] loadFile(Plate template, File dataFile) throws ExperimentExcep } @Override - public List loadMultiGridFile(Plate template, File dataFile) throws ExperimentException + public List loadMultiGridFile(Plate template, FileLike dataFile) throws ExperimentException { String fileName = dataFile.getName().toLowerCase(); if (fileName.endsWith(".xls") || fileName.endsWith(".xlsx")) { DataLoaderFactory factory = DataLoader.get().findFactory(dataFile, null); - try (DataLoader loader = factory.createLoader(dataFile, false)) + try (InputStream in = dataFile.openInputStream(); + DataLoader loader = factory.createLoader(in, false)) { return PlateUtils.parseAllGrids(dataFile, loader.load(), template.getRows(), template.getColumns(), this); } diff --git a/flow/enginesrc/org/labkey/flow/Main.java b/flow/enginesrc/org/labkey/flow/Main.java index c5ec419e7..7258035a4 100644 --- a/flow/enginesrc/org/labkey/flow/Main.java +++ b/flow/enginesrc/org/labkey/flow/Main.java @@ -95,7 +95,7 @@ private static Workspace readWorkspace(File file, boolean printWarnings) private static File uniqueFile(File dir, String name) { - File file = new File(dir, name); + File file = FileUtil.appendName(dir, name); if (file.exists()) { String base = name; @@ -108,7 +108,7 @@ private static File uniqueFile(File dir, String name) } for (int i = 1; file.exists(); i++) - file = new File(dir, base + i + ext); + file = FileUtil.appendName(dir, base + i + ext); } return file; @@ -244,7 +244,7 @@ private static void writeAnalysis(File outDir, String name, Workspace workspace, { XmlOptions options = new XmlOptions(); options.setSavePrettyPrint(); - doc.save(new File(outDir, name), options); + doc.save(FileUtil.appendName(outDir, name), options); } catch (IOException ioe) { diff --git a/flow/enginesrc/org/labkey/flow/analysis/model/FCS.java b/flow/enginesrc/org/labkey/flow/analysis/model/FCS.java index 5aabca400..cf1cc7750 100644 --- a/flow/enginesrc/org/labkey/flow/analysis/model/FCS.java +++ b/flow/enginesrc/org/labkey/flow/analysis/model/FCS.java @@ -19,6 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.StringUtilsLabKey; import java.io.*; @@ -368,7 +369,7 @@ public boolean accept(File dir, String name) return ext.equals(".fcs") || ext.equals(".facs") || ext.equals(".lmd"); } else - return isFCSFile(new File(dir,name)); + return isFCSFile(FileUtil.appendName(dir, name)); } } diff --git a/flow/enginesrc/org/labkey/flow/analysis/web/PlotTests.java b/flow/enginesrc/org/labkey/flow/analysis/web/PlotTests.java index c932b2d5a..9ab2b06e6 100644 --- a/flow/enginesrc/org/labkey/flow/analysis/web/PlotTests.java +++ b/flow/enginesrc/org/labkey/flow/analysis/web/PlotTests.java @@ -20,6 +20,7 @@ import org.junit.Test; import org.labkey.api.util.FileUtil; import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.Path; import org.labkey.api.writer.PrintWriters; import org.labkey.flow.analysis.model.Analysis; import org.labkey.flow.analysis.model.CompensationMatrix; @@ -78,11 +79,10 @@ private Map dumpPlots(File outDir, CompensationMatrix comp, A for (FCSAnalyzer.GraphResult graph : graphs) { String imgName = AnalysisSerializer.generateFriendlyImageName(graph.spec); - File image = new File(outDir, imgName); + File image = FileUtil.appendName(outDir, imgName); FileOutputStream fos = new FileOutputStream(image); IOUtils.write(graph.bytes, fos); images.put(imgName, graph.spec); - System.out.println(" " + graph.spec); } return images; } @@ -106,7 +106,7 @@ private void compare(File outDir, File workspaceFile, File fcsFile, File expecte generateHtml(outDir, workspaceFile, fcsFile, expectedImageDir, generatedImages); Desktop desktop = Desktop.getDesktop(); - desktop.browse(new File(outDir, "index.html").toURI()); + desktop.browse(FileUtil.appendName(outDir, "index.html").toURI()); } private void generateHtml(File outDir, File workspaceFile, File fcsFile, File expectedImageDir, Map generatedImages) throws IOException @@ -162,8 +162,8 @@ private void generateHtml(File outDir, File workspaceFile, File fcsFile, File ex sb.append(""); sb.append(""); sb.append(""); - sb.append(""); - sb.append(""); + sb.append(""); + sb.append(""); sb.append(""); } @@ -174,7 +174,7 @@ private void generateHtml(File outDir, File workspaceFile, File fcsFile, File ex sb.append(""); sb.append(""); - try (PrintWriter writer = PrintWriters.getPrintWriter(new File(outDir, "index.html"))) + try (PrintWriter writer = PrintWriters.getPrintWriter(FileUtil.appendName(outDir, "index.html"))) { writer.append(sb.toString()); writer.flush(); @@ -198,7 +198,7 @@ public void generatePlotsAndCompare(File outDir, File workspaceFile, File fcsFil @Test public void advanced() throws Exception { - File outDir = new File(outDir(), "flow/advanced"); + File outDir = FileUtil.appendPath(outDir(), Path.parse("flow/advanced")); File workspaceFile = JunitUtil.getSampleData(null, "flow/advanced/advanced-v7.6.5.wsp"); File fcsFile = JunitUtil.getSampleData(null, "flow/advanced/931115-B02- Sample 01.fcs"); File expectedImages = JunitUtil.getSampleData(null, "flow/advanced/931115-B02_graphs_v7.6.5"); @@ -212,10 +212,10 @@ public void advanced() throws Exception @Test public void HVTN078() throws Exception { - File outDir = new File(outDir(), "HVTN/HVTN078"); - File workspaceFile = new File(dataDir(), "HVTN/HVTN078/1325-L-078.xml"); - File fcsFile = new File(dataDir(), "HVTN/HVTN078/1012833.fcs"); - File expectedImages = new File(dataDir(), "HVTN/HVTN078/1012833_graphs_v9.4.10"); + File outDir = FileUtil.appendPath(outDir(), Path.parse("HVTN/HVTN078")); + File workspaceFile = FileUtil.appendPath(dataDir(), Path.parse("HVTN/HVTN078/1325-L-078.xml")); + File fcsFile = FileUtil.appendPath(dataDir(), Path.parse("HVTN/HVTN078/1012833.fcs")); + File expectedImages = FileUtil.appendPath(dataDir(), Path.parse("HVTN/HVTN078/1012833_graphs_v9.4.10")); generatePlotsAndCompare(outDir, workspaceFile, fcsFile, expectedImages); } @@ -227,10 +227,10 @@ public void HVTN078() throws Exception @Test public void IAVI315() throws Exception { - File outDir = new File(outDir(), "IAVI/315"); - File workspaceFile = new File(dataDir(), "IAVI/315/workspace.xml"); - File fcsFile = new File(dataDir(), "IAVI/315/SEB_SEB315_A12.fcs"); - File expectedImages = new File(dataDir(), "IAVI/315/SEB_SEB315_A12_graphs_v9.4.10"); + File outDir = FileUtil.appendPath(outDir(), Path.parse("IAVI/315")); + File workspaceFile = FileUtil.appendPath(dataDir(), Path.parse("IAVI/315/workspace.xml")); + File fcsFile = FileUtil.appendPath(dataDir(), Path.parse("IAVI/315/SEB_SEB315_A12.fcs")); + File expectedImages = FileUtil.appendPath(dataDir(), Path.parse("IAVI/315/SEB_SEB315_A12_graphs_v9.4.10")); generatePlotsAndCompare(outDir, workspaceFile, fcsFile, expectedImages); } @@ -240,10 +240,10 @@ public void IAVI315() throws Exception @Test public void ITN027AI() throws Exception { - File outDir = new File(outDir(), "ITN/ITN027AI"); - File workspaceFile = new File(dataDir(), "ITN/ITN027AI/ITN027AI_tube131.xml"); - File fcsFile = new File(dataDir(), "ITN/ITN027AI/ITN-131-01.LMD"); - File expectedImages = new File(dataDir(), "ITN/ITN027AI/ITN-131-01_graphs_v8.8.7"); + File outDir = FileUtil.appendPath(outDir(), Path.parse("ITN/ITN027AI")); + File workspaceFile = FileUtil.appendPath(dataDir(), Path.parse("ITN/ITN027AI/ITN027AI_tube131.xml")); + File fcsFile = FileUtil.appendPath(dataDir(), Path.parse("ITN/ITN027AI/ITN-131-01.LMD")); + File expectedImages = FileUtil.appendPath(dataDir(), Path.parse("ITN/ITN027AI/ITN-131-01_graphs_v8.8.7")); generatePlotsAndCompare(outDir, workspaceFile, fcsFile, expectedImages); } @@ -251,10 +251,10 @@ public void ITN027AI() throws Exception @Test public void ITN030ST() throws Exception { - File outDir = new File(outDir(), "ITN/ITN030ST"); - File workspaceFile = new File(dataDir(), "ITN/ITN030ST/workspace.wsp"); - File fcsFile = new File(dataDir(), "ITN/ITN030ST/10047201_SH01_I007.fcs"); - File expectedImages = new File(dataDir(), "ITN/ITN030ST/10047201_SH01_I007_graphs_v7.6.5"); + File outDir = FileUtil.appendPath(outDir(), Path.parse("ITN/ITN030ST")); + File workspaceFile = FileUtil.appendPath(dataDir(), Path.parse("ITN/ITN030ST/workspace.wsp")); + File fcsFile = FileUtil.appendPath(dataDir(), Path.parse("ITN/ITN030ST/10047201_SH01_I007.fcs")); + File expectedImages = FileUtil.appendPath(dataDir(), Path.parse("ITN/ITN030ST/10047201_SH01_I007_graphs_v7.6.5")); generatePlotsAndCompare(outDir, workspaceFile, fcsFile, expectedImages); } @@ -262,10 +262,10 @@ public void ITN030ST() throws Exception @Test public void ITNPilot() throws Exception { - File outDir = new File(outDir(), "ITN/Pilot"); - File workspaceFile = new File(dataDir(), "ITN/Pilot/workspace.xml"); - File fcsFile = new File(dataDir(), "ITN/Pilot/ITN64.fcs"); - File expectedImages = new File(dataDir(), "ITN/Pilot/ITN64_graphs_v9.4.10"); + File outDir = FileUtil.appendPath(outDir(), Path.parse("ITN/Pilot")); + File workspaceFile = FileUtil.appendPath(dataDir(), Path.parse("ITN/Pilot/workspace.xml")); + File fcsFile = FileUtil.appendPath(dataDir(), Path.parse("ITN/Pilot/ITN64.fcs")); + File expectedImages = FileUtil.appendPath(dataDir(), Path.parse("ITN/Pilot/ITN64_graphs_v9.4.10")); generatePlotsAndCompare(outDir, workspaceFile, fcsFile, expectedImages); } @@ -273,10 +273,10 @@ public void ITNPilot() throws Exception @Test public void LabKeyDemo() throws Exception { - File outDir = new File(outDir(), "labkey-demo"); - File workspaceFile = new File(dataDir(), "labkey-demo/labkey-demo.xml"); - File fcsFile = new File(dataDir(), "labkey-demo/119166.fcs"); - File expectedImages = new File(dataDir(), "labkey-demo/119166_graphs_v9.4.10"); + File outDir = FileUtil.appendPath(outDir(), Path.parse("labkey-demo")); + File workspaceFile = FileUtil.appendPath(dataDir(), Path.parse("labkey-demo/labkey-demo.xml")); + File fcsFile = FileUtil.appendPath(dataDir(), Path.parse("labkey-demo/119166.fcs")); + File expectedImages = FileUtil.appendPath(dataDir(), Path.parse("labkey-demo/119166_graphs_v9.4.10")); generatePlotsAndCompare(outDir, workspaceFile, fcsFile, expectedImages); } @@ -284,10 +284,10 @@ public void LabKeyDemo() throws Exception @Test public void LetvinFACSCalibur() throws Exception { - File outDir = new File(outDir(), "Letvin/FACSCalibur"); - File workspaceFile = new File(dataDir(), "Letvin/FACSCalibur/workspace.xml"); - File fcsFile = new File(dataDir(), "Letvin/FACSCalibur/64.001"); - File expectedImages = new File(dataDir(), "Letvin/FACSCalibur/64.001_graphs_v9.4.10"); + File outDir = FileUtil.appendPath(outDir(), Path.parse("Letvin/FACSCalibur")); + File workspaceFile = FileUtil.appendPath(dataDir(), Path.parse("Letvin/FACSCalibur/workspace.xml")); + File fcsFile = FileUtil.appendPath(dataDir(), Path.parse("Letvin/FACSCalibur/64.001")); + File expectedImages = FileUtil.appendPath(dataDir(), Path.parse("Letvin/FACSCalibur/64.001_graphs_v9.4.10")); generatePlotsAndCompare(outDir, workspaceFile, fcsFile, expectedImages); } @@ -295,10 +295,10 @@ public void LetvinFACSCalibur() throws Exception @Test public void LetvinLargeFCS() throws Exception { - File outDir = new File(outDir(), "Letvin/LargeFCS"); - File workspaceFile = new File(dataDir(), "Letvin/LargeFCS/workspace.xml"); - File fcsFile = new File(dataDir(), "Letvin/LargeFCS/BLOOD P11C STIM_AS31_B04.fcs"); - File expectedImages = new File(dataDir(), "Letvin/LargeFCS/AS31_B04_graphs_v9.4.10"); + File outDir = FileUtil.appendPath(outDir(), Path.parse("Letvin/LargeFCS")); + File workspaceFile = FileUtil.appendPath(dataDir(), Path.parse("Letvin/LargeFCS/workspace.xml")); + File fcsFile = FileUtil.appendPath(dataDir(), Path.parse("Letvin/LargeFCS/BLOOD P11C STIM_AS31_B04.fcs")); + File expectedImages = FileUtil.appendPath(dataDir(), Path.parse("Letvin/LargeFCS/AS31_B04_graphs_v9.4.10")); generatePlotsAndCompare(outDir, workspaceFile, fcsFile, expectedImages); } @@ -306,10 +306,10 @@ public void LetvinLargeFCS() throws Exception @Test public void YirongWang() throws Exception { - File outDir = new File(outDir(), "YirongWang"); - File workspaceFile = new File(dataDir(), "YirongWang/workspace.wsp"); - File fcsFile = new File(dataDir(), "YirongWang/001.fcs"); - File expectedImages = new File(dataDir(), "YirongWang/001_graphs_v7.6.5"); + File outDir = FileUtil.appendPath(outDir(), Path.parse("YirongWang")); + File workspaceFile = FileUtil.appendPath(dataDir(), Path.parse("YirongWang/workspace.wsp")); + File fcsFile = FileUtil.appendPath(dataDir(), Path.parse("YirongWang/001.fcs")); + File expectedImages = FileUtil.appendPath(dataDir(), Path.parse("YirongWang/001_graphs_v7.6.5")); generatePlotsAndCompare(outDir, workspaceFile, fcsFile, expectedImages); } diff --git a/flow/enginesrc/org/labkey/flow/persist/AnalysisSerializer.java b/flow/enginesrc/org/labkey/flow/persist/AnalysisSerializer.java index 9dee7589b..8884fb628 100644 --- a/flow/enginesrc/org/labkey/flow/persist/AnalysisSerializer.java +++ b/flow/enginesrc/org/labkey/flow/persist/AnalysisSerializer.java @@ -262,9 +262,9 @@ public static File extractArchive(File file, FileLike tempDir) { statisticsFile = file; } - else if (file.isDirectory() && new File(file, AnalysisSerializer.STATISTICS_FILENAME).isFile()) + else if (file.isDirectory() && FileUtil.appendName(file, AnalysisSerializer.STATISTICS_FILENAME).isFile()) { - statisticsFile = new File(file, AnalysisSerializer.STATISTICS_FILENAME); + statisticsFile = FileUtil.appendName(file, AnalysisSerializer.STATISTICS_FILENAME); } else if (file.getName().endsWith(".zip")) { diff --git a/flow/src/org/labkey/flow/controllers/WorkspaceData.java b/flow/src/org/labkey/flow/controllers/WorkspaceData.java index ea7addd89..5108717e3 100644 --- a/flow/src/org/labkey/flow/controllers/WorkspaceData.java +++ b/flow/src/org/labkey/flow/controllers/WorkspaceData.java @@ -416,7 +416,7 @@ private static IWorkspace readWorkspace(File file, String path) throws Workspace { try { - if (file.isDirectory() && new File(file, AnalysisSerializer.STATISTICS_FILENAME).isFile()) + if (file.isDirectory() && FileUtil.appendName(file, AnalysisSerializer.STATISTICS_FILENAME).isFile()) { return AnalysisSerializer.readAnalysis(file); } diff --git a/flow/src/org/labkey/flow/controllers/run/RunController.java b/flow/src/org/labkey/flow/controllers/run/RunController.java index bfdd9f4ef..af765567b 100644 --- a/flow/src/org/labkey/flow/controllers/run/RunController.java +++ b/flow/src/org/labkey/flow/controllers/run/RunController.java @@ -658,7 +658,7 @@ VirtualFile createVirtualFile(ExportAnalysisForm form, String name) throws IOExc if (_exportToScriptLocation != null) dir = new File(_exportToScriptLocation); else - dir = new File(FileUtil.getTempDirectory(), "flow-export-to-script"); + dir = FileUtil.appendName(FileUtil.getTempDirectory(), "flow-export-to-script"); if ("zip".equalsIgnoreCase(_exportToScriptFormat)) { @@ -667,7 +667,7 @@ VirtualFile createVirtualFile(ExportAnalysisForm form, String name) throws IOExc } else { - File child = new File(dir, FileUtil.makeLegalName(name + "_" + getTimestamp())); + File child = FileUtil.appendName(dir, FileUtil.makeLegalName(name + "_" + getTimestamp())); FileUtil.mkdirs(child); return new FileSystemFile(child); } @@ -780,7 +780,7 @@ public ExportToScriptJob(String guid, String exportToScriptPath, String exportTo _deleteOnComplete = deleteOnComplete; // setup the log file - File logFile = new File(root.getLogDirectory(), FileUtil.makeFileNameWithTimestamp("export-to-script", "log")); + File logFile = FileUtil.appendName(root.getLogDirectory(), FileUtil.makeFileNameWithTimestamp("export-to-script", "log")); setLogFile(logFile); } diff --git a/flow/src/org/labkey/flow/data/FlowAssayProvider.java b/flow/src/org/labkey/flow/data/FlowAssayProvider.java index edcd9e22f..6a1f2c4a5 100644 --- a/flow/src/org/labkey/flow/data/FlowAssayProvider.java +++ b/flow/src/org/labkey/flow/data/FlowAssayProvider.java @@ -72,7 +72,6 @@ import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.mvc.Controller; -import java.io.File; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -221,7 +220,7 @@ public AssayRunCreator getRunCreator() } @Override - public List getDataCollectors(Map uploadedFiles, AssayRunUploadForm context) + public List getDataCollectors(@Nullable Map uploadedFiles, AssayRunUploadForm context) { throw new UnsupportedOperationException(); } diff --git a/flow/src/org/labkey/flow/persist/FlowDataHandler.java b/flow/src/org/labkey/flow/persist/FlowDataHandler.java index 074d06f70..2a4ae5c0d 100644 --- a/flow/src/org/labkey/flow/persist/FlowDataHandler.java +++ b/flow/src/org/labkey/flow/persist/FlowDataHandler.java @@ -38,7 +38,7 @@ import org.labkey.flow.flowdata.xml.FlowData; import org.labkey.flow.flowdata.xml.FlowdataDocument; -import java.io.File; +import org.labkey.vfs.FileLike; import java.io.OutputStream; import java.net.URI; import java.util.List; @@ -63,7 +63,7 @@ public void beforeDeleteData(List datas, User user) } @Override - public void exportFile(ExpData data, File dataFile, User user, OutputStream out) + public void exportFile(ExpData data, FileLike dataFile, User user, OutputStream out) { try { @@ -115,7 +115,7 @@ public Priority getPriority(ExpData data) } @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException + public void importFile(@NotNull ExpData data, @NotNull FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException { try { @@ -123,7 +123,7 @@ public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgr { if (AttributeSetHelper.fromData(data) == null) { - FlowdataDocument doc = FlowdataDocument.Factory.parse(dataFile); + FlowdataDocument doc = FlowdataDocument.Factory.parse(dataFile.openInputStream()); FlowData flowdata = doc.getFlowdata(); URI uriFile = null; if (flowdata.getUri() != null) @@ -142,7 +142,7 @@ public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgr else if (dataFile.getName().endsWith("." + EXT_SCRIPT)) { FlowScript script = new FlowScript(data); - script.setAnalysisScript(info.getUser(), PageFlowUtil.getFileContentsAsString(dataFile)); + script.setAnalysisScript(info.getUser(), PageFlowUtil.getStreamContentsAsString(dataFile.openInputStream())); } } catch (Exception e) diff --git a/flow/src/org/labkey/flow/script/FlowExperimentJob.java b/flow/src/org/labkey/flow/script/FlowExperimentJob.java index b9cf44bf6..8396bae55 100644 --- a/flow/src/org/labkey/flow/script/FlowExperimentJob.java +++ b/flow/src/org/labkey/flow/script/FlowExperimentJob.java @@ -69,7 +69,7 @@ public FlowExperimentJob(ViewBackgroundInfo info, PipeRoot root, String experime private void initStatus() throws IOException { String guid = GUID.makeGUID(); - File logFile = new File(_containerFolder, guid + ".flow.log"); + File logFile = FileUtil.appendName(_containerFolder, guid + ".flow.log"); logFile.createNewFile(); setLogFile(logFile); } @@ -138,7 +138,7 @@ protected boolean checkProcessPath(File path, FlowProtocolStep step) protected File getWorkingFolder(Container container) throws IOException { File dirRoot = FlowAnalyzer.getAnalysisDirectory(); - File dirFolder = new File(dirRoot, "Folder" + container.getRowId()); + File dirFolder = FileUtil.appendName(dirRoot, "Folder" + container.getRowId()); if (!dirFolder.exists()) { if (!FileUtil.mkdirs(dirFolder)) @@ -163,7 +163,7 @@ public File createAnalysisDirectory(String dirName, FlowProtocolStep step) throw } for (int i = 1; ; i ++) { - File dirData = new File(dirRun, step.getLabel() + i); + File dirData = FileUtil.appendName(dirRun, step.getLabel() + i); if (!dirData.exists()) { if (!FileUtil.mkdirs(dirData)) diff --git a/flow/src/org/labkey/flow/script/ImportResultsJob.java b/flow/src/org/labkey/flow/script/ImportResultsJob.java index 7cd95b4a8..33e18e044 100644 --- a/flow/src/org/labkey/flow/script/ImportResultsJob.java +++ b/flow/src/org/labkey/flow/script/ImportResultsJob.java @@ -212,7 +212,7 @@ else if (getRunFilePathRoot() != null) // UNDONE: comp matrix } - File statisticsFile = new File(_analysisPathRoot, AnalysisSerializer.STATISTICS_FILENAME); + File statisticsFile = FileUtil.appendName(_analysisPathRoot, AnalysisSerializer.STATISTICS_FILENAME); FlowRun run = saveAnalysis(getUser(), getContainer(), getExperiment(), _analysisRunName, statisticsFile, getOriginalImportedFile(), @@ -229,7 +229,7 @@ _analysisRunName, statisticsFile, getOriginalImportedFile(), ); // Add attachments to the run - File attachmentsDir = new File(_analysisPathRoot, "attachments"); + File attachmentsDir = FileUtil.appendName(_analysisPathRoot, "attachments"); if (attachmentsDir.isDirectory()) { AttachmentService svc = AttachmentService.get(); diff --git a/flow/src/org/labkey/flow/script/ScriptXarSource.java b/flow/src/org/labkey/flow/script/ScriptXarSource.java index 26071919c..a356d85fa 100644 --- a/flow/src/org/labkey/flow/script/ScriptXarSource.java +++ b/flow/src/org/labkey/flow/script/ScriptXarSource.java @@ -41,12 +41,12 @@ public ScriptXarSource(ExperimentArchiveDocument doc, File root, File workingDir _root = root; _doc = doc; _workingDirectory = workingDirectory; - _logFile = new File(_workingDirectory, "flow.xar.log"); + _logFile = FileUtil.appendName(_workingDirectory, "flow.xar.log"); // For informational purposes, write out the XAR file. try { - File xarfile = new File(_workingDirectory, "flow.xar.xml"); + File xarfile = FileUtil.appendName(_workingDirectory, "flow.xar.xml"); try (FileWriter writer = new FileWriter(xarfile)) { diff --git a/luminex/src/org/labkey/luminex/LuminexDataHandler.java b/luminex/src/org/labkey/luminex/LuminexDataHandler.java index 4a02dd20b..dd97666a8 100644 --- a/luminex/src/org/labkey/luminex/LuminexDataHandler.java +++ b/luminex/src/org/labkey/luminex/LuminexDataHandler.java @@ -165,17 +165,17 @@ public DataType getDataType() } @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException + public void importFile(@NotNull ExpData data, @NotNull FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException { if (!dataFile.exists()) { - log.warn("Could not find file " + dataFile.getAbsolutePath() + " on disk for data with LSID " + data.getLSID()); + log.warn("Could not find file " + dataFile + " on disk for data with LSID " + data.getLSID()); return; } ExpRun expRun = data.getRun(); if (expRun == null) { - throw new ExperimentException("Could not load Luminex file " + dataFile.getAbsolutePath() + " because it is not owned by an experiment run"); + throw new ExperimentException("Could not load Luminex file " + dataFile + " because it is not owned by an experiment run"); } LuminexExcelParser parser; @@ -187,7 +187,7 @@ public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgr } else { - parser = new LuminexExcelParser(expRun.getProtocol(), Collections.singleton(dataFile)); + parser = LuminexExcelParser.create(expRun.getProtocol(), Collections.singleton(dataFile)); } // The parser has already collapsed the data from multiple files into a single set of data, // so don't bother importing it twice if it came from separate files. This can happen if you aren't using a diff --git a/luminex/src/org/labkey/luminex/LuminexExclusionPipelineJob.java b/luminex/src/org/labkey/luminex/LuminexExclusionPipelineJob.java index 8cb66f28d..a48d694e0 100644 --- a/luminex/src/org/labkey/luminex/LuminexExclusionPipelineJob.java +++ b/luminex/src/org/labkey/luminex/LuminexExclusionPipelineJob.java @@ -27,8 +27,6 @@ import org.labkey.api.util.URLHelper; import org.labkey.api.view.ViewBackgroundInfo; -import java.io.File; - public class LuminexExclusionPipelineJob extends PipelineJob { private LuminexSaveExclusionsForm _form; @@ -46,8 +44,7 @@ public LuminexExclusionPipelineJob(ViewBackgroundInfo info, PipeRoot root, Lumin { super(LuminexAssayProvider.NAME, info, root); - File logFile = new File(root.getLogDirectory(), FileUtil.makeFileNameWithTimestamp("luminex_exclusion", "log")); - setLogFile(logFile); + setLogFile(root.getLogDirectoryFileLike(true).resolveChild(FileUtil.makeFileNameWithTimestamp("luminex_exclusion", "log")).toNioPathForWrite()); _form = form; _exclusionType = LuminexManager.ExclusionType.valueOf(form.getTableName()); diff --git a/microarray/src/org/labkey/api/study/assay/matrix/AbstractMatrixRunCreator.java b/microarray/src/org/labkey/api/study/assay/matrix/AbstractMatrixRunCreator.java index e1a3e9038..850e11628 100644 --- a/microarray/src/org/labkey/api/study/assay/matrix/AbstractMatrixRunCreator.java +++ b/microarray/src/org/labkey/api/study/assay/matrix/AbstractMatrixRunCreator.java @@ -133,7 +133,7 @@ protected void addInputMaterials( // Attach the materials found in the matrix file to the run try { - File dataFile = getPrimaryFile(context); + FileLike dataFile = getPrimaryFile(context); try (TabLoader loader = AbstractMatrixDataHandler.createTabLoader(dataFile, getIdColumnName(), getIdColumnAliases())) { ColumnDescriptor[] cols = loader.getColumns(); @@ -156,13 +156,10 @@ protected void addInputMaterials( } } - private File getPrimaryFile(AssayRunUploadContext context) throws ExperimentException + private FileLike getPrimaryFile(AssayRunUploadContext context) throws ExperimentException { Map files = context.getUploadedData(); assert files.containsKey(AssayDataCollector.PRIMARY_FILE); - FileLike fl = files.get(AssayDataCollector.PRIMARY_FILE); - if (null == fl) - return null; - return fl.toNioPathForRead().toFile(); + return files.get(AssayDataCollector.PRIMARY_FILE); } } diff --git a/microarray/src/org/labkey/microarray/matrix/ExpressionMatrixDataHandler.java b/microarray/src/org/labkey/microarray/matrix/ExpressionMatrixDataHandler.java index e58779a2e..7fc716cbf 100644 --- a/microarray/src/org/labkey/microarray/matrix/ExpressionMatrixDataHandler.java +++ b/microarray/src/org/labkey/microarray/matrix/ExpressionMatrixDataHandler.java @@ -18,6 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.jetbrains.annotations.NotNull; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.collections.LongHashMap; import org.labkey.api.data.Container; @@ -42,6 +43,7 @@ import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.microarray.MicroarrayManager; import org.labkey.microarray.query.MicroarrayUserSchema; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -78,19 +80,19 @@ public DataType getDataType() { return ExpressionMatrixAssayProvider.DATA_TYPE; } - + @Override - public void importFile(ExpData data, File dataFile, ViewBackgroundInfo info, Logger log, XarContext context) throws ExperimentException + public void importFile(@NotNull ExpData data, @NotNull FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException { if (!dataFile.exists()) { - log.warn("Could not find file " + dataFile.getAbsolutePath() + " on disk for data with LSID " + data.getLSID()); + log.warn("Could not find file " + dataFile + " on disk for data with LSID " + data.getLSID()); return; } ExpRun expRun = data.getRun(); if (expRun == null) { - throw new ExperimentException("Could not load ExpressionMatrix file " + dataFile.getAbsolutePath() + " because it is not owned by an experiment run"); + throw new ExperimentException("Could not load ExpressionMatrix file " + dataFile + " because it is not owned by an experiment run"); } try diff --git a/ms2/src/org/labkey/ms2/MS2Controller.java b/ms2/src/org/labkey/ms2/MS2Controller.java index 204f3f677..c29bf5081 100644 --- a/ms2/src/org/labkey/ms2/MS2Controller.java +++ b/ms2/src/org/labkey/ms2/MS2Controller.java @@ -4819,7 +4819,7 @@ public void export(DetailsForm form, HttpServletResponse response, BindException if (!paramsFile.exists() && TPPTask.FT_PEP_XML.isType(run.getFileName())) { String basename = TPPTask.FT_PEP_XML.getBaseName(new File(run.getPath() + "/" + run.getFileName())); - paramsFile = new File(paramsFile.getParentFile(), basename + "." + run.getParamsFileName()); + paramsFile = FileUtil.appendName(paramsFile.getParentFile(), basename + "." + run.getParamsFileName()); } } if (!NetworkDrive.exists(paramsFile)) diff --git a/ms2/src/org/labkey/ms2/MS2Manager.java b/ms2/src/org/labkey/ms2/MS2Manager.java index 8c99e5275..11d1d9f72 100644 --- a/ms2/src/org/labkey/ms2/MS2Manager.java +++ b/ms2/src/org/labkey/ms2/MS2Manager.java @@ -88,6 +88,8 @@ import org.labkey.ms2.reader.RandomAccessMzxmlIteratorFactory; import org.labkey.ms2.reader.RelativeQuantAnalysisSummary; import org.labkey.ms2.reader.SimpleScan; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import javax.xml.stream.XMLStreamException; import java.io.File; @@ -337,7 +339,7 @@ private static ExpRun wrapRun(MS2Run run, User user) throws ExperimentException try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction()) { Container container = run.getContainer(); - final File pepXMLFile = new File(run.getPath(), run.getFileName()); + final FileLike pepXMLFile = FileSystemLike.wrapFile(new File(run.getPath(), run.getFileName())); // Check if this ExpData existingPepXmlData = ExperimentService.get().getExpDataByURL(pepXMLFile, container); @@ -383,7 +385,7 @@ public Path getLogFilePath() @Override public Path getRootPath() { - return pepXMLFile.getParentFile().toPath(); + return pepXMLFile.getParent().toNioPathForRead(); } @Override @@ -443,7 +445,7 @@ public ExperimentArchiveDocument getDocument() outputDatas.put(protXMLData, TPPTask.PROT_XML_INPUT_ROLE); } - expRun.setFilePathRoot(pepXMLFile.getParentFile()); + expRun.setFilePathRoot(pepXMLFile.getParent().toNioPathForRead().toFile()); ViewBackgroundInfo info = new ViewBackgroundInfo(container, user, null); expRun = ExperimentService.get().saveSimpleExperimentRun(expRun, Collections.emptyMap(), inputDatas, Collections.emptyMap(), outputDatas, Collections.emptyMap(), info, LOG, false); diff --git a/ms2/src/org/labkey/ms2/PepXmlExperimentDataHandler.java b/ms2/src/org/labkey/ms2/PepXmlExperimentDataHandler.java index cd2ba0b74..4947a7efe 100644 --- a/ms2/src/org/labkey/ms2/PepXmlExperimentDataHandler.java +++ b/ms2/src/org/labkey/ms2/PepXmlExperimentDataHandler.java @@ -30,6 +30,8 @@ import org.labkey.api.view.ActionURL; import org.labkey.api.view.UnauthorizedException; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import javax.xml.stream.XMLStreamException; import java.io.File; @@ -58,8 +60,9 @@ protected boolean shouldImport(ExpData data, XarContext context) } @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException + public void importFile(@NotNull ExpData data, @NotNull FileLike dataFileLike, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException { + File dataFile = FileSystemLike.toFile(dataFileLike); if (!shouldImport(data, context)) { log.info("Skipping import of file " + dataFile); diff --git a/ms2/src/org/labkey/ms2/PepXmlImporter.java b/ms2/src/org/labkey/ms2/PepXmlImporter.java index 5490f3234..240e85efd 100644 --- a/ms2/src/org/labkey/ms2/PepXmlImporter.java +++ b/ms2/src/org/labkey/ms2/PepXmlImporter.java @@ -24,6 +24,7 @@ import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.file.AbstractFileAnalysisProtocol; import org.labkey.api.security.User; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.NetworkDrive; import org.labkey.api.util.PepXMLFileType; import org.labkey.api.util.massSpecDataFileType; @@ -277,7 +278,7 @@ protected File getMzXMLFile(MS2Loader.PeptideFraction fraction) // Check two directories up from the pepXML file, where the pipeline normally reads the mzXML file. if (dir.getParentFile() != null && dir.getParentFile().getParentFile() != null) { - f = new File(dir.getParentFile().getParentFile(), mzXMLFile.getName()); + f = FileUtil.appendName(dir.getParentFile().getParentFile(), mzXMLFile.getName()); if (NetworkDrive.exists(f) && f.isFile()) { return f; @@ -296,7 +297,7 @@ protected File getMzXMLFile(MS2Loader.PeptideFraction fraction) return f; } } - f = new File(dir, mzXMLFile.getName()); + f = FileUtil.appendName(dir, mzXMLFile.getName()); if (NetworkDrive.exists(f) && f.isFile()) { return f; diff --git a/ms2/src/org/labkey/ms2/ProteinProphetExperimentDataHandler.java b/ms2/src/org/labkey/ms2/ProteinProphetExperimentDataHandler.java index 8a8c8f538..539d16aa8 100644 --- a/ms2/src/org/labkey/ms2/ProteinProphetExperimentDataHandler.java +++ b/ms2/src/org/labkey/ms2/ProteinProphetExperimentDataHandler.java @@ -29,6 +29,8 @@ import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.ms2.pipeline.TPPTask; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import javax.xml.stream.XMLStreamException; import java.io.File; @@ -46,8 +48,9 @@ public DataType getDataType() } @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException + public void importFile(@NotNull ExpData data, @NotNull FileLike dataFileLike, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException { + File dataFile = FileSystemLike.toFile(dataFileLike); if (context.getJob() != null && "false".equalsIgnoreCase(context.getJob().getParameters().get(IMPORT_PROPHET_RESULTS))) { log.info("Skipping import of file " + dataFile); diff --git a/ms2/src/org/labkey/ms2/ProteinProphetImporter.java b/ms2/src/org/labkey/ms2/ProteinProphetImporter.java index 7333fad91..2ec2f927c 100644 --- a/ms2/src/org/labkey/ms2/ProteinProphetImporter.java +++ b/ms2/src/org/labkey/ms2/ProteinProphetImporter.java @@ -30,6 +30,7 @@ import org.labkey.api.protein.fasta.FastaProtein; import org.labkey.api.query.AliasManager; import org.labkey.api.reader.SimpleXMLStreamReader; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.NetworkDrive; import org.labkey.api.util.Pair; import org.labkey.api.util.PossiblyGZIPpedFileInputStreamFactory; @@ -358,7 +359,7 @@ private MS2Run importRun(ViewBackgroundInfo info, Logger log) throws IOException if (pepXMLFile == null) { // Second, try the file name in the XML in the current directory - pepXMLFile = new File(_file.getParentFile(), new File(pepXMLFileName).getName()); + pepXMLFile = FileUtil.appendName(_file.getParentFile(), new File(pepXMLFileName).getName()); attemptedFiles.add(pepXMLFile.getAbsolutePath()); if (!NetworkDrive.exists(pepXMLFile)) { diff --git a/ms2/test/src/org/labkey/test/ms2/MS2TestBase.java b/ms2/test/src/org/labkey/test/ms2/MS2TestBase.java index 2cca5bfd4..32ae2163a 100644 --- a/ms2/test/src/org/labkey/test/ms2/MS2TestBase.java +++ b/ms2/test/src/org/labkey/test/ms2/MS2TestBase.java @@ -16,6 +16,8 @@ package org.labkey.test.ms2; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Path; import org.labkey.test.BaseWebDriverTest; import org.labkey.test.TestFileUtils; import org.labkey.test.TestTimeoutException; @@ -112,12 +114,12 @@ protected void cleanPipe(String search_type) return; File rootDir = new File(PIPELINE_PATH); - delete(new File(rootDir, "bov_sample/xars")); - delete(new File(rootDir, "bov_sample/"+search_type+"/test1/CAexample_mini.log")); - delete(new File(rootDir, "bov_sample/"+search_type+"/test2")); - delete(new File(rootDir, ".labkey/protocols/mass_spec/TestMS2Protocol.xml")); - delete(new File(rootDir, ".labkey/protocols/"+search_type+"/default.xml")); - delete(new File(rootDir, ".labkey/protocols/"+search_type+"/test2.xml")); + delete(FileUtil.appendPath(rootDir, Path.parse("bov_sample/xars"))); + delete(FileUtil.appendPath(rootDir, Path.parse("bov_sample/"+search_type+"/test1/CAexample_mini.log"))); + delete(FileUtil.appendPath(rootDir, Path.parse("bov_sample/"+search_type+"/test2"))); + delete(FileUtil.appendPath(rootDir, Path.parse(".labkey/protocols/mass_spec/TestMS2Protocol.xml"))); + delete(FileUtil.appendPath(rootDir, Path.parse(".labkey/protocols/"+search_type+"/default.xml"))); + delete(FileUtil.appendPath(rootDir, Path.parse(".labkey/protocols/"+search_type+"/test2.xml"))); } protected void navigateToFolder(String folderName) diff --git a/ms2/test/src/org/labkey/test/ms2/QuantitationTest.java b/ms2/test/src/org/labkey/test/ms2/QuantitationTest.java index 3aad4168c..4dabc7955 100644 --- a/ms2/test/src/org/labkey/test/ms2/QuantitationTest.java +++ b/ms2/test/src/org/labkey/test/ms2/QuantitationTest.java @@ -17,6 +17,8 @@ import org.junit.Test; import org.junit.experimental.categories.Category; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Path; import org.labkey.test.BaseWebDriverTest; import org.labkey.test.Locator; import org.labkey.test.categories.Daily; @@ -105,7 +107,7 @@ protected void basicChecks() protected void cleanPipe(String search_type) { File rootDir = new File(PIPELINE_PATH); - delete(new File(rootDir, ".labkey/protocols/"+search_type+"/" + LIBRA_PROTOCOL_NAME + ".xml")); - delete(new File(rootDir, "bov_sample/"+search_type+"/" + LIBRA_PROTOCOL_NAME)); + delete(FileUtil.appendPath(rootDir, Path.parse(".labkey/protocols/"+search_type+"/" + LIBRA_PROTOCOL_NAME + ".xml"))); + delete(FileUtil.appendPath(rootDir, Path.parse("bov_sample/"+search_type+"/" + LIBRA_PROTOCOL_NAME))); } } diff --git a/ms2/test/src/org/labkey/test/tests/ms2/MS2ExtensionsTest.java b/ms2/test/src/org/labkey/test/tests/ms2/MS2ExtensionsTest.java index e74d761ad..318d6c8fe 100644 --- a/ms2/test/src/org/labkey/test/tests/ms2/MS2ExtensionsTest.java +++ b/ms2/test/src/org/labkey/test/tests/ms2/MS2ExtensionsTest.java @@ -16,6 +16,8 @@ package org.labkey.test.tests.ms2; import org.junit.experimental.categories.Category; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Path; import org.labkey.test.BaseWebDriverTest; import org.labkey.test.Locator; import org.labkey.test.categories.Daily; @@ -103,9 +105,7 @@ private void compareWithCriteria(String matchCriteria) private void cleanPipeline() { - if (PIPELINE_PATH == null) - return; File rootDir = new File(PIPELINE_PATH); - delete(new File(rootDir, ".labkey/protocols/rollup/Protocol Rollup 1.xml")); + delete(FileUtil.appendPath(rootDir, Path.parse(".labkey/protocols/rollup/Protocol Rollup 1.xml"))); } } diff --git a/ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java b/ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java index a0380297a..90b0b0aba 100644 --- a/ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java +++ b/ms2/test/src/org/labkey/test/tests/ms2/MS2Test.java @@ -18,6 +18,8 @@ import org.junit.Assert; import org.junit.experimental.categories.Category; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Path; import org.labkey.test.BaseWebDriverTest; import org.labkey.test.Locator; import org.labkey.test.SortDirection; @@ -1212,7 +1214,7 @@ private void queryValidationTest() private void cleanPipeline() { File rootDir = new File(PIPELINE_PATH); - delete(new File(rootDir, ".labkey/protocols/rollup/Protocol Rollup 1.xml")); + delete(FileUtil.appendPath(rootDir, Path.parse(".labkey/protocols/rollup/Protocol Rollup 1.xml"))); } //TODO: Create MS2RunView component for this stuff diff --git a/ms2/test/src/org/labkey/test/tests/ms2/MascotTest.java b/ms2/test/src/org/labkey/test/tests/ms2/MascotTest.java index fe37686ad..e6dfca20e 100644 --- a/ms2/test/src/org/labkey/test/tests/ms2/MascotTest.java +++ b/ms2/test/src/org/labkey/test/tests/ms2/MascotTest.java @@ -20,6 +20,8 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Path; import org.labkey.test.BaseWebDriverTest; import org.labkey.test.Locator; import org.labkey.test.SortDirection; @@ -505,6 +507,6 @@ protected void cleanPipe(String search_type) return; File rootDir = new File(PIPELINE_PATH); - delete(new File(rootDir, "databases/mascot")); + delete(FileUtil.appendPath(rootDir, Path.parse("databases/mascot"))); } } diff --git a/ms2/test/src/org/labkey/test/tests/ms2/PeaksTest.java b/ms2/test/src/org/labkey/test/tests/ms2/PeaksTest.java index bf81a75eb..9e22aec9c 100644 --- a/ms2/test/src/org/labkey/test/tests/ms2/PeaksTest.java +++ b/ms2/test/src/org/labkey/test/tests/ms2/PeaksTest.java @@ -19,6 +19,8 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Path; import org.labkey.test.Locator; import org.labkey.test.TestTimeoutException; import org.labkey.test.categories.Daily; @@ -128,6 +130,6 @@ protected void cleanPipe(String search_type) super.cleanPipe(search_type); File rootDir = new File(PIPELINE_PATH); - delete(new File(rootDir, "peaks/peaks.log")); + delete(FileUtil.appendPath(rootDir, Path.parse("peaks/peaks.log"))); } } diff --git a/nab/src/org/labkey/nab/NabAssayController.java b/nab/src/org/labkey/nab/NabAssayController.java index b2756e82c..95c24ef72 100644 --- a/nab/src/org/labkey/nab/NabAssayController.java +++ b/nab/src/org/labkey/nab/NabAssayController.java @@ -124,11 +124,11 @@ import org.labkey.api.view.ViewContext; import org.labkey.nab.qc.NabWellQCFlag; import org.labkey.nab.query.NabProtocolSchema; +import org.labkey.vfs.FileLike; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; -import java.io.File; import java.io.Serializable; import java.text.DecimalFormat; import java.util.ArrayList; @@ -235,12 +235,12 @@ public ModelAndView getView(RenderAssayForm form, BindException errors) throws E { throw new NotFoundException("Run " + form.getRowId() + " does not exist."); } - File file = getDataHandler(run).getDataFile(run); + FileLike file = getDataHandler(run).getDataFile(run); if (file == null) { throw new NotFoundException("Data file for run " + run.getName() + " was not found. Deleted from the file system?"); } - PageFlowUtil.streamFile(getViewContext().getResponse(), file, true); + PageFlowUtil.streamFile(getViewContext().getResponse(), file.toNioPathForRead().toFile(), true); return null; } @@ -437,7 +437,7 @@ public void setReupload(boolean reupload) public class DeleteRunAction extends FormHandlerAction { private ExpRun _run; - private File _file; + private FileLike _file; @Override public void validateCommand(DeleteRunForm form, Errors errors) @@ -473,7 +473,7 @@ public URLHelper getSuccessURL(DeleteRunForm form) if (form.isReupload()) { ActionURL reuploadURL = new ActionURL(NabUploadWizardAction.class, getContainer()); - reuploadURL.addParameter("dataFile", _file.getPath()); + reuploadURL.addParameter("dataFile", _file.toNioPathForRead().toFile().getPath()); return reuploadURL; } @@ -641,7 +641,7 @@ public void export(SampleSpreadsheetForm sampleSpreadsheetForm, HttpServletRespo } AssayProvider provider = AssayService.get().getProvider(protocol); - if (provider == null || !(provider instanceof NabAssayProvider nabProvider)) + if (!(provider instanceof NabAssayProvider nabProvider)) { String message = "Protocol " + sampleSpreadsheetForm.getProtocol() + " is not a NAb protocol: " + protocol.getName(); throw new NotFoundException(message); @@ -906,13 +906,12 @@ private Map serializePlate(Plate plate, DilutionAssayRun assay) Set colPos = new TreeSet<>(); Set rowPos = new TreeSet<>(); - List colOrder = new ArrayList<>(); for (Position position : virusGroup.getPositions()) { colPos.add(position.getColumn()); rowPos.add(position.getRow()); } - colOrder.addAll(colPos); + List colOrder = new ArrayList<>(colPos); colPos.clear(); for (Position position : cellGroup.getPositions()) { diff --git a/nab/src/org/labkey/nab/NabDataHandler.java b/nab/src/org/labkey/nab/NabDataHandler.java index da1c83ff1..4d8665d11 100644 --- a/nab/src/org/labkey/nab/NabDataHandler.java +++ b/nab/src/org/labkey/nab/NabDataHandler.java @@ -18,6 +18,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.poi.ss.usermodel.DataValidationHelper; import org.jetbrains.annotations.Nullable; import org.labkey.api.assay.plate.AbstractPlateBasedAssayProvider; import org.labkey.api.assay.dilution.DilutionAssayRun; @@ -36,16 +37,21 @@ import org.labkey.api.dataiterator.MapDataIterator; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.XarContext; import org.labkey.api.exp.api.DataType; import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.api.ExpMaterial; import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.qc.DataLoaderSettings; +import org.labkey.api.qc.ValidationDataHandler; import org.labkey.api.query.BatchValidationException; import org.labkey.api.security.User; import org.labkey.api.util.Pair; +import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.nab.query.NabProtocolSchema; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -57,7 +63,7 @@ import java.util.List; import java.util.Map; -public abstract class NabDataHandler extends DilutionDataHandler +public abstract class NabDataHandler extends DilutionDataHandler implements ValidationDataHandler { public static final Logger LOG = LogManager.getLogger(NabDataHandler.class); @@ -122,6 +128,8 @@ protected void importRows(ExpData data, ExpRun run, ExpProtocol protocol, DataIt populateWellData(protocol, run, user, cutoffFormats, wellGroupNameToNabSpecimen); } + + /** * Populates cutoff and AUC information from the passed in raw data */ @@ -259,6 +267,18 @@ private void _populateDilutionStats(ExpData data, ExpRun run, ExpProtocol protoc } } + @Override + public Map getValidationDataMap(ExpData data, FileLike dataFile, ViewBackgroundInfo info, Logger log, XarContext context, DataLoaderSettings settings) throws ExperimentException + { + DilutionDataFileParser parser = getDataFileParser(data, dataFile, info); + + Map datas = new HashMap<>(); + List> rows = parser.getResults(); + datas.put(NAB_TRANSFORMED_DATA_TYPE, MapDataIterator.of(rows)); + + return datas; + } + @Override public void beforeDeleteData(List datas, User user) throws ExperimentException { @@ -275,7 +295,7 @@ public void beforeDeleteData(List datas, User user) throws ExperimentEx /** * Parse a list of values into multiple plates. */ - protected List parseList(File dataFile, List> rows, String locationColumnHeader, String resultColumnHeader, + protected List parseList(FileLike dataFile, List> rows, String locationColumnHeader, String resultColumnHeader, int maxPlates, int expectedRows, int expectedCols, List errors) { int wellsPerPlate = expectedRows * expectedCols; @@ -307,13 +327,13 @@ protected List parseList(File dataFile, List> ro } Integer value = null; - if (dataValue instanceof Integer) - value = (Integer)dataValue; - if (dataValue instanceof String) + if (dataValue instanceof Integer i) + value = i; + if (dataValue instanceof String s) { try { - double d = Double.valueOf((String)dataValue); + double d = Double.valueOf(s); value = (int)Math.round(d); } catch (NumberFormatException nfe) @@ -364,7 +384,7 @@ protected List parseList(File dataFile, List> ro * Translate a well location value, e.g. "B04", into a (row, column) pair of coordinates. */ @Nullable - protected Pair getWellLocation(File dataFile, String locationColumnHeader, int expectedRows, int expectedCols, Map line, int lineNumber) throws ExperimentException + protected Pair getWellLocation(FileLike dataFile, String locationColumnHeader, int expectedRows, int expectedCols, Map line, int lineNumber) throws ExperimentException { Object locationValue = line.get(locationColumnHeader); if (!(locationValue instanceof String location) || location.length() < 2) diff --git a/nab/src/org/labkey/nab/PlateParserTests.java b/nab/src/org/labkey/nab/PlateParserTests.java index 858bed504..1a37e3ba4 100644 --- a/nab/src/org/labkey/nab/PlateParserTests.java +++ b/nab/src/org/labkey/nab/PlateParserTests.java @@ -23,6 +23,8 @@ import org.labkey.api.exp.ExperimentException; import org.labkey.api.util.JunitUtil; import org.labkey.api.util.Pair; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.IOException; @@ -42,13 +44,13 @@ public PlateParserTests() _context = new Mockery(); } - public double[][] parse(File file, Plate template) throws ExperimentException + public double[][] parse(FileLike file, Plate template) throws ExperimentException { SinglePlateNabDataHandler handler = new SinglePlateNabDataHandler(); return handler.getCellValues(file, template); } - public void assertCells(String nabFile, double[][] expected, File file, Plate template) throws ExperimentException + public void assertCells(String nabFile, double[][] expected, FileLike file, Plate template) throws ExperimentException { double[][] actual = parse(file, template); if (actual == null) @@ -62,10 +64,10 @@ public void assertCells(String nabFile, double[][] expected, File file, Plate te } } - public double[][] parseExpected(File file) throws IOException + public double[][] parseExpected(FileLike file) throws IOException { List values = new ArrayList<>(8); - for (String line : Files.readAllLines(file.toPath(), Charset.defaultCharset())) + for (String line : Files.readAllLines(file.toNioPathForRead(), Charset.defaultCharset())) { String[] s = line.split("\\t"); double[] cells = new double[s.length]; @@ -122,9 +124,9 @@ public void parseSinglePlates() throws Exception File nabFile = JunitUtil.getSampleData(null, test.first); File expectedFile = JunitUtil.getSampleData(null, test.second); - final double[][] expected = parseExpected(expectedFile); + final double[][] expected = parseExpected(FileSystemLike.wrapFile(expectedFile)); Plate template = template(nabFile.getName(), expected.length, expected[0].length); - assertCells(test.first, expected, nabFile, template); + assertCells(test.first, expected, FileSystemLike.wrapFile(nabFile), template); } } diff --git a/nab/src/org/labkey/nab/SinglePlateNabDataHandler.java b/nab/src/org/labkey/nab/SinglePlateNabDataHandler.java index 45b435f0b..45d028beb 100644 --- a/nab/src/org/labkey/nab/SinglePlateNabDataHandler.java +++ b/nab/src/org/labkey/nab/SinglePlateNabDataHandler.java @@ -16,7 +16,6 @@ package org.labkey.nab; -import org.apache.logging.log4j.Logger; import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.jetbrains.annotations.NotNull; import org.labkey.api.assay.AssayDataType; @@ -38,18 +37,15 @@ import org.labkey.api.data.TableInfo; import org.labkey.api.data.statistics.StatsService; import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.MapDataIterator; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.Lsid; import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.XarContext; import org.labkey.api.exp.api.DataType; import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.api.ExpMaterial; import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.qc.DataLoaderSettings; import org.labkey.api.assay.transform.TransformDataHandler; import org.labkey.api.query.FieldKey; import org.labkey.api.query.FilteredTable; @@ -61,7 +57,6 @@ import org.labkey.api.reader.TabLoader; import org.labkey.api.security.User; import org.labkey.api.util.FileType; -import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.nab.query.NabVirusDomainKind; import org.labkey.vfs.FileLike; @@ -104,17 +99,18 @@ protected DilutionAssayRun createDilutionAssayRun(DilutionAssayProvider provi } @Override - protected double[][] getCellValues(final File dataFile, Plate nabTemplate) throws ExperimentException + protected double[][] getCellValues(final FileLike dataFile, Plate nabTemplate) throws ExperimentException { final int expectedRows = nabTemplate.getRows(); final int expectedCols = nabTemplate.getColumns(); try { + File f = dataFile.toNioPathForRead().toFile(); // Special case for excel - The ExcelLoader only returns data for a single sheet so we need to create a new ExcelLoader for each sheet - if (ExcelLoader.isExcel(dataFile)) + if (ExcelLoader.isExcel(f)) { - try (ExcelFactory.WorkbookMetadata md = ExcelFactory.getMetadata(dataFile)) + try (ExcelFactory.WorkbookMetadata md = ExcelFactory.getMetadata(f)) { for (int i = 0, len = md.getSheetNames().size(); i < len; i++) { @@ -135,7 +131,7 @@ DataLoader createGrid() DataLoader createLoader(boolean hasColumnHeaders) { - ExcelLoader loader = new ExcelLoader(dataFile, md, hasColumnHeaders, null); + ExcelLoader loader = new ExcelLoader(f, md, hasColumnHeaders, null); loader.setInferTypes(false); loader.setIncludeBlankLines(true); loader.setSheetIndex(sheetNum); @@ -163,13 +159,13 @@ void close(DataLoader loader) @Override DataLoader createList() throws IOException, ExperimentException { - return createLoader(dataFile, true); + return createLoader(f, true); } @Override DataLoader createGrid() throws IOException, ExperimentException { - return createLoader(dataFile, false); + return createLoader(f, false); } @Override @@ -200,7 +196,7 @@ protected abstract static class Load abstract void close(DataLoader loader); } - protected double[][] parse(File dataFile, Load load, int expectedRows, int expectedCols) throws ExperimentException, IOException + protected double[][] parse(FileLike dataFile, Load load, int expectedRows, int expectedCols) throws ExperimentException, IOException { // First, attempt to parse list-style data using column headers. DataLoader loader = load.createList(); @@ -355,18 +351,6 @@ public void importTransformDataMap(ExpData data, AssayRunUploadContext contex importRows(data, run, context.getProtocol(), dataMap, context.getUser()); } - @Override - public Map getValidationDataMap(ExpData data, FileLike dataFile, ViewBackgroundInfo info, Logger log, XarContext context, DataLoaderSettings settings) throws ExperimentException - { - DilutionDataFileParser parser = getDataFileParser(data, dataFile, info); - - Map datas = new HashMap<>(); - List> rows = parser.getResults(); - datas.put(NAB_TRANSFORMED_DATA_TYPE, MapDataIterator.of(rows)); - - return datas; - } - @Override public Priority getPriority(ExpData data) { @@ -393,7 +377,7 @@ public void beforeDeleteData(List datas, User user) throws ExperimentEx AssayProvider provider = AssayService.get().getProvider(protocol); AssayProtocolSchema protocolSchema = provider.createProtocolSchema(user, protocol.getContainer(), protocol, null); TableInfo virusTable = protocolSchema.createTable(DilutionManager.VIRUS_TABLE_NAME, null); - if (virusTable instanceof FilteredTable ft) + if (virusTable instanceof FilteredTable ft) { if (virusTable.getColumn(FieldKey.fromParts(NabVirusDomainKind.DATLSID_COLUMN_NAME)) != null) { diff --git a/nab/src/org/labkey/nab/multiplate/CrossPlateDilutionNabDataHandler.java b/nab/src/org/labkey/nab/multiplate/CrossPlateDilutionNabDataHandler.java index f90d51892..f3a8fa682 100644 --- a/nab/src/org/labkey/nab/multiplate/CrossPlateDilutionNabDataHandler.java +++ b/nab/src/org/labkey/nab/multiplate/CrossPlateDilutionNabDataHandler.java @@ -68,12 +68,7 @@ protected Map> getMaterialWellGroupMapping(Dilution throw new ExperimentException("Unable to find sample metadata for sample well group \"" + name + "\": your sample metadata file may contain incorrect well group names, or it may not list all required samples."); } - List materialWellGroups = mapping.get(material); - if (materialWellGroups == null) - { - materialWellGroups = new ArrayList<>(); - mapping.put(material, materialWellGroups); - } + List materialWellGroups = mapping.computeIfAbsent(material, k -> new ArrayList<>()); materialWellGroups.add(specimenGroup); } } diff --git a/nab/src/org/labkey/nab/multiplate/HighThroughputNabDataHandler.java b/nab/src/org/labkey/nab/multiplate/HighThroughputNabDataHandler.java index e68bf6623..7b4178301 100644 --- a/nab/src/org/labkey/nab/multiplate/HighThroughputNabDataHandler.java +++ b/nab/src/org/labkey/nab/multiplate/HighThroughputNabDataHandler.java @@ -16,7 +16,6 @@ package org.labkey.nab.multiplate; import org.apache.commons.lang3.math.NumberUtils; -import org.apache.logging.log4j.Logger; import org.labkey.api.assay.AssayRunUploadContext; import org.labkey.api.assay.dilution.DilutionManager; import org.labkey.api.assay.dilution.SampleProperty; @@ -27,31 +26,24 @@ import org.labkey.api.assay.plate.WellGroup; import org.labkey.api.collections.IntHashMap; import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.MapDataIterator; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.XarContext; -import org.labkey.api.exp.api.DataType; import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.api.ExpMaterial; import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.qc.DataLoaderSettings; import org.labkey.api.assay.transform.TransformDataHandler; import org.labkey.api.reader.ColumnDescriptor; import org.labkey.api.reader.DataLoader; import org.labkey.api.reader.ExcelLoader; import org.labkey.api.reader.TabLoader; -import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.nab.NabAssayProvider; import org.labkey.nab.NabDataHandler; import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -72,18 +64,18 @@ protected String getPreferredDataFileExtension() protected static final String LOCATION_COLUMNN_HEADER = "Well Location"; @Override - protected List createPlates(File dataFile, Plate template) throws ExperimentException + protected List createPlates(FileLike dataFile, Plate template) throws ExperimentException { DataLoader loader = null; try { if (dataFile.getName().toLowerCase().endsWith(".csv")) { - loader = new TabLoader(dataFile, true); + loader = new TabLoader(dataFile.openInputStream(), true, null); ((TabLoader) loader).parseAsCSV(); } else - loader = new ExcelLoader(dataFile, true); + loader = new ExcelLoader(dataFile.openInputStream(), true, null); final int expectedRows = template.getRows(); final int expectedCols = template.getColumns(); @@ -150,12 +142,12 @@ protected List createPlates(ExpRun run, Plate template, boolean recalcSta } @Override - protected double[][] getCellValues(final File dataFile, Plate nabTemplate) + protected double[][] getCellValues(final FileLike dataFile, Plate nabTemplate) { throw new IllegalStateException("getCellValues should not be called for High Throughput handlers."); } - protected List parse(File dataFile, ColumnDescriptor[] columns, List> rows, int expectedRows, int expectedCols) throws ExperimentException + protected List parse(FileLike dataFile, ColumnDescriptor[] columns, List> rows, int expectedRows, int expectedCols) throws ExperimentException { // attempt to parse list-style data if (columns != null && columns.length > 0) @@ -239,18 +231,6 @@ protected void prepareWellGroups(List groups, ExpMaterial sampleInput applyDilution(wells, sampleInput, properties, reverseDirection, sampleProperties); } - @Override - public Map getValidationDataMap(ExpData data, FileLike dataFile, ViewBackgroundInfo info, Logger log, XarContext context, DataLoaderSettings settings) throws ExperimentException - { - DilutionDataFileParser parser = getDataFileParser(data, dataFile, info); - - Map datas = new HashMap<>(); - List> rows = parser.getResults(); - datas.put(NAB_TRANSFORMED_DATA_TYPE, MapDataIterator.of(rows)); - - return datas; - } - @Override public void importTransformDataMap(ExpData data, AssayRunUploadContext context, ExpRun run, DataIteratorBuilder dataMap) throws ExperimentException { diff --git a/nab/src/org/labkey/nab/multiplate/SinglePlateDilutionNabDataHandler.java b/nab/src/org/labkey/nab/multiplate/SinglePlateDilutionNabDataHandler.java index 5538e1bb3..92b6897ab 100644 --- a/nab/src/org/labkey/nab/multiplate/SinglePlateDilutionNabDataHandler.java +++ b/nab/src/org/labkey/nab/multiplate/SinglePlateDilutionNabDataHandler.java @@ -41,8 +41,8 @@ import org.labkey.api.util.Pair; import org.labkey.nab.NabAssayProvider; import org.labkey.nab.NabManager; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -69,18 +69,18 @@ public DataType getDataType() } @Override - protected List createPlates(File dataFile, Plate template) throws ExperimentException + protected List createPlates(FileLike dataFile, Plate template) throws ExperimentException { DataLoader loader = null; try { if (dataFile.getName().toLowerCase().endsWith(".csv")) { - loader = new TabLoader(dataFile, true); + loader = new TabLoader(dataFile.openInputStream(), true, null); ((TabLoader) loader).parseAsCSV(); } else - loader = new ExcelLoader(dataFile, true); + loader = new ExcelLoader(dataFile.openInputStream(), true, null); int wellsPerPlate = template.getRows() * template.getColumns(); diff --git a/nab/src/org/labkey/nab/multiplate/SinglePlateDilutionSamplePropertyHelper.java b/nab/src/org/labkey/nab/multiplate/SinglePlateDilutionSamplePropertyHelper.java index d77ee32d5..c030a7fde 100644 --- a/nab/src/org/labkey/nab/multiplate/SinglePlateDilutionSamplePropertyHelper.java +++ b/nab/src/org/labkey/nab/multiplate/SinglePlateDilutionSamplePropertyHelper.java @@ -28,8 +28,11 @@ import org.labkey.nab.NabAssayProvider; import jakarta.servlet.http.HttpServletRequest; +import org.labkey.vfs.FileLike; + import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -51,12 +54,13 @@ public Map> getSampleProperties(HttpServletR if (_sampleProperties != null) return _sampleProperties; - File metadataFile = getSampleMetadata(request); + FileLike metadataFile = getSampleMetadata(request); if (metadataFile == null) return null; Map> allProperties = new HashMap<>(); - try (ExcelLoader loader = new ExcelLoader(metadataFile, true)) + try (InputStream in = metadataFile.openInputStream(); + ExcelLoader loader = new ExcelLoader(in, true, null)) { Map sampleGroupNames = getSampleWellGroupNameMap(); diff --git a/protein/api-src/org/labkey/api/protein/annotation/DefaultAnnotationLoader.java b/protein/api-src/org/labkey/api/protein/annotation/DefaultAnnotationLoader.java index acea11120..f793694d8 100644 --- a/protein/api-src/org/labkey/api/protein/annotation/DefaultAnnotationLoader.java +++ b/protein/api-src/org/labkey/api/protein/annotation/DefaultAnnotationLoader.java @@ -26,6 +26,7 @@ import org.labkey.api.util.NetworkDrive; import org.labkey.api.util.URLHelper; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.FileNotFoundException; @@ -53,13 +54,13 @@ public DefaultAnnotationLoader(File file, ViewBackgroundInfo info, PipeRoot pipe { throw new IOException("No pipeline root configured for the /Shared project"); } - File logDir = pipelineRoot.resolvePath("proteinAnnotationImport"); - FileUtil.mkdir(logDir); + FileLike logDir = pipelineRoot.resolvePathToFileLike("proteinAnnotationImport"); + logDir.mkdir(); if (!logDir.isDirectory()) { throw new IOException("Could not create directory for log file: " + logDir); } - setLogFile(new File(logDir, file.getName() + "." + DateUtil.formatDateTime(new Date(), FORMAT_STRING) + ".log")); + setLogFile(logDir.resolveChild(file.getName() + "." + DateUtil.formatDateTime(new Date(), FORMAT_STRING) + ".log").toNioPathForWrite()); } @Override diff --git a/signalData/test/src/org/labkey/test/tests/signaldata/SignalDataRawTest.java b/signalData/test/src/org/labkey/test/tests/signaldata/SignalDataRawTest.java index 8b6d9ec9f..f540aa42e 100644 --- a/signalData/test/src/org/labkey/test/tests/signaldata/SignalDataRawTest.java +++ b/signalData/test/src/org/labkey/test/tests/signaldata/SignalDataRawTest.java @@ -20,6 +20,8 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Path; import org.labkey.test.BaseWebDriverTest; import org.labkey.test.Locator; import org.labkey.test.categories.Daily; @@ -169,7 +171,7 @@ public void testFileImport() getFile(String.join("/", ASSAY_DATA_LOC, RESULT_FILENAME_5)), getFile(String.join("/", ASSAY_DATA_LOC, RESULT_FILENAME_6)), getFile(String.join("/", ASSAY_DATA_LOC, RESULT_FILENAME_7)) - ), Collections.EMPTY_MAP, 4); + ), Collections.emptyMap(), 4); // test import of files with a subset of the metadata files importRun(SignalDataInitializer.RAW_SignalData_ASSAY, @@ -189,7 +191,7 @@ public void testFileImport() List.of( getFile(String.join("/", ASSAY_DATA_LOC, RESULT_FILENAME_6)), getFile(String.join("/", ASSAY_DATA_LOC, RESULT_FILENAME_7)) - ), Collections.EMPTY_MAP, 4); + ), Collections.emptyMap(), 4); } @Test @@ -287,7 +289,7 @@ private void uploadWithErrorAction(String assayName, uploadPage.waitForProgressBars(uploadCount); uploadPage.setRunIDField(runId); - Window dialog = uploadPage.saveRunExpectingError(getDriver()); + Window dialog = uploadPage.saveRunExpectingError(getDriver()); String actualMsg = dialog.getBody(); @@ -350,7 +352,7 @@ private SignalDataAssayBeginPage importRun( private File getFile(String relativePath) { - File file = new File(SignalDataInitializer.RAW_SignalData_SAMPLE_DATA, relativePath); + File file = FileUtil.appendPath(SignalDataInitializer.RAW_SignalData_SAMPLE_DATA, Path.parse(relativePath)); if (!file.exists()) throw new RuntimeException("Can't find path: " + file.getAbsolutePath()); return file; diff --git a/viability/src/org/labkey/viability/ViabilityAssayDataHandler.java b/viability/src/org/labkey/viability/ViabilityAssayDataHandler.java index f0e9a9465..12975fa7c 100644 --- a/viability/src/org/labkey/viability/ViabilityAssayDataHandler.java +++ b/viability/src/org/labkey/viability/ViabilityAssayDataHandler.java @@ -45,6 +45,7 @@ import org.labkey.api.assay.AbstractAssayTsvDataHandler; import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayService; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.JunitUtil; import org.labkey.api.util.Pair; import org.labkey.api.view.ActionURL; @@ -430,7 +431,7 @@ private File getViabilitySampleDirectory() throws IOException @Test public void testTsv() throws Exception { - ViabilityTsvDataHandler.Parser parser = new ViabilityTsvDataHandler.Parser(null, null, new File(getViabilitySampleDirectory(), "simple.tsv")); + ViabilityTsvDataHandler.Parser parser = new ViabilityTsvDataHandler.Parser(null, null, FileUtil.appendName(getViabilitySampleDirectory(), "simple.tsv")); List> rows = parser.getResultData(); assertEquals("Expected 3 rows", 3, rows.size()); @@ -459,7 +460,7 @@ public void testTsv() throws Exception @Test public void testGuava() throws Exception { - GuavaDataHandler.Parser parser = new GuavaDataHandler.Parser(null, null, new File(getViabilitySampleDirectory(), "small.VIA.csv")); + GuavaDataHandler.Parser parser = new GuavaDataHandler.Parser(null, null, FileUtil.appendName(getViabilitySampleDirectory(), "small.VIA.csv")); List> rows = parser.getResultData(); assertEquals("Expected 7 rows", 7, rows.size()); @@ -488,7 +489,7 @@ public void testGuava() throws Exception @Test public void testViabilityAndCountFormat() throws Exception { - GuavaDataHandler.Parser parser = new GuavaDataHandler.Parser(null, null, new File(getViabilitySampleDirectory(), "muse_ex.csv")); + GuavaDataHandler.Parser parser = new GuavaDataHandler.Parser(null, null, FileUtil.appendName(getViabilitySampleDirectory(), "muse_ex.csv")); List> rows = parser.getResultData(); assertEquals("Expected 8 rows", 8, rows.size()); @@ -518,7 +519,7 @@ public void testViabilityAndCountFormat() throws Exception @Test public void testExpressPlus() throws Exception { - GuavaDataHandler.Parser parser = new GuavaDataHandler.Parser(null, null, new File(getViabilitySampleDirectory(), "122810.EP5.CSV")); + GuavaDataHandler.Parser parser = new GuavaDataHandler.Parser(null, null, FileUtil.appendName(getViabilitySampleDirectory(), "122810.EP5.CSV")); List> rows = parser.getResultData(); assertEquals("Expected 16 rows", 16, rows.size()); diff --git a/viability/src/org/labkey/viability/ViabilityAssayProvider.java b/viability/src/org/labkey/viability/ViabilityAssayProvider.java index a596a336e..be8b5c04b 100644 --- a/viability/src/org/labkey/viability/ViabilityAssayProvider.java +++ b/viability/src/org/labkey/viability/ViabilityAssayProvider.java @@ -452,7 +452,7 @@ public PipelineProvider getPipelineProvider() } @Override - public List getDataCollectors(Map uploadedFiles, AssayRunUploadForm context) + public List getDataCollectors(@Nullable Map uploadedFiles, AssayRunUploadForm context) { ViabilityAssayRunUploadForm form = (ViabilityAssayRunUploadForm)context; if (form.getReRunId() != null && !form.isDelete())