diff --git a/api/src/org/labkey/api/assay/AbstractAssayProvider.java b/api/src/org/labkey/api/assay/AbstractAssayProvider.java index 6a6c72f562b..666153638d5 100644 --- a/api/src/org/labkey/api/assay/AbstractAssayProvider.java +++ b/api/src/org/labkey/api/assay/AbstractAssayProvider.java @@ -115,6 +115,7 @@ import org.labkey.api.view.NavTree; import org.labkey.api.view.NotFoundException; import org.labkey.api.view.ViewContext; +import org.labkey.vfs.FileLike; import org.labkey.vfs.FileSystemLike; import org.springframework.validation.BindException; import org.springframework.web.servlet.ModelAndView; @@ -612,7 +613,7 @@ public List>> createDefaultDomains(Cont } @Override - public List getDataCollectors(@Nullable Map uploadedFiles, AssayRunUploadForm context) + public List getDataCollectors(@Nullable Map uploadedFiles, AssayRunUploadForm context) { return getDataCollectors(uploadedFiles, context, true); } @@ -623,7 +624,7 @@ public String getResourceName() return getName(); } - public List getDataCollectors(@Nullable Map uploadedFiles, AssayRunUploadForm context, boolean allowFileReuseOnReRun) + public List getDataCollectors(@Nullable Map uploadedFiles, AssayRunUploadForm context, boolean allowFileReuseOnReRun) { List result = new ArrayList<>(); if (!PipelineDataCollector.getFileQueue(context).isEmpty()) @@ -636,7 +637,7 @@ public List getDataCollectors(@Nullable Map up { // In the re-run scenario, figure out what files to offer up for reuse - Map reusableFiles = new HashMap<>(); + Map reusableFiles = new HashMap<>(); // Include any files that were uploaded as part of this request if (uploadedFiles != null && !uploadedFiles.isEmpty()) { @@ -674,14 +675,15 @@ else if (inputDatas.size() > 1) // Filter out any files that aren't under the current pipeline root, since we won't be able to resolve // them successfully due to security restrictions for what's an allowable input to the new run. See issue 18387. PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(context.getContainer()); - for (Iterator> iter = reusableFiles.entrySet().iterator(); iter.hasNext(); ) + for (Iterator> iter = reusableFiles.entrySet().iterator(); iter.hasNext(); ) { - Map.Entry entry = iter.next(); + Map.Entry entry = iter.next(); // If it's not under the current pipeline root - if (pipeRoot == null || !pipeRoot.isUnderRoot(entry.getValue())) + if (pipeRoot == null || !pipeRoot.isUnderRoot(entry.getValue().toNioPathForRead())) { - // Remove it from the collection + // Remove it from both collections iter.remove(); + reusableFiles.remove(entry.getKey()); } } @@ -691,8 +693,7 @@ else if (inputDatas.size() > 1) // to reuse or re-upload if (!reusableFiles.isEmpty()) { - var reusableFileLike = FileSystemLike.wrapFiles(reusableFiles); - result.add(new PreviouslyUploadedDataCollector<>(reusableFileLike)); + result.add(new PreviouslyUploadedDataCollector<>(reusableFiles)); } result.add(new FileUploadDataCollector<>(getMaxFileInputs())); } @@ -708,8 +709,7 @@ else if (inputDatas.size() > 1) // Normal (non-rerun) scenario if (uploadedFiles != null) { - var uploadedFileLikes = FileSystemLike.wrapFiles(uploadedFiles); - result.add(new PreviouslyUploadedDataCollector<>(uploadedFileLikes)); + result.add(new PreviouslyUploadedDataCollector<>(uploadedFiles)); } result.add(new FileUploadDataCollector<>(getMaxFileInputs())); } @@ -717,12 +717,17 @@ else if (inputDatas.size() > 1) return result; } - private void addReusableData(Map reusableFiles, ExpData inputData) + private void addReusableData(Map reusableFiles, ExpData inputData) { // Not all datas are associated with a file if (inputData.getFile() != null) { - reusableFiles.put(AssayDataCollector.PRIMARY_FILE + (reusableFiles.isEmpty() ? "" : Integer.toString(reusableFiles.size())), inputData.getFile()); + String key = AssayDataCollector.PRIMARY_FILE + (reusableFiles.isEmpty() ? "" : Integer.toString(reusableFiles.size())); + File f = inputData.getFile(); + if (f != null) + { + reusableFiles.put(key, org.labkey.vfs.FileSystemLike.wrapFile(f)); + } } } @@ -733,7 +738,7 @@ public AssayRunCreator getRunCreator() } @Override - public ExpProtocol createAssayDefinition(User user, Container container, String name, String description, ExpProtocol.Status status, XarContext context) + public ExpProtocol createAssayDefinition(User user, Container container, String name, String description, ExpProtocol.Status status, @NotNull XarContext context) throws ExperimentException { String protocolLsid = getAssayProtocolLsid(container, name, context); diff --git a/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java b/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java index f8f3f3b2158..715ec17b739 100644 --- a/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java +++ b/api/src/org/labkey/api/assay/AbstractAssayTsvDataHandler.java @@ -143,19 +143,12 @@ public String toString() protected abstract boolean allowEmptyData(); @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException + public void importFile(@NotNull ExpData data, @NotNull FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException { - importFile(data, dataFile, info, log, context, true); + importFile(data, dataFile, info, log, context, true, false); } - @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context, boolean allowLookupByAlternateKey) throws ExperimentException - { - importFile(data, dataFile, info, log, context, allowLookupByAlternateKey, false); - } - - @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context, boolean allowLookupByAlternateKey, boolean autoFillDefaultResultColumns) throws ExperimentException + protected void importFile(@NotNull ExpData data, @NotNull FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context, boolean allowLookupByAlternateKey, boolean autoFillDefaultResultColumns) throws ExperimentException { ExpProtocolApplication sourceApplication = data.getSourceApplication(); if (sourceApplication == null) @@ -175,7 +168,7 @@ public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgr // type conversion error). settings.setBestEffortConversion(true); - FileLike fo = FileSystemLike.wrapFile(dataFile); + FileLike fo = dataFile; Map rawData = getValidationDataMap(data, fo, info, log, context, settings); assert(rawData.size() <= 1); try diff --git a/api/src/org/labkey/api/assay/AbstractTempDirDataCollector.java b/api/src/org/labkey/api/assay/AbstractTempDirDataCollector.java index 904bdfea196..cca1e9fd544 100644 --- a/api/src/org/labkey/api/assay/AbstractTempDirDataCollector.java +++ b/api/src/org/labkey/api/assay/AbstractTempDirDataCollector.java @@ -47,31 +47,7 @@ public abstract class AbstractTempDirDataCollector) context).getUploadAttemptID(); - - // Cleanup files other than input generated by transform scripts - FileLike tempDir = ensureSubdirectory(context.getContainer(), TEMP_DIR_NAME); - FileLike uploadAttemptDir = tempDir.resolveChild(uploadAttemptID); - if (NetworkDrive.exists(uploadAttemptDir)) - { - try - { - FileUtils.deleteDirectory(uploadAttemptDir.toNioPathForWrite().toFile()); - } - catch (IOException e) - { - // Delete quietly - } - } - } - + @Override public void initDir(ContextType context) throws ExperimentException { @@ -234,12 +210,6 @@ public Map uploadComplete(ContextType context, @Nullable ExpRu return result; } - /** @return the preferred name for the run given the primary data file */ - protected String getPreferredAssayId(File primaryFile) - { - return primaryFile.getName(); - } - protected String getPreferredAssayId(FileLike primaryFile) { return primaryFile.getName(); diff --git a/api/src/org/labkey/api/assay/AssayDataCollector.java b/api/src/org/labkey/api/assay/AssayDataCollector.java index d713cc9cf10..90027eb4baa 100644 --- a/api/src/org/labkey/api/assay/AssayDataCollector.java +++ b/api/src/org/labkey/api/assay/AssayDataCollector.java @@ -23,7 +23,6 @@ import org.labkey.api.view.HttpView; import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.util.Map; @@ -57,7 +56,7 @@ public String getButtonText() } /** @return the UI to plug into the import wizard for the user to somehow select/upload the file */ - HttpView getView(ContextType context) throws ExperimentException; + HttpView getView(ContextType context) throws ExperimentException; /** @return the name for this AssayDataCollector. Needs to be unique within the set of data collectors for any given import attempt */ String getShortName(); @@ -81,5 +80,5 @@ default void initDir(ContextType context) throws ExperimentException {} * @return null if the file was uploaded as part of the import */ @Nullable - default File getOriginalFileLocation() { return null; } + default FileLike getOriginalFileLocation() { return null; } } diff --git a/api/src/org/labkey/api/assay/AssayFileWriter.java b/api/src/org/labkey/api/assay/AssayFileWriter.java index 327151f9b62..5164c5689e3 100644 --- a/api/src/org/labkey/api/assay/AssayFileWriter.java +++ b/api/src/org/labkey/api/assay/AssayFileWriter.java @@ -33,7 +33,6 @@ import org.labkey.api.util.NetworkDrive; import org.labkey.api.view.ViewContext; import org.labkey.vfs.FileLike; -import org.labkey.vfs.FileSystemLike; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartHttpServletRequest; @@ -189,8 +188,7 @@ protected FileLike getFileTargetDir(ContextType context) throws ExperimentExcept return ensureUploadDirectory(context.getContainer()); } - /* TODO: this is a really awkward transition between File->FileLike (files come from FileQueue) */ - public Map savePipelineFiles(ContextType context, Map files) throws ExperimentException, IOException + public Map savePipelineFiles(ContextType context, Map files) throws ExperimentException, IOException { Map savedFiles = CollectionUtils.enforceValueClass(new TreeMap<>(), FileLike.class); if (context.getRequest() instanceof MultipartHttpServletRequest) @@ -201,27 +199,24 @@ public Map savePipelineFiles(ContextType context, Map savePostedFiles(ContextType context, Set parameterNames, boolean allowMultiple, boolean ensureExpData) throws ExperimentException, IOException + public Map savePostedFiles(ContextType context, @NotNull Set parameterNames, boolean allowMultiple, boolean ensureExpData) throws ExperimentException, IOException { Map files = CollectionUtils.enforceValueClass(new TreeMap<>(), FileLike.class); Set originalFileNames = new HashSet<>(); @@ -244,7 +239,7 @@ public Map savePostedFiles(ContextType context, Set pa while (iter.hasNext()) { Map.Entry> entry = iter.next(); - if (parameterNames == null || parameterNames.contains(entry.getKey())) + if (parameterNames.contains(entry.getKey())) { List multipartFiles = entry.getValue(); boolean isAfterFirstFile = false; diff --git a/api/src/org/labkey/api/assay/AssayProvider.java b/api/src/org/labkey/api/assay/AssayProvider.java index 1e8dd43ecd3..d94874809db 100644 --- a/api/src/org/labkey/api/assay/AssayProvider.java +++ b/api/src/org/labkey/api/assay/AssayProvider.java @@ -59,10 +59,8 @@ import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.mvc.Controller; -import java.io.File; import java.io.IOException; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -116,17 +114,7 @@ enum ReRunSupport AssayRunCreator getRunCreator(); /** @return all the legal data collectors that the user can choose from for the current import attempt */ - // TODO File->FileLike - List getDataCollectors(Map uploadedFiles, AssayRunUploadForm context); - - default List getDataCollectorsFileObject(Map uploadedFileObjects, AssayRunUploadForm context) - { - Map map = new HashMap<>(); - if (uploadedFileObjects != null) - for (var entry : uploadedFileObjects.entrySet()) - map.put(entry.getKey(), entry.getValue().toNioPathForRead().toFile()); - return getDataCollectors(map, context); - } + List getDataCollectors(Map uploadedFileObjects, AssayRunUploadForm context); /** * @return the name of the assay provider. diff --git a/api/src/org/labkey/api/assay/AssayRunDatabaseContext.java b/api/src/org/labkey/api/assay/AssayRunDatabaseContext.java index 7be9b1a8e0a..f19138d5369 100644 --- a/api/src/org/labkey/api/assay/AssayRunDatabaseContext.java +++ b/api/src/org/labkey/api/assay/AssayRunDatabaseContext.java @@ -182,14 +182,14 @@ public Map getUploadedData() throws ExperimentException @Nullable @Override - public File getOriginalFileLocation() + public FileLike getOriginalFileLocation() { for (ExpData data : _run.getOutputDatas(_provider.getDataType())) { File f = data.getFile(); if (f != null) { - return f.getParentFile(); + return FileSystemLike.wrapFile(f.getParentFile()); } } return null; diff --git a/api/src/org/labkey/api/assay/AssayRunUploadContext.java b/api/src/org/labkey/api/assay/AssayRunUploadContext.java index f2d7a81fc25..4b69318e848 100644 --- a/api/src/org/labkey/api/assay/AssayRunUploadContext.java +++ b/api/src/org/labkey/api/assay/AssayRunUploadContext.java @@ -213,7 +213,7 @@ default void init() throws ExperimentException * @return null if the file was uploaded as part of the import */ @Nullable - default File getOriginalFileLocation() + default FileLike getOriginalFileLocation() { return null; } diff --git a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java index c0896b81cc7..f48283bf482 100644 --- a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java +++ b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java @@ -607,7 +607,15 @@ protected void importStandardResultData( { ExperimentDataHandler dataHandler = insertedData.findDataHandler(); - dataHandler.importFile(insertedData, insertedData.getFile(), info, logger, xarContext, context.isAllowLookupByAlternateKey(), context.shouldAutoFillDefaultResultColumns()); + FileLike fileLike = FileSystemLike.wrapFile(insertedData.getFile()); + if (dataHandler instanceof AbstractAssayTsvDataHandler tsvHandler) + { + tsvHandler.importFile(insertedData, fileLike, info, logger, xarContext, context.isAllowLookupByAlternateKey(), context.shouldAutoFillDefaultResultColumns()); + } + else + { + dataHandler.importFile(insertedData, fileLike, info, logger, xarContext); + } } } } diff --git a/api/src/org/labkey/api/assay/FileUploadDataCollector.java b/api/src/org/labkey/api/assay/FileUploadDataCollector.java index 2f018065ee8..98551e1df12 100644 --- a/api/src/org/labkey/api/assay/FileUploadDataCollector.java +++ b/api/src/org/labkey/api/assay/FileUploadDataCollector.java @@ -23,7 +23,6 @@ import org.labkey.api.view.JspView; import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.HashMap; @@ -38,7 +37,7 @@ public class FileUploadDataCollector> extends AbstractTempDirDataCollector { private final int _maxFileInputs; - private final Map _reusableFiles; + private final Map _reusableFiles; // Name of the form for the file. private final String _fileInputName; @@ -52,12 +51,12 @@ public FileUploadDataCollector(int maxFileInputs) this(maxFileInputs, Collections.emptyMap()); } - public FileUploadDataCollector(int maxFileInputs, Map reusableFiles) + public FileUploadDataCollector(int maxFileInputs, Map reusableFiles) { this(maxFileInputs, reusableFiles, PRIMARY_FILE); } - public FileUploadDataCollector(int maxFileInputs, Map reusableFiles, String fileInputName) + public FileUploadDataCollector(int maxFileInputs, Map reusableFiles, String fileInputName) { _maxFileInputs = maxFileInputs; _reusableFiles = Collections.unmodifiableMap(reusableFiles); @@ -69,12 +68,12 @@ public FileUploadDataCollector(int maxFileInputs, Map reusableFile } @Override - public HttpView getView(ContextType context) + public HttpView getView(ContextType context) { - return new JspView("/org/labkey/api/assay/fileUpload.jsp", this); + return new JspView<>("/org/labkey/api/assay/fileUpload.jsp", this); } - public Map getReusableFiles() + public Map getReusableFiles() { return _reusableFiles; } diff --git a/api/src/org/labkey/api/assay/PipelineDataCollector.java b/api/src/org/labkey/api/assay/PipelineDataCollector.java index b484d562181..2f2eb520491 100644 --- a/api/src/org/labkey/api/assay/PipelineDataCollector.java +++ b/api/src/org/labkey/api/assay/PipelineDataCollector.java @@ -22,6 +22,7 @@ import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.util.DOM; import org.labkey.api.util.HtmlString; import org.labkey.api.util.HtmlStringBuilder; import org.labkey.api.util.Pair; @@ -30,7 +31,6 @@ import jakarta.servlet.http.HttpSession; import org.labkey.vfs.FileLike; -import org.labkey.vfs.FileSystemLike; import java.io.File; import java.io.FileNotFoundException; @@ -41,8 +41,6 @@ import java.util.List; import java.util.Map; -import static org.labkey.api.util.HtmlString.unsafe; - /** * Data collector that supplies files the user previously selected through the pipeline/file browser. * @@ -55,25 +53,25 @@ public PipelineDataCollector() { } - private File _originalFileLocation = null; + private FileLike _originalFileLocation = null; @Override - public HttpView getView(ContextType context) throws ExperimentException + public HttpView getView(ContextType context) throws ExperimentException { return new HtmlView(getHTML(context)); } public HtmlString getHTML(ContextType context) { - Map files = getCurrentFilesForDisplay(context); + Map files = getCurrentFilesForDisplay(context); if (files.isEmpty()) { - return unsafe("
No files have been selected.
"); + return DOM.createHtml(DOM.DIV(DOM.at(DOM.cl("labkey-error")), "No files have been selected.")); } HtmlStringBuilder html = HtmlStringBuilder.of(); html.startTag("ul"); - for (File file : files.values()) + for (FileLike file : files.values()) { html.startTag("li"); html.append(file.getName()); @@ -102,9 +100,9 @@ protected int getAdditionalFileSetCount(ContextType context) } /** @return the files to be processed for the current upload attempt */ - protected Map getCurrentFilesForDisplay(ContextType context) + protected Map getCurrentFilesForDisplay(ContextType context) { - List> files = getFileQueue(context); + List> files = getFileQueue(context); if (files.isEmpty()) { return Collections.emptyMap(); @@ -121,33 +119,32 @@ public String getShortName() @Override public String getDescription(ContextType context) { - List> allFiles = getFileQueue(context); + List> allFiles = getFileQueue(context); if (allFiles.isEmpty()) { return ""; } - Map files = allFiles.get(0); + Map files = allFiles.get(0); return (files.size() > 1 ? files.size() + " files" : "One file ") + " from the Data Pipeline in " + files.values().iterator().next().getParent(); } - public static synchronized void setFileCollection(HttpSession session, Container c, ExpProtocol protocol, List> files) + public static synchronized void setFileCollection(HttpSession session, Container c, ExpProtocol protocol, List> files) { - List> existingFiles = getFileQueue(session, c, protocol); + List> existingFiles = getFileQueue(session, c, protocol); existingFiles.clear(); existingFiles.addAll(files); } - /* TODO File->FileLike convert FileQueue */ - public static List> getFileQueue(AssayRunUploadContext context) + public static List> getFileQueue(AssayRunUploadContext context) { return getFileQueue(context.getRequest().getSession(true), context.getContainer(), context.getProtocol()); } - private static List> getFileQueue(HttpSession session, Container c, ExpProtocol protocol) + private static List> getFileQueue(HttpSession session, Container c, ExpProtocol protocol) { // Use the protocol's RowId instead the ExpProtocol itself because it will be serialized as part of the session // state when Tomcat is shut down cleanly - Map, List>> collections = (Map, List>>) session.getAttribute(PipelineDataCollector.class.getName()); + Map, List>> collections = (Map, List>>) session.getAttribute(PipelineDataCollector.class.getName()); if (collections == null) { collections = new HashMap<>(); @@ -171,43 +168,37 @@ public static synchronized void clearFileQueue(HttpSession session, Container c, @NotNull public Map createData(ContextType context) throws IOException, ExperimentException { - List> files = getFileQueue(context); + List> files = getFileQueue(context); if (files.isEmpty()) { throw new FileNotFoundException("No files from the pipeline directory have been selected"); } - Map currentFiles = files.get(0); + Map currentFiles = files.get(0); if (!currentFiles.isEmpty()) { - _originalFileLocation = currentFiles.values().iterator().next().getParentFile(); + _originalFileLocation = currentFiles.values().iterator().next().getParent(); } return savePipelineFiles(context, currentFiles); } @Nullable @Override - public File getOriginalFileLocation() + public FileLike getOriginalFileLocation() { return _originalFileLocation; } - @Override - public boolean isVisible() - { - return true; - } - // When importing via pipeline, the file is already on the server so return the path of that file @Nullable @Override protected FileLike getFilePath(ContextType context, @Nullable ExpRun run, FileLike tempDirFile) { - Map files = getFileQueue(context).get(0); - for (File file : files.values()) + Map files = getFileQueue(context).get(0); + for (FileLike file : files.values()) { if (file.getName().equals(tempDirFile.getName())) - return FileSystemLike.wrapFile(file); + return file; } return null; @@ -225,7 +216,7 @@ protected void handleTempFile(File tempDirFile, File assayDirFile) public Map uploadComplete(ContextType context, @Nullable ExpRun run) throws ExperimentException { Map result = super.uploadComplete(context, run); - List> files = getFileQueue(context); + List> files = getFileQueue(context); if (!files.isEmpty()) { files.remove(0); diff --git a/api/src/org/labkey/api/assay/PreviouslyUploadedDataCollector.java b/api/src/org/labkey/api/assay/PreviouslyUploadedDataCollector.java index 11588d03805..293b90fb64b 100644 --- a/api/src/org/labkey/api/assay/PreviouslyUploadedDataCollector.java +++ b/api/src/org/labkey/api/assay/PreviouslyUploadedDataCollector.java @@ -16,7 +16,7 @@ package org.labkey.api.assay; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.jetbrains.annotations.NotNull; import org.labkey.api.collections.CollectionUtils; import org.labkey.api.data.Container; @@ -27,7 +27,6 @@ import org.labkey.api.view.InsertView; import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.LinkedHashMap; @@ -100,19 +99,19 @@ public HtmlView getView(ContextType context) sb.append(separator); separator = ", "; sb.append(entry.getValue().getName()); - sb.append(getHiddenFormElementHTML(context.getContainer(), entry.getKey(), entry.getValue().toNioPathForRead().toFile())); + sb.append(getHiddenFormElementHTML(context.getContainer(), entry.getKey(), entry.getValue())); } return new HtmlView(sb); } - public HtmlString getHiddenFormElementHTML(Container container, String formElementName, File file) + public HtmlString getHiddenFormElementHTML(Container container, String formElementName, FileLike file) { PipeRoot pipeRoot = getPipelineRoot(container); HtmlStringBuilder sb = HtmlStringBuilder.of(); sb.unsafeAppend(" getFilesFromRequest(ContextType context, Type type Map result = CollectionUtils.enforceValueClass(new LinkedHashMap<>(), FileLike.class); for (int i = 0; i < paths.length; i++) { - result.put(names[i], pipelineRoot.resolvePathToFileLike(StringUtils.replace(paths[i],"\\","/"))); + result.put(names[i], pipelineRoot.resolvePathToFileLike(Strings.CS.replace(paths[i],"\\","/"))); } return result; } diff --git a/api/src/org/labkey/api/assay/TsvDataHandler.java b/api/src/org/labkey/api/assay/TsvDataHandler.java index 52bbc6a1b67..d8d88c447d2 100644 --- a/api/src/org/labkey/api/assay/TsvDataHandler.java +++ b/api/src/org/labkey/api/assay/TsvDataHandler.java @@ -118,19 +118,13 @@ protected boolean shouldAddInputMaterials() } @Override - public boolean hasContentToExport(ExpData data, File file) - { - return hasContentToExport(data, null != file ? file.toPath() : null); - } - - @Override - public boolean hasContentToExport(ExpData data, Path file) + public boolean hasContentToExport(ExpData data, org.labkey.vfs.FileLike file) { return data.isFinalRunOutput(); } @Override - public void exportFile(ExpData data, Path dataFile, String rootFilePath, User user, OutputStream out) throws ExperimentException + public void exportFile(ExpData data, org.labkey.vfs.FileLike dataFile, User user, OutputStream out) throws ExperimentException { if (data.isFinalRunOutput()) { @@ -154,7 +148,7 @@ public void exportFile(ExpData data, Path dataFile, String rootFilePath, User us try { - File tempFile = FileUtil.createTempFile(FileUtil.getBaseName(FileUtil.getFileName(dataFile)), ".tsv"); + File tempFile = FileUtil.createTempFile("assay_export", ".tsv"); // Figure out the subset of columns to actually export in the TSV, see Issue 36746 Set ignored = Set.of(FieldKey.fromParts("Run"), FieldKey.fromParts("RowId"), FieldKey.fromParts("DataId"), FieldKey.fromParts("Folder")); @@ -163,10 +157,7 @@ public void exportFile(ExpData data, Path dataFile, String rootFilePath, User us { if (!ignored.contains(column.getFieldKey())) { - if (PropertyType.FILE_LINK == column.getPropertyType() && !StringUtils.isEmpty(rootFilePath)) - displayColumns.add(new TsvDataHandler.ExportFileLinkColumn(column, rootFilePath)); - else - displayColumns.add(column.getRenderer()); + displayColumns.add(column.getRenderer()); } } diff --git a/api/src/org/labkey/api/assay/actions/AssayRunUploadForm.java b/api/src/org/labkey/api/assay/actions/AssayRunUploadForm.java index 534f7193b44..0d5f0aeeb24 100644 --- a/api/src/org/labkey/api/assay/actions/AssayRunUploadForm.java +++ b/api/src/org/labkey/api/assay/actions/AssayRunUploadForm.java @@ -247,9 +247,9 @@ private List getDataCollectors() @Nullable @Override - public File getOriginalFileLocation() + public FileLike getOriginalFileLocation() { - AssayDataCollector collector = getSelectedDataCollector(); + AssayDataCollector collector = getSelectedDataCollector(); if (collector != null) { return collector.getOriginalFileLocation(); @@ -356,14 +356,14 @@ public Map getAdditionalPostedFiles(List <%@ page import="java.util.Collections" %> <%@ page import="java.util.Map" %> +<%@ page import="org.labkey.vfs.FileLike" %> <%@ page extends="org.labkey.api.jsp.JspBase" %> <%@ taglib prefix="labkey" uri="http://www.labkey.org/taglib" %> <%! @@ -99,7 +100,7 @@ // Add an entry for all file groups that can be reused from a previous upload <% PreviouslyUploadedDataCollector reuseDataCollector = new PreviouslyUploadedDataCollector(Collections.emptyMap(), PreviouslyUploadedDataCollector.Type.ReRun); - for (Map.Entry entry : bean.getReusableFiles().entrySet()) { %> + for (Map.Entry entry : bean.getReusableFiles().entrySet()) { %> addFileUploadInputRow(null, <%= q(entry.getValue().getName())%>, <%= q(reuseDataCollector.getHiddenFormElementHTML(getContainer(), entry.getKey(), entry.getValue()))%>); <% } %> diff --git a/api/src/org/labkey/api/assay/pipeline/AssayRunAsyncContext.java b/api/src/org/labkey/api/assay/pipeline/AssayRunAsyncContext.java index eb60cff2dcd..e8b97177c41 100644 --- a/api/src/org/labkey/api/assay/pipeline/AssayRunAsyncContext.java +++ b/api/src/org/labkey/api/assay/pipeline/AssayRunAsyncContext.java @@ -42,7 +42,6 @@ import jakarta.servlet.http.HttpServletRequest; import org.labkey.vfs.FileLike; -import java.io.File; import java.io.Serializable; import java.util.Collections; import java.util.HashMap; @@ -78,7 +77,7 @@ public class AssayRunAsyncContext implements private ReImportOption _reImportOption; private boolean _allowCrossRunFileInputs; - private File _originalFileLocation; + private FileLike _originalFileLocation; // Cached values that aren't serializable private transient User _user; @@ -411,7 +410,7 @@ public Logger getLogger() @Nullable @Override - public File getOriginalFileLocation() + public FileLike getOriginalFileLocation() { return _originalFileLocation; } diff --git a/api/src/org/labkey/api/assay/plate/PlateMetadataDataHandler.java b/api/src/org/labkey/api/assay/plate/PlateMetadataDataHandler.java index 5240e686322..b9c26edc22f 100644 --- a/api/src/org/labkey/api/assay/plate/PlateMetadataDataHandler.java +++ b/api/src/org/labkey/api/assay/plate/PlateMetadataDataHandler.java @@ -12,8 +12,7 @@ import org.labkey.api.exp.api.ExpDataRunInput; import org.labkey.api.util.FileType; import org.labkey.api.view.ViewBackgroundInfo; - -import java.io.File; +import org.labkey.vfs.FileLike; public class PlateMetadataDataHandler extends AbstractAssayTsvDataHandler { @@ -57,22 +56,10 @@ public Priority getPriority(ExpData data) } @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException + public void importFile(@NotNull ExpData data, @NotNull FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException { // this is just a noop data handler, the actual parsing and importing of the plate metadata needs to happen in // AbstractAssayTsvDataHandler.addAssayPlateMetadata because we need to access the inserted result data to get at the // lsids. } - - @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context, boolean allowLookupByAlternateKey) throws ExperimentException - { - // see description above - } - - @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context, boolean allowLookupByAlternateKey, boolean autoFillDefaultResultColumns) throws ExperimentException - { - // see description above - } } diff --git a/api/src/org/labkey/api/assay/transform/AnalysisScript.java b/api/src/org/labkey/api/assay/transform/AnalysisScript.java index 308b06ca052..e43020d4cd1 100644 --- a/api/src/org/labkey/api/assay/transform/AnalysisScript.java +++ b/api/src/org/labkey/api/assay/transform/AnalysisScript.java @@ -4,10 +4,12 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import org.labkey.api.util.UnexpectedException; import org.labkey.vfs.FileLike; import org.labkey.vfs.FileSystemLike; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -17,18 +19,18 @@ public class AnalysisScript { - FileLike _script; + final FileLike _script; Set _operations = new HashSet<>(); public AnalysisScript(File script, Set operations) { - _script = new FileSystemLike.Builder(script).build().getRoot(); + this(script); _operations = operations; } private AnalysisScript(File script, List operations) { - _script = new FileSystemLike.Builder(script).build().getRoot(); + this(script); for (String op : operations) { if (op != null) @@ -36,6 +38,18 @@ private AnalysisScript(File script, List operations) } } + private AnalysisScript(File script) + { + try + { + _script = FileSystemLike.wrapFile(script.getParentFile(), script); + } + catch (IOException e) + { + throw UnexpectedException.wrap(e); + } + } + public FileLike getScript() { return _script; diff --git a/api/src/org/labkey/api/assay/transform/DataTransformService.java b/api/src/org/labkey/api/assay/transform/DataTransformService.java index 5b65275fc45..0b46f8387c8 100644 --- a/api/src/org/labkey/api/assay/transform/DataTransformService.java +++ b/api/src/org/labkey/api/assay/transform/DataTransformService.java @@ -94,7 +94,7 @@ public TransformResult transformAndValidate( continue; // read the contents of the script file - File scriptFile = analysisScript.getScript().toNioPathForRead().toFile(); + FileLike scriptFile = analysisScript.getScript(); if (scriptFile.exists()) { if (operation == TransformOperation.UPDATE) @@ -104,7 +104,7 @@ public TransformResult transformAndValidate( // TODO: don't bother slurping the contents of binary scripts StringBuilder sb = new StringBuilder(); - try (BufferedReader br = Readers.getReader(scriptFile)) + try (BufferedReader br = Readers.getReader(scriptFile.openInputStream())) { String l; while ((l = br.readLine()) != null) @@ -115,7 +115,7 @@ public TransformResult transformAndValidate( throw new ValidationException(e.getMessage()); } ScriptEngine engine = null; - String ext = FileUtil.getExtension(scriptFile); + String ext = FileUtil.getExtension(scriptFile.getName()); if (ext != null) { engine = LabKeyScriptEngineManager.get() @@ -143,7 +143,7 @@ public TransformResult transformAndValidate( FileLike runInfo = files.getKey(); bindings.put(ExternalScriptEngine.WORKING_DIRECTORY, scriptDir.toNioPathForWrite().toString()); - bindings.put(ExternalScriptEngine.SCRIPT_PATH, scriptFile.getAbsolutePath()); + bindings.put(ExternalScriptEngine.SCRIPT_PATH, scriptFile.toNioPathForRead().toFile().getAbsolutePath()); Map paramMap = new HashMap<>(); @@ -168,7 +168,7 @@ public TransformResult transformAndValidate( } else { - rewrittenScriptFile = FileSystemLike.wrapFile(scriptFile); + rewrittenScriptFile = scriptFile; } // process any output from the transformation script @@ -228,20 +228,20 @@ public TransformResult transformAndValidate( } else { - throw new ValidationException("The transform script, " + scriptFile.getAbsolutePath() + ", configured for this assay does not exist. Please check " + + throw new ValidationException("The transform script, " + scriptFile + ", configured for this assay does not exist. Please check " + "the configuration for this assay design."); } } return result; } - public void addStandardParameters(@Nullable HttpServletRequest request, @Nullable Container container, @Nullable File scriptFile, @Nullable String apiKey, @NotNull Map paramMap) + public void addStandardParameters(@Nullable HttpServletRequest request, @Nullable Container container, @Nullable FileLike scriptFile, @Nullable String apiKey, @NotNull Map paramMap) { if (scriptFile != null) { - File srcDir = scriptFile.getParentFile(); + FileLike srcDir = scriptFile.getParent(); if (srcDir != null && srcDir.exists()) - paramMap.put(SRC_DIR_REPLACEMENT, srcDir.getAbsolutePath().replaceAll("\\\\", "/")); + paramMap.put(SRC_DIR_REPLACEMENT, srcDir.toNioPathForRead().toFile().getAbsolutePath().replaceAll("\\\\", "/")); } paramMap.put(R_SESSIONID_REPLACEMENT, getSessionInfo(request, apiKey)); paramMap.put(LEGACY_SESSION_COOKIE_NAME_REPLACEMENT, getSessionCookieName(request)); @@ -263,15 +263,14 @@ private String getSessionCookieName(@Nullable HttpServletRequest request) return SecurityManager.TRANSFORM_SESSION_ID; } - private FileLike getScriptDir(ExpProtocol protocol, File scriptFile, boolean isDefault) throws IOException + private FileLike getScriptDir(ExpProtocol protocol, FileLike scriptFile, boolean isDefault) throws IOException { FileLike tempDir = FileUtil.getTempDirectoryFileLike(); FileLike tempRoot = tempDir.resolveChild(ExternalScriptEngine.DEFAULT_WORKING_DIRECTORY); if (isDefault && scriptFile.exists()) { - // TODO getScriptDir(FileLike scriptFile); - tempDir = new FileSystemLike.Builder(scriptFile.getParentFile()).readwrite().root(); + tempDir = scriptFile.getParent(); tempRoot = tempDir.resolveChild("TransformAndValidationFiles"); } diff --git a/api/src/org/labkey/api/data/FileSqlScriptProvider.java b/api/src/org/labkey/api/data/FileSqlScriptProvider.java index 092ac063ae6..016e5c0a61b 100644 --- a/api/src/org/labkey/api/data/FileSqlScriptProvider.java +++ b/api/src/org/labkey/api/data/FileSqlScriptProvider.java @@ -31,6 +31,7 @@ import org.labkey.api.module.ModuleLoader; import org.labkey.api.resource.Resource; import org.labkey.api.settings.AppProps; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Path; import org.labkey.api.vcs.Vcs; @@ -164,7 +165,7 @@ Returned set can be empty (i.e., schemas that have no scripts) // Returns all script file names listed in the specified file private @NotNull Collection getScriptsFromFile(SqlDialect dialect, String filename) throws IOException { - Path path = Path.parse(_module.getSqlScriptsPath(dialect)).append(filename); + Path path = _module.getSqlScriptsPath(dialect).append(filename); Resource r = _module.getModuleResource(path); return null != r ? PageFlowUtil.getStreamContentsAsList(r.getInputStream(), true) : Collections.emptyList(); @@ -205,7 +206,7 @@ protected String getContents(DbSchema schema, String filename) throws SqlScriptE { try { - Path path = Path.parse(_module.getSqlScriptsPath(schema.getSqlDialect())).append(filename); + Path path = _module.getSqlScriptsPath(schema.getSqlDialect()).append(filename); Resource r = _module.getModuleResource(path); if (null == r || !r.isFile()) throw new SqlScriptException("File not found: " + path, filename); @@ -249,7 +250,7 @@ public void saveScript(DbSchema schema, String description, String contents, boo if (!scriptsDir.exists()) throw new IllegalStateException("SQL scripts directory not found"); - File file = new File(scriptsDir, description); + File file = FileUtil.appendName(scriptsDir, description); boolean exists = file.exists(); if (exists && !overwrite) @@ -279,12 +280,12 @@ public File getScriptDirectory(SqlDialect dialect) if (isBlank(_module.getSourcePath())) return null; - String scriptsPath = _module.getSqlScriptsPath(dialect); - File scriptsDir = new File(new File(_module.getSourcePath(), "resources"), scriptsPath); + Path scriptsPath = _module.getSqlScriptsPath(dialect); + File scriptsDir = FileUtil.appendPath(FileUtil.appendName(new File(_module.getSourcePath()), "resources"), scriptsPath); // Handle file structure of old file-based modules, e.g., reagent if (!scriptsDir.exists()) - scriptsDir = new File(_module.getSourcePath(), scriptsPath); + scriptsDir = FileUtil.appendPath(new File(_module.getSourcePath()), scriptsPath); return scriptsDir; } diff --git a/api/src/org/labkey/api/data/TSVGridWriter.java b/api/src/org/labkey/api/data/TSVGridWriter.java index 043b44cf9a8..a678c886d39 100644 --- a/api/src/org/labkey/api/data/TSVGridWriter.java +++ b/api/src/org/labkey/api/data/TSVGridWriter.java @@ -22,6 +22,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.collections.ResultSetRowMapFactory; import org.labkey.api.query.FieldKey; +import org.labkey.api.util.FileUtil; import org.labkey.api.view.HttpView; import java.io.File; @@ -266,7 +267,7 @@ private List writeResultSetBatches(Results results, File outputDir, String private File startBatchFile(File outputDir, String baseName, String extension, int batchSize, int totalBatches) throws IOException { String batchId = batchSize == 0 ? "" : "-" + totalBatches; - File file = new File(outputDir, baseName + batchId + extension); + File file = FileUtil.appendName(outputDir, baseName + batchId + extension); prepare(file); writeFileHeader(); if (isHeaderRowVisible()) diff --git a/api/src/org/labkey/api/data/TempTableTracker.java b/api/src/org/labkey/api/data/TempTableTracker.java index bc9d62e0709..3ce1c067499 100644 --- a/api/src/org/labkey/api/data/TempTableTracker.java +++ b/api/src/org/labkey/api/data/TempTableTracker.java @@ -212,7 +212,7 @@ static void synchronizeLog(boolean loadFile) try { if (null == tempTableLog) - tempTableLog = new RandomAccessFile(new File(FileUtil.getTempDirectory(), LOGFILE), "rwd"); + tempTableLog = new RandomAccessFile(FileUtil.appendName(FileUtil.getTempDirectory(), LOGFILE), "rwd"); if (loadFile) { @@ -344,7 +344,7 @@ public void shutdownStarted() { join(5000); } - catch (InterruptedException e) {} + catch (InterruptedException ignored) {} synchronizeLog(false); } } diff --git a/api/src/org/labkey/api/data/WorkbookContainerType.java b/api/src/org/labkey/api/data/WorkbookContainerType.java index 24e17d4c27d..b3796ae673e 100644 --- a/api/src/org/labkey/api/data/WorkbookContainerType.java +++ b/api/src/org/labkey/api/data/WorkbookContainerType.java @@ -435,7 +435,7 @@ public void testFileRootOverride() throws Exception File parentRoot = FileContentService.get().getFileRoot(parent); File origSubRoot = FileContentService.get().getFileRoot(sub); File origWbRoot = FileContentService.get().getFileRoot(wbOverride); - File wbOverrideRoot = new File(FileContentService.get().getSiteDefaultRoot(), "_foo"); + File wbOverrideRoot = FileUtil.appendName(FileContentService.get().getSiteDefaultRoot(), "_foo"); FileContentService.get().setFileRoot(wbOverride, wbOverrideRoot); @@ -448,9 +448,9 @@ public void testFileRootOverride() throws Exception Assert.assertEquals("Incorrect file root", wbOverrideRoot, FileContentService.get().getFileRoot(wbOverride)); Assert.assertEquals("Should not have changed", origWbRoot, FileContentService.get().getDefaultRoot(wbOverride, false)); - Assert.assertEquals("Incorrect file root", new File(parentRoot, wbDefaultRoot.getName()), FileContentService.get().getFileRoot(wbDefaultRoot)); + Assert.assertEquals("Incorrect file root", FileUtil.appendName(parentRoot, wbDefaultRoot.getName()), FileContentService.get().getFileRoot(wbDefaultRoot)); - File test = new File(FileContentService.get().getFileRoot(wbOverride), "/@files/test.txt"); + File test = FileUtil.appendPath(FileContentService.get().getFileRoot(wbOverride), Path.parse("/@files/test.txt")); if (!test.getParentFile().exists()) { FileUtil.mkdirs(test.getParentFile()); diff --git a/api/src/org/labkey/api/data/bigiron/AbstractClrInstallationManager.java b/api/src/org/labkey/api/data/bigiron/AbstractClrInstallationManager.java index 25bd23681ed..e001dc1f1ef 100644 --- a/api/src/org/labkey/api/data/bigiron/AbstractClrInstallationManager.java +++ b/api/src/org/labkey/api/data/bigiron/AbstractClrInstallationManager.java @@ -231,7 +231,7 @@ private String getEnableClrStatements(SqlDialect dialect) throws SqlScriptExcept { Module core = ModuleLoader.getInstance().getCoreModule(); String enableClrFilename = "enable_clr.sql"; - Path path = Path.parse(core.getSqlScriptsPath(dialect)).append(enableClrFilename); + Path path = core.getSqlScriptsPath(dialect).append(enableClrFilename); Resource r = core.getModuleResource(path); try diff --git a/api/src/org/labkey/api/exp/ExperimentDataHandler.java b/api/src/org/labkey/api/exp/ExperimentDataHandler.java index 69cd303f9ba..3c7cba267a4 100644 --- a/api/src/org/labkey/api/exp/ExperimentDataHandler.java +++ b/api/src/org/labkey/api/exp/ExperimentDataHandler.java @@ -25,14 +25,12 @@ import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.security.User; -import org.labkey.api.util.FileUtil; import org.labkey.api.util.URLHelper; import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.OutputStream; -import java.nio.file.Path; import java.util.List; /** @@ -56,36 +54,13 @@ default String getFileName(ExpData data, String defaultName) * Import whatever content from the file is destined for storage in the database. Typically persisted in a schema * owned by the module that holds the implementation of the ExperimentDataHandler. */ - // TODO File->FileLike - void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException; - default void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context, boolean allowLookupByAlternateKey) throws ExperimentException - { - importFile(data, dataFile, info, log, context); - } - - // TODO File->FileLike - default void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context, boolean allowLookupByAlternateKey, boolean autoFillDefaultResultColumns) throws ExperimentException - { - importFile(data, dataFile, info, log, context); - } - - // TODO File->FileLike - default void importFile(@NotNull ExpData data, Path dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException - { - if (FileUtil.hasCloudScheme(dataFile)) - throw new ExperimentException(this.getClass().getName() + " does not support importFile on a cloud path"); - importFile(data, dataFile.toFile(), info, log, context); - } + void importFile(@NotNull ExpData data, @NotNull FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException; /** * Stream the content of this data object. Typically, this just streams the bytes of the file from disk, but could * create something based exclusively on what's in the database. */ - // TODO File->FileLike - void exportFile(ExpData data, File dataFile, User user, OutputStream out) throws ExperimentException; - default void exportFile(ExpData data, Path dataFile, String rootFilePath, User user, OutputStream out) throws ExperimentException - { - } + void exportFile(ExpData data, @Nullable FileLike dataFile, User user, OutputStream out) throws ExperimentException; /** @return URL to the imported version of the data, like a grid view over a database table or a custom details page */ @Nullable @@ -106,11 +81,10 @@ default void exportFile(ExpData data, Path dataFile, String rootFilePath, User u */ void deleteData(ExpData data, Container container, User user); - // TODO File->FileLike - boolean hasContentToExport(ExpData data, File file); - default boolean hasContentToExport(ExpData data, Path file) + // Preferred: use FileLike + default boolean hasContentToExport(ExpData data, @Nullable FileLike file) { - return false; + return file != null && file.exists() && file.isFile(); } default void runMoved(ExpData newData, Container container, Container targetContainer, String oldRunLSID, String newRunLSID, User user, long oldDataRowID) throws ExperimentException diff --git a/api/src/org/labkey/api/exp/api/AbstractExperimentDataHandler.java b/api/src/org/labkey/api/exp/api/AbstractExperimentDataHandler.java index 97e09b9391f..1ca266c0e90 100644 --- a/api/src/org/labkey/api/exp/api/AbstractExperimentDataHandler.java +++ b/api/src/org/labkey/api/exp/api/AbstractExperimentDataHandler.java @@ -21,14 +21,11 @@ import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.Lsid; import org.labkey.api.security.User; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.NetworkDrive; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.List; /** @@ -46,19 +43,13 @@ public String getFileName(ExpData data, String defaultName) } @Override - public void exportFile(ExpData data, File dataFile, User user, OutputStream out) throws ExperimentException - { - exportFile(data, dataFile.toPath(), null, user, out); - } - - @Override - public void exportFile(ExpData data, Path dataFile, String rootFilePath, User user, OutputStream out) throws ExperimentException + public void exportFile(ExpData data, FileLike dataFile, User user, OutputStream out) throws ExperimentException { if (dataFile != null) { - try + try (InputStream is = dataFile.openInputStream()) { - Files.copy(dataFile, out); + is.transferTo(out); } catch (IOException e) { @@ -67,26 +58,16 @@ public void exportFile(ExpData data, Path dataFile, String rootFilePath, User us } } - @Override - public void beforeDeleteData(List data, User user) throws ExperimentException - { - } @Override - public boolean hasContentToExport(ExpData data, File file) + public void beforeDeleteData(List data, User user) throws ExperimentException { - return hasContentToExport(data, file.toPath()); } @Override - public boolean hasContentToExport(ExpData data, Path path) + public boolean hasContentToExport(ExpData data, FileLike file) { - if (!FileUtil.hasCloudScheme(path)) - { - File file = path.toFile(); - return NetworkDrive.exists(file) && file.isFile(); - } - return Files.exists(path) && !Files.isDirectory(path); + return file != null && file.exists() && file.isFile(); } @Override diff --git a/api/src/org/labkey/api/exp/api/DefaultExperimentDataHandler.java b/api/src/org/labkey/api/exp/api/DefaultExperimentDataHandler.java index 24ad6f495ab..8c4600200a1 100644 --- a/api/src/org/labkey/api/exp/api/DefaultExperimentDataHandler.java +++ b/api/src/org/labkey/api/exp/api/DefaultExperimentDataHandler.java @@ -24,6 +24,7 @@ import org.labkey.api.security.User; import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; import java.io.File; @@ -41,7 +42,7 @@ public DataType getDataType() } @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) + public void importFile(@NotNull ExpData data, @NotNull FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) { log.debug("No ExperimentDataHandler registered for data file " + data.getDataFileURI() + ", no special loading will be done on this file."); } diff --git a/api/src/org/labkey/api/exp/api/ExperimentService.java b/api/src/org/labkey/api/exp/api/ExperimentService.java index cc3c1cca948..bdf36dc10f8 100644 --- a/api/src/org/labkey/api/exp/api/ExperimentService.java +++ b/api/src/org/labkey/api/exp/api/ExperimentService.java @@ -84,6 +84,7 @@ import org.labkey.api.view.NotFoundException; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.api.view.ViewContext; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -715,6 +716,12 @@ static void validateParentAlias(Map aliasMap, Set reserv */ ExpData getExpDataByURL(File f, @Nullable Container c); + /** + * Get the most recently created ExpData for the file, if it exists. + * @see #getAllExpDataByURL(Path, Container) + */ + ExpData getExpDataByURL(FileLike f, @Nullable Container c); + /** * Get the most recently created ExpData for the path, if it exists. * @see #getAllExpDataByURL(Path, Container) @@ -991,8 +998,6 @@ List getExpProtocolsWithParameterValue( List importXar(XarSource source, PipelineJob pipelineJob, XarImportOptions options) throws ExperimentException; - File exportXarForRuns(User user, Set runIds, Long expRowId, XarExportOptions options) throws NotFoundException, IOException, ExperimentException; - /** * Create an experiment run to represent the work that the task's job has done so far. * The job's recorded actions will be marked as completed after creating the ExpRun so subsequent diff --git a/api/src/org/labkey/api/files/FileSystemWatcherImpl.java b/api/src/org/labkey/api/files/FileSystemWatcherImpl.java index 880074aaa2a..5f8339d4352 100644 --- a/api/src/org/labkey/api/files/FileSystemWatcherImpl.java +++ b/api/src/org/labkey/api/files/FileSystemWatcherImpl.java @@ -489,7 +489,7 @@ public static class TestCase extends Assert public void testFileWatchers() throws IOException, InterruptedException { File root = FileUtil.createTempDirectory("fileWatcherTest").toFile(); //Conversion should be safe as temp dir should be on a local URI //TODO should convert to Path for consistencies sake - File testFolder = new File(root, "test"); + File testFolder = FileUtil.appendName(root, "test"); // Do it twice to ensure that everything gets cleaned up on directory delete and a new watcher can be added // to a new version of the same directory @@ -553,13 +553,13 @@ public void overflow() assertNotNull(plm); assertEquals(1, plm._list.size()); - File a = new File(testFolder, "a"); - File b = new File(testFolder, "b"); - File c = new File(testFolder, "c"); + File a = FileUtil.appendName(testFolder, "a"); + File b = FileUtil.appendName(testFolder, "b"); + File c = FileUtil.appendName(testFolder, "c"); - assertTrue(a.createNewFile()); - assertTrue(b.createNewFile()); - assertTrue(c.createNewFile()); + assertTrue(FileUtil.createNewFile(a)); + assertTrue(FileUtil.createNewFile(b)); + assertTrue(FileUtil.createNewFile(c)); waitForEvents(events, 3); diff --git a/api/src/org/labkey/api/jsp/JspClassLoader.java b/api/src/org/labkey/api/jsp/JspClassLoader.java index 7f00130da8a..3df284ef7c6 100644 --- a/api/src/org/labkey/api/jsp/JspClassLoader.java +++ b/api/src/org/labkey/api/jsp/JspClassLoader.java @@ -22,6 +22,8 @@ import org.labkey.api.util.ConfigurationException; import jakarta.servlet.ServletContext; +import org.labkey.api.util.FileUtil; + import java.io.File; import java.net.MalformedURLException; import java.net.URL; @@ -71,7 +73,7 @@ private boolean scanForJspJars() { // use m.getExplodedPath() instead of getModuleResource() otherwise, we will look for the // jars in the source directory on dev machines - var libDir = new File(m.getExplodedPath(), "lib"); + var libDir = FileUtil.appendName(m.getExplodedPath(), "lib"); if (!libDir.exists()) continue; var listing = libDir.listFiles((dir, name) -> name.contains("_jsp") && name.endsWith(".jar")); diff --git a/api/src/org/labkey/api/jsp/RecompilingJspClassLoader.java b/api/src/org/labkey/api/jsp/RecompilingJspClassLoader.java index bd4101ceb70..dfff873721a 100644 --- a/api/src/org/labkey/api/jsp/RecompilingJspClassLoader.java +++ b/api/src/org/labkey/api/jsp/RecompilingJspClassLoader.java @@ -31,6 +31,8 @@ import jakarta.servlet.ServletContext; import jakarta.servlet.ServletException; +import org.labkey.api.util.Path; + import javax.tools.JavaCompiler; import javax.tools.ToolProvider; import java.io.ByteArrayOutputStream; @@ -70,7 +72,7 @@ public Class loadClass(ServletContext context, String jspFilename) throws ClassN { File jspJavaFileBuildDirectory = new File(finder.getBuildPath() + JSP_JAVA_PATH); File jspClassesFileBuildDirectory = new File(finder.getBuildPath() + JSP_CLASSES_DIR); - File classFile = new File(jspClassesFileBuildDirectory, JSP_PACKAGE_PATH + compiledJspPath + ".class"); + File classFile = FileUtil.appendPath(jspClassesFileBuildDirectory, Path.parse(JSP_PACKAGE_PATH + compiledJspPath + ".class")); File sourceFile = null; if (null != finder.getSourcePath()) sourceFile = new File(getCompleteSourcePath(finder.getSourcePath(), getSourceJspPath(jspFilename))); diff --git a/api/src/org/labkey/api/module/DefaultModule.java b/api/src/org/labkey/api/module/DefaultModule.java index 8935a9d88df..e654802bb8d 100644 --- a/api/src/org/labkey/api/module/DefaultModule.java +++ b/api/src/org/labkey/api/module/DefaultModule.java @@ -1029,7 +1029,7 @@ public final Set getSqlScripts(@NotNull DbSchema schema) { SqlDialect dialect = schema.getSqlDialect(); - String sqlScriptsPath = getSqlScriptsPath(dialect); + Path sqlScriptsPath = getSqlScriptsPath(dialect); Resource dir = getModuleResource(sqlScriptsPath); if (dir == null || !dir.isCollection()) return Collections.emptySet(); @@ -1046,9 +1046,9 @@ public final Set getSqlScripts(@NotNull DbSchema schema) } @Override - public final String getSqlScriptsPath(@NotNull SqlDialect dialect) + public final Path getSqlScriptsPath(@NotNull SqlDialect dialect) { - return "schemas/dbscripts/" + dialect.getSQLScriptPath() + "/"; + return Path.parse("schemas/dbscripts/" + dialect.getSQLScriptPath() + "/"); } @Override diff --git a/api/src/org/labkey/api/module/MockModule.java b/api/src/org/labkey/api/module/MockModule.java index 84e7b1215d4..ee2c70dbc23 100644 --- a/api/src/org/labkey/api/module/MockModule.java +++ b/api/src/org/labkey/api/module/MockModule.java @@ -367,7 +367,7 @@ public Set getSqlScripts(@Nullable DbSchema schema) } @Override - public String getSqlScriptsPath(@NotNull SqlDialect dialect) + public Path getSqlScriptsPath(@NotNull SqlDialect dialect) { return null; } diff --git a/api/src/org/labkey/api/module/Module.java b/api/src/org/labkey/api/module/Module.java index cf938f6e959..e64bb9fe929 100644 --- a/api/src/org/labkey/api/module/Module.java +++ b/api/src/org/labkey/api/module/Module.java @@ -355,7 +355,7 @@ default String getBuildTime() * @param dialect The sql dialect for the scripts * @return The script file path */ - String getSqlScriptsPath(@NotNull SqlDialect dialect); + Path getSqlScriptsPath(@NotNull SqlDialect dialect); /** * handle a http request diff --git a/api/src/org/labkey/api/module/ModuleLoader.java b/api/src/org/labkey/api/module/ModuleLoader.java index a3f8f5b9f91..1dfd255e7c0 100644 --- a/api/src/org/labkey/api/module/ModuleLoader.java +++ b/api/src/org/labkey/api/module/ModuleLoader.java @@ -1214,7 +1214,7 @@ else if (!moduleNamePattern.matcher(module.getName()).matches()) private Module loadModuleFromProperties(ApplicationContext parentContext, File moduleDir) throws ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException { //check for simple .properties file - File modulePropsFile = new File(moduleDir, "config/module.properties"); + File modulePropsFile = FileUtil.appendPath(moduleDir, Path.parse("config/module.properties")); Map props = Collections.emptyMap(); if (modulePropsFile.exists()) { diff --git a/api/src/org/labkey/api/pipeline/AbstractSpecimenTransformTask.java b/api/src/org/labkey/api/pipeline/AbstractSpecimenTransformTask.java index 0e23dfd2a9f..42d9aa62f8a 100644 --- a/api/src/org/labkey/api/pipeline/AbstractSpecimenTransformTask.java +++ b/api/src/org/labkey/api/pipeline/AbstractSpecimenTransformTask.java @@ -55,24 +55,6 @@ public AbstractSpecimenTransformTask(PipelineJob job) _job = job; } - @Deprecated - public void transform(File input, File output) throws PipelineJobException - { - throw new UnsupportedOperationException("Use the Path version of the transform method"); - } - - public void transform(Path input, Path output) throws PipelineJobException - { - try - { - transform(input.toFile(), output.toFile()); - } - catch (UnsupportedOperationException e) - { - throw new PipelineJobException("Error transforming input file.", e); - } - } - /** * Transform a row of data from the parsed input data file the transformed specimen row * During the transform, it's expected that any lab, primary, derivative and additive diff --git a/api/src/org/labkey/api/pipeline/LocalDirectory.java b/api/src/org/labkey/api/pipeline/LocalDirectory.java index c614f3f5470..f537ce2e94f 100644 --- a/api/src/org/labkey/api/pipeline/LocalDirectory.java +++ b/api/src/org/labkey/api/pipeline/LocalDirectory.java @@ -102,7 +102,7 @@ public LocalDirectory(Container container, String moduleName, PipeRoot pipeRoot, try { File containerDir = ensureContainerDir(container); - _localDirectoryFile = new File(containerDir, FileUtil.makeFileNameWithTimestamp("_temp_")); + _localDirectoryFile = FileUtil.appendName(containerDir, FileUtil.makeFileNameWithTimestamp("_temp_")); ensureLocalDirectory(); } @@ -179,7 +179,7 @@ public File copyToLocalDirectory(String url, Logger log) String filename = FileUtil.getFileName(path); try { - File tempFile = new File(_localDirectoryFile, filename); + File tempFile = FileUtil.appendName(_localDirectoryFile, filename); if (!Files.exists(tempFile.toPath())) { Files.copy(path, tempFile.toPath(), StandardCopyOption.COPY_ATTRIBUTES); @@ -214,7 +214,7 @@ public static File copyToContainerDirectory(@NotNull Container container, @NotNu try { File containerDir = ensureContainerDir(container); - File tempFile = new File(containerDir, tempFileName); + File tempFile = FileUtil.appendName(containerDir, tempFileName); if (!Files.exists(tempFile.toPath())) { log.debug("Copying file to container's temp directory: "+ FileUtil.pathToString(remotePath)); @@ -295,12 +295,12 @@ private void deleteDirectory(Path dir) private static File getModuleLocalTempDirectory() { File tempDir = FileUtil.getTempDirectory(); // tomcat/temp or similar - return new File(tempDir, FileUtil.makeLegalName(PipelineService.MODULE_NAME + "_temp")); + return FileUtil.appendName(tempDir, FileUtil.makeLegalName(PipelineService.MODULE_NAME + "_temp")); } public static File getContainerLocalTempDirectory(Container container) { - return new File(getModuleLocalTempDirectory(), FileUtil.makeLegalName(container.getName() + "_" + container.getId())); + return FileUtil.appendName(getModuleLocalTempDirectory(), FileUtil.makeLegalName(container.getName() + "_" + container.getId())); } public Path getRemoteLogFilePath() diff --git a/api/src/org/labkey/api/pipeline/PipeRoot.java b/api/src/org/labkey/api/pipeline/PipeRoot.java index 4996535e3ad..d71491f9586 100644 --- a/api/src/org/labkey/api/pipeline/PipeRoot.java +++ b/api/src/org/labkey/api/pipeline/PipeRoot.java @@ -108,11 +108,15 @@ public interface PipeRoot extends SecurableResource /** @return relative path to the file from the root. null if the file isn't under the root. Does not include a leading slash */ String relativePath(File file); + /** @return relative path to the file from the root. null if the file isn't under the root. Does not include a leading slash */ + String relativePath(FileLike file); + /** @return relative path to the file from the root. null if the path isn't under the root. Does not include a leading slash */ String relativePath(Path file); /** @return whether the file specified is a child of the pipeline root */ boolean isUnderRoot(File file); + boolean isUnderRoot(FileLike file); boolean isUnderRoot(Path file); boolean hasPermission(Container container, User user, Class perm); diff --git a/api/src/org/labkey/api/pipeline/PipelineJob.java b/api/src/org/labkey/api/pipeline/PipelineJob.java index 18e595703a3..e872a09ce1f 100644 --- a/api/src/org/labkey/api/pipeline/PipelineJob.java +++ b/api/src/org/labkey/api/pipeline/PipelineJob.java @@ -476,7 +476,7 @@ public static File findUniqueLogFile(File primaryFile, String baseName) String validBaseName = FileUtil.makeLegalName(baseName); // need to look in current and archived dirs for any unused log file names (issue 20987) File fileLog = FT_LOG.newFile(primaryFile.getParentFile(), validBaseName); - File archivedDir = new File(primaryFile.getParentFile(), AssayFileWriter.ARCHIVED_DIR_NAME); + File archivedDir = FileUtil.appendName(primaryFile.getParentFile(), AssayFileWriter.ARCHIVED_DIR_NAME); File fileLogArchived = FT_LOG.newFile(archivedDir, validBaseName); int index = 1; diff --git a/api/src/org/labkey/api/pipeline/PipelineProtocolFactory.java b/api/src/org/labkey/api/pipeline/PipelineProtocolFactory.java index 20a29692af8..97e4d4dcba6 100644 --- a/api/src/org/labkey/api/pipeline/PipelineProtocolFactory.java +++ b/api/src/org/labkey/api/pipeline/PipelineProtocolFactory.java @@ -52,8 +52,8 @@ public static Path getProtocolRootDir(PipeRoot root) public static File locateProtocolRootDir(File rootDir, File systemDir) { - File protocolRootDir = new File(systemDir, _pipelineProtocolDir); - File protocolRootDirLegacy = new File(rootDir, _pipelineProtocolDir); + File protocolRootDir = FileUtil.appendName(systemDir, _pipelineProtocolDir); + File protocolRootDirLegacy = FileUtil.appendName(rootDir, _pipelineProtocolDir); if (NetworkDrive.exists(protocolRootDirLegacy)) protocolRootDirLegacy.renameTo(protocolRootDir); return protocolRootDir; diff --git a/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisProtocolFactory.java b/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisProtocolFactory.java index 12b60e2c06e..28fe1bf96e7 100644 --- a/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisProtocolFactory.java +++ b/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisProtocolFactory.java @@ -178,11 +178,11 @@ public void initSystemDirectory(File rootDir, File systemDir) File protocolRootDir = locateProtocolRootDir(rootDir, systemDir); // Make sure the defaults for this particular protocol are in the right place. - File fileLegacyDefaults = new File(rootDir, getLegacyDefaultParametersFileName()); + File fileLegacyDefaults = FileUtil.appendName(rootDir, getLegacyDefaultParametersFileName()); if (NetworkDrive.exists(fileLegacyDefaults)) { - File protocolDir = new File(protocolRootDir, getName()); - fileLegacyDefaults.renameTo(new File(protocolDir, getDefaultParametersFileName())); + File protocolDir = FileUtil.appendName(protocolRootDir, getName()); + fileLegacyDefaults.renameTo(FileUtil.appendName(protocolDir, getDefaultParametersFileName())); } } diff --git a/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java b/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java index 1c576c17623..552d468f729 100644 --- a/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java +++ b/api/src/org/labkey/api/qc/TsvDataExchangeHandler.java @@ -488,9 +488,9 @@ private Map getContextProperties(DataTransformService.TransformO { map.put(Props.assayId.name(), StringUtils.defaultString(context.getName())); map.put(Props.runComments.name(), StringUtils.defaultString(context.getComments())); - File originalFileLocation = context.getOriginalFileLocation(); + FileLike originalFileLocation = context.getOriginalFileLocation(); if (originalFileLocation != null) - map.put(Props.originalFileLocation.name(), originalFileLocation.getPath()); + map.put(Props.originalFileLocation.name(), originalFileLocation.toNioPathForRead().toFile().getPath()); } map.put(Props.baseUrl.name(), AppProps.getInstance().getBaseServerUrl() + AppProps.getInstance().getContextPath()); @@ -1200,7 +1200,7 @@ private static class RunInfo { private File _errorFile; private String _warningSevLevel; - private File _originalFileLocation; + private FileLike _originalFileLocation; private ExpProtocol _protocol; public void setProtocol(ExpProtocol protocol) @@ -1222,12 +1222,12 @@ public boolean isBackgroundUpload() // Original file location used to determine if file was already on the server or not. Used for cleanup when // there is an error. - public File getOriginalFileLocation() + public FileLike getOriginalFileLocation() { return _originalFileLocation; } - public void setOriginalFileLocation(File originalFileLocation) + public void setOriginalFileLocation(FileLike originalFileLocation) { _originalFileLocation = originalFileLocation; } @@ -1399,12 +1399,5 @@ public Logger getLogger() { return null; } - - @Nullable - @Override - public File getOriginalFileLocation() - { - return null; - } } } diff --git a/api/src/org/labkey/api/reader/DataLoaderService.java b/api/src/org/labkey/api/reader/DataLoaderService.java index 25396ef5585..ac1c63f4312 100644 --- a/api/src/org/labkey/api/reader/DataLoaderService.java +++ b/api/src/org/labkey/api/reader/DataLoaderService.java @@ -47,10 +47,10 @@ static void setInstance(DataLoaderService impl) void registerFactory(@NotNull DataLoaderFactory factory); @Nullable - DataLoaderFactory findFactory(File file, @Nullable FileType guessFormat); + DataLoaderFactory findFactory(FileLike file, @Nullable FileType guessFormat); @Nullable - DataLoaderFactory findFactory(File file, String contentType, @Nullable FileType guessFormat); + DataLoaderFactory findFactory(FileLike file, String contentType, @Nullable FileType guessFormat); @Nullable DataLoaderFactory findFactory(String filename, String contentType, InputStream is, @Nullable FileType guessFormat); diff --git a/api/src/org/labkey/api/reader/ExcelFactory.java b/api/src/org/labkey/api/reader/ExcelFactory.java index 0d344340d3e..83f1a895978 100644 --- a/api/src/org/labkey/api/reader/ExcelFactory.java +++ b/api/src/org/labkey/api/reader/ExcelFactory.java @@ -57,6 +57,7 @@ import org.labkey.api.data.ExcelWriter; import org.labkey.api.reader.jxl.JxlWorkbook; import org.labkey.api.settings.AppProps; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.JunitUtil; import org.labkey.api.util.logging.LogHelper; import org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSheet; @@ -180,7 +181,7 @@ public static WorkbookMetadata getMetadata(@NotNull File file) throws IOExceptio { return getMetadata(opc); } - // might be OLE2 formet + // might be OLE2 format catch (InvalidFormatException|UnsupportedFileFormatException e) { Workbook wb = create(file); @@ -584,11 +585,11 @@ public static Cell getCell(Sheet sheet, int colIdx, int rowIdx) /** Supports .xls (BIFF8 only), and .xlsx */ @NotNull - public static JSONArray convertExcelToJSON(InputStream in, boolean extended) throws IOException + public static JSONArray convertExcelToJSON(InputStream in, boolean extended, int maxRows) throws IOException { try (Workbook workbook = WorkbookFactory.create(in)) { - return convertExcelToJSON(workbook, extended, -1); + return convertExcelToJSON(workbook, extended, maxRows); } } @@ -919,9 +920,9 @@ public void testExcelFileImportShouldFail() throws Exception { File dataloading = JunitUtil.getSampleData(null, "dataLoading/excel"); - attemptImportExpectError(new File(dataloading, "doesntexist.xls"), FileNotFoundException.class); - attemptImportExpectError(new File(dataloading, ""), FileNotFoundException.class); - attemptImportExpectError(new File(dataloading, "notreallyexcel.xls"), InvalidFormatException.class); + attemptImportExpectError(FileUtil.appendName(dataloading, "doesntexist.xls"), FileNotFoundException.class); + attemptImportExpectError(FileUtil.appendName(dataloading, ""), FileNotFoundException.class); + attemptImportExpectError(FileUtil.appendName(dataloading, "notreallyexcel.xls"), InvalidFormatException.class); } private void attemptImportExpectError(File excelFile, Class exceptionClass) diff --git a/api/src/org/labkey/api/reader/ExcelLoader.java b/api/src/org/labkey/api/reader/ExcelLoader.java index c2dc153bdd0..de2535a195d 100644 --- a/api/src/org/labkey/api/reader/ExcelLoader.java +++ b/api/src/org/labkey/api/reader/ExcelLoader.java @@ -46,6 +46,7 @@ import org.labkey.api.util.FileUtil; import org.labkey.api.util.JunitUtil; import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.vfs.FileSystemLike; import org.xml.sax.Attributes; import org.xml.sax.InputSource; import org.xml.sax.XMLReader; @@ -804,11 +805,11 @@ public void detect() throws Exception { File excelSamplesRoot = JunitUtil.getSampleData(null, "dataLoading/excel"); - assertTrue(isExcel(new File(excelSamplesRoot, "ExcelLoaderTest.xls"))); - assertTrue(isExcel(new File(excelSamplesRoot, "SimpleExcelFile.xls"))); - assertTrue(isExcel(new File(excelSamplesRoot, "SimpleExcelFile.xlsx"))); - assertTrue(isExcel(new File(excelSamplesRoot, "fruits.xls"))); - assertTrue(isExcel(new File(excelSamplesRoot, "DatesWithSeconds.xlsx"))); + assertTrue(isExcel(FileUtil.appendName(excelSamplesRoot, "ExcelLoaderTest.xls"))); + assertTrue(isExcel(FileUtil.appendName(excelSamplesRoot, "SimpleExcelFile.xls"))); + assertTrue(isExcel(FileUtil.appendName(excelSamplesRoot, "SimpleExcelFile.xlsx"))); + assertTrue(isExcel(FileUtil.appendName(excelSamplesRoot, "fruits.xls"))); + assertTrue(isExcel(FileUtil.appendName(excelSamplesRoot, "DatesWithSeconds.xlsx"))); // Issue 22153: detect xls file without extension assertTrue(isExcel(JunitUtil.getSampleData(null, "Nab/seaman/MS010407"))); @@ -817,13 +818,13 @@ public void detect() throws Exception DataLoaderService svc = DataLoaderService.get(); if (svc != null) { - DataLoaderFactory factory = svc.findFactory(JunitUtil.getSampleData(null, "Nab/seaman/MS010407"), null); + DataLoaderFactory factory = svc.findFactory(FileSystemLike.wrapFile(JunitUtil.getSampleData(null, "Nab/seaman/MS010407")), null); assertTrue(factory instanceof ExcelLoader.Factory); } - assertFalse(isExcel(new File(excelSamplesRoot, "notreallyexcel.xls"))); - assertFalse(isExcel(new File(excelSamplesRoot, "notreallyexcel.xlsx"))); - assertFalse(isExcel(new File(excelSamplesRoot, "fruits.tsv"))); + assertFalse(isExcel(FileUtil.appendName(excelSamplesRoot, "notreallyexcel.xls"))); + assertFalse(isExcel(FileUtil.appendName(excelSamplesRoot, "notreallyexcel.xlsx"))); + assertFalse(isExcel(FileUtil.appendName(excelSamplesRoot, "fruits.tsv"))); } @Test @@ -958,7 +959,7 @@ private ExcelLoader getExcelLoader(String filename) throws IOException if (!excelSamplesRoot.canRead()) throw new IOException("Could not read excel samples in: " + excelSamplesRoot); - File excelFile = new File(excelSamplesRoot, filename); + File excelFile = FileUtil.appendName(excelSamplesRoot, filename); return new ExcelLoader(excelFile, true); } diff --git a/api/src/org/labkey/api/reports/ExternalScriptEngine.java b/api/src/org/labkey/api/reports/ExternalScriptEngine.java index 8ea29e50c61..0f1fec57695 100644 --- a/api/src/org/labkey/api/reports/ExternalScriptEngine.java +++ b/api/src/org/labkey/api/reports/ExternalScriptEngine.java @@ -191,7 +191,7 @@ protected File getWorkingDir(ScriptContext context) else { File tempDir = new File(System.getProperty("java.io.tmpdir")); - _workingDirectory = new File(tempDir, DEFAULT_WORKING_DIRECTORY); + _workingDirectory = FileUtil.appendName(tempDir, DEFAULT_WORKING_DIRECTORY); } if (!_workingDirectory.exists()) @@ -440,10 +440,10 @@ protected File writeScriptFile(String script, ScriptContext context, List replacem return; try { - File mapFile = new File(cacheDir, SUBSTITUTION_MAP); + File mapFile = FileUtil.appendName(cacheDir, SUBSTITUTION_MAP); for (ParamReplacement param : replacements) { List> changes = new ArrayList<>(); for (File src : param.getFiles()) { - File dst = new File(cacheDir, src.getName()); + File dst = FileUtil.appendName(cacheDir, src.getName()); if (src.exists() && FileUtil.createTempFile(dst)) { FileUtil.copyFile(src, dst); @@ -482,7 +482,7 @@ protected boolean getCachedReport(ViewContext context, List re try { - replacements.addAll(ParamReplacementSvc.get().fromFile(new File(cacheDir, SUBSTITUTION_MAP))); + replacements.addAll(ParamReplacementSvc.get().fromFile(FileUtil.appendName(cacheDir, SUBSTITUTION_MAP))); return !replacements.isEmpty(); } catch (Exception e) diff --git a/api/src/org/labkey/api/reports/report/ScriptReport.java b/api/src/org/labkey/api/reports/report/ScriptReport.java index 774fc97f2f5..42278885423 100644 --- a/api/src/org/labkey/api/reports/report/ScriptReport.java +++ b/api/src/org/labkey/api/reports/report/ScriptReport.java @@ -279,7 +279,7 @@ public static File getTempRoot(ReportDescriptor descriptor) public static File getDefaultTempRoot() { File tempDir = new File(System.getProperty("java.io.tmpdir")); - return new File(tempDir, REPORT_DIR); + return FileUtil.appendName(tempDir, REPORT_DIR); } public static FileLike getTempRootFileLike(ReportDescriptor descriptor) diff --git a/api/src/org/labkey/api/reports/report/r/RReportJob.java b/api/src/org/labkey/api/reports/report/r/RReportJob.java index 8e96dbbfd93..79c220902a5 100644 --- a/api/src/org/labkey/api/reports/report/r/RReportJob.java +++ b/api/src/org/labkey/api/reports/report/r/RReportJob.java @@ -18,8 +18,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.labkey.api.data.JdbcType; import org.labkey.api.exp.PropertyType; @@ -61,7 +59,6 @@ */ public class RReportJob extends PipelineJob implements Serializable { - private static final Logger _log = LogManager.getLogger(RReportJob.class); private static final ThreadLocal _jobIdentifier = new ThreadLocal<>(); public static final String PROCESSING_STATUS = "Processing"; @@ -300,7 +297,7 @@ protected void processOutputs(RReport report, List outputSubst List newFiles = new ArrayList<>(); for (File file : replacement.getFiles()) { - File newFile = new File(parentDir, file.getName()); + File newFile = FileUtil.appendName(parentDir, file.getName()); FileUtils.moveFile(file, newFile); newFiles.add(newFile); } @@ -315,7 +312,7 @@ protected void processOutputs(RReport report, List outputSubst // move the remaining files and delete the pipeline specific directory for (File file : reportDir.listFiles()) { - File newFile = new File(parentDir, file.getName()); + File newFile = FileUtil.appendName(parentDir, file.getName()); // special handling for log file if (LOG_FILE_NAME.equalsIgnoreCase(file.getName())) { @@ -341,10 +338,10 @@ protected void processOutputs(RReport report, List outputSubst } } FileUtils.deleteDirectory(reportDir); - substitutionMap = new File(reportDir.getParent(), RReport.SUBSTITUTION_MAP); + substitutionMap = FileUtil.appendName(reportDir.getParentFile(), RReport.SUBSTITUTION_MAP); } else - substitutionMap = new File(reportDir, RReport.SUBSTITUTION_MAP); + substitutionMap = FileUtil.appendName(reportDir, RReport.SUBSTITUTION_MAP); ParamReplacementSvc.get().toFile(outputSubst, substitutionMap); } } diff --git a/api/src/org/labkey/api/reports/report/r/view/DownloadOutputView.java b/api/src/org/labkey/api/reports/report/r/view/DownloadOutputView.java index 31a0027933c..3ee179b603b 100644 --- a/api/src/org/labkey/api/reports/report/r/view/DownloadOutputView.java +++ b/api/src/org/labkey/api/reports/report/r/view/DownloadOutputView.java @@ -62,7 +62,7 @@ protected String renderInternalAsString(File file) if (file != null && file.exists() && (file.length() > 0)) { - File newFile = moveToTemp(file, "RReportPdf"); + File newFile = moveToTemp(file); // file hasn't been saved yet String key = ImageUtil.setFileInSession(getViewContext().getRequest(), newFile); downloadUrl = PageFlowUtil.urlProvider(ReportUrls.class).urlStreamFile(getViewContext().getContainer()). diff --git a/api/src/org/labkey/api/reports/report/r/view/DownloadParamReplacement.java b/api/src/org/labkey/api/reports/report/r/view/DownloadParamReplacement.java index 7d85234bb7a..10bff2618d2 100644 --- a/api/src/org/labkey/api/reports/report/r/view/DownloadParamReplacement.java +++ b/api/src/org/labkey/api/reports/report/r/view/DownloadParamReplacement.java @@ -49,7 +49,7 @@ protected final File getSubstitution(File directory, String extension) fileName = getName().concat(extension); if (directory != null) - file = new File(directory, fileName); + file = FileUtil.appendName(directory, fileName); } if (file != null) addFile(file); diff --git a/api/src/org/labkey/api/reports/report/r/view/ImageOutput.java b/api/src/org/labkey/api/reports/report/r/view/ImageOutput.java index fee09049e6c..b89061204d9 100644 --- a/api/src/org/labkey/api/reports/report/r/view/ImageOutput.java +++ b/api/src/org/labkey/api/reports/report/r/view/ImageOutput.java @@ -128,7 +128,7 @@ protected String renderInternalAsString(File file) if (!_deleteFile) imgFile = file; else - imgFile = moveToTemp(file, "RReportImg"); + imgFile = moveToTemp(file); if (imgFile != null) { diff --git a/api/src/org/labkey/api/reports/report/r/view/ROutputView.java b/api/src/org/labkey/api/reports/report/r/view/ROutputView.java index 45598777218..a20d34d6ffb 100644 --- a/api/src/org/labkey/api/reports/report/r/view/ROutputView.java +++ b/api/src/org/labkey/api/reports/report/r/view/ROutputView.java @@ -148,11 +148,11 @@ protected void renderTitle(Object model, PrintWriter out) out.write(sb.toString()); } - protected File moveToTemp(File file, String prefix) + protected File moveToTemp(File file) { File root = ScriptEngineReport.getTempRoot(ReportService.get().createDescriptorInstance(RReportDescriptor.TYPE)); - File newFile = new File(root, FileUtil.makeFileNameWithTimestamp(FileUtil.getBaseName(file.getName()), FileUtil.getExtension(file))); + File newFile = FileUtil.appendName(root, FileUtil.makeFileNameWithTimestamp(FileUtil.getBaseName(file.getName()), FileUtil.getExtension(file))); newFile.delete(); LOG.debug("Moving '" + file.getAbsolutePath() + "' to '" + newFile.getAbsolutePath() + "'"); diff --git a/api/src/org/labkey/api/settings/AppPropsImpl.java b/api/src/org/labkey/api/settings/AppPropsImpl.java index 8a134521ca2..82833c35df5 100644 --- a/api/src/org/labkey/api/settings/AppPropsImpl.java +++ b/api/src/org/labkey/api/settings/AppPropsImpl.java @@ -33,6 +33,7 @@ import org.labkey.api.security.UserManager; import org.labkey.api.security.UserPrincipal; import org.labkey.api.util.ExceptionReportingLevel; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.GUID; import org.labkey.api.util.MothershipReport; import org.labkey.api.util.PageFlowUtil; @@ -267,7 +268,7 @@ public String getPipelineToolsDirectory() { @SuppressWarnings("ConstantConditions") File webappDir = new File(ModuleLoader.getServletContext().getRealPath("")); - File binDir = new File(webappDir.getParentFile(), "bin"); + File binDir = FileUtil.appendName(webappDir.getParentFile(), "bin"); return lookupStringValue(pipelineToolsDirectory, binDir.getAbsolutePath()); } diff --git a/api/src/org/labkey/api/study/SpecimenTransform.java b/api/src/org/labkey/api/study/SpecimenTransform.java index 5842bfcadc7..d00b7731f19 100644 --- a/api/src/org/labkey/api/study/SpecimenTransform.java +++ b/api/src/org/labkey/api/study/SpecimenTransform.java @@ -53,19 +53,7 @@ public interface SpecimenTransform */ FileType getFileType(); - /** - * Transform the input file into a specimen archive that a basic specimen import can - * process. - */ - @Deprecated - void transform(@Nullable PipelineJob job, File input, File outputArchive) throws PipelineJobException; - - default void transform(@Nullable PipelineJob job, Path input, Path outputArchive) throws PipelineJobException - { - //TODO this should be implemented in the inheriting classes - // defaulting for now to prevent build issues - transform(job, input.toFile(), outputArchive.toFile()); - } + void transform(@Nullable PipelineJob job, Path input, Path outputArchive) throws PipelineJobException; /** * An optional post transform step. diff --git a/api/src/org/labkey/api/util/DotRunner.java b/api/src/org/labkey/api/util/DotRunner.java index aa3a8296dd8..b975a34b2b7 100644 --- a/api/src/org/labkey/api/util/DotRunner.java +++ b/api/src/org/labkey/api/util/DotRunner.java @@ -88,7 +88,7 @@ public static void testDotPath(File baseDirectory) throws ExperimentException { try { - File testVersion = new File(baseDirectory, "dottest.txt"); + File testVersion = FileUtil.appendName(baseDirectory, "dottest.txt"); if (testVersion.exists()) return; diff --git a/api/src/org/labkey/api/util/FileUtil.java b/api/src/org/labkey/api/util/FileUtil.java index 88753e6b1f2..d7675c3092d 100644 --- a/api/src/org/labkey/api/util/FileUtil.java +++ b/api/src/org/labkey/api/util/FileUtil.java @@ -1559,6 +1559,12 @@ else if (ch == '-' && * Returns the absolute path to a file. On Windows and Mac, corrects casing in file paths to match the * canonical path. */ + @NotNull + public static FileLike getAbsoluteCaseSensitiveFile(@NotNull FileLike file) + { + return FileSystemLike.wrapFile(getAbsoluteCaseSensitiveFile(file.toNioPathForRead().toFile())); + } + @NotNull public static File getAbsoluteCaseSensitiveFile(@NotNull File file) { diff --git a/api/src/org/labkey/api/util/JunitUtil.java b/api/src/org/labkey/api/util/JunitUtil.java index 24575abe785..492c1156d9d 100644 --- a/api/src/org/labkey/api/util/JunitUtil.java +++ b/api/src/org/labkey/api/util/JunitUtil.java @@ -201,7 +201,8 @@ public static void createRaces(final Runnable runnable, final int threads, final { // Modules might have null sourcePath on TeamCity, so crawl for test/sampledata directories and populate // a map the first time, then stash the map for future lookups. - String name = module.getExplodedFileLike().getName(); + // Convert to a Path since this is the root for the FileLike so asking for its name returns an empty string + String name = module.getExplodedFileLike().toNioPathForRead().getFileName().toString(); synchronized (MAP_LOCK) { @@ -232,11 +233,16 @@ public static void createRaces(final Runnable runnable, final int threads, final } sampleDataDir = _sampleDataDirectories.get(name); + + if (sampleDataDir == null) + { + Assert.fail("Sample data directory not found for module: " + name + ". Known directories: " + _sampleDataDirectories); + } } } } - File file = new File(sampleDataDir, relativePath); + File file = FileUtil.appendPath(sampleDataDir, Path.parse(relativePath)); if (file.exists()) return file; diff --git a/api/src/org/labkey/api/util/MaintenancePipelineJob.java b/api/src/org/labkey/api/util/MaintenancePipelineJob.java index c0ccf7ba091..eaee550dffe 100644 --- a/api/src/org/labkey/api/util/MaintenancePipelineJob.java +++ b/api/src/org/labkey/api/util/MaintenancePipelineJob.java @@ -22,7 +22,6 @@ import org.labkey.api.util.SystemMaintenance.MaintenanceTask; import org.labkey.api.view.ViewBackgroundInfo; -import java.io.File; import java.util.Collection; /** @@ -43,7 +42,7 @@ protected MaintenancePipelineJob(@JsonProperty("_tasks") Collection tasks) { super("SystemMaintenance", info, pipeRoot); - setLogFile(new File(pipeRoot.getLogDirectory(), FileUtil.makeFileNameWithTimestamp("system_maintenance", "log")).toPath()); + setLogFile(pipeRoot.getLogDirectoryFileLike(true).resolveChild(FileUtil.makeFileNameWithTimestamp("system_maintenance", "log")).toNioPathForWrite()); _tasks = tasks; } diff --git a/api/src/org/labkey/api/writer/FileSystemFile.java b/api/src/org/labkey/api/writer/FileSystemFile.java index dce4ee72d14..298c2f0b42f 100644 --- a/api/src/org/labkey/api/writer/FileSystemFile.java +++ b/api/src/org/labkey/api/writer/FileSystemFile.java @@ -81,7 +81,7 @@ public String getLocation() @Override public PrintWriter getPrintWriter(String filename) throws IOException { - File file = new File(_root.toFile(), makeLegalName(filename)); + File file = FileUtil.appendName(_root.toFile(), makeLegalName(filename)); return PrintWriters.getPrintWriter(file); } diff --git a/api/src/org/labkey/vfs/FileSystemLike.java b/api/src/org/labkey/vfs/FileSystemLike.java index 7fc22bef58a..953c002553e 100644 --- a/api/src/org/labkey/vfs/FileSystemLike.java +++ b/api/src/org/labkey/vfs/FileSystemLike.java @@ -45,7 +45,7 @@ * *

* implementation notes: - * - This is meant to be a wrapper over java.nio.file.Path, java.io.File or org.apache.commons.vfs2.FileObject or other implementaions. + * - This is meant to be a wrapper over java.nio.file.Path, java.io.File or org.apache.commons.vfs2.FileObject or other implementations. * However, it is still lower level than Resource. For instance, it does not know about Permissions or ContentType, etc. *
* - FileLike objects always present String path and util.Path relative to the FileSystemLike root. @@ -196,6 +196,8 @@ public FileLike root() /** Helper for partially converted code. Parent dir must exist. */ static FileLike wrapFile(File f) { + if (f == null) + return null; FileLike p = new Builder(f.getParentFile()).root(); return p.resolveChild(f.getName()); } diff --git a/api/src/org/labkey/vfs/vfsTestCase.jsp b/api/src/org/labkey/vfs/vfsTestCase.jsp index 68de7c6dd93..1c60019ccc6 100644 --- a/api/src/org/labkey/vfs/vfsTestCase.jsp +++ b/api/src/org/labkey/vfs/vfsTestCase.jsp @@ -229,8 +229,8 @@ fs = fs.getCachingFileSystem(); FileLike root = fs.getRoot(); File ROOT = root.toNioPathForWrite().toFile(); - File DIR = new File(ROOT,"caching"); - FileUtil.deleteDir(new File(ROOT,"caching")); + File DIR = FileUtil.appendName(ROOT,"caching"); + FileUtil.deleteDir(DIR); FileLike dir = root.resolveChild("caching"); assertFalse(dir.exists()); @@ -243,12 +243,12 @@ assertTrue(dir.isDirectory()); assertTrue(dir.getChildren().isEmpty()); - File FILE = new File(DIR, "a.txt"); + File FILE = FileUtil.appendName(DIR, "a.txt"); FileLike file = dir.resolveChild("a.txt"); assertFalse(file.exists()); assertFalse(file.isFile()); assertTrue(dir.getChildren().isEmpty()); - FILE.createNewFile(); + FileUtil.createNewFile(FILE); assertFalse(file.exists()); assertFalse(file.isFile()); assertTrue(dir.getChildren().isEmpty()); diff --git a/assay/api-src/org/labkey/api/assay/dilution/DilutionDataHandler.java b/assay/api-src/org/labkey/api/assay/dilution/DilutionDataHandler.java index 43ecb9ee7cc..e4eabafa1ff 100644 --- a/assay/api-src/org/labkey/api/assay/dilution/DilutionDataHandler.java +++ b/assay/api-src/org/labkey/api/assay/dilution/DilutionDataHandler.java @@ -111,10 +111,10 @@ public DilutionDataHandler(String dataLsidPrefix) protected class DilutionDataFileParser { private final ExpData _data; - private final File _dataFile; + private final FileLike _dataFile; private final ViewBackgroundInfo _info; - public DilutionDataFileParser(ExpData data, File dataFile, ViewBackgroundInfo info) + public DilutionDataFileParser(ExpData data, FileLike dataFile, ViewBackgroundInfo info) { _data = data; _dataFile = dataFile; @@ -136,7 +136,7 @@ public List> getResults() throws ExperimentException * plate data versus the saved table data. If use run for plates is false (use dataFile) then * stats are always recalculated. */ - public List> calculateDilutionStats(ExpRun run, User user, @Nullable File dataFile, boolean useRunForPlates, boolean recalculateStats) throws ExperimentException + public List> calculateDilutionStats(ExpRun run, User user, @Nullable FileLike dataFile, boolean useRunForPlates, boolean recalculateStats) throws ExperimentException { try { DilutionAssayRun assayResults = getAssayResults(run, user, dataFile, null, useRunForPlates, recalculateStats); @@ -238,12 +238,12 @@ else if (Double.POSITIVE_INFINITY == icValue) results.put(name + OORDisplayColumnFactory.OOR_INDICATOR_COLUMN_SUFFIX, outOfRange); } - protected ExperimentException createParseError(File dataFile, String msg) + protected ExperimentException createParseError(FileLike dataFile, String msg) { return createParseError(dataFile, msg, null); } - protected ExperimentException createParseError(File dataFile, String msg, @Nullable Exception cause) + protected ExperimentException createParseError(FileLike dataFile, String msg, @Nullable Exception cause) { StringBuilder fullMessage = new StringBuilder("There was an error parsing "); fullMessage.append(dataFile.getName()).append(".\n"); @@ -290,14 +290,14 @@ public DilutionAssayRun getAssayResults(ExpRun run, User user, @Nullable StatsSe return getAssayResults(run, user, getDataFile(run), fit, true, false); } - public File getDataFile(ExpRun run) + public FileLike getDataFile(ExpRun run) { if (run == null) return null; List outputDatas = run.getOutputDatas(getDataType()); if (outputDatas == null || outputDatas.size() != 1) throw new IllegalStateException(getResourceName(run) + " runs should have a single data output."); - File dataFile = outputDatas.get(0).getFile(); + FileLike dataFile = outputDatas.get(0).getFileLike(); if (!dataFile.exists()) return null; return dataFile; @@ -312,7 +312,7 @@ public String getResourceName(ExpRun run) return provider != null ? provider.getResourceName() : "Assay"; } - protected List createPlates(File dataFile, Plate template) throws ExperimentException + protected List createPlates(FileLike dataFile, Plate template) throws ExperimentException { return Collections.singletonList(PlateService.get().createPlate(template, getCellValues(dataFile, template), null, PlateService.NO_RUNID, 1)); } @@ -335,9 +335,9 @@ protected List createPlates(ExpRun run, Plate template, boolean recalcSta return Collections.singletonList(plate); } - protected abstract double[][] getCellValues(final File dataFile, Plate nabTemplate) throws ExperimentException; + protected abstract double[][] getCellValues(final FileLike dataFile, Plate nabTemplate) throws ExperimentException; - protected DilutionAssayRun getAssayResults(ExpRun run, User user, @Nullable File dataFile, @Nullable StatsService.CurveFitType fit, + protected DilutionAssayRun getAssayResults(ExpRun run, User user, @Nullable FileLike dataFile, @Nullable StatsService.CurveFitType fit, boolean useRunForPlates, boolean recalcStats) throws ExperimentException { ExpProtocol protocol = ExperimentService.get().getExpProtocol(run.getProtocol().getLSID()); @@ -473,18 +473,13 @@ protected abstract DilutionAssayRun createDilutionAssayRun(DilutionAssayProvider public abstract Map getDilutionSummaries(User user, StatsService.CurveFitType fit, long... dataObjectIds) throws ExperimentException, SQLException; - protected DilutionDataFileParser getDataFileParser(ExpData data, File dataFile, ViewBackgroundInfo info) - { - return new DilutionDataFileParser(data, dataFile, info); - } - final protected DilutionDataFileParser getDataFileParser(ExpData data, FileLike dataFile, ViewBackgroundInfo info) { - return new DilutionDataFileParser(data, dataFile.toNioPathForRead().toFile(), info); + return new DilutionDataFileParser(data, dataFile, info); } @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException + public void importFile(@NotNull ExpData data, @NotNull FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException { ExpRun run = data.getRun(); ExpProtocol protocol = run.getProtocol(); @@ -807,7 +802,7 @@ private void _populateWellData(ExpProtocol protocol, ExpRun run, User user, Map< if (populatePlatesFromFile) { - File dataFile = getDataFile(run); + FileLike dataFile = getDataFile(run); if (null == dataFile) throw new MissingDataFileException("Data file not found."); plates = createPlates(dataFile, nabTemplate); diff --git a/assay/api-src/org/labkey/api/assay/matrix/AbstractMatrixDataHandler.java b/assay/api-src/org/labkey/api/assay/matrix/AbstractMatrixDataHandler.java index bfc8e084733..861e0dc834c 100644 --- a/assay/api-src/org/labkey/api/assay/matrix/AbstractMatrixDataHandler.java +++ b/assay/api-src/org/labkey/api/assay/matrix/AbstractMatrixDataHandler.java @@ -51,13 +51,12 @@ import org.labkey.api.query.ValidationException; import org.labkey.api.reader.ColumnDescriptor; import org.labkey.api.reader.DataLoader; -import org.labkey.api.reader.DataLoaderFactory; import org.labkey.api.reader.TabLoader; import org.labkey.api.security.User; import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; @@ -113,26 +112,13 @@ public DataType getDataType() } @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException + public void importFile(@NotNull ExpData data, @NotNull org.labkey.vfs.FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) throws ExperimentException { } - public static DataLoader createLoader(File file, String idColumnName) throws IOException, ExperimentException + public static TabLoader createTabLoader(FileLike file, String idColumnName, Set aliases) throws IOException, ExperimentException { - return createLoader(file, idColumnName, Collections.emptySet()); - } - - public static DataLoader createLoader(File file, String idColumnName, Set aliases) throws IOException, ExperimentException - { - DataLoaderFactory factory = DataLoader.get().findFactory(file, null); - DataLoader loader = factory.createLoader(file, true); - ensureColumns(idColumnName, aliases, loader.getColumns()); - return loader; - } - - public static TabLoader createTabLoader(File file, String idColumnName, Set aliases) throws IOException, ExperimentException - { - TabLoader loader = new TabLoader(file, true); + TabLoader loader = new TabLoader(file.openInputStream(), true, null); ensureColumns(idColumnName, aliases, loader.getColumns()); return loader; } diff --git a/assay/api-src/org/labkey/api/assay/nab/Luc5Assay.java b/assay/api-src/org/labkey/api/assay/nab/Luc5Assay.java index c98fe6e968e..580ee487433 100644 --- a/assay/api-src/org/labkey/api/assay/nab/Luc5Assay.java +++ b/assay/api-src/org/labkey/api/assay/nab/Luc5Assay.java @@ -26,8 +26,8 @@ import org.labkey.api.assay.plate.Plate; import org.labkey.api.assay.plate.Position; import org.labkey.api.assay.plate.WellData; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.Serializable; import java.util.List; import java.util.Map; @@ -42,7 +42,7 @@ public abstract class Luc5Assay implements Serializable, DilutionCurve.PercentCa private final Long _runRowId; private final int[] _cutoffs; private Map _cutoffFormats; - private File _dataFile; + private FileLike _dataFile; protected StatsService.CurveFitType _renderedCurveFitType; private boolean _lockAxes; @@ -191,12 +191,12 @@ public void setCutoffFormats(Map cutoffFormats) _cutoffFormats = cutoffFormats; } - public File getDataFile() + public FileLike getDataFile() { return _dataFile; } - public void setDataFile(File dataFile) + public void setDataFile(FileLike dataFile) { _dataFile = dataFile; } diff --git a/assay/api-src/org/labkey/api/assay/plate/AbstractPlateBasedAssayProvider.java b/assay/api-src/org/labkey/api/assay/plate/AbstractPlateBasedAssayProvider.java index 17d2bbc2a87..1e97a198f6a 100644 --- a/assay/api-src/org/labkey/api/assay/plate/AbstractPlateBasedAssayProvider.java +++ b/assay/api-src/org/labkey/api/assay/plate/AbstractPlateBasedAssayProvider.java @@ -51,8 +51,8 @@ import org.labkey.api.util.Pair; import org.labkey.api.view.HtmlView; import org.labkey.api.view.InsertView; +import org.labkey.vfs.FileLike; -import java.io.File; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -127,7 +127,7 @@ public AssayRunCreator getRunCreator() } @Override - public File getSampleMetadataFile(Container container, int runId) + public FileLike getSampleMetadataFile(Container container, int runId) { ExpRun run = ExperimentService.get().getExpRun(runId); if (!run.getContainer().equals(container)) @@ -151,7 +151,7 @@ public File getSampleMetadataFile(Container container, int runId) for (Map.Entry entry : sampleDerivationInputs.entrySet()) { if (SAMPLE_METADATA_INPUT_ROLE.equals(entry.getValue())) - return entry.getKey().getFile(); + return entry.getKey().getFileLike(); } return null; } diff --git a/assay/api-src/org/labkey/api/assay/plate/AbstractPlateReader.java b/assay/api-src/org/labkey/api/assay/plate/AbstractPlateReader.java index 180c86276a9..4758e7d7cd0 100644 --- a/assay/api-src/org/labkey/api/assay/plate/AbstractPlateReader.java +++ b/assay/api-src/org/labkey/api/assay/plate/AbstractPlateReader.java @@ -18,8 +18,8 @@ import org.apache.commons.lang3.math.NumberUtils; import org.labkey.api.exp.ExperimentException; import org.labkey.api.query.ValidationException; +import org.labkey.vfs.FileLike; -import java.io.File; import java.util.List; /** @@ -51,7 +51,7 @@ else if (dblValue == WELL_OFF_SCALE) } @Override - public List loadMultiGridFile(Plate template, File dataFile) throws ExperimentException + public List loadMultiGridFile(Plate template, FileLike dataFile) throws ExperimentException { throw new UnsupportedOperationException("loading multiple grids for this reader implementation is not supported"); } diff --git a/assay/api-src/org/labkey/api/assay/plate/ExcelPlateReader.java b/assay/api-src/org/labkey/api/assay/plate/ExcelPlateReader.java index 77d72f0eaf4..daf6fb39de9 100644 --- a/assay/api-src/org/labkey/api/assay/plate/ExcelPlateReader.java +++ b/assay/api-src/org/labkey/api/assay/plate/ExcelPlateReader.java @@ -24,9 +24,10 @@ import org.labkey.api.exp.ExperimentException; import org.labkey.api.reader.DataLoader; import org.labkey.api.reader.DataLoaderFactory; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.List; /** @@ -46,10 +47,11 @@ public String getType() } @Override - public double[][] loadFile(Plate template, File dataFile) throws ExperimentException + public double[][] loadFile(Plate template, FileLike dataFile) throws ExperimentException { DataLoaderFactory factory = DataLoader.get().findFactory(dataFile, null); - try (DataLoader loader = factory.createLoader(dataFile, false)) + try (InputStream in = dataFile.openInputStream(); + DataLoader loader = factory.createLoader(in, false)) { return PlateUtils.parseGrid(dataFile, loader.load(), template.getRows(), template.getColumns(), this); } @@ -60,10 +62,11 @@ public double[][] loadFile(Plate template, File dataFile) throws ExperimentExcep } @Override - public List loadMultiGridFile(Plate template, File dataFile) throws ExperimentException + public List loadMultiGridFile(Plate template, FileLike dataFile) throws ExperimentException { DataLoaderFactory factory = DataLoader.get().findFactory(dataFile, null); - try (DataLoader loader = factory.createLoader(dataFile, false)) + try (InputStream in = dataFile.openInputStream(); + DataLoader loader = factory.createLoader(in, false)) { return PlateUtils.parseAllGrids(dataFile, loader.load(), template.getRows(), template.getColumns(), this); } diff --git a/assay/api-src/org/labkey/api/assay/plate/PlateBasedAssayProvider.java b/assay/api-src/org/labkey/api/assay/plate/PlateBasedAssayProvider.java index 853cdb1738d..4c716f2c652 100644 --- a/assay/api-src/org/labkey/api/assay/plate/PlateBasedAssayProvider.java +++ b/assay/api-src/org/labkey/api/assay/plate/PlateBasedAssayProvider.java @@ -25,6 +25,7 @@ import org.labkey.api.assay.AssayProvider; import org.labkey.api.study.assay.ParticipantVisitResolverType; import org.labkey.api.study.assay.SampleMetadataInputFormat; +import org.labkey.vfs.FileLike; import java.io.File; import java.util.Collection; @@ -39,7 +40,7 @@ public interface PlateBasedAssayProvider extends AssayProvider void setPlate(Container container, ExpProtocol protocol, Plate plate); @Nullable Plate getPlate(Container container, ExpProtocol protocol); - File getSampleMetadataFile(Container container, int runId); + FileLike getSampleMetadataFile(Container container, int runId); @Nullable PlateReader getPlateReader(String readerName); SampleMetadataInputFormat[] getSupportedMetadataInputFormats(); SampleMetadataInputFormat getMetadataInputFormat(ExpProtocol protocol); diff --git a/assay/api-src/org/labkey/api/assay/plate/PlateBasedRunCreator.java b/assay/api-src/org/labkey/api/assay/plate/PlateBasedRunCreator.java index e4f0e634e44..9872724fce5 100644 --- a/assay/api-src/org/labkey/api/assay/plate/PlateBasedRunCreator.java +++ b/assay/api-src/org/labkey/api/assay/plate/PlateBasedRunCreator.java @@ -44,6 +44,7 @@ import org.labkey.api.util.FileType; import org.labkey.api.util.FileUtil; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; import java.io.File; import java.util.Arrays; @@ -189,7 +190,7 @@ private Map createDerivedMaterials(AssayRunUploadContext if (getProvider().getMetadataInputFormat(context.getProtocol()) == SampleMetadataInputFormat.FILE_BASED) { PlateSampleFilePropertyHelper helper = (PlateSampleFilePropertyHelper) getProvider().getSamplePropertyHelper((PlateUploadForm) context, null); - File metadataFile = helper.getMetadataFile(); + FileLike metadataFile = helper.getMetadataFile(); sampleMetadataFile = ExperimentService.get().createData(context.getContainer(), SAMPLE_METADATA_FILE_TYPE); sampleMetadataFile.setDataFileURI(FileUtil.getAbsoluteCaseSensitiveFile(metadataFile).toURI()); sampleMetadataFile.setName(metadataFile.getName()); diff --git a/assay/api-src/org/labkey/api/assay/plate/PlateReader.java b/assay/api-src/org/labkey/api/assay/plate/PlateReader.java index 45105ff5e24..fee10c40fc8 100644 --- a/assay/api-src/org/labkey/api/assay/plate/PlateReader.java +++ b/assay/api-src/org/labkey/api/assay/plate/PlateReader.java @@ -18,8 +18,8 @@ import org.labkey.api.exp.ExperimentException; import org.labkey.api.query.ValidationException; +import org.labkey.vfs.FileLike; -import java.io.File; import java.util.List; /** @@ -41,14 +41,14 @@ public interface PlateReader /** * Parse the specified datafile and populate an array of well values */ - double[][] loadFile(Plate template, File dataFile) throws ExperimentException; + double[][] loadFile(Plate template, FileLike dataFile) throws ExperimentException; /** * Parse the specified datafile and populate a map of array of well values. This is designed to process files * that have multiple grids of data embedded and the caller is interested in all of the data. The parser will * attempt to annotate the grids with metadata that it may discover during parsing. */ - List loadMultiGridFile(Plate template, File dataFile) throws ExperimentException; + List loadMultiGridFile(Plate template, FileLike dataFile) throws ExperimentException; /** * Determines whether the specified well value should be used in any analytical calculations diff --git a/assay/api-src/org/labkey/api/assay/plate/PlateSampleFilePropertyHelper.java b/assay/api-src/org/labkey/api/assay/plate/PlateSampleFilePropertyHelper.java index 9afd4f74ade..e749867e848 100644 --- a/assay/api-src/org/labkey/api/assay/plate/PlateSampleFilePropertyHelper.java +++ b/assay/api-src/org/labkey/api/assay/plate/PlateSampleFilePropertyHelper.java @@ -55,8 +55,6 @@ import org.springframework.web.multipart.MultipartHttpServletRequest; import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.HashMap; @@ -88,7 +86,7 @@ public class PlateSampleFilePropertyHelper extends PlateSamplePropertyHelper protected Map> _sampleProperties; protected String _metadataNoun = "Sample"; protected String _wellGroupColumnName = SAMPLE_WELLGROUP_COLUMN; - private File _metadataFile; + private FileLike _metadataFile; public PlateSampleFilePropertyHelper(Container container, ExpProtocol protocol, List domainProperties, Plate template, SampleMetadataInputFormat inputFormat) { @@ -137,7 +135,7 @@ public Map> getPostedPropertyValues(HttpServ return new HashMap<>(sampleProperties); } - protected File getSampleMetadata(HttpServletRequest request) throws ExperimentException + protected FileLike getSampleMetadata(HttpServletRequest request) throws ExperimentException { if (_metadataFile != null) return _metadataFile; @@ -146,7 +144,7 @@ protected File getSampleMetadata(HttpServletRequest request) throws ExperimentEx { String relativePath = request.getParameter(METADATA_PREVUPLOAD_LOCATION); PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(_container); - _metadataFile = pipelineRoot.resolvePath(relativePath); + _metadataFile = pipelineRoot.resolvePathToFileLike(relativePath); return _metadataFile; } @@ -155,7 +153,7 @@ protected File getSampleMetadata(HttpServletRequest request) throws ExperimentEx { String relativePath = request.getParameter(PRIMARY_PREV_UPLOAD_PATH); PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(_container); - _metadataFile = pipelineRoot.resolvePath(relativePath); + _metadataFile = pipelineRoot.resolvePathToFileLike(relativePath); return _metadataFile; } @@ -167,15 +165,15 @@ protected File getSampleMetadata(HttpServletRequest request) throws ExperimentEx try { FileLike uploadDirectory = AssayFileWriter.ensureUploadDirectory(_container); - _metadataFile = FileUtil.findUniqueFileName(metadata.getOriginalFilename(), uploadDirectory).toNioPathForWrite().toFile(); - try (BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(_metadataFile)); + _metadataFile = FileUtil.findUniqueFileName(metadata.getOriginalFilename(), uploadDirectory); + try (BufferedOutputStream fos = new BufferedOutputStream(_metadataFile.openOutputStream()); InputStream is = metadata.getInputStream()) { IOUtils.copy(is, fos); FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(_container, "Adding sample metadata file " + metadata.getOriginalFilename() + "."); event.setProvidedFileName(metadata.getOriginalFilename()); event.setFile(_metadataFile.getName()); - event.setDirectory(_metadataFile.getParent()); + event.setDirectory(_metadataFile.getParent().toNioPathForRead().toFile().getPath()); AuditLogService.get().addEvent(null, event); } } @@ -219,7 +217,7 @@ public Map> getSampleProperties(HttpServletR if (_sampleProperties != null) return _sampleProperties; - File metadataFile = getSampleMetadata(request); + FileLike metadataFile = getSampleMetadata(request); if (metadataFile == null) return null; @@ -282,7 +280,7 @@ protected void validateMetadataRow(Map row, String wellGroupName } } - public File getMetadataFile() + public FileLike getMetadataFile() { return _metadataFile; } @@ -312,7 +310,7 @@ protected Object getValue(Map row, DomainProperty property) @Override public void addSampleColumns(InsertView view, User user, final AssayRunUploadForm defaultValueContext, final boolean errorReshow) { - final File reshowFile; + final FileLike reshowFile; if (errorReshow) reshowFile = getMetadataFile(); else if (defaultValueContext instanceof PlateUploadForm && diff --git a/assay/api-src/org/labkey/api/assay/plate/PlateUtils.java b/assay/api-src/org/labkey/api/assay/plate/PlateUtils.java index 31f306194db..40c0c8ab60a 100644 --- a/assay/api-src/org/labkey/api/assay/plate/PlateUtils.java +++ b/assay/api-src/org/labkey/api/assay/plate/PlateUtils.java @@ -23,6 +23,7 @@ import org.labkey.api.collections.RowMap; import org.labkey.api.collections.RowMapFactory; import org.labkey.api.query.ValidationException; +import org.labkey.vfs.FileLike; import java.io.File; import java.util.ArrayList; @@ -71,7 +72,7 @@ public double[][] getData() * Search for a grid of numbers that has the expected number of rows and columns. * TODO: Find multiple plates in "RC121306.xls" while ignoring duplicate plate found in "20131218_0004.txt" */ - public static List parseAllGrids(File dataFile, List> rows, int expectedRows, int expectedCols, PlateReader reader) + public static List parseAllGrids(FileLike dataFile, List> rows, int expectedRows, int expectedCols, PlateReader reader) { return _parseGrids(dataFile, rows, expectedRows, expectedCols, true, reader); } @@ -80,8 +81,8 @@ public static List parseAllGrids(File dataFile, List> rows, int expectedRows, int expectedCols, @Nullable PlateReader reader) + + public static double @Nullable[][] parseGrid(FileLike dataFile, List> rows, int expectedRows, int expectedCols, @Nullable PlateReader reader) { List grids = _parseGrids(dataFile, rows, expectedRows, expectedCols, false, reader); if (!grids.isEmpty()) @@ -97,7 +98,7 @@ public static double[][] parseGrid(File dataFile, List> rows * TODO: Find multiple plates in "RC121306.xls" while ignoring duplicate plate found in "20131218_0004.txt" */ private static List _parseGrids( - File dataFile, + FileLike dataFile, List> rows, int expectedRows, int expectedCols, diff --git a/assay/api-src/org/labkey/api/assay/plate/TextPlateReader.java b/assay/api-src/org/labkey/api/assay/plate/TextPlateReader.java index 1e763de3a8e..9257c5b348e 100644 --- a/assay/api-src/org/labkey/api/assay/plate/TextPlateReader.java +++ b/assay/api-src/org/labkey/api/assay/plate/TextPlateReader.java @@ -20,8 +20,8 @@ import org.labkey.api.exp.ExperimentException; import org.labkey.api.query.ValidationException; import org.labkey.api.reader.Readers; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.io.LineNumberReader; import java.util.ArrayList; @@ -44,13 +44,13 @@ public String getType() } @Override - public double[][] loadFile(Plate template, File dataFile) throws ExperimentException + public double[][] loadFile(Plate template, FileLike dataFile) throws ExperimentException { String fileName = dataFile.getName().toLowerCase(); if (!fileName.endsWith(".txt")) throw new ExperimentException("Unable to load data file: Invalid Format"); - try (LineNumberReader reader = new LineNumberReader(Readers.getReader(dataFile))) + try (LineNumberReader reader = new LineNumberReader(Readers.getReader(dataFile.openInputStream()))) { List data = new ArrayList<>(); String line; diff --git a/assay/src/org/labkey/assay/AssayIntegrationTestCase.jsp b/assay/src/org/labkey/assay/AssayIntegrationTestCase.jsp index 463eb17340d..674ad8d39fd 100644 --- a/assay/src/org/labkey/assay/AssayIntegrationTestCase.jsp +++ b/assay/src/org/labkey/assay/AssayIntegrationTestCase.jsp @@ -95,6 +95,7 @@ <%@ page import="org.jetbrains.annotations.Nullable" %> <%@ page import="java.io.IOException" %> <%@ page import="org.apache.commons.collections.MapUtils" %> +<%@ page import="org.labkey.vfs.FileSystemLike" %> <%@ page extends="org.labkey.api.jsp.JspTest.BVT" %> <%! @@ -259,7 +260,7 @@ HttpSession session = TestContext.get().getRequest().getSession(); // Use the AssayFileUploadForm and the PipelineDataCollector to simulate the user selecting a file from the file browser. - PipelineDataCollector.setFileCollection(session, c, assayProtocol, List.of(Map.of(AssayDataCollector.PRIMARY_FILE, file))); + PipelineDataCollector.setFileCollection(session, c, assayProtocol, List.of(Map.of(AssayDataCollector.PRIMARY_FILE, FileSystemLike.wrapFile(file)))); // Use a multipart request to trigger the AssayFileWriter.savePipelineFiles() code path that copies files into the assay upload directory var mockRequest = new MockMultipartHttpServletRequest(); diff --git a/assay/src/org/labkey/assay/FileBasedModuleDataHandler.java b/assay/src/org/labkey/assay/FileBasedModuleDataHandler.java index 203e170897e..ac5e462e9df 100644 --- a/assay/src/org/labkey/assay/FileBasedModuleDataHandler.java +++ b/assay/src/org/labkey/assay/FileBasedModuleDataHandler.java @@ -30,8 +30,7 @@ import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewBackgroundInfo; - -import java.io.File; +import org.labkey.vfs.FileLike; /** * User: jeckels @@ -68,7 +67,7 @@ public ActionURL getContentURL(ExpData data) } @Override - public void importFile(@NotNull ExpData data, File dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) + public void importFile(@NotNull ExpData data, @NotNull FileLike dataFile, @NotNull ViewBackgroundInfo info, @NotNull Logger log, @NotNull XarContext context) { } } diff --git a/assay/src/org/labkey/assay/TextAreaDataCollector.java b/assay/src/org/labkey/assay/TextAreaDataCollector.java index 78e4f0ce6d0..2e3eff2528d 100644 --- a/assay/src/org/labkey/assay/TextAreaDataCollector.java +++ b/assay/src/org/labkey/assay/TextAreaDataCollector.java @@ -27,7 +27,6 @@ import org.labkey.vfs.FileLike; import java.io.ByteArrayInputStream; -import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.Map; @@ -41,7 +40,7 @@ public class TextAreaDataCollector\n" + "