diff --git a/api/src/org/labkey/api/admin/FolderImportContext.java b/api/src/org/labkey/api/admin/FolderImportContext.java index 006b704a66a..786f5a31650 100644 --- a/api/src/org/labkey/api/admin/FolderImportContext.java +++ b/api/src/org/labkey/api/admin/FolderImportContext.java @@ -29,11 +29,10 @@ import org.labkey.api.util.XmlValidationException; import org.labkey.api.writer.VirtualFile; import org.labkey.folder.xml.FolderDocument; +import org.labkey.vfs.FileLike; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -47,7 +46,7 @@ */ public class FolderImportContext extends AbstractFolderContext { - private Path _folderXml; + private FileLike _folderXml; private String _xarJobId; @@ -66,7 +65,7 @@ public FolderImportContext() super(null, null, null, null, null, null); } - public FolderImportContext(User user, Container c, Path folderXml, Set dataTypes, LoggerGetter logger, VirtualFile root) + public FolderImportContext(User user, Container c, FileLike folderXml, Set dataTypes, LoggerGetter logger, VirtualFile root) { super(user, c, null, dataTypes, logger, root); _folderXml = folderXml; @@ -112,17 +111,17 @@ public synchronized FolderDocument getDocument() throws ImportException return folderDoc; } - private FolderDocument readFolderDocument(Path folderXml) throws ImportException, IOException + private FolderDocument readFolderDocument(FileLike folderXml) throws ImportException, IOException { - if (!Files.exists(folderXml)) - throw new ImportException(folderXml.getFileName() + " file does not exist."); + if (!folderXml.exists()) + throw new ImportException(folderXml.getName() + " file does not exist."); FolderDocument folderDoc; - try (InputStream inputStream = Files.newInputStream(folderXml)) + try (InputStream inputStream = folderXml.openInputStream()) { folderDoc = FolderDocument.Factory.parse(inputStream, XmlBeansUtil.getDefaultParseOptions()); - XmlBeansUtil.validateXmlDocument(folderDoc, folderXml.getFileName().toString()); + XmlBeansUtil.validateXmlDocument(folderDoc, folderXml.getName()); } catch (XmlException | XmlValidationException e) { diff --git a/api/src/org/labkey/api/admin/ImportOptions.java b/api/src/org/labkey/api/admin/ImportOptions.java index 2f917d84f5d..61e7b23c2ae 100644 --- a/api/src/org/labkey/api/admin/ImportOptions.java +++ b/api/src/org/labkey/api/admin/ImportOptions.java @@ -19,6 +19,7 @@ import org.labkey.api.data.Activity; import org.labkey.api.security.User; import org.labkey.api.security.UserManager; +import org.labkey.vfs.FileLike; import java.nio.file.Path; import java.util.Collection; @@ -41,7 +42,7 @@ public class ImportOptions private final Collection _messages = new LinkedList<>(); private Set _dataTypes; private Activity _activity; - private Path _analysisDir; + private FileLike _analysisDir; private String _folderArchiveSourceName = null; private boolean _isNewFolderImport; // if we know the target folder is empty, can skip certain merge logic @@ -142,12 +143,12 @@ public void setActivity(Activity activity) _activity = activity; } - public Path getAnalysisDir() + public FileLike getAnalysisDir() { return _analysisDir; } - public void setAnalysisDir(Path analysisDir) + public void setAnalysisDir(FileLike analysisDir) { _analysisDir = analysisDir; } diff --git a/api/src/org/labkey/api/admin/InvalidFileException.java b/api/src/org/labkey/api/admin/InvalidFileException.java index c156765b723..e5182175593 100644 --- a/api/src/org/labkey/api/admin/InvalidFileException.java +++ b/api/src/org/labkey/api/admin/InvalidFileException.java @@ -20,18 +20,19 @@ import org.apache.xmlbeans.XmlException; import org.labkey.api.util.XmlValidationException; import org.labkey.api.writer.VirtualFile; +import org.labkey.vfs.FileLike; import java.io.File; import java.nio.file.Path; public class InvalidFileException extends ImportException { - @Deprecated // prefer the Path version - public InvalidFileException(File root, File file, Throwable t) + public InvalidFileException(FileLike root, FileLike file, Throwable t) { - this(root.toPath(), file.toPath(), t); + this(root.toNioPathForRead(), file.toNioPathForRead(), t); } + @Deprecated // prefer the FileLike version public InvalidFileException(Path root, Path file, Throwable t) { super(getErrorString(root, file, t.getMessage())); @@ -42,21 +43,11 @@ public InvalidFileException(VirtualFile root, File file, Throwable t) super(getErrorString(root.getRelativePath(file.getName()), t.getMessage())); } - public InvalidFileException(File root, File file, XmlException e) - { - super(getErrorString(root, file, e)); - } - public InvalidFileException(VirtualFile root, File file, XmlException e) { super(getErrorString(root, file, e)); } - public InvalidFileException(File root, File file, XmlValidationException e) - { - super(getErrorString(root, file, (String)null), e); - } - public InvalidFileException(VirtualFile root, File file, XmlValidationException e) { super(getErrorString(root.getRelativePath(file.getName()), null), e); diff --git a/api/src/org/labkey/api/assay/AbstractAssayProvider.java b/api/src/org/labkey/api/assay/AbstractAssayProvider.java index 666153638d5..5dc3c2cf5a1 100644 --- a/api/src/org/labkey/api/assay/AbstractAssayProvider.java +++ b/api/src/org/labkey/api/assay/AbstractAssayProvider.java @@ -116,7 +116,6 @@ import org.labkey.api.view.NotFoundException; import org.labkey.api.view.ViewContext; import org.labkey.vfs.FileLike; -import org.labkey.vfs.FileSystemLike; import org.springframework.validation.BindException; import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.mvc.Controller; @@ -285,12 +284,7 @@ public ActionURL linkToStudy(User user, Container assayDataContainer, ExpProtoco dataMap.put(StudyPublishService.TARGET_STUDY_PROPERTY_NAME, targetStudyContainer); // Remember which rows we're planning to link, partitioned by the target study - Set rowIds = rowIdsByTargetContainer.get(targetStudyContainer); - if (rowIds == null) - { - rowIds = new HashSet<>(); - rowIdsByTargetContainer.put(targetStudyContainer, rowIds); - } + Set rowIds = rowIdsByTargetContainer.computeIfAbsent(targetStudyContainer, k -> new HashSet<>()); rowIds.add(publishKey.getDataId()); dataMaps.add(dataMap); @@ -732,7 +726,7 @@ private void addReusableData(Map reusableFiles, } @Override - public AssayRunCreator getRunCreator() + public AssayRunCreator getRunCreator() { return new DefaultAssayRunCreator<>(this); } @@ -1459,13 +1453,13 @@ public AssaySaveHandler getSaveHandler() } @Override - public AssayRunUploadContext.Factory createRunUploadFactory(ExpProtocol protocol, ViewContext context) + public AssayRunUploadContext.Factory> createRunUploadFactory(ExpProtocol protocol, ViewContext context) { return new AssayRunUploadContextImpl.Factory<>(protocol, this, context); } @Override - public AssayRunUploadContext.Factory createRunUploadFactory(ExpProtocol protocol, User user, Container c) + public AssayRunUploadContext.Factory> createRunUploadFactory(ExpProtocol protocol, User user, Container c) { return new AssayRunUploadContextImpl.Factory<>(protocol, this, user, c); } @@ -1518,15 +1512,15 @@ public DataExchangeHandler createDataExchangeHandler() } @Override - public AssayRunDatabaseContext createRunDatabaseContext(ExpRun run, User user, HttpServletRequest request) + public AssayRunDatabaseContext createRunDatabaseContext(ExpRun run, User user, HttpServletRequest request) { - return new AssayRunDatabaseContext(run, user, request); + return new AssayRunDatabaseContext<>(run, user, request); } @Override public AssayRunAsyncContext createRunAsyncContext(AssayRunUploadContext context) throws IOException, ExperimentException { - return new AssayRunAsyncContext(context); + return new AssayRunAsyncContext<>(context); } @Override @@ -1698,16 +1692,6 @@ public Pair getAssayResultRowIdFromLsid(Container containe return Pair.of(protocol, rowId); } - @Override - public @Nullable ActionURL getResultRowURL(Container container, Lsid lsid) - { - var pair = getAssayResultRowIdFromLsid(container, lsid); - if (pair == null) - return null; - - return PageFlowUtil.urlProvider(AssayUrls.class).getAssayResultRowURL(this, container, pair.first, pair.second); - } - @Override public boolean supportsFlagColumnType(ExpProtocol.AssayDomainTypes type) { @@ -2141,10 +2125,7 @@ private void updateDataFileUrl(List runs, Container sourceContainer, Con AuditLogService.get().addEvent(user, event); } } - catch (Exception e) - { - - } + catch (Exception ignored) {} } } @@ -2152,7 +2133,7 @@ protected void moveAssayResults(List runs, ExpProtocol protocol, Contain { String tableName = AssayProtocolSchema.DATA_TABLE_NAME; AssaySchema schema = createProtocolSchema(user, targetContainer, protocol, null); - FilteredTable assayResultTable = (FilteredTable) schema.getTable(tableName); + FilteredTable assayResultTable = (FilteredTable) schema.getTable(tableName); if (assayResultTable == null) return; @@ -2161,7 +2142,7 @@ protected void moveAssayResults(List runs, ExpProtocol protocol, Contain record AssayFileMoveReference(String sourceFilePath, File updatedFile, String runName, String fieldName) {} - private void updateResultFiles(FilteredTable assayResultTable, List runs, ExpProtocol assayProtocol, Container sourceContainer, Container targetContainer, User user, AssayMoveData assayMoveData) throws ExperimentException + private void updateResultFiles(FilteredTable assayResultTable, List runs, ExpProtocol assayProtocol, Container sourceContainer, Container targetContainer, User user, AssayMoveData assayMoveData) throws ExperimentException { FileContentService fileContentService = FileContentService.get(); if (fileContentService == null) diff --git a/api/src/org/labkey/api/assay/AssayProvider.java b/api/src/org/labkey/api/assay/AssayProvider.java index d94874809db..0bb5ce2cc5c 100644 --- a/api/src/org/labkey/api/assay/AssayProvider.java +++ b/api/src/org/labkey/api/assay/AssayProvider.java @@ -297,7 +297,7 @@ enum Scope */ DataExchangeHandler createDataExchangeHandler(); /** Make a context that knows how to update a run that's already been stored in the database */ - AssayRunDatabaseContext createRunDatabaseContext(ExpRun run, User user, HttpServletRequest request); + AssayRunDatabaseContext createRunDatabaseContext(ExpRun run, User user, HttpServletRequest request); /** * Make a context that knows how to do the import in the background, on a separate thread * (and therefore detached from the HTTP request that might have spawned it) @@ -322,11 +322,6 @@ enum Scope @Nullable Pair getAssayResultRowIdFromLsid(Container container, Lsid assayResultRowLsid); - /** - * Get the URL for an assay result row's LSID. - */ - @Nullable ActionURL getResultRowURL(Container container, Lsid lsid); - /** * Return a SQL pattern that can be used to match a protocol's LSID to this AssayProvider. * The pattern must match a protocol's LSID in the same manner as diff --git a/api/src/org/labkey/api/assay/AssayService.java b/api/src/org/labkey/api/assay/AssayService.java index 56db398f0b2..c41726c1103 100644 --- a/api/src/org/labkey/api/assay/AssayService.java +++ b/api/src/org/labkey/api/assay/AssayService.java @@ -249,6 +249,8 @@ ParticipantVisitResolver createResolver(User user, ExpRun run, @Nullable ExpProt /** Returns the lineage "role" for an assay run/result property. */ @NotNull String getPropertyInputLineageRole(@NotNull DomainProperty dp); + AssayDomainService createAssayDomainService(User user, Container container); + interface ResultsCheckHelper { @NotNull Logger getLogger(); diff --git a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java index 4d22109f358..f7fbae5aaa8 100644 --- a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java +++ b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java @@ -221,9 +221,9 @@ private ExpExperiment saveExperimentRunAsync(AssayRunUploadContext // Choose another file as the primary primaryFile = context.getUploadedData().entrySet().iterator().next().getValue(); } - primaryFile = Objects.requireNonNull(primaryFile); + Objects.requireNonNull(primaryFile); AssayRunAsyncContext asyncContext = context.getProvider().createRunAsyncContext(context); - final AssayUploadPipelineJob pipelineJob = new AssayUploadPipelineJob( + final AssayUploadPipelineJob pipelineJob = new AssayUploadPipelineJob<>( asyncContext, info, batch, diff --git a/api/src/org/labkey/api/assay/transform/AnalysisScript.java b/api/src/org/labkey/api/assay/transform/AnalysisScript.java index e43020d4cd1..350a2ec936a 100644 --- a/api/src/org/labkey/api/assay/transform/AnalysisScript.java +++ b/api/src/org/labkey/api/assay/transform/AnalysisScript.java @@ -40,13 +40,13 @@ private AnalysisScript(File script, List operations) private AnalysisScript(File script) { - try + if (!script.exists()) { - _script = FileSystemLike.wrapFile(script.getParentFile(), script); + _script = new FileSystemLike.Builder(script).build().getRoot(); } - catch (IOException e) + else { - throw UnexpectedException.wrap(e); + _script = FileSystemLike.wrapFile(script); } } diff --git a/api/src/org/labkey/api/cloud/CloudArchiveImporterSupport.java b/api/src/org/labkey/api/cloud/CloudArchiveImporterSupport.java index a55bfee6af7..856eb9837f1 100644 --- a/api/src/org/labkey/api/cloud/CloudArchiveImporterSupport.java +++ b/api/src/org/labkey/api/cloud/CloudArchiveImporterSupport.java @@ -26,7 +26,7 @@ public interface CloudArchiveImporterSupport default void downloadCloudArchive(@NotNull PipelineJob job, @NotNull Path studyXml, BindException errors) throws UnsupportedOperationException { //check if cloud based pipeline root, and study xml hasn't been downloaded already - if (!studyXml.startsWith(job.getPipeRoot().getImportDirectory().toPath().toAbsolutePath())) + if (!studyXml.startsWith(job.getPipeRoot().getImportDirectory().toNioPathForRead().toAbsolutePath())) { if (CloudStoreService.get() != null) //proxy of is Cloud Module enabled for the current job/container { diff --git a/api/src/org/labkey/api/cloud/CloudStoreService.java b/api/src/org/labkey/api/cloud/CloudStoreService.java index 6d517d6fabb..7803bee33f3 100644 --- a/api/src/org/labkey/api/cloud/CloudStoreService.java +++ b/api/src/org/labkey/api/cloud/CloudStoreService.java @@ -23,6 +23,8 @@ import org.labkey.api.services.ServiceRegistry; import org.labkey.api.util.Pair; import org.labkey.api.webdav.WebdavResource; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.nio.file.Path; import java.util.Collection; @@ -141,6 +143,17 @@ default Collection getEnabledCloudStores(Container container, boolean ex @Nullable Path getPath(Container container, String storeName, org.labkey.api.util.Path path); + /** + * Return nio.Path to cloud file/directory + */ + @Nullable + FileLike getFileLike(Container container, String storeName, org.labkey.api.util.Path path); + + /** + * Return FileSystem for cloud root + */ + public @Nullable FileSystemLike getFileSystemLike(Container container, String configName); + /** * Return path relative to cloud store */ diff --git a/api/src/org/labkey/api/data/TSVGridWriter.java b/api/src/org/labkey/api/data/TSVGridWriter.java index a678c886d39..70f54014457 100644 --- a/api/src/org/labkey/api/data/TSVGridWriter.java +++ b/api/src/org/labkey/api/data/TSVGridWriter.java @@ -22,10 +22,9 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.collections.ResultSetRowMapFactory; import org.labkey.api.query.FieldKey; -import org.labkey.api.util.FileUtil; import org.labkey.api.view.HttpView; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; @@ -198,7 +197,7 @@ private int writeBody(Results results) * @return List of the output Files. */ @NotNull - public List writeBatchFiles(@NotNull File outputDir, @NotNull String baseName, @Nullable String extension, int batchSize, @Nullable FieldKey batchColumn) + public List writeBatchFiles(@NotNull FileLike outputDir, @NotNull String baseName, @Nullable String extension, int batchSize, @Nullable FieldKey batchColumn) { extension = StringUtils.trimToEmpty(extension); String ext = "".equals(extension) || extension.startsWith(".") ? extension : "." + extension; @@ -211,13 +210,13 @@ public List writeBatchFiles(@NotNull File outputDir, @NotNull String baseN } @NotNull - private List writeResultSetBatches(Results results, File outputDir, String baseName, String extension, int batchSize, @Nullable FieldKey batchColumn) throws IOException + private List writeResultSetBatches(Results results, FileLike outputDir, String baseName, String extension, int batchSize, @Nullable FieldKey batchColumn) throws IOException { int currentBatchSize = 0; int totalBatches = 1; Object previousBatchColumnValue = null; Object newBatchColumnValue; - List outputFiles = new ArrayList<>(); + List outputFiles = new ArrayList<>(); outputFiles.add(startBatchFile(outputDir, baseName, extension, batchSize, totalBatches)); RenderContext ctx = getRenderContext(); ctx.setResults(results); @@ -264,11 +263,11 @@ private List writeResultSetBatches(Results results, File outputDir, String } @NotNull - private File startBatchFile(File outputDir, String baseName, String extension, int batchSize, int totalBatches) throws IOException + private FileLike startBatchFile(FileLike outputDir, String baseName, String extension, int batchSize, int totalBatches) throws IOException { String batchId = batchSize == 0 ? "" : "-" + totalBatches; - File file = FileUtil.appendName(outputDir, baseName + batchId + extension); - prepare(file); + FileLike file = outputDir.resolveChild(baseName + batchId + extension); + prepare(file.openOutputStream()); writeFileHeader(); if (isHeaderRowVisible()) writeColumnHeaders(); diff --git a/api/src/org/labkey/api/exp/AbstractFileXarSource.java b/api/src/org/labkey/api/exp/AbstractFileXarSource.java index 1b57d449e29..b52b70d40c3 100644 --- a/api/src/org/labkey/api/exp/AbstractFileXarSource.java +++ b/api/src/org/labkey/api/exp/AbstractFileXarSource.java @@ -27,6 +27,7 @@ import org.labkey.api.util.FileUtil; import org.labkey.api.util.NetworkDrive; import org.labkey.api.util.XmlBeansUtil; +import org.labkey.vfs.FileLike; import java.io.IOException; import java.io.InputStream; @@ -42,9 +43,9 @@ */ public abstract class AbstractFileXarSource extends XarSource { - protected Path _xmlFile; + protected FileLike _xmlFile; - protected Path getXmlFile() + protected FileLike getXmlFile() { return _xmlFile; } @@ -77,7 +78,7 @@ public ExperimentArchiveDocument getDocument() throws XmlException, IOException try { NetworkDrive.exists(getXmlFile()); - fIn = Files.newInputStream(getXmlFile()); + fIn = getXmlFile().openInputStream(); return ExperimentArchiveDocument.Factory.parse(fIn, XmlBeansUtil.getDefaultParseOptions()); } finally @@ -88,9 +89,7 @@ public ExperimentArchiveDocument getDocument() throws XmlException, IOException { fIn.close(); } - catch (IOException e) - { - } + catch (IOException ignored) {} } } } @@ -99,7 +98,7 @@ public ExperimentArchiveDocument getDocument() throws XmlException, IOException @Nullable public Path getRootPath() { - return null != getXmlFile()? getXmlFile().getParent(): null; + return null != getXmlFile()? getXmlFile().toNioPathForRead().getParent(): null; } @Override @@ -137,15 +136,15 @@ public String canonicalizeDataFileURL(String dataFileURL) } } - public static Path getLogFileFor(Path f) throws IOException + public static FileLike getLogFileFor(FileLike f) throws IOException { - Path xarDirectory = f.getParent(); - if (!Files.exists(xarDirectory)) + FileLike xarDirectory = f.getParent(); + if (!xarDirectory.exists()) { throw new IOException("Xar file parent directory does not exist"); } - String xarShortName = f.getFileName().toString(); + String xarShortName = f.getName(); int index = xarShortName.toLowerCase().lastIndexOf(".xml"); if (index == -1) { @@ -157,6 +156,6 @@ public static Path getLogFileFor(Path f) throws IOException xarShortName = xarShortName.substring(0, index); } - return xarDirectory.resolve(xarShortName + LOG_FILE_NAME_SUFFIX); + return xarDirectory.resolveChild(xarShortName + LOG_FILE_NAME_SUFFIX); } } diff --git a/api/src/org/labkey/api/exp/FileXarSource.java b/api/src/org/labkey/api/exp/FileXarSource.java index d518701264e..ccdcc593f1c 100644 --- a/api/src/org/labkey/api/exp/FileXarSource.java +++ b/api/src/org/labkey/api/exp/FileXarSource.java @@ -19,6 +19,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Container; import org.labkey.api.pipeline.PipelineJob; +import org.labkey.vfs.FileLike; import java.io.IOException; import java.nio.file.Path; @@ -30,25 +31,25 @@ */ public class FileXarSource extends AbstractFileXarSource { - public FileXarSource(Path file, PipelineJob job) + public FileXarSource(FileLike file, PipelineJob job) { super(job); - _xmlFile = file.normalize(); + _xmlFile = file; } - public FileXarSource(Path file, PipelineJob job, Container targetContainer, @Nullable Map substitutions) + public FileXarSource(FileLike file, PipelineJob job, Container targetContainer, @Nullable Map substitutions) { super(job.getDescription(), targetContainer, job.getUser(), job, substitutions); _xmlFile = file; } - public FileXarSource(Path file, PipelineJob job, Container targetContainer) + public FileXarSource(FileLike file, PipelineJob job, Container targetContainer) { this(file, job, targetContainer, null); } @Override - public Path getLogFilePath() throws IOException + public FileLike getLogFilePath() throws IOException { return getLogFileFor(_xmlFile); } diff --git a/api/src/org/labkey/api/exp/XarContext.java b/api/src/org/labkey/api/exp/XarContext.java index cc5fbfbede4..1c368d4935f 100644 --- a/api/src/org/labkey/api/exp/XarContext.java +++ b/api/src/org/labkey/api/exp/XarContext.java @@ -29,6 +29,8 @@ import org.labkey.api.settings.AppProps; import org.labkey.api.util.GUID; import org.labkey.api.util.NetworkDrive; +import org.labkey.api.util.Path; +import org.labkey.vfs.FileLike; import java.io.File; import java.net.URI; diff --git a/api/src/org/labkey/api/exp/XarSource.java b/api/src/org/labkey/api/exp/XarSource.java index 084b367f7cd..b3e4475bfe4 100644 --- a/api/src/org/labkey/api/exp/XarSource.java +++ b/api/src/org/labkey/api/exp/XarSource.java @@ -32,6 +32,7 @@ import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.security.User; import org.labkey.api.util.FileUtil; +import org.labkey.vfs.FileLike; import java.io.IOException; import java.io.Serializable; @@ -122,7 +123,7 @@ public final String getCanonicalDataFileURL(String dataFileURL) throws XarFormat protected abstract String canonicalizeDataFileURL(String dataFileURL) throws XarFormatException; - public abstract Path getLogFilePath() throws IOException; + public abstract FileLike getLogFilePath() throws IOException; /** * Called before trying to import this XAR to let the source set up any resources that are required diff --git a/api/src/org/labkey/api/exp/api/ExperimentService.java b/api/src/org/labkey/api/exp/api/ExperimentService.java index bdf36dc10f8..f1ca905d090 100644 --- a/api/src/org/labkey/api/exp/api/ExperimentService.java +++ b/api/src/org/labkey/api/exp/api/ExperimentService.java @@ -987,7 +987,7 @@ List getExpProtocolsWithParameterValue( * * @return the job responsible for doing the work */ - PipelineJob importXarAsync(ViewBackgroundInfo info, File file, String description, PipeRoot root) throws IOException; + PipelineJob importXarAsync(ViewBackgroundInfo info, FileLike file, String description, PipeRoot root) throws IOException; /** * Loads the xar synchronously, in the context of the pipelineJob diff --git a/api/src/org/labkey/api/files/FileContentService.java b/api/src/org/labkey/api/files/FileContentService.java index 9791ba68902..ee10724bdd4 100644 --- a/api/src/org/labkey/api/files/FileContentService.java +++ b/api/src/org/labkey/api/files/FileContentService.java @@ -30,6 +30,7 @@ import org.labkey.api.services.ServiceRegistry; import org.labkey.api.util.FileUtil; import org.labkey.api.webdav.WebdavResource; +import org.labkey.vfs.FileLike; import java.io.File; import java.net.URI; @@ -340,6 +341,9 @@ enum PathType { full, serverRelative, folderRelative } @Nullable URI getWebDavUrl(@NotNull Path path, @NotNull Container container, @NotNull PathType type); + @Nullable + URI getWebDavUrl(@NotNull FileLike path, @NotNull Container container, @NotNull PathType type); + /** * Ensure an entry in the exp.data table exists for all files in the container's file root. */ diff --git a/api/src/org/labkey/api/pipeline/AnalyzeForm.java b/api/src/org/labkey/api/pipeline/AnalyzeForm.java index 9a69e371a9c..0fe86132e9a 100644 --- a/api/src/org/labkey/api/pipeline/AnalyzeForm.java +++ b/api/src/org/labkey/api/pipeline/AnalyzeForm.java @@ -21,6 +21,7 @@ import org.labkey.api.security.User; import org.labkey.api.util.FileType; import org.labkey.api.util.FileUtil; +import org.labkey.vfs.FileLike; import java.nio.file.Path; @@ -64,7 +65,7 @@ public AnalyzeForm(Container container, User user, String taskId, String protoco setProtocolName(protocolName); } - public void initStatus(AbstractFileAnalysisProtocol protocol, Path dirData, Path dirAnalysis) + public void initStatus(AbstractFileAnalysisProtocol protocol, FileLike dirData, FileLike dirAnalysis) { if (fileInputStatus != null) return; @@ -80,7 +81,7 @@ public void initStatus(AbstractFileAnalysisProtocol protocol, Path dirData, Path fileInputStatus[len] = initStatusFile(protocol, dirData, dirAnalysis, null, false); } - private String initStatusFile(AbstractFileAnalysisProtocol protocol, Path dirData, Path dirAnalysis, + private String initStatusFile(AbstractFileAnalysisProtocol protocol, FileLike dirData, FileLike dirAnalysis, String fileInputName, boolean statusSingle) { if (protocol == null) @@ -88,7 +89,7 @@ private String initStatusFile(AbstractFileAnalysisProtocol protocol, Path dirDat return UNKNOWN_STATUS; } - Path fileStatus = null; + FileLike fileStatus = null; if (!statusSingle) { @@ -97,7 +98,7 @@ private String initStatusFile(AbstractFileAnalysisProtocol protocol, Path dirDat } else if (fileInputName != null) { - Path fileInput = FileUtil.appendName(dirData, fileInputName); + FileLike fileInput = dirData.resolveChild(fileInputName); FileType ft = protocol.findInputType(fileInput); if (ft != null) fileStatus = PipelineJob.FT_LOG.newFile(dirAnalysis, ft.getBaseName(fileInput)); diff --git a/api/src/org/labkey/api/pipeline/LocalDirectory.java b/api/src/org/labkey/api/pipeline/LocalDirectory.java index f537ce2e94f..68c67ed130f 100644 --- a/api/src/org/labkey/api/pipeline/LocalDirectory.java +++ b/api/src/org/labkey/api/pipeline/LocalDirectory.java @@ -48,24 +48,23 @@ public class LocalDirectory implements Serializable private final Path _remoteDir; private Path _logFile; private final String _baseLogFileName; - private final String _moduleName; - public static LocalDirectory create(@NotNull PipeRoot root, @NotNull String moduleName) + public static LocalDirectory create(@NotNull PipeRoot root) { - return create(root, moduleName, "dummyLogFile", root.isCloudRoot() ? "dummy" : root.getRootPath().getPath()); + return create(root, "dummyLogFile", root.isCloudRoot() ? "dummy" : root.getRootPath().getPath()); } @Deprecated //Prefer to use a Path for workingDir -- can be local or remote, but should match with root - public static LocalDirectory create(@NotNull PipeRoot root, @NotNull String moduleName, @NotNull String baseLogFileName, @NotNull String workingDir) + public static LocalDirectory create(@NotNull PipeRoot root, @NotNull String baseLogFileName, @NotNull String workingDir) { - return create(root, moduleName, baseLogFileName, Path.of(workingDir)); + return create(root, baseLogFileName, Path.of(workingDir)); } - public static LocalDirectory create(@NotNull PipeRoot root, @NotNull String moduleName, @NotNull String baseLogFileName, @NotNull Path workingDir) + public static LocalDirectory create(@NotNull PipeRoot root, @NotNull String baseLogFileName, @NotNull Path workingDir) { return !root.isCloudRoot() ? - new LocalDirectory(workingDir.toFile(), moduleName, baseLogFileName) : - new LocalDirectory(root.getContainer(), moduleName, root, baseLogFileName); + new LocalDirectory(workingDir.toFile(), baseLogFileName) : + new LocalDirectory(root.getContainer(), root, baseLogFileName); } @JsonCreator @@ -74,7 +73,6 @@ private LocalDirectory( @JsonProperty("_isTemporary") boolean isTemporary, @JsonProperty("_pipeRoot") PipeRoot pipeRoot, @JsonProperty("_baseLogFileName") String baseLogFileName, - @JsonProperty("_moduleName") String moduleName, @JsonProperty("_remoteDir") Path remoteDir) { _localDirectoryFile = localDirectoryFile; @@ -82,22 +80,20 @@ private LocalDirectory( _pipeRoot = pipeRoot; _remoteDir = remoteDir != null ? remoteDir : _pipeRoot == null ? null : _pipeRoot.getRootNioPath(); //Using _piperoot as default for backwards compatability _baseLogFileName = baseLogFileName; - _moduleName = moduleName; } // Constructor for runs and actions when pipeline root is cloud - public LocalDirectory(Container container, String moduleName, PipeRoot pipeRoot, String basename) + public LocalDirectory(Container container, PipeRoot pipeRoot, String basename) { - this(container, moduleName, pipeRoot, basename, null); + this(container, pipeRoot, basename, null); } - public LocalDirectory(Container container, String moduleName, PipeRoot pipeRoot, String basename, Path remoteDir) + public LocalDirectory(Container container, PipeRoot pipeRoot, String basename, Path remoteDir) { _isTemporary = true; _pipeRoot = pipeRoot; _remoteDir = remoteDir != null ? remoteDir : _pipeRoot == null ? null : _pipeRoot.getRootNioPath(); //Using _piperoot as default for backwards compatability _baseLogFileName = basename; - _moduleName = moduleName; try { @@ -113,13 +109,12 @@ public LocalDirectory(Container container, String moduleName, PipeRoot pipeRoot, } // Constructor when pipeline root not in cloud - public LocalDirectory(@NotNull File localDirectory, String moduleName, String basename) + public LocalDirectory(@NotNull File localDirectory, String basename) { _localDirectoryFile = localDirectory; _isTemporary = false; _pipeRoot = null; _baseLogFileName = basename; - _moduleName = moduleName; _remoteDir = null; } diff --git a/api/src/org/labkey/api/pipeline/PipeRoot.java b/api/src/org/labkey/api/pipeline/PipeRoot.java index d71491f9586..80138b4b122 100644 --- a/api/src/org/labkey/api/pipeline/PipeRoot.java +++ b/api/src/org/labkey/api/pipeline/PipeRoot.java @@ -25,7 +25,6 @@ import org.labkey.api.security.User; import org.labkey.api.security.permissions.Permission; import org.labkey.vfs.FileLike; -import org.labkey.vfs.FileSystemLike; import java.io.File; import java.net.URI; @@ -89,21 +88,16 @@ public interface PipeRoot extends SecurableResource @Nullable FileLike resolvePathToFileLike(String relativePath); - /** - * Get a local directory that can be used for importing (Read/Write) - * - * Cloud: Uses a temp directory - * Default: Uses folder within the file root - */ @NotNull - File getImportDirectory(); + FileLike getImportDirectory(); /** * Delete the import directory and its contents + * * @return File object for import directory * @throws DirectoryNotDeletedException if import directory exists and cannot be deleted */ - Path deleteImportDirectory(@Nullable Logger log) throws DirectoryNotDeletedException; + FileLike deleteImportDirectory(@Nullable Logger log) throws DirectoryNotDeletedException; /** @return relative path to the file from the root. null if the file isn't under the root. Does not include a leading slash */ String relativePath(File file); @@ -125,18 +119,8 @@ public interface PipeRoot extends SecurableResource /** Creates a .labkey directory if it's not present and returns it. Used for things like protocol definition files, * log files for some upgrade tasks, etc. Its contents are generally not exposed directly to the user */ - @Deprecated // prefer ensureSystemFileLike() @NotNull - File ensureSystemDirectory(); - - @Deprecated // prefer ensureSystemFileLike() - @NotNull - Path ensureSystemDirectoryPath(); - - default FileLike ensureSystemFileLike() - { - return new FileSystemLike.Builder(ensureSystemDirectory()).readwrite().root(); - } + FileLike ensureSystemDirectory(); /** @return the entityId for this pipeline root, used to store permissions */ String getEntityId(); diff --git a/api/src/org/labkey/api/pipeline/PipelineJob.java b/api/src/org/labkey/api/pipeline/PipelineJob.java index e872a09ce1f..f1c2a45aa9f 100644 --- a/api/src/org/labkey/api/pipeline/PipelineJob.java +++ b/api/src/org/labkey/api/pipeline/PipelineJob.java @@ -25,6 +25,7 @@ import datadog.trace.api.Trace; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -62,6 +63,8 @@ import org.labkey.api.writer.ContainerUser; import org.labkey.api.writer.PrintWriters; import org.labkey.remoteapi.query.Filter; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import org.quartz.CronExpression; import java.io.BufferedReader; @@ -135,6 +138,11 @@ public void clearActionSet(ExpRun run) _actionSet = new RecordedActionSet(); } + public FileLike getLogFileLike() + { + return FileSystemLike.wrapFile(getLogFilePath()); + } + public enum TaskStatus { /** Job is in the queue, waiting for its turn to run */ @@ -430,12 +438,18 @@ public PipeRoot getPipeRoot() return _pipeRoot; } - @Deprecated //Please switch to the Path version + @Deprecated //Please switch to the FileLike version public void setLogFile(File logFile) { setLogFile(logFile.toPath()); } + public void setLogFile(FileLike logFile) + { + setLogFile(logFile.toNioPathForWrite()); + } + + @Deprecated //Please switch to the FileLike version public void setLogFile(Path logFile) { // Set Log file path and clear/reset logger @@ -1859,7 +1873,7 @@ public String serializeJob(boolean ensureDeserialize) public static String getClassNameFromJson(String serialized) { // Expect [ "org.labkey....", {.... - if (StringUtils.startsWith(serialized, "[")) + if (Strings.CS.startsWith(serialized, "[")) { return StringUtils.substringBetween(serialized, "\""); } @@ -2006,12 +2020,11 @@ protected Path getWorkingDirectoryString() * Note: Override getDefaultLocalDirectoryString if piperoot isn't the desired local directory * * @param pipeRoot Pipeline's root directory - * @param moduleName supplying the pipeline * @param baseLogFileName base name of the log file */ - protected final void setupLocalDirectoryAndJobLog(PipeRoot pipeRoot, String moduleName, String baseLogFileName) + protected final void setupLocalDirectoryAndJobLog(PipeRoot pipeRoot, String baseLogFileName) { - LocalDirectory localDirectory = LocalDirectory.create(pipeRoot, moduleName, baseLogFileName, getWorkingDirectoryString()); + LocalDirectory localDirectory = LocalDirectory.create(pipeRoot, baseLogFileName, getWorkingDirectoryString()); setLocalDirectory(localDirectory); setLogFile(localDirectory.determineLogFile()); } diff --git a/api/src/org/labkey/api/pipeline/PipelineProtocol.java b/api/src/org/labkey/api/pipeline/PipelineProtocol.java index f9700804715..c1fc87fd905 100644 --- a/api/src/org/labkey/api/pipeline/PipelineProtocol.java +++ b/api/src/org/labkey/api/pipeline/PipelineProtocol.java @@ -20,16 +20,13 @@ import org.fhcrc.cpas.pipeline.protocol.xml.PipelineProtocolPropsDocument; import org.labkey.api.util.FileUtil; import org.labkey.api.util.NetworkDrive; +import org.labkey.api.writer.PrintWriters; +import org.labkey.vfs.FileLike; import java.beans.PropertyDescriptor; -import java.io.BufferedWriter; -import java.io.File; import java.io.IOException; +import java.io.PrintWriter; import java.lang.reflect.InvocationTargetException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.HashMap; import java.util.Map; @@ -46,10 +43,6 @@ public abstract class PipelineProtocol private String name; private String template; - public PipelineProtocol() - { - } - public PipelineProtocol(String name) { this.name = name; @@ -65,7 +58,7 @@ public void setName(String name) this.name = name; } - public abstract PipelineProtocolFactory getFactory(); + public abstract PipelineProtocolFactory getFactory(); public void validateToSave(PipeRoot root, boolean validateName, boolean abortOnExists) throws PipelineValidationException { @@ -100,7 +93,7 @@ else if (!getFactory().isValidProtocolName(name)) throw new PipelineValidationException("The name '" + name + "' is not a valid protocol name."); } - public Path getDefinitionFile(PipeRoot root) + public FileLike getDefinitionFile(PipeRoot root) { return getFactory().getProtocolFile(root, name, false); } @@ -152,17 +145,11 @@ protected Map getSaveProperties() return propMap; } - @Deprecated - public void save(File file) throws IOException - { - save(file.toPath()); - } - - private void ensureDir(Path dir) throws IOException + private void ensureDir(FileLike dir) throws IOException { try { - if (!Files.exists(dir)) + if (!dir.exists()) { FileUtil.createDirectories(dir); } @@ -173,9 +160,9 @@ private void ensureDir(Path dir) throws IOException } } - public void save(Path file) throws IOException + public void save(FileLike file) throws IOException { - Path dir = file.getParent(); + FileLike dir = file.getParent(); try { ensureDir(dir); @@ -210,9 +197,9 @@ public void save(Path file) throws IOException XmlOptions opts = new XmlOptions() .setSavePrettyPrint() .setSaveImplicitNamespaces(mapNS); - try (BufferedWriter bfw = Files.newBufferedWriter(file, StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) + try (PrintWriter pw = PrintWriters.getPrintWriter(file.toNioPathForWrite())) { - doc.save(bfw, opts); + doc.save(pw, opts); } } diff --git a/api/src/org/labkey/api/pipeline/PipelineProtocolFactory.java b/api/src/org/labkey/api/pipeline/PipelineProtocolFactory.java index 97e4d4dcba6..94dced631af 100644 --- a/api/src/org/labkey/api/pipeline/PipelineProtocolFactory.java +++ b/api/src/org/labkey/api/pipeline/PipelineProtocolFactory.java @@ -21,14 +21,15 @@ import org.fhcrc.cpas.pipeline.protocol.xml.PipelineProtocolPropsDocument; import org.labkey.api.util.FileUtil; import org.labkey.api.util.NetworkDrive; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Path; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; /** @@ -44,10 +45,10 @@ public abstract class PipelineProtocolFactory private static final Logger LOG = LogManager.getLogger(PipelineProtocolFactory.class); - public static Path getProtocolRootDir(PipeRoot root) + public static FileLike getProtocolRootDir(PipeRoot root) { - Path systemDir = root.ensureSystemDirectoryPath(); - return systemDir.resolve(_pipelineProtocolDir); + FileLike systemDir = root.ensureSystemDirectory(); + return systemDir.resolveChild(_pipelineProtocolDir); } public static File locateProtocolRootDir(File rootDir, File systemDir) @@ -63,7 +64,7 @@ public static File locateProtocolRootDir(File rootDir, File systemDir) public T load(PipeRoot root, String name, boolean archived) throws IOException { - Path file = getProtocolFile(root, name, archived); + FileLike file = getProtocolFile(root, name, archived); try { Map mapNS = new HashMap<>(); @@ -71,7 +72,7 @@ public T load(PipeRoot root, String name, boolean archived) throws IOException XmlOptions opts = new XmlOptions().setLoadSubstituteNamespaces(mapNS); PipelineProtocolPropsDocument doc = - PipelineProtocolPropsDocument.Factory.parse(Files.newInputStream(file), opts); + PipelineProtocolPropsDocument.Factory.parse(file.openInputStream(), opts); PipelineProtocolPropsDocument.PipelineProtocolProps ppp = doc.getPipelineProtocolProps(); String type = ppp.getType(); @@ -104,7 +105,7 @@ public T load(PipeRoot root, String name, boolean archived) throws IOException } catch (Exception e) { - throw new IOException("Failed to load protocol document " + file.toAbsolutePath() + ".", e); + throw new IOException("Failed to load protocol document " + file + ".", e); } } @@ -115,48 +116,42 @@ public boolean isValidProtocolName(String name) public boolean exists(PipeRoot root, String name, boolean archived) { - return Files.exists(getProtocolFile(root, name, archived)); + return getProtocolFile(root, name, archived).exists(); } - public Path getProtocolDir(PipeRoot root, boolean archived) + public FileLike getProtocolDir(PipeRoot root, boolean archived) { - Path protocolDir = getProtocolRootDir(root).resolve(getName()); + FileLike protocolDir = getProtocolRootDir(root).resolveChild(getName()); if (archived) - protocolDir = protocolDir.resolve(_archivedProtocolDir); + protocolDir = protocolDir.resolveChild(_archivedProtocolDir); return protocolDir; } - public Path getProtocolFile(PipeRoot root, String name, boolean archived) + public FileLike getProtocolFile(PipeRoot root, String name, boolean archived) { - return FileUtil.appendName(getProtocolDir(root, archived), name + ".xml"); + return getProtocolDir(root, archived).resolveChild(name + ".xml"); } /** @return sorted list of protocol names */ - public String[] getProtocolNames(PipeRoot root, Path dirData, boolean archived) + public String[] getProtocolNames(PipeRoot root, FileLike dirData, boolean archived) { HashSet setNames = new HashSet<>(); // Add .xml files - File[] files = getProtocolDir(root, archived).toFile().listFiles(f -> f.getName().endsWith(".xml") && !f.isDirectory()); - if (files != null) + List files = getProtocolDir(root, archived).getChildren(f -> f.getName().endsWith(".xml") && !f.isDirectory()); + for (FileLike file : files) { - for (File file : files) - { - final String name = file.getName(); - setNames.add(name.substring(0, name.lastIndexOf('.'))); - } + final String name = file.getName(); + setNames.add(name.substring(0, name.lastIndexOf('.'))); } // Add all directories that already exist in the analysis root. if (dirData != null && !archived) { - files = dirData.resolve(getName()).toFile().listFiles(File::isDirectory); + files = dirData.resolveChild(getName()).getChildren(FileLike::isDirectory); - if (files != null) - { - for (File file : files) - setNames.add(file.getName()); - } + for (FileLike file : files) + setNames.add(file.getName()); } String[] vals = setNames.toArray(new String[0]); @@ -179,22 +174,22 @@ public boolean changeArchiveStatus(PipeRoot root, String name, boolean moveToArc { if (moveToArchive) { - Path archiveDir = getProtocolDir(root, true); - if (!Files.exists(archiveDir)) + FileLike archiveDir = getProtocolDir(root, true); + if (!archiveDir.exists()) { FileUtil.createDirectories(archiveDir); } - else if (Files.isRegularFile(archiveDir)) + else if (archiveDir.isFile()) { LOG.error("Unable to create archived directory because a file with that name exists in the protocol directory: " - + getProtocolDir(root, false).toAbsolutePath()); + + getProtocolDir(root, false)); return false; } } try { - Files.move(getProtocolFile(root, name, !moveToArchive), getProtocolFile(root, name, moveToArchive)); + Files.move(getProtocolFile(root, name, !moveToArchive).toNioPathForWrite(), getProtocolFile(root, name, moveToArchive).toNioPathForWrite()); } catch (IOException e) { @@ -215,25 +210,25 @@ else if (Files.isRegularFile(archiveDir)) */ public boolean deleteProtocolFile(PipeRoot root, String name) { - Path protocolFile = getProtocolFile(root, name, false); + FileLike protocolFile = getProtocolFile(root, name, false); //If it doesn't exist, check archive - if (!Files.exists(protocolFile)) + if (!protocolFile.exists()) protocolFile = getProtocolFile(root, name, true); //If it still doesn't exist, move on - if (!Files.exists(protocolFile)) + if (!protocolFile.exists()) { return true; // We don't care if the file doesn't exist } try { - return Files.deleteIfExists(protocolFile); + return protocolFile.delete(); } catch (IOException e) { - LogManager.getLogger(PipelineProtocolFactory.class).debug("Error attempting to delete protocol file " + protocolFile, e); + LOG.debug("Error attempting to delete protocol file " + protocolFile, e); return false; } } diff --git a/api/src/org/labkey/api/pipeline/PipelineProvider.java b/api/src/org/labkey/api/pipeline/PipelineProvider.java index a5cadc1a1a3..618b46eed5a 100644 --- a/api/src/org/labkey/api/pipeline/PipelineProvider.java +++ b/api/src/org/labkey/api/pipeline/PipelineProvider.java @@ -208,13 +208,6 @@ public String getName() * @param rootDir the pipeline root directory on disk * @param systemDir the system directory itself */ - @Deprecated - public void initSystemDirectory(File rootDir, File systemDir) - { - if (null != rootDir && null != systemDir) - initSystemDirectory(rootDir.toPath(), systemDir.toPath()); - } - public void initSystemDirectory(Path rootDir, Path systemDir) { } diff --git a/api/src/org/labkey/api/pipeline/PipelineService.java b/api/src/org/labkey/api/pipeline/PipelineService.java index 0bd346a4781..f87384fb70d 100644 --- a/api/src/org/labkey/api/pipeline/PipelineService.java +++ b/api/src/org/labkey/api/pipeline/PipelineService.java @@ -33,6 +33,7 @@ import org.labkey.api.view.HttpView; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.api.view.ViewContext; +import org.labkey.vfs.FileLike; import org.springframework.validation.BindException; import java.io.File; @@ -195,7 +196,7 @@ void rememberLastProtocolSetting(PipelineProtocolFactory factory, Container c TableInfo getJobsTable(User user, Container container, @Nullable ContainerFilter cf); - boolean runFolderImportJob(Container c, User user, ActionURL url, Path folderXml, String originalFilename, PipeRoot pipelineRoot, ImportOptions options); + boolean runFolderImportJob(Container c, User user, ActionURL url, FileLike folderXml, String originalFilename, PipeRoot pipelineRoot, ImportOptions options); /** * Register a folder archive source implementation. A FolderArchiveSource creates folder artifacts that can be @@ -224,10 +225,10 @@ void rememberLastProtocolSetting(PipelineProtocolFactory factory, Container c class PathAnalysisProperties { private final PipeRoot _pipeRoot; - private final Path _dirData; + private final FileLike _dirData; private final AbstractFileAnalysisProtocolFactory _factory; - public PathAnalysisProperties(PipeRoot pipeRoot, Path dirData, AbstractFileAnalysisProtocolFactory factory) + public PathAnalysisProperties(PipeRoot pipeRoot, FileLike dirData, AbstractFileAnalysisProtocolFactory factory) { _pipeRoot = pipeRoot; _dirData = dirData; @@ -240,7 +241,7 @@ public PipeRoot getPipeRoot() } @Nullable - public Path getDirData() + public FileLike getDirData() { return _dirData; } diff --git a/api/src/org/labkey/api/pipeline/PipelineStatusFile.java b/api/src/org/labkey/api/pipeline/PipelineStatusFile.java index e5337df65a8..63a656e2ccc 100644 --- a/api/src/org/labkey/api/pipeline/PipelineStatusFile.java +++ b/api/src/org/labkey/api/pipeline/PipelineStatusFile.java @@ -17,6 +17,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Container; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -37,6 +38,10 @@ interface StatusReader @Deprecated PipelineStatusFile getStatusFile(File logFile); PipelineStatusFile getStatusFile(Container container, Path logFile); + default PipelineStatusFile getStatusFile(Container container, FileLike logFile) + { + return getStatusFile(container, logFile.toNioPathForRead()); + } PipelineStatusFile getStatusFile(long rowId); diff --git a/api/src/org/labkey/api/pipeline/PipelineUrls.java b/api/src/org/labkey/api/pipeline/PipelineUrls.java index e292f64de05..c0c098db078 100644 --- a/api/src/org/labkey/api/pipeline/PipelineUrls.java +++ b/api/src/org/labkey/api/pipeline/PipelineUrls.java @@ -22,6 +22,7 @@ import org.labkey.api.data.Container; import org.labkey.api.util.URLHelper; import org.labkey.api.view.ActionURL; +import org.labkey.vfs.FileLike; import java.nio.file.Path; @@ -41,7 +42,7 @@ public interface PipelineUrls extends UrlProvider ActionURL urlActions(Container container); - ActionURL urlStartFolderImport(Container container, @NotNull Path archiveFile, @Nullable ImportOptions options, boolean fromTemplateSourceFolder); + ActionURL urlStartFolderImport(Container container, @NotNull FileLike archiveFile, @Nullable ImportOptions options, boolean fromTemplateSourceFolder); ActionURL urlCreatePipelineTrigger(Container container, String pipelineId, @Nullable ActionURL returnUrl); diff --git a/api/src/org/labkey/api/pipeline/RecordedAction.java b/api/src/org/labkey/api/pipeline/RecordedAction.java index c895b3681ff..23abf52bcdb 100644 --- a/api/src/org/labkey/api/pipeline/RecordedAction.java +++ b/api/src/org/labkey/api/pipeline/RecordedAction.java @@ -21,6 +21,7 @@ import org.labkey.api.exp.PropertyType; import org.labkey.api.util.FileUtil; import org.labkey.api.util.Pair; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.Serializable; @@ -93,6 +94,11 @@ public void addInput(File input, String role) addInput(input.toURI(), role); } + public void addInput(FileLike input, String role) + { + addInput(input.toNioPathForRead().toFile(), role); + } + private boolean uriExists(URI toTest, Set set) { for (DataFile df : set) diff --git a/api/src/org/labkey/api/pipeline/RecordedActionSet.java b/api/src/org/labkey/api/pipeline/RecordedActionSet.java index cc03d80a71f..d0496a22f9e 100644 --- a/api/src/org/labkey/api/pipeline/RecordedActionSet.java +++ b/api/src/org/labkey/api/pipeline/RecordedActionSet.java @@ -19,9 +19,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.labkey.vfs.FileLike; import java.net.URI; -import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; @@ -84,9 +84,9 @@ public Map getOtherInputs() return _otherInputs; } - public void add(Path inputFile, String inputRole) + public void add(FileLike inputFile, String inputRole) { - _otherInputs.put(inputFile.toUri(), inputRole); + _otherInputs.put(inputFile.toNioPathForRead().toUri(), inputRole); } public void add(RecordedActionSet set) diff --git a/api/src/org/labkey/api/pipeline/WorkDirectory.java b/api/src/org/labkey/api/pipeline/WorkDirectory.java index be806a421e9..e60c24985db 100644 --- a/api/src/org/labkey/api/pipeline/WorkDirectory.java +++ b/api/src/org/labkey/api/pipeline/WorkDirectory.java @@ -17,6 +17,7 @@ import org.labkey.api.pipeline.cmd.TaskPath; import org.labkey.api.util.FileType; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -56,9 +57,6 @@ enum Function { /** Informs the WorkDirectory that a new file is being created. It is treated as a Function.output */ File newFile(FileType type); - /** Informs the WorkDirectory that a new file is being created. */ - File newFile(Function f, FileType type); - /** * Indicates that a file is to be used as input. The implementation can choose whether it needs to be copied, unless * forceCopy is true (in which case it will always be copied to the work directory @@ -66,6 +64,12 @@ enum Function { */ File inputFile(File fileInput, boolean forceCopy) throws IOException; + default File inputFile(FileLike fileInput, boolean forceCopy) throws IOException + { + return inputFile(fileInput.toNioPathForRead().toFile(), forceCopy); + } + + /** * Indicates that a file is to be used as input. The implementation can choose whether it needs to be copied, unless * forceCopy is true (in which case it will always be copied to the work directory. This version of the method allows the caller diff --git a/api/src/org/labkey/api/pipeline/browse/PipelinePathForm.java b/api/src/org/labkey/api/pipeline/browse/PipelinePathForm.java index b037848edf1..7db45135681 100644 --- a/api/src/org/labkey/api/pipeline/browse/PipelinePathForm.java +++ b/api/src/org/labkey/api/pipeline/browse/PipelinePathForm.java @@ -21,13 +21,11 @@ import org.labkey.api.pipeline.PipeRoot; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.util.FileUtil; import org.labkey.api.util.NetworkDrive; import org.labkey.api.view.NotFoundException; import org.labkey.api.view.ViewForm; +import org.labkey.vfs.FileLike; -import java.io.File; -import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -91,16 +89,16 @@ public void setFileIds(int[] fileIds) * For ExpData IDs provided, ensures the files exists and the user has read permission on the associated container. The files do not need to be located in the same directory. * Throws NotFoundException if no files are specified, invalid files are specified, there's no pipeline root, etc. */ - public List getValidatedFiles(Container c) + public List getValidatedFiles(Container c) { return getValidatedFiles(c, false); } - public List getValidatedFiles(Container c, boolean allowNonExistentFiles) + public List getValidatedFiles(Container c, boolean allowNonExistentFiles) { PipeRoot pr = getPipeRoot(c); - File dir = pr.resolvePath(getPath()); + FileLike dir = pr.resolvePathToFileLike(getPath()); if (dir == null || !dir.exists()) throw new NotFoundException("Could not find path " + getPath()); @@ -109,10 +107,10 @@ public List getValidatedFiles(Container c, boolean allowNonExistentFiles) throw new NotFoundException("No files specified"); } - List result = new ArrayList<>(); + List result = new ArrayList<>(); for (String fileName : _file) { - File f = pr.resolvePath(getPath() + "/" + fileName); + FileLike f = pr.resolvePathToFileLike(getPath() + "/" + fileName); if (!allowNonExistentFiles && !NetworkDrive.exists(f)) { throw new NotFoundException("Could not find file '" + fileName + "' in '" + getPath() + "'"); @@ -136,7 +134,7 @@ public List getValidatedFiles(Container c, boolean allowNonExistentFiles) throw new NotFoundException("Insufficient permissions for file '" + data.getFile()); } - File file = data.getFile(); + FileLike file = data.getFileLike(); if (!allowNonExistentFiles && !NetworkDrive.exists(file)) { throw new NotFoundException("Could not find file '" + file + "'"); @@ -150,55 +148,12 @@ public List getValidatedFiles(Container c, boolean allowNonExistentFiles) public List getValidatedPaths(Container c, boolean allowNonExistentFiles) { - PipeRoot pr = getPipeRoot(c); - - Path dir = pr.resolveToNioPath(getPath()); - if (dir == null || !Files.exists(dir)) - throw new NotFoundException("Could not find path " + getPath()); - - if ((getFile() == null || getFile().length == 0) && (getFileIds() == null || getFileIds().length == 0)) - { - throw new NotFoundException("No files specified"); - } - + List files = getValidatedFiles(c, allowNonExistentFiles); List result = new ArrayList<>(); - for (String fileName : _file) + for (FileLike file : files) { - Path path = pr.resolveToNioPath(getPath() + "/" + fileName); - if (!allowNonExistentFiles && (null == path || !Files.exists(path))) - { - throw new NotFoundException("Could not find file '" + fileName + "' in '" + getPath() + "'"); - } - if (null != path) - result.add(path); - } - - ExperimentService es = ExperimentService.get(); - if (_fileIds != null) - { - for (int fileId : _fileIds) - { - ExpData data = es.getExpData(fileId); - if(data == null) - { - throw new NotFoundException("Could not find file associated with Data Id: '" + fileId); - } - - if (!data.getContainer().hasPermission(getUser(), ReadPermission.class)) - { - throw new NotFoundException("Insufficient permissions for file '" + data.getFile()); - } - - Path path = pr.resolveToNioPath(data.getDataFileURI().getPath()); - if (!allowNonExistentFiles && (null == path || !Files.exists(path))) - { - throw new NotFoundException("Could not find file '" + FileUtil.getFileName(path) + "'"); - } - if (null != path) - result.add(path); - } + result.add(file.toNioPathForRead()); } - return result; } @@ -211,13 +166,18 @@ public PipeRoot getPipeRoot(Container c) } /** Verifies that only a single file was selected and returns it, throwing an exception if there isn't exactly one */ - @Deprecated //prefer the nio.Path version: getValidatedSinglePath - public File getValidatedSingleFile(Container c) + public FileLike getValidatedSingleFile(Container c) { - return getValidatedSinglePath(c).toFile(); + List files = getValidatedFiles(c); + if (files.size() != 1) + { + throw new IllegalArgumentException("Expected a single file but got " + files.size()); + } + return files.get(0); } /** Verifies that only a single file was selected and returns it, throwing an exception if there isn't exactly one */ + @Deprecated // use the FileLike version public Path getValidatedSinglePath(Container c) { List files = getValidatedPaths(c, false); diff --git a/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisJob.java b/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisJob.java index 766e2421a95..396a2b31348 100644 --- a/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisJob.java +++ b/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisJob.java @@ -15,10 +15,10 @@ */ package org.labkey.api.pipeline.file; +import io.micrometer.common.util.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.labkey.api.exp.PropertyType; import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.api.ExperimentService; @@ -36,10 +36,10 @@ import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; @@ -61,11 +61,10 @@ abstract public class AbstractFileAnalysisJob extends PipelineJob implements Fil private String _protocolName; private String _joinedBaseName; private String _baseName; - private Path _dirData; - private Path _dirAnalysis; - private Path _fileParameters; - private Path _fileJobInfo; - private List _filesInput; + private FileLike _dirData; + private FileLike _dirAnalysis; + private FileLike _fileParameters; + private List _filesInput; private List _inputTypes; private boolean _splittable = true; @@ -77,39 +76,14 @@ abstract public class AbstractFileAnalysisJob extends PipelineJob implements Fil // For serialization protected AbstractFileAnalysisJob() {} - @Deprecated //Prefer the Path version, retained for backwards compatbility - public AbstractFileAnalysisJob(AbstractFileAnalysisProtocol protocol, + public AbstractFileAnalysisJob(@NotNull AbstractFileAnalysisProtocol protocol, String providerName, ViewBackgroundInfo info, PipeRoot root, String protocolName, - File fileParameters, - List filesInput, - boolean splittable, - boolean writeJobInfoFile) throws IOException - { - this( - protocol, - providerName, - info, - root, - protocolName, - fileParameters.toPath(), - filesInput.stream().map(File::toPath).collect(Collectors.toList()), - splittable, - writeJobInfoFile - ); - } - - public AbstractFileAnalysisJob(@NotNull AbstractFileAnalysisProtocol protocol, - String providerName, - ViewBackgroundInfo info, - PipeRoot root, - String protocolName, - Path fileParameters, - List filesInput, - boolean splittable, - boolean writeJobInfoFile) throws IOException + FileLike fileParameters, + List filesInput, + boolean splittable) throws IOException { super(providerName, info, root); @@ -127,13 +101,13 @@ public AbstractFileAnalysisJob(@NotNull AbstractFileAnalysisProtocol protocol, // Check for explicitly set default parameters. Otherwise use the default. String paramDefaults = _parametersOverrides.get("list path, default parameters"); - Path fileDefaults; + FileLike fileDefaults; if (paramDefaults != null) - fileDefaults = getPipeRoot().resolveToNioPath(paramDefaults); + fileDefaults = getPipeRoot().resolvePathToFileLike(paramDefaults); else fileDefaults = protocol.getFactory().getDefaultParametersFile(root); - _parametersDefaults = fileDefaults != null && Files.exists(fileDefaults) ? + _parametersDefaults = fileDefaults != null && fileDefaults.exists() ? getInputParameters(fileDefaults).getInputParameters() : Collections.emptyMap(); @@ -155,7 +129,7 @@ public AbstractFileAnalysisJob(@NotNull AbstractFileAnalysisProtocol protocol, } String logFile = protocol.timestampLog() ? FileUtil.makeFileNameWithTimestamp(_baseName) : _baseName; - setupLocalDirectoryAndJobLog(getPipeRoot(), "FileAnalysis", logFile); + setupLocalDirectoryAndJobLog(getPipeRoot(), logFile); } /** @@ -164,15 +138,15 @@ public AbstractFileAnalysisJob(@NotNull AbstractFileAnalysisProtocol protocol, @Override protected Path getWorkingDirectoryString() { - return _dirAnalysis.toAbsolutePath(); + return _dirAnalysis.toNioPathForWrite().toAbsolutePath(); } - public AbstractFileAnalysisJob(AbstractFileAnalysisJob job, File fileInput) + public AbstractFileAnalysisJob(AbstractFileAnalysisJob job, FileLike fileInput) { this(job, Collections.singletonList(fileInput)); } - public AbstractFileAnalysisJob(AbstractFileAnalysisJob job, List filesInput) + public AbstractFileAnalysisJob(AbstractFileAnalysisJob job, List filesInput) { super(job); @@ -188,11 +162,11 @@ public AbstractFileAnalysisJob(AbstractFileAnalysisJob job, List filesInpu _joinedBaseName = job._joinedBaseName; // Change parameters which are specific to the fraction job. - _filesInput = filesInput.stream().map(File::toPath).collect(Collectors.toList()); + _filesInput = new ArrayList<>(filesInput); _inputTypes = FileType.findTypes(job._inputTypes, _filesInput); _baseName = (_inputTypes.isEmpty() ? filesInput.get(0).getName() : _inputTypes.get(0).getBaseName(filesInput.get(0))); - setupLocalDirectoryAndJobLog(getPipeRoot(), "FileAnalysis", _baseName); + setupLocalDirectoryAndJobLog(getPipeRoot(), _baseName); } @Override @@ -222,7 +196,7 @@ public List createSplitJobs() return super.createSplitJobs(); ArrayList jobs = new ArrayList<>(); - for (File file : getInputFiles()) + for (FileLike file : _filesInput) jobs.add(createSingleFileJob(file)); return Collections.unmodifiableList(jobs); } @@ -235,7 +209,7 @@ public TaskPipeline getTaskPipeline() abstract public TaskId getTaskPipelineId(); - abstract public AbstractFileAnalysisJob createSingleFileJob(File file); + abstract public AbstractFileAnalysisJob createSingleFileJob(FileLike file); @Override public String getProtocolName() @@ -259,7 +233,7 @@ public String getJoinedBaseName() public List getSplitBaseNames() { ArrayList baseNames = new ArrayList<>(); - for (Path fileInput : _filesInput) + for (FileLike fileInput : _filesInput) { for (FileType ft : _inputTypes) { @@ -278,7 +252,7 @@ public String getBaseNameForFileType(FileType fileType) { if (fileType != null) { - for (Path fileInput : _filesInput) + for (FileLike fileInput : _filesInput) { if (fileType.isType(fileInput)) return fileType.getBaseName(fileInput); @@ -289,13 +263,7 @@ public String getBaseNameForFileType(FileType fileType) } @Override - public File getDataDirectory() - { - return _dirData.toFile(); - } - - @Override - public Path getDataDirectoryPath() + public FileLike getDataDirectoryFileLike() { return _dirData; } @@ -303,13 +271,7 @@ public Path getDataDirectoryPath() @Override public File getAnalysisDirectory() { - return _dirAnalysis.toFile(); - } - - @Override - public Path getAnalysisDirectoryPath() - { - return _dirAnalysis; + return _dirAnalysis.toNioPathForWrite().toFile(); } @Override @@ -357,28 +319,13 @@ public List getInputFiles() @Override public List getInputFilePaths() { - return _filesInput; - } - - @Override - @Nullable - public File getJobInfoFile() - { - return _fileJobInfo.toFile(); - } - - - @Override - @Nullable - public Path getJobInfoFilePath() - { - return _fileJobInfo; + return _filesInput.stream().map(FileLike::toNioPathForRead).toList(); } @Override public File getParametersFile() { - return _fileParameters.toFile(); + return _fileParameters.toNioPathForRead().toFile(); } @Override @@ -407,10 +354,10 @@ public ParamParser getInputParameters() throws IOException return getInputParameters(_fileParameters); } - public ParamParser getInputParameters(Path parametersFile) throws IOException + public ParamParser getInputParameters(FileLike parametersFile) throws IOException { ParamParser parser = createParamParser(); - parser.parse(Files.newInputStream(parametersFile)); + parser.parse(parametersFile.openInputStream()); if (parser.getErrors() != null) { ParamParser.Error err = parser.getErrors()[0]; @@ -428,7 +375,7 @@ public ParamParser getInputParameters(Path parametersFile) throws IOException return parser; } - private void logParameters(String description, Path file, Map parameters) + private void logParameters(String description, FileLike file, Map parameters) { _log.debug(description + " " + parameters.size() + " parameters (" + file + "):"); for (Map.Entry entry : new TreeMap<>(parameters).entrySet()) @@ -445,7 +392,7 @@ public ParamParser createParamParser() @Override public String getDescription() { - return getDataDescription(getDataDirectoryPath(), getBaseName(), getJoinedBaseName(), getProtocolName(), getInputFilePaths()); + return getDataDescription(getDataDirectoryFileLike(), getBaseName(), getJoinedBaseName(), getProtocolName(), _filesInput); } @Override @@ -460,25 +407,19 @@ public ActionURL getStatusHref() return null; } - @Deprecated //prefer Path version - public static String getDataDescription(File dirData, String baseName, String joinedBaseName, String protocolName) - { - return getDataDescription(dirData.toPath(), baseName, joinedBaseName, protocolName, Collections.emptyList()); - } - - public static String getDataDescription(Path dirData, String baseName, String joinedBaseName, String protocolName, List inputFiles) + public static String getDataDescription(FileLike dirData, String baseName, String joinedBaseName, String protocolName, List inputFiles) { String dataName = ""; if (dirData != null) { - dataName = dirData.getFileName().toString(); + dataName = dirData.getName(); // Can't remember why we would ever need the "xml" check. We may get an extra "." in the path, // so check for that and remove it. if (".".equals(dataName) || "xml".equals(dataName)) { dirData = dirData.getParent(); if (dirData != null) - dataName = dirData.getFileName().toString(); + dataName = dirData.getName(); } } @@ -490,14 +431,17 @@ public static String getDataDescription(Path dirData, String baseName, String jo description.append("/"); description.append(baseName); } - description.append(" (").append(protocolName).append(")"); + if (!StringUtils.isEmpty(protocolName)) + { + description.append(" (").append(protocolName).append(")"); + } // input files if (!inputFiles.isEmpty()) { description.append(" ("); //p.getFileName returns the full S3 path -- S3fs bug? - description.append(inputFiles.stream().map(FileUtil::getFileName).collect(Collectors.joining(","))); + description.append(inputFiles.stream().map(FileLike::getName).collect(Collectors.joining(","))); description.append(")"); } return description.toString(); diff --git a/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisProtocol.java b/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisProtocol.java index a75a2b01925..2f13f675526 100644 --- a/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisProtocol.java +++ b/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisProtocol.java @@ -32,6 +32,7 @@ import org.labkey.api.util.XmlBeansUtil; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.api.writer.PrintWriters; +import org.labkey.vfs.FileLike; import org.xml.sax.InputSource; import javax.xml.parsers.DocumentBuilder; @@ -41,14 +42,11 @@ import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import java.io.BufferedReader; -import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.io.StringReader; import java.io.StringWriter; import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -141,32 +139,21 @@ public String getJoinedBaseName() return getName(); } - public String getBaseName(File file) - { - return getBaseName(file.toPath()); - } - - public String getBaseName(Path file) + public String getBaseName(FileLike file) { FileType ft = findInputType(file); if (ft == null) - return file.getFileName().toString(); + return file.getName(); return ft.getBaseName(file); } - public File getAnalysisDir(File dirData, PipeRoot root) - { - return getFactory().getAnalysisDir(dirData.toPath(), getName(), root).toFile(); - } - - @Deprecated //Prefer Path version - public File getParametersFile(File dirData, PipeRoot root) + public FileLike getAnalysisDir(FileLike dirData, PipeRoot root) { - return getParametersFile(dirData.toPath(), root).toFile(); + return getFactory().getAnalysisDir(dirData, getName(), root); } - public Path getParametersFile(Path dirData, PipeRoot root) + public FileLike getParametersFile(FileLike dirData, PipeRoot root) { return getFactory().getParametersFile(dirData, getName(), root); } @@ -177,20 +164,14 @@ public void saveDefinition(PipeRoot root) throws IOException save(getFactory().getProtocolFile(root, getName(), false), null, null); } - @Deprecated //Prefer Path version - public void saveInstance(File file, Container c) throws IOException - { - saveInstance(file.toPath(), c); - } - - public void saveInstance(Path file, Container c) throws IOException + public void saveInstance(FileLike file, Container c) throws IOException { Map addParams = new HashMap<>(); addParams.put(PipelineJob.PIPELINE_EMAIL_ADDRESS_PARAM, email); save(file, null, addParams); } - protected void save(Path file, Map addParams, Map instanceParams) throws IOException + protected void save(FileLike file, Map addParams, Map instanceParams) throws IOException { if (xml == null || xml.isEmpty()) { @@ -210,11 +191,12 @@ protected void save(Path file, Map addParams, Map addParams, Map getFactory(); public abstract JOB createPipelineJob(ViewBackgroundInfo info, - PipeRoot root, List filesInput, - Path fileParameters, @Nullable Map variableMap) throws IOException; + PipeRoot root, List filesInput, + FileLike fileParameters, @Nullable Map variableMap) throws IOException; public boolean timestampLog() { diff --git a/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisProtocolFactory.java b/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisProtocolFactory.java index 28fe1bf96e7..b370206472c 100644 --- a/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisProtocolFactory.java +++ b/api/src/org/labkey/api/pipeline/file/AbstractFileAnalysisProtocolFactory.java @@ -25,28 +25,23 @@ import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineJobService; import org.labkey.api.pipeline.PipelineProtocolFactory; -import org.labkey.api.pipeline.PipelineProvider; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.TaskPipeline; +import org.labkey.api.reader.Readers; import org.labkey.api.util.FileUtil; import org.labkey.api.util.NetworkDrive; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.writer.PrintWriters; +import org.labkey.vfs.FileLike; import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.PrintWriter; import java.io.Reader; import java.io.StringReader; -import java.nio.charset.Charset; -import java.nio.file.Files; import java.nio.file.InvalidPathException; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.List; /** * Base class for protocol factories that are primarily focused on analyzing data files (as opposed to other types of resources) @@ -96,13 +91,13 @@ public String getLegacyDefaultParametersFileName() * @param root pipeline root under which the files are stored * @return analysis directory */ - public Path getAnalysisDir(Path dirData, String protocolName, PipeRoot root) + public FileLike getAnalysisDir(FileLike dirData, String protocolName, PipeRoot root) { - Path defaultFile = dirData.resolve(getName()).resolve(protocolName); + FileLike defaultFile = dirData.resolveChild(getName()).resolveChild(protocolName); // Check if the pipeline root wants us to write somewhere else, because the source file might be in a read-only // pipeline location String relativePath = root.relativePath(defaultFile); - return root.resolveToNioPath(relativePath); + return root.resolvePathToFileLike(relativePath); } /** @@ -113,32 +108,26 @@ public boolean isProtocolTypeFile(File file) return NetworkDrive.exists(new File(file.getParent(), getParametersFileName())); } - @Deprecated - public File getParametersFile(@Nullable File dirData, String protocolName, PipeRoot root) - { - Path result = getParametersFile(dirData == null? null : dirData.toPath(), protocolName, root); - return result != null ? result.toFile() : null; - } /** - * Get the parameters file location, given a directory containing the mass spec data. + * Get the parameters file location, given a directory containing the input files to the job. * - * @param dirData mass spec data directory + * @param dirData input data directory * @param protocolName name of protocol for analysis * @param root pipeline root under which the files are stored * @return parameters file */ @Nullable - public Path getParametersFile(@Nullable Path dirData, String protocolName, PipeRoot root) + public FileLike getParametersFile(@Nullable FileLike dirData, String protocolName, PipeRoot root) { if (dirData == null) { return null; } - Path defaultFile = getAnalysisDir(dirData, protocolName, root).resolve(getParametersFileName()); + FileLike defaultFile = getAnalysisDir(dirData, protocolName, root).resolveChild(getParametersFileName()); // Check if the pipeline root wants us to write somewhere else, because the source file might be in a read-only // pipeline location String relativePath = root.relativePath(defaultFile); - return root.resolveToNioPath(relativePath); + return root.resolvePathToFileLike(relativePath); } /** @@ -147,9 +136,9 @@ public Path getParametersFile(@Nullable Path dirData, String protocolName, PipeR * @param root pipeline root directory * @return default parameters file */ - public Path getDefaultParametersFile(PipeRoot root) + public FileLike getDefaultParametersFile(PipeRoot root) { - return getProtocolDir(root, false).resolve(getDefaultParametersFileName()); + return getProtocolDir(root, false).resolveChild(getDefaultParametersFileName()); } /** @@ -164,7 +153,7 @@ public void ensureDefaultParameters(PipeRoot root) throws IOException } @Override - public String[] getProtocolNames(PipeRoot root, Path dirData, boolean archived) + public String[] getProtocolNames(PipeRoot root, FileLike dirData, boolean archived) { String[] protocolNames = super.getProtocolNames(root, dirData, archived); @@ -224,10 +213,10 @@ public T load(PipeRoot root, String name, boolean archived) throws IOException return instance; } - public T loadInstance(Path file, Container container) throws IOException + public T loadInstance(FileLike file, Container container) throws IOException { ParamParser parser = createParamParser(); - try (InputStream is = Files.newInputStream(file)) + try (InputStream is = file.openInputStream()) { parser.parse(is); if (parser.getErrors() != null) @@ -251,8 +240,8 @@ public T loadInstance(Path file, Container container) throws IOException public String getDefaultParametersXML(PipeRoot root) throws IOException { - Path fileDefault = getDefaultParametersFile(root); - if (!Files.exists(fileDefault)) + FileLike fileDefault = getDefaultParametersFile(root); + if (!fileDefault.exists()) return null; return new FileDefaultsReader(fileDefault).readXML(); @@ -260,9 +249,9 @@ public String getDefaultParametersXML(PipeRoot root) throws IOException protected static class FileDefaultsReader extends DefaultsReader { - private final Path _fileDefaults; + private final FileLike _fileDefaults; - public FileDefaultsReader(Path fileDefaults) + public FileDefaultsReader(FileLike fileDefaults) { _fileDefaults = fileDefaults; } @@ -270,7 +259,7 @@ public FileDefaultsReader(Path fileDefaults) @Override public Reader createReader() throws IOException { - return Files.newBufferedReader(_fileDefaults, Charset.defaultCharset()); + return Readers.getReader(_fileDefaults.openInputStream()); } } @@ -313,10 +302,10 @@ public void setDefaultParametersXML(PipeRoot root, String xml) throws IOExceptio throw new IllegalArgumentException("Line " + err.getLine() + ": " + err.getMessage()); } - Path fileDefault = getDefaultParametersFile(root); + FileLike fileDefault = getDefaultParametersFile(root); FileUtil.createDirectories(fileDefault.getParent()); - try (BufferedWriter writer = Files.newBufferedWriter(fileDefault, Charset.defaultCharset(), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) + try (PrintWriter writer = PrintWriters.getPrintWriter(fileDefault.openOutputStream())) { writer.write(xml, 0, xml.length()); } @@ -327,31 +316,12 @@ public void setDefaultParametersXML(PipeRoot root, String xml) throws IOExceptio } } - public static >, F extends AbstractFileAnalysisProtocolFactory> - F fromFile(Class clazz, File file) - { - List providers = PipelineService.get().getPipelineProviders(); - for (PipelineProvider provider : providers) - { - if (!(clazz.isInstance(provider))) - continue; - - T mprovider = (T) provider; - F factory = mprovider.getProtocolFactory(file); - if (factory != null) - return factory; - } - - // TODO: Return some default? - return null; - } - @Nullable - public AbstractFileAnalysisProtocol getProtocol(PipeRoot root, Path dirData, String protocolName, boolean archived) + public AbstractFileAnalysisProtocol getProtocol(PipeRoot root, FileLike dirData, String protocolName, boolean archived) { try { - Path protocolFile = getParametersFile(dirData, protocolName, root); + FileLike protocolFile = getParametersFile(dirData, protocolName, root); AbstractFileAnalysisProtocol result; if (NetworkDrive.exists(protocolFile)) { @@ -362,9 +332,16 @@ public AbstractFileAnalysisProtocol getProtocol(PipeRoot root, Path dirData, } else { - protocolFile = getProtocolFile(root, protocolName, archived); - if (protocolFile == null || !Files.exists(protocolFile)) + try + { + protocolFile = getProtocolFile(root, protocolName, archived); + if (protocolFile == null || !protocolFile.exists()) + return null; + } + catch (InvalidPathException e) + { return null; + } result = load(root, protocolName, archived); } diff --git a/api/src/org/labkey/api/pipeline/file/FileAnalysisJobSupport.java b/api/src/org/labkey/api/pipeline/file/FileAnalysisJobSupport.java index 52c9752230a..0a23361774e 100644 --- a/api/src/org/labkey/api/pipeline/file/FileAnalysisJobSupport.java +++ b/api/src/org/labkey/api/pipeline/file/FileAnalysisJobSupport.java @@ -19,8 +19,12 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.pipeline.ParamParser; import org.labkey.api.util.FileType; +import org.labkey.api.util.UnexpectedException; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -63,26 +67,26 @@ public interface FileAnalysisJobSupport /** * @return the directory in which the original input file resides. */ - @Deprecated //Prefer the getDataDirectoryPath version as File return type doesn't support full URIs very well - File getDataDirectory(); - default Path getDataDirectoryPath() + @Deprecated //Prefer the getDataDirectoryFileLike version as File return type doesn't support full URIs very well + default File getDataDirectory() { - // TODO This needs implementation in derived classes... - // This is typically safe but may cause an error if FileSystem provider isn't configured - return getDataDirectory().toPath(); + return getDataDirectoryFileLike().toNioPathForWrite().toFile(); } + FileLike getDataDirectoryFileLike(); + /** * @return the directory where the input files reside, and where the * final analysis should end up. */ - @Deprecated // Please use getAnalysisDirectoryPath instead, as File objects may have issues with full URIs + @Deprecated // Please use getAnalysisDirectoryFileLike File getAnalysisDirectory(); - default Path getAnalysisDirectoryPath() + + default FileLike getAnalysisDirectoryFileLike() { // TODO This needs implementation in derived classes... // This is typically safe but may cause an error if FileSystem provider isn't configured - return getAnalysisDirectory().toPath(); + return FileSystemLike.wrapFile(getAnalysisDirectory()); } /** @@ -90,14 +94,31 @@ default Path getAnalysisDirectoryPath() * This allows the task definitions to name files they require as input, * and the pipeline definition to specify where those files should come from. */ - @Deprecated // Please use findInputPath instead, as File objects may have issues with full URIs + @Deprecated // Please use findInputFileLike instead, as File objects may have issues with full URIs File findInputFile(String name); + @Deprecated // Please use findInputFileLike instead, as File objects may have issues with full URIs default Path findInputPath(String filepath) { // TODO This needs implementation in derived classes... // This is typically safe but may cause an error if FileSystem provider isn't configured return findInputFile(filepath).toPath(); } + default FileLike findInputFileLike(String filepath) + { + File file = findInputFile(filepath); + if (file != null) + { + try + { + return FileSystemLike.wrapFile(getDataDirectory(), file); + } + catch (IOException e) + { + throw UnexpectedException.wrap(e); + } + } + return null; + } /** * Returns a file for use as output in the pipeline, given its name. @@ -142,40 +163,12 @@ default Path findOutputPath(@NotNull String outputDir, @NotNull String filename) @Deprecated //Use Path based versions File getParametersFile(); - /** - * @return the job info file used to provide the external executable or script task with input file context. - */ - @Nullable - @Deprecated //Use Path based versions - File getJobInfoFile(); - /** * @return a list of all input files analyzed. */ @Deprecated List getInputFiles(); - - /** - * @return the parameters input file used to drive the pipeline. - */ - @Nullable - default Path getParametersFilePath() - { - //Implemented as such for backwards compatibility - return getParametersFile() == null ? null : getParametersFile().toPath(); - } - - /** - * @return the job info file used to provide the external executable or script task with input file context. - */ - @Nullable - default Path getJobInfoFilePath() - { - //Implemented as such for backwards compatibility - return getJobInfoFile() == null? null : getJobInfoFile().toPath(); - } - default List getInputFilePaths() { //Implemented as such for backwards compatibility diff --git a/api/src/org/labkey/api/pipeline/file/FileAnalysisTaskPipeline.java b/api/src/org/labkey/api/pipeline/file/FileAnalysisTaskPipeline.java index 04360c27cba..469338e5879 100644 --- a/api/src/org/labkey/api/pipeline/file/FileAnalysisTaskPipeline.java +++ b/api/src/org/labkey/api/pipeline/file/FileAnalysisTaskPipeline.java @@ -24,6 +24,7 @@ import org.labkey.api.util.FileType; import org.labkey.api.util.ReturnURLString; import org.labkey.api.util.URLHelper; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.FileFilter; @@ -31,6 +32,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.function.Predicate; /** * FileAnalysisTaskPipeline @@ -38,13 +40,19 @@ */ public interface FileAnalysisTaskPipeline extends TaskPipeline { - interface FilePathFilter extends FileFilter, DirectoryStream.Filter + interface FilePathFilter extends FileFilter, DirectoryStream.Filter, Predicate { @Override boolean accept(File file); @Override boolean accept(Path path); + + @Override + default boolean test(FileLike fileLike) + { + return accept(fileLike.toNioPathForRead()); + } } diff --git a/api/src/org/labkey/api/reports/report/r/RReportJob.java b/api/src/org/labkey/api/reports/report/r/RReportJob.java index 79c220902a5..8090a2bada0 100644 --- a/api/src/org/labkey/api/reports/report/r/RReportJob.java +++ b/api/src/org/labkey/api/reports/report/r/RReportJob.java @@ -100,7 +100,7 @@ protected void init(@NotNull String executingContainerId) { _jobIdentifier.set(getJobGUID()); FileLike logFile = report.getReportDirFileLike(executingContainerId).resolveChild(LOG_FILE_NAME); - setLogFile(logFile.toNioPathForWrite()); + setLogFile(logFile); } } diff --git a/api/src/org/labkey/api/study/SpecimenTransform.java b/api/src/org/labkey/api/study/SpecimenTransform.java index d00b7731f19..e62f3af113e 100644 --- a/api/src/org/labkey/api/study/SpecimenTransform.java +++ b/api/src/org/labkey/api/study/SpecimenTransform.java @@ -23,6 +23,7 @@ import org.labkey.api.security.User; import org.labkey.api.util.FileType; import org.labkey.api.view.ActionURL; +import org.labkey.vfs.FileLike; import java.io.File; import java.nio.file.Path; @@ -75,7 +76,7 @@ public interface SpecimenTransform * @param importConfig configuration object * @param inputArchive the file to write the externally sourced data into */ - void importFromExternalSource(@Nullable PipelineJob job, ExternalImportConfig importConfig, File inputArchive) throws PipelineJobException; + void importFromExternalSource(@Nullable PipelineJob job, ExternalImportConfig importConfig, FileLike inputArchive) throws PipelineJobException; interface ExternalImportConfig { diff --git a/api/src/org/labkey/api/util/FileType.java b/api/src/org/labkey/api/util/FileType.java index 680795aafaa..d874803027c 100644 --- a/api/src/org/labkey/api/util/FileType.java +++ b/api/src/org/labkey/api/util/FileType.java @@ -334,6 +334,11 @@ public String getName(File parentDir, String basename) return getName(parentDir.toPath(), basename); } + public String getName(FileLike parentDir, String basename) + { + return getName(parentDir.toNioPathForRead(), basename); + } + public String getName(Path parentDir, String basename) { if (_suffixes.size() > 1) @@ -430,6 +435,11 @@ public String getBaseName(File file) return getBaseName(file.toPath()); } + public String getBaseName(FileLike file) + { + return getBaseName(file.toNioPathForRead()); + } + public String getBaseName(@NotNull java.nio.file.Path file) { String fileName = file.getFileName().toString(); @@ -469,6 +479,11 @@ public File newFile(File parent, String basename) return FileUtil.appendName(parent, getName(parent, basename)); } + public FileLike newFile(FileLike parent, String basename) + { + return parent.resolveChild(getName(parent, basename)); + } + public Path newFile(Path parent, String basename) { return FileUtil.appendName(parent, getName(parent, basename)); @@ -657,15 +672,15 @@ public String toString() } @NotNull - public static List findTypes(@NotNull List types, @NotNull List files) + public static List findTypes(@NotNull List types, @NotNull List files) { ArrayList foundTypes = new ArrayList<>(); // This O(n*m), but these are usually very short lists. for (FileType type : types) { - for (Path file : files) + for (FileLike file : files) { - if (type.isType(file.getFileName().toString())) + if (type.isType(file.getName())) { foundTypes.add(type); break; diff --git a/api/src/org/labkey/api/util/FileUtil.java b/api/src/org/labkey/api/util/FileUtil.java index d7675c3092d..fd358042a19 100644 --- a/api/src/org/labkey/api/util/FileUtil.java +++ b/api/src/org/labkey/api/util/FileUtil.java @@ -33,9 +33,11 @@ import org.labkey.api.cloud.CloudStoreService; import org.labkey.api.data.Container; import org.labkey.api.files.FileContentService; +import org.labkey.api.pipeline.PipelineService; import org.labkey.api.security.Crypt; import org.labkey.api.settings.AppProps; import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.NotFoundException; import org.labkey.api.view.UnauthorizedException; import org.labkey.vfs.FileLike; import org.labkey.vfs.FileSystemLike; @@ -212,15 +214,24 @@ public static boolean deleteDir(File dir) return deleteDir(dir, null); } + public static boolean deleteDir(@NotNull FileLike dir) + { + return deleteDir(dir.toNioPathForWrite(), null); + } + + public static boolean deleteDir(@NotNull FileLike dir, @Nullable Logger log) + { + return deleteDir(dir.toNioPathForWrite(), log); + } @Deprecated - public static boolean deleteDir(@NotNull File dir, Logger log) + public static boolean deleteDir(@NotNull File dir, @Nullable Logger log) { return deleteDir(dir.toPath(), log); } - public static boolean deleteDir(Path dir, Logger log) + public static boolean deleteDir(@NotNull Path dir, @Nullable Logger log) { //TODO seems like this could be reworked to use Files.walkFileTree log = log == null ? LOG : log; @@ -450,6 +461,12 @@ public static boolean mkdirs(FileLike file, boolean checkFileName) throws IOExce } + public static FileLike createDirectory(FileLike path) throws IOException + { + createDirectory(path.toNioPathForWrite(), AppProps.getInstance().isInvalidFilenameBlocked()); + return path; + } + public static Path createDirectory(Path path) throws IOException { return createDirectory(path, AppProps.getInstance().isInvalidFilenameBlocked()); @@ -477,8 +494,7 @@ public static void createDirectories(FileLike file) throws IOException { if (!file.getFileSystem().canWriteFiles()) throw new UnauthorizedException(); - File target = toFileForWrite(file); - createDirectories(target.toPath(), AppProps.getInstance().isInvalidFilenameBlocked()); + createDirectories(file.toNioPathForWrite(), AppProps.getInstance().isInvalidFilenameBlocked()); } @@ -606,6 +622,11 @@ public static String getBaseName(File file, int dots) return getBaseName(file.getName(), dots); } + public static String getBaseName(FileLike file, int dots) + { + return getBaseName(file.toNioPathForRead().toFile(), dots); + } + /** * Remove text right of and including the last period in a file's name. @@ -617,6 +638,11 @@ public static String getBaseName(File file) return getBaseName(file, 1); } + public static String getBaseName(FileLike file) + { + return getBaseName(file, 1); + } + /** * Returns the file name extension without the dot, null if there @@ -926,7 +952,17 @@ public static Path stringToPath(Container container, String str) public static Path stringToPath(Container container, String str, boolean isEncoded) { if (!FileUtil.hasCloudScheme(str)) - return new File(createUri(str, isEncoded)).toPath(); + { + URI uri = createUri(str, isEncoded); + if (!uri.isAbsolute()) + { + return PipelineService.get().findPipelineRoot(container).resolveToNioPath(str); + } + else + { + return new File(uri).toPath(); + } + } else return Objects.requireNonNull(CloudStoreService.get()).getPathFromUrl(container, PageFlowUtil.decode(str)/*decode everything not just the space*/); } @@ -1085,6 +1121,16 @@ public static String matchPathLists(List home, List file) return path.toString(); } + public static void copyFile(FileLike src, FileLike dst) throws IOException + { + try (InputStream in = src.openInputStream(); + OutputStream out = dst.openOutputStream()) + { + copyData(in, out); + } + } + + public static void copyFile(File src, File dst) throws IOException { try (FileInputStream is = new FileInputStream(src); diff --git a/api/src/org/labkey/api/util/MaintenancePipelineJob.java b/api/src/org/labkey/api/util/MaintenancePipelineJob.java index eaee550dffe..ae0d00215ef 100644 --- a/api/src/org/labkey/api/util/MaintenancePipelineJob.java +++ b/api/src/org/labkey/api/util/MaintenancePipelineJob.java @@ -42,7 +42,7 @@ protected MaintenancePipelineJob(@JsonProperty("_tasks") Collection tasks) { super("SystemMaintenance", info, pipeRoot); - setLogFile(pipeRoot.getLogDirectoryFileLike(true).resolveChild(FileUtil.makeFileNameWithTimestamp("system_maintenance", "log")).toNioPathForWrite()); + setLogFile(pipeRoot.getLogDirectoryFileLike(true).resolveChild(FileUtil.makeFileNameWithTimestamp("system_maintenance", "log"))); _tasks = tasks; } diff --git a/study/src/org/labkey/study/writer/OutputStreamWrapper.java b/api/src/org/labkey/api/util/OutputStreamWrapper.java similarity index 79% rename from study/src/org/labkey/study/writer/OutputStreamWrapper.java rename to api/src/org/labkey/api/util/OutputStreamWrapper.java index 8506020f233..f076893ac67 100644 --- a/study/src/org/labkey/study/writer/OutputStreamWrapper.java +++ b/api/src/org/labkey/api/util/OutputStreamWrapper.java @@ -1,91 +1,89 @@ -/* - * Copyright (c) 2009-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.study.writer; - -import java.io.IOException; -import java.io.OutputStream; - -/* -* User: adam -* Date: Sep 5, 2009 -* Time: 6:30:12 PM -*/ - -// Allows overriding arbitrary OutputStreams. A little strange, since OutputStream is an abstract class not an interface... -public class OutputStreamWrapper extends OutputStream -{ - private final OutputStream _out; - - public OutputStreamWrapper(OutputStream out) - { - _out = out; - } - - @Override - public void write(int b) throws IOException - { - _out.write(b); - } - - @Override - public void write(byte[] b) throws IOException - { - _out.write(b); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException - { - _out.write(b, off, len); - } - - @Override - public void flush() throws IOException - { - _out.flush(); - } - - @Override - public void close() throws IOException - { - _out.close(); - } - - @Override - public int hashCode() - { - return _out.hashCode(); - } - - @Override - public boolean equals(Object obj) - { - return _out.equals(obj); - } - - @Override - protected Object clone() throws CloneNotSupportedException - { - throw new CloneNotSupportedException(); - } - - @Override - public String toString() - { - return _out.toString(); - } -} +/* + * Copyright (c) 2009-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.util; + +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Allows overriding arbitrary OutputStreams. A little strange, since OutputStream is an abstract class not an interface... + */ +public class OutputStreamWrapper extends OutputStream +{ + private final OutputStream _out; + + public OutputStreamWrapper(OutputStream out) + { + _out = out; + } + + @Override + public void write(int b) throws IOException + { + _out.write(b); + } + + @Override + public void write(byte @NotNull[] b) throws IOException + { + _out.write(b); + } + + @Override + public void write(byte @NotNull[] b, int off, int len) throws IOException + { + _out.write(b, off, len); + } + + @Override + public void flush() throws IOException + { + _out.flush(); + } + + @Override + public void close() throws IOException + { + _out.close(); + } + + @Override + public int hashCode() + { + return _out.hashCode(); + } + + @Override + public boolean equals(Object obj) + { + return _out.equals(obj); + } + + @Override + protected Object clone() throws CloneNotSupportedException + { + throw new CloneNotSupportedException(); + } + + @Override + public String toString() + { + return _out.toString(); + } +} diff --git a/api/src/org/labkey/api/util/Path.java b/api/src/org/labkey/api/util/Path.java index b7153dc8352..f317e471695 100644 --- a/api/src/org/labkey/api/util/Path.java +++ b/api/src/org/labkey/api/util/Path.java @@ -20,9 +20,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.commons.collections4.iterators.ArrayIterator; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.junit.Assert; import org.junit.Test; +import org.labkey.api.util.logging.LogHelper; import java.io.IOException; import java.io.Serializable; @@ -43,6 +45,8 @@ public class Path implements Serializable, Comparable, Iterable { + private static final Logger LOG = LogHelper.getLogger(Path.class, "Internal path parsing"); + final private boolean _caseSensitive; final private int _hash; final private String[] _path; @@ -90,6 +94,14 @@ protected Path(String[] path, int length, boolean abs, boolean dir, boolean case _isDirectory = dir; _hash = computeHash(_path, _length); _caseSensitive = caseSensitive; + + for (String s : path) + { + if (s != null && s.contains("\\")) + { + LOG.debug("Path contains backslash: {}", s, new Exception("Path contains backslash")); + } + } } // Create an instance from a java.nio.file.Path diff --git a/api/src/org/labkey/api/util/PossiblyGZIPpedFileInputStreamFactory.java b/api/src/org/labkey/api/util/PossiblyGZIPpedFileInputStreamFactory.java index f43864a88dc..ec063643fc1 100644 --- a/api/src/org/labkey/api/util/PossiblyGZIPpedFileInputStreamFactory.java +++ b/api/src/org/labkey/api/util/PossiblyGZIPpedFileInputStreamFactory.java @@ -15,9 +15,9 @@ */ package org.labkey.api.util; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; +import org.labkey.vfs.FileLike; + +import java.io.IOException; import java.io.InputStream; import java.util.zip.GZIPInputStream; @@ -36,9 +36,9 @@ abstract public class PossiblyGZIPpedFileInputStreamFactory { private static final int STREAM_BUFFER_SIZE = 128 * 1024; - static public InputStream getStream(File f) throws FileNotFoundException + static public InputStream getStream(FileLike f) throws IOException { - FileInputStream fis = new FileInputStream(f); + InputStream fis = f.openInputStream(); try { return new GZIPInputStream(fis, STREAM_BUFFER_SIZE); @@ -54,7 +54,7 @@ static public InputStream getStream(File f) throws FileNotFoundException { // seems unlikely at this point } - return new FileInputStream(f); + return f.openInputStream(); } } } diff --git a/api/src/org/labkey/api/writer/FileSystemFile.java b/api/src/org/labkey/api/writer/FileSystemFile.java index 298c2f0b42f..2ec36c79d15 100644 --- a/api/src/org/labkey/api/writer/FileSystemFile.java +++ b/api/src/org/labkey/api/writer/FileSystemFile.java @@ -23,6 +23,7 @@ import org.labkey.api.util.MinorConfigurationException; import org.labkey.api.util.XmlBeansUtil; import org.labkey.api.util.XmlValidationException; +import org.labkey.vfs.FileLike; import java.io.BufferedInputStream; import java.io.File; @@ -59,6 +60,11 @@ public FileSystemFile(File root) this(root.toPath()); } + public FileSystemFile(FileLike root) + { + this(root.toNioPathForWrite()); + } + public FileSystemFile(Path root) { try diff --git a/api/src/org/labkey/api/writer/ZipUtil.java b/api/src/org/labkey/api/writer/ZipUtil.java index f486ff24e3d..5bafbf1c8c4 100644 --- a/api/src/org/labkey/api/writer/ZipUtil.java +++ b/api/src/org/labkey/api/writer/ZipUtil.java @@ -23,6 +23,9 @@ import org.labkey.api.util.ResponseHelper; import jakarta.servlet.http.HttpServletResponse; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; + import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; @@ -46,16 +49,21 @@ */ public class ZipUtil { - // Unzip a zipped file archive to the specified directory - @Deprecated - public static List unzipToDirectory(File zipFile, File unzipDir) throws IOException + public static List unzipToDirectory(Path zipFile, Path unzipDir) throws IOException { - return unzipToDirectory(zipFile.toPath(), unzipDir.toPath()).stream().map(Path::toFile).collect(Collectors.toList()); + return unzipToDirectory(zipFile, unzipDir, null); } - public static List unzipToDirectory(Path zipFile, Path unzipDir) throws IOException + public static List unzipToDirectory(FileLike zipFile, FileLike unzipDir) throws IOException { - return unzipToDirectory(zipFile, unzipDir, null); + List paths = unzipToDirectory(zipFile.toNioPathForRead(), unzipDir.toNioPathForWrite(), null); + File rootFile = unzipDir.toNioPathForRead().toFile(); + List result = new ArrayList<>(); + for (Path path : paths) + { + result.add(FileSystemLike.wrapFile(rootFile, path.toFile())); + } + return result; } @Deprecated @@ -69,12 +77,6 @@ public static List unzipToDirectory(Path zipFile, Path unzipDir, @Nullable return unzipToDirectory(zipFile, unzipDir, log, false); } - @Deprecated - public static List unzipToDirectory(File zipFile, File unzipDir, @Nullable Logger log, boolean includeFolder) throws IOException - { - return unzipToDirectory(zipFile.toPath(), unzipDir.toPath(), log, includeFolder).stream().map(Path::toFile).collect(Collectors.toList()); - } - // Unzip an archive to the specified directory; log each file if Logger is non-null public static List unzipToDirectory(Path zipFile, Path unzipDir, @Nullable Logger log, boolean includeFolder) throws IOException { diff --git a/api/src/org/labkey/vfs/AbstractFileLike.java b/api/src/org/labkey/vfs/AbstractFileLike.java index 1d12f19baf0..be250592b04 100644 --- a/api/src/org/labkey/vfs/AbstractFileLike.java +++ b/api/src/org/labkey/vfs/AbstractFileLike.java @@ -1,10 +1,15 @@ package org.labkey.vfs; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import org.labkey.api.util.Path; import org.labkey.api.view.UnauthorizedException; import java.io.IOException; +@JsonSerialize(using = FileLike.FileLikeSerializer.class) +@JsonDeserialize(using = FileLike.FileLikeDeserializer.class) abstract public class AbstractFileLike implements FileLike { final Path path; @@ -115,4 +120,14 @@ public String toString() { return toURI().toString(); } + + protected void _serialize(JsonGenerator gen) throws IOException + { + FileSystemLike fs = getFileSystem(); + gen.writeStringField("fs", fs.getClass().getSimpleName()); + gen.writeStringField("rootUri", fs.getURI().toString()); + gen.writeBooleanField("canReadFiles", fs.canReadFiles()); + gen.writeBooleanField("canWriteFiles", fs.canWriteFiles()); + gen.writeStringField("path", getPath().toString()); + } } diff --git a/api/src/org/labkey/vfs/FileLike.java b/api/src/org/labkey/vfs/FileLike.java index 0466976cfd7..56854a42eb1 100644 --- a/api/src/org/labkey/vfs/FileLike.java +++ b/api/src/org/labkey/vfs/FileLike.java @@ -20,7 +20,9 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URI; +import java.nio.file.InvalidPathException; import java.util.List; +import java.util.function.Predicate; @JsonSerialize(using = FileLike.FileLikeSerializer.class) @JsonDeserialize(using = FileLike.FileLikeDeserializer.class) @@ -75,10 +77,10 @@ default FileLike resolveChild(Path.Part name) default FileLike resolveChild(String name) { if (".".equals(name) || "..".equals(name)) - throw new IllegalArgumentException("Cannot resolve child '" + name + "'"); + throw new InvalidPathException(name, "Cannot resolve child"); Path path = Path.parse(name); if (1 != path.size()) - throw new IllegalArgumentException("Cannot resolve child '" + name + "'"); + throw new InvalidPathException(name, "Cannot resolve child"); return resolveFile(path); } @@ -87,6 +89,12 @@ default FileLike resolveChild(String name) @NotNull List getChildren(); + @NotNull + default List getChildren(Predicate filter) + { + return getChildren().stream().filter(filter).toList(); + } + /** * Does not create parent directories */ @@ -119,43 +127,31 @@ default FileLike resolveChild(String name) InputStream openInputStream() throws IOException; - - class FileLikeSerializer extends StdSerializer + class FileLikeSerializer extends StdSerializer { public FileLikeSerializer() { this(null); } - public FileLikeSerializer(Class t) + public FileLikeSerializer(Class t) { super(t); } - public void _serialize(FileLike value, JsonGenerator gen, SerializerProvider provider) throws IOException - { - FileSystemLike fs = value.getFileSystem(); - gen.writeStringField("rootUri", fs.getURI().toString()); - gen.writeBooleanField("canReadFiles", fs.canReadFiles()); - gen.writeBooleanField("canWriteFiles", fs.canWriteFiles()); - gen.writeStringField("path", value.getPath().toString()); - if (fs instanceof FileSystemVFS) - gen.writeBooleanField("vfs", true); - } - @Override - public void serialize(FileLike value, JsonGenerator gen, SerializerProvider provider) throws IOException + public void serialize(AbstractFileLike value, JsonGenerator gen, SerializerProvider provider) throws IOException { gen.writeStartObject(); - _serialize(value, gen, provider); + value._serialize(gen); gen.writeEndObject(); } @Override - public void serializeWithType(FileLike value, JsonGenerator gen, SerializerProvider provider, TypeSerializer typeSer) throws IOException + public void serializeWithType(AbstractFileLike value, JsonGenerator gen, SerializerProvider provider, TypeSerializer typeSer) throws IOException { WritableTypeId typeIdDef = typeSer.writeTypePrefix(gen, typeSer.typeId(value, JsonToken.START_OBJECT)); - _serialize(value, gen, provider); + value._serialize(gen); typeSer.writeTypeSuffix(gen, typeIdDef); } } @@ -190,7 +186,12 @@ else if (canReadFiles) b.readonly(); if (vfs) b.vfs(); - return b.build().resolveFile(Path.parse(path)); + // for cloud config + if (node.has("containerId")) + b.container(node.get("containerId").asText()); + if (node.has("configName")) + b.config(node.get("configName").asText()); + return b.build(ctx).resolveFile(Path.parse(path)); } } } diff --git a/api/src/org/labkey/vfs/FileSystemLike.java b/api/src/org/labkey/vfs/FileSystemLike.java index 953c002553e..14d9d0edcdd 100644 --- a/api/src/org/labkey/vfs/FileSystemLike.java +++ b/api/src/org/labkey/vfs/FileSystemLike.java @@ -1,7 +1,10 @@ package org.labkey.vfs; +import com.fasterxml.jackson.databind.DeserializationContext; +import org.labkey.api.cloud.CloudStoreService; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; import org.labkey.api.pipeline.PipeRoot; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.util.FileUtil; @@ -116,6 +119,8 @@ class Builder boolean canDeleteRoot = false; boolean memCheck = true; boolean caching = false; + String containerId = null; + String configName = null; public Builder(URI uri) { @@ -170,14 +175,62 @@ public Builder noMemCheck() return this; } + public Builder container(String containerId) + { + this.containerId = containerId; + return this; + } + + public Builder config(String configName) + { + this.configName = configName; + return this; + } + + + public final String S3_SCHEME = "s3"; // S3FileSystemProvider.getScheme(); + + public FileSystemLike build(DeserializationContext ctx) + { + if (null != ctx && S3_SCHEME.equals(uri.getScheme())) + { + Map map; + if (null == (map = (Map) ctx.getAttribute(FileSystemLike.Builder.class.getName()))) + ctx.setAttribute(FileSystemLike.Builder.class.getName(), (map = new HashMap<>())); + Path fsKey = new Path(containerId, configName); + FileSystemLike cloud = map.get(fsKey); + if (null == cloud) + { + // cloud is always caching we don't care if caching values match + cloud = build(); + map.put(fsKey, cloud); + } + return cloud; + } + return build(); + } + public FileSystemLike build() { var scheme = defaultIfBlank(uri.getScheme(), FILE_SCHEME); + FileSystemLike ret; - if (defaultVfs || !FILE_SCHEME.equals(scheme)) + if (S3_SCHEME.equals(scheme)) + { + Container c = ContainerManager.getForId(containerId); + if (null == c) + throw new RuntimeException("Container not found: " + containerId); + ret = CloudStoreService.get().getFileSystemLike(c, configName); + } + else if (defaultVfs && !FILE_SCHEME.equals(scheme)) + { ret = new FileSystemVFS(uri, canReadFiles, canWriteFiles, canDeleteRoot); + } else + { ret = new FileSystemLocal(uri, canReadFiles, canWriteFiles, canDeleteRoot); + } + if (caching) ret = ret.getCachingFileSystem(); if (!memCheck) @@ -274,11 +327,15 @@ static Map wrapFiles(Map files) } /** + * Deprecated - stop passing around absolute paths through HTTP form submissions, and refactor to use + * pipeline root relative paths. + * * Verify that the provided path is within the Pipeline for the container and is usable as file * @param container scope and context * @param filePath to verify * @return A FileLike object representation of the provided file path relative to the container's pipeline root */ + @Deprecated static FileLike getVerifiedFileLike(Container container, String filePath) { if (filePath == null) diff --git a/api/src/org/labkey/vfs/FileSystemLocal.java b/api/src/org/labkey/vfs/FileSystemLocal.java index 9c878f56fe2..284b9173b29 100644 --- a/api/src/org/labkey/vfs/FileSystemLocal.java +++ b/api/src/org/labkey/vfs/FileSystemLocal.java @@ -2,10 +2,12 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.labkey.api.util.FileUtil; import org.labkey.api.util.Path; import org.labkey.api.util.UnexpectedException; +import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.UnauthorizedException; import java.io.File; @@ -29,6 +31,8 @@ public class FileSystemLocal extends AbstractFileSystemLike final _FileLike root; final boolean caching; + private static final Logger LOG = LogHelper.getLogger(FileSystemLocal.class, "Files backed by local storage"); + FileSystemLocal(URI uri, boolean canRead, boolean canWrite, boolean canDeleteRoot) { this(uri, false, canRead, canWrite, canDeleteRoot); @@ -71,6 +75,7 @@ public FileLike getRoot() @Override public FileLike resolveFile(Path path) { + LOG.debug("Resolving {} against {}", path, nioRoot); path = path.absolute().normalize(); if (null == path) throw new IllegalArgumentException("Path could not be resolved"); @@ -89,6 +94,11 @@ _FileLike createFileLike(Path path, File file) return new _FileLike(path, file); } + @Override + public String toString() + { + return "FileSystemLocal " + nioRoot + " caching=" + caching; + } @JsonSerialize(using = FileLike.FileLikeSerializer.class) @JsonDeserialize(using = FileLike.FileLikeDeserializer.class) diff --git a/api/src/org/labkey/vfs/FileSystemVFS.java b/api/src/org/labkey/vfs/FileSystemVFS.java index 641b4863283..71fe21c406d 100644 --- a/api/src/org/labkey/vfs/FileSystemVFS.java +++ b/api/src/org/labkey/vfs/FileSystemVFS.java @@ -1,5 +1,6 @@ package org.labkey.vfs; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import org.apache.commons.lang3.StringUtils; @@ -309,5 +310,12 @@ public boolean equals(Object obj) return false; return vfs.getName().equals(other.vfs.getName()); } + + @Override + public void _serialize(JsonGenerator gen) throws IOException + { + super._serialize(gen); + gen.writeBooleanField("vfs", true); + } } } diff --git a/assay/api-src/org/labkey/api/assay/plate/AbstractPlateBasedAssayProvider.java b/assay/api-src/org/labkey/api/assay/plate/AbstractPlateBasedAssayProvider.java index 1e97a198f6a..e6e1b184ef0 100644 --- a/assay/api-src/org/labkey/api/assay/plate/AbstractPlateBasedAssayProvider.java +++ b/assay/api-src/org/labkey/api/assay/plate/AbstractPlateBasedAssayProvider.java @@ -22,7 +22,6 @@ import org.labkey.api.assay.AbstractTsvAssayProvider; import org.labkey.api.assay.AssayDataType; import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayRunCreator; import org.labkey.api.assay.AssayRunUploadContext; import org.labkey.api.assay.AssayService; import org.labkey.api.assay.actions.PlateUploadForm; @@ -121,9 +120,9 @@ public List>> createDefaultDomains(Cont } @Override - public AssayRunCreator getRunCreator() + public PlateBasedRunCreator getRunCreator() { - return new PlateBasedRunCreator(this); + return new PlateBasedRunCreator<>(this); } @Override @@ -385,7 +384,7 @@ public boolean collectPropertyOnUpload(AssayRunUploadContext uploadContext, S } } - public static class CurveFitTableInfo extends EnumTableInfo + public static class CurveFitTableInfo extends EnumTableInfo { PlateBasedAssayProvider _provider; diff --git a/assay/api-src/org/labkey/api/assay/plate/PlateBasedAssayProvider.java b/assay/api-src/org/labkey/api/assay/plate/PlateBasedAssayProvider.java index 4c716f2c652..0dd120dc38b 100644 --- a/assay/api-src/org/labkey/api/assay/plate/PlateBasedAssayProvider.java +++ b/assay/api-src/org/labkey/api/assay/plate/PlateBasedAssayProvider.java @@ -27,7 +27,6 @@ import org.labkey.api.study.assay.SampleMetadataInputFormat; import org.labkey.vfs.FileLike; -import java.io.File; import java.util.Collection; /** diff --git a/assay/src/org/labkey/assay/AssayDomainServiceImpl.java b/assay/src/org/labkey/assay/AssayDomainServiceImpl.java index 0885a3dd071..e23579241b7 100644 --- a/assay/src/org/labkey/assay/AssayDomainServiceImpl.java +++ b/assay/src/org/labkey/assay/AssayDomainServiceImpl.java @@ -57,7 +57,6 @@ import org.labkey.api.gwt.client.model.GWTContainer; import org.labkey.api.gwt.client.model.GWTDomain; import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.gwt.server.BaseRemoteService; import org.labkey.api.query.MetadataUnavailableException; import org.labkey.api.query.QueryService; import org.labkey.api.query.ValidationException; @@ -71,7 +70,7 @@ import org.labkey.api.util.UnexpectedException; import org.labkey.api.view.ActionURL; import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.ViewContext; +import org.labkey.api.writer.ContainerUser; import org.labkey.assay.actions.SetDefaultValuesAssayAction; import org.labkey.assay.plate.PlateManager; import org.labkey.assay.query.AssayDbSchema; @@ -86,13 +85,28 @@ import java.util.Map; import java.util.Set; -public class AssayDomainServiceImpl extends BaseRemoteService implements AssayDomainService +public class AssayDomainServiceImpl implements AssayDomainService, ContainerUser { public static final Logger LOG = LogManager.getLogger(AssayDomainServiceImpl.class); + private final User _user; + private final Container _container; - public AssayDomainServiceImpl(ViewContext context) + public AssayDomainServiceImpl(User user, Container container) { - super(context); + _user = user; + _container = container; + } + + @Override + public User getUser() + { + return _user; + } + + @Override + public Container getContainer() + { + return _container; } @Override diff --git a/assay/src/org/labkey/assay/AssayIntegrationTestCase.jsp b/assay/src/org/labkey/assay/AssayIntegrationTestCase.jsp index d66cbfd7755..bf601732f18 100644 --- a/assay/src/org/labkey/assay/AssayIntegrationTestCase.jsp +++ b/assay/src/org/labkey/assay/AssayIntegrationTestCase.jsp @@ -137,7 +137,7 @@ private Pair createAssay(ViewContext context, boolean editableRunsAndResults, boolean includeSampleTypeLookups) throws Exception { // create assay design - AssayDomainService assayDomainService = new AssayDomainServiceImpl(context); + AssayDomainService assayDomainService = new AssayDomainServiceImpl(context.getUser(), context.getContainer()); GWTProtocol assayTemplate = assayDomainService.getAssayTemplate("General"); assayTemplate.setName(ASSAY_NAME); assayTemplate.setEditableRuns(true); @@ -721,7 +721,7 @@ var sampleTypeName = "Does Not Exist"; var lookupName = "Field100 ABCDEFGHIJKLMNOPQRSTUVWXYZ%()=+-[]_|*`'\":;<>?!@#^AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTU)"; var assayPair = createAssay(context, true, true); - var domainService = new AssayDomainServiceImpl(context); + var domainService = new AssayDomainServiceImpl(context.getUser(), context.getContainer()); var gwtProtocol = domainService.getAssayDefinition(assayPair.second.getRowId(), false); // Update the assay design to refer to a non-existent sample type diff --git a/assay/src/org/labkey/assay/AssayManager.java b/assay/src/org/labkey/assay/AssayManager.java index 0ac76d4dca4..5c901c8c27c 100644 --- a/assay/src/org/labkey/assay/AssayManager.java +++ b/assay/src/org/labkey/assay/AssayManager.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.assay.AbstractAssayProvider; import org.labkey.api.assay.AssayColumnInfoRenderer; +import org.labkey.api.assay.AssayDomainService; import org.labkey.api.assay.AssayFileWriter; import org.labkey.api.assay.AssayFlagHandler; import org.labkey.api.assay.AssayHeaderLinkProvider; @@ -681,11 +682,6 @@ public void indexAssay(SearchService.TaskIndexingQueue queue, ExpProtocol protoc queue.addResource(r); } - private boolean shouldIndexBatch(ExpExperiment batch) - { - return batch != null && shouldIndexProtocolBatches(batch.getBatchProtocol()); - } - /** * Only index batches for assays that have batch properties on their domain */ @@ -1095,4 +1091,10 @@ List getRegisteredAssayProviders() // result in lineage inputs which we are unable to disambiguate when processing based on "role". return dp.getName(); } + + @Override + public AssayDomainService createAssayDomainService(User user, Container container) + { + return new AssayDomainServiceImpl(user, container); + } } diff --git a/assay/src/org/labkey/assay/actions/GetProtocolAction.java b/assay/src/org/labkey/assay/actions/GetProtocolAction.java index 83799fc419d..8846544911e 100644 --- a/assay/src/org/labkey/assay/actions/GetProtocolAction.java +++ b/assay/src/org/labkey/assay/actions/GetProtocolAction.java @@ -83,7 +83,7 @@ public Object execute(DesignerForm form, BindException errors) throws Exception } else if (expProtocol.getContainer().hasPermission(getUser(), ReadPermission.class)) { - AssayDomainService svc = new AssayDomainServiceImpl(getViewContext()); + AssayDomainService svc = new AssayDomainServiceImpl(getViewContext().getUser(), getViewContext().getContainer()); GWTProtocol ret = svc.getAssayDefinition(form.getProtocolId(), form.isCopy()); if (ret == null) { @@ -99,7 +99,7 @@ else if (expProtocol.getContainer().hasPermission(getUser(), ReadPermission.clas else if (form.getProviderName() != null) { // get the assay template - AssayDomainService svc = new AssayDomainServiceImpl(getViewContext()); + AssayDomainService svc = new AssayDomainServiceImpl(getViewContext().getUser(), getViewContext().getContainer()); GWTProtocol ret = svc.getAssayTemplate(form.getProviderName()); return success("Generated assay template for provider '" + form.getProviderName() + "'", ret); } diff --git a/assay/src/org/labkey/assay/actions/SaveProtocolAction.java b/assay/src/org/labkey/assay/actions/SaveProtocolAction.java index 4f60003af4e..4721d83058c 100644 --- a/assay/src/org/labkey/assay/actions/SaveProtocolAction.java +++ b/assay/src/org/labkey/assay/actions/SaveProtocolAction.java @@ -59,7 +59,7 @@ public Object execute(GWTProtocol protocol, BindException errors) throws Excepti if (protocol.getName() != null) protocol.setName(protocol.getName().trim()); - AssayDomainService svc = new AssayDomainServiceImpl(getViewContext()); + AssayDomainService svc = new AssayDomainServiceImpl(getViewContext().getUser(), getViewContext().getContainer()); GWTProtocol updated = svc.saveChanges(protocol, true); return success((isNew ? "Created" : "Updated") + " assay protocol '" + updated.getName() + "'", updated); } diff --git a/assay/src/org/labkey/assay/data/generator/AssayDesignGenerator.java b/assay/src/org/labkey/assay/data/generator/AssayDesignGenerator.java index 445b7f6a361..5b1102e114f 100644 --- a/assay/src/org/labkey/assay/data/generator/AssayDesignGenerator.java +++ b/assay/src/org/labkey/assay/data/generator/AssayDesignGenerator.java @@ -11,8 +11,6 @@ import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.query.ValidationException; import org.labkey.api.util.CPUTimer; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.api.view.ViewContext; import org.labkey.assay.AssayDomainServiceImpl; import org.labkey.assay.AssayManager; @@ -61,7 +59,7 @@ public void generateAssayDesigns(String namePrefix) throws ValidationException, private void createStandardAssayDesign(String name) throws ValidationException { // create assay design - AssayDomainService assayDomainService = new AssayDomainServiceImpl(new ViewContext(new ViewBackgroundInfo(getContainer(), getUser(), null))); + AssayDomainService assayDomainService = new AssayDomainServiceImpl(getUser(), getContainer()); GWTProtocol assayTemplate = assayDomainService.getAssayTemplate("General"); assayTemplate.setName(name); diff --git a/assay/test/src/org/labkey/test/tests/assay/AssayReimportIndexTest.java b/assay/test/src/org/labkey/test/tests/assay/AssayReimportIndexTest.java index 856102368f7..8c5db2ba1a2 100644 --- a/assay/test/src/org/labkey/test/tests/assay/AssayReimportIndexTest.java +++ b/assay/test/src/org/labkey/test/tests/assay/AssayReimportIndexTest.java @@ -8,6 +8,7 @@ import org.labkey.test.Locator; import org.labkey.test.categories.Assays; import org.labkey.test.categories.Daily; +import org.labkey.test.components.assay.AssayConstants; import org.labkey.test.pages.assay.AssayImportPage; import org.labkey.test.pages.assay.AssayRunsPage; import org.labkey.test.params.FieldDefinition; @@ -73,7 +74,7 @@ public void testIndexLatestAssayRun() clickButton("Next"); AssayImportPage importPage = new AssayImportPage(getDriver()); importPage.setNamedInputText("Name", firstRun); - importPage.setNamedTextAreaValue("TextAreaDataCollector.textArea", firstRunData); + importPage.setNamedTextAreaValue(AssayConstants.TEXT_AREA_DATA_COLLECTOR_TEXT_AREA_NAME, firstRunData); importPage.clickSaveAndFinish(); SearchAdminAPIHelper.waitForIndexer(); @@ -92,7 +93,7 @@ public void testIndexLatestAssayRun() clickButton("Next"); AssayImportPage importPage2 = new AssayImportPage(getDriver()); importPage2.setNamedInputText("Name", secondRun); - importPage2.setNamedTextAreaValue("TextAreaDataCollector.textArea", secondRunData); + importPage2.setNamedTextAreaValue(AssayConstants.TEXT_AREA_DATA_COLLECTOR_TEXT_AREA_NAME, secondRunData); importPage2.clickSaveAndFinish(); SearchAdminAPIHelper.waitForIndexer(); diff --git a/core/src/org/labkey/core/CoreController.java b/core/src/org/labkey/core/CoreController.java index 96068322b9a..5837de0c470 100644 --- a/core/src/org/labkey/core/CoreController.java +++ b/core/src/org/labkey/core/CoreController.java @@ -1936,7 +1936,8 @@ private List> getCloudArchiveImporters(FolderImporterForm fo private boolean isCloudArchive(FolderImporterForm form) { - return FileUtil.hasCloudScheme(form.getArchiveFilePath()); + String path = form.getArchiveFilePath(); + return StringUtils.isNotBlank(path) && FileUtil.hasCloudScheme(form.getArchiveFilePath()); } private List> getSelectableImporters(FolderImporterForm form, List registeredImporters) throws Exception diff --git a/core/src/org/labkey/core/admin/AdminController.java b/core/src/org/labkey/core/admin/AdminController.java index e993f7dbdf4..52a2c70875a 100644 --- a/core/src/org/labkey/core/admin/AdminController.java +++ b/core/src/org/labkey/core/admin/AdminController.java @@ -333,6 +333,7 @@ import org.labkey.data.xml.TablesDocument; import org.labkey.filters.ContentSecurityPolicyFilter; import org.labkey.security.xml.GroupEnumType; +import org.labkey.vfs.FileLike; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.validation.BindException; import org.springframework.validation.Errors; @@ -5325,7 +5326,7 @@ public boolean handlePost(ImportFolderForm form, BindException errors) throws Ex User user = getUser(); Container container = getContainer(); PipeRoot pipelineRoot; - Path pipelineUnzipDir; // Should be local & writable + FileLike pipelineUnzipDir; // Should be local & writable PipelineUrls pipelineUrlProvider; if (form.getOrigin() == null) @@ -5375,8 +5376,8 @@ public boolean handlePost(ImportFolderForm form, BindException errors) throws Ex } // get the folder.xml file from the unzipped import archive - Path archiveXml = pipelineUnzipDir.resolve("folder.xml"); - if (!Files.exists(archiveXml)) + FileLike archiveXml = pipelineUnzipDir.resolveChild("folder.xml"); + if (!archiveXml.exists()) { errors.reject("folderImport", "This archive doesn't contain a folder.xml file."); return false; @@ -5395,7 +5396,7 @@ public boolean handlePost(ImportFolderForm form, BindException errors) throws Ex return !errors.hasErrors(); } - private @Nullable FolderImportConfig getFolderFromZipArchive(Path pipelineUnzipDir, BindException errors) + private @Nullable FolderImportConfig getFolderFromZipArchive(FileLike pipelineUnzipDir, BindException errors) { // user chose to import from a zip file Map map = getFileMap(); @@ -5419,17 +5420,17 @@ public boolean handlePost(ImportFolderForm form, BindException errors) throws Ex // copy and unzip the uploaded import archive zip file to the pipeline unzip dir try { - Path pipelineUnzipFile = pipelineUnzipDir.resolve(originalFilename); + FileLike pipelineUnzipFile = pipelineUnzipDir.resolveFile(org.labkey.api.util.Path.parse(originalFilename)); // Check that the resolved file is under the pipelineUnzipDir - if (!pipelineUnzipFile.normalize().startsWith(pipelineUnzipDir.normalize())) + if (!pipelineUnzipFile.toNioPathForRead().normalize().startsWith(pipelineUnzipDir.toNioPathForRead().normalize())) { errors.reject("folderImport", "Invalid file path - must be within the unzip directory"); return null; } FileUtil.createDirectories(pipelineUnzipFile.getParent()); // Non-pipeline import sometimes fails here on Windows (shrug) - FileUtil.createFile(pipelineUnzipFile); - try (OutputStream os = Files.newOutputStream(pipelineUnzipFile)) + FileUtil.createNewFile(pipelineUnzipFile, true); + try (OutputStream os = pipelineUnzipFile.openOutputStream()) { FileUtil.copyData(zipFile.getInputStream(), os); } @@ -5456,7 +5457,7 @@ public boolean handlePost(ImportFolderForm form, BindException errors) throws Ex } } - private FolderImportConfig getFolderImportConfigFromTemplateFolder(final ImportFolderForm form, final Path pipelineUnzipDir, final BindException errors) throws Exception + private FolderImportConfig getFolderImportConfigFromTemplateFolder(final ImportFolderForm form, final FileLike pipelineUnzipDir, final BindException errors) throws Exception { // user choose to import from a template source folder Container sourceContainer = form.getSourceTemplateFolderContainer(); @@ -5468,7 +5469,12 @@ private FolderImportConfig getFolderImportConfigFromTemplateFolder(final ImportF PHI.NotPHI, false, false, false, new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); FolderWriterImpl writer = new FolderWriterImpl(); String zipFileName = FileUtil.makeFileNameWithTimestamp(sourceContainer.getName(), "folder.zip"); - try (ZipFile zip = new ZipFile(pipelineUnzipDir, zipFileName)) + FileLike implicitZipFile = pipelineUnzipDir.resolveChild(zipFileName); + if (!pipelineUnzipDir.isDirectory()) + pipelineUnzipDir.mkdirs(); + implicitZipFile.createFile(); + try (OutputStream out = implicitZipFile.openOutputStream(); + ZipFile zip = new ZipFile(out, false)) { writer.write(sourceContainer, ctx, zip); } @@ -5476,26 +5482,25 @@ private FolderImportConfig getFolderImportConfigFromTemplateFolder(final ImportF { errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); } - Path implicitZipFile = pipelineUnzipDir.resolve(zipFileName); // To support the simple import option unzip the zip file to the pipeline unzip dir of the current container ZipUtil.unzipToDirectory(implicitZipFile, pipelineUnzipDir); return new FolderImportConfig( StringUtils.isNotEmpty(form.getSourceTemplateFolderId()), - implicitZipFile.getFileName().toString(), + implicitZipFile.getName(), implicitZipFile, null ); } private static class FolderImportConfig { - Path pipelineUnzipFile; + FileLike pipelineUnzipFile; String originalFileName; - Path archiveFile; + FileLike archiveFile; boolean fromTemplateSourceFolder; - public FolderImportConfig(boolean fromTemplateSourceFolder, String originalFileName, Path archiveFile, @Nullable Path pipelineUnzipFile) + public FolderImportConfig(boolean fromTemplateSourceFolder, String originalFileName, FileLike archiveFile, @Nullable FileLike pipelineUnzipFile) { this.originalFileName = originalFileName; this.archiveFile = archiveFile; diff --git a/core/src/org/labkey/core/admin/CopyFileRootPipelineJob.java b/core/src/org/labkey/core/admin/CopyFileRootPipelineJob.java index 30bdc3d62f2..155f3816c41 100644 --- a/core/src/org/labkey/core/admin/CopyFileRootPipelineJob.java +++ b/core/src/org/labkey/core/admin/CopyFileRootPipelineJob.java @@ -41,7 +41,6 @@ import org.labkey.api.util.TestContext; import org.labkey.api.util.URLHelper; import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.core.CoreModule; import org.labkey.core.admin.AdminController.MigrateFilesOption; import java.io.FileNotFoundException; @@ -84,7 +83,7 @@ protected CopyFileRootPipelineJob( String baseLogFileName = FileUtil.makeFileNameWithTimestamp( FileUtil.getBaseName("copy_directory_fileroot_change", 1).replace(" ", "_")); - setupLocalDirectoryAndJobLog(pipeRoot, CoreModule.CORE_MODULE_NAME, baseLogFileName); + setupLocalDirectoryAndJobLog(pipeRoot, baseLogFileName); } @Override diff --git a/core/src/org/labkey/core/admin/ValidateDomainsPipelineJob.java b/core/src/org/labkey/core/admin/ValidateDomainsPipelineJob.java index eff12e58b64..6cdc7d51b3f 100644 --- a/core/src/org/labkey/core/admin/ValidateDomainsPipelineJob.java +++ b/core/src/org/labkey/core/admin/ValidateDomainsPipelineJob.java @@ -23,8 +23,8 @@ import org.labkey.api.util.URLHelper; import org.labkey.api.util.UnexpectedException; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; /** @@ -43,7 +43,7 @@ public ValidateDomainsPipelineJob(ViewBackgroundInfo info, PipeRoot root) try { - File logFile = FileUtil.createTempFile("validateDomains", ".log", root.ensureSystemDirectory()); + FileLike logFile = FileUtil.createTempFile("validateDomains", ".log", root.ensureSystemDirectory()); setLogFile(logFile); } catch (IOException e) diff --git a/experiment/src/org/labkey/experiment/CompressedInputStreamXarSource.java b/experiment/src/org/labkey/experiment/CompressedInputStreamXarSource.java index 485247f6930..58b2572578e 100644 --- a/experiment/src/org/labkey/experiment/CompressedInputStreamXarSource.java +++ b/experiment/src/org/labkey/experiment/CompressedInputStreamXarSource.java @@ -10,6 +10,7 @@ import org.labkey.api.security.User; import org.labkey.api.util.FileUtil; import org.labkey.api.util.XmlBeansUtil; +import org.labkey.vfs.FileLike; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; @@ -19,7 +20,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Path; import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -32,10 +32,10 @@ public class CompressedInputStreamXarSource extends AbstractFileXarSource { private final InputStream _xarInputStream; - private final Path _logFile; + private final FileLike _logFile; private String _xml; - public CompressedInputStreamXarSource(InputStream xarInputStream, Path xarFile, Path logFile, @Nullable PipelineJob job, User user, Container container, @Nullable Map substitutions) + public CompressedInputStreamXarSource(InputStream xarInputStream, FileLike xarFile, FileLike logFile, @Nullable PipelineJob job, User user, Container container, @Nullable Map substitutions) { super(job == null ? null : job.getDescription(), container, user, job, substitutions); _xarInputStream = xarInputStream; @@ -86,7 +86,7 @@ public ExperimentArchiveDocument getDocument() throws XmlException, IOException } @Override - public Path getLogFilePath() + public FileLike getLogFilePath() { return _logFile; } diff --git a/experiment/src/org/labkey/experiment/ScriptsResourceProvider.java b/experiment/src/org/labkey/experiment/ScriptsResourceProvider.java index 0ec70c020ab..910bff397f1 100644 --- a/experiment/src/org/labkey/experiment/ScriptsResourceProvider.java +++ b/experiment/src/org/labkey/experiment/ScriptsResourceProvider.java @@ -40,7 +40,7 @@ public class ScriptsResourceProvider implements WebdavService.Provider java.nio.file.Path root = svc.getFileRootPath(c); if (root != null) { - if (!FileUtil.hasCloudScheme(root) && NetworkDrive.exists(root.toFile())) + if (!FileUtil.hasCloudScheme(root) && NetworkDrive.exists(root)) { result.add(FileContentService.SCRIPTS_LINK); } diff --git a/experiment/src/org/labkey/experiment/XarExportPipelineJob.java b/experiment/src/org/labkey/experiment/XarExportPipelineJob.java index bec82562dfc..660d6bc1ef2 100644 --- a/experiment/src/org/labkey/experiment/XarExportPipelineJob.java +++ b/experiment/src/org/labkey/experiment/XarExportPipelineJob.java @@ -73,7 +73,7 @@ public XarExportPipelineJob(ViewBackgroundInfo info, PipeRoot root, String fileN _exportFile = exportedXarsDir.resolveChild(_fileName).toNioPathForWrite().toFile(); - setLogFile(exportedXarsDir.resolveChild(fileName + ".log").toNioPathForWrite()); + setLogFile(exportedXarsDir.resolveChild(fileName + ".log")); header("Experiment export to " + _exportFile.getName()); } diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index f3d51bf42b7..c4d81da28ee 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -8164,9 +8164,9 @@ public List getExpProtocolsWithParameterValue( } @Override - public PipelineJob importXarAsync(ViewBackgroundInfo info, File file, String description, PipeRoot root) throws IOException + public PipelineJob importXarAsync(ViewBackgroundInfo info, FileLike file, String description, PipeRoot root) throws IOException { - ExperimentPipelineJob job = new ExperimentPipelineJob(info, file.toPath(), description, false, root); + ExperimentPipelineJob job = new ExperimentPipelineJob(info, file, description, false, root); try { PipelineService.get().queueJob(job); diff --git a/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java b/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java index 00644fd3577..bd3c8e9359b 100644 --- a/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java +++ b/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java @@ -23,9 +23,9 @@ import jakarta.servlet.http.HttpSession; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.apache.poi.ss.usermodel.Workbook; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -4969,7 +4969,7 @@ private ActionURL exportXAR(@NotNull XarExportSelection selection, @Nullable LSI fileName = fixupExportName(fileName); String xarXmlFileName = null; - if (StringUtils.endsWithIgnoreCase(fileName, ".xar")) + if (Strings.CI.endsWith(fileName, ".xar")) xarXmlFileName = fileName + ".xml"; switch (exportType) @@ -6731,10 +6731,10 @@ public Object execute(Object o, BindException errors) throws Exception } PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(getContainer()); - Path systemDir = pipeRoot.ensureSystemDirectoryPath(); - Path uploadDir = systemDir.resolve("UploadedXARs"); + FileLike systemDir = pipeRoot.ensureSystemDirectory(); + FileLike uploadDir = systemDir.resolveChild("UploadedXARs"); FileUtil.createDirectories(uploadDir); - if (!Files.isDirectory(uploadDir)) + if (!uploadDir.isDirectory()) { errors.reject(ERROR_MSG, "Unable to create a 'system/UploadedXARs' directory under the pipeline root"); return false; @@ -6744,18 +6744,18 @@ public Object execute(Object o, BindException errors) throws Exception { userDirName = GUEST_DIRECTORY_NAME; } - Path userDir = FileUtil.appendName(uploadDir, userDirName); + FileLike userDir = uploadDir.resolveChild(userDirName); FileUtil.createDirectories(userDir); - if (!Files.isDirectory(userDir)) + if (!userDir.isDirectory()) { errors.reject(ERROR_MSG, "Unable to create an 'UploadedXARs/" + userDirName + "' directory under the pipeline root"); return false; } - Path xarFile = FileUtil.appendName(userDir, formFile.getOriginalFilename()); + FileLike xarFile = userDir.resolveChild(formFile.getOriginalFilename()); // As this is multi-part will need to use finally to close, to prevent a stream closure exception - try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(xarFile))) + try (OutputStream out = new BufferedOutputStream(xarFile.openOutputStream())) { out.write(bytes); } @@ -6786,17 +6786,17 @@ public void validateCommand(ImportXarForm target, Errors errors) @Override public boolean handlePost(ImportXarForm form, BindException errors) throws Exception { - for (File f : form.getValidatedFiles(getContainer())) + for (FileLike f : form.getValidatedFiles(getContainer())) { if (f.isFile()) { - ExperimentPipelineJob job = new ExperimentPipelineJob(getViewBackgroundInfo(), f.toPath(), "Experiment Import", false, form.getPipeRoot(getContainer())); + ExperimentPipelineJob job = new ExperimentPipelineJob(getViewBackgroundInfo(), f, "Experiment Import", false, form.getPipeRoot(getContainer())); // TODO: Configure module resources with the appropriate log location per container if (form.getModule() != null) { FileLike logFile = form.getPipeRoot(getContainer()).getLogDirectoryFileLike(true).resolveChild("module-resource-xar.log"); - job.setLogFile(logFile.toNioPathForWrite()); + job.setLogFile(logFile); } PipelineService.get().queueJob(job); @@ -6827,16 +6827,16 @@ public Object execute(ImportXarForm form, BindException errors) throws Exception ApiSimpleResponse response = new ApiSimpleResponse(); List> archives = new ArrayList<>(); - for (File f : form.getValidatedFiles(getContainer())) + for (FileLike f : form.getValidatedFiles(getContainer())) { Map archive = new HashMap<>(); - ExperimentPipelineJob job = new ExperimentPipelineJob(getViewBackgroundInfo(), f.toPath(), "Experiment Import", false, form.getPipeRoot(getContainer())); + ExperimentPipelineJob job = new ExperimentPipelineJob(getViewBackgroundInfo(), f, "Experiment Import", false, form.getPipeRoot(getContainer())); // TODO: Configure module resources with the appropriate log location per container if (form.getModule() != null) { FileLike logFile = form.getPipeRoot(getContainer()).getLogDirectoryFileLike(true).resolveChild("module-resource-xar.log"); - job.setLogFile(logFile.toNioPathForWrite()); + job.setLogFile(logFile); } PipelineService.get().queueJob(job); diff --git a/experiment/src/org/labkey/experiment/controllers/exp/ImportXarForm.java b/experiment/src/org/labkey/experiment/controllers/exp/ImportXarForm.java index f0ca5ddd765..306971e4a7c 100644 --- a/experiment/src/org/labkey/experiment/controllers/exp/ImportXarForm.java +++ b/experiment/src/org/labkey/experiment/controllers/exp/ImportXarForm.java @@ -23,6 +23,8 @@ import org.labkey.api.resource.Resource; import org.labkey.api.util.NetworkDrive; import org.labkey.api.view.NotFoundException; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.File; import java.util.ArrayList; @@ -51,7 +53,7 @@ public void setModule(String module) * default pipeline xar processing. */ @Override - public List getValidatedFiles(Container c, boolean allowNonExistentFiles) + public List getValidatedFiles(Container c, boolean allowNonExistentFiles) { if (_module == null) return super.getValidatedFiles(c, allowNonExistentFiles); @@ -70,7 +72,7 @@ public List getValidatedFiles(Container c, boolean allowNonExistentFiles) throw new NotFoundException("Could not find path " + getPath()); } - List files = new ArrayList<>(); + List files = new ArrayList<>(); for (String fileName : getFile()) { Resource rf = m.getModuleResource(getPath() + "/" + fileName); @@ -85,7 +87,7 @@ public List getValidatedFiles(Container c, boolean allowNonExistentFiles) { throw new NotFoundException("Could not find file '" + f + "'"); } - files.add(f); + files.add(FileSystemLike.wrapFile(f)); } return files; diff --git a/experiment/src/org/labkey/experiment/pipeline/ExperimentPipelineJob.java b/experiment/src/org/labkey/experiment/pipeline/ExperimentPipelineJob.java index 9df4b071384..1571ec5d2f2 100644 --- a/experiment/src/org/labkey/experiment/pipeline/ExperimentPipelineJob.java +++ b/experiment/src/org/labkey/experiment/pipeline/ExperimentPipelineJob.java @@ -31,10 +31,9 @@ import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.sql.BatchUpdateException; import java.util.List; @@ -48,14 +47,14 @@ public class ExperimentPipelineJob extends PipelineJob private static final Object _experimentLock = new Object(); - private final Path _xarFile; + private final FileLike _xarFile; private final String _description; private final boolean _deleteExistingRuns; private transient XarSource _xarSource; @JsonCreator - protected ExperimentPipelineJob(@JsonProperty("_xarFile") Path xarFile, + protected ExperimentPipelineJob(@JsonProperty("_xarFile") FileLike xarFile, @JsonProperty("_description") String description, @JsonProperty("_deleteExistingRuns") boolean deleteExistingRuns) { @@ -65,26 +64,20 @@ protected ExperimentPipelineJob(@JsonProperty("_xarFile") Path xarFile, _deleteExistingRuns = deleteExistingRuns; } - @Deprecated //Prefer the Path version - public ExperimentPipelineJob(ViewBackgroundInfo info, File file, String description, boolean deleteExistingRuns, PipeRoot root) throws IOException - { - this(info, file.toPath(), description, deleteExistingRuns, root); - } - - public ExperimentPipelineJob(ViewBackgroundInfo info, Path file, String description, boolean deleteExistingRuns, PipeRoot root) throws IOException + public ExperimentPipelineJob(ViewBackgroundInfo info, FileLike file, String description, boolean deleteExistingRuns, PipeRoot root) throws IOException { super(ExperimentPipelineProvider.NAME, info, root); _xarFile = file; - _description = description + " - " + file.getFileName().toString(); + _description = description + " - " + file.getName(); _deleteExistingRuns = deleteExistingRuns; XarSource xarSource = getXarSource(); header("XAR Import from " + xarSource.toString()); } - protected XarSource createXarSource(Path file) + protected XarSource createXarSource(FileLike file) { - String name = file.getFileName().toString().toLowerCase(); + String name = file.getName().toLowerCase(); if (name.endsWith(".xar") || name.endsWith(".zip")) { return new CompressedXarSource(file, this); diff --git a/experiment/src/org/labkey/experiment/pipeline/ExperimentPipelineProvider.java b/experiment/src/org/labkey/experiment/pipeline/ExperimentPipelineProvider.java index 4d797234e81..307413bbb58 100644 --- a/experiment/src/org/labkey/experiment/pipeline/ExperimentPipelineProvider.java +++ b/experiment/src/org/labkey/experiment/pipeline/ExperimentPipelineProvider.java @@ -49,7 +49,7 @@ public static Path getMoveDirectory(PipeRoot pr) private static Path getExperimentDirectory(PipeRoot pr, String name) { - Path systemDir = pr.ensureSystemDirectoryPath(); + Path systemDir = pr.ensureSystemDirectory().toNioPathForRead(); return systemDir.resolve(DIR_NAME_EXPERIMENT).resolve(name); } diff --git a/experiment/src/org/labkey/experiment/pipeline/MoveRunsPipelineJob.java b/experiment/src/org/labkey/experiment/pipeline/MoveRunsPipelineJob.java index f19b153b332..9ede8686a9d 100644 --- a/experiment/src/org/labkey/experiment/pipeline/MoveRunsPipelineJob.java +++ b/experiment/src/org/labkey/experiment/pipeline/MoveRunsPipelineJob.java @@ -57,7 +57,7 @@ public MoveRunsPipelineJob(ViewBackgroundInfo info, Container sourceContainer, l _sourceContainer = sourceContainer; String baseLogFileName = FileUtil.makeFileNameWithTimestamp("moveRun", ".log"); - setupLocalDirectoryAndJobLog(getPipeRoot(), ExperimentService.MODULE_NAME, baseLogFileName); + setupLocalDirectoryAndJobLog(getPipeRoot(), baseLogFileName); getLogger().info(getDescription()); for (var runId : _runIds) diff --git a/experiment/src/org/labkey/experiment/pipeline/MoveRunsTask.java b/experiment/src/org/labkey/experiment/pipeline/MoveRunsTask.java index acc6a3ec7ee..12d9ec8b57f 100644 --- a/experiment/src/org/labkey/experiment/pipeline/MoveRunsTask.java +++ b/experiment/src/org/labkey/experiment/pipeline/MoveRunsTask.java @@ -39,6 +39,8 @@ import org.labkey.experiment.XarReader; import org.labkey.experiment.api.ExpRunImpl; import org.labkey.experiment.api.ExperimentServiceImpl; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.ByteArrayOutputStream; import java.io.File; @@ -252,9 +254,9 @@ public String canonicalizeDataFileURL(String dataFileURL) } @Override - public Path getLogFilePath() + public FileLike getLogFilePath() { - return _logFile.toPath(); + return FileSystemLike.wrapFile(_logFile); } public String toString() diff --git a/experiment/src/org/labkey/experiment/pipeline/SampleReloadTask.java b/experiment/src/org/labkey/experiment/pipeline/SampleReloadTask.java index aa6c3ee30e2..74e7159a485 100644 --- a/experiment/src/org/labkey/experiment/pipeline/SampleReloadTask.java +++ b/experiment/src/org/labkey/experiment/pipeline/SampleReloadTask.java @@ -68,7 +68,7 @@ public RecordedActionSet run() { PipelineJob job = getJob(); FileAnalysisJobSupport support = job.getJobSupport(FileAnalysisJobSupport.class); - job.setLogFile(FileUtil.appendName(support.getDataDirectory(), FileUtil.makeFileNameWithTimestamp("triggered_sample_reload", "log"))); + job.setLogFile(support.getDataDirectoryFileLike().resolveChild(FileUtil.makeFileNameWithTimestamp("triggered_sample_reload", "log"))); Map params = support.getParameters(); job.setStatus("RELOADING", "Job started at: " + DateUtil.nowISO()); diff --git a/experiment/src/org/labkey/experiment/pipeline/XarGeneratorSource.java b/experiment/src/org/labkey/experiment/pipeline/XarGeneratorSource.java index dc7b4a4be83..be9b9cf220e 100644 --- a/experiment/src/org/labkey/experiment/pipeline/XarGeneratorSource.java +++ b/experiment/src/org/labkey/experiment/pipeline/XarGeneratorSource.java @@ -18,6 +18,7 @@ import org.fhcrc.cpas.exp.xml.ExperimentArchiveDocument; import org.labkey.api.exp.AbstractFileXarSource; import org.labkey.api.pipeline.PipelineJob; +import org.labkey.vfs.FileLike; import java.nio.file.Path; @@ -27,14 +28,14 @@ */ public class XarGeneratorSource extends AbstractFileXarSource { - public XarGeneratorSource(PipelineJob job, Path xarFile) + public XarGeneratorSource(PipelineJob job, FileLike xarFile) { super(job); _xmlFile = xarFile; } @Override - public Path getLogFilePath() + public FileLike getLogFilePath() { throw new UnsupportedOperationException(); } diff --git a/experiment/src/org/labkey/experiment/pipeline/XarGeneratorTask.java b/experiment/src/org/labkey/experiment/pipeline/XarGeneratorTask.java index f86c683d814..f0e60638a03 100644 --- a/experiment/src/org/labkey/experiment/pipeline/XarGeneratorTask.java +++ b/experiment/src/org/labkey/experiment/pipeline/XarGeneratorTask.java @@ -39,14 +39,13 @@ import org.labkey.experiment.DataURLRelativizer; import org.labkey.api.exp.xar.LSIDRelativizer; import org.labkey.experiment.XarExporter; +import org.labkey.vfs.FileLike; import java.io.BufferedOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -105,10 +104,10 @@ public List getProtocolActionNames() return Collections.emptyList(); } - protected Path getXarFile(PipelineJob job) + protected FileLike getXarFile(PipelineJob job) { FileAnalysisJobSupport jobSupport = job.getJobSupport(FileAnalysisJobSupport.class); - return getOutputType().newFile(jobSupport.getAnalysisDirectoryPath(), jobSupport.getBaseName()); + return getOutputType().newFile(jobSupport.getAnalysisDirectoryFileLike(), jobSupport.getBaseName()); } @Override @@ -159,7 +158,7 @@ public RecordedActionSet run() throws PipelineJobException Set importedRuns = new HashSet<>(); if (_factory.isLoadFiles()) { - Path permanentXAR = _factory.getXarFile(getJob()); + FileLike permanentXAR = _factory.getXarFile(getJob()); if (NetworkDrive.exists(permanentXAR)) { // Be sure that it's been imported (and not already deleted from the database) @@ -176,7 +175,7 @@ public RecordedActionSet run() throws PipelineJobException // Load the data files for this run importedRuns.addAll(ExperimentService.get().importXar(new FileXarSource(getLoadingXarFile(), getJob()), getJob(), false)); - Files.move(getLoadingXarFile(), permanentXAR); + Files.move(getLoadingXarFile().toNioPathForWrite(), permanentXAR.toNioPathForWrite()); } } else @@ -225,19 +224,19 @@ public RecordedActionSet run() throws PipelineJobException @Override public void writeToDisk(ExpRun run) throws PipelineJobException { - Path f = getLoadingXarFile(); - Path tempFile = f.getParent().resolve(f.getFileName().toString() + ".temp"); + FileLike f = getLoadingXarFile(); + FileLike tempFile = f.getParent().resolveChild(f.getName() + ".temp"); try { XarExporter exporter = new XarExporter(LSIDRelativizer.FOLDER_RELATIVE, DataURLRelativizer.RUN_RELATIVE_LOCATION.createURLRewriter(), getJob().getUser(), getJob().getContainer()); exporter.addExperimentRun(run); - try (OutputStream fOut = new BufferedOutputStream(Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE))) + try (OutputStream fOut = new BufferedOutputStream(tempFile.openOutputStream())) { exporter.dumpXML(fOut); fOut.close(); - Files.move(tempFile, f, StandardCopyOption.ATOMIC_MOVE); + Files.move(tempFile.toNioPathForWrite(), f.toNioPathForWrite(), StandardCopyOption.ATOMIC_MOVE); } } catch (ExperimentException | IOException e) @@ -246,9 +245,9 @@ public void writeToDisk(ExpRun run) throws PipelineJobException } } - private Path getLoadingXarFile() + private FileLike getLoadingXarFile() { - Path xarPath = _factory.getXarFile(getJob()); - return xarPath.resolve(xarPath + ".loading"); + FileLike xarPath = _factory.getXarFile(getJob()); + return xarPath.getParent().resolveChild(xarPath.getName() + ".loading"); } } diff --git a/experiment/src/org/labkey/experiment/samples/AbstractExpFolderImporter.java b/experiment/src/org/labkey/experiment/samples/AbstractExpFolderImporter.java index 750b80d1628..5147ae170f4 100644 --- a/experiment/src/org/labkey/experiment/samples/AbstractExpFolderImporter.java +++ b/experiment/src/org/labkey/experiment/samples/AbstractExpFolderImporter.java @@ -46,10 +46,11 @@ import org.labkey.experiment.api.SampleTypeServiceImpl; import org.labkey.experiment.xar.FolderXarImporterFactory; import org.labkey.experiment.xar.XarImportContext; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; import java.nio.file.Path; import java.sql.SQLException; import java.util.Collections; @@ -90,9 +91,9 @@ public void process(@Nullable PipelineJob job, FolderImportContext ctx, VirtualF if (xarDir != null) { // #44384 Generate a relative Path object for the folder's VirtualFile - Path xarDirPath = Path.of(xarDir.getLocation()); - Path typesXarFile = null; - Path runsXarFile = null; + FileLike xarDirPath = FileSystemLike.wrapFile(Path.of(xarDir.getLocation()).toAbsolutePath()); + FileLike typesXarFile = null; + FileLike runsXarFile = null; Logger log = ctx.getLogger(); if (null != job) @@ -104,14 +105,14 @@ public void process(@Nullable PipelineJob job, FolderImportContext ctx, VirtualF if (isXarTypesFile(file)) { if (typesXarFile == null) - typesXarFile = xarDirPath.resolve(file); + typesXarFile = xarDirPath.resolveChild(file); else log.error("More than one types XAR file found in the sample type directory: ", file); } else if (file.equalsIgnoreCase(XAR_RUNS_NAME) || file.equalsIgnoreCase(XAR_RUNS_XML_NAME)) { if (runsXarFile == null) - runsXarFile = xarDirPath.resolve(file); + runsXarFile = xarDirPath.resolveChild(file); else log.error("More than one runs XAR file found in the sample type directory: ", file); } @@ -126,8 +127,8 @@ else if (file.equalsIgnoreCase(XAR_RUNS_NAME) || file.equalsIgnoreCase(XAR_RUNS_ { if (typesXarFile != null) { - Path logFile = null; - if (Files.exists(typesXarFile)) + FileLike logFile = null; + if (typesXarFile.exists()) logFile = CompressedInputStreamXarSource.getLogFileFor(typesXarFile); XarReader typesReader = getXarReader(job, ctx, root, typesXarFile); XarContext xarContext = typesReader.getXarSource().getXarContext(); @@ -142,10 +143,10 @@ else if (file.equalsIgnoreCase(XAR_RUNS_NAME) || file.equalsIgnoreCase(XAR_RUNS_ if (runsXarFile != null) { XarSource runsXarSource; - if (runsXarFile.getFileName().toString().toLowerCase().endsWith(".xar.xml")) + if (runsXarFile.getName().toLowerCase().endsWith(".xar.xml")) runsXarSource = new FileXarSource(runsXarFile, job, ctx.getContainer(), ctx.getXarJobIdContext()); else - runsXarSource = new CompressedInputStreamXarSource(xarDir.getInputStream(runsXarFile.getFileName().toString()), runsXarFile, logFile, job, ctx.getUser(), ctx.getContainer(), ctx.getXarJobIdContext()); + runsXarSource = new CompressedInputStreamXarSource(xarDir.getInputStream(runsXarFile.getName()), runsXarFile, logFile, job, ctx.getUser(), ctx.getContainer(), ctx.getXarJobIdContext()); try { runsXarSource.init(); @@ -155,7 +156,7 @@ else if (file.equalsIgnoreCase(XAR_RUNS_NAME) || file.equalsIgnoreCase(XAR_RUNS_ log.error("Failed to initialize runs XAR source", e); throw(e); } - log.info("Importing the runs XAR file: " + runsXarFile.getFileName().toString()); + log.info("Importing the runs XAR file: " + runsXarFile.getName()); XarReader runsReader = new FolderXarImporterFactory.FolderExportXarReader(runsXarSource, job); runsReader.setStrictValidateExistingSampleType(xarCtx.isStrictValidateExistingSampleType()); runsReader.parseAndLoad(false, ctx.getAuditBehaviorType()); @@ -188,14 +189,14 @@ else if (file.equalsIgnoreCase(XAR_RUNS_NAME) || file.equalsIgnoreCase(XAR_RUNS_ } } - protected XarReader getXarReader(@Nullable PipelineJob job, FolderImportContext ctx, VirtualFile root, Path typesXarFile) throws IOException, ExperimentException + protected XarReader getXarReader(@Nullable PipelineJob job, FolderImportContext ctx, VirtualFile root, FileLike typesXarFile) throws IOException, ExperimentException { VirtualFile xarDir = getXarDir(root); Logger log = ctx.getLogger(); - Path logFile = null; + FileLike logFile = null; // we don't need the log file in cases where the xarFile is a virtual file and not in the file system - if (Files.exists(typesXarFile)) + if (typesXarFile.exists()) logFile = CompressedInputStreamXarSource.getLogFileFor(typesXarFile); if (job == null) @@ -206,10 +207,10 @@ protected XarReader getXarReader(@Nullable PipelineJob job, FolderImportContext XarSource typesXarSource; - if (typesXarFile.getFileName().toString().toLowerCase().endsWith(".xar.xml")) + if (typesXarFile.getName().toLowerCase().endsWith(".xar.xml")) typesXarSource = new FileXarSource(typesXarFile, job, ctx.getContainer(), ctx.getXarJobIdContext()); else - typesXarSource = new CompressedInputStreamXarSource(xarDir.getInputStream(typesXarFile.getFileName().toString()), typesXarFile, logFile, job, ctx.getUser(), ctx.getContainer(), ctx.getXarJobIdContext()); + typesXarSource = new CompressedInputStreamXarSource(xarDir.getInputStream(typesXarFile.getName()), typesXarFile, logFile, job, ctx.getUser(), ctx.getContainer(), ctx.getXarJobIdContext()); try { typesXarSource.init(); diff --git a/experiment/src/org/labkey/experiment/samples/SampleStatusFolderImporter.java b/experiment/src/org/labkey/experiment/samples/SampleStatusFolderImporter.java index 7ade7bb4675..7d3e441d744 100644 --- a/experiment/src/org/labkey/experiment/samples/SampleStatusFolderImporter.java +++ b/experiment/src/org/labkey/experiment/samples/SampleStatusFolderImporter.java @@ -15,6 +15,8 @@ import org.labkey.api.util.FileUtil; import org.labkey.api.writer.VirtualFile; import org.labkey.experiment.XarReader; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; import java.nio.file.Path; import java.util.HashMap; @@ -52,8 +54,8 @@ public void process(@Nullable PipelineJob job, FolderImportContext ctx, VirtualF if (xarDir != null) { // #44384 Generate a relative Path object for the folder's VirtualFile - Path xarDirPath = Path.of(xarDir.getLocation()); - Path typesXarFile = null; + FileLike xarDirPath = FileSystemLike.wrapFile(Path.of(xarDir.getLocation()).toAbsolutePath()); + FileLike typesXarFile = null; Map sampleStatusDataFiles = new HashMap<>(); Logger log = ctx.getLogger(); @@ -66,7 +68,7 @@ public void process(@Nullable PipelineJob job, FolderImportContext ctx, VirtualF if (file.equalsIgnoreCase(XAR_TYPES_NAME) || file.equalsIgnoreCase(XAR_TYPES_XML_NAME)) { if (typesXarFile == null) - typesXarFile = xarDirPath.resolve(file); + typesXarFile = xarDirPath.resolveChild(file); else log.error("More than one types XAR file found in the sample type directory: ", file); } diff --git a/experiment/src/org/labkey/experiment/xar/CompressedXarSource.java b/experiment/src/org/labkey/experiment/xar/CompressedXarSource.java index 9463c79c15d..2c8ed5d4738 100644 --- a/experiment/src/org/labkey/experiment/xar/CompressedXarSource.java +++ b/experiment/src/org/labkey/experiment/xar/CompressedXarSource.java @@ -24,6 +24,7 @@ import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.util.FileUtil; import org.labkey.api.writer.ZipUtil; +import org.labkey.vfs.FileLike; import java.io.IOException; import java.nio.file.Files; @@ -38,9 +39,9 @@ */ public class CompressedXarSource extends AbstractFileXarSource { - private final Path _xarFile; + private final FileLike _xarFile; - public CompressedXarSource(Path xarFile, PipelineJob job) + public CompressedXarSource(FileLike xarFile, PipelineJob job) { super(job); _xarFile = xarFile; @@ -53,13 +54,13 @@ public CompressedXarSource(Path xarFile, PipelineJob job) * This may not be the same as the the Container returned by job.getContainer(). * @param substitutions Additional context substitutions */ - public CompressedXarSource(Path xarFile, PipelineJob job, Container targetContainer, @Nullable Map substitutions) + public CompressedXarSource(FileLike xarFile, PipelineJob job, Container targetContainer, @Nullable Map substitutions) { super(job.getDescription(), targetContainer, job.getUser(), job, substitutions); _xarFile = xarFile; } - public CompressedXarSource(Path xarFile, PipelineJob job, Container targetContainer) + public CompressedXarSource(FileLike xarFile, PipelineJob job, Container targetContainer) { this(xarFile, job, targetContainer, null); } @@ -67,19 +68,19 @@ public CompressedXarSource(Path xarFile, PipelineJob job, Container targetContai @Override public void init() throws ExperimentException, IOException { - Path outputDir = _xarFile.resolve(_xarFile + ".exploded"); + FileLike outputDir = _xarFile.getParent().resolveChild(_xarFile.getName() + ".exploded"); FileUtil.deleteDir(outputDir); - if (Files.exists(outputDir)) + if (outputDir.exists()) { throw new ExperimentException("Failed to clean up old directory " + outputDir); } FileUtil.createDirectories(outputDir); - if (!Files.isDirectory(outputDir)) + if (!outputDir.isDirectory()) { throw new ExperimentException("Failed to create directory " + outputDir); } - List xarContents; + List xarContents; try { xarContents = ZipUtil.unzipToDirectory(_xarFile, outputDir); @@ -89,7 +90,7 @@ public void init() throws ExperimentException, IOException throw new ExperimentException("Failed to extract XAR file: " + _xarFile, e); } - List xarFiles = xarContents.stream().filter(f -> f.getFileName().toString().toLowerCase().endsWith(".xar.xml")).collect(Collectors.toList()); + List xarFiles = xarContents.stream().filter(f -> f.getName().toLowerCase().endsWith(".xar.xml")).toList(); if (xarFiles.isEmpty()) { @@ -106,7 +107,7 @@ else if (xarFiles.size() > 1) } @Override - public Path getLogFilePath() + public FileLike getLogFilePath() { try { diff --git a/experiment/src/org/labkey/experiment/xar/FolderXarImporterFactory.java b/experiment/src/org/labkey/experiment/xar/FolderXarImporterFactory.java index b430a42a436..18326f029fd 100644 --- a/experiment/src/org/labkey/experiment/xar/FolderXarImporterFactory.java +++ b/experiment/src/org/labkey/experiment/xar/FolderXarImporterFactory.java @@ -32,8 +32,8 @@ import org.labkey.api.writer.VirtualFile; import org.labkey.experiment.XarReader; import org.labkey.experiment.pipeline.ExperimentPipelineJob; - -import java.nio.file.Path; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; /** * User: vsharma @@ -100,7 +100,7 @@ public void process(PipelineJob job, FolderImportContext ctx, VirtualFile root) throw e; } - Path xarFile = xarSourceWrapper.getXarFile(); + FileLike xarFile = xarSourceWrapper.getXarFile(); if (xarFile == null) { ctx.getLogger().error("Could not find a xar file in the xar directory."); @@ -120,7 +120,7 @@ public void process(PipelineJob job, FolderImportContext ctx, VirtualFile root) job = new ExperimentPipelineJob(bgInfo, xarFile, "Xar import", false, pipeRoot) { @Override - protected XarSource createXarSource(Path file) + protected XarSource createXarSource(FileLike file) { // Assume this is a .xar or a .zip file return xarSourceWrapper.getXarSource(this); @@ -165,7 +165,7 @@ private static class FolderExportXarSourceWrapper private final VirtualFile _xarDir; private final FolderImportContext _importContext; - private Path _xarFile; + private FileLike _xarFile; private XarSource _xarSource; public FolderExportXarSourceWrapper(VirtualFile xarDir, FolderImportContext ctx) @@ -185,13 +185,13 @@ public void init() { if (file.toLowerCase().endsWith(".xar") || file.toLowerCase().endsWith(".xar.xml")) { - _xarFile = FileUtil.getPath(_importContext.getContainer(), FileUtil.createUri(_xarDir.getLocation())).resolve(file); + _xarFile = FileSystemLike.wrapFile(FileUtil.getPath(_importContext.getContainer(), FileUtil.createUri(_xarDir.getLocation())).resolve(file)); break; } } } - public Path getXarFile() + public FileLike getXarFile() { return _xarFile; } @@ -200,7 +200,7 @@ public XarSource getXarSource(PipelineJob job) { if (_xarSource == null) { - if (getXarFile().getFileName().toString().toLowerCase().endsWith(".xar.xml")) + if (getXarFile().getName().toLowerCase().endsWith(".xar.xml")) { _xarSource = new FileXarSource( getXarFile(), diff --git a/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java b/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java index f3ed5d2eaff..371168c722a 100644 --- a/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java +++ b/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java @@ -93,6 +93,7 @@ import org.labkey.api.view.template.Warnings; import org.labkey.api.webdav.WebdavResource; import org.labkey.api.webdav.WebdavService; +import org.labkey.vfs.FileLike; import java.beans.PropertyChangeEvent; import java.io.BufferedWriter; @@ -1367,6 +1368,13 @@ public String getAbsolutePathFromDataFileUrl(String dataFileUrl, Container conta return FileUtil.getAbsolutePath(container, FileUtil.createUri(dataFileUrl)); } + @Nullable + @Override + public URI getWebDavUrl(@NotNull FileLike path, @NotNull Container container, @NotNull PathType type) + { + return getWebDavUrl(path.toNioPathForRead(), container, type); + } + @Nullable @Override public URI getWebDavUrl(@NotNull java.nio.file.Path path, @NotNull Container container, @NotNull PathType type) diff --git a/list/src/org/labkey/list/model/ListImporter.java b/list/src/org/labkey/list/model/ListImporter.java index fb7e140e05e..12f9c6c1786 100644 --- a/list/src/org/labkey/list/model/ListImporter.java +++ b/list/src/org/labkey/list/model/ListImporter.java @@ -272,21 +272,34 @@ private boolean processSingle(VirtualFile sourceDir, ListDefinition def, String String src = ti.getColumn(def.getKeyName()).getJdbcDefaultValue(); if (null != src) { - String sequence = ""; + SQLFragment keyupdate; - int start = src.indexOf('\''); - int end = src.lastIndexOf('\''); - - if (end > start) + // To best support crazy column names, reuse the regclass in the nextval() call when present + if (src.startsWith("nextval(") && src.endsWith("::regclass)")) + { + String setVal = src.replace("nextval(", "setval("); + setVal = setVal.replace("::regclass)", "::regclass, "); + keyupdate = new SQLFragment("SELECT ").append(SQLFragment.unsafe(setVal)); + } + else { - sequence = src.substring(start + 1, end); + // In case there are sequences in the wild with different syntax, fall back on + // our previous strategy + String sequence = ""; + + int start = src.indexOf('\''); + int end = src.lastIndexOf('\''); + + if (end > start) + + sequence = src.substring(start + 1, end); if (!sequence.toLowerCase().startsWith("list.")) sequence = "list." + sequence; + keyupdate = new SQLFragment("SELECT setval(").appendStringLiteral(sequence, dialect); } String keyStorageColName = def.getDomain().getPropertyByName(def.getKeyName()).getPropertyDescriptor().getStorageColumnName(); - SQLFragment keyupdate = new SQLFragment("SELECT setval(").appendStringLiteral(sequence, dialect); - keyupdate.append(", coalesce((SELECT MAX(").append(dialect.quoteIdentifier(keyStorageColName.toLowerCase())).append(")+1 FROM ").append(tableName); + keyupdate.append(" coalesce((SELECT MAX(").appendIdentifier(dialect.makeDatabaseIdentifier(keyStorageColName.toLowerCase())).append(")+1 FROM ").append(tableName); keyupdate.append("), 1), false)"); new SqlExecutor(ti.getSchema()).execute(keyupdate); } diff --git a/pipeline/src/org/labkey/pipeline/PipelineController.java b/pipeline/src/org/labkey/pipeline/PipelineController.java index a3eb64035ad..d8636fee02f 100644 --- a/pipeline/src/org/labkey/pipeline/PipelineController.java +++ b/pipeline/src/org/labkey/pipeline/PipelineController.java @@ -113,6 +113,7 @@ import org.labkey.pipeline.api.PipelineServiceImpl; import org.labkey.pipeline.api.PipelineStatusManager; import org.labkey.pipeline.status.StatusController; +import org.labkey.vfs.FileLike; import org.springframework.beans.MutablePropertyValues; import org.springframework.validation.BindException; import org.springframework.validation.Errors; @@ -1170,10 +1171,10 @@ public class ImportFolderFromPipelineAction extends SimpleRedirectAction { private String _navTrail = "Import Folder"; - private java.nio.file.Path _archiveFile; + private FileLike _archiveFile; @Override public void validateCommand(StartFolderImportForm form, Errors errors) @@ -1217,7 +1218,8 @@ else if (form.getFilePath() == null) } else { - _archiveFile = PipelineManager.validateFolderImportFileNioPath(form.getFilePath(), currentPipelineRoot, errors); + // We no longer support absolute paths - should be relative to the pipeline root + _archiveFile = currentPipelineRoot.resolvePathToFileLike(form.getFilePath()); // Be sure that import container has a valid pipeline root PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(getContainer()); @@ -1245,12 +1247,11 @@ public boolean handlePost(StartFolderImportForm form, BindException errors) thro { User user = getUser(); boolean success = true; - Map containerArchiveXmlMap = new HashMap<>(); + Map containerArchiveXmlMap = new HashMap<>(); - if (Files.exists(_archiveFile)) + if (_archiveFile.exists()) { - - java.nio.file.Path archiveXml = PipelineManager.getArchiveXmlFile(getContainer(), _archiveFile, "folder.xml", errors); + FileLike archiveXml = PipelineManager.getArchiveXmlFile(getContainer(), _archiveFile, "folder.xml", errors); if (errors.hasErrors()) return false; @@ -1280,12 +1281,12 @@ public boolean handlePost(StartFolderImportForm form, BindException errors) thro return success; } - private boolean createImportPipelineJob(Container container, User user, ImportOptions options, java.nio.file.Path archiveXml) + private boolean createImportPipelineJob(Container container, User user, ImportOptions options, FileLike archiveXml) { PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(container); ActionURL url = getViewContext().getActionURL(); - return PipelineService.get().runFolderImportJob(container, user, url, archiveXml, _archiveFile.getFileName().toString(), pipelineRoot, options); + return PipelineService.get().runFolderImportJob(container, user, url, archiveXml, _archiveFile.getName(), pipelineRoot, options); } @Override @@ -1626,11 +1627,11 @@ public ActionURL urlActions(Container container) } @Override - public ActionURL urlStartFolderImport(Container container, @NotNull java.nio.file.Path archiveFile, @Nullable ImportOptions options, boolean fromTemplateSourceFolder) + public ActionURL urlStartFolderImport(Container container, @NotNull FileLike archiveFile, @Nullable ImportOptions options, boolean fromTemplateSourceFolder) { ActionURL url = new ActionURL(StartFolderImportAction.class, container); - return addStartImportParameters(url, archiveFile, options, fromTemplateSourceFolder); + return addStartImportParameters(container, url, archiveFile, options, fromTemplateSourceFolder); } @Override @@ -1647,9 +1648,10 @@ public ActionURL urlCreatePipelineTrigger(Container container, String pipelineId return url; } - private ActionURL addStartImportParameters(ActionURL url, @NotNull java.nio.file.Path file, @Nullable ImportOptions options, boolean fromTemplateSourceFolder) + private ActionURL addStartImportParameters(Container container, ActionURL url, @NotNull FileLike file, @Nullable ImportOptions options, boolean fromTemplateSourceFolder) { - url.addParameter("filePath", file.toAbsolutePath().toString()); + PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(container); + url.addParameter("filePath", pipelineRoot.relativePath(file)); url.addParameter("validateQueries", options == null || !options.isSkipQueryValidation()); url.addParameter("createSharedDatasets", options == null || options.isCreateSharedDatasets()); if (options != null) diff --git a/pipeline/src/org/labkey/pipeline/PipelineModule.java b/pipeline/src/org/labkey/pipeline/PipelineModule.java index bc3fcda9dcc..e3e357c300a 100644 --- a/pipeline/src/org/labkey/pipeline/PipelineModule.java +++ b/pipeline/src/org/labkey/pipeline/PipelineModule.java @@ -21,12 +21,15 @@ import org.labkey.api.admin.sitevalidation.SiteValidationService; import org.labkey.api.audit.AuditLogService; import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.DbSchema; import org.labkey.api.data.RuntimeSQLException; import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableSelector; import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.files.FileContentService; import org.labkey.api.files.TableUpdaterFileListener; @@ -253,6 +256,7 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) result.put("jmsType", PipelineService.get().getJmsType().toString()); result.put("pipelineRootCount", PipelineService.get().getAllPipelineRoots().size()); + result.put("supplementalDirectories", new TableSelector(PipelineSchema.getInstance().getTableInfoPipelineRoots(), new SimpleFilter("SupplementalPath", null, CompareType.NONBLANK), null).getRowCount()); return result; }); diff --git a/pipeline/src/org/labkey/pipeline/analysis/AnalysisController.java b/pipeline/src/org/labkey/pipeline/analysis/AnalysisController.java index a59f1ba0ce4..0a53f7d1062 100644 --- a/pipeline/src/org/labkey/pipeline/analysis/AnalysisController.java +++ b/pipeline/src/org/labkey/pipeline/analysis/AnalysisController.java @@ -69,6 +69,7 @@ import org.labkey.api.view.NotFoundException; import org.labkey.api.view.ViewForm; import org.labkey.api.writer.ContainerUser; +import org.labkey.vfs.FileLike; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; @@ -77,7 +78,6 @@ import java.io.IOException; import java.io.StringReader; import java.nio.charset.Charset; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -201,7 +201,7 @@ public ApiResponse execute(AnalyzeForm form, BindException errors) PipelineService.PathAnalysisProperties props = PipelineService.get().getFileAnalysisProperties(getContainer(), form.getTaskId(), form.getPath()); AbstractFileAnalysisProtocol protocol = props.getFactory().getProtocol(props.getPipeRoot(), props.getDirData(), form.getProtocolName(), false); //NOTE: if protocol if null, initFileStatus() will return a result of UNKNOWN - Path dirAnalysis = props.getFactory().getAnalysisDir(props.getDirData(), form.getProtocolName(), props.getPipeRoot()); + FileLike dirAnalysis = props.getFactory().getAnalysisDir(props.getDirData(), form.getProtocolName(), props.getPipeRoot()); form.initStatus(protocol, props.getDirData(), dirAnalysis); boolean isRetry = false; @@ -251,17 +251,17 @@ public ApiResponse execute(AnalyzeForm form, BindException errors) if (wbRoot == null || !wbRoot.isValid()) continue; - File wbDirData = null; + FileLike wbDirData = null; if (form.getPath() != null) { - wbDirData = wbRoot.resolvePath(form.getPath()); + wbDirData = wbRoot.resolvePathToFileLike(form.getPath()); if (!NetworkDrive.exists(wbDirData)) continue; } - for (String protocolName : props.getFactory().getProtocolNames(wbRoot, wbDirData.toPath(), false)) + for (String protocolName : props.getFactory().getProtocolNames(wbRoot, wbDirData, false)) { - protocols.put(getProtocolJson(protocolName, wbRoot, wbDirData.toPath(), props.getFactory())); + protocols.put(getProtocolJson(protocolName, wbRoot, wbDirData, props.getFactory())); } } } @@ -273,7 +273,7 @@ public ApiResponse execute(AnalyzeForm form, BindException errors) return new ApiSimpleResponse(result); } - protected JSONObject getProtocolJson(String protocolName, PipeRoot root, @Nullable Path dirData, AbstractFileAnalysisProtocolFactory factory) throws NotFoundException + protected JSONObject getProtocolJson(String protocolName, PipeRoot root, @Nullable FileLike dirData, AbstractFileAnalysisProtocolFactory factory) throws NotFoundException { JSONObject protocol = new JSONObject(); AbstractFileAnalysisProtocol pipelineProtocol = factory.getProtocol(root, dirData, protocolName, false); diff --git a/pipeline/src/org/labkey/pipeline/analysis/FileAnalysisJob.java b/pipeline/src/org/labkey/pipeline/analysis/FileAnalysisJob.java index 76239b6bfff..43331da9776 100644 --- a/pipeline/src/org/labkey/pipeline/analysis/FileAnalysisJob.java +++ b/pipeline/src/org/labkey/pipeline/analysis/FileAnalysisJob.java @@ -26,10 +26,10 @@ import org.labkey.api.util.FileType; import org.labkey.api.util.NetworkDrive; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -54,19 +54,18 @@ public FileAnalysisJob(FileAnalysisProtocol protocol, PipeRoot root, TaskId taskPipelineId, String protocolName, - Path fileParameters, - List filesInput, + FileLike fileParameters, + List filesInput, @Nullable Map variableMap, - boolean splittable, - boolean writeJobInfoFile) throws IOException + boolean splittable) throws IOException { - super(protocol, providerName, info, root, protocolName, fileParameters, filesInput, splittable, writeJobInfoFile); + super(protocol, providerName, info, root, protocolName, fileParameters, filesInput, splittable); _taskPipelineId = taskPipelineId; _variableMap = variableMap; } - public FileAnalysisJob(FileAnalysisJob job, File fileInput) + public FileAnalysisJob(FileAnalysisJob job, FileLike fileInput) { super(job, fileInput); @@ -101,7 +100,7 @@ public TaskId getTaskPipelineId() } @Override - public AbstractFileAnalysisJob createSingleFileJob(File file) + public AbstractFileAnalysisJob createSingleFileJob(FileLike file) { return new FileAnalysisJob(this, file); } @@ -109,7 +108,7 @@ public AbstractFileAnalysisJob createSingleFileJob(File file) @Override public FileAnalysisTaskPipeline getTaskPipeline() { - TaskPipeline tp = super.getTaskPipeline(); + TaskPipeline tp = super.getTaskPipeline(); if (tp == null) { LOG.warn("Task pipeline " + _taskPipelineId + " not found."); diff --git a/pipeline/src/org/labkey/pipeline/analysis/FileAnalysisProtocol.java b/pipeline/src/org/labkey/pipeline/analysis/FileAnalysisProtocol.java index 9eae9d0db3f..f79a53aad95 100644 --- a/pipeline/src/org/labkey/pipeline/analysis/FileAnalysisProtocol.java +++ b/pipeline/src/org/labkey/pipeline/analysis/FileAnalysisProtocol.java @@ -23,9 +23,9 @@ import org.labkey.api.pipeline.file.AbstractFileAnalysisProtocol; import org.labkey.api.util.FileType; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; import java.io.IOException; -import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -60,16 +60,15 @@ public void setFactory(FileAnalysisProtocolFactory factory) } @Override - public AbstractFileAnalysisJob createPipelineJob(ViewBackgroundInfo info, PipeRoot root, List filesInput, - Path fileParameters, @Nullable Map variableMap + public AbstractFileAnalysisJob createPipelineJob(ViewBackgroundInfo info, PipeRoot root, List filesInput, + FileLike fileParameters, @Nullable Map variableMap ) throws IOException { TaskId id = _factory.getPipeline().getId(); boolean splittable = _factory.getPipeline().isSplittable(); - boolean writeJobInfoFile = _factory.getPipeline().isWriteJobInfoFile(); return new FileAnalysisJob(this, FileAnalysisPipelineProvider.name, info, root, - id, getName(), fileParameters, filesInput, variableMap, splittable, writeJobInfoFile); + id, getName(), fileParameters, filesInput, variableMap, splittable); } } diff --git a/pipeline/src/org/labkey/pipeline/api/AbstractWorkDirectory.java b/pipeline/src/org/labkey/pipeline/api/AbstractWorkDirectory.java index 803641f78ad..c648668e556 100644 --- a/pipeline/src/org/labkey/pipeline/api/AbstractWorkDirectory.java +++ b/pipeline/src/org/labkey/pipeline/api/AbstractWorkDirectory.java @@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.labkey.api.assay.AssayFileWriter; import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.RecordedAction; import org.labkey.api.pipeline.WorkDirFactory; @@ -268,28 +267,15 @@ private Set outputFile(TaskPath tp, String role, RecordedAction action) th List filesWork = getWorkFiles(WorkDirectory.Function.output, tp); for (File fileWork : filesWork) { - File fileOutput; + File fileOutput = switch (tp.getOutputLocation()) + { + case ANALYSIS_DIR -> new File(_support.getAnalysisDirectory(), fileWork.getName()); + case DATA_DIR -> new File(_support.getDataDirectory(), fileWork.getName()); + case PATH -> _support.findOutputFile(tp.getOutputDir(), fileWork.getName()); + default -> _support.findOutputFile(fileWork.getName()); + }; // Check if the output is specifically flagged to go into a special location - switch (tp.getOutputLocation()) - { - case ANALYSIS_DIR: - fileOutput = new File(_support.getAnalysisDirectory(), fileWork.getName()); - break; - - case DATA_DIR: - fileOutput = new File(_support.getDataDirectory(), fileWork.getName()); - break; - - case PATH: - fileOutput = _support.findOutputFile(tp.getOutputDir(), fileWork.getName()); - break; - - case DEFAULT: - default: - fileOutput = _support.findOutputFile(fileWork.getName()); - break; - } if (fileOutput != null) { @@ -383,23 +369,8 @@ private File getDir(Function f, String name) @Override public File newFile(FileType type) { - return newFile(Function.output, type); - } - - @Override - public File newFile(Function f, FileType type) - { - if (f == Function.input) - { - // that null arg to type.getName causes it to try all known filename extensions instead of just default - return newFile(f, type.getName((File)null, _support.getBaseName())); - } - else if (f == Function.output) - { - // TODO: Issue 20143: pipeline: Custom output directory for task outputs - return newFile(f, type.getName(_dir, _support.getBaseName())); - } - throw new IllegalArgumentException("input or output expected"); + // TODO: Issue 20143: pipeline: Custom output directory for task outputs + return newFile(Function.output, type.getName(_dir, _support.getBaseName())); } @Override diff --git a/pipeline/src/org/labkey/pipeline/api/ParamParserImpl.java b/pipeline/src/org/labkey/pipeline/api/ParamParserImpl.java index b429281159e..196580e0f2a 100644 --- a/pipeline/src/org/labkey/pipeline/api/ParamParserImpl.java +++ b/pipeline/src/org/labkey/pipeline/api/ParamParserImpl.java @@ -127,27 +127,32 @@ protected void addError(Error error) @Override public void parse(InputStream inputStream) { - try + if (inputStream != null) { - DocumentBuilder db = XmlBeansUtil.DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); + try (inputStream) + { + try + { + DocumentBuilder db = XmlBeansUtil.DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); - InputSource source = new InputSource(new InputStreamReader(inputStream)); - _doc = db.parse(source); - _doc.setXmlStandalone(true); // Added to help with new Transformer-based getXML() - validateDocument(); - } - catch (SAXParseException e) - { - // Subtract 1 from the line number, since we added the DOCTYPE line - addError(new ErrorImpl(e.getMessage(), e.getLineNumber(), e.getColumnNumber())); - } - catch (Exception e) - { - addError(new ErrorImpl(e.toString())); - } - finally - { - try { inputStream.close(); } catch (IOException ignored) {} + InputSource source = new InputSource(new InputStreamReader(inputStream)); + _doc = db.parse(source); + _doc.setXmlStandalone(true); // Added to help with new Transformer-based getXML() + validateDocument(); + } + catch (SAXParseException e) + { + // Subtract 1 from the line number, since we added the DOCTYPE line + addError(new ErrorImpl(e.getMessage(), e.getLineNumber(), e.getColumnNumber())); + } + catch (Exception e) + { + addError(new ErrorImpl(e.toString())); + } + } + catch (IOException ignored) + { + } } } @@ -169,7 +174,7 @@ public Error[] getErrors() { if (_errors == null || _errors.isEmpty()) return null; - return _errors.toArray(new ErrorImpl[0]); + return _errors.toArray(new Error[0]); } @Override @@ -231,7 +236,6 @@ protected void validateDocument() if (!VAL_INPUT.equals(type)) { addError(new ErrorImpl("Note type '" + type + "' not supported.")); - continue; } } diff --git a/pipeline/src/org/labkey/pipeline/api/PipeRootImpl.java b/pipeline/src/org/labkey/pipeline/api/PipeRootImpl.java index 51af9631832..df8f896ca74 100644 --- a/pipeline/src/org/labkey/pipeline/api/PipeRootImpl.java +++ b/pipeline/src/org/labkey/pipeline/api/PipeRootImpl.java @@ -55,6 +55,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; public class PipeRootImpl implements PipeRoot { @@ -146,44 +147,33 @@ public PipeRootImpl(PipelineRoot root) @Override @NotNull - public File ensureSystemDirectory() + public FileLike ensureSystemDirectory() { - Path path = ensureSystemDirectoryPath(); - if (FileUtil.hasCloudScheme(path)) - throw new RuntimeException("System Dir is not on file system."); - return path.toFile(); - } - - @Override - @NotNull - public Path ensureSystemDirectoryPath() - { - Path root = getRootNioPath(); - Path systemDir = root.resolve(SYSTEM_DIRECTORY_NAME); - if (!Files.exists(systemDir)) + FileLike root = getRootFileLike(); + FileLike systemDir = root.resolveChild(SYSTEM_DIRECTORY_NAME); + if (!systemDir.exists()) { try { FileUtil.createDirectories(systemDir); - - Path systemDirLegacy = root.resolve(SYSTEM_DIRECTORY_LEGACY); - if (Files.exists(systemDirLegacy)) + FileLike systemDirLegacy = root.resolveChild(SYSTEM_DIRECTORY_LEGACY); + if (systemDirLegacy.exists() && !isCloudRoot()) { // Legacy means it must be on file system - File legacyDir = systemDirLegacy.toFile(); - for (File f : legacyDir.listFiles()) - f.renameTo(systemDir.toFile()); + File sysDir = systemDirLegacy.toNioPathForRead().toFile(); + File legacyDir = systemDirLegacy.toNioPathForWrite().toFile(); + for (File f : Objects.requireNonNullElse(legacyDir.listFiles(),new File[0])) + f.renameTo(sysDir); } for (PipelineProvider provider : PipelineService.get().getPipelineProviders()) - provider.initSystemDirectory(root, systemDir); + provider.initSystemDirectory(root.toNioPathForWrite(), systemDir.toNioPathForWrite()); } catch (IOException e) { throw new RuntimeException(e); } } - return systemDir; } @@ -227,7 +217,11 @@ public Path getRootNioPath() @Override public @NotNull FileLike getRootFileLike() { - return new FileSystemLike.Builder(getRootPath()).readwrite().root(); + var ret = resolvePathToFileLike(""); + // this should not return null unless there a configuration problem. + if (null == ret) + throw new IllegalStateException("Could not resolve pipeline path."); + return ret; } @Override @@ -384,6 +378,15 @@ public File resolvePath(org.labkey.api.util.Path path) { var parsedPath = org.labkey.api.util.Path.parse(relativePath); + if (ROOT_BASE.cloud.equals(_defaultRoot)) + { + // Return the path to the default location + var combinedPath = StringUtils.isNotBlank(_uris.get(0).getPath()) ? + org.labkey.api.util.Path.parse(_uris.get(0).getPath()).append(parsedPath) : + parsedPath; + return CloudStoreService.get().getFileLike(getContainer(), _cloudStoreName, combinedPath); + } + var pair = _resolveRoot(parsedPath); if (null == pair) return null; @@ -479,28 +482,22 @@ public Path resolveToNioPathFromUrl(String url) return null; } - /** - * Get a local directory that can be used for importing (Read/Write) - * - * Cloud: Uses temp directory - * Default: Uses file root - */ @Override @NotNull - public File getImportDirectory() + public FileLike getImportDirectory() { // If pipeline root is in File system, return that; otherwise return temp directory - File root = isCloudRoot() ? - FileUtil.getTempDirectory() : - getRootPath(); - return FileUtil.appendName(root, PipelineService.UNZIP_DIR); + FileLike root = isCloudRoot() ? + FileUtil.getTempDirectoryFileLike() : + getRootFileLike(); + return root.resolveChild(PipelineService.UNZIP_DIR); } @Override - public Path deleteImportDirectory(@Nullable Logger logger) throws DirectoryNotDeletedException + public FileLike deleteImportDirectory(@Nullable Logger logger) throws DirectoryNotDeletedException { - Path importDir = getImportDirectory().toPath(); - if (Files.exists(importDir) && !FileUtil.deleteDir(importDir, logger)) + FileLike importDir = getImportDirectory(); + if (importDir.exists() && !FileUtil.deleteDir(importDir, logger)) { throw new DirectoryNotDeletedException("Could not delete the directory \"" + PipelineService.UNZIP_DIR + "\""); } @@ -534,6 +531,7 @@ public String relativePath(Path path) if (!strPath.toLowerCase().startsWith(strRoot.toLowerCase())) return null; String ret = strPath.substring(strRoot.length()); + ret = ret.replace('\\', '/'); if (ret.startsWith(File.separator) || ret.startsWith("/")) { return ret.substring(1); diff --git a/pipeline/src/org/labkey/pipeline/api/PipelineManager.java b/pipeline/src/org/labkey/pipeline/api/PipelineManager.java index c1b2f30b6e3..44f2b9682bb 100644 --- a/pipeline/src/org/labkey/pipeline/api/PipelineManager.java +++ b/pipeline/src/org/labkey/pipeline/api/PipelineManager.java @@ -82,10 +82,10 @@ import org.labkey.folder.xml.FolderDocument; import org.labkey.pipeline.query.TriggerConfigurationsTable; import org.labkey.pipeline.status.StatusController; +import org.labkey.vfs.FileLike; import org.springframework.validation.BindException; import org.springframework.validation.Errors; -import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -149,11 +149,11 @@ public static PipelineRoot findPipelineRoot(@NotNull Container container, String } - static public PipelineRoot[] getPipelineRoots(String type) + static public List getPipelineRoots(String type) { SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Type"), type); - return new TableSelector(pipeline.getTableInfoPipelineRoots(), filter, null).getArray(PipelineRoot.class); + return new TableSelector(pipeline.getTableInfoPipelineRoots(), filter, null).getArrayList(PipelineRoot.class); } static public void setPipelineRoot(User user, Container container, URI[] roots, String type, @@ -809,32 +809,27 @@ else if (!pipeRoot.isCloudRoot() && !pipeRoot.isUnderRoot(archiveFile)) // T } @Nullable - private static Path expandZipLocally(PipeRoot pipelineRoot, Path archiveFile, BindException errors) + private static FileLike expandZipLocally(PipeRoot pipelineRoot, FileLike archiveFile, BindException errors) { try { // check if the archive file already exists in the unzip dir of this pipeline root - Path importDir = pipelineRoot.getImportDirectory().toPath(); - if (!archiveFile.getParent().toAbsolutePath().toString().equalsIgnoreCase(importDir.toAbsolutePath().toString())) + FileLike importDir = pipelineRoot.getImportDirectory(); + if (!archiveFile.getParent().equals(importDir)) importDir = pipelineRoot.deleteImportDirectory(null); - boolean shouldUnzip = Files.notExists(importDir); + boolean shouldUnzip = !importDir.exists(); if (!shouldUnzip) { - try (Stream pathStream = Files.list(importDir)) - { - shouldUnzip = pathStream.noneMatch(s -> s.getFileName().toString().equalsIgnoreCase(archiveFile.getFileName().toString())); - } + Stream pathStream = importDir.getChildren().stream(); + shouldUnzip = pathStream.noneMatch(s -> s.getName().equalsIgnoreCase(archiveFile.getName())); } if (shouldUnzip) { // Only unzip once - try (InputStream is = Files.newInputStream(archiveFile)) - { - ZipUtil.unzipToDirectory(is, importDir); - } + ZipUtil.unzipToDirectory(archiveFile, importDir); } return importDir; @@ -856,13 +851,13 @@ private static Path expandZipLocally(PipeRoot pipelineRoot, Path archiveFile, Bi return null; } - private static Path getImportXmlFile(@NotNull PipeRoot pipelineRoot, @NotNull Path archiveFile, @NotNull String xmlFileName, BindException errors) throws InvalidFileException + private static FileLike getImportXmlFile(@NotNull PipeRoot pipelineRoot, @NotNull FileLike archiveFile, @NotNull String xmlFileName, BindException errors) throws InvalidFileException { - Path xmlFile = archiveFile; + FileLike xmlFile = archiveFile; - if (archiveFile.getFileName().toString().toLowerCase().endsWith(".zip")) + if (archiveFile.getName().toLowerCase().endsWith(".zip")) { - Path importDir = expandZipLocally(pipelineRoot, archiveFile, errors); + FileLike importDir = expandZipLocally(pipelineRoot, archiveFile, errors); if (importDir != null) { xmlFile = getXmlFilePathFromArchive(importDir, archiveFile, xmlFileName); @@ -874,42 +869,42 @@ private static Path getImportXmlFile(@NotNull PipeRoot pipelineRoot, @NotNull Pa return xmlFile; } - public static @NotNull Path getXmlFilePathFromArchive(@NotNull Path importDir, Path archiveFile, @NotNull String xmlFileName) throws InvalidFileException + public static @NotNull FileLike getXmlFilePathFromArchive(@NotNull FileLike importDir, FileLike archiveFile, @NotNull String xmlFileName) throws InvalidFileException { // when importing a folder archive for a study, the study.xml file may not be at the root - if ("study.xml".equalsIgnoreCase(xmlFileName) && archiveFile.getFileName().toString().toLowerCase().endsWith(".folder.zip")) + if ("study.xml".equalsIgnoreCase(xmlFileName) && archiveFile.getName().toLowerCase().endsWith(".folder.zip")) { - File folderXml = new File(importDir.toFile(), "folder.xml"); + FileLike folderXml = importDir.resolveChild("folder.xml"); FolderDocument folderDoc; - try + try (InputStream in = folderXml.openInputStream()) { - folderDoc = FolderDocument.Factory.parse(folderXml, XmlBeansUtil.getDefaultParseOptions()); + folderDoc = FolderDocument.Factory.parse(in, XmlBeansUtil.getDefaultParseOptions()); XmlBeansUtil.validateXmlDocument(folderDoc, xmlFileName); } catch (Exception e) { - throw new InvalidFileException(folderXml.getParentFile().toPath(), folderXml.toPath(), e); + throw new InvalidFileException(folderXml.toString(), e); } if (folderDoc.getFolder().isSetStudy()) { - importDir = importDir.resolve(folderDoc.getFolder().getStudy().getDir()); + importDir = importDir.resolveFile(org.labkey.api.util.Path.parse(folderDoc.getFolder().getStudy().getDir())); } } - return importDir.toAbsolutePath().resolve(xmlFileName); + return importDir.resolveChild(xmlFileName); } - public static Path getArchiveXmlFile(Container container, Path archiveFile, String xmlFileName, BindException errors) throws InvalidFileException + public static FileLike getArchiveXmlFile(Container container, FileLike archiveFile, String xmlFileName, BindException errors) throws InvalidFileException { PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(container); - Path xmlFile = getImportXmlFile(pipelineRoot, archiveFile, xmlFileName, errors); + FileLike xmlFile = getImportXmlFile(pipelineRoot, archiveFile, xmlFileName, errors); // if this is an import from a source template folder that has been previously implicitly exported // to the unzip dir (without ever creating a zip file) then just look there for the xmlFile. - if (pipelineRoot != null && Files.isDirectory(archiveFile)) + if (pipelineRoot != null && archiveFile.isDirectory()) { - xmlFile = java.nio.file.Path.of(archiveFile.toString(), xmlFileName); + xmlFile = archiveFile.resolveChild(xmlFileName); } return xmlFile; diff --git a/pipeline/src/org/labkey/pipeline/api/PipelineServiceImpl.java b/pipeline/src/org/labkey/pipeline/api/PipelineServiceImpl.java index 3f2e1db3837..41a86ec5bfb 100644 --- a/pipeline/src/org/labkey/pipeline/api/PipelineServiceImpl.java +++ b/pipeline/src/org/labkey/pipeline/api/PipelineServiceImpl.java @@ -89,6 +89,7 @@ import org.labkey.pipeline.mule.ResumableDescriptor; import org.labkey.pipeline.status.PipelineQueryView; import org.labkey.pipeline.trigger.PipelineTriggerManager; +import org.labkey.vfs.FileLike; import org.mule.MuleManager; import org.mule.umo.UMODescriptor; import org.mule.umo.UMOException; @@ -320,10 +321,8 @@ public boolean hasValidPipelineRoot(Container container) @Override public Map getAllPipelineRoots() { - PipelineRoot[] pipelines = PipelineManager.getPipelineRoots(PRIMARY_ROOT); - Map result = new HashMap<>(); - for (PipelineRoot pipeline : pipelines) + for (PipelineRoot pipeline : PipelineManager.getPipelineRoots(PRIMARY_ROOT)) { PipeRoot p = new PipeRootImpl(pipeline); if (p.getContainer() != null) @@ -456,7 +455,7 @@ public List getClusterStartupArguments() { List args = new ArrayList<>(); args.add(System.getProperty("java.home") + "/bin/java" + (SystemUtils.IS_OS_WINDOWS ? ".exe" : "")); - File labkeyBootstrap = new File(new File(System.getProperty("catalina.home")), "labkeyBootstrap.jar"); + File labkeyBootstrap = FileUtil.appendName(new File(System.getProperty("catalina.home")), "labkeyBootstrap.jar"); if (!labkeyBootstrap.exists()) { @@ -754,7 +753,7 @@ public void setTriggeredTime(Container container, User user, int triggerConfigId } @Override - public boolean runFolderImportJob(Container c, User user, ActionURL url, Path folderXml, String originalFilename, PipeRoot pipelineRoot, ImportOptions options) + public boolean runFolderImportJob(Container c, User user, ActionURL url, FileLike folderXml, String originalFilename, PipeRoot pipelineRoot, ImportOptions options) { try { @@ -814,7 +813,7 @@ public boolean runGenerateFolderArchiveAndImportJob(Container c, User user, Acti public boolean runGenerateFolderArchiveAndImportJob(Container c, User user, ActionURL url, ImportOptions options) { PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(c); - Path folderXml = new File(pipelineRoot.getRootPath(), "folder.xml").toPath(); + FileLike folderXml = pipelineRoot.resolvePathToFileLike("folder.xml"); return runFolderImportJob(c, user, null, folderXml, "folder.xml", pipelineRoot, options); } @@ -848,10 +847,10 @@ public PathAnalysisProperties getFileAnalysisProperties(Container c, String task if (pr == null || !pr.isValid()) throw new NotFoundException(); - Path dirData = null; + FileLike dirData = null; if (path != null) { - dirData = pr.resolveToNioPath(path); + dirData = pr.resolvePathToFileLike(path); if (!NetworkDrive.exists(dirData)) throw new NotFoundException("Could not resolve path: " + path); } @@ -887,7 +886,7 @@ public String startFileAnalysis(AnalyzeForm form, @Nullable Map TaskPipeline taskPipeline = PipelineJobService.get().getTaskPipeline(form.getTaskId()); PathAnalysisProperties props = getFileAnalysisProperties(context.getContainer(), form.getTaskId(), form.getPath()); PipeRoot root = props.getPipeRoot(); - Path dirData = props.getDirData(); + FileLike dirData = props.getDirData(); AbstractFileAnalysisProtocolFactory factory = props.getFactory(); if (dirData == null) @@ -897,10 +896,10 @@ public String startFileAnalysis(AnalyzeForm form, @Nullable Map if (taskPipeline.isUseUniqueAnalysisDirectory()) { - dirData = FileUtil.appendName(dirData, form.getProtocolName() + "_" + FileUtil.getTimestamp()); - if (!Files.exists(FileUtil.createDirectories(dirData))) + dirData = dirData.resolveChild(form.getProtocolName() + "_" + FileUtil.getTimestamp()); + if (!FileUtil.createDirectory(dirData).exists()) { - throw new IOException("Failed to create unique analysis directory: " + FileUtil.getAbsoluteCaseSensitiveFile(dirData.toFile()).getAbsolutePath()); + throw new IOException("Failed to create unique analysis directory: " + FileUtil.getAbsoluteCaseSensitiveFile(dirData)); } } AbstractFileAnalysisProtocol protocol = factory.getProtocol(root, dirData, form.getProtocolName(), false); @@ -964,16 +963,16 @@ public String startFileAnalysis(AnalyzeForm form, @Nullable Map protocol.getFactory().ensureDefaultParameters(root); - Path fileParameters = protocol.getParametersFile(dirData, root); + FileLike fileParameters = protocol.getParametersFile(dirData, root); // Make sure configure.xml file exists for the job when it runs. - if (fileParameters != null && !Files.exists(fileParameters)) + if (fileParameters != null && !fileParameters.exists()) { protocol.setEmail(context.getUser().getEmail()); protocol.saveInstance(fileParameters, context.getContainer()); } boolean allowNonExistentFiles = form.isAllowNonExistentFiles() != null ? form.isAllowNonExistentFiles() : false; - List filesInputList = form.getValidatedPaths(context.getContainer(), allowNonExistentFiles); + List filesInputList = form.getValidatedFiles(context.getContainer(), allowNonExistentFiles); if (form.isActiveJobs()) { @@ -982,17 +981,17 @@ public String startFileAnalysis(AnalyzeForm form, @Nullable Map if (taskPipeline.isUseUniqueAnalysisDirectory()) { - for (Path inputFile : filesInputList) + for (FileLike inputFile : filesInputList) { try { - Files.move(inputFile, FileUtil.appendName(dirData, inputFile.getFileName().toString())); + Files.move(inputFile.toNioPathForWrite(), dirData.resolveChild(inputFile.getName()).toNioPathForWrite()); } catch (IOException e) { if (!allowNonExistentFiles) { - throw new IOException("Failed to move input file into unique directory: " + FileUtil.getAbsoluteCaseSensitivePath(context.getContainer(), inputFile).toAbsolutePath()); + throw new IOException("Failed to move input file into unique directory: " + FileUtil.getAbsoluteCaseSensitiveFile(inputFile)); } } } diff --git a/pipeline/src/org/labkey/pipeline/importer/FolderImportJob.java b/pipeline/src/org/labkey/pipeline/importer/FolderImportJob.java index 3e61809e97c..b18cc778537 100644 --- a/pipeline/src/org/labkey/pipeline/importer/FolderImportJob.java +++ b/pipeline/src/org/labkey/pipeline/importer/FolderImportJob.java @@ -39,6 +39,7 @@ import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.api.writer.FileSystemFile; import org.labkey.api.writer.VirtualFile; +import org.labkey.vfs.FileLike; import java.nio.file.Path; @@ -69,13 +70,13 @@ protected FolderImportJob(@JsonProperty("_ctx") FolderImportContext ctx, @JsonPr _ctx.setLoggerGetter(new PipelineJobLoggerGetter(this)); } - public FolderImportJob(Container c, User user, ActionURL url, Path folderXml, String originalFilename, PipeRoot pipeRoot, ImportOptions options) + public FolderImportJob(Container c, User user, ActionURL url, FileLike folderXml, String originalFilename, PipeRoot pipeRoot, ImportOptions options) { super("FolderImport", new ViewBackgroundInfo(c, user, url), pipeRoot); - _root = new FileSystemFile(folderXml.getParent()); + _root = new FileSystemFile(folderXml.getParent().toNioPathForRead()); _originalFilename = originalFilename; _folderArchiveSourceName = options.getFolderArchiveSourceName(); // Optional FolderArchiveSource name. If non-null, will be invoked to generate the archive before import. - setupLocalDirectoryAndJobLog(pipeRoot, "FolderImport", FolderImportProvider.generateLogFilename("folder_load")); + setupLocalDirectoryAndJobLog(pipeRoot, FolderImportProvider.generateLogFilename("folder_load")); _ctx = new FolderImportContext(user, c, folderXml, options.getDataTypes(), new PipelineJobLoggerGetter(this), _root); _ctx.setSkipQueryValidation(options.isSkipQueryValidation()); _ctx.setCreateSharedDatasets(options.isCreateSharedDatasets()); @@ -112,7 +113,7 @@ public String getFolderArchiveSourceName() } @Override - public TaskPipeline getTaskPipeline() + public TaskPipeline getTaskPipeline() { return PipelineJobService.get().getTaskPipeline(new TaskId(FolderImportJob.class)); } diff --git a/pipeline/src/org/labkey/pipeline/importer/FolderImportTask.java b/pipeline/src/org/labkey/pipeline/importer/FolderImportTask.java index e31bf15e830..2d703a034ef 100644 --- a/pipeline/src/org/labkey/pipeline/importer/FolderImportTask.java +++ b/pipeline/src/org/labkey/pipeline/importer/FolderImportTask.java @@ -75,13 +75,13 @@ public RecordedActionSet run() throws PipelineJobException { FileAnalysisJobSupport support = job.getJobSupport(FileAnalysisJobSupport.class); ImportOptions options = new ImportOptions(job.getContainerId(), job.getUser().getUserId()); - options.setAnalysisDir(support.getDataDirectory().toPath()); + options.setAnalysisDir(support.getDataDirectoryFileLike()); - job = new FolderImportJob(job.getContainer(), job.getUser(), null, support.findInputPath(FOLDER_XML), FOLDER_XML, job.getPipeRoot(), options); + job = new FolderImportJob(job.getContainer(), job.getUser(), null, support.findInputFileLike(FOLDER_XML), FOLDER_XML, job.getPipeRoot(), options); job.setStatus(PipelineJob.TaskStatus.running.toString(), "Starting folder import job", true); importContext = ((FolderImportJob) job).getImportContext(); - vf = new FileSystemFile(support.getDataDirectory()); + vf = new FileSystemFile(support.getDataDirectoryFileLike()); } /* Standard Pipeline triggered job */ else @@ -165,7 +165,7 @@ public Factory() } @Override - public PipelineJob.Task createTask(PipelineJob job) + public FolderImportTask createTask(PipelineJob job) { return new FolderImportTask(this, job); } diff --git a/specimen/src/org/labkey/specimen/actions/SpecimenController.java b/specimen/src/org/labkey/specimen/actions/SpecimenController.java index d3d180977fb..2d05110bb03 100644 --- a/specimen/src/org/labkey/specimen/actions/SpecimenController.java +++ b/specimen/src/org/labkey/specimen/actions/SpecimenController.java @@ -186,6 +186,7 @@ import org.labkey.specimen.view.SpecimenRequestNotificationEmailTemplate; import org.labkey.specimen.view.SpecimenSearchWebPart; import org.labkey.specimen.view.SpecimenWebPart; +import org.labkey.vfs.FileLike; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.validation.ObjectError; @@ -919,7 +920,7 @@ public boolean isMerge() } } - public static void submitSpecimenBatch(Container c, User user, ActionURL url, File f, PipeRoot root, boolean merge) throws IOException + public static void submitSpecimenBatch(Container c, User user, ActionURL url, FileLike f, PipeRoot root, boolean merge) throws IOException { if (null == f || !f.exists() || !f.isFile()) throw new NotFoundException(); @@ -942,7 +943,7 @@ public boolean handlePost(PipelineForm form, BindException errors) throws Except Container c = getContainer(); PipeRoot root = PipelineService.get().findPipelineRoot(c); boolean first = true; - for (File f : form.getValidatedFiles(c)) + for (FileLike f : form.getValidatedFiles(c)) { // Only possibly overwrite when the first archive is loaded: boolean merge = !first || form.isMerge(); @@ -977,13 +978,13 @@ public boolean handlePost(PipelineForm form, BindException errors) throws Except { Container c = getContainer(); String path = form.getPath(); - File f = null; + FileLike f = null; PipeRoot root = PipelineService.get().findPipelineRoot(c); if (path != null) { if (root != null) - f = root.resolvePath(path); + f = root.resolvePathToFileLike(path); } submitSpecimenBatch(c, getUser(), getViewContext().getActionURL(), f, root, form.isMerge()); @@ -1083,18 +1084,18 @@ public class ImportSpecimenDataAction extends SimpleViewAction @Override public ModelAndView getView(PipelineForm form, BindException bindErrors) { - List dataFiles = form.getValidatedFiles(getContainer()); + List dataFiles = form.getValidatedFiles(getContainer()); List archives = new ArrayList<>(); List errors = new ArrayList<>(); _filePaths = form.getFile(); - for (File dataFile : dataFiles) + for (FileLike dataFile : dataFiles) { if (null == dataFile || !dataFile.exists() || !dataFile.isFile()) { throw new NotFoundException(); } - if (!dataFile.canRead()) + if (!dataFile.toNioPathForRead().toFile().canRead()) errors.add("Can't read data file: " + dataFile); SpecimenArchive archive = new SpecimenArchive(dataFile); diff --git a/specimen/src/org/labkey/specimen/pipeline/SpecimenArchive.java b/specimen/src/org/labkey/specimen/pipeline/SpecimenArchive.java index dc43d69d266..781b8d8681a 100644 --- a/specimen/src/org/labkey/specimen/pipeline/SpecimenArchive.java +++ b/specimen/src/org/labkey/specimen/pipeline/SpecimenArchive.java @@ -19,8 +19,8 @@ import org.labkey.api.data.Container; import org.labkey.api.study.SpecimenService; import org.labkey.api.study.SpecimenTransform; +import org.labkey.vfs.FileLike; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Date; @@ -36,14 +36,14 @@ */ public class SpecimenArchive { - private final File _definitionFile; + private final FileLike _definitionFile; - public SpecimenArchive(File definitionFile) + public SpecimenArchive(FileLike definitionFile) { _definitionFile = definitionFile; } - public File getDefinitionFile() + public FileLike getDefinitionFile() { return _definitionFile; } @@ -56,13 +56,13 @@ public List getEntryDescriptions(Container container) throws I { if (transform.getFileType().isType(_definitionFile)) { - entryList.add(new EntryDescription(_definitionFile.getName(), _definitionFile.length(), new Date(_definitionFile.lastModified()))); + entryList.add(new EntryDescription(_definitionFile.getName(), _definitionFile.getSize(), new Date(_definitionFile.getLastModified()))); return entryList; } } // standard non-transformed specimen archive - try (ZipFile zip = new ZipFile(_definitionFile)) + try (ZipFile zip = new ZipFile(_definitionFile.toNioPathForRead().toFile())) { Enumeration entries = zip.entries(); while (entries.hasMoreElements()) diff --git a/specimen/src/org/labkey/specimen/pipeline/SpecimenBatch.java b/specimen/src/org/labkey/specimen/pipeline/SpecimenBatch.java index e072c092d94..b9c6a2acbcd 100644 --- a/specimen/src/org/labkey/specimen/pipeline/SpecimenBatch.java +++ b/specimen/src/org/labkey/specimen/pipeline/SpecimenBatch.java @@ -29,6 +29,7 @@ import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.Serializable; @@ -48,7 +49,7 @@ public class SpecimenBatch extends StudyBatch implements Serializable, SpecimenJ // For serialization protected SpecimenBatch() {} - public SpecimenBatch(ViewBackgroundInfo info, File definitionFile, PipeRoot root, boolean merge) + public SpecimenBatch(ViewBackgroundInfo info, FileLike definitionFile, PipeRoot root, boolean merge) { super(info, definitionFile, root); _isMerge = merge; @@ -78,7 +79,7 @@ public ActionURL getStatusHref() @Override public Path getSpecimenArchivePath() { - return _definitionFile.toPath(); + return _definitionFile.toNioPathForRead(); } @Override diff --git a/specimen/src/org/labkey/specimen/pipeline/SpecimenReloadJob.java b/specimen/src/org/labkey/specimen/pipeline/SpecimenReloadJob.java index c635f80c900..4ddbbb6e6ac 100644 --- a/specimen/src/org/labkey/specimen/pipeline/SpecimenReloadJob.java +++ b/specimen/src/org/labkey/specimen/pipeline/SpecimenReloadJob.java @@ -22,6 +22,7 @@ import org.labkey.api.study.SpecimenTransform; import org.labkey.api.util.FileUtil; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.Serializable; @@ -41,13 +42,13 @@ public SpecimenReloadJob(ViewBackgroundInfo info, PipeRoot root, String transfor { super(info, null, root, false); - File logFile = new File(root.getRootPath(), FileUtil.makeFileNameWithTimestamp("specimen_reload", "log")); + FileLike logFile = root.resolvePathToFileLike(FileUtil.makeFileNameWithTimestamp("specimen_reload", "log")); setLogFile(logFile); _transformName = transformName; } @Override - public void setSpecimenArchive(File archiveFile) + public void setSpecimenArchive(FileLike archiveFile) { _definitionFile = archiveFile; } diff --git a/specimen/src/org/labkey/specimen/pipeline/SpecimenReloadJobSupport.java b/specimen/src/org/labkey/specimen/pipeline/SpecimenReloadJobSupport.java index 42f6a61a49c..2ecc7cd0306 100644 --- a/specimen/src/org/labkey/specimen/pipeline/SpecimenReloadJobSupport.java +++ b/specimen/src/org/labkey/specimen/pipeline/SpecimenReloadJobSupport.java @@ -16,6 +16,7 @@ package org.labkey.specimen.pipeline; import org.labkey.api.study.SpecimenTransform; +import org.labkey.vfs.FileLike; import java.io.File; @@ -24,7 +25,7 @@ */ public interface SpecimenReloadJobSupport extends SpecimenJobSupport { - void setSpecimenArchive(File archiveFile); + void setSpecimenArchive(FileLike archiveFile); String getSpecimenTransform(); diff --git a/specimen/src/org/labkey/specimen/pipeline/SpecimenReloadTask.java b/specimen/src/org/labkey/specimen/pipeline/SpecimenReloadTask.java index 9cd2bf6de14..6a88f0e86a9 100644 --- a/specimen/src/org/labkey/specimen/pipeline/SpecimenReloadTask.java +++ b/specimen/src/org/labkey/specimen/pipeline/SpecimenReloadTask.java @@ -58,8 +58,7 @@ public RecordedActionSet run() throws PipelineJobException PipeRoot root = PipelineService.get().findPipelineRoot(job.getContainer()); if (root != null) { - FileLike archiveFileLike = root.getRootFileLike().resolveChild(FileUtil.makeFileNameWithTimestamp("specimen_reload", transform.getFileType().getDefaultSuffix())); - File archive = FileSystemLike.toFile(archiveFileLike); + FileLike archive = root.getRootFileLike().resolveChild(FileUtil.makeFileNameWithTimestamp("specimen_reload", transform.getFileType().getDefaultSuffix())); transform.importFromExternalSource(job, support.getExternalImportConfig(), archive); support.setSpecimenArchive(archive); diff --git a/study/api-src/org/labkey/api/study/pipeline/StudyBatch.java b/study/api-src/org/labkey/api/study/pipeline/StudyBatch.java index 6e2564b018c..fe8382aee8c 100644 --- a/study/api-src/org/labkey/api/study/pipeline/StudyBatch.java +++ b/study/api-src/org/labkey/api/study/pipeline/StudyBatch.java @@ -24,6 +24,7 @@ import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.vfs.FileLike; import java.io.File; import java.io.IOException; @@ -36,12 +37,12 @@ */ public abstract class StudyBatch extends PipelineJob implements Serializable { - protected File _definitionFile; + protected FileLike _definitionFile; // For serialization protected StudyBatch() {} - public StudyBatch(ViewBackgroundInfo info, File definitionFile, PipeRoot root) + public StudyBatch(ViewBackgroundInfo info, FileLike definitionFile, PipeRoot root) { super("Study", info, root); _definitionFile = definitionFile; @@ -74,9 +75,4 @@ public void submit() throws IOException throw new IOException(e); } } - - public File getDefinitionFile() - { - return _definitionFile; - } } diff --git a/study/src/org/labkey/study/controllers/StudyController.java b/study/src/org/labkey/study/controllers/StudyController.java index 55aa0fee64b..3ab2ad09db9 100644 --- a/study/src/org/labkey/study/controllers/StudyController.java +++ b/study/src/org/labkey/study/controllers/StudyController.java @@ -287,6 +287,7 @@ import org.labkey.study.visitmanager.VisitManager; import org.labkey.study.visitmanager.VisitManager.VisitStatistic; import org.labkey.study.xml.DatasetsDocument; +import org.labkey.vfs.FileLike; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; @@ -3953,7 +3954,7 @@ public void validateCommand(ResetPipelinePathForm form, Errors errors) @Override public boolean handlePost(ResetPipelinePathForm form, BindException errors) throws Exception { - for (File f : form.getValidatedFiles(getContainer())) + for (FileLike f : form.getValidatedFiles(getContainer())) { if (f.isFile() && f.getName().endsWith(".lock")) { @@ -4120,7 +4121,7 @@ public ModelAndView getView(PipelinePathForm form, BindException errors) throws { Container c = getContainer(); - File definitionFile = form.getValidatedSingleFile(c); + File definitionFile = form.getValidatedSingleFile(c).toNioPathForRead().toFile(); path = form.getPath(); if (!path.endsWith("/")) { diff --git a/study/src/org/labkey/study/controllers/publish/PublishController.java b/study/src/org/labkey/study/controllers/publish/PublishController.java index 16a5e29c0e9..c675c1e3aaa 100644 --- a/study/src/org/labkey/study/controllers/publish/PublishController.java +++ b/study/src/org/labkey/study/controllers/publish/PublishController.java @@ -314,7 +314,7 @@ public AutoLinkPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot pipeRoot, _runIds = form.getRunId(); _autoLinkCategory = form.getAutoLinkCategory(); - setLogFile(FileUtil.appendName(pipeRoot.getRootPath(), FileUtil.makeFileNameWithTimestamp("auto_link_to_study", "log")).toPath()); + setLogFile(pipeRoot.resolvePathToFileLike(FileUtil.makeFileNameWithTimestamp("auto_link_to_study", "log"))); } @Override diff --git a/study/src/org/labkey/study/importer/CreateChildStudyPipelineJob.java b/study/src/org/labkey/study/importer/CreateChildStudyPipelineJob.java index c6e959829d7..5c98fa3eee2 100644 --- a/study/src/org/labkey/study/importer/CreateChildStudyPipelineJob.java +++ b/study/src/org/labkey/study/importer/CreateChildStudyPipelineJob.java @@ -327,7 +327,7 @@ public boolean run(ViewContext context) } finally { - if (!success && _destFolderCreated) + if (!success && _destFolderCreated && getDstContainer() != null) ContainerManager.delete(getDstContainer(), getUser()); } diff --git a/study/src/org/labkey/study/importer/StudyImporterFactory.java b/study/src/org/labkey/study/importer/StudyImporterFactory.java index 8519052b6d5..2692e0d40ad 100644 --- a/study/src/org/labkey/study/importer/StudyImporterFactory.java +++ b/study/src/org/labkey/study/importer/StudyImporterFactory.java @@ -155,7 +155,7 @@ public void process(@Nullable PipelineJob job, FolderImportContext ctx, VirtualF if (useLocalImportDir) { //TODO this should be done from the import context getSpecimenArchive specimenFile = job.getPipeRoot().getRootNioPath().relativize(specimenFile); - specimenFile = job.getPipeRoot().getImportDirectory().toPath().resolve(specimenFile); + specimenFile = job.getPipeRoot().getImportDirectory().toNioPathForRead().resolve(specimenFile); } SpecimenMigrationService.get().importSpecimenArchive(specimenFile, job, studyImportContext, false, false); diff --git a/study/src/org/labkey/study/model/StudyManager.java b/study/src/org/labkey/study/model/StudyManager.java index 207daa48924..409da6b5536 100644 --- a/study/src/org/labkey/study/model/StudyManager.java +++ b/study/src/org/labkey/study/model/StudyManager.java @@ -2795,9 +2795,9 @@ private void deleteStudyDesignData(Container c, User user, List study { for (TableInfo tinfo : studyDesignTables) { - if (tinfo instanceof FilteredTable) + if (tinfo instanceof FilteredTable ft) { - Table.delete(((FilteredTable)tinfo).getRealTable(), new SimpleFilter(FieldKey.fromParts("Container"), c)); + Table.delete(ft.getRealTable(), new SimpleFilter(FieldKey.fromParts("Container"), c)); } } } diff --git a/study/src/org/labkey/study/visitmanager/PurgeParticipantsJob.java b/study/src/org/labkey/study/visitmanager/PurgeParticipantsJob.java index 118dc10328a..59a7d774e41 100644 --- a/study/src/org/labkey/study/visitmanager/PurgeParticipantsJob.java +++ b/study/src/org/labkey/study/visitmanager/PurgeParticipantsJob.java @@ -29,7 +29,7 @@ public PurgeParticipantsJob() PurgeParticipantsJob(ViewBackgroundInfo info, PipeRoot pipeRoot) { super("StudyParticipantPurge", info, pipeRoot); - setLogFile(pipeRoot.getLogDirectoryFileLike(true).resolveChild(FileUtil.makeFileNameWithTimestamp("purge_participants", "log")).toNioPathForWrite()); + setLogFile(pipeRoot.getLogDirectoryFileLike(true).resolveChild(FileUtil.makeFileNameWithTimestamp("purge_participants", "log"))); } @Override diff --git a/study/test/src/org/labkey/test/tests/study/TargetStudyTest.java b/study/test/src/org/labkey/test/tests/study/TargetStudyTest.java index f88217e037a..db6cf90735b 100644 --- a/study/test/src/org/labkey/test/tests/study/TargetStudyTest.java +++ b/study/test/src/org/labkey/test/tests/study/TargetStudyTest.java @@ -180,7 +180,7 @@ protected void uploadRuns() click(AssayConstants.TEXT_AREA_DATA_PROVIDER_LOCATOR); String data2 = data1.replace("StudyNotExist", ""); - setFormElement(Locator.name("TextAreaDataCollector.textArea"), data2); + setFormElement(Locator.name(AssayConstants.TEXT_AREA_DATA_COLLECTOR_TEXT_AREA_NAME), data2); clickButton("Save and Finish"); assertNoLabKeyErrors();