Add audit logging for file fields and renaming of files

api/src/org/labkey/api/ApiModule.java

@@ -25,7 +25,6 @@
 import org.labkey.api.action.ApiXmlWriter;
 import org.labkey.api.action.SpringActionController;
 import org.labkey.api.admin.SubfolderWriter;
-import org.labkey.api.assay.AssayFileWriter;
 import org.labkey.api.assay.AssayResultsFileWriter;
 import org.labkey.api.assay.ReplacedRunFilter;
 import org.labkey.api.assay.sample.MaterialInputRoleComparator;
@@ -445,7 +444,6 @@ public void registerServlets(ServletContext servletCtx)
             Table.IsSelectTestCase.class,
             ValidEmail.TestCase.class,
             URIUtil.TestCase.class,
-            AssayFileWriter.TestCase.class,
             AssayResultsFileWriter.TestCase.class
         );
     }

api/src/org/labkey/api/action/AbstractFileUploadAction.java

@@ -15,6 +15,8 @@
  */
 package org.labkey.api.action;
 
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
 import org.labkey.api.util.ExceptionUtil;
 import org.labkey.api.util.FileUtil;
 import org.labkey.api.util.PageFlowUtil;
@@ -28,8 +30,6 @@
 import org.springframework.web.multipart.MultipartHttpServletRequest;
 import org.springframework.web.servlet.ModelAndView;
 
-import jakarta.servlet.http.HttpServletRequest;
-import jakarta.servlet.http.HttpServletResponse;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileOutputStream;

api/src/org/labkey/api/assay/AbstractAssayProvider.java

@@ -31,6 +31,7 @@
 import org.labkey.api.assay.transform.DataExchangeHandler;
 import org.labkey.api.assay.transform.DataTransformService;
 import org.labkey.api.audit.AuditLogService;
+import org.labkey.api.audit.provider.FileSystemAuditProvider;
 import org.labkey.api.data.ActionButton;
 import org.labkey.api.data.ButtonBar;
 import org.labkey.api.data.ColumnInfo;
@@ -136,7 +137,6 @@
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.ListIterator;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -1798,7 +1798,7 @@ public record AssayFileMoveData(ExpRun run, Container sourceContainer, String fi
     public record AssayMoveData(Map<String, Integer> counts, Map<Integer, List<AssayFileMoveData>> fileMovesByRunId) {}
 
     @Override
-    public void moveRuns(List<ExpRun> runs, Container targetContainer, User user, AbstractAssayProvider.AssayMoveData assayMoveData) throws ExperimentException
+    public void moveRuns(List<ExpRun> runs, Container targetContainer, User user, AssayMoveData assayMoveData) throws ExperimentException
     {
         if (runs.isEmpty())
             return;
@@ -1968,7 +1968,7 @@ private void moveRunsBatch(List<ExpRun> runs, Container sourceContainer, Contain
                     if (updatedFile != null)
                     {
                         if (!fileMoveReferences.containsKey(sourceFileName))
-                            fileMoveReferences.put(sourceFileName, new AssayFileMoveReference(sourceFileName, updatedFile, experiment.getName()));
+                            fileMoveReferences.put(sourceFileName, new AssayFileMoveReference(sourceFileName, updatedFile, experiment.getName(), fileProp.getName()));
 
                         if (!fileMoveCounts.containsKey(sourceFileName))
                             fileMoveCounts.put(sourceFileName, 0);
@@ -1991,6 +1991,11 @@ private void moveRunsBatch(List<ExpRun> runs, Container sourceContainer, Contain
                     throw new ExperimentException("Assay batch " + ref.runName + " cannot be moved since it references a shared file: " + sourceFile.getName());
 
                 fileContentService.fireFileMoveEvent(sourceFile.toPath(), ref.updatedFile.toPath(), user, sourceContainer, targetContainer);
+                FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + sourceContainer.getPath() + " to " + targetContainer.getPath() + ".");
+                event.setProvidedFileName(sourceFile.getName());
+                event.setFile(ref.updatedFile.getName());
+                event.setDirectory(ref.updatedFile.getParent());
+                AuditLogService.get().addEvent(user, event);
             }
 
         }
@@ -2050,7 +2055,7 @@ private void updateRunFiles(List<ExpRun> runs, Container sourceContainer, Contai
                 if (updatedFile != null)
                 {
                     if (!fileMoveReferences.containsKey(sourceFileName))
-                        fileMoveReferences.put(sourceFileName, new AssayFileMoveReference(sourceFileName, updatedFile, run.getName()));
+                        fileMoveReferences.put(sourceFileName, new AssayFileMoveReference(sourceFileName, updatedFile, run.getName(), fileProp.getName()));
 
                     if (!fileMoveCounts.containsKey(sourceFileName))
                         fileMoveCounts.put(sourceFileName, 0);
@@ -2072,6 +2077,12 @@ private void updateRunFiles(List<ExpRun> runs, Container sourceContainer, Contai
                 throw new ExperimentException("Assay run " + ref.runName + " cannot be moved since it references a shared file: " + sourceFile.getName());
 
             fileContentService.fireFileMoveEvent(sourceFile.toPath(), ref.updatedFile.toPath(), user, sourceContainer, targetContainer);
+            FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + sourceContainer.getPath() + " to " + targetContainer.getPath() + ".");
+            event.setProvidedFileName(sourceFile.getName());
+            event.setFile(ref.updatedFile.getName());
+            event.setDirectory(ref.updatedFile.getParent());
+            event.setFieldName(ref.fieldName);
+            AuditLogService.get().addEvent(user, event);
         }
 
     }
@@ -2116,6 +2127,11 @@ private void updateDataFileUrl(List<ExpRun> runs, Container sourceContainer, Con
                     movedFiles.putIfAbsent(runId, new ArrayList<>());
                     movedFiles.get(runId).add(new AssayFileMoveData(run, run.getContainer(), null, sourceFile, updatedFile));
                     fileContentService.fireFileMoveEvent(sourceFile.toPath(), updatedFile.toPath(), user, sourceContainer, targetContainer);
+                    FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + sourceContainer.getPath() + " to " + targetContainer.getPath() + ".");
+                    event.setProvidedFileName(sourceFile.getName());
+                    event.setFile(updatedFile.getName());
+                    event.setDirectory(updatedFile.getParent());
+                    AuditLogService.get().addEvent(user, event);
                 }
             }
             catch (Exception e)
@@ -2136,7 +2152,7 @@ protected void moveAssayResults(List<ExpRun> runs, ExpProtocol protocol, Contain
         updateResultFiles(assayResultTable, runs, protocol, sourceContainer, targetContainer, user, assayMoveData);
     }
 
-    record AssayFileMoveReference(String sourceFilePath, File updatedFile, String runName) {}
+    record AssayFileMoveReference(String sourceFilePath, File updatedFile, String runName, String fieldName) {}
 
     private void updateResultFiles(FilteredTable assayResultTable, List<ExpRun> runs, ExpProtocol assayProtocol, Container sourceContainer, Container targetContainer, User user, AssayMoveData assayMoveData) throws ExperimentException
     {
@@ -2169,10 +2185,10 @@ private void updateResultFiles(FilteredTable assayResultTable, List<ExpRun> runs
         Map<Integer, ExpRun> runMap = new HashMap<>();
         runs.forEach(run -> runMap.put(run.getRowId(), run));
 
-        Map<String, AssayFileMoveReference> fileMoveReferences = new HashMap<>();
-        Map<String, List<Integer>> fileMoveResultRowIds = new HashMap<>();
         for (String fileField : fileFields)
         {
+            Map<String, AssayFileMoveReference> fileMoveReferences = new HashMap<>();
+            Map<String, List<Integer>> fileMoveResultRowIds = new HashMap<>();
             var fileColumn = assayResultTable.getColumn(fileField);
             TableSelector ts = new TableSelector(assayResultTable, assayResultTable.getColumns("rowid", "run", fileField), filter, null);
             Map<String, Object>[] resultFiles = ts.getMapArray();
@@ -2194,7 +2210,7 @@ private void updateResultFiles(FilteredTable assayResultTable, List<ExpRun> runs
                     ExpRun run = runMap.get(resultRunId);
 
                     if (!fileMoveReferences.containsKey(sourceFileName))
-                        fileMoveReferences.put(sourceFileName, new AssayFileMoveReference(sourceFileName, updatedFile, run.getName()));
+                        fileMoveReferences.put(sourceFileName, new AssayFileMoveReference(sourceFileName, updatedFile, run.getName(),fileField));
 
                     movedFiles.putIfAbsent(resultRunId, new ArrayList<>());
                     movedFiles.get(resultRunId).add(new AssayFileMoveData(run, run.getContainer(), fileField, sourceFile, updatedFile));
@@ -2234,6 +2250,12 @@ private void updateResultFiles(FilteredTable assayResultTable, List<ExpRun> runs
                         .append(" WHERE rowId ").appendInClause(fileMoveResultRowIds.get(sourceFileName), realTable.getSqlDialect());
                 new SqlExecutor(assayResultTable.getSchema()).execute(updateSql);
 
+                FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + sourceContainer.getPath() + " to " + targetContainer.getPath() + ".");
+                event.setProvidedFileName(sourceFile.getName());
+                event.setFile(updatedFile.getName());
+                event.setDirectory(updatedFile.getParent());
+                event.setFieldName(ref.fieldName);
+                AuditLogService.get().addEvent(user, event);
             }
 
         }

api/src/org/labkey/api/assay/AbstractTempDirDataCollector.java

@@ -16,16 +16,18 @@
 package org.labkey.api.assay;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
 import org.labkey.api.assay.actions.AssayRunUploadForm;
+import org.labkey.api.audit.AuditLogService;
+import org.labkey.api.audit.provider.FileSystemAuditProvider;
 import org.labkey.api.exp.ExperimentException;
 import org.labkey.api.exp.api.ExpData;
 import org.labkey.api.exp.api.ExpRun;
 import org.labkey.api.query.BatchValidationException;
 import org.labkey.api.util.FileUtil;
 import org.labkey.api.util.NetworkDrive;
+import org.labkey.api.util.logging.LogHelper;
 import org.labkey.vfs.FileLike;
 
 import java.io.File;
@@ -43,17 +45,16 @@
 public abstract class AbstractTempDirDataCollector<ContextType extends AssayRunUploadContext<? extends AssayProvider>> extends AbstractAssayDataCollector<ContextType>
 {
     protected boolean _uploadComplete = false;
-    private static final String TMPFILE = "tmp";
 
-    private static final Logger LOG = LogManager.getLogger(AbstractTempDirDataCollector.class);
+    private static final Logger LOG = LogHelper.getLogger(AbstractTempDirDataCollector.class, "Activity related to data collection into a temporary directory");
 
     private void removeTempDir(ContextType context) throws ExperimentException
     {
         // TODO: remove this instanceof check
         if (!(context instanceof AssayRunUploadForm))
             return;
 
-        String uploadAttemptID = ((AssayRunUploadForm)context).getUploadAttemptID();
+        String uploadAttemptID = ((AssayRunUploadForm<?>) context).getUploadAttemptID();
 
         // Cleanup files other than input generated by transform scripts
         FileLike tempDir = ensureSubdirectory(context.getContainer(), TEMP_DIR_NAME);
@@ -71,24 +72,6 @@ private void removeTempDir(ContextType context) throws ExperimentException
         }
     }
 
-    private void removeFiles(ContextType context, Collection<Map<String, File>> assayFiles) throws ExperimentException
-    {
-        for (Map<String, File> fileMap : assayFiles)
-        {
-            for (File file : fileMap.values())
-            {
-                FileLike assayFile = ensureUploadDirectory(context.getContainer(), DIR_NAME).resolveChild(file.getName());
-                FileUtils.deleteQuietly(assayFile.toNioPathForWrite().toFile());
-            }
-        }
-    }
-
-    protected void uploadFailed(ContextType context, List<Map<String, File>> assayFiles) throws ExperimentException
-    {
-        removeTempDir(context);
-        removeFiles(context, assayFiles);
-    }
-
     @Override
     public void initDir(ContextType context) throws ExperimentException
     {
@@ -99,7 +82,7 @@ public void initDir(ContextType context) throws ExperimentException
             if (!(context instanceof AssayRunUploadForm))
                 return;
 
-            String uploadAttemptID = ((AssayRunUploadForm)context).getUploadAttemptID();
+            String uploadAttemptID = ((AssayRunUploadForm<?>)context).getUploadAttemptID();
 
             // Cleanup transform script output files if generated by warning feature
             FileLike tempDir = AssayFileWriter.ensureSubdirectory(context.getContainer(), TEMP_DIR_NAME);
@@ -140,7 +123,7 @@ protected FileLike getFileTargetDir(ContextType context) throws ExperimentExcept
             try
             {
                 FileLike tempDir = ensureSubdirectory(context.getContainer(), TEMP_DIR_NAME);
-                uploadAttemptDir = tempDir.resolveChild(((AssayRunUploadForm)context).getUploadAttemptID());
+                uploadAttemptDir = tempDir.resolveChild(((AssayRunUploadForm<?>)context).getUploadAttemptID());
 
                 if (!NetworkDrive.exists(uploadAttemptDir))
                 {
@@ -169,7 +152,7 @@ protected FileLike getFileTargetDir(ContextType context) throws ExperimentExcept
     protected FileLike getFilePath(ContextType context, @Nullable ExpRun run, FileLike tempDirFile) throws ExperimentException
     {
         FileLike assayDir = ensureUploadDirectory(context.getContainer());
-        return findUniqueFileName(tempDirFile.getName(), assayDir);
+        return FileUtil.findUniqueFileName(tempDirFile.getName(), assayDir);
     }
 
     // This is the default case to move the primary file from the temp directory to the assayData directory
@@ -229,10 +212,15 @@ public Map<String, FileLike> uploadComplete(ContextType context, @Nullable ExpRu
                         run.save(context.getUser());
                     }
                     handleTempFile(tempDirFile.toNioPathForWrite().toFile(), assayDirFile.toNioPathForWrite().toFile());
+                    FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(context.getContainer(),  "Primary file provided for assay import");
+                    event.setProvidedFileName(tempDirFile.getName());
+                    event.setFile(assayDirFile.getName());
+                    event.setDirectory(assayDirFile.getParent().toURI().getPath());
+                    AuditLogService.get().addEvent(context.getUser(), event);
                 }
                 else
                 {
-                    LOG.warn("Unable to resolve import/upload file location for file: " + tempDirFile);
+                    LOG.warn("Unable to resolve import/upload file location for file: {}", tempDirFile);
                 }
             }
             FileUtils.deleteDirectory(tempDir.toNioPathForWrite().toFile());