Configurable docroot via MicroProfile Config

doc/sphinx-guides/source/installation/config.rst

@@ -1771,8 +1771,8 @@ protocol, host, and port number and should not include a trailing slash.
 dataverse.files.directory
 +++++++++++++++++++++++++
 
-Please provide an absolute path to a directory backed by some mounted file system. This directory is used for a number
-of purposes:
+Providing an explicit location here makes it easier to reuse some mounted filesystem and we recommend doing so
+to avoid filled up disks, aid in performance, etc. This directory is used for a number of purposes:
 
 1. ``<dataverse.files.directory>/temp`` after uploading, data is temporarily stored here for ingest and/or before
    shipping to the final storage destination.
@@ -1785,24 +1785,51 @@ of purposes:
    under certain conditions. This directory may also be used by file stores for :ref:`permanent file storage <storage-files-dir>`,
    but this is controlled by other, store-specific settings.
 
-Defaults to ``/tmp/dataverse``. Can also be set via *MicroProfile Config API* sources, e.g. the environment variable
-``DATAVERSE_FILES_DIRECTORY``. Defaults to ``${STORAGE_DIR}`` for profile ``ct``, important for the
-:ref:`Dataverse Application Image <app-locations>`.
+Notes:
+
+- Please provide an absolute path to a directory backed by some mounted file system.
+- Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_DIRECTORY``.
+- Defaults to ``/tmp/dataverse`` in a :doc:`default installation <installation-main>`.
+- Defaults to ``${STORAGE_DIR}`` using our :ref:`Dataverse container <app-locations>` (resolving to ``/dv``).
+- During startup, this directory will be checked for existence and write access. It will be created for you
+  if missing. If it cannot be created or does not have proper write access, application deployment will fail.
 
 .. _dataverse.files.uploads:
 
 dataverse.files.uploads
 +++++++++++++++++++++++
 
-Configure a folder to store the incoming file stream during uploads (before transfering to `${dataverse.files.directory}/temp`).
+Configure a folder to store the incoming file stream during uploads (before transfering to ``${dataverse.files.directory}/temp``).
+Providing an explicit location here makes it easier to reuse some mounted filesystem.
 Please also see :ref:`temporary-file-storage` for more details.
-You can use an absolute path or a relative, which is relative to the application server domain directory.
 
-Defaults to ``./uploads``, which resolves to ``/usr/local/payara6/glassfish/domains/domain1/uploads`` in a default
-installation.
+Notes:
+
+- Please provide an absolute path to a directory backed by some mounted file system.
+- Defaults to ``${com.sun.aas.instanceRoot}/uploads`` in a :doc:`default installation <installation-main>`
+  (resolving to ``/usr/local/payara6/glassfish/domains/domain1/uploads``).
+- Defaults to ``${STORAGE_DIR}/uploads`` using our :ref:`Dataverse container <app-locations>` (resolving to ``/dv/uploads``).
+- Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_UPLOADS``.
+- During startup, this directory will be checked for existence and write access. It will be created for you
+  if missing. If it cannot be created or does not have proper write access, application deployment will fail.
+
+.. _dataverse.files.docroot:
+
+dataverse.files.docroot
++++++++++++++++++++++++
+
+Configure a folder to store and retrieve additional materials like user uploaded collection logos, generated sitemaps,
+and so on. Providing an explicit location here makes it easier to reuse some mounted filesystem.
+See also logo customization above.
+
+Notes:
 
-Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_UPLOADS``.
-Defaults to ``${STORAGE_DIR}/uploads`` for profile ``ct``, important for the :ref:`Dataverse Application Image <app-locations>`.
+- Defaults to ``${com.sun.aas.instanceRoot}/docroot`` in a :doc:`default installation <installation-main>`
+  (resolves to ``/usr/local/payara6/glassfish/domains/domain1/docroot``).
+- Defaults to ``${STORAGE_DIR}/docroot`` using our :ref:`Dataverse container <app-locations>` (resolving to ``/dv/docroot``).
+- Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_DOCROOT``.
+- During startup, this directory will be checked for existence and write access. It will be created for you
+  if missing. If it cannot be created or does not have proper write access, application deployment will fail.
 
 dataverse.auth.password-reset-timeout-in-minutes
 ++++++++++++++++++++++++++++++++++++++++++++++++

docker-compose-dev.yml

@@ -28,6 +28,7 @@ services:
     depends_on:
       - dev_postgres
       - dev_solr
+      - dev_dv_initializer
     volumes:
       - ./docker-dev-volumes/app/data:/dv
       - ./docker-dev-volumes/app/secrets:/secrets
@@ -48,6 +49,17 @@ services:
     networks:
       - dataverse
 
+  dev_dv_initializer:
+    container_name: "dev_dv_initializer"
+    image: gdcc/configbaker:unstable
+    restart: "no"
+    command:
+      - sh
+      - -c
+      - "fix-fs-perms.sh dv"
+    volumes:
+      - ./docker-dev-volumes/app/data:/dv
+
   dev_postgres:
     container_name: "dev_postgres"
     hostname: postgres

src/main/docker/Dockerfile

@@ -29,6 +29,11 @@ FROM $BASE_IMAGE
 # See also https://download.eclipse.org/microprofile/microprofile-config-3.0/microprofile-config-spec-3.0.html#configprofile
 ENV MP_CONFIG_PROFILE=ct
 
+# Workaround to configure upload directories by default to useful place until we can have variable lookups in
+# defaults for glassfish-web.xml and other places.
+ENV DATAVERSE_FILES_UPLOADS="${STORAGE_DIR}/uploads"
+ENV DATAVERSE_FILES_DOCROOT="${STORAGE_DIR}/docroot"
+
 # Copy app and deps from assembly in proper layers
 COPY --chown=payara:payara maven/deps ${DEPLOY_DIR}/dataverse/WEB-INF/lib/
 COPY --chown=payara:payara maven/app ${DEPLOY_DIR}/dataverse/

src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java

@@ -399,16 +399,7 @@ private File getLogo(Dataverse dataverse) {
 
         DataverseTheme theme = dataverse.getDataverseTheme(); 
         if (theme != null && theme.getLogo() != null && !theme.getLogo().isEmpty()) {
-            Properties p = System.getProperties();
-            String domainRoot = p.getProperty("com.sun.aas.instanceRoot");
-
-            if (domainRoot != null && !"".equals(domainRoot)) {
-                return new File (domainRoot + File.separator + 
-                    "docroot" + File.separator + 
-                    "logos" + File.separator + 
-                    dataverse.getLogoOwnerId() + File.separator + 
-                    theme.getLogo());
-            }
+            return ThemeWidgetFragment.getLogoDir(dataverse.getLogoOwnerId()).resolve(theme.getLogo()).toFile();
         }
 
         return null;         

src/main/java/edu/harvard/iq/dataverse/ThemeWidgetFragment.java

@@ -7,13 +7,15 @@
 
 import edu.harvard.iq.dataverse.engine.command.Command;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseThemeCommand;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
 import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.util.logging.Level;
@@ -49,6 +51,8 @@ public class ThemeWidgetFragment implements java.io.Serializable {
     static final String DEFAULT_TEXT_COLOR = "888888";
     private static final Logger logger = Logger.getLogger(ThemeWidgetFragment.class.getCanonicalName());   
 
+    public static final String LOGOS_SUBDIR = "logos";
+    public static final String LOGOS_TEMP_SUBDIR = LOGOS_SUBDIR + File.separator + "temp";
 
     private File tempDir;
     private File uploadedFile;
@@ -86,12 +90,18 @@ public void setTaglineInput(HtmlInputText taglineInput) {
     }
 
 
-
+    public static Path getLogoDir(String ownerId) {
+        return Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), LOGOS_SUBDIR, ownerId);
+    }
 
-    private  void createTempDir() {
+    private void createTempDir() {
           try {
-            File tempRoot = Files.createDirectories(Paths.get("../docroot/logos/temp")).toFile();
-            tempDir = Files.createTempDirectory(tempRoot.toPath(),editDv.getId().toString()).toFile();
+            // Create the temporary space if not yet existing (will silently ignore preexisting)
+            // Note that the docroot directory is checked within ConfigCheckService for presence and write access.
+            Path tempRoot = Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), LOGOS_TEMP_SUBDIR);
+            Files.createDirectories(tempRoot);
+
+            this.tempDir = Files.createTempDirectory(tempRoot, editDv.getId().toString()).toFile();
         } catch (IOException e) {
             throw new RuntimeException("Error creating temp directory", e); // improve error handling
         }

src/main/java/edu/harvard/iq/dataverse/api/Access.java

@@ -31,6 +31,7 @@
 import edu.harvard.iq.dataverse.RoleAssignment;
 import edu.harvard.iq.dataverse.UserNotification;
 import edu.harvard.iq.dataverse.UserNotificationServiceBean;
+import edu.harvard.iq.dataverse.ThemeWidgetFragment;
 
 import static edu.harvard.iq.dataverse.api.Datasets.handleVersion;
 
@@ -1196,16 +1197,7 @@ private File getLogo(Dataverse dataverse) {
 
         DataverseTheme theme = dataverse.getDataverseTheme(); 
         if (theme != null && theme.getLogo() != null && !theme.getLogo().equals("")) {
-            Properties p = System.getProperties();
-            String domainRoot = p.getProperty("com.sun.aas.instanceRoot");
-
-            if (domainRoot != null && !"".equals(domainRoot)) {
-                return new File (domainRoot + File.separator + 
-                    "docroot" + File.separator + 
-                    "logos" + File.separator + 
-                    dataverse.getLogoOwnerId() + File.separator + 
-                    theme.getLogo());
-            }
+            return ThemeWidgetFragment.getLogoDir(dataverse.getLogoOwnerId()).resolve(theme.getLogo()).toFile();
         }
 
         return null;         

src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java

@@ -976,6 +976,8 @@ public Response listAssignments(@Context ContainerRequestContext crc, @PathParam
      */
 //    File tempDir;
 //
+//    TODO: Code duplicate in ThemeWidgetFragment. Maybe extract, make static and put some place else?
+//          Important: at least use JvmSettings.DOCROOT_DIRECTORY and not the hardcoded location!
 //    private void createTempDir(Dataverse editDv) {
 //        try {
 //            File tempRoot = java.nio.file.Files.createDirectories(Paths.get("../docroot/logos/temp")).toFile();

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseThemeCommand.java

@@ -1,6 +1,7 @@
 package edu.harvard.iq.dataverse.engine.command.impl;
 
 import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.ThemeWidgetFragment;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
@@ -22,7 +23,6 @@
 public class UpdateDataverseThemeCommand extends AbstractCommand<Dataverse> {
     private final Dataverse editedDv;
     private final File uploadedFile;
-    private final Path logoPath = Paths.get("../docroot/logos");
     private String locate;
 
     public UpdateDataverseThemeCommand(Dataverse editedDv, File uploadedFile, DataverseRequest aRequest, String location) {
@@ -44,7 +44,7 @@ public UpdateDataverseThemeCommand(Dataverse editedDv, File uploadedFile, Datave
     public Dataverse execute(CommandContext ctxt) throws CommandException {
         // Get current dataverse, so we can delete current logo file if necessary
         Dataverse currentDv = ctxt.dataverses().find(editedDv.getId());
-        File logoFileDir = new File(logoPath.toFile(), editedDv.getId().toString());
+        File logoFileDir = ThemeWidgetFragment.getLogoDir(editedDv.getId().toString()).toFile();
         File currentFile=null;
 
         if (locate.equals("FOOTER")){

src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java

@@ -0,0 +1,81 @@
+package edu.harvard.iq.dataverse.settings;
+
+import edu.harvard.iq.dataverse.util.FileUtil;
+
+import jakarta.annotation.PostConstruct;
+import jakarta.ejb.DependsOn;
+import jakarta.ejb.Singleton;
+import jakarta.ejb.Startup;
+import java.io.IOException;
+import java.nio.file.FileSystemException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+@Startup
+@Singleton
+@DependsOn("StartupFlywayMigrator")
+public class ConfigCheckService {
+
+    private static final Logger logger = Logger.getLogger(ConfigCheckService.class.getCanonicalName());
+
+    public static class ConfigurationError extends RuntimeException {
+        public ConfigurationError(String message) {
+            super(message);
+        }
+    }
+
+    @PostConstruct
+    public void startup() {
+        if (!checkSystemDirectories()) {
+            throw new ConfigurationError("Not all configuration checks passed successfully. See logs above.");
+        }
+    }
+
+    /**
+     * In this method, we check the existence and write-ability of all important directories we use during
+     * normal operations. It does not include checks for the storage system. If directories are not available,
+     * try to create them (and fail when not allowed to).
+     *
+     * @return True if all checks successful, false otherwise.
+     */
+    public boolean checkSystemDirectories() {
+        Map<Path, String> paths = Map.of(
+                Path.of(JvmSettings.UPLOADS_DIRECTORY.lookup()), "temporary JSF upload space (see " + JvmSettings.UPLOADS_DIRECTORY.getScopedKey() + ")",
+                Path.of(FileUtil.getFilesTempDirectory()), "temporary processing space (see " + JvmSettings.FILES_DIRECTORY.getScopedKey() + ")",
+                Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup()), "docroot space (see " + JvmSettings.DOCROOT_DIRECTORY.getScopedKey() + ")");
+
+        boolean success = true;
+        for (Path path : paths.keySet()) {
+            // Check if the configured path is absolute - avoid potential problems with relative paths this way
+            if (! path.isAbsolute()) {
+                logger.log(Level.SEVERE, () -> "Configured directory " + path + " for " + paths.get(path) + " is not absolute");
+                success = false;
+                continue;
+            }
+
+            if (! Files.exists(path)) {
+                try {
+                    Files.createDirectories(path);
+                } catch (IOException e) {
+                    String details;
+                    if (e instanceof FileSystemException) {
+                        details = ": " + e.getClass();
+                    } else {
+                        details = "";
+                    }
+
+                    logger.log(Level.SEVERE, () -> "Could not create directory " + path + " for " + paths.get(path) + details);
+                    success = false;
+                }
+            } else if (!Files.isWritable(path)) {
+                logger.log(Level.SEVERE, () -> "Directory " + path + " for " + paths.get(path) + " exists, but is not writeable");
+                success = false;
+            }
+        }
+        return success;
+    }
+
+}

src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java

@@ -47,6 +47,8 @@ public enum JvmSettings {
     // FILES SETTINGS
     SCOPE_FILES(PREFIX, "files"),
     FILES_DIRECTORY(SCOPE_FILES, "directory"),
+    UPLOADS_DIRECTORY(SCOPE_FILES, "uploads"),
+    DOCROOT_DIRECTORY(SCOPE_FILES, "docroot"),
     GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"),
 
     // SOLR INDEX SETTINGS

src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java

@@ -3,6 +3,8 @@
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DvObjectContainer;
+import edu.harvard.iq.dataverse.settings.ConfigCheckService;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.xml.XmlValidator;
 import java.io.File;
@@ -210,16 +212,17 @@ public static boolean stageFileExists() {
         }
         return false;
     }
-
+
+    /**
+     * Lookup the location where to generate the sitemap.
+     *
+     * Note: the location is checked to be configured, does exist and is writeable in
+     * {@link ConfigCheckService#checkSystemDirectories()}
+     *
+     * @return Sitemap storage location ([docroot]/sitemap)
+     */
     private static String getSitemapPathString() {
-        String sitemapPathString = "/tmp";
-        // i.e. /usr/local/glassfish4/glassfish/domains/domain1
-        String domainRoot = System.getProperty("com.sun.aas.instanceRoot");
-        if (domainRoot != null) {
-            // Note that we write to a directory called "sitemap" but we serve just "/sitemap.xml" using PrettyFaces.
-            sitemapPathString = domainRoot + File.separator + "docroot" + File.separator + "sitemap";
-        }
-        return sitemapPathString;
+        return JvmSettings.DOCROOT_DIRECTORY.lookup() + File.separator + "sitemap";
 
     }
 }