From ac49dc6641f1f39f266c149407c6db67dfdeba30 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Fri, 12 Sep 2025 15:58:25 -0700 Subject: [PATCH 1/2] Don't need to check for SearchService; it falls back to a noop now --- .../announcements/AnnouncementModule.java | 506 +- .../org/labkey/api/data/ContainerManager.java | 6252 ++-- .../api/search/SearchResultTemplate.java | 194 +- .../org/labkey/api/search/SearchService.java | 2 +- .../org/labkey/api/util/ExceptionUtil.java | 3443 ++- .../api/webdav/AbstractWebdavResource.java | 1455 +- assay/src/org/labkey/assay/AssayModule.java | 51 +- .../labkey/core/CoreContainerListener.java | 245 +- core/src/org/labkey/core/CoreModule.java | 3547 ++- .../labkey/core/admin/AdminController.java | 24594 ++++++++-------- .../attachment/AttachmentServiceImpl.java | 3755 ++- .../org/labkey/core/webdav/DavController.java | 8 +- .../labkey/experiment/ExpDataIterators.java | 19 +- .../labkey/experiment/ExperimentModule.java | 2146 +- .../controllers/exp/ExperimentController.java | 16708 ++++++----- issues/src/org/labkey/issue/IssuesModule.java | 471 +- list/src/org/labkey/list/ListModule.java | 451 +- .../org/labkey/list/model/ListManager.java | 2742 +- .../labkey/pipeline/api/PipelineManager.java | 1836 +- query/src/org/labkey/query/QueryModule.java | 845 +- .../search/SearchContainerListener.java | 112 +- .../org/labkey/search/SearchController.java | 2386 +- .../src/org/labkey/search/SearchModule.java | 536 +- .../search/model/LuceneSearchServiceImpl.java | 2 +- .../org/labkey/search/model/SavePaths.java | 7 +- .../org/labkey/search/view/indexerAdmin.jsp | 12 +- .../org/labkey/search/view/indexerStats.jsp | 13 +- .../org/labkey/search/view/searchStats.jsp | 10 +- .../org/labkey/study/model/StudyManager.java | 9918 ++++--- wiki/src/org/labkey/wiki/WikiController.java | 8 +- wiki/src/org/labkey/wiki/WikiModule.java | 513 +- 31 files changed, 41328 insertions(+), 41459 deletions(-) diff --git a/announcements/src/org/labkey/announcements/AnnouncementModule.java b/announcements/src/org/labkey/announcements/AnnouncementModule.java index d80184cc661..30a4807ef80 100644 --- a/announcements/src/org/labkey/announcements/AnnouncementModule.java +++ b/announcements/src/org/labkey/announcements/AnnouncementModule.java @@ -1,255 +1,251 @@ -/* - * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.announcements; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.announcements.api.AnnouncementServiceImpl; -import org.labkey.announcements.api.TourServiceImpl; -import org.labkey.announcements.config.AnnouncementEmailConfig; -import org.labkey.announcements.model.AnnouncementDigestProvider; -import org.labkey.announcements.model.AnnouncementManager; -import org.labkey.announcements.model.AnnouncementType; -import org.labkey.announcements.model.DiscussionServiceImpl; -import org.labkey.announcements.model.DiscussionWebPartFactory; -import org.labkey.announcements.model.InsertMessagePermission; -import org.labkey.announcements.model.MessageBoardContributorRole; -import org.labkey.announcements.model.SecureMessageBoardReadPermission; -import org.labkey.announcements.model.SecureMessageBoardRespondPermission; -import org.labkey.announcements.query.AnnouncementSchema; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.announcements.CommSchema; -import org.labkey.api.announcements.DiscussionService; -import org.labkey.api.announcements.api.AnnouncementService; -import org.labkey.api.announcements.api.TourService; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.provider.MessageAuditProvider; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.TableSelector; -import org.labkey.api.message.digest.DailyMessageDigest; -import org.labkey.api.message.settings.MessageConfigService; -import org.labkey.api.module.DefaultModule; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.query.FieldKey; -import org.labkey.api.rss.RSSService; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.UserManager; -import org.labkey.api.security.roles.EditorRole; -import org.labkey.api.security.roles.Role; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.emailTemplate.EmailTemplateService; -import org.labkey.api.view.AlwaysAvailableWebPartFactory; -import org.labkey.api.view.Portal; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.view.WebPartView; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * NOTE: Wiki handles some of the shared Communications module stuff. - * e.g. it handles ContainerListener and Attachments - */ -public class AnnouncementModule extends DefaultModule implements SearchService.DocumentProvider -{ - public static final String WEB_PART_NAME = "Messages"; - public static final String NAME = "Announcements"; - - public AnnouncementModule() - { - setLabel("Message Board and Discussion Service"); - - RSSServiceImpl i = new RSSServiceImpl(); - RSSService.setInstance(i); - } - - @Override - public String getName() - { - return NAME; - } - - @Override - public @Nullable Double getSchemaVersion() - { - return 25.000; - } - - @Override - protected void init() - { - addController("announcements", AnnouncementsController.class); - AnnouncementService.setInstance(new AnnouncementServiceImpl()); - - addController("tours", ToursController.class); - - AnnouncementSchema.register(this); - DiscussionService.setInstance(new DiscussionServiceImpl()); - EmailTemplateService.get().registerTemplate(AnnouncementManager.NotificationEmailTemplate.class); - EmailTemplateService.get().registerTemplate(AnnouncementDigestProvider.DailyDigestEmailTemplate.class); - - AttachmentService.get().registerAttachmentType(AnnouncementType.get()); - } - - @Override - @NotNull - protected Collection createWebPartFactories() - { - return List.of( - new AnnouncementsController.AnnouncementWebPartFactory(WEB_PART_NAME), - new AlwaysAvailableWebPartFactory(WEB_PART_NAME + " List") - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext parentCtx, @NotNull Portal.WebPart webPart) - { - return new AnnouncementsController.AnnouncementListWebPart(parentCtx); - } - }, - new DiscussionWebPartFactory() - ); - } - - @Override - public boolean hasScripts() - { - return true; - } - - @Override - public String getTabName(ViewContext context) - { - return "Messages"; - } - - - @Override - public void doStartup(ModuleContext moduleContext) - { - ContainerManager.addContainerListener(new AnnouncementContainerListener()); - UserManager.addUserListener(new AnnouncementUserListener()); - AuditLogService.get().registerAuditType(new MessageAuditProvider()); - - TourListener tourListener = new TourListener(); - ContainerManager.addContainerListener(tourListener); - - // Editors can read and respond to secure message boards - RoleManager.registerPermission(new SecureMessageBoardReadPermission()); - RoleManager.registerPermission(new SecureMessageBoardRespondPermission()); - Role editor = RoleManager.getRole(EditorRole.class); - editor.addPermission(SecureMessageBoardReadPermission.class); - editor.addPermission(SecureMessageBoardRespondPermission.class); - - RoleManager.registerPermission(new InsertMessagePermission(),true); - RoleManager.registerRole(new MessageBoardContributorRole()); - - // initialize message digests - DailyMessageDigest.getInstance().addProvider(new AnnouncementDigestProvider()); - - // add a config provider for announcements - MessageConfigService.get().registerConfigType(new AnnouncementEmailConfig()); - - SearchService ss = SearchService.get(); - - if (null != ss) - { - ss.addSearchCategory(AnnouncementManager.searchCategory); - ss.addDocumentProvider(this); - } - - FolderSerializationRegistry fsr = FolderSerializationRegistry.get(); - if (null != fsr) - { - fsr.addFactories(new NotificationSettingsWriterFactory(), new NotificationSettingsImporterFactory()); - } - - TourService.setInstance(new TourServiceImpl()); - - UsageMetricsService.get().registerUsageMetrics(NAME, () -> Map.of("discussions", Map.of( - "rootEnabled", LookAndFeelProperties.getInstance(ContainerManager.getRoot()).isDiscussionEnabled(), - "projectsEnabled", ContainerManager.getProjects().stream() - .filter(project -> LookAndFeelProperties.getInstance(project).isDiscussionEnabled()) - .count(), - "createdByYear", new TableSelector(CommSchema.getInstance().getTableInfoAnnouncements(), Collections.singleton("Created"), - new SimpleFilter(FieldKey.fromString("DiscussionSrcUrl"), null, CompareType.NONBLANK), null - ).stream(Date.class) - .collect(Collectors.groupingBy(date -> date.getYear() + 1900, Collectors.counting())) - ))); - } - - - @Override - public void startBackgroundThreads() - { - DailyMessageDigest.getInstance().initializeTimer(); - } - - @Override - @NotNull - public Set getIntegrationTests() - { - return Set.of( - AnnouncementManager.TestCase.class - ); - } - - @Override - @NotNull - public Set getSchemaNames() - { - return PageFlowUtil.set(CommSchema.getInstance().getSchemaName()); - } - - @Override - @NotNull - public Collection getSummary(Container c) - { - List list = new ArrayList<>(1); - long count = AnnouncementManager.getMessageCount(c); - - if (count > 0) - list.add(count + " " + (count > 1 ? "Messages/Responses" : "Message")); - - return list; - } - - - @Override - public void enumerateDocuments(SearchService.TaskIndexingQueue queue, final Date modifiedSince) - { - queue.addRunnable((q) -> AnnouncementManager.indexMessages(q, modifiedSince)); - } - - @Override - public void indexDeleted() - { - new SqlExecutor(CommSchema.getInstance().getSchema()).execute("UPDATE comm.announcements SET LastIndexed=NULL"); - } -} +/* + * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.announcements; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.announcements.api.AnnouncementServiceImpl; +import org.labkey.announcements.api.TourServiceImpl; +import org.labkey.announcements.config.AnnouncementEmailConfig; +import org.labkey.announcements.model.AnnouncementDigestProvider; +import org.labkey.announcements.model.AnnouncementManager; +import org.labkey.announcements.model.AnnouncementType; +import org.labkey.announcements.model.DiscussionServiceImpl; +import org.labkey.announcements.model.DiscussionWebPartFactory; +import org.labkey.announcements.model.InsertMessagePermission; +import org.labkey.announcements.model.MessageBoardContributorRole; +import org.labkey.announcements.model.SecureMessageBoardReadPermission; +import org.labkey.announcements.model.SecureMessageBoardRespondPermission; +import org.labkey.announcements.query.AnnouncementSchema; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.announcements.CommSchema; +import org.labkey.api.announcements.DiscussionService; +import org.labkey.api.announcements.api.AnnouncementService; +import org.labkey.api.announcements.api.TourService; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.provider.MessageAuditProvider; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.TableSelector; +import org.labkey.api.message.digest.DailyMessageDigest; +import org.labkey.api.message.settings.MessageConfigService; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.query.FieldKey; +import org.labkey.api.rss.RSSService; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.roles.EditorRole; +import org.labkey.api.security.roles.Role; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.emailTemplate.EmailTemplateService; +import org.labkey.api.view.AlwaysAvailableWebPartFactory; +import org.labkey.api.view.Portal; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * NOTE: Wiki handles some of the shared Communications module stuff. + * e.g. it handles ContainerListener and Attachments + */ +public class AnnouncementModule extends DefaultModule implements SearchService.DocumentProvider +{ + public static final String WEB_PART_NAME = "Messages"; + public static final String NAME = "Announcements"; + + public AnnouncementModule() + { + setLabel("Message Board and Discussion Service"); + + RSSServiceImpl i = new RSSServiceImpl(); + RSSService.setInstance(i); + } + + @Override + public String getName() + { + return NAME; + } + + @Override + public @Nullable Double getSchemaVersion() + { + return 25.000; + } + + @Override + protected void init() + { + addController("announcements", AnnouncementsController.class); + AnnouncementService.setInstance(new AnnouncementServiceImpl()); + + addController("tours", ToursController.class); + + AnnouncementSchema.register(this); + DiscussionService.setInstance(new DiscussionServiceImpl()); + EmailTemplateService.get().registerTemplate(AnnouncementManager.NotificationEmailTemplate.class); + EmailTemplateService.get().registerTemplate(AnnouncementDigestProvider.DailyDigestEmailTemplate.class); + + AttachmentService.get().registerAttachmentType(AnnouncementType.get()); + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + return List.of( + new AnnouncementsController.AnnouncementWebPartFactory(WEB_PART_NAME), + new AlwaysAvailableWebPartFactory(WEB_PART_NAME + " List") + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext parentCtx, @NotNull Portal.WebPart webPart) + { + return new AnnouncementsController.AnnouncementListWebPart(parentCtx); + } + }, + new DiscussionWebPartFactory() + ); + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + public String getTabName(ViewContext context) + { + return "Messages"; + } + + + @Override + public void doStartup(ModuleContext moduleContext) + { + ContainerManager.addContainerListener(new AnnouncementContainerListener()); + UserManager.addUserListener(new AnnouncementUserListener()); + AuditLogService.get().registerAuditType(new MessageAuditProvider()); + + TourListener tourListener = new TourListener(); + ContainerManager.addContainerListener(tourListener); + + // Editors can read and respond to secure message boards + RoleManager.registerPermission(new SecureMessageBoardReadPermission()); + RoleManager.registerPermission(new SecureMessageBoardRespondPermission()); + Role editor = RoleManager.getRole(EditorRole.class); + editor.addPermission(SecureMessageBoardReadPermission.class); + editor.addPermission(SecureMessageBoardRespondPermission.class); + + RoleManager.registerPermission(new InsertMessagePermission(),true); + RoleManager.registerRole(new MessageBoardContributorRole()); + + // initialize message digests + DailyMessageDigest.getInstance().addProvider(new AnnouncementDigestProvider()); + + // add a config provider for announcements + MessageConfigService.get().registerConfigType(new AnnouncementEmailConfig()); + + SearchService ss = SearchService.get(); + ss.addSearchCategory(AnnouncementManager.searchCategory); + ss.addDocumentProvider(this); + + FolderSerializationRegistry fsr = FolderSerializationRegistry.get(); + if (null != fsr) + { + fsr.addFactories(new NotificationSettingsWriterFactory(), new NotificationSettingsImporterFactory()); + } + + TourService.setInstance(new TourServiceImpl()); + + UsageMetricsService.get().registerUsageMetrics(NAME, () -> Map.of("discussions", Map.of( + "rootEnabled", LookAndFeelProperties.getInstance(ContainerManager.getRoot()).isDiscussionEnabled(), + "projectsEnabled", ContainerManager.getProjects().stream() + .filter(project -> LookAndFeelProperties.getInstance(project).isDiscussionEnabled()) + .count(), + "createdByYear", new TableSelector(CommSchema.getInstance().getTableInfoAnnouncements(), Collections.singleton("Created"), + new SimpleFilter(FieldKey.fromString("DiscussionSrcUrl"), null, CompareType.NONBLANK), null + ).stream(Date.class) + .collect(Collectors.groupingBy(date -> date.getYear() + 1900, Collectors.counting())) + ))); + } + + + @Override + public void startBackgroundThreads() + { + DailyMessageDigest.getInstance().initializeTimer(); + } + + @Override + @NotNull + public Set getIntegrationTests() + { + return Set.of( + AnnouncementManager.TestCase.class + ); + } + + @Override + @NotNull + public Set getSchemaNames() + { + return PageFlowUtil.set(CommSchema.getInstance().getSchemaName()); + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + List list = new ArrayList<>(1); + long count = AnnouncementManager.getMessageCount(c); + + if (count > 0) + list.add(count + " " + (count > 1 ? "Messages/Responses" : "Message")); + + return list; + } + + + @Override + public void enumerateDocuments(SearchService.TaskIndexingQueue queue, final Date modifiedSince) + { + queue.addRunnable((q) -> AnnouncementManager.indexMessages(q, modifiedSince)); + } + + @Override + public void indexDeleted() + { + new SqlExecutor(CommSchema.getInstance().getSchema()).execute("UPDATE comm.announcements SET LastIndexed=NULL"); + } +} diff --git a/api/src/org/labkey/api/data/ContainerManager.java b/api/src/org/labkey/api/data/ContainerManager.java index 286eef5c511..db4f235b41b 100644 --- a/api/src/org/labkey/api/data/ContainerManager.java +++ b/api/src/org/labkey/api/data/ContainerManager.java @@ -1,3128 +1,3124 @@ -/* - * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.data; - -import com.google.common.base.Enums; -import org.apache.commons.collections4.MultiValuedMap; -import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.xmlbeans.XmlObject; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.labkey.api.Constants; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.FolderExportContext; -import org.labkey.api.admin.FolderImportContext; -import org.labkey.api.admin.FolderImporterImpl; -import org.labkey.api.admin.FolderWriterImpl; -import org.labkey.api.admin.StaticLoggerGetter; -import org.labkey.api.attachments.AttachmentParent; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.provider.ContainerAuditProvider; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.ConcurrentHashSet; -import org.labkey.api.collections.IntHashMap; -import org.labkey.api.data.Container.ContainerException; -import org.labkey.api.data.Container.LockState; -import org.labkey.api.data.PropertyManager.WritablePropertyMap; -import org.labkey.api.data.SimpleFilter.InClause; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.data.validator.ColumnValidators; -import org.labkey.api.event.PropertyChange; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.module.FolderType; -import org.labkey.api.module.FolderTypeManager; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.portal.ProjectUrls; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.Group; -import org.labkey.api.security.MutableSecurityPolicy; -import org.labkey.api.security.SecurityLogger; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.SecurityPolicyManager; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.CreateProjectPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.roles.AuthorRole; -import org.labkey.api.security.roles.ReaderRole; -import org.labkey.api.security.roles.Role; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.test.TestTimeout; -import org.labkey.api.test.TestWhen; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JunitUtil; -import org.labkey.api.util.MinorConfigurationException; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Path; -import org.labkey.api.util.QuietCloser; -import org.labkey.api.util.ReentrantLockWithName; -import org.labkey.api.util.ResultSetUtil; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.FolderTab; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NavTreeManager; -import org.labkey.api.view.Portal; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.ViewContext; -import org.labkey.api.writer.MemoryVirtualFile; -import org.labkey.folder.xml.FolderDocument; -import org.labkey.remoteapi.collections.CaseInsensitiveHashMap; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; - -import java.beans.PropertyChangeEvent; -import java.beans.PropertyChangeListener; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import static org.labkey.api.action.SpringActionController.ERROR_GENERIC; - -/** - * This class manages a hierarchy of collections, backed by a database table called Containers. - * Containers are named using filesystem-like paths e.g. /proteomics/comet/. Each path - * maps to a UID and set of permissions. The current security scheme allows ACLs - * to be specified explicitly on the directory or completely inherited. ACLs are not combined. - *

- * NOTE: we act like java.io.File(). Paths start with forward-slash, but do not end with forward-slash. - * The root container's name is '/'. This means that it is not always the case that - * me.getPath() == me.getParent().getPath() + "/" + me.getName() - *

- * The synchronization goals are to keep invalid containers from creeping into the cache. For example, once - * a container is deleted, it should never get put back in the cache. We accomplish this by synchronizing on - * the removal from the cache, and the database lookup/cache insertion. While a container is in the middle - * of being deleted, it's OK for other clients to see it because FKs enforce that it's always internally - * consistent, even if some of the data has already been deleted. - */ -public class ContainerManager -{ - private static final Logger LOG = LogHelper.getLogger(ContainerManager.class, "Container (projects, folders, and workbooks) retrieval and management"); - private static final CoreSchema CORE = CoreSchema.getInstance(); - - private static final String PROJECT_LIST_ID = "Projects"; - - public static final String HOME_PROJECT_PATH = "/home"; - public static final String DEFAULT_SUPPORT_PROJECT_PATH = HOME_PROJECT_PATH + "/support"; - - private static final Cache CACHE_PATH = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Containers by Path"); - private static final Cache CACHE_ENTITY_ID = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Containers by EntityId"); - private static final Cache> CACHE_CHILDREN = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Child EntityIds of Containers"); - private static final ReentrantLock DATABASE_QUERY_LOCK = new ReentrantLockWithName(ContainerManager.class, "DATABASE_QUERY_LOCK"); - public static final String FOLDER_TYPE_PROPERTY_SET_NAME = "folderType"; - public static final String FOLDER_TYPE_PROPERTY_NAME = "name"; - public static final String FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN = "ctFolderTypeOverridden"; - public static final String TABFOLDER_CHILDREN_DELETED = "tabChildrenDeleted"; - public static final String AUDIT_SETTINGS_PROPERTY_SET_NAME = "containerAuditSettings"; - public static final String REQUIRE_USER_COMMENTS_PROPERTY_NAME = "requireUserComments"; - - private static final List _resourceProviders = new CopyOnWriteArrayList<>(); - - // containers that are being constructed, used to suppress events before fireCreateContainer() - private static final Set _constructing = new ConcurrentHashSet<>(); - - - /** enum of properties you can see in property change events */ - public enum Property - { - Name, - Parent, - Policy, - /** The default or active set of modules in the container has changed */ - Modules, - FolderType, - WebRoot, - AttachmentDirectory, - PipelineRoot, - Title, - Description, - SiteRoot, - StudyChange, - EndpointDirectory, - CloudStores - } - - static Path makePath(Container parent, String name) - { - if (null == parent) - return new Path(name); - return parent.getParsedPath().append(name, true); - } - - public static Container createMockContainer() - { - return new Container(null, "MockContainer", "01234567-8901-2345-6789-012345678901", 99999999, 0, new Date(), User.guest.getUserId(), true); - } - - private static Container createRoot() - { - Map m = new HashMap<>(); - m.put("Parent", null); - m.put("Name", ""); - Table.insert(null, CORE.getTableInfoContainers(), m); - - return getRoot(); - } - - private static DbScope.Transaction ensureTransaction() - { - return CORE.getSchema().getScope().ensureTransaction(DATABASE_QUERY_LOCK); - } - - private static int getNewChildSortOrder(Container parent) - { - int nextSortOrderVal = 0; - - List children = parent.getChildren(); - if (children != null) - { - for (Container child : children) - { - // find the max sort order value for the set of children - nextSortOrderVal = Math.max(nextSortOrderVal, child.getSortOrder()); - } - } - - // custom sorting applies: put new container at the end. - if (nextSortOrderVal > 0) - return nextSortOrderVal + 1; - - // we're sorted alphabetically - return 0; - } - - // TODO: Make private and force callers to use ensureContainer instead? - // TODO: Handle root creation here? - @NotNull - public static Container createContainer(Container parent, String name, @NotNull User user) - { - return createContainer(parent, name, null, null, NormalContainerType.NAME, user, null, null); - } - - public static final String WORKBOOK_DBSEQUENCE_NAME = "org.labkey.api.data.Workbooks"; - - // TODO: Pass in FolderType (separate from the container type of workbook, etc) and transact it with container creation? - @NotNull - public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user) - { - return createContainer(parent, name, title, description, type, user, null, null); - } - - @NotNull - public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user, @Nullable String auditMsg) - { - return createContainer(parent, name, title, description, type, user, auditMsg, null); - } - - @NotNull - public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user, @Nullable String auditMsg, - Consumer configureContainer) - { - ContainerType cType = ContainerTypeRegistry.get().getType(type); - if (cType == null) - throw new IllegalArgumentException("Unknown container type: " + type); - - // TODO: move this to ContainerType? - long sortOrder; - if (cType instanceof WorkbookContainerType) - { - sortOrder = DbSequenceManager.get(parent, WORKBOOK_DBSEQUENCE_NAME).next(); - - // Default workbook names are simply "" - if (name == null) - name = String.valueOf(sortOrder); - } - else - { - sortOrder = getNewChildSortOrder(parent); - } - - if (!parent.canHaveChildren()) - throw new IllegalArgumentException("Parent of a container must not be a " + parent.getContainerType().getName()); - - StringBuilder error = new StringBuilder(); - if (!Container.isLegalName(name, parent.isRoot(), error)) - throw new ApiUsageException(error.toString()); - - if (!Container.isLegalTitle(title, error)) - throw new ApiUsageException(error.toString()); - - Path path = makePath(parent, name); - SQLException sqlx = null; - Map insertMap = null; - - GUID entityId = new GUID(); - Container c; - - try - { - _constructing.add(entityId); - - try - { - Map m = new CaseInsensitiveHashMap<>(); - m.put("Parent", parent.getId()); - m.put("Name", name); - m.put("Title", title); - m.put("SortOrder", sortOrder); - m.put("EntityId", entityId); - if (null != description) - m.put("Description", description); - m.put("Type", type); - insertMap = Table.insert(user, CORE.getTableInfoContainers(), m); - } - catch (RuntimeSQLException x) - { - if (!x.isConstraintException()) - throw x; - sqlx = x.getSQLException(); - } - - _clearChildrenFromCache(parent); - - c = insertMap == null ? null : getForId(entityId); - - if (null == c) - { - if (null != sqlx) - throw new RuntimeSQLException(sqlx); - else - throw new RuntimeException("Container for path '" + path + "' was not created properly."); - } - - User savePolicyUser = user; - if (c.isProject() && !c.hasPermission(user, AdminPermission.class) && ContainerManager.getRoot().hasPermission(user, CreateProjectPermission.class)) - { - // Special case for project creators who don't necessarily yet have permission to save the policy of - // the project they just created - savePolicyUser = User.getAdminServiceUser(); - } - - // Workbooks inherit perms from their parent so don't create a policy if this is a workbook - if (c.isContainerFor(ContainerType.DataType.permissions)) - { - SecurityManager.setAdminOnlyPermissions(c, savePolicyUser); - } - - _removeFromCache(c, true); // seems odd, but it removes c.getProject() which clears other things from the cache - - // Initialize the list of active modules in the Container - c.getActiveModules(true, true, user); - - if (c.isProject()) - { - SecurityManager.createNewProjectGroups(c, savePolicyUser); - } - else - { - // If current user does NOT have admin permission on this container or the project has been - // explicitly set to have new subfolders inherit permissions, then inherit permissions - // (otherwise they would not be able to see the folder) - boolean hasAdminPermission = c.hasPermission(user, AdminPermission.class); - if ((!hasAdminPermission && !user.hasRootAdminPermission()) || SecurityManager.shouldNewSubfoldersInheritPermissions(c.getProject())) - SecurityManager.setInheritPermissions(c); - } - - // NOTE parent caches some info about children (e.g. hasWorkbookChildren) - // since mutating cached objects is frowned upon, just uncache parent - // CONSIDER: we could perhaps only uncache if the child is a workbook, but I think this reasonable - _removeFromCache(parent, true); - - if (null != configureContainer) - configureContainer.accept(c); - } - finally - { - _constructing.remove(entityId); - } - - fireCreateContainer(c, user, auditMsg); - - return c; - } - - public static void addSecurableResourceProvider(ContainerSecurableResourceProvider provider) - { - _resourceProviders.add(provider); - } - - public static List getSecurableResourceProviders() - { - return Collections.unmodifiableList(_resourceProviders); - } - - public static Container createContainerFromTemplate(Container parent, String name, String title, Container templateContainer, User user, FolderExportContext exportCtx, Consumer afterCreateHandler) throws Exception - { - MemoryVirtualFile vf = new MemoryVirtualFile(); - - // export objects from the source template folder - FolderWriterImpl writer = new FolderWriterImpl(); - writer.write(templateContainer, exportCtx, vf); - - // create the new target container - Container c = createContainer(parent, name, title, null, NormalContainerType.NAME, user, null, afterCreateHandler); - - // import objects into the target folder - XmlObject folderXml = vf.getXmlBean("folder.xml"); - if (folderXml instanceof FolderDocument folderDoc) - { - FolderImportContext importCtx = new FolderImportContext(user, c, folderDoc, null, new StaticLoggerGetter(LogManager.getLogger(FolderImporterImpl.class)), vf); - - FolderImporterImpl importer = new FolderImporterImpl(); - importer.process(null, importCtx, vf); - } - - return c; - } - - public static void setRequireAuditComments(Container container, User user, @NotNull Boolean required) - { - WritablePropertyMap props = PropertyManager.getWritableProperties(container, AUDIT_SETTINGS_PROPERTY_SET_NAME, true); - String originalValue = props.get(REQUIRE_USER_COMMENTS_PROPERTY_NAME); - props.put(REQUIRE_USER_COMMENTS_PROPERTY_NAME, required.toString()); - props.save(); - - addAuditEvent(user, container, - "Changed " + REQUIRE_USER_COMMENTS_PROPERTY_NAME + " from \"" + - originalValue + "\" to \"" + required + "\""); - } - - public static void setFolderType(Container c, FolderType folderType, User user, BindException errors) - { - FolderType oldType = c.getFolderType(); - - if (folderType.equals(oldType)) - return; - - List errorStrings = new ArrayList<>(); - - if (!c.isProject() && folderType.isProjectOnlyType()) - errorStrings.add("Cannot set a subfolder to " + folderType.getName() + " because it is a project-only folder type."); - - // Check for any containers that need to be moved into container tabs - if (errorStrings.isEmpty() && folderType.hasContainerTabs()) - { - List childTabFoldersNonMatchingTypes = new ArrayList<>(); - List containersBecomingTabs = findAndCheckContainersMatchingTabs(c, folderType, childTabFoldersNonMatchingTypes, errorStrings); - - if (errorStrings.isEmpty()) - { - if (!containersBecomingTabs.isEmpty()) - { - // Make containers tab container; Folder tab will find them by name - try (DbScope.Transaction transaction = ensureTransaction()) - { - for (Container container : containersBecomingTabs) - updateType(container, TabContainerType.NAME, user); - - transaction.commit(); - } - } - - // Check these and change type unless they were overridden explicitly - for (Container container : childTabFoldersNonMatchingTypes) - { - if (!isContainerTabTypeOverridden(container)) - { - FolderTab newTab = folderType.findTab(container.getName()); - assert null != newTab; // There must be a tab because it caused the container to get into childTabFoldersNonMatchingTypes - FolderType newType = newTab.getFolderType(); - if (null == newType) - newType = FolderType.NONE; // default to NONE - setFolderType(container, newType, user, errors); - } - } - } - } - - if (errorStrings.isEmpty()) - { - oldType.unconfigureContainer(c, user); - WritablePropertyMap props = PropertyManager.getWritableProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME, true); - props.put(FOLDER_TYPE_PROPERTY_NAME, folderType.getName()); - - if (c.isContainerTab()) - { - boolean containerTabTypeOverridden = false; - FolderTab tab = c.getParent().getFolderType().findTab(c.getName()); - if (null != tab && !folderType.equals(tab.getFolderType())) - containerTabTypeOverridden = true; - props.put(FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN, Boolean.toString(containerTabTypeOverridden)); - } - props.save(); - - notifyContainerChange(c.getId(), Property.FolderType, user); - folderType.configureContainer(c, user); // Configure new only after folder type has been changed - - // TODO: Not needed? I don't think we've changed the container's state. - _removeFromCache(c, false); - } - else - { - for (String errorString : errorStrings) - errors.reject(SpringActionController.ERROR_MSG, errorString); - } - } - - public static void checkContainerValidity(Container c) throws ContainerException - { - // Check container for validity; in rare cases user may have changed their custom folderType.xml and caused - // duplicate subfolders (same name) to exist - // Get list of child containers that are not container tabs, but match container tabs; these are bad - FolderType folderType = getFolderType(c); - List errorStrings = new ArrayList<>(); - List childTabFoldersNonMatchingTypes = new ArrayList<>(); - List containersMatchingTabs = findAndCheckContainersMatchingTabs(c, folderType, childTabFoldersNonMatchingTypes, errorStrings); - if (!containersMatchingTabs.isEmpty()) - { - throw new Container.ContainerException("Folder " + c.getPath() + - " has a subfolder with the same name as a container tab folder, which is an invalid state." + - " This may have been caused by changing the folder type's tabs after this folder was set to its folder type." + - " An administrator should either delete the offending subfolder or change the folder's folder type.\n"); - } - } - - public static List findAndCheckContainersMatchingTabs(Container c, FolderType folderType, - List childTabFoldersNonMatchingTypes, List errorStrings) - { - List containersMatchingTabs = new ArrayList<>(); - for (FolderTab folderTab : folderType.getDefaultTabs()) - { - if (folderTab.getTabType() == FolderTab.TAB_TYPE.Container) - { - for (Container child : c.getChildren()) - { - if (child.getName().equalsIgnoreCase(folderTab.getName())) - { - if (!child.getFolderType().getName().equalsIgnoreCase(folderTab.getFolderTypeName())) - { - if (child.isContainerTab()) - childTabFoldersNonMatchingTypes.add(child); // Tab type doesn't match child tab folder - else - errorStrings.add("Child folder " + child.getName() + - " matches container tab, but folder type " + child.getFolderType().getName() + " doesn't match tab's folder type " + - folderTab.getFolderTypeName() + "."); - } - - int childCount = child.getChildren().size(); - if (childCount > 0) - { - errorStrings.add("Child folder " + child.getName() + - " matches container tab, but cannot be converted to a tab folder because it has " + childCount + " children."); - } - - if (!child.isConvertibleToTab()) - { - errorStrings.add("Child folder " + child.getName() + - " matches container tab, but cannot be converted to a tab folder because it is a " + child.getContainerNoun() + "."); - } - - if (!child.isContainerTab()) - containersMatchingTabs.add(child); - - break; // we found name match; can't be another - } - } - } - } - return containersMatchingTabs; - } - - private static final Set containersWithBadFolderTypes = new ConcurrentHashSet<>(); - - @NotNull - public static FolderType getFolderType(Container c) - { - String name = getFolderTypeName(c); - FolderType folderType; - - if (null != name) - { - folderType = FolderTypeManager.get().getFolderType(name); - - if (null == folderType) - { - // If we're upgrading then folder types won't be defined yet... don't warn in that case. - if (!ModuleLoader.getInstance().isUpgradeInProgress() && - !ModuleLoader.getInstance().isUpgradeRequired() && - !containersWithBadFolderTypes.contains(c)) - { - LOG.warn("No such folder type " + name + " for folder " + c.toString()); - containersWithBadFolderTypes.add(c); - } - - folderType = FolderType.NONE; - } - } - else - folderType = FolderType.NONE; - - return folderType; - } - - /** - * Most code should call getFolderType() instead. - * Useful for finding the name of the folder type BEFORE startup is complete, so the FolderType itself - * may not be available. - */ - @Nullable - public static String getFolderTypeName(Container c) - { - Map props = PropertyManager.getProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME); - return props.get(FOLDER_TYPE_PROPERTY_NAME); - } - - - @NotNull - public static Map getFolderTypeNameContainerCounts(Container root) - { - Map nameCounts = new TreeMap<>(); - for (Container c : getAllChildren(root)) - { - Integer count = nameCounts.get(c.getFolderType().getName()); - if (null == count) - { - count = Integer.valueOf(0); - } - nameCounts.put(c.getFolderType().getName(), ++count); - } - return nameCounts; - } - - @NotNull - public static Map getProductFoldersMetrics(@NotNull FolderType folderType) - { - Container root = getRoot(); - Map metrics = new TreeMap<>(); - List counts = new ArrayList<>(); - for (Container c : root.getChildren()) - { - if (!c.getFolderType().getName().equals(folderType.getName())) - continue; - - int childCount = c.getChildren().stream().filter(Container::isInFolderNav).toList().size(); - counts.add(childCount); - } - - int totalFolderTypeMatch = counts.size(); - if (totalFolderTypeMatch == 0) - return metrics; - - Collections.sort(counts); - int median = counts.get((totalFolderTypeMatch - 1)/2); - if (totalFolderTypeMatch % 2 == 0 ) - { - int low = counts.get(totalFolderTypeMatch/2 - 1); - int high = counts.get(totalFolderTypeMatch/2); - median = Math.round((low + high) / 2.0f); - } - int maxProjectsCount = counts.get(totalFolderTypeMatch - 1); - int totalProjectsCount = counts.stream().mapToInt(Integer::intValue).sum(); - int averageProjectsCount = Math.round((float) totalProjectsCount /totalFolderTypeMatch); - - metrics.put("totalSubProjectsCount", totalProjectsCount); - metrics.put("averageSubProjectsPerHomeProject", averageProjectsCount); - metrics.put("medianSubProjectsCountPerHomeProject", median); - metrics.put("maxSubProjectsCountInHomeProject", maxProjectsCount); - - return metrics; - } - - public static boolean isContainerTabTypeThisOrChildrenOverridden(Container c) - { - if (isContainerTabTypeOverridden(c)) - return true; - if (c.getFolderType().hasContainerTabs()) - { - for (Container child : c.getChildren()) - { - if (child.isContainerTab() && isContainerTabTypeOverridden(child)) - return true; - } - } - return false; - } - - public static boolean isContainerTabTypeOverridden(Container c) - { - Map props = PropertyManager.getProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME); - String overridden = props.get(FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN); - return (null != overridden) && overridden.equalsIgnoreCase("true"); - } - - private static void setContainerTabDeleted(Container c, String tabName, String folderTypeName) - { - // Add prop in this category - WritablePropertyMap props = PropertyManager.getWritableProperties(c, TABFOLDER_CHILDREN_DELETED, true); - props.put(getDeletedTabKey(tabName, folderTypeName), "true"); - props.save(); - } - - public static void clearContainerTabDeleted(Container c, String tabName, String folderTypeName) - { - WritablePropertyMap props = PropertyManager.getWritableProperties(c, TABFOLDER_CHILDREN_DELETED, true); - String key = getDeletedTabKey(tabName, folderTypeName); - if (props.containsKey(key)) - { - props.remove(key); - props.save(); - } - } - - public static boolean hasContainerTabBeenDeleted(Container c, String tabName, String folderTypeName) - { - // We keep arbitrary number of deleted children tabs using suffix 0, 1, 2.... - Map props = PropertyManager.getProperties(c, TABFOLDER_CHILDREN_DELETED); - return props.containsKey(getDeletedTabKey(tabName, folderTypeName)); - } - - private static String getDeletedTabKey(String tabName, String folderTypeName) - { - return tabName + "-TABDELETED-FOLDER-" + folderTypeName; - } - - @NotNull - public static Container ensureContainer(@NotNull String path, @NotNull User user) - { - return ensureContainer(Path.parse(path), user); - } - - @NotNull - public static Container ensureContainer(@NotNull Path path, @NotNull User user) - { - Container c = null; - - try - { - c = getForPath(path); - } - catch (RootContainerException e) - { - // Ignore this -- root doesn't exist yet - } - - if (null == c) - { - if (path.isEmpty()) - c = createRoot(); - else - { - Path parentPath = path.getParent(); - c = ensureContainer(parentPath, user); - c = createContainer(c, path.getName(), null, null, NormalContainerType.NAME, user); - } - } - return c; - } - - - @NotNull - public static Container ensureContainer(Container parent, String name, User user) - { - // NOTE: Running outside a tx doesn't seem to be necessary. -// if (CORE.getSchema().getScope().isTransactionActive()) -// throw new IllegalStateException("Transaction should not be active"); - - Container c = null; - - try - { - c = getForPath(makePath(parent,name)); - } - catch (RootContainerException e) - { - // Ignore this -- root doesn't exist yet - } - - if (null == c) - { - c = createContainer(parent, name, user); - } - return c; - } - - public static void updateDescription(Container container, String description, User user) - throws ValidationException - { - ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, description); - - //For some reason, there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET Description=? WHERE RowID=?").add(description).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - String oldValue = container.getDescription(); - _removeFromCache(container, false); - container = getForRowId(container.getRowId()); - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(container, user, Property.Description, oldValue, description); - firePropertyChangeEvent(evt); - } - - public static void updateSearchable(Container container, boolean searchable, User user) - { - //For some reason, there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET Searchable=? WHERE RowID=?").add(searchable).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - } - - public static void updateLockState(Container container, LockState lockState, @NotNull Runnable auditRunnable) - { - //For some reason there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET LockState = ?, ExpirationDate = NULL WHERE RowID = ?").add(lockState).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - - auditRunnable.run(); - } - - public static List getExcludedProjects() - { - return getProjects().stream() - .filter(p->p.getLockState() == Container.LockState.Excluded) - .collect(Collectors.toList()); - } - - public static List getNonExcludedProjects() - { - return getProjects().stream() - .filter(p->p.getLockState() != Container.LockState.Excluded) - .collect(Collectors.toList()); - } - - public static void setExcludedProjects(Collection ids, @NotNull Runnable auditRunnable) - { - // First clear all existing "Excluded" states - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET LockState = NULL, ExpirationDate = NULL WHERE LockState = ?").add(LockState.Excluded); - new SqlExecutor(CORE.getSchema()).execute(sql); - - // Now set the passed-in projects to "Excluded" - if (!ids.isEmpty()) - { - ColumnInfo entityIdCol = CORE.getTableInfoContainers().getColumn("EntityId"); - Filter inClauseFilter = new SimpleFilter(new InClause(entityIdCol.getFieldKey(), ids)); - SQLFragment frag = new SQLFragment("UPDATE "); - frag.append(CORE.getTableInfoContainers().getSelectName()); - frag.append(" SET LockState = ?, ExpirationDate = NULL "); - frag.add(LockState.Excluded); - frag.append(inClauseFilter.getSQLFragment(CORE.getSqlDialect(), "c", Map.of(entityIdCol.getFieldKey(), entityIdCol))); - new SqlExecutor(CORE.getSchema()).execute(frag); - } - - clearCache(); - - auditRunnable.run(); - } - - public static void archiveContainer(User user, Container container, boolean archive) - { - if (container.isRoot() || container.isProject() || container.isAppHomeFolder()) - throw new ApiUsageException("Archive action not supported for this folder."); - - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers().getSelectName()); - if (archive) - { - sql.append(" SET LockState = ? "); - sql.add(LockState.Archived); - sql.append(" WHERE LockState IS NULL "); - } - else - { - sql.append(" SET LockState = NULL WHERE LockState = ? "); - sql.add(LockState.Archived); - } - sql.append("AND EntityId = ? "); - sql.add(container.getEntityId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - clearCache(); - - addAuditEvent(user, container, archive ? "Container has been archived." : "Archived container has been restored."); - } - - public static void updateExpirationDate(Container container, LocalDate expirationDate, @NotNull Runnable auditRunnable) - { - //For some reason there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - // Note: jTDS doesn't support LocalDate, so convert to java.sql.Date - sql.append(" SET ExpirationDate = ? WHERE RowID = ?").add(java.sql.Date.valueOf(expirationDate)).add(container.getRowId()); - - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - - auditRunnable.run(); - } - - public static void updateType(Container container, String newType, User user) - { - //For some reason there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET Type=? WHERE RowID=?").add(newType).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - } - - public static void updateTitle(Container container, String title, User user) - throws ValidationException - { - ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, title); - - //For some reason there is no primary key defined on core.containers - //so we can't use Table.update here - SQLFragment sql = new SQLFragment("UPDATE "); - sql.append(CORE.getTableInfoContainers()); - sql.append(" SET Title=? WHERE RowID=?").add(title).add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(sql); - - _removeFromCache(container, false); - String oldValue = container.getTitle(); - container = getForRowId(container.getRowId()); - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(container, user, Property.Title, oldValue, title); - firePropertyChangeEvent(evt); - } - - public static void uncache(Container c) - { - _removeFromCache(c, true); - } - - public static final String SHARED_CONTAINER_PATH = "/Shared"; - - @NotNull - public static Container getSharedContainer() - { - return ensureContainer(Path.parse(SHARED_CONTAINER_PATH), User.getAdminServiceUser()); - } - - public static List getChildren(Container parent) - { - return new ArrayList<>(getChildrenMap(parent).values()); - } - - // Default is to include all types of children, as seems only appropriate - public static List getChildren(Container parent, User u, Class perm) - { - return getChildren(parent, u, perm, null, ContainerTypeRegistry.get().getTypeNames()); - } - - public static List getChildren(Container parent, User u, Class perm, Set roles) - { - return getChildren(parent, u, perm, roles, ContainerTypeRegistry.get().getTypeNames()); - } - - public static List getChildren(Container parent, User u, Class perm, String typeIncluded) - { - return getChildren(parent, u, perm, null, Collections.singleton(typeIncluded)); - } - - public static List getChildren(Container parent, User u, Class perm, Set roles, Set includedTypes) - { - List children = new ArrayList<>(); - for (Container child : getChildrenMap(parent).values()) - if (includedTypes.contains(child.getContainerType().getName()) && child.hasPermission(u, perm, roles)) - children.add(child); - - return children; - } - - public static List getAllChildren(Container parent, User u) - { - return getAllChildren(parent, u, ReadPermission.class, null, ContainerTypeRegistry.get().getTypeNames()); - } - - public static List getAllChildren(Container parent, User u, Class perm) - { - return getAllChildren(parent, u, perm, null, ContainerTypeRegistry.get().getTypeNames()); - } - - // Default is to include all types of children - public static List getAllChildren(Container parent, User u, Class perm, Set roles) - { - return getAllChildren(parent, u, perm, roles, ContainerTypeRegistry.get().getTypeNames()); - } - - public static List getAllChildren(Container parent, User u, Class perm, String typeIncluded) - { - return getAllChildren(parent, u, perm, null, Collections.singleton(typeIncluded)); - } - - public static List getAllChildren(Container parent, User u, Class perm, Set roles, Set typesIncluded) - { - Set allChildren = getAllChildren(parent); - List result = new ArrayList<>(allChildren.size()); - - for (Container container : allChildren) - { - if (typesIncluded.contains(container.getContainerType().getName()) && container.hasPermission(u, perm, roles)) - { - result.add(container); - } - } - - return result; - } - - // Returns the next available child container name based on the baseName - public static String getAvailableChildContainerName(Container c, String baseName) - { - List children = getChildren(c); - Map folders = new HashMap<>(children.size() * 2); - for (Container child : children) - folders.put(child.getName(), child); - - String availableContainerName = baseName; - int i = 1; - while (folders.containsKey(availableContainerName)) - { - availableContainerName = baseName + " " + i++; - } - - return availableContainerName; - } - - // Returns true only if user has the specified permission in the entire container tree starting at root - public static boolean hasTreePermission(Container root, User u, Class perm) - { - for (Container c : getAllChildren(root)) - if (!c.hasPermission(u, perm)) - return false; - - return true; - } - - private static Map getChildrenMap(Container parent) - { - if (!parent.canHaveChildren()) - { - // Optimization to avoid database query (important because some installs have tens of thousands of - // workbooks) when the container is a workbook, which is not allowed to have children - return Collections.emptyMap(); - } - - List childIds = CACHE_CHILDREN.get(parent.getEntityId()); - if (null == childIds) - { - try (DbScope.Transaction t = ensureTransaction()) - { - List children = new SqlSelector(CORE.getSchema(), - "SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE Parent = ? ORDER BY SortOrder, LOWER(Name)", - parent.getId()).getArrayList(Container.class); - - childIds = new ArrayList<>(children.size()); - for (Container c : children) - { - childIds.add(c.getEntityId()); - _addToCache(c); - } - childIds = Collections.unmodifiableList(childIds); - CACHE_CHILDREN.put(parent.getEntityId(), childIds); - // No database changes to commit, but need to decrement the transaction counter - t.commit(); - } - } - - if (childIds.isEmpty()) - return Collections.emptyMap(); - - // Use a LinkedHashMap to preserve the order defined by the user - they're not necessarily alphabetical - Map ret = new LinkedHashMap<>(); - for (GUID id : childIds) - { - Container c = getForId(id); - if (null != c) - ret.put(c.getName(), c); - } - return Collections.unmodifiableMap(ret); - } - - public static Container getForRowId(int id) - { - Selector selector = new SqlSelector(CORE.getSchema(), new SQLFragment("SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE RowId = ?", id)); - return selector.getObject(Container.class); - } - - public static @Nullable Container getForId(@NotNull GUID guid) - { - return guid != null ? getForId(guid.toString()) : null; - } - - public static @Nullable Container getForId(@Nullable String id) - { - //if the input string is not a GUID, just return null, - //so that we don't get a SQLException when the database - //tries to convert it to a unique identifier. - if (!GUID.isGUID(id)) - return null; - - GUID guid = new GUID(id); - - Container d = CACHE_ENTITY_ID.get(guid); - if (null != d) - return d; - - try (DbScope.Transaction t = ensureTransaction()) - { - Container result = new SqlSelector( - CORE.getSchema(), - "SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE EntityId = ?", - id).getObject(Container.class); - if (result != null) - { - result = _addToCache(result); - } - // No database changes to commit, but need to decrement the counter - t.commit(); - - return result; - } - } - - public static Container getChild(Container c, String name) - { - Path path = c.getParsedPath().append(name); - - Container d = _getFromCachePath(path); - if (null != d) - return d; - - Map map = getChildrenMap(c); - return map.get(name); - } - - - public static Container getForURL(@NotNull ActionURL url) - { - Container ret = getForPath(url.getExtraPath()); - if (ret == null) - ret = getForId(StringUtils.strip(url.getExtraPath(), "/")); - return ret; - } - - - public static Container getForPath(@NotNull String path) - { - if (GUID.isGUID(path)) - { - Container c = getForId(path); - if (c != null) - return c; - } - - Path p = Path.parse(path); - return getForPath(p); - } - - public static Container getForPath(Path path) - { - Container d = _getFromCachePath(path); - if (null != d) - return d; - - // Special case for ROOT -- we want to throw instead of returning null - if (path.equals(Path.rootPath)) - { - try (DbScope.Transaction t = ensureTransaction()) - { - TableInfo tinfo = CORE.getTableInfoContainers(); - - // Unusual, but possible -- if cache loader hits an exception it can end up caching null - if (null == tinfo) - throw new RootContainerException("Container table could not be retrieved from the cache"); - - // This might be called at bootstrap, before schemas have been created - if (tinfo.getTableType() == DatabaseTableType.NOT_IN_DB) - throw new RootContainerException("Container table has not been created"); - - Container result = new SqlSelector(CORE.getSchema(),"SELECT * FROM " + tinfo + " WHERE Parent IS NULL").getObject(Container.class); - - if (result == null) - throw new RootContainerException("Root container does not exist"); - - _addToCache(result); - // No database changes to commit, but need to decrement the counter - t.commit(); - return result; - } - } - else - { - Path parent = path.getParent(); - String name = path.getName(); - Container dirParent = getForPath(parent); - - if (null == dirParent) - return null; - - Map map = getChildrenMap(dirParent); - return map.get(name); - } - } - - public static class RootContainerException extends RuntimeException - { - private RootContainerException(String message, Throwable cause) - { - super(message, cause); - } - - private RootContainerException(String message) - { - super(message); - } - } - - public static Container getRoot() - { - try - { - return getForPath("/"); - } - catch (MinorConfigurationException e) - { - // If the server is misconfigured, rethrow so some callers don't swallow it and other callers don't end up - // reporting it to mothership, Issue 50843. - throw e; - } - catch (Exception e) - { - // Some callers catch and ignore this exception, e.g., early in the bootstrap process - throw new RootContainerException("Root container can't be retrieved", e); - } - } - - public static void saveAliasesForContainer(Container container, List aliases, User user) - { - Set originalAliases = new CaseInsensitiveHashSet(getAliasesForContainer(container)); - Set newAliases = new CaseInsensitiveHashSet(aliases); - - if (originalAliases.equals(newAliases)) - { - return; - } - - try (DbScope.Transaction transaction = ensureTransaction()) - { - // Delete all of the aliases for the current container, plus any of the aliases that might be associated - // with another container right now - SQLFragment deleteSQL = new SQLFragment(); - deleteSQL.append("DELETE FROM "); - deleteSQL.append(CORE.getTableInfoContainerAliases()); - deleteSQL.append(" WHERE ContainerRowId = ? "); - deleteSQL.add(container.getRowId()); - if (!aliases.isEmpty()) - { - deleteSQL.append(" OR Path IN ("); - String separator = ""; - for (String alias : aliases) - { - deleteSQL.append(separator); - separator = ", "; - deleteSQL.append("LOWER(?)"); - deleteSQL.add(alias); - } - deleteSQL.append(")"); - } - new SqlExecutor(CORE.getSchema()).execute(deleteSQL); - - // Store the alias as LOWER() so that we can query against it using the index - for (String alias : newAliases) - { - SQLFragment insertSQL = new SQLFragment(); - insertSQL.append("INSERT INTO "); - insertSQL.append(CORE.getTableInfoContainerAliases()); - insertSQL.append(" (Path, ContainerRowId) VALUES (LOWER(?), ?)"); - insertSQL.add(alias); - insertSQL.add(container.getRowId()); - new SqlExecutor(CORE.getSchema()).execute(insertSQL); - } - - addAuditEvent(user, container, - "Changed folder aliases from \"" + - StringUtils.join(originalAliases, ", ") + "\" to \"" + - StringUtils.join(newAliases, ", ") + "\""); - - transaction.commit(); - } - } - - // Abstract base class used for attaching system resources (favorite icons, logos, stylesheets, sso auth logos) to folders and projects - public static abstract class ContainerParent implements AttachmentParent - { - private final Container _c; - - protected ContainerParent(Container c) - { - _c = c; - } - - @Override - public String getEntityId() - { - return _c.getId(); - } - - @Override - public String getContainerId() - { - return _c.getId(); - } - - public Container getContainer() - { - return _c; - } - } - - public static Container getHomeContainer() - { - return getForPath(HOME_PROJECT_PATH); - } - - public static List getProjects() - { - return getChildren(getRoot()); - } - - public static NavTree getProjectList(ViewContext context, boolean includeChildren) - { - User user = context.getUser(); - Container currentProject = context.getContainer().getProject(); - String projectNavTreeId = PROJECT_LIST_ID; - if (currentProject != null) - projectNavTreeId += currentProject.getId(); - - NavTree navTree = (NavTree) NavTreeManager.getFromCache(projectNavTreeId, context); - if (null != navTree) - return navTree; - - NavTree list = new NavTree("Projects"); - List projects = getProjects(); - - for (Container project : projects) - { - boolean shouldDisplay = project.shouldDisplay(user) && project.hasPermission("getProjectList()", user, ReadPermission.class); - boolean includeCurrentProject = includeChildren && currentProject != null && currentProject.equals(project); - - if (shouldDisplay || includeCurrentProject) - { - ActionURL startURL = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(project); - - if (includeChildren) - list.addChild(getFolderListForUser(project, context)); - else if (project.equals(getHomeContainer())) - list.addChild(new NavTree("Home", startURL)); - else - list.addChild(project.getTitle(), startURL); - } - } - - list.setId(projectNavTreeId); - NavTreeManager.cacheTree(list, context.getUser()); - - return list; - } - - public static NavTree getFolderListForUser(final Container project, ViewContext viewContext) - { - final boolean isNavAccessOpen = AppProps.getInstance().isNavigationAccessOpen(); - final Container c = viewContext.getContainer(); - final String cacheKey = isNavAccessOpen ? project.getId() : c.getId(); - - NavTree tree = (NavTree) NavTreeManager.getFromCache(cacheKey, viewContext); - if (null != tree) - return tree; - - try - { - assert SecurityLogger.indent("getFolderListForUser()"); - - User user = viewContext.getUser(); - String projectId = project.getId(); - - List folders = new ArrayList<>(getAllChildren(project)); - - Collections.sort(folders); - - Set containersInTree = new HashSet<>(); - - Map m = new HashMap<>(); - Map permission = new HashMap<>(); - - for (Container f : folders) - { - if (!f.isInFolderNav()) - continue; - - boolean hasPolicyRead = f.hasPermission(user, ReadPermission.class); - - boolean skip = ( - !hasPolicyRead || - !f.shouldDisplay(user) || - !f.hasPermission(user, ReadPermission.class) - ); - - //Always put the project and current container in... - if (skip && !f.equals(project) && !f.equals(c)) - continue; - - //HACK to make home link consistent... - String name = f.getTitle(); - if (name.equals("home") && f.equals(getHomeContainer())) - name = "Home"; - - NavTree t = new NavTree(name); - - // 34137: Support folder path expansion for containers where label != name - t.setId(f.getId()); - if (hasPolicyRead) - { - ActionURL url = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(f); - t.setHref(url.getEncodedLocalURIString()); - } - - boolean addFolder = false; - - if (isNavAccessOpen) - { - addFolder = true; - } - else - { - // 32718: If navigation access is not open then hide projects that aren't directly - // accessible in site folder navigation. - - if (f.equals(c) || f.isRoot() || (hasPolicyRead && f.isProject())) - { - // In current container, root, or readable project - addFolder = true; - } - else - { - boolean isAscendant = f.isDescendant(c); - boolean isDescendant = c.isDescendant(f); - boolean inActivePath = isAscendant || isDescendant; - boolean hasAncestryRead = false; - - if (inActivePath) - { - Container leaf = isAscendant ? f : c; - Container localRoot = isAscendant ? c : f; - - List ancestors = containersToRootList(leaf); - Collections.reverse(ancestors); - - for (Container p : ancestors) - { - if (!permission.containsKey(p.getId())) - permission.put(p.getId(), p.hasPermission(user, ReadPermission.class)); - boolean hasRead = permission.get(p.getId()); - - if (p.equals(localRoot)) - { - hasAncestryRead = hasRead; - break; - } - else if (!hasRead) - { - hasAncestryRead = false; - break; - } - } - } - else - { - hasAncestryRead = containersToRoot(f).stream().allMatch(p -> { - if (!permission.containsKey(p.getId())) - permission.put(p.getId(), p.hasPermission(user, ReadPermission.class)); - return permission.get(p.getId()); - }); - } - - if (hasPolicyRead && hasAncestryRead && inActivePath) - { - // Is in the direct readable lineage of the current container - addFolder = true; - } - else if (hasPolicyRead && f.getParent().equals(c.getParent())) - { - // Is a readable sibling of the current container - addFolder = true; - } - else if (hasAncestryRead) - { - // Is a part of a fully readable ancestry - addFolder = true; - } - } - - if (!addFolder) - LOG.debug("isNavAccessOpen restriction: \"" + f.getPath() + "\""); - } - - if (addFolder) - { - containersInTree.add(f); - m.put(f.getId(), t); - } - } - - //Ensure parents of any accessible folder are in the tree. If not add them with no link. - for (Container treeContainer : containersInTree) - { - if (!treeContainer.equals(project) && !containersInTree.contains(treeContainer.getParent())) - { - Set containersToRoot = containersToRoot(treeContainer); - //Possible will be added more than once, if several children are accessible, but that's OK... - for (Container missing : containersToRoot) - { - if (!m.containsKey(missing.getId())) - { - if (isNavAccessOpen) - { - NavTree noLinkTree = new NavTree(missing.getName()); - noLinkTree.setId(missing.getId()); - m.put(missing.getId(), noLinkTree); - } - else - { - if (!permission.containsKey(missing.getId())) - permission.put(missing.getId(), missing.hasPermission(user, ReadPermission.class)); - - if (!permission.get(missing.getId())) - { - NavTree noLinkTree = new NavTree(missing.getName()); - m.put(missing.getId(), noLinkTree); - } - else - { - NavTree linkTree = new NavTree(missing.getName()); - ActionURL url = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(missing); - linkTree.setHref(url.getEncodedLocalURIString()); - m.put(missing.getId(), linkTree); - } - } - } - } - } - } - - for (Container f : folders) - { - if (f.getId().equals(projectId)) - continue; - - NavTree child = m.get(f.getId()); - if (null == child) - continue; - - NavTree parent = m.get(f.getParent().getId()); - assert null != parent; //This should not happen anymore, we assure all parents are in tree. - if (null != parent) - parent.addChild(child); - } - - NavTree projectTree = m.get(projectId); - - projectTree.setId(cacheKey); - - NavTreeManager.cacheTree(projectTree, user); - return projectTree; - } - finally - { - assert SecurityLogger.outdent(); - } - } - - public static Set containersToRoot(Container child) - { - Set containersOnPath = new HashSet<>(); - Container current = child; - while (current != null && !current.isRoot()) - { - containersOnPath.add(current); - current = current.getParent(); - } - - return containersOnPath; - } - - /** - * Provides a sorted list of containers from the root to the child container provided. - * It does not include the root node. - * @param child Container from which the search is sourced. - * @return List sorted in order of distance from root. - */ - public static List containersToRootList(Container child) - { - List containers = new ArrayList<>(); - Container current = child; - while (current != null && !current.isRoot()) - { - containers.add(current); - current = current.getParent(); - } - - Collections.reverse(containers); - return containers; - } - - // Move a container to another part of the container tree. Careful: this method DOES NOT prevent you from orphaning - // an entire tree (e.g., by setting a container's parent to one of its children); the UI in AdminController does this. - // - // NOTE: Beware side-effect of changing ACLs and GROUPS if a container changes projects - // - // @return true if project has changed (should probably redirect to security page) - public static boolean move(Container c, final Container newParent, User user) throws ValidationException - { - if (!isRenameable(c)) - { - throw new IllegalArgumentException("Can't move container " + c.getPath()); - } - - try (QuietCloser ignored = lockForMutation(MutatingOperation.move, c)) - { - List errors = new ArrayList<>(); - for (ContainerListener listener : getListeners()) - { - try - { - errors.addAll(listener.canMove(c, newParent, user)); - } - catch (Exception e) - { - ExceptionUtil.logExceptionToMothership(null, new IllegalStateException(listener.getClass().getName() + ".canMove() threw an exception or violated @NotNull contract")); - } - } - if (!errors.isEmpty()) - { - ValidationException exception = new ValidationException(); - for (String error : errors) - { - exception.addError(new SimpleValidationError(error)); - } - throw exception; - } - - if (c.getParent().getId().equals(newParent.getId())) - return false; - - Container oldParent = c.getParent(); - Container oldProject = c.getProject(); - Container newProject = newParent.isRoot() ? c : newParent.getProject(); - - boolean changedProjects = !oldProject.getId().equals(newProject.getId()); - - // Synchronize the transaction, but not the listeners -- see #9901 - try (DbScope.Transaction t = ensureTransaction()) - { - new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET Parent = ? WHERE EntityId = ?", newParent.getId(), c.getId()); - - // Refresh the container directly from the database so the container reflects the new parent, isProject(), etc. - c = getForRowId(c.getRowId()); - - // this could be done in the trigger, but I prefer to put it in the transaction - if (changedProjects) - SecurityManager.changeProject(c, oldProject, newProject, user); - - clearCache(); - - try - { - ExperimentService.get().moveContainer(c, oldParent, newParent); - } - catch (ExperimentException e) - { - throw new RuntimeException(e); - } - - // Clear after the commit has propagated the state to other threads and transactions - // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own - t.addCommitTask(() -> - { - clearCache(); - getChildrenMap(newParent); // reload the cache - }, DbScope.CommitTaskOption.POSTCOMMIT); - - t.commit(); - } - - Container newContainer = getForId(c.getId()); - fireMoveContainer(newContainer, oldParent, user); - - return changedProjects; - } - } - - public static void rename(@NotNull Container c, User user, String name) - { - rename(c, user, name, c.getTitle(), false); - } - - /** - * Transacted method to rename a container. Optionally, supports updating the title and aliasing the - * original container path when the name is changed (as name changes result in a new container path). - */ - public static Container rename(@NotNull Container c, User user, String name, @Nullable String title, boolean addAlias) - { - try (QuietCloser ignored = lockForMutation(MutatingOperation.rename, c); - DbScope.Transaction tx = ensureTransaction()) - { - final String oldName = c.getName(); - final String newName = StringUtils.trimToNull(name); - boolean isRenaming = !oldName.equals(newName); - StringBuilder errors = new StringBuilder(); - - // Rename - if (isRenaming) - { - // Issue 16221: Don't allow renaming of system reserved folders (e.g. /Shared, home, root, etc). - if (!isRenameable(c)) - throw new ApiUsageException("This folder may not be renamed as it is reserved by the system."); - - if (!Container.isLegalName(newName, c.isProject(), errors)) - throw new ApiUsageException(errors.toString()); - - // Issue 19061: Unable to do case-only container rename - if (c.getParent().hasChild(newName) && !c.equals(c.getParent().getChild(newName))) - { - if (c.getParent().isRoot()) - throw new ApiUsageException("The server already has a project with this name."); - throw new ApiUsageException("The " + (c.getParent().isProject() ? "project " : "folder ") + c.getParent().getPath() + " already has a folder with this name."); - } - - new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET Name=? WHERE EntityId=?", newName, c.getId()); - clearCache(); // Clear the entire cache, since containers cache their full paths - // Get new version since name has changed. - Container renamedContainer = getForId(c.getId()); - fireRenameContainer(renamedContainer, user, oldName); - // Clear again after the commit has propagated the state to other threads and transactions - // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own - tx.addCommitTask(ContainerManager::clearCache, DbScope.CommitTaskOption.POSTCOMMIT); - - // Alias - if (addAlias) - { - // Intentionally use original container rather than the already renamedContainer - List newAliases = new ArrayList<>(getAliasesForContainer(c)); - newAliases.add(c.getPath()); - saveAliasesForContainer(c, newAliases, user); - } - } - - // Title - if (!c.getTitle().equals(title)) - { - if (!Container.isLegalTitle(title, errors)) - throw new ApiUsageException(errors.toString()); - updateTitle(c, title, user); - } - - tx.commit(); - } - catch (ValidationException e) - { - throw new IllegalArgumentException(e); - } - - return getForId(c.getId()); - } - - public static void setChildOrderToAlphabetical(Container parent) - { - setChildOrder(parent.getChildren(), true); - } - - public static void setChildOrder(Container parent, List orderedChildren) throws ContainerException - { - for (Container child : orderedChildren) - { - if (child == null || child.getParent() == null || !child.getParent().equals(parent)) // #13481 - throw new ContainerException("Invalid parent container of " + (child == null ? "null child container" : child.getPath())); - } - setChildOrder(orderedChildren, false); - } - - private static void setChildOrder(List siblings, boolean resetToAlphabetical) - { - try (DbScope.Transaction t = ensureTransaction()) - { - for (int index = 0; index < siblings.size(); index++) - { - Container current = siblings.get(index); - new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET SortOrder = ? WHERE EntityId = ?", - resetToAlphabetical ? 0 : index, current.getId()); - } - // Clear after the commit has propagated the state to other threads and transactions - // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own - t.addCommitTask(ContainerManager::clearCache, DbScope.CommitTaskOption.POSTCOMMIT); - - t.commit(); - } - } - - private enum MutatingOperation - { - delete, - rename, - move - } - - private static final Map mutatingContainers = Collections.synchronizedMap(new IntHashMap<>()); - - private static QuietCloser lockForMutation(MutatingOperation op, Container c) - { - return lockForMutation(op, Collections.singletonList(c)); - } - - private static QuietCloser lockForMutation(MutatingOperation op, Collection containers) - { - List ids = new ArrayList<>(containers.size()); - synchronized (mutatingContainers) - { - for (Container container : containers) - { - MutatingOperation currentOp = mutatingContainers.get(container.getRowId()); - if (currentOp != null) - { - throw new ApiUsageException("Cannot start a " + op + " operation on " + container.getPath() + ". It is currently undergoing a " + currentOp); - } - ids.add(container.getRowId()); - } - ids.forEach(id -> mutatingContainers.put(id, op)); - } - return () -> - { - synchronized (mutatingContainers) - { - ids.forEach(mutatingContainers::remove); - } - }; - } - - // Delete containers from the database - private static boolean delete(final Collection containers, User user, @Nullable String comment) - { - // Do this check before we bother with any synchronization - for (Container container : containers) - { - if (!isDeletable(container)) - { - throw new ApiUsageException("Cannot delete container: " + container.getPath()); - } - } - - try (QuietCloser ignored = lockForMutation(MutatingOperation.delete, containers)) - { - boolean deleted = true; - for (Container c : containers) - { - deleted = deleted && delete(c, user, comment); - } - return deleted; - } - } - - // Delete a container from the database - private static boolean delete(final Container c, User user, @Nullable String comment) - { - // Verify method isn't called inappropriately - if (mutatingContainers.get(c.getRowId()) != MutatingOperation.delete) - { - throw new IllegalStateException("Container not flagged as being deleted: " + c.getPath()); - } - - LOG.debug("Starting container delete for " + c.getContainerNoun(true) + " " + c.getPath()); - - SearchService ss = SearchService.get(); - if (ss != null) - { - // Tell the search indexer to drop work for the container that's about to be deleted - ss.purgeForContainer(c); - } - - DbScope.RetryFn tryDeleteContainer = (tx) -> - { - // Verify that no children exist - Selector sel = new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("Parent"), c), null); - - if (sel.exists()) - { - _removeFromCache(c, true); - return false; - } - - if (c.shouldRemoveFromPortal()) - { - // Need to remove portal page, too; container name is page's pageId and in container's parent container - Portal.PortalPage page = Portal.getPortalPage(c.getParent(), c.getName()); - if (null != page) // Be safe - Portal.deletePage(page); - - // Tell parent - setContainerTabDeleted(c.getParent(), c.getName(), c.getParent().getFolderType().getName()); - } - - fireDeleteContainer(c, user); - - SqlExecutor sqlExecutor = new SqlExecutor(CORE.getSchema()); - sqlExecutor.execute("DELETE FROM " + CORE.getTableInfoContainerAliases() + " WHERE ContainerRowId=?", c.getRowId()); - sqlExecutor.execute("DELETE FROM " + CORE.getTableInfoContainers() + " WHERE EntityId=?", c.getId()); - // now that the container is actually gone, delete all ACLs (better to have an ACL w/o object than object w/o ACL) - SecurityPolicyManager.removeAll(c); - // and delete all container-based sequences - DbSequenceManager.deleteAll(c); - - ExperimentService experimentService = ExperimentService.get(); - if (experimentService != null) - experimentService.removeContainerDataTypeExclusions(c.getId()); - - // After we've committed the transaction, be sure that we remove this container from the cache - // See https://www.labkey.org/issues/home/Developer/issues/details.view?issueId=17015 - tx.addCommitTask(() -> - { - // Be sure that we've waited until any threads that might be populating the cache have finished - // before we guarantee that we've removed this now-deleted container - DATABASE_QUERY_LOCK.lock(); - try - { - _removeFromCache(c, true); - } - finally - { - DATABASE_QUERY_LOCK.unlock(); - } - }, DbScope.CommitTaskOption.POSTCOMMIT); - String auditComment = c.getContainerNoun(true) + " " + c.getPath() + " was deleted"; - if (comment != null) - auditComment = auditComment.concat(". " + comment); - addAuditEvent(user, c, auditComment); - return true; - }; - - boolean success = CORE.getSchema().getScope().executeWithRetry(tryDeleteContainer); - if (success) - { - LOG.debug("Completed container delete for " + c.getContainerNoun(true) + " " + c.getPath()); - } - else - { - LOG.warn("Failed to delete container: " + c.getPath()); - } - return success; - } - - /** - * Delete a single container. Primarily for use by tests. - */ - public static boolean delete(final Container c, User user) - { - return delete(List.of(c), user, null); - } - - public static boolean isDeletable(Container c) - { - return !isSystemContainer(c); - } - - public static boolean isRenameable(Container c) - { - return !isSystemContainer(c); - } - - /** System containers include the root container, /Home, and /Shared */ - public static boolean isSystemContainer(Container c) - { - return c.equals(getRoot()) || c.equals(getHomeContainer()) || c.equals(getSharedContainer()); - } - - /** Has the container already been deleted or is it in the process of being deleted? */ - public static boolean exists(@Nullable Container c) - { - return c != null && null != getForId(c.getEntityId()) && mutatingContainers.get(c.getRowId()) != MutatingOperation.delete; - } - - public static void deleteAll(Container root, User user, @Nullable String comment) throws UnauthorizedException - { - if (!hasTreePermission(root, user, DeletePermission.class)) - throw new UnauthorizedException("You don't have delete permissions to all folders"); - - LOG.debug("Starting container (and children) delete for " + root.getContainerNoun(true) + " " + root.getPath()); - Set depthFirst = getAllChildrenDepthFirst(root); - depthFirst.add(root); - - delete(depthFirst, user, comment); - - LOG.debug("Completed container (and children) delete for " + root.getContainerNoun(true) + " " + root.getPath()); - } - - public static void deleteAll(Container root, User user) throws UnauthorizedException - { - deleteAll(root, user, null); - } - - private static void addAuditEvent(User user, Container c, String comment) - { - if (user != null) - { - AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, c, comment); - AuditLogService.get().addEvent(user, event); - } - } - - private static Set getAllChildrenDepthFirst(Container c) - { - Set set = new LinkedHashSet<>(); - getAllChildrenDepthFirst(c, set); - return set; - } - - private static void getAllChildrenDepthFirst(Container c, Collection list) - { - for (Container child : c.getChildren()) - { - getAllChildrenDepthFirst(child, list); - list.add(child); - } - } - - private static Container _getFromCachePath(Path path) - { - return CACHE_PATH.get(path); - } - - private static Container _addToCache(Container c) - { - assert DATABASE_QUERY_LOCK.isHeldByCurrentThread() : "Any cache modifications must be synchronized at a " + - "higher level so that we ensure that the container to be inserted still exists and hasn't been deleted"; - CACHE_ENTITY_ID.put(c.getEntityId(), c); - CACHE_PATH.put(c.getParsedPath(), c); - return c; - } - - private static void _clearChildrenFromCache(Container c) - { - CACHE_CHILDREN.remove(c.getEntityId()); - navTreeManageUncache(c); - } - - /** @param hierarchyChange whether the shape of the container tree has changed */ - private static void _removeFromCache(Container c, boolean hierarchyChange) - { - CACHE_ENTITY_ID.remove(c.getEntityId()); - CACHE_PATH.remove(c.getParsedPath()); - - if (hierarchyChange) - { - // This is strictly keeping track of the parent/child relationships themselves so it only needs to be - // cleared when the tree changes - CACHE_CHILDREN.clear(); - } - - navTreeManageUncache(c); - } - - public static void clearCache() - { - CACHE_PATH.clear(); - CACHE_ENTITY_ID.clear(); - CACHE_CHILDREN.clear(); - - // UNDONE: NavTreeManager should register a ContainerListener - NavTreeManager.uncacheAll(); - } - - private static void navTreeManageUncache(Container c) - { - // UNDONE: NavTreeManager should register a ContainerListener - NavTreeManager.uncacheTree(PROJECT_LIST_ID); - NavTreeManager.uncacheTree(getRoot().getId()); - - Container project = c.getProject(); - if (project != null) - { - NavTreeManager.uncacheTree(project.getId()); - NavTreeManager.uncacheTree(PROJECT_LIST_ID + project.getId()); - } - } - - public static void notifyContainerChange(String id, Property prop) - { - notifyContainerChange(id, prop, null); - } - - public static void notifyContainerChange(String id, Property prop, @Nullable User u) - { - if (_constructing.contains(new GUID(id))) - return; - - Container c = getForId(id); - if (null != c) - { - _removeFromCache(c, false); - c = getForId(id); // load a fresh container since the original might be stale. - if (null != c) - { - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, u, prop, null, null); - firePropertyChangeEvent(evt); - } - } - } - - - /** Recursive, including root node */ - public static Set getAllChildren(Container root) - { - Set children = getAllChildrenDepthFirst(root); - children.add(root); - - return Collections.unmodifiableSet(children); - } - - /** - * Return all children of the root node, including root node, which have the given active module - */ - @NotNull - public static Set getAllChildrenWithModule(@NotNull Container root, @NotNull Module module) - { - Set children = new HashSet<>(); - for (Container candidate : getAllChildren(root)) - { - if (candidate.getActiveModules().contains(module)) - children.add(candidate); - } - return Collections.unmodifiableSet(children); - } - - public static long getContainerCount() - { - return new TableSelector(CORE.getTableInfoContainers()).getRowCount(); - } - - public static long getWorkbookCount() - { - return new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("type"), "workbook"), null).getRowCount(); - } - - public static long getArchivedContainerCount() - { - return new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("lockstate"), "Archived"), null).getRowCount(); - } - - public static long getAuditCommentRequiredCount() - { - SQLFragment sql = new SQLFragment( - "SELECT COUNT(*) FROM\n" + - " core.containers c\n" + - " JOIN prop.propertysets ps on c.entityid = ps.objectid\n" + - " JOIN prop.properties p on p.\"set\" = ps.\"set\"\n" + - "WHERE ps.category = '" + AUDIT_SETTINGS_PROPERTY_SET_NAME + "' AND p.name='"+ REQUIRE_USER_COMMENTS_PROPERTY_NAME + "' and p.value='true'"); - return new SqlSelector(CORE.getSchema(), sql).getObject(Long.class); - } - - - /** Retrieve entire container hierarchy */ - public static MultiValuedMap getContainerTree() - { - final MultiValuedMap mm = new ArrayListValuedHashMap<>(); - - // Get all containers and parents - SqlSelector selector = new SqlSelector(CORE.getSchema(), "SELECT Parent, EntityId FROM " + CORE.getTableInfoContainers() + " ORDER BY SortOrder, LOWER(Name) ASC"); - - selector.forEach(rs -> { - String parentId = rs.getString(1); - Container parent = (parentId != null ? getForId(parentId) : null); - Container child = getForId(rs.getString(2)); - - if (null != child) - mm.put(parent, child); - }); - - return mm; - } - - /** - * Returns a branch of the container tree including only the root and its descendants - * @param root The root container - * @return MultiMap of containers including root and its descendants - */ - public static MultiValuedMap getContainerTree(Container root) - { - //build a multimap of only the container ids - final MultiValuedMap mmIds = new ArrayListValuedHashMap<>(); - - // Get all containers and parents - Selector selector = new SqlSelector(CORE.getSchema(), "SELECT Parent, EntityId FROM " + CORE.getTableInfoContainers() + " ORDER BY SortOrder, LOWER(Name) ASC"); - - selector.forEach(rs -> mmIds.put(rs.getString(1), rs.getString(2))); - - //now find the root and build a MultiMap of it and its descendants - MultiValuedMap mm = new ArrayListValuedHashMap<>(); - mm.put(null, root); - addChildren(root, mmIds, mm); - return mm; - } - - private static void addChildren(Container c, MultiValuedMap mmIds, MultiValuedMap mm) - { - Collection childIds = mmIds.get(c.getId()); - if (null != childIds) - { - for (String childId : childIds) - { - Container child = getForId(childId); - if (null != child) - { - mm.put(c, child); - addChildren(child, mmIds, mm); - } - } - } - } - - public static Set getContainerSet(MultiValuedMap mm, User user, Class perm) - { - Collection containers = mm.values(); - if (null == containers) - return new HashSet<>(); - - return containers - .stream() - .filter(c -> c.hasPermission(user, perm)) - .collect(Collectors.toSet()); - } - - - public static SQLFragment getIdsAsCsvList(Set containers, SqlDialect d) - { - if (containers.isEmpty()) - return new SQLFragment("(NULL)"); // WHERE x IN (NULL) should match no rows - - SQLFragment csvList = new SQLFragment("("); - String comma = ""; - for (Container container : containers) - { - csvList.append(comma); - comma = ","; - csvList.appendValue(container, d); - } - csvList.append(")"); - - return csvList; - } - - - public static List getIds(User user, Class perm) - { - Set containers = getContainerSet(getContainerTree(), user, perm); - - List ids = new ArrayList<>(containers.size()); - - for (Container c : containers) - ids.add(c.getId()); - - return ids; - } - - - // - // ContainerListener - // - - public interface ContainerListener extends PropertyChangeListener - { - enum Order {First, Last} - - /** Called after a new container has been created */ - void containerCreated(Container c, User user); - - default void containerCreated(Container c, User user, @Nullable String auditMsg) - { - containerCreated(c, user); - } - - /** Called immediately prior to deleting the row from core.containers */ - void containerDeleted(Container c, User user); - - /** Called after the container has been moved to its new parent */ - void containerMoved(Container c, Container oldParent, User user); - - /** - * Called prior to moving a container, to find out if there are any issues that would prevent a successful move - * @return a list of errors that should prevent the move from happening, if any - */ - @NotNull - Collection canMove(Container c, Container newParent, User user); - - @Override - void propertyChange(PropertyChangeEvent evt); - } - - public static abstract class AbstractContainerListener implements ContainerListener - { - @Override - public void containerCreated(Container c, User user) - {} - - @Override - public void containerDeleted(Container c, User user) - {} - - @Override - public void containerMoved(Container c, Container oldParent, User user) - {} - - @NotNull - @Override - public Collection canMove(Container c, Container newParent, User user) - { - return Collections.emptyList(); - } - - @Override - public void propertyChange(PropertyChangeEvent evt) - {} - } - - - public static class ContainerPropertyChangeEvent extends PropertyChangeEvent implements PropertyChange - { - public final Property property; - public final Container container; - public User user; - - public ContainerPropertyChangeEvent(Container c, @Nullable User user, Property p, Object oldValue, Object newValue) - { - super(c, p.name(), oldValue, newValue); - container = c; - this.user = user; - property = p; - } - - public ContainerPropertyChangeEvent(Container c, Property p, Object oldValue, Object newValue) - { - this(c, null, p, oldValue, newValue); - } - - @Override - public Property getProperty() - { - return property; - } - } - - - // Thread-safe list implementation that allows iteration and modifications without external synchronization - private static final List _listeners = new CopyOnWriteArrayList<>(); - private static final List _laterListeners = new CopyOnWriteArrayList<>(); - - // These listeners are executed in the order they are registered, before the "Last" listeners - public static void addContainerListener(ContainerListener listener) - { - addContainerListener(listener, ContainerListener.Order.First); - } - - - // Explicitly request "Last" ordering via this method. "Last" listeners execute after all "First" listeners. - public static void addContainerListener(ContainerListener listener, ContainerListener.Order order) - { - if (ContainerListener.Order.First == order) - _listeners.add(listener); - else - _laterListeners.add(listener); - } - - - public static void removeContainerListener(ContainerListener listener) - { - _listeners.remove(listener); - _laterListeners.remove(listener); - } - - - private static List getListeners() - { - List combined = new ArrayList<>(_listeners.size() + _laterListeners.size()); - combined.addAll(_listeners); - combined.addAll(_laterListeners); - - return combined; - } - - - private static List getListenersReversed() - { - List combined = new LinkedList<>(); - - // Copy to guarantee consistency between .listIterator() and .size() - List copy = new ArrayList<>(_listeners); - ListIterator iter = copy.listIterator(copy.size()); - - // Iterate in reverse - while(iter.hasPrevious()) - combined.add(iter.previous()); - - // Copy to guarantee consistency between .listIterator() and .size() - // Add elements from the laterList in reverse order so that Core is fired last - List laterCopy = new ArrayList<>(_laterListeners); - ListIterator laterIter = laterCopy.listIterator(laterCopy.size()); - - // Iterate in reverse - while(laterIter.hasPrevious()) - combined.add(laterIter.previous()); - - return combined; - } - - - protected static void fireCreateContainer(Container c, User user, @Nullable String auditMsg) - { - List list = getListeners(); - - for (ContainerListener cl : list) - { - try - { - cl.containerCreated(c, user, auditMsg); - } - catch (Throwable t) - { - LOG.error("fireCreateContainer for " + cl.getClass().getName(), t); - } - } - } - - - protected static void fireDeleteContainer(Container c, User user) - { - List list = getListenersReversed(); - - for (ContainerListener l : list) - { - LOG.debug("Deleting " + c.getPath() + ": fireDeleteContainer for " + l.getClass().getName()); - try - { - l.containerDeleted(c, user); - } - catch (RuntimeException e) - { - LOG.error("fireDeleteContainer for " + l.getClass().getName(), e); - - // Fail fast (first Throwable aborts iteration), #17560 - throw e; - } - } - } - - - protected static void fireRenameContainer(Container c, User user, String oldValue) - { - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, user, Property.Name, oldValue, c.getName()); - firePropertyChangeEvent(evt); - } - - - protected static void fireMoveContainer(Container c, Container oldParent, User user) - { - List list = getListeners(); - - for (ContainerListener cl : list) - { - // While we would ideally transact the full container move, that will likely cause long-blocking - // queries and/or deadlocks. For now, at least transact each separate move handler independently - try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) - { - cl.containerMoved(c, oldParent, user); - transaction.commit(); - } - } - ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, user, Property.Parent, oldParent, c.getParent()); - firePropertyChangeEvent(evt); - } - - - public static void firePropertyChangeEvent(ContainerPropertyChangeEvent evt) - { - if (_constructing.contains(evt.container.getEntityId())) - return; - - List list = getListeners(); - for (ContainerListener l : list) - { - try - { - l.propertyChange(evt); - } - catch (Throwable t) - { - LOG.error("firePropertyChangeEvent for " + l.getClass().getName(), t); - } - } - } - - private static final List MODULE_DEPENDENCY_PROVIDERS = new CopyOnWriteArrayList<>(); - - public static void registerModuleDependencyProvider(ModuleDependencyProvider provider) - { - MODULE_DEPENDENCY_PROVIDERS.add(provider); - } - - public static void forEachModuleDependencyProvider(Consumer action) - { - MODULE_DEPENDENCY_PROVIDERS.forEach(action); - } - - // Compliance module adds a locked project handler that checks permissions; without that, this implementation - // is used, and projects are never locked - static volatile LockedProjectHandler LOCKED_PROJECT_HANDLER = (project, user, contextualRoles, lockState) -> false; - - // Replaces any previously set LockedProjectHandler - public static void setLockedProjectHandler(LockedProjectHandler handler) - { - LOCKED_PROJECT_HANDLER = handler; - } - - public static Container createDefaultSupportContainer() - { - LOG.info("Creating default support container: " + DEFAULT_SUPPORT_PROJECT_PATH); - // create a "support" container. Admins can do anything, - // Users can read/write, Guests can read. - return bootstrapContainer(DEFAULT_SUPPORT_PROJECT_PATH, - RoleManager.getRole(AuthorRole.class), - RoleManager.getRole(ReaderRole.class) - ); - } - - public static void removeDefaultSupportContainer(User user) - { - Container support = getDefaultSupportContainer(); - if (support != null) - { - LOG.info("Removing default support container: " + DEFAULT_SUPPORT_PROJECT_PATH); - ContainerManager.delete(support, user); - } - } - - public static Container getDefaultSupportContainer() - { - return getForPath(DEFAULT_SUPPORT_PROJECT_PATH); - } - - public static List getAliasesForContainer(Container c) - { - return Collections.unmodifiableList(new SqlSelector(CORE.getSchema(), - new SQLFragment("SELECT Path FROM " + CORE.getTableInfoContainerAliases() + " WHERE ContainerRowId = ? ORDER BY Path", - c.getRowId())).getArrayList(String.class)); - } - - @Nullable - public static Container resolveContainerPathAlias(String path) - { - return resolveContainerPathAlias(path, false); - } - - @Nullable - private static Container resolveContainerPathAlias(String path, boolean top) - { - // Strip any trailing slashes - while (path.endsWith("/")) - { - path = path.substring(0, path.length() - 1); - } - - // Simple case -- resolve directly (sans alias) - Container aliased = getForPath(path); - if (aliased != null) - return aliased; - - // Simple case -- directly resolve from database - aliased = getForPathAlias(path); - if (aliased != null) - return aliased; - - // At the leaf and the container was not found - if (top) - return null; - - List splits = Arrays.asList(path.split("/")); - String subPath = ""; - for (int i=0; i < splits.size()-1; i++) // minus 1 due to leaving off last container - { - if (!splits.get(i).isEmpty()) - subPath += "/" + splits.get(i); - } - - aliased = resolveContainerPathAlias(subPath, false); - - if (aliased == null) - return null; - - String leafPath = aliased.getPath() + "/" + splits.get(splits.size()-1); - return resolveContainerPathAlias(leafPath, true); - } - - @Nullable - private static Container getForPathAlias(String path) - { - // We store the path as lower-case, so we don't need to also LOWER() on the value in core.ContainerAliases, letting the DB use the index - Container[] ret = new SqlSelector(CORE.getSchema(), - "SELECT * FROM " + CORE.getTableInfoContainers() + " c, " + CORE.getTableInfoContainerAliases() + " ca WHERE ca.ContainerRowId = c.RowId AND ca.path = LOWER(?)", - path).getArray(Container.class); - - return ret.length == 0 ? null : ret[0]; - } - - public static Container getMoveTargetContainer(@Nullable String queryName, @NotNull Container sourceContainer, User user, @Nullable String targetIdOrPath, Errors errors) - { - if (targetIdOrPath == null) - { - errors.reject(ERROR_GENERIC, "A target container must be specified for the move operation."); - return null; - } - - Container _targetContainer = getContainerForIdOrPath(targetIdOrPath); - if (_targetContainer == null) - { - errors.reject(ERROR_GENERIC, "The target container was not found: " + targetIdOrPath + "."); - return null; - } - - if (!_targetContainer.hasPermission(user, InsertPermission.class)) - { - String _queryName = queryName == null ? "this table" : "'" + queryName + "'"; - errors.reject(ERROR_GENERIC, "You do not have permission to move rows from " + _queryName + " to the target container: " + targetIdOrPath + "."); - return null; - } - - if (!isValidTargetContainer(sourceContainer, _targetContainer)) - { - errors.reject(ERROR_GENERIC, "Invalid target container for the move operation: " + targetIdOrPath + "."); - return null; - } - return _targetContainer; - } - - private static Container getContainerForIdOrPath(String targetContainer) - { - Container c = ContainerManager.getForId(targetContainer); - if (c == null) - c = ContainerManager.getForPath(targetContainer); - - return c; - } - - // targetContainer must be in the same app project at this time - // i.e. child of current project, project of current child, sibling within project - private static boolean isValidTargetContainer(Container current, Container target) - { - if (current.isRoot() || target.isRoot()) - return false; - - // Allow moving to the current container since we now allow the chosen entities to be from different containers - if (current.equals(target)) - return true; - - boolean moveFromProjectToChild = current.isProject() && target.getParent().equals(current); - boolean moveFromChildToProject = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target); - boolean moveFromChildToSibling = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target.getParent()); - - return moveFromProjectToChild || moveFromChildToProject || moveFromChildToSibling; - } - - public static int updateContainer(TableInfo dataTable, String idField, Collection ids, Container targetContainer, User user, boolean withModified) - { - try (DbScope.Transaction transaction = dataTable.getSchema().getScope().ensureTransaction()) - { - SQLFragment dataUpdate = new SQLFragment("UPDATE ").append(dataTable) - .append(" SET container = ").appendValue(targetContainer.getEntityId()); - if (withModified) - { - dataUpdate.append(", modified = ").appendValue(new Date()); - dataUpdate.append(", modifiedby = ").appendValue(user.getUserId()); - } - dataUpdate.append(" WHERE ").append(idField); - dataTable.getSchema().getSqlDialect().appendInClauseSql(dataUpdate, ids); - int numUpdated = new SqlExecutor(dataTable.getSchema()).execute(dataUpdate); - transaction.commit(); - - return numUpdated; - } - } - - /** - * If a container at the given path does not exist, create one and set permissions. If the container does exist, - * permissions are only set if there is no explicit ACL for the container. This prevents us from resetting - * permissions if all users are dropped. Implicitly done as an admin-level service user. - */ - @NotNull - public static Container bootstrapContainer(String path, @NotNull Role userRole, @Nullable Role guestRole) - { - Container c = null; - User user = User.getAdminServiceUser(); - - try - { - c = getForPath(path); - } - catch (RootContainerException e) - { - // Ignore this -- root doesn't exist yet - } - boolean newContainer = false; - - if (c == null) - { - LOG.debug("Creating new container for path '" + path + "'"); - newContainer = true; - c = ensureContainer(path, user); - } - - // Only set permissions if there are no explicit permissions - // set for this object or we just created it - Integer policyCount = null; - if (!newContainer) - { - policyCount = new SqlSelector(CORE.getSchema(), - "SELECT COUNT(*) FROM " + CORE.getTableInfoPolicies() + " WHERE ResourceId = ?", - c.getId()).getObject(Integer.class); - } - - if (newContainer || 0 == policyCount.intValue()) - { - LOG.debug("Setting permissions for '" + path + "'"); - MutableSecurityPolicy policy = new MutableSecurityPolicy(c); - policy.addRoleAssignment(SecurityManager.getGroup(Group.groupUsers), userRole); - if (guestRole != null) - policy.addRoleAssignment(SecurityManager.getGroup(Group.groupGuests), guestRole); - SecurityPolicyManager.savePolicy(policy, user); - } - - return c; - } - - /** - * @param container the container being created. May be null if we haven't actually created it yet - * @param parent the parent of the container being created. Used in case the container doesn't actually exist yet. - * @return the list of standard steps and any extra ones based on the container's FolderType - */ - public static List getCreateContainerWizardSteps(@Nullable Container container, @NotNull Container parent) - { - List navTrail = new ArrayList<>(); - - boolean isProject = parent.isRoot(); - - navTrail.add(new NavTree(isProject ? "Create Project" : "Create Folder")); - navTrail.add(new NavTree("Users / Permissions")); - if (isProject) - navTrail.add(new NavTree("Project Settings")); - if (container != null) - navTrail.addAll(container.getFolderType().getExtraSetupSteps(container)); - return navTrail; - } - - @TestTimeout(120) @TestWhen(TestWhen.When.BVT) - public static class TestCase extends Assert implements ContainerListener - { - Map _containers = new HashMap<>(); - Container _testRoot = null; - - @Before - public void setUp() - { - if (null == _testRoot) - { - Container junit = JunitUtil.getTestContainer(); - _testRoot = ensureContainer(junit, "ContainerManager$TestCase-" + GUID.makeGUID(), TestContext.get().getUser()); - addContainerListener(this); - } - } - - @After - public void tearDown() - { - removeContainerListener(this); - if (null != _testRoot) - deleteAll(_testRoot, TestContext.get().getUser()); - } - - @Test - public void testImproperFolderNamesBlocked() - { - String[] badNames = {"", "f\\o", "f/o", "f\\\\o", "foo;", "@foo", "foo" + '\u001F', '\u0000' + "foo", "fo" + '\u007F' + "o", "" + '\u009F'}; - - for (String name: badNames) - { - try - { - Container c = createContainer(_testRoot, name, TestContext.get().getUser()); - try - { - assertTrue(delete(c, TestContext.get().getUser())); - } - catch (Exception ignored) {} - fail("Should have thrown exception when trying to create container with name: " + name); - } - catch (ApiUsageException e) - { - // Do nothing, this is expected - } - } - } - - @Test - public void testCreateDeleteContainers() - { - int count = 20; - Random random = new Random(); - MultiValuedMap mm = new ArrayListValuedHashMap<>(); - - for (int i = 1; i <= count; i++) - { - int parentId = random.nextInt(i); - String parentName = 0 == parentId ? _testRoot.getName() : String.valueOf(parentId); - String childName = String.valueOf(i); - mm.put(parentName, childName); - } - - logNode(mm, _testRoot.getName(), 0); - for (int i=0; i<2; i++) //do this twice to make sure the containers were *really* deleted - { - createContainers(mm, _testRoot.getName(), _testRoot); - assertEquals(count, _containers.size()); - cleanUpChildren(mm, _testRoot.getName(), _testRoot); - assertEquals(0, _containers.size()); - } - } - - @Test - public void testCache() - { - assertEquals(0, _containers.size()); - assertEquals(0, getChildren(_testRoot).size()); - - Container one = createContainer(_testRoot, "one", TestContext.get().getUser()); - assertEquals(1, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(0, getChildren(one).size()); - - Container oneA = createContainer(one, "A", TestContext.get().getUser()); - assertEquals(2, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(1, getChildren(one).size()); - assertEquals(0, getChildren(oneA).size()); - - Container oneB = createContainer(one, "B", TestContext.get().getUser()); - assertEquals(3, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(2, getChildren(one).size()); - assertEquals(0, getChildren(oneB).size()); - - Container deleteme = createContainer(one, "deleteme", TestContext.get().getUser()); - assertEquals(4, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(3, getChildren(one).size()); - assertEquals(0, getChildren(deleteme).size()); - - assertTrue(delete(deleteme, TestContext.get().getUser())); - assertEquals(3, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(2, getChildren(one).size()); - - Container oneC = createContainer(one, "C", TestContext.get().getUser()); - assertEquals(4, _containers.size()); - assertEquals(1, getChildren(_testRoot).size()); - assertEquals(3, getChildren(one).size()); - assertEquals(0, getChildren(oneC).size()); - - assertTrue(delete(oneC, TestContext.get().getUser())); - assertTrue(delete(oneB, TestContext.get().getUser())); - assertEquals(1, getChildren(one).size()); - - assertTrue(delete(oneA, TestContext.get().getUser())); - assertEquals(0, getChildren(one).size()); - - assertTrue(delete(one, TestContext.get().getUser())); - assertEquals(0, getChildren(_testRoot).size()); - assertEquals(0, _containers.size()); - } - - @Test - public void testFolderType() - { - // Test all folder types - List folderTypes = new ArrayList<>(FolderTypeManager.get().getAllFolderTypes()); - for (FolderType folderType : folderTypes) - { - if (!folderType.isProjectOnlyType()) // Dataspace can't be subfolder - testOneFolderType(folderType); - } - } - - private void testOneFolderType(FolderType folderType) - { - LOG.info("testOneFolderType(" + folderType.getName() + "): creating container"); - Container newFolder = createContainer(_testRoot, "folderTypeTest", TestContext.get().getUser()); - FolderType ft = newFolder.getFolderType(); - assertEquals(FolderType.NONE, ft); - - Container newFolderFromCache = getForId(newFolder.getId()); - assertNotNull(newFolderFromCache); - assertEquals(FolderType.NONE, newFolderFromCache.getFolderType()); - LOG.info("testOneFolderType(" + folderType.getName() + "): setting folder type"); - newFolder.setFolderType(folderType, TestContext.get().getUser()); - - newFolderFromCache = getForId(newFolder.getId()); - assertNotNull(newFolderFromCache); - assertEquals(newFolderFromCache.getFolderType().getName(), folderType.getName()); - assertEquals(newFolderFromCache.getFolderType().getDescription(), folderType.getDescription()); - - LOG.info("testOneFolderType(" + folderType.getName() + "): deleteAll"); - deleteAll(newFolder, TestContext.get().getUser()); // There might be subfolders because of container tabs - LOG.info("testOneFolderType(" + folderType.getName() + "): deleteAll complete"); - Container deletedContainer = getForId(newFolder.getId()); - - if (deletedContainer != null) - { - fail("Expected container with Id " + newFolder.getId() + " to be deleted, but found " + deletedContainer + ". Folder type was " + folderType); - } - } - - private static void createContainers(MultiValuedMap mm, String name, Container parent) - { - Collection nodes = mm.get(name); - - if (null == nodes) - return; - - for (String childName : nodes) - { - Container child = createContainer(parent, childName, TestContext.get().getUser()); - createContainers(mm, childName, child); - } - } - - private static void cleanUpChildren(MultiValuedMap mm, String name, Container parent) - { - Collection nodes = mm.get(name); - - if (null == nodes) - return; - - for (String childName : nodes) - { - Container child = getForPath(makePath(parent, childName)); - cleanUpChildren(mm, childName, child); - assertTrue(delete(child, TestContext.get().getUser())); - } - } - - private static void logNode(MultiValuedMap mm, String name, int offset) - { - Collection nodes = mm.get(name); - - if (null == nodes) - return; - - for (String childName : nodes) - { - LOG.debug(StringUtils.repeat(" ", offset) + childName); - logNode(mm, childName, offset + 1); - } - } - - // ContainerListener - @Override - public void propertyChange(PropertyChangeEvent evt) - { - } - - @Override - public void containerCreated(Container c, User user) - { - if (null == _testRoot || !c.getParsedPath().startsWith(_testRoot.getParsedPath())) - return; - _containers.put(c.getParsedPath(), c); - } - - - @Override - public void containerDeleted(Container c, User user) - { - _containers.remove(c.getParsedPath()); - } - - @Override - public void containerMoved(Container c, Container oldParent, User user) - { - } - - @NotNull - @Override - public Collection canMove(Container c, Container newParent, User user) - { - return Collections.emptyList(); - } - } - - static - { - ObjectFactory.Registry.register(Container.class, new ContainerFactory()); - } - - public static class ContainerFactory implements ObjectFactory - { - @Override - public Container fromMap(Map m) - { - throw new UnsupportedOperationException(); - } - - @Override - public Container fromMap(Container bean, Map m) - { - throw new UnsupportedOperationException(); - } - - @Override - public Map toMap(Container bean, Map m) - { - throw new UnsupportedOperationException(); - } - - @Override - public Container handle(ResultSet rs) throws SQLException - { - String id; - Container d; - String parentId = rs.getString("Parent"); - String name = rs.getString("Name"); - id = rs.getString("EntityId"); - int rowId = rs.getInt("RowId"); - int sortOrder = rs.getInt("SortOrder"); - Date created = rs.getTimestamp("Created"); - int createdBy = rs.getInt("CreatedBy"); - // _ts - String description = rs.getString("Description"); - String type = rs.getString("Type"); - String title = rs.getString("Title"); - boolean searchable = rs.getBoolean("Searchable"); - String lockStateString = rs.getString("LockState"); - LockState lockState = null != lockStateString ? Enums.getIfPresent(LockState.class, lockStateString).or(LockState.Unlocked) : LockState.Unlocked; - - LocalDate expirationDate = rs.getObject("ExpirationDate", LocalDate.class); - - // Could be running upgrade code before these recent columns have been added to the table. Use a find map - // to determine if they are present. Issue 51692. These checks could be removed after creation of these - // columns is incorporated into the bootstrap scripts. - Map findMap = ResultSetUtil.getFindMap(rs.getMetaData()); - Long fileRootSize = findMap.containsKey("FileRootSize") ? (Long)rs.getObject("FileRootSize") : null; // getObject() and cast because getLong() returns 0 for null - LocalDateTime fileRootLastCrawled = findMap.containsKey("FileRootLastCrawled") ? rs.getObject("FileRootLastCrawled", LocalDateTime.class) : null; - - Container dirParent = null; - if (null != parentId) - dirParent = getForId(parentId); - - d = new Container(dirParent, name, id, rowId, sortOrder, created, createdBy, searchable); - d.setDescription(description); - d.setType(type); - d.setTitle(title); - d.setLockState(lockState); - d.setExpirationDate(expirationDate); - d.setFileRootSize(fileRootSize); - d.setFileRootLastCrawled(fileRootLastCrawled); - return d; - } - - @Override - public ArrayList handleArrayList(ResultSet rs) throws SQLException - { - ArrayList list = new ArrayList<>(); - while (rs.next()) - { - list.add(handle(rs)); - } - return list; - } - } - - public static Container createFakeContainer(@Nullable String name, @Nullable Container parent) - { - return new Container(parent, name, GUID.makeGUID(), 1, 0, new Date(), 0, false); - } -} +/* + * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.data; + +import com.google.common.base.Enums; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.xmlbeans.XmlObject; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.labkey.api.Constants; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.FolderExportContext; +import org.labkey.api.admin.FolderImportContext; +import org.labkey.api.admin.FolderImporterImpl; +import org.labkey.api.admin.FolderWriterImpl; +import org.labkey.api.admin.StaticLoggerGetter; +import org.labkey.api.attachments.AttachmentParent; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.provider.ContainerAuditProvider; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.ConcurrentHashSet; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.data.Container.ContainerException; +import org.labkey.api.data.Container.LockState; +import org.labkey.api.data.PropertyManager.WritablePropertyMap; +import org.labkey.api.data.SimpleFilter.InClause; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.data.validator.ColumnValidators; +import org.labkey.api.event.PropertyChange; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.module.FolderType; +import org.labkey.api.module.FolderTypeManager; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.portal.ProjectUrls; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.Group; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.SecurityLogger; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.CreateProjectPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.roles.AuthorRole; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.security.roles.Role; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.test.TestTimeout; +import org.labkey.api.test.TestWhen; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.MinorConfigurationException; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; +import org.labkey.api.util.QuietCloser; +import org.labkey.api.util.ReentrantLockWithName; +import org.labkey.api.util.ResultSetUtil; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.FolderTab; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NavTreeManager; +import org.labkey.api.view.Portal; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.ViewContext; +import org.labkey.api.writer.MemoryVirtualFile; +import org.labkey.folder.xml.FolderDocument; +import org.labkey.remoteapi.collections.CaseInsensitiveHashMap; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; + +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.labkey.api.action.SpringActionController.ERROR_GENERIC; + +/** + * This class manages a hierarchy of collections, backed by a database table called Containers. + * Containers are named using filesystem-like paths e.g. /proteomics/comet/. Each path + * maps to a UID and set of permissions. The current security scheme allows ACLs + * to be specified explicitly on the directory or completely inherited. ACLs are not combined. + *

+ * NOTE: we act like java.io.File(). Paths start with forward-slash, but do not end with forward-slash. + * The root container's name is '/'. This means that it is not always the case that + * me.getPath() == me.getParent().getPath() + "/" + me.getName() + *

+ * The synchronization goals are to keep invalid containers from creeping into the cache. For example, once + * a container is deleted, it should never get put back in the cache. We accomplish this by synchronizing on + * the removal from the cache, and the database lookup/cache insertion. While a container is in the middle + * of being deleted, it's OK for other clients to see it because FKs enforce that it's always internally + * consistent, even if some of the data has already been deleted. + */ +public class ContainerManager +{ + private static final Logger LOG = LogHelper.getLogger(ContainerManager.class, "Container (projects, folders, and workbooks) retrieval and management"); + private static final CoreSchema CORE = CoreSchema.getInstance(); + + private static final String PROJECT_LIST_ID = "Projects"; + + public static final String HOME_PROJECT_PATH = "/home"; + public static final String DEFAULT_SUPPORT_PROJECT_PATH = HOME_PROJECT_PATH + "/support"; + + private static final Cache CACHE_PATH = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Containers by Path"); + private static final Cache CACHE_ENTITY_ID = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Containers by EntityId"); + private static final Cache> CACHE_CHILDREN = CacheManager.getCache(Constants.getMaxContainers(), CacheManager.DAY, "Child EntityIds of Containers"); + private static final ReentrantLock DATABASE_QUERY_LOCK = new ReentrantLockWithName(ContainerManager.class, "DATABASE_QUERY_LOCK"); + public static final String FOLDER_TYPE_PROPERTY_SET_NAME = "folderType"; + public static final String FOLDER_TYPE_PROPERTY_NAME = "name"; + public static final String FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN = "ctFolderTypeOverridden"; + public static final String TABFOLDER_CHILDREN_DELETED = "tabChildrenDeleted"; + public static final String AUDIT_SETTINGS_PROPERTY_SET_NAME = "containerAuditSettings"; + public static final String REQUIRE_USER_COMMENTS_PROPERTY_NAME = "requireUserComments"; + + private static final List _resourceProviders = new CopyOnWriteArrayList<>(); + + // containers that are being constructed, used to suppress events before fireCreateContainer() + private static final Set _constructing = new ConcurrentHashSet<>(); + + + /** enum of properties you can see in property change events */ + public enum Property + { + Name, + Parent, + Policy, + /** The default or active set of modules in the container has changed */ + Modules, + FolderType, + WebRoot, + AttachmentDirectory, + PipelineRoot, + Title, + Description, + SiteRoot, + StudyChange, + EndpointDirectory, + CloudStores + } + + static Path makePath(Container parent, String name) + { + if (null == parent) + return new Path(name); + return parent.getParsedPath().append(name, true); + } + + public static Container createMockContainer() + { + return new Container(null, "MockContainer", "01234567-8901-2345-6789-012345678901", 99999999, 0, new Date(), User.guest.getUserId(), true); + } + + private static Container createRoot() + { + Map m = new HashMap<>(); + m.put("Parent", null); + m.put("Name", ""); + Table.insert(null, CORE.getTableInfoContainers(), m); + + return getRoot(); + } + + private static DbScope.Transaction ensureTransaction() + { + return CORE.getSchema().getScope().ensureTransaction(DATABASE_QUERY_LOCK); + } + + private static int getNewChildSortOrder(Container parent) + { + int nextSortOrderVal = 0; + + List children = parent.getChildren(); + if (children != null) + { + for (Container child : children) + { + // find the max sort order value for the set of children + nextSortOrderVal = Math.max(nextSortOrderVal, child.getSortOrder()); + } + } + + // custom sorting applies: put new container at the end. + if (nextSortOrderVal > 0) + return nextSortOrderVal + 1; + + // we're sorted alphabetically + return 0; + } + + // TODO: Make private and force callers to use ensureContainer instead? + // TODO: Handle root creation here? + @NotNull + public static Container createContainer(Container parent, String name, @NotNull User user) + { + return createContainer(parent, name, null, null, NormalContainerType.NAME, user, null, null); + } + + public static final String WORKBOOK_DBSEQUENCE_NAME = "org.labkey.api.data.Workbooks"; + + // TODO: Pass in FolderType (separate from the container type of workbook, etc) and transact it with container creation? + @NotNull + public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user) + { + return createContainer(parent, name, title, description, type, user, null, null); + } + + @NotNull + public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user, @Nullable String auditMsg) + { + return createContainer(parent, name, title, description, type, user, auditMsg, null); + } + + @NotNull + public static Container createContainer(Container parent, String name, @Nullable String title, @Nullable String description, String type, @NotNull User user, @Nullable String auditMsg, + Consumer configureContainer) + { + ContainerType cType = ContainerTypeRegistry.get().getType(type); + if (cType == null) + throw new IllegalArgumentException("Unknown container type: " + type); + + // TODO: move this to ContainerType? + long sortOrder; + if (cType instanceof WorkbookContainerType) + { + sortOrder = DbSequenceManager.get(parent, WORKBOOK_DBSEQUENCE_NAME).next(); + + // Default workbook names are simply "" + if (name == null) + name = String.valueOf(sortOrder); + } + else + { + sortOrder = getNewChildSortOrder(parent); + } + + if (!parent.canHaveChildren()) + throw new IllegalArgumentException("Parent of a container must not be a " + parent.getContainerType().getName()); + + StringBuilder error = new StringBuilder(); + if (!Container.isLegalName(name, parent.isRoot(), error)) + throw new ApiUsageException(error.toString()); + + if (!Container.isLegalTitle(title, error)) + throw new ApiUsageException(error.toString()); + + Path path = makePath(parent, name); + SQLException sqlx = null; + Map insertMap = null; + + GUID entityId = new GUID(); + Container c; + + try + { + _constructing.add(entityId); + + try + { + Map m = new CaseInsensitiveHashMap<>(); + m.put("Parent", parent.getId()); + m.put("Name", name); + m.put("Title", title); + m.put("SortOrder", sortOrder); + m.put("EntityId", entityId); + if (null != description) + m.put("Description", description); + m.put("Type", type); + insertMap = Table.insert(user, CORE.getTableInfoContainers(), m); + } + catch (RuntimeSQLException x) + { + if (!x.isConstraintException()) + throw x; + sqlx = x.getSQLException(); + } + + _clearChildrenFromCache(parent); + + c = insertMap == null ? null : getForId(entityId); + + if (null == c) + { + if (null != sqlx) + throw new RuntimeSQLException(sqlx); + else + throw new RuntimeException("Container for path '" + path + "' was not created properly."); + } + + User savePolicyUser = user; + if (c.isProject() && !c.hasPermission(user, AdminPermission.class) && ContainerManager.getRoot().hasPermission(user, CreateProjectPermission.class)) + { + // Special case for project creators who don't necessarily yet have permission to save the policy of + // the project they just created + savePolicyUser = User.getAdminServiceUser(); + } + + // Workbooks inherit perms from their parent so don't create a policy if this is a workbook + if (c.isContainerFor(ContainerType.DataType.permissions)) + { + SecurityManager.setAdminOnlyPermissions(c, savePolicyUser); + } + + _removeFromCache(c, true); // seems odd, but it removes c.getProject() which clears other things from the cache + + // Initialize the list of active modules in the Container + c.getActiveModules(true, true, user); + + if (c.isProject()) + { + SecurityManager.createNewProjectGroups(c, savePolicyUser); + } + else + { + // If current user does NOT have admin permission on this container or the project has been + // explicitly set to have new subfolders inherit permissions, then inherit permissions + // (otherwise they would not be able to see the folder) + boolean hasAdminPermission = c.hasPermission(user, AdminPermission.class); + if ((!hasAdminPermission && !user.hasRootAdminPermission()) || SecurityManager.shouldNewSubfoldersInheritPermissions(c.getProject())) + SecurityManager.setInheritPermissions(c); + } + + // NOTE parent caches some info about children (e.g. hasWorkbookChildren) + // since mutating cached objects is frowned upon, just uncache parent + // CONSIDER: we could perhaps only uncache if the child is a workbook, but I think this reasonable + _removeFromCache(parent, true); + + if (null != configureContainer) + configureContainer.accept(c); + } + finally + { + _constructing.remove(entityId); + } + + fireCreateContainer(c, user, auditMsg); + + return c; + } + + public static void addSecurableResourceProvider(ContainerSecurableResourceProvider provider) + { + _resourceProviders.add(provider); + } + + public static List getSecurableResourceProviders() + { + return Collections.unmodifiableList(_resourceProviders); + } + + public static Container createContainerFromTemplate(Container parent, String name, String title, Container templateContainer, User user, FolderExportContext exportCtx, Consumer afterCreateHandler) throws Exception + { + MemoryVirtualFile vf = new MemoryVirtualFile(); + + // export objects from the source template folder + FolderWriterImpl writer = new FolderWriterImpl(); + writer.write(templateContainer, exportCtx, vf); + + // create the new target container + Container c = createContainer(parent, name, title, null, NormalContainerType.NAME, user, null, afterCreateHandler); + + // import objects into the target folder + XmlObject folderXml = vf.getXmlBean("folder.xml"); + if (folderXml instanceof FolderDocument folderDoc) + { + FolderImportContext importCtx = new FolderImportContext(user, c, folderDoc, null, new StaticLoggerGetter(LogManager.getLogger(FolderImporterImpl.class)), vf); + + FolderImporterImpl importer = new FolderImporterImpl(); + importer.process(null, importCtx, vf); + } + + return c; + } + + public static void setRequireAuditComments(Container container, User user, @NotNull Boolean required) + { + WritablePropertyMap props = PropertyManager.getWritableProperties(container, AUDIT_SETTINGS_PROPERTY_SET_NAME, true); + String originalValue = props.get(REQUIRE_USER_COMMENTS_PROPERTY_NAME); + props.put(REQUIRE_USER_COMMENTS_PROPERTY_NAME, required.toString()); + props.save(); + + addAuditEvent(user, container, + "Changed " + REQUIRE_USER_COMMENTS_PROPERTY_NAME + " from \"" + + originalValue + "\" to \"" + required + "\""); + } + + public static void setFolderType(Container c, FolderType folderType, User user, BindException errors) + { + FolderType oldType = c.getFolderType(); + + if (folderType.equals(oldType)) + return; + + List errorStrings = new ArrayList<>(); + + if (!c.isProject() && folderType.isProjectOnlyType()) + errorStrings.add("Cannot set a subfolder to " + folderType.getName() + " because it is a project-only folder type."); + + // Check for any containers that need to be moved into container tabs + if (errorStrings.isEmpty() && folderType.hasContainerTabs()) + { + List childTabFoldersNonMatchingTypes = new ArrayList<>(); + List containersBecomingTabs = findAndCheckContainersMatchingTabs(c, folderType, childTabFoldersNonMatchingTypes, errorStrings); + + if (errorStrings.isEmpty()) + { + if (!containersBecomingTabs.isEmpty()) + { + // Make containers tab container; Folder tab will find them by name + try (DbScope.Transaction transaction = ensureTransaction()) + { + for (Container container : containersBecomingTabs) + updateType(container, TabContainerType.NAME, user); + + transaction.commit(); + } + } + + // Check these and change type unless they were overridden explicitly + for (Container container : childTabFoldersNonMatchingTypes) + { + if (!isContainerTabTypeOverridden(container)) + { + FolderTab newTab = folderType.findTab(container.getName()); + assert null != newTab; // There must be a tab because it caused the container to get into childTabFoldersNonMatchingTypes + FolderType newType = newTab.getFolderType(); + if (null == newType) + newType = FolderType.NONE; // default to NONE + setFolderType(container, newType, user, errors); + } + } + } + } + + if (errorStrings.isEmpty()) + { + oldType.unconfigureContainer(c, user); + WritablePropertyMap props = PropertyManager.getWritableProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME, true); + props.put(FOLDER_TYPE_PROPERTY_NAME, folderType.getName()); + + if (c.isContainerTab()) + { + boolean containerTabTypeOverridden = false; + FolderTab tab = c.getParent().getFolderType().findTab(c.getName()); + if (null != tab && !folderType.equals(tab.getFolderType())) + containerTabTypeOverridden = true; + props.put(FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN, Boolean.toString(containerTabTypeOverridden)); + } + props.save(); + + notifyContainerChange(c.getId(), Property.FolderType, user); + folderType.configureContainer(c, user); // Configure new only after folder type has been changed + + // TODO: Not needed? I don't think we've changed the container's state. + _removeFromCache(c, false); + } + else + { + for (String errorString : errorStrings) + errors.reject(SpringActionController.ERROR_MSG, errorString); + } + } + + public static void checkContainerValidity(Container c) throws ContainerException + { + // Check container for validity; in rare cases user may have changed their custom folderType.xml and caused + // duplicate subfolders (same name) to exist + // Get list of child containers that are not container tabs, but match container tabs; these are bad + FolderType folderType = getFolderType(c); + List errorStrings = new ArrayList<>(); + List childTabFoldersNonMatchingTypes = new ArrayList<>(); + List containersMatchingTabs = findAndCheckContainersMatchingTabs(c, folderType, childTabFoldersNonMatchingTypes, errorStrings); + if (!containersMatchingTabs.isEmpty()) + { + throw new Container.ContainerException("Folder " + c.getPath() + + " has a subfolder with the same name as a container tab folder, which is an invalid state." + + " This may have been caused by changing the folder type's tabs after this folder was set to its folder type." + + " An administrator should either delete the offending subfolder or change the folder's folder type.\n"); + } + } + + public static List findAndCheckContainersMatchingTabs(Container c, FolderType folderType, + List childTabFoldersNonMatchingTypes, List errorStrings) + { + List containersMatchingTabs = new ArrayList<>(); + for (FolderTab folderTab : folderType.getDefaultTabs()) + { + if (folderTab.getTabType() == FolderTab.TAB_TYPE.Container) + { + for (Container child : c.getChildren()) + { + if (child.getName().equalsIgnoreCase(folderTab.getName())) + { + if (!child.getFolderType().getName().equalsIgnoreCase(folderTab.getFolderTypeName())) + { + if (child.isContainerTab()) + childTabFoldersNonMatchingTypes.add(child); // Tab type doesn't match child tab folder + else + errorStrings.add("Child folder " + child.getName() + + " matches container tab, but folder type " + child.getFolderType().getName() + " doesn't match tab's folder type " + + folderTab.getFolderTypeName() + "."); + } + + int childCount = child.getChildren().size(); + if (childCount > 0) + { + errorStrings.add("Child folder " + child.getName() + + " matches container tab, but cannot be converted to a tab folder because it has " + childCount + " children."); + } + + if (!child.isConvertibleToTab()) + { + errorStrings.add("Child folder " + child.getName() + + " matches container tab, but cannot be converted to a tab folder because it is a " + child.getContainerNoun() + "."); + } + + if (!child.isContainerTab()) + containersMatchingTabs.add(child); + + break; // we found name match; can't be another + } + } + } + } + return containersMatchingTabs; + } + + private static final Set containersWithBadFolderTypes = new ConcurrentHashSet<>(); + + @NotNull + public static FolderType getFolderType(Container c) + { + String name = getFolderTypeName(c); + FolderType folderType; + + if (null != name) + { + folderType = FolderTypeManager.get().getFolderType(name); + + if (null == folderType) + { + // If we're upgrading then folder types won't be defined yet... don't warn in that case. + if (!ModuleLoader.getInstance().isUpgradeInProgress() && + !ModuleLoader.getInstance().isUpgradeRequired() && + !containersWithBadFolderTypes.contains(c)) + { + LOG.warn("No such folder type " + name + " for folder " + c.toString()); + containersWithBadFolderTypes.add(c); + } + + folderType = FolderType.NONE; + } + } + else + folderType = FolderType.NONE; + + return folderType; + } + + /** + * Most code should call getFolderType() instead. + * Useful for finding the name of the folder type BEFORE startup is complete, so the FolderType itself + * may not be available. + */ + @Nullable + public static String getFolderTypeName(Container c) + { + Map props = PropertyManager.getProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME); + return props.get(FOLDER_TYPE_PROPERTY_NAME); + } + + + @NotNull + public static Map getFolderTypeNameContainerCounts(Container root) + { + Map nameCounts = new TreeMap<>(); + for (Container c : getAllChildren(root)) + { + Integer count = nameCounts.get(c.getFolderType().getName()); + if (null == count) + { + count = Integer.valueOf(0); + } + nameCounts.put(c.getFolderType().getName(), ++count); + } + return nameCounts; + } + + @NotNull + public static Map getProductFoldersMetrics(@NotNull FolderType folderType) + { + Container root = getRoot(); + Map metrics = new TreeMap<>(); + List counts = new ArrayList<>(); + for (Container c : root.getChildren()) + { + if (!c.getFolderType().getName().equals(folderType.getName())) + continue; + + int childCount = c.getChildren().stream().filter(Container::isInFolderNav).toList().size(); + counts.add(childCount); + } + + int totalFolderTypeMatch = counts.size(); + if (totalFolderTypeMatch == 0) + return metrics; + + Collections.sort(counts); + int median = counts.get((totalFolderTypeMatch - 1)/2); + if (totalFolderTypeMatch % 2 == 0 ) + { + int low = counts.get(totalFolderTypeMatch/2 - 1); + int high = counts.get(totalFolderTypeMatch/2); + median = Math.round((low + high) / 2.0f); + } + int maxProjectsCount = counts.get(totalFolderTypeMatch - 1); + int totalProjectsCount = counts.stream().mapToInt(Integer::intValue).sum(); + int averageProjectsCount = Math.round((float) totalProjectsCount /totalFolderTypeMatch); + + metrics.put("totalSubProjectsCount", totalProjectsCount); + metrics.put("averageSubProjectsPerHomeProject", averageProjectsCount); + metrics.put("medianSubProjectsCountPerHomeProject", median); + metrics.put("maxSubProjectsCountInHomeProject", maxProjectsCount); + + return metrics; + } + + public static boolean isContainerTabTypeThisOrChildrenOverridden(Container c) + { + if (isContainerTabTypeOverridden(c)) + return true; + if (c.getFolderType().hasContainerTabs()) + { + for (Container child : c.getChildren()) + { + if (child.isContainerTab() && isContainerTabTypeOverridden(child)) + return true; + } + } + return false; + } + + public static boolean isContainerTabTypeOverridden(Container c) + { + Map props = PropertyManager.getProperties(c, FOLDER_TYPE_PROPERTY_SET_NAME); + String overridden = props.get(FOLDER_TYPE_PROPERTY_TABTYPE_OVERRIDDEN); + return (null != overridden) && overridden.equalsIgnoreCase("true"); + } + + private static void setContainerTabDeleted(Container c, String tabName, String folderTypeName) + { + // Add prop in this category + WritablePropertyMap props = PropertyManager.getWritableProperties(c, TABFOLDER_CHILDREN_DELETED, true); + props.put(getDeletedTabKey(tabName, folderTypeName), "true"); + props.save(); + } + + public static void clearContainerTabDeleted(Container c, String tabName, String folderTypeName) + { + WritablePropertyMap props = PropertyManager.getWritableProperties(c, TABFOLDER_CHILDREN_DELETED, true); + String key = getDeletedTabKey(tabName, folderTypeName); + if (props.containsKey(key)) + { + props.remove(key); + props.save(); + } + } + + public static boolean hasContainerTabBeenDeleted(Container c, String tabName, String folderTypeName) + { + // We keep arbitrary number of deleted children tabs using suffix 0, 1, 2.... + Map props = PropertyManager.getProperties(c, TABFOLDER_CHILDREN_DELETED); + return props.containsKey(getDeletedTabKey(tabName, folderTypeName)); + } + + private static String getDeletedTabKey(String tabName, String folderTypeName) + { + return tabName + "-TABDELETED-FOLDER-" + folderTypeName; + } + + @NotNull + public static Container ensureContainer(@NotNull String path, @NotNull User user) + { + return ensureContainer(Path.parse(path), user); + } + + @NotNull + public static Container ensureContainer(@NotNull Path path, @NotNull User user) + { + Container c = null; + + try + { + c = getForPath(path); + } + catch (RootContainerException e) + { + // Ignore this -- root doesn't exist yet + } + + if (null == c) + { + if (path.isEmpty()) + c = createRoot(); + else + { + Path parentPath = path.getParent(); + c = ensureContainer(parentPath, user); + c = createContainer(c, path.getName(), null, null, NormalContainerType.NAME, user); + } + } + return c; + } + + + @NotNull + public static Container ensureContainer(Container parent, String name, User user) + { + // NOTE: Running outside a tx doesn't seem to be necessary. +// if (CORE.getSchema().getScope().isTransactionActive()) +// throw new IllegalStateException("Transaction should not be active"); + + Container c = null; + + try + { + c = getForPath(makePath(parent,name)); + } + catch (RootContainerException e) + { + // Ignore this -- root doesn't exist yet + } + + if (null == c) + { + c = createContainer(parent, name, user); + } + return c; + } + + public static void updateDescription(Container container, String description, User user) + throws ValidationException + { + ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, description); + + //For some reason, there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET Description=? WHERE RowID=?").add(description).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + String oldValue = container.getDescription(); + _removeFromCache(container, false); + container = getForRowId(container.getRowId()); + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(container, user, Property.Description, oldValue, description); + firePropertyChangeEvent(evt); + } + + public static void updateSearchable(Container container, boolean searchable, User user) + { + //For some reason, there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET Searchable=? WHERE RowID=?").add(searchable).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + } + + public static void updateLockState(Container container, LockState lockState, @NotNull Runnable auditRunnable) + { + //For some reason there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET LockState = ?, ExpirationDate = NULL WHERE RowID = ?").add(lockState).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + + auditRunnable.run(); + } + + public static List getExcludedProjects() + { + return getProjects().stream() + .filter(p->p.getLockState() == Container.LockState.Excluded) + .collect(Collectors.toList()); + } + + public static List getNonExcludedProjects() + { + return getProjects().stream() + .filter(p->p.getLockState() != Container.LockState.Excluded) + .collect(Collectors.toList()); + } + + public static void setExcludedProjects(Collection ids, @NotNull Runnable auditRunnable) + { + // First clear all existing "Excluded" states + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET LockState = NULL, ExpirationDate = NULL WHERE LockState = ?").add(LockState.Excluded); + new SqlExecutor(CORE.getSchema()).execute(sql); + + // Now set the passed-in projects to "Excluded" + if (!ids.isEmpty()) + { + ColumnInfo entityIdCol = CORE.getTableInfoContainers().getColumn("EntityId"); + Filter inClauseFilter = new SimpleFilter(new InClause(entityIdCol.getFieldKey(), ids)); + SQLFragment frag = new SQLFragment("UPDATE "); + frag.append(CORE.getTableInfoContainers().getSelectName()); + frag.append(" SET LockState = ?, ExpirationDate = NULL "); + frag.add(LockState.Excluded); + frag.append(inClauseFilter.getSQLFragment(CORE.getSqlDialect(), "c", Map.of(entityIdCol.getFieldKey(), entityIdCol))); + new SqlExecutor(CORE.getSchema()).execute(frag); + } + + clearCache(); + + auditRunnable.run(); + } + + public static void archiveContainer(User user, Container container, boolean archive) + { + if (container.isRoot() || container.isProject() || container.isAppHomeFolder()) + throw new ApiUsageException("Archive action not supported for this folder."); + + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers().getSelectName()); + if (archive) + { + sql.append(" SET LockState = ? "); + sql.add(LockState.Archived); + sql.append(" WHERE LockState IS NULL "); + } + else + { + sql.append(" SET LockState = NULL WHERE LockState = ? "); + sql.add(LockState.Archived); + } + sql.append("AND EntityId = ? "); + sql.add(container.getEntityId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + clearCache(); + + addAuditEvent(user, container, archive ? "Container has been archived." : "Archived container has been restored."); + } + + public static void updateExpirationDate(Container container, LocalDate expirationDate, @NotNull Runnable auditRunnable) + { + //For some reason there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + // Note: jTDS doesn't support LocalDate, so convert to java.sql.Date + sql.append(" SET ExpirationDate = ? WHERE RowID = ?").add(java.sql.Date.valueOf(expirationDate)).add(container.getRowId()); + + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + + auditRunnable.run(); + } + + public static void updateType(Container container, String newType, User user) + { + //For some reason there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET Type=? WHERE RowID=?").add(newType).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + } + + public static void updateTitle(Container container, String title, User user) + throws ValidationException + { + ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, title); + + //For some reason there is no primary key defined on core.containers + //so we can't use Table.update here + SQLFragment sql = new SQLFragment("UPDATE "); + sql.append(CORE.getTableInfoContainers()); + sql.append(" SET Title=? WHERE RowID=?").add(title).add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(sql); + + _removeFromCache(container, false); + String oldValue = container.getTitle(); + container = getForRowId(container.getRowId()); + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(container, user, Property.Title, oldValue, title); + firePropertyChangeEvent(evt); + } + + public static void uncache(Container c) + { + _removeFromCache(c, true); + } + + public static final String SHARED_CONTAINER_PATH = "/Shared"; + + @NotNull + public static Container getSharedContainer() + { + return ensureContainer(Path.parse(SHARED_CONTAINER_PATH), User.getAdminServiceUser()); + } + + public static List getChildren(Container parent) + { + return new ArrayList<>(getChildrenMap(parent).values()); + } + + // Default is to include all types of children, as seems only appropriate + public static List getChildren(Container parent, User u, Class perm) + { + return getChildren(parent, u, perm, null, ContainerTypeRegistry.get().getTypeNames()); + } + + public static List getChildren(Container parent, User u, Class perm, Set roles) + { + return getChildren(parent, u, perm, roles, ContainerTypeRegistry.get().getTypeNames()); + } + + public static List getChildren(Container parent, User u, Class perm, String typeIncluded) + { + return getChildren(parent, u, perm, null, Collections.singleton(typeIncluded)); + } + + public static List getChildren(Container parent, User u, Class perm, Set roles, Set includedTypes) + { + List children = new ArrayList<>(); + for (Container child : getChildrenMap(parent).values()) + if (includedTypes.contains(child.getContainerType().getName()) && child.hasPermission(u, perm, roles)) + children.add(child); + + return children; + } + + public static List getAllChildren(Container parent, User u) + { + return getAllChildren(parent, u, ReadPermission.class, null, ContainerTypeRegistry.get().getTypeNames()); + } + + public static List getAllChildren(Container parent, User u, Class perm) + { + return getAllChildren(parent, u, perm, null, ContainerTypeRegistry.get().getTypeNames()); + } + + // Default is to include all types of children + public static List getAllChildren(Container parent, User u, Class perm, Set roles) + { + return getAllChildren(parent, u, perm, roles, ContainerTypeRegistry.get().getTypeNames()); + } + + public static List getAllChildren(Container parent, User u, Class perm, String typeIncluded) + { + return getAllChildren(parent, u, perm, null, Collections.singleton(typeIncluded)); + } + + public static List getAllChildren(Container parent, User u, Class perm, Set roles, Set typesIncluded) + { + Set allChildren = getAllChildren(parent); + List result = new ArrayList<>(allChildren.size()); + + for (Container container : allChildren) + { + if (typesIncluded.contains(container.getContainerType().getName()) && container.hasPermission(u, perm, roles)) + { + result.add(container); + } + } + + return result; + } + + // Returns the next available child container name based on the baseName + public static String getAvailableChildContainerName(Container c, String baseName) + { + List children = getChildren(c); + Map folders = new HashMap<>(children.size() * 2); + for (Container child : children) + folders.put(child.getName(), child); + + String availableContainerName = baseName; + int i = 1; + while (folders.containsKey(availableContainerName)) + { + availableContainerName = baseName + " " + i++; + } + + return availableContainerName; + } + + // Returns true only if user has the specified permission in the entire container tree starting at root + public static boolean hasTreePermission(Container root, User u, Class perm) + { + for (Container c : getAllChildren(root)) + if (!c.hasPermission(u, perm)) + return false; + + return true; + } + + private static Map getChildrenMap(Container parent) + { + if (!parent.canHaveChildren()) + { + // Optimization to avoid database query (important because some installs have tens of thousands of + // workbooks) when the container is a workbook, which is not allowed to have children + return Collections.emptyMap(); + } + + List childIds = CACHE_CHILDREN.get(parent.getEntityId()); + if (null == childIds) + { + try (DbScope.Transaction t = ensureTransaction()) + { + List children = new SqlSelector(CORE.getSchema(), + "SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE Parent = ? ORDER BY SortOrder, LOWER(Name)", + parent.getId()).getArrayList(Container.class); + + childIds = new ArrayList<>(children.size()); + for (Container c : children) + { + childIds.add(c.getEntityId()); + _addToCache(c); + } + childIds = Collections.unmodifiableList(childIds); + CACHE_CHILDREN.put(parent.getEntityId(), childIds); + // No database changes to commit, but need to decrement the transaction counter + t.commit(); + } + } + + if (childIds.isEmpty()) + return Collections.emptyMap(); + + // Use a LinkedHashMap to preserve the order defined by the user - they're not necessarily alphabetical + Map ret = new LinkedHashMap<>(); + for (GUID id : childIds) + { + Container c = getForId(id); + if (null != c) + ret.put(c.getName(), c); + } + return Collections.unmodifiableMap(ret); + } + + public static Container getForRowId(int id) + { + Selector selector = new SqlSelector(CORE.getSchema(), new SQLFragment("SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE RowId = ?", id)); + return selector.getObject(Container.class); + } + + public static @Nullable Container getForId(@NotNull GUID guid) + { + return guid != null ? getForId(guid.toString()) : null; + } + + public static @Nullable Container getForId(@Nullable String id) + { + //if the input string is not a GUID, just return null, + //so that we don't get a SQLException when the database + //tries to convert it to a unique identifier. + if (!GUID.isGUID(id)) + return null; + + GUID guid = new GUID(id); + + Container d = CACHE_ENTITY_ID.get(guid); + if (null != d) + return d; + + try (DbScope.Transaction t = ensureTransaction()) + { + Container result = new SqlSelector( + CORE.getSchema(), + "SELECT * FROM " + CORE.getTableInfoContainers() + " WHERE EntityId = ?", + id).getObject(Container.class); + if (result != null) + { + result = _addToCache(result); + } + // No database changes to commit, but need to decrement the counter + t.commit(); + + return result; + } + } + + public static Container getChild(Container c, String name) + { + Path path = c.getParsedPath().append(name); + + Container d = _getFromCachePath(path); + if (null != d) + return d; + + Map map = getChildrenMap(c); + return map.get(name); + } + + + public static Container getForURL(@NotNull ActionURL url) + { + Container ret = getForPath(url.getExtraPath()); + if (ret == null) + ret = getForId(StringUtils.strip(url.getExtraPath(), "/")); + return ret; + } + + + public static Container getForPath(@NotNull String path) + { + if (GUID.isGUID(path)) + { + Container c = getForId(path); + if (c != null) + return c; + } + + Path p = Path.parse(path); + return getForPath(p); + } + + public static Container getForPath(Path path) + { + Container d = _getFromCachePath(path); + if (null != d) + return d; + + // Special case for ROOT -- we want to throw instead of returning null + if (path.equals(Path.rootPath)) + { + try (DbScope.Transaction t = ensureTransaction()) + { + TableInfo tinfo = CORE.getTableInfoContainers(); + + // Unusual, but possible -- if cache loader hits an exception it can end up caching null + if (null == tinfo) + throw new RootContainerException("Container table could not be retrieved from the cache"); + + // This might be called at bootstrap, before schemas have been created + if (tinfo.getTableType() == DatabaseTableType.NOT_IN_DB) + throw new RootContainerException("Container table has not been created"); + + Container result = new SqlSelector(CORE.getSchema(),"SELECT * FROM " + tinfo + " WHERE Parent IS NULL").getObject(Container.class); + + if (result == null) + throw new RootContainerException("Root container does not exist"); + + _addToCache(result); + // No database changes to commit, but need to decrement the counter + t.commit(); + return result; + } + } + else + { + Path parent = path.getParent(); + String name = path.getName(); + Container dirParent = getForPath(parent); + + if (null == dirParent) + return null; + + Map map = getChildrenMap(dirParent); + return map.get(name); + } + } + + public static class RootContainerException extends RuntimeException + { + private RootContainerException(String message, Throwable cause) + { + super(message, cause); + } + + private RootContainerException(String message) + { + super(message); + } + } + + public static Container getRoot() + { + try + { + return getForPath("/"); + } + catch (MinorConfigurationException e) + { + // If the server is misconfigured, rethrow so some callers don't swallow it and other callers don't end up + // reporting it to mothership, Issue 50843. + throw e; + } + catch (Exception e) + { + // Some callers catch and ignore this exception, e.g., early in the bootstrap process + throw new RootContainerException("Root container can't be retrieved", e); + } + } + + public static void saveAliasesForContainer(Container container, List aliases, User user) + { + Set originalAliases = new CaseInsensitiveHashSet(getAliasesForContainer(container)); + Set newAliases = new CaseInsensitiveHashSet(aliases); + + if (originalAliases.equals(newAliases)) + { + return; + } + + try (DbScope.Transaction transaction = ensureTransaction()) + { + // Delete all of the aliases for the current container, plus any of the aliases that might be associated + // with another container right now + SQLFragment deleteSQL = new SQLFragment(); + deleteSQL.append("DELETE FROM "); + deleteSQL.append(CORE.getTableInfoContainerAliases()); + deleteSQL.append(" WHERE ContainerRowId = ? "); + deleteSQL.add(container.getRowId()); + if (!aliases.isEmpty()) + { + deleteSQL.append(" OR Path IN ("); + String separator = ""; + for (String alias : aliases) + { + deleteSQL.append(separator); + separator = ", "; + deleteSQL.append("LOWER(?)"); + deleteSQL.add(alias); + } + deleteSQL.append(")"); + } + new SqlExecutor(CORE.getSchema()).execute(deleteSQL); + + // Store the alias as LOWER() so that we can query against it using the index + for (String alias : newAliases) + { + SQLFragment insertSQL = new SQLFragment(); + insertSQL.append("INSERT INTO "); + insertSQL.append(CORE.getTableInfoContainerAliases()); + insertSQL.append(" (Path, ContainerRowId) VALUES (LOWER(?), ?)"); + insertSQL.add(alias); + insertSQL.add(container.getRowId()); + new SqlExecutor(CORE.getSchema()).execute(insertSQL); + } + + addAuditEvent(user, container, + "Changed folder aliases from \"" + + StringUtils.join(originalAliases, ", ") + "\" to \"" + + StringUtils.join(newAliases, ", ") + "\""); + + transaction.commit(); + } + } + + // Abstract base class used for attaching system resources (favorite icons, logos, stylesheets, sso auth logos) to folders and projects + public static abstract class ContainerParent implements AttachmentParent + { + private final Container _c; + + protected ContainerParent(Container c) + { + _c = c; + } + + @Override + public String getEntityId() + { + return _c.getId(); + } + + @Override + public String getContainerId() + { + return _c.getId(); + } + + public Container getContainer() + { + return _c; + } + } + + public static Container getHomeContainer() + { + return getForPath(HOME_PROJECT_PATH); + } + + public static List getProjects() + { + return getChildren(getRoot()); + } + + public static NavTree getProjectList(ViewContext context, boolean includeChildren) + { + User user = context.getUser(); + Container currentProject = context.getContainer().getProject(); + String projectNavTreeId = PROJECT_LIST_ID; + if (currentProject != null) + projectNavTreeId += currentProject.getId(); + + NavTree navTree = (NavTree) NavTreeManager.getFromCache(projectNavTreeId, context); + if (null != navTree) + return navTree; + + NavTree list = new NavTree("Projects"); + List projects = getProjects(); + + for (Container project : projects) + { + boolean shouldDisplay = project.shouldDisplay(user) && project.hasPermission("getProjectList()", user, ReadPermission.class); + boolean includeCurrentProject = includeChildren && currentProject != null && currentProject.equals(project); + + if (shouldDisplay || includeCurrentProject) + { + ActionURL startURL = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(project); + + if (includeChildren) + list.addChild(getFolderListForUser(project, context)); + else if (project.equals(getHomeContainer())) + list.addChild(new NavTree("Home", startURL)); + else + list.addChild(project.getTitle(), startURL); + } + } + + list.setId(projectNavTreeId); + NavTreeManager.cacheTree(list, context.getUser()); + + return list; + } + + public static NavTree getFolderListForUser(final Container project, ViewContext viewContext) + { + final boolean isNavAccessOpen = AppProps.getInstance().isNavigationAccessOpen(); + final Container c = viewContext.getContainer(); + final String cacheKey = isNavAccessOpen ? project.getId() : c.getId(); + + NavTree tree = (NavTree) NavTreeManager.getFromCache(cacheKey, viewContext); + if (null != tree) + return tree; + + try + { + assert SecurityLogger.indent("getFolderListForUser()"); + + User user = viewContext.getUser(); + String projectId = project.getId(); + + List folders = new ArrayList<>(getAllChildren(project)); + + Collections.sort(folders); + + Set containersInTree = new HashSet<>(); + + Map m = new HashMap<>(); + Map permission = new HashMap<>(); + + for (Container f : folders) + { + if (!f.isInFolderNav()) + continue; + + boolean hasPolicyRead = f.hasPermission(user, ReadPermission.class); + + boolean skip = ( + !hasPolicyRead || + !f.shouldDisplay(user) || + !f.hasPermission(user, ReadPermission.class) + ); + + //Always put the project and current container in... + if (skip && !f.equals(project) && !f.equals(c)) + continue; + + //HACK to make home link consistent... + String name = f.getTitle(); + if (name.equals("home") && f.equals(getHomeContainer())) + name = "Home"; + + NavTree t = new NavTree(name); + + // 34137: Support folder path expansion for containers where label != name + t.setId(f.getId()); + if (hasPolicyRead) + { + ActionURL url = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(f); + t.setHref(url.getEncodedLocalURIString()); + } + + boolean addFolder = false; + + if (isNavAccessOpen) + { + addFolder = true; + } + else + { + // 32718: If navigation access is not open then hide projects that aren't directly + // accessible in site folder navigation. + + if (f.equals(c) || f.isRoot() || (hasPolicyRead && f.isProject())) + { + // In current container, root, or readable project + addFolder = true; + } + else + { + boolean isAscendant = f.isDescendant(c); + boolean isDescendant = c.isDescendant(f); + boolean inActivePath = isAscendant || isDescendant; + boolean hasAncestryRead = false; + + if (inActivePath) + { + Container leaf = isAscendant ? f : c; + Container localRoot = isAscendant ? c : f; + + List ancestors = containersToRootList(leaf); + Collections.reverse(ancestors); + + for (Container p : ancestors) + { + if (!permission.containsKey(p.getId())) + permission.put(p.getId(), p.hasPermission(user, ReadPermission.class)); + boolean hasRead = permission.get(p.getId()); + + if (p.equals(localRoot)) + { + hasAncestryRead = hasRead; + break; + } + else if (!hasRead) + { + hasAncestryRead = false; + break; + } + } + } + else + { + hasAncestryRead = containersToRoot(f).stream().allMatch(p -> { + if (!permission.containsKey(p.getId())) + permission.put(p.getId(), p.hasPermission(user, ReadPermission.class)); + return permission.get(p.getId()); + }); + } + + if (hasPolicyRead && hasAncestryRead && inActivePath) + { + // Is in the direct readable lineage of the current container + addFolder = true; + } + else if (hasPolicyRead && f.getParent().equals(c.getParent())) + { + // Is a readable sibling of the current container + addFolder = true; + } + else if (hasAncestryRead) + { + // Is a part of a fully readable ancestry + addFolder = true; + } + } + + if (!addFolder) + LOG.debug("isNavAccessOpen restriction: \"" + f.getPath() + "\""); + } + + if (addFolder) + { + containersInTree.add(f); + m.put(f.getId(), t); + } + } + + //Ensure parents of any accessible folder are in the tree. If not add them with no link. + for (Container treeContainer : containersInTree) + { + if (!treeContainer.equals(project) && !containersInTree.contains(treeContainer.getParent())) + { + Set containersToRoot = containersToRoot(treeContainer); + //Possible will be added more than once, if several children are accessible, but that's OK... + for (Container missing : containersToRoot) + { + if (!m.containsKey(missing.getId())) + { + if (isNavAccessOpen) + { + NavTree noLinkTree = new NavTree(missing.getName()); + noLinkTree.setId(missing.getId()); + m.put(missing.getId(), noLinkTree); + } + else + { + if (!permission.containsKey(missing.getId())) + permission.put(missing.getId(), missing.hasPermission(user, ReadPermission.class)); + + if (!permission.get(missing.getId())) + { + NavTree noLinkTree = new NavTree(missing.getName()); + m.put(missing.getId(), noLinkTree); + } + else + { + NavTree linkTree = new NavTree(missing.getName()); + ActionURL url = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(missing); + linkTree.setHref(url.getEncodedLocalURIString()); + m.put(missing.getId(), linkTree); + } + } + } + } + } + } + + for (Container f : folders) + { + if (f.getId().equals(projectId)) + continue; + + NavTree child = m.get(f.getId()); + if (null == child) + continue; + + NavTree parent = m.get(f.getParent().getId()); + assert null != parent; //This should not happen anymore, we assure all parents are in tree. + if (null != parent) + parent.addChild(child); + } + + NavTree projectTree = m.get(projectId); + + projectTree.setId(cacheKey); + + NavTreeManager.cacheTree(projectTree, user); + return projectTree; + } + finally + { + assert SecurityLogger.outdent(); + } + } + + public static Set containersToRoot(Container child) + { + Set containersOnPath = new HashSet<>(); + Container current = child; + while (current != null && !current.isRoot()) + { + containersOnPath.add(current); + current = current.getParent(); + } + + return containersOnPath; + } + + /** + * Provides a sorted list of containers from the root to the child container provided. + * It does not include the root node. + * @param child Container from which the search is sourced. + * @return List sorted in order of distance from root. + */ + public static List containersToRootList(Container child) + { + List containers = new ArrayList<>(); + Container current = child; + while (current != null && !current.isRoot()) + { + containers.add(current); + current = current.getParent(); + } + + Collections.reverse(containers); + return containers; + } + + // Move a container to another part of the container tree. Careful: this method DOES NOT prevent you from orphaning + // an entire tree (e.g., by setting a container's parent to one of its children); the UI in AdminController does this. + // + // NOTE: Beware side-effect of changing ACLs and GROUPS if a container changes projects + // + // @return true if project has changed (should probably redirect to security page) + public static boolean move(Container c, final Container newParent, User user) throws ValidationException + { + if (!isRenameable(c)) + { + throw new IllegalArgumentException("Can't move container " + c.getPath()); + } + + try (QuietCloser ignored = lockForMutation(MutatingOperation.move, c)) + { + List errors = new ArrayList<>(); + for (ContainerListener listener : getListeners()) + { + try + { + errors.addAll(listener.canMove(c, newParent, user)); + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(null, new IllegalStateException(listener.getClass().getName() + ".canMove() threw an exception or violated @NotNull contract")); + } + } + if (!errors.isEmpty()) + { + ValidationException exception = new ValidationException(); + for (String error : errors) + { + exception.addError(new SimpleValidationError(error)); + } + throw exception; + } + + if (c.getParent().getId().equals(newParent.getId())) + return false; + + Container oldParent = c.getParent(); + Container oldProject = c.getProject(); + Container newProject = newParent.isRoot() ? c : newParent.getProject(); + + boolean changedProjects = !oldProject.getId().equals(newProject.getId()); + + // Synchronize the transaction, but not the listeners -- see #9901 + try (DbScope.Transaction t = ensureTransaction()) + { + new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET Parent = ? WHERE EntityId = ?", newParent.getId(), c.getId()); + + // Refresh the container directly from the database so the container reflects the new parent, isProject(), etc. + c = getForRowId(c.getRowId()); + + // this could be done in the trigger, but I prefer to put it in the transaction + if (changedProjects) + SecurityManager.changeProject(c, oldProject, newProject, user); + + clearCache(); + + try + { + ExperimentService.get().moveContainer(c, oldParent, newParent); + } + catch (ExperimentException e) + { + throw new RuntimeException(e); + } + + // Clear after the commit has propagated the state to other threads and transactions + // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own + t.addCommitTask(() -> + { + clearCache(); + getChildrenMap(newParent); // reload the cache + }, DbScope.CommitTaskOption.POSTCOMMIT); + + t.commit(); + } + + Container newContainer = getForId(c.getId()); + fireMoveContainer(newContainer, oldParent, user); + + return changedProjects; + } + } + + public static void rename(@NotNull Container c, User user, String name) + { + rename(c, user, name, c.getTitle(), false); + } + + /** + * Transacted method to rename a container. Optionally, supports updating the title and aliasing the + * original container path when the name is changed (as name changes result in a new container path). + */ + public static Container rename(@NotNull Container c, User user, String name, @Nullable String title, boolean addAlias) + { + try (QuietCloser ignored = lockForMutation(MutatingOperation.rename, c); + DbScope.Transaction tx = ensureTransaction()) + { + final String oldName = c.getName(); + final String newName = StringUtils.trimToNull(name); + boolean isRenaming = !oldName.equals(newName); + StringBuilder errors = new StringBuilder(); + + // Rename + if (isRenaming) + { + // Issue 16221: Don't allow renaming of system reserved folders (e.g. /Shared, home, root, etc). + if (!isRenameable(c)) + throw new ApiUsageException("This folder may not be renamed as it is reserved by the system."); + + if (!Container.isLegalName(newName, c.isProject(), errors)) + throw new ApiUsageException(errors.toString()); + + // Issue 19061: Unable to do case-only container rename + if (c.getParent().hasChild(newName) && !c.equals(c.getParent().getChild(newName))) + { + if (c.getParent().isRoot()) + throw new ApiUsageException("The server already has a project with this name."); + throw new ApiUsageException("The " + (c.getParent().isProject() ? "project " : "folder ") + c.getParent().getPath() + " already has a folder with this name."); + } + + new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET Name=? WHERE EntityId=?", newName, c.getId()); + clearCache(); // Clear the entire cache, since containers cache their full paths + // Get new version since name has changed. + Container renamedContainer = getForId(c.getId()); + fireRenameContainer(renamedContainer, user, oldName); + // Clear again after the commit has propagated the state to other threads and transactions + // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own + tx.addCommitTask(ContainerManager::clearCache, DbScope.CommitTaskOption.POSTCOMMIT); + + // Alias + if (addAlias) + { + // Intentionally use original container rather than the already renamedContainer + List newAliases = new ArrayList<>(getAliasesForContainer(c)); + newAliases.add(c.getPath()); + saveAliasesForContainer(c, newAliases, user); + } + } + + // Title + if (!c.getTitle().equals(title)) + { + if (!Container.isLegalTitle(title, errors)) + throw new ApiUsageException(errors.toString()); + updateTitle(c, title, user); + } + + tx.commit(); + } + catch (ValidationException e) + { + throw new IllegalArgumentException(e); + } + + return getForId(c.getId()); + } + + public static void setChildOrderToAlphabetical(Container parent) + { + setChildOrder(parent.getChildren(), true); + } + + public static void setChildOrder(Container parent, List orderedChildren) throws ContainerException + { + for (Container child : orderedChildren) + { + if (child == null || child.getParent() == null || !child.getParent().equals(parent)) // #13481 + throw new ContainerException("Invalid parent container of " + (child == null ? "null child container" : child.getPath())); + } + setChildOrder(orderedChildren, false); + } + + private static void setChildOrder(List siblings, boolean resetToAlphabetical) + { + try (DbScope.Transaction t = ensureTransaction()) + { + for (int index = 0; index < siblings.size(); index++) + { + Container current = siblings.get(index); + new SqlExecutor(CORE.getSchema()).execute("UPDATE " + CORE.getTableInfoContainers() + " SET SortOrder = ? WHERE EntityId = ?", + resetToAlphabetical ? 0 : index, current.getId()); + } + // Clear after the commit has propagated the state to other threads and transactions + // Do this in a commit task in case we've joined another existing DbScope.Transaction instead of starting our own + t.addCommitTask(ContainerManager::clearCache, DbScope.CommitTaskOption.POSTCOMMIT); + + t.commit(); + } + } + + private enum MutatingOperation + { + delete, + rename, + move + } + + private static final Map mutatingContainers = Collections.synchronizedMap(new IntHashMap<>()); + + private static QuietCloser lockForMutation(MutatingOperation op, Container c) + { + return lockForMutation(op, Collections.singletonList(c)); + } + + private static QuietCloser lockForMutation(MutatingOperation op, Collection containers) + { + List ids = new ArrayList<>(containers.size()); + synchronized (mutatingContainers) + { + for (Container container : containers) + { + MutatingOperation currentOp = mutatingContainers.get(container.getRowId()); + if (currentOp != null) + { + throw new ApiUsageException("Cannot start a " + op + " operation on " + container.getPath() + ". It is currently undergoing a " + currentOp); + } + ids.add(container.getRowId()); + } + ids.forEach(id -> mutatingContainers.put(id, op)); + } + return () -> + { + synchronized (mutatingContainers) + { + ids.forEach(mutatingContainers::remove); + } + }; + } + + // Delete containers from the database + private static boolean delete(final Collection containers, User user, @Nullable String comment) + { + // Do this check before we bother with any synchronization + for (Container container : containers) + { + if (!isDeletable(container)) + { + throw new ApiUsageException("Cannot delete container: " + container.getPath()); + } + } + + try (QuietCloser ignored = lockForMutation(MutatingOperation.delete, containers)) + { + boolean deleted = true; + for (Container c : containers) + { + deleted = deleted && delete(c, user, comment); + } + return deleted; + } + } + + // Delete a container from the database + private static boolean delete(final Container c, User user, @Nullable String comment) + { + // Verify method isn't called inappropriately + if (mutatingContainers.get(c.getRowId()) != MutatingOperation.delete) + { + throw new IllegalStateException("Container not flagged as being deleted: " + c.getPath()); + } + + LOG.debug("Starting container delete for " + c.getContainerNoun(true) + " " + c.getPath()); + + // Tell the search indexer to drop work for the container that's about to be deleted + SearchService.get().purgeForContainer(c); + + DbScope.RetryFn tryDeleteContainer = (tx) -> + { + // Verify that no children exist + Selector sel = new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("Parent"), c), null); + + if (sel.exists()) + { + _removeFromCache(c, true); + return false; + } + + if (c.shouldRemoveFromPortal()) + { + // Need to remove portal page, too; container name is page's pageId and in container's parent container + Portal.PortalPage page = Portal.getPortalPage(c.getParent(), c.getName()); + if (null != page) // Be safe + Portal.deletePage(page); + + // Tell parent + setContainerTabDeleted(c.getParent(), c.getName(), c.getParent().getFolderType().getName()); + } + + fireDeleteContainer(c, user); + + SqlExecutor sqlExecutor = new SqlExecutor(CORE.getSchema()); + sqlExecutor.execute("DELETE FROM " + CORE.getTableInfoContainerAliases() + " WHERE ContainerRowId=?", c.getRowId()); + sqlExecutor.execute("DELETE FROM " + CORE.getTableInfoContainers() + " WHERE EntityId=?", c.getId()); + // now that the container is actually gone, delete all ACLs (better to have an ACL w/o object than object w/o ACL) + SecurityPolicyManager.removeAll(c); + // and delete all container-based sequences + DbSequenceManager.deleteAll(c); + + ExperimentService experimentService = ExperimentService.get(); + if (experimentService != null) + experimentService.removeContainerDataTypeExclusions(c.getId()); + + // After we've committed the transaction, be sure that we remove this container from the cache + // See https://www.labkey.org/issues/home/Developer/issues/details.view?issueId=17015 + tx.addCommitTask(() -> + { + // Be sure that we've waited until any threads that might be populating the cache have finished + // before we guarantee that we've removed this now-deleted container + DATABASE_QUERY_LOCK.lock(); + try + { + _removeFromCache(c, true); + } + finally + { + DATABASE_QUERY_LOCK.unlock(); + } + }, DbScope.CommitTaskOption.POSTCOMMIT); + String auditComment = c.getContainerNoun(true) + " " + c.getPath() + " was deleted"; + if (comment != null) + auditComment = auditComment.concat(". " + comment); + addAuditEvent(user, c, auditComment); + return true; + }; + + boolean success = CORE.getSchema().getScope().executeWithRetry(tryDeleteContainer); + if (success) + { + LOG.debug("Completed container delete for " + c.getContainerNoun(true) + " " + c.getPath()); + } + else + { + LOG.warn("Failed to delete container: " + c.getPath()); + } + return success; + } + + /** + * Delete a single container. Primarily for use by tests. + */ + public static boolean delete(final Container c, User user) + { + return delete(List.of(c), user, null); + } + + public static boolean isDeletable(Container c) + { + return !isSystemContainer(c); + } + + public static boolean isRenameable(Container c) + { + return !isSystemContainer(c); + } + + /** System containers include the root container, /Home, and /Shared */ + public static boolean isSystemContainer(Container c) + { + return c.equals(getRoot()) || c.equals(getHomeContainer()) || c.equals(getSharedContainer()); + } + + /** Has the container already been deleted or is it in the process of being deleted? */ + public static boolean exists(@Nullable Container c) + { + return c != null && null != getForId(c.getEntityId()) && mutatingContainers.get(c.getRowId()) != MutatingOperation.delete; + } + + public static void deleteAll(Container root, User user, @Nullable String comment) throws UnauthorizedException + { + if (!hasTreePermission(root, user, DeletePermission.class)) + throw new UnauthorizedException("You don't have delete permissions to all folders"); + + LOG.debug("Starting container (and children) delete for " + root.getContainerNoun(true) + " " + root.getPath()); + Set depthFirst = getAllChildrenDepthFirst(root); + depthFirst.add(root); + + delete(depthFirst, user, comment); + + LOG.debug("Completed container (and children) delete for " + root.getContainerNoun(true) + " " + root.getPath()); + } + + public static void deleteAll(Container root, User user) throws UnauthorizedException + { + deleteAll(root, user, null); + } + + private static void addAuditEvent(User user, Container c, String comment) + { + if (user != null) + { + AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, c, comment); + AuditLogService.get().addEvent(user, event); + } + } + + private static Set getAllChildrenDepthFirst(Container c) + { + Set set = new LinkedHashSet<>(); + getAllChildrenDepthFirst(c, set); + return set; + } + + private static void getAllChildrenDepthFirst(Container c, Collection list) + { + for (Container child : c.getChildren()) + { + getAllChildrenDepthFirst(child, list); + list.add(child); + } + } + + private static Container _getFromCachePath(Path path) + { + return CACHE_PATH.get(path); + } + + private static Container _addToCache(Container c) + { + assert DATABASE_QUERY_LOCK.isHeldByCurrentThread() : "Any cache modifications must be synchronized at a " + + "higher level so that we ensure that the container to be inserted still exists and hasn't been deleted"; + CACHE_ENTITY_ID.put(c.getEntityId(), c); + CACHE_PATH.put(c.getParsedPath(), c); + return c; + } + + private static void _clearChildrenFromCache(Container c) + { + CACHE_CHILDREN.remove(c.getEntityId()); + navTreeManageUncache(c); + } + + /** @param hierarchyChange whether the shape of the container tree has changed */ + private static void _removeFromCache(Container c, boolean hierarchyChange) + { + CACHE_ENTITY_ID.remove(c.getEntityId()); + CACHE_PATH.remove(c.getParsedPath()); + + if (hierarchyChange) + { + // This is strictly keeping track of the parent/child relationships themselves so it only needs to be + // cleared when the tree changes + CACHE_CHILDREN.clear(); + } + + navTreeManageUncache(c); + } + + public static void clearCache() + { + CACHE_PATH.clear(); + CACHE_ENTITY_ID.clear(); + CACHE_CHILDREN.clear(); + + // UNDONE: NavTreeManager should register a ContainerListener + NavTreeManager.uncacheAll(); + } + + private static void navTreeManageUncache(Container c) + { + // UNDONE: NavTreeManager should register a ContainerListener + NavTreeManager.uncacheTree(PROJECT_LIST_ID); + NavTreeManager.uncacheTree(getRoot().getId()); + + Container project = c.getProject(); + if (project != null) + { + NavTreeManager.uncacheTree(project.getId()); + NavTreeManager.uncacheTree(PROJECT_LIST_ID + project.getId()); + } + } + + public static void notifyContainerChange(String id, Property prop) + { + notifyContainerChange(id, prop, null); + } + + public static void notifyContainerChange(String id, Property prop, @Nullable User u) + { + if (_constructing.contains(new GUID(id))) + return; + + Container c = getForId(id); + if (null != c) + { + _removeFromCache(c, false); + c = getForId(id); // load a fresh container since the original might be stale. + if (null != c) + { + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, u, prop, null, null); + firePropertyChangeEvent(evt); + } + } + } + + + /** Recursive, including root node */ + public static Set getAllChildren(Container root) + { + Set children = getAllChildrenDepthFirst(root); + children.add(root); + + return Collections.unmodifiableSet(children); + } + + /** + * Return all children of the root node, including root node, which have the given active module + */ + @NotNull + public static Set getAllChildrenWithModule(@NotNull Container root, @NotNull Module module) + { + Set children = new HashSet<>(); + for (Container candidate : getAllChildren(root)) + { + if (candidate.getActiveModules().contains(module)) + children.add(candidate); + } + return Collections.unmodifiableSet(children); + } + + public static long getContainerCount() + { + return new TableSelector(CORE.getTableInfoContainers()).getRowCount(); + } + + public static long getWorkbookCount() + { + return new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("type"), "workbook"), null).getRowCount(); + } + + public static long getArchivedContainerCount() + { + return new TableSelector(CORE.getTableInfoContainers(), new SimpleFilter(FieldKey.fromParts("lockstate"), "Archived"), null).getRowCount(); + } + + public static long getAuditCommentRequiredCount() + { + SQLFragment sql = new SQLFragment( + "SELECT COUNT(*) FROM\n" + + " core.containers c\n" + + " JOIN prop.propertysets ps on c.entityid = ps.objectid\n" + + " JOIN prop.properties p on p.\"set\" = ps.\"set\"\n" + + "WHERE ps.category = '" + AUDIT_SETTINGS_PROPERTY_SET_NAME + "' AND p.name='"+ REQUIRE_USER_COMMENTS_PROPERTY_NAME + "' and p.value='true'"); + return new SqlSelector(CORE.getSchema(), sql).getObject(Long.class); + } + + + /** Retrieve entire container hierarchy */ + public static MultiValuedMap getContainerTree() + { + final MultiValuedMap mm = new ArrayListValuedHashMap<>(); + + // Get all containers and parents + SqlSelector selector = new SqlSelector(CORE.getSchema(), "SELECT Parent, EntityId FROM " + CORE.getTableInfoContainers() + " ORDER BY SortOrder, LOWER(Name) ASC"); + + selector.forEach(rs -> { + String parentId = rs.getString(1); + Container parent = (parentId != null ? getForId(parentId) : null); + Container child = getForId(rs.getString(2)); + + if (null != child) + mm.put(parent, child); + }); + + return mm; + } + + /** + * Returns a branch of the container tree including only the root and its descendants + * @param root The root container + * @return MultiMap of containers including root and its descendants + */ + public static MultiValuedMap getContainerTree(Container root) + { + //build a multimap of only the container ids + final MultiValuedMap mmIds = new ArrayListValuedHashMap<>(); + + // Get all containers and parents + Selector selector = new SqlSelector(CORE.getSchema(), "SELECT Parent, EntityId FROM " + CORE.getTableInfoContainers() + " ORDER BY SortOrder, LOWER(Name) ASC"); + + selector.forEach(rs -> mmIds.put(rs.getString(1), rs.getString(2))); + + //now find the root and build a MultiMap of it and its descendants + MultiValuedMap mm = new ArrayListValuedHashMap<>(); + mm.put(null, root); + addChildren(root, mmIds, mm); + return mm; + } + + private static void addChildren(Container c, MultiValuedMap mmIds, MultiValuedMap mm) + { + Collection childIds = mmIds.get(c.getId()); + if (null != childIds) + { + for (String childId : childIds) + { + Container child = getForId(childId); + if (null != child) + { + mm.put(c, child); + addChildren(child, mmIds, mm); + } + } + } + } + + public static Set getContainerSet(MultiValuedMap mm, User user, Class perm) + { + Collection containers = mm.values(); + if (null == containers) + return new HashSet<>(); + + return containers + .stream() + .filter(c -> c.hasPermission(user, perm)) + .collect(Collectors.toSet()); + } + + + public static SQLFragment getIdsAsCsvList(Set containers, SqlDialect d) + { + if (containers.isEmpty()) + return new SQLFragment("(NULL)"); // WHERE x IN (NULL) should match no rows + + SQLFragment csvList = new SQLFragment("("); + String comma = ""; + for (Container container : containers) + { + csvList.append(comma); + comma = ","; + csvList.appendValue(container, d); + } + csvList.append(")"); + + return csvList; + } + + + public static List getIds(User user, Class perm) + { + Set containers = getContainerSet(getContainerTree(), user, perm); + + List ids = new ArrayList<>(containers.size()); + + for (Container c : containers) + ids.add(c.getId()); + + return ids; + } + + + // + // ContainerListener + // + + public interface ContainerListener extends PropertyChangeListener + { + enum Order {First, Last} + + /** Called after a new container has been created */ + void containerCreated(Container c, User user); + + default void containerCreated(Container c, User user, @Nullable String auditMsg) + { + containerCreated(c, user); + } + + /** Called immediately prior to deleting the row from core.containers */ + void containerDeleted(Container c, User user); + + /** Called after the container has been moved to its new parent */ + void containerMoved(Container c, Container oldParent, User user); + + /** + * Called prior to moving a container, to find out if there are any issues that would prevent a successful move + * @return a list of errors that should prevent the move from happening, if any + */ + @NotNull + Collection canMove(Container c, Container newParent, User user); + + @Override + void propertyChange(PropertyChangeEvent evt); + } + + public static abstract class AbstractContainerListener implements ContainerListener + { + @Override + public void containerCreated(Container c, User user) + {} + + @Override + public void containerDeleted(Container c, User user) + {} + + @Override + public void containerMoved(Container c, Container oldParent, User user) + {} + + @NotNull + @Override + public Collection canMove(Container c, Container newParent, User user) + { + return Collections.emptyList(); + } + + @Override + public void propertyChange(PropertyChangeEvent evt) + {} + } + + + public static class ContainerPropertyChangeEvent extends PropertyChangeEvent implements PropertyChange + { + public final Property property; + public final Container container; + public User user; + + public ContainerPropertyChangeEvent(Container c, @Nullable User user, Property p, Object oldValue, Object newValue) + { + super(c, p.name(), oldValue, newValue); + container = c; + this.user = user; + property = p; + } + + public ContainerPropertyChangeEvent(Container c, Property p, Object oldValue, Object newValue) + { + this(c, null, p, oldValue, newValue); + } + + @Override + public Property getProperty() + { + return property; + } + } + + + // Thread-safe list implementation that allows iteration and modifications without external synchronization + private static final List _listeners = new CopyOnWriteArrayList<>(); + private static final List _laterListeners = new CopyOnWriteArrayList<>(); + + // These listeners are executed in the order they are registered, before the "Last" listeners + public static void addContainerListener(ContainerListener listener) + { + addContainerListener(listener, ContainerListener.Order.First); + } + + + // Explicitly request "Last" ordering via this method. "Last" listeners execute after all "First" listeners. + public static void addContainerListener(ContainerListener listener, ContainerListener.Order order) + { + if (ContainerListener.Order.First == order) + _listeners.add(listener); + else + _laterListeners.add(listener); + } + + + public static void removeContainerListener(ContainerListener listener) + { + _listeners.remove(listener); + _laterListeners.remove(listener); + } + + + private static List getListeners() + { + List combined = new ArrayList<>(_listeners.size() + _laterListeners.size()); + combined.addAll(_listeners); + combined.addAll(_laterListeners); + + return combined; + } + + + private static List getListenersReversed() + { + List combined = new LinkedList<>(); + + // Copy to guarantee consistency between .listIterator() and .size() + List copy = new ArrayList<>(_listeners); + ListIterator iter = copy.listIterator(copy.size()); + + // Iterate in reverse + while(iter.hasPrevious()) + combined.add(iter.previous()); + + // Copy to guarantee consistency between .listIterator() and .size() + // Add elements from the laterList in reverse order so that Core is fired last + List laterCopy = new ArrayList<>(_laterListeners); + ListIterator laterIter = laterCopy.listIterator(laterCopy.size()); + + // Iterate in reverse + while(laterIter.hasPrevious()) + combined.add(laterIter.previous()); + + return combined; + } + + + protected static void fireCreateContainer(Container c, User user, @Nullable String auditMsg) + { + List list = getListeners(); + + for (ContainerListener cl : list) + { + try + { + cl.containerCreated(c, user, auditMsg); + } + catch (Throwable t) + { + LOG.error("fireCreateContainer for " + cl.getClass().getName(), t); + } + } + } + + + protected static void fireDeleteContainer(Container c, User user) + { + List list = getListenersReversed(); + + for (ContainerListener l : list) + { + LOG.debug("Deleting " + c.getPath() + ": fireDeleteContainer for " + l.getClass().getName()); + try + { + l.containerDeleted(c, user); + } + catch (RuntimeException e) + { + LOG.error("fireDeleteContainer for " + l.getClass().getName(), e); + + // Fail fast (first Throwable aborts iteration), #17560 + throw e; + } + } + } + + + protected static void fireRenameContainer(Container c, User user, String oldValue) + { + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, user, Property.Name, oldValue, c.getName()); + firePropertyChangeEvent(evt); + } + + + protected static void fireMoveContainer(Container c, Container oldParent, User user) + { + List list = getListeners(); + + for (ContainerListener cl : list) + { + // While we would ideally transact the full container move, that will likely cause long-blocking + // queries and/or deadlocks. For now, at least transact each separate move handler independently + try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) + { + cl.containerMoved(c, oldParent, user); + transaction.commit(); + } + } + ContainerPropertyChangeEvent evt = new ContainerPropertyChangeEvent(c, user, Property.Parent, oldParent, c.getParent()); + firePropertyChangeEvent(evt); + } + + + public static void firePropertyChangeEvent(ContainerPropertyChangeEvent evt) + { + if (_constructing.contains(evt.container.getEntityId())) + return; + + List list = getListeners(); + for (ContainerListener l : list) + { + try + { + l.propertyChange(evt); + } + catch (Throwable t) + { + LOG.error("firePropertyChangeEvent for " + l.getClass().getName(), t); + } + } + } + + private static final List MODULE_DEPENDENCY_PROVIDERS = new CopyOnWriteArrayList<>(); + + public static void registerModuleDependencyProvider(ModuleDependencyProvider provider) + { + MODULE_DEPENDENCY_PROVIDERS.add(provider); + } + + public static void forEachModuleDependencyProvider(Consumer action) + { + MODULE_DEPENDENCY_PROVIDERS.forEach(action); + } + + // Compliance module adds a locked project handler that checks permissions; without that, this implementation + // is used, and projects are never locked + static volatile LockedProjectHandler LOCKED_PROJECT_HANDLER = (project, user, contextualRoles, lockState) -> false; + + // Replaces any previously set LockedProjectHandler + public static void setLockedProjectHandler(LockedProjectHandler handler) + { + LOCKED_PROJECT_HANDLER = handler; + } + + public static Container createDefaultSupportContainer() + { + LOG.info("Creating default support container: " + DEFAULT_SUPPORT_PROJECT_PATH); + // create a "support" container. Admins can do anything, + // Users can read/write, Guests can read. + return bootstrapContainer(DEFAULT_SUPPORT_PROJECT_PATH, + RoleManager.getRole(AuthorRole.class), + RoleManager.getRole(ReaderRole.class) + ); + } + + public static void removeDefaultSupportContainer(User user) + { + Container support = getDefaultSupportContainer(); + if (support != null) + { + LOG.info("Removing default support container: " + DEFAULT_SUPPORT_PROJECT_PATH); + ContainerManager.delete(support, user); + } + } + + public static Container getDefaultSupportContainer() + { + return getForPath(DEFAULT_SUPPORT_PROJECT_PATH); + } + + public static List getAliasesForContainer(Container c) + { + return Collections.unmodifiableList(new SqlSelector(CORE.getSchema(), + new SQLFragment("SELECT Path FROM " + CORE.getTableInfoContainerAliases() + " WHERE ContainerRowId = ? ORDER BY Path", + c.getRowId())).getArrayList(String.class)); + } + + @Nullable + public static Container resolveContainerPathAlias(String path) + { + return resolveContainerPathAlias(path, false); + } + + @Nullable + private static Container resolveContainerPathAlias(String path, boolean top) + { + // Strip any trailing slashes + while (path.endsWith("/")) + { + path = path.substring(0, path.length() - 1); + } + + // Simple case -- resolve directly (sans alias) + Container aliased = getForPath(path); + if (aliased != null) + return aliased; + + // Simple case -- directly resolve from database + aliased = getForPathAlias(path); + if (aliased != null) + return aliased; + + // At the leaf and the container was not found + if (top) + return null; + + List splits = Arrays.asList(path.split("/")); + String subPath = ""; + for (int i=0; i < splits.size()-1; i++) // minus 1 due to leaving off last container + { + if (!splits.get(i).isEmpty()) + subPath += "/" + splits.get(i); + } + + aliased = resolveContainerPathAlias(subPath, false); + + if (aliased == null) + return null; + + String leafPath = aliased.getPath() + "/" + splits.get(splits.size()-1); + return resolveContainerPathAlias(leafPath, true); + } + + @Nullable + private static Container getForPathAlias(String path) + { + // We store the path as lower-case, so we don't need to also LOWER() on the value in core.ContainerAliases, letting the DB use the index + Container[] ret = new SqlSelector(CORE.getSchema(), + "SELECT * FROM " + CORE.getTableInfoContainers() + " c, " + CORE.getTableInfoContainerAliases() + " ca WHERE ca.ContainerRowId = c.RowId AND ca.path = LOWER(?)", + path).getArray(Container.class); + + return ret.length == 0 ? null : ret[0]; + } + + public static Container getMoveTargetContainer(@Nullable String queryName, @NotNull Container sourceContainer, User user, @Nullable String targetIdOrPath, Errors errors) + { + if (targetIdOrPath == null) + { + errors.reject(ERROR_GENERIC, "A target container must be specified for the move operation."); + return null; + } + + Container _targetContainer = getContainerForIdOrPath(targetIdOrPath); + if (_targetContainer == null) + { + errors.reject(ERROR_GENERIC, "The target container was not found: " + targetIdOrPath + "."); + return null; + } + + if (!_targetContainer.hasPermission(user, InsertPermission.class)) + { + String _queryName = queryName == null ? "this table" : "'" + queryName + "'"; + errors.reject(ERROR_GENERIC, "You do not have permission to move rows from " + _queryName + " to the target container: " + targetIdOrPath + "."); + return null; + } + + if (!isValidTargetContainer(sourceContainer, _targetContainer)) + { + errors.reject(ERROR_GENERIC, "Invalid target container for the move operation: " + targetIdOrPath + "."); + return null; + } + return _targetContainer; + } + + private static Container getContainerForIdOrPath(String targetContainer) + { + Container c = ContainerManager.getForId(targetContainer); + if (c == null) + c = ContainerManager.getForPath(targetContainer); + + return c; + } + + // targetContainer must be in the same app project at this time + // i.e. child of current project, project of current child, sibling within project + private static boolean isValidTargetContainer(Container current, Container target) + { + if (current.isRoot() || target.isRoot()) + return false; + + // Allow moving to the current container since we now allow the chosen entities to be from different containers + if (current.equals(target)) + return true; + + boolean moveFromProjectToChild = current.isProject() && target.getParent().equals(current); + boolean moveFromChildToProject = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target); + boolean moveFromChildToSibling = !current.isProject() && current.getParent().isProject() && current.getParent().equals(target.getParent()); + + return moveFromProjectToChild || moveFromChildToProject || moveFromChildToSibling; + } + + public static int updateContainer(TableInfo dataTable, String idField, Collection ids, Container targetContainer, User user, boolean withModified) + { + try (DbScope.Transaction transaction = dataTable.getSchema().getScope().ensureTransaction()) + { + SQLFragment dataUpdate = new SQLFragment("UPDATE ").append(dataTable) + .append(" SET container = ").appendValue(targetContainer.getEntityId()); + if (withModified) + { + dataUpdate.append(", modified = ").appendValue(new Date()); + dataUpdate.append(", modifiedby = ").appendValue(user.getUserId()); + } + dataUpdate.append(" WHERE ").append(idField); + dataTable.getSchema().getSqlDialect().appendInClauseSql(dataUpdate, ids); + int numUpdated = new SqlExecutor(dataTable.getSchema()).execute(dataUpdate); + transaction.commit(); + + return numUpdated; + } + } + + /** + * If a container at the given path does not exist, create one and set permissions. If the container does exist, + * permissions are only set if there is no explicit ACL for the container. This prevents us from resetting + * permissions if all users are dropped. Implicitly done as an admin-level service user. + */ + @NotNull + public static Container bootstrapContainer(String path, @NotNull Role userRole, @Nullable Role guestRole) + { + Container c = null; + User user = User.getAdminServiceUser(); + + try + { + c = getForPath(path); + } + catch (RootContainerException e) + { + // Ignore this -- root doesn't exist yet + } + boolean newContainer = false; + + if (c == null) + { + LOG.debug("Creating new container for path '" + path + "'"); + newContainer = true; + c = ensureContainer(path, user); + } + + // Only set permissions if there are no explicit permissions + // set for this object or we just created it + Integer policyCount = null; + if (!newContainer) + { + policyCount = new SqlSelector(CORE.getSchema(), + "SELECT COUNT(*) FROM " + CORE.getTableInfoPolicies() + " WHERE ResourceId = ?", + c.getId()).getObject(Integer.class); + } + + if (newContainer || 0 == policyCount.intValue()) + { + LOG.debug("Setting permissions for '" + path + "'"); + MutableSecurityPolicy policy = new MutableSecurityPolicy(c); + policy.addRoleAssignment(SecurityManager.getGroup(Group.groupUsers), userRole); + if (guestRole != null) + policy.addRoleAssignment(SecurityManager.getGroup(Group.groupGuests), guestRole); + SecurityPolicyManager.savePolicy(policy, user); + } + + return c; + } + + /** + * @param container the container being created. May be null if we haven't actually created it yet + * @param parent the parent of the container being created. Used in case the container doesn't actually exist yet. + * @return the list of standard steps and any extra ones based on the container's FolderType + */ + public static List getCreateContainerWizardSteps(@Nullable Container container, @NotNull Container parent) + { + List navTrail = new ArrayList<>(); + + boolean isProject = parent.isRoot(); + + navTrail.add(new NavTree(isProject ? "Create Project" : "Create Folder")); + navTrail.add(new NavTree("Users / Permissions")); + if (isProject) + navTrail.add(new NavTree("Project Settings")); + if (container != null) + navTrail.addAll(container.getFolderType().getExtraSetupSteps(container)); + return navTrail; + } + + @TestTimeout(120) @TestWhen(TestWhen.When.BVT) + public static class TestCase extends Assert implements ContainerListener + { + Map _containers = new HashMap<>(); + Container _testRoot = null; + + @Before + public void setUp() + { + if (null == _testRoot) + { + Container junit = JunitUtil.getTestContainer(); + _testRoot = ensureContainer(junit, "ContainerManager$TestCase-" + GUID.makeGUID(), TestContext.get().getUser()); + addContainerListener(this); + } + } + + @After + public void tearDown() + { + removeContainerListener(this); + if (null != _testRoot) + deleteAll(_testRoot, TestContext.get().getUser()); + } + + @Test + public void testImproperFolderNamesBlocked() + { + String[] badNames = {"", "f\\o", "f/o", "f\\\\o", "foo;", "@foo", "foo" + '\u001F', '\u0000' + "foo", "fo" + '\u007F' + "o", "" + '\u009F'}; + + for (String name: badNames) + { + try + { + Container c = createContainer(_testRoot, name, TestContext.get().getUser()); + try + { + assertTrue(delete(c, TestContext.get().getUser())); + } + catch (Exception ignored) {} + fail("Should have thrown exception when trying to create container with name: " + name); + } + catch (ApiUsageException e) + { + // Do nothing, this is expected + } + } + } + + @Test + public void testCreateDeleteContainers() + { + int count = 20; + Random random = new Random(); + MultiValuedMap mm = new ArrayListValuedHashMap<>(); + + for (int i = 1; i <= count; i++) + { + int parentId = random.nextInt(i); + String parentName = 0 == parentId ? _testRoot.getName() : String.valueOf(parentId); + String childName = String.valueOf(i); + mm.put(parentName, childName); + } + + logNode(mm, _testRoot.getName(), 0); + for (int i=0; i<2; i++) //do this twice to make sure the containers were *really* deleted + { + createContainers(mm, _testRoot.getName(), _testRoot); + assertEquals(count, _containers.size()); + cleanUpChildren(mm, _testRoot.getName(), _testRoot); + assertEquals(0, _containers.size()); + } + } + + @Test + public void testCache() + { + assertEquals(0, _containers.size()); + assertEquals(0, getChildren(_testRoot).size()); + + Container one = createContainer(_testRoot, "one", TestContext.get().getUser()); + assertEquals(1, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(0, getChildren(one).size()); + + Container oneA = createContainer(one, "A", TestContext.get().getUser()); + assertEquals(2, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(1, getChildren(one).size()); + assertEquals(0, getChildren(oneA).size()); + + Container oneB = createContainer(one, "B", TestContext.get().getUser()); + assertEquals(3, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(2, getChildren(one).size()); + assertEquals(0, getChildren(oneB).size()); + + Container deleteme = createContainer(one, "deleteme", TestContext.get().getUser()); + assertEquals(4, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(3, getChildren(one).size()); + assertEquals(0, getChildren(deleteme).size()); + + assertTrue(delete(deleteme, TestContext.get().getUser())); + assertEquals(3, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(2, getChildren(one).size()); + + Container oneC = createContainer(one, "C", TestContext.get().getUser()); + assertEquals(4, _containers.size()); + assertEquals(1, getChildren(_testRoot).size()); + assertEquals(3, getChildren(one).size()); + assertEquals(0, getChildren(oneC).size()); + + assertTrue(delete(oneC, TestContext.get().getUser())); + assertTrue(delete(oneB, TestContext.get().getUser())); + assertEquals(1, getChildren(one).size()); + + assertTrue(delete(oneA, TestContext.get().getUser())); + assertEquals(0, getChildren(one).size()); + + assertTrue(delete(one, TestContext.get().getUser())); + assertEquals(0, getChildren(_testRoot).size()); + assertEquals(0, _containers.size()); + } + + @Test + public void testFolderType() + { + // Test all folder types + List folderTypes = new ArrayList<>(FolderTypeManager.get().getAllFolderTypes()); + for (FolderType folderType : folderTypes) + { + if (!folderType.isProjectOnlyType()) // Dataspace can't be subfolder + testOneFolderType(folderType); + } + } + + private void testOneFolderType(FolderType folderType) + { + LOG.info("testOneFolderType(" + folderType.getName() + "): creating container"); + Container newFolder = createContainer(_testRoot, "folderTypeTest", TestContext.get().getUser()); + FolderType ft = newFolder.getFolderType(); + assertEquals(FolderType.NONE, ft); + + Container newFolderFromCache = getForId(newFolder.getId()); + assertNotNull(newFolderFromCache); + assertEquals(FolderType.NONE, newFolderFromCache.getFolderType()); + LOG.info("testOneFolderType(" + folderType.getName() + "): setting folder type"); + newFolder.setFolderType(folderType, TestContext.get().getUser()); + + newFolderFromCache = getForId(newFolder.getId()); + assertNotNull(newFolderFromCache); + assertEquals(newFolderFromCache.getFolderType().getName(), folderType.getName()); + assertEquals(newFolderFromCache.getFolderType().getDescription(), folderType.getDescription()); + + LOG.info("testOneFolderType(" + folderType.getName() + "): deleteAll"); + deleteAll(newFolder, TestContext.get().getUser()); // There might be subfolders because of container tabs + LOG.info("testOneFolderType(" + folderType.getName() + "): deleteAll complete"); + Container deletedContainer = getForId(newFolder.getId()); + + if (deletedContainer != null) + { + fail("Expected container with Id " + newFolder.getId() + " to be deleted, but found " + deletedContainer + ". Folder type was " + folderType); + } + } + + private static void createContainers(MultiValuedMap mm, String name, Container parent) + { + Collection nodes = mm.get(name); + + if (null == nodes) + return; + + for (String childName : nodes) + { + Container child = createContainer(parent, childName, TestContext.get().getUser()); + createContainers(mm, childName, child); + } + } + + private static void cleanUpChildren(MultiValuedMap mm, String name, Container parent) + { + Collection nodes = mm.get(name); + + if (null == nodes) + return; + + for (String childName : nodes) + { + Container child = getForPath(makePath(parent, childName)); + cleanUpChildren(mm, childName, child); + assertTrue(delete(child, TestContext.get().getUser())); + } + } + + private static void logNode(MultiValuedMap mm, String name, int offset) + { + Collection nodes = mm.get(name); + + if (null == nodes) + return; + + for (String childName : nodes) + { + LOG.debug(StringUtils.repeat(" ", offset) + childName); + logNode(mm, childName, offset + 1); + } + } + + // ContainerListener + @Override + public void propertyChange(PropertyChangeEvent evt) + { + } + + @Override + public void containerCreated(Container c, User user) + { + if (null == _testRoot || !c.getParsedPath().startsWith(_testRoot.getParsedPath())) + return; + _containers.put(c.getParsedPath(), c); + } + + + @Override + public void containerDeleted(Container c, User user) + { + _containers.remove(c.getParsedPath()); + } + + @Override + public void containerMoved(Container c, Container oldParent, User user) + { + } + + @NotNull + @Override + public Collection canMove(Container c, Container newParent, User user) + { + return Collections.emptyList(); + } + } + + static + { + ObjectFactory.Registry.register(Container.class, new ContainerFactory()); + } + + public static class ContainerFactory implements ObjectFactory + { + @Override + public Container fromMap(Map m) + { + throw new UnsupportedOperationException(); + } + + @Override + public Container fromMap(Container bean, Map m) + { + throw new UnsupportedOperationException(); + } + + @Override + public Map toMap(Container bean, Map m) + { + throw new UnsupportedOperationException(); + } + + @Override + public Container handle(ResultSet rs) throws SQLException + { + String id; + Container d; + String parentId = rs.getString("Parent"); + String name = rs.getString("Name"); + id = rs.getString("EntityId"); + int rowId = rs.getInt("RowId"); + int sortOrder = rs.getInt("SortOrder"); + Date created = rs.getTimestamp("Created"); + int createdBy = rs.getInt("CreatedBy"); + // _ts + String description = rs.getString("Description"); + String type = rs.getString("Type"); + String title = rs.getString("Title"); + boolean searchable = rs.getBoolean("Searchable"); + String lockStateString = rs.getString("LockState"); + LockState lockState = null != lockStateString ? Enums.getIfPresent(LockState.class, lockStateString).or(LockState.Unlocked) : LockState.Unlocked; + + LocalDate expirationDate = rs.getObject("ExpirationDate", LocalDate.class); + + // Could be running upgrade code before these recent columns have been added to the table. Use a find map + // to determine if they are present. Issue 51692. These checks could be removed after creation of these + // columns is incorporated into the bootstrap scripts. + Map findMap = ResultSetUtil.getFindMap(rs.getMetaData()); + Long fileRootSize = findMap.containsKey("FileRootSize") ? (Long)rs.getObject("FileRootSize") : null; // getObject() and cast because getLong() returns 0 for null + LocalDateTime fileRootLastCrawled = findMap.containsKey("FileRootLastCrawled") ? rs.getObject("FileRootLastCrawled", LocalDateTime.class) : null; + + Container dirParent = null; + if (null != parentId) + dirParent = getForId(parentId); + + d = new Container(dirParent, name, id, rowId, sortOrder, created, createdBy, searchable); + d.setDescription(description); + d.setType(type); + d.setTitle(title); + d.setLockState(lockState); + d.setExpirationDate(expirationDate); + d.setFileRootSize(fileRootSize); + d.setFileRootLastCrawled(fileRootLastCrawled); + return d; + } + + @Override + public ArrayList handleArrayList(ResultSet rs) throws SQLException + { + ArrayList list = new ArrayList<>(); + while (rs.next()) + { + list.add(handle(rs)); + } + return list; + } + } + + public static Container createFakeContainer(@Nullable String name, @Nullable Container parent) + { + return new Container(parent, name, GUID.makeGUID(), 1, 0, new Date(), 0, false); + } +} diff --git a/api/src/org/labkey/api/search/SearchResultTemplate.java b/api/src/org/labkey/api/search/SearchResultTemplate.java index 92cc7972793..209e62f1957 100644 --- a/api/src/org/labkey/api/search/SearchResultTemplate.java +++ b/api/src/org/labkey/api/search/SearchResultTemplate.java @@ -1,99 +1,95 @@ -/* - * Copyright (c) 2012-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.search; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.Container; -import org.labkey.api.search.SearchService.SearchCategory; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.ViewContext; - -import java.util.List; - -public interface SearchResultTemplate -{ - @Nullable String getName(); - - /** - * Return null for default behavior (using "category" parameter on the URL) or return a space-separated list of category names to override. - */ - @Nullable String getCategories(); - - /** - * Return null for default behavior (using search scope on the URL) or return a search scope to override. - */ - @Nullable SearchScope getSearchScope(); - - @NotNull String getResultNameSingular(); - - @NotNull String getResultNamePlural(); - - boolean includeNavigationLinks(); - - boolean includeAdvanceUI(); - - @Nullable HtmlString getExtraHtml(ViewContext ctx); - - @Nullable HtmlString getHiddenInputsHtml(ViewContext ctx); - - String reviseQuery(ViewContext ctx, String q); - - default void addNavTrail(NavTree root, ViewContext ctx, @NotNull SearchScope scope, @Nullable String category) - { - Container c = ctx.getContainer(); - String title = "Search"; - - switch (scope) - { - case All: - title += " site"; - break; - case Project: - Container project = c.getProject(); - if (null != project) - title += " project '" + project.getName() + "'"; - break; - case Folder: - case FolderAndSubfolders: - title += " folder '"; - if (c.getName().isEmpty()) - title += "root'"; - else - title += c.getName() + "'"; - break; - } - - SearchService ss = SearchService.get(); - if (ss != null) - { - List categories = ss.getCategories(category); - - if (null != categories) - { - List list = categories.stream() - .map(SearchCategory::getDescription) - .toList(); - - title += " for " + StringUtilsLabKey.joinWithConjunction(list, "and"); - } - } - - root.addChild(title); - } -} +/* + * Copyright (c) 2012-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.search; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.search.SearchService.SearchCategory; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.ViewContext; + +import java.util.List; + +public interface SearchResultTemplate +{ + @Nullable String getName(); + + /** + * Return null for default behavior (using "category" parameter on the URL) or return a space-separated list of category names to override. + */ + @Nullable String getCategories(); + + /** + * Return null for default behavior (using search scope on the URL) or return a search scope to override. + */ + @Nullable SearchScope getSearchScope(); + + @NotNull String getResultNameSingular(); + + @NotNull String getResultNamePlural(); + + boolean includeNavigationLinks(); + + boolean includeAdvanceUI(); + + @Nullable HtmlString getExtraHtml(ViewContext ctx); + + @Nullable HtmlString getHiddenInputsHtml(ViewContext ctx); + + String reviseQuery(ViewContext ctx, String q); + + default void addNavTrail(NavTree root, ViewContext ctx, @NotNull SearchScope scope, @Nullable String category) + { + Container c = ctx.getContainer(); + String title = "Search"; + + switch (scope) + { + case All: + title += " site"; + break; + case Project: + Container project = c.getProject(); + if (null != project) + title += " project '" + project.getName() + "'"; + break; + case Folder: + case FolderAndSubfolders: + title += " folder '"; + if (c.getName().isEmpty()) + title += "root'"; + else + title += c.getName() + "'"; + break; + } + + List categories = SearchService.get().getCategories(category); + + if (null != categories) + { + List list = categories.stream() + .map(SearchCategory::getDescription) + .toList(); + + title += " for " + StringUtilsLabKey.joinWithConjunction(list, "and"); + } + + root.addChild(title); + } +} diff --git a/api/src/org/labkey/api/search/SearchService.java b/api/src/org/labkey/api/search/SearchService.java index 214024ba7a7..270077e3487 100644 --- a/api/src/org/labkey/api/search/SearchService.java +++ b/api/src/org/labkey/api/search/SearchService.java @@ -388,7 +388,7 @@ public String normalizeHref(Path contextPath, Container c) DbSchema getSchema(); - WebPartView getSearchView(boolean includeSubfolders, int textBoxWidth, boolean includeHelpLink, boolean isWebpart); + WebPartView getSearchView(boolean includeSubfolders, int textBoxWidth, boolean includeHelpLink, boolean isWebpart); SearchResult search(SearchOptions options) throws IOException; diff --git a/api/src/org/labkey/api/util/ExceptionUtil.java b/api/src/org/labkey/api/util/ExceptionUtil.java index 6c623fe76c6..8ba91854dac 100644 --- a/api/src/org/labkey/api/util/ExceptionUtil.java +++ b/api/src/org/labkey/api/util/ExceptionUtil.java @@ -1,1722 +1,1721 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.util; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.core.LoggerContext; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.action.ApiResponseWriter; -import org.labkey.api.action.ApiSimpleResponse; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.miniprofiler.MiniProfiler; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.LoginUrls; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.BadRequestException; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.RedirectException; -import org.labkey.api.view.RequestBasicAuthException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.ViewServlet; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.PageConfig; -import org.labkey.api.webdav.DavException; -import org.labkey.api.writer.PrintWriters; -import org.springframework.dao.DataAccessResourceFailureException; - -import jakarta.servlet.ServletException; -import jakarta.servlet.ServletOutputStream; -import jakarta.servlet.WriteListener; -import jakarta.servlet.http.Cookie; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletRequestWrapper; -import jakarta.servlet.http.HttpServletResponse; -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.EOFException; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringReader; -import java.io.StringWriter; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Proxy; -import java.net.MalformedURLException; -import java.net.SocketException; -import java.net.URISyntaxException; -import java.security.Principal; -import java.sql.BatchUpdateException; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.TreeMap; -import java.util.WeakHashMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class ExceptionUtil -{ - public static final String REQUEST_EXCEPTION_ATTRIBUTE = ExceptionUtil.class.getName() + "$exception"; - public static final String CALCULATED_COLUMN_SQL_TAG = "/* CALCULATED-EXPRESSION-COLUMN-QUERY */"; - - private static final JobRunner JOB_RUNNER = new JobRunner("Mothership Reporting", 1); - private static final Logger LOG = LogHelper.getLogger(ExceptionUtil.class, "Handles rendering of errors during requests"); - /** - * Remember all the exceptions we've seen since the server started up. - * Key is the exception's hash as calculated by {@link #hashStackTrace(ExceptionStackTrace)}. - */ - private static final Map EXCEPTION_TALLIES = Collections.synchronizedMap(new HashMap<>()); - - private static class ExceptionTally - { - /** Total number of times the exception has happened */ - private final AtomicInteger _count = new AtomicInteger(0); - /** Timestamp of the last time we reported it */ - private long _lastReported = System.currentTimeMillis(); - } - - private ExceptionUtil() - { - } - - public static String renderStackTrace(@Nullable StackTraceElement[] stackTrace) - { - return renderStackTrace(stackTrace, 2); - } - - public static String renderStackTrace(@Nullable StackTraceElement[] stackTrace, int linesToSkip) - { - if (stackTrace == null) - { - return MiniProfiler.NO_STACK_TRACE_AVAILABLE; - } - StringBuilder trace = new StringBuilder(); - - for (int i = linesToSkip; i < stackTrace.length; i++) - { - String line = String.valueOf(stackTrace[i]); - if (line.startsWith("javax.servlet.http.HttpServlet.service(")) - break; - trace.append("\n\tat "); - trace.append(line); - } - - return trace.toString(); - } - - @NotNull - public static Throwable unwrapException(@NotNull Throwable ex) - { - Throwable cause=ex; - - while (null != cause) - { - ex = cause; - cause = null; - - if (ex.getClass() == RuntimeException.class || ex.getClass() == UnexpectedException.class || ex.getClass() == RuntimeSQLException.class || ex instanceof InvocationTargetException || ex instanceof com.google.gwt.user.server.rpc.UnexpectedException) - { - cause = ex.getCause(); - } - else if (ex.getClass() == ServletException.class && ((ServletException)ex).getRootCause() != null) - { - ex = ((ServletException)ex).getRootCause(); - } - else if (ex instanceof BatchUpdateException) - { - cause = ((BatchUpdateException)ex).getNextException(); - } - } - - return ex; - } - - public static HtmlString renderException(Throwable e) - { - StringWriter sw = new StringWriter(); - e.printStackTrace(new PrintWriter(sw)); - String s = PageFlowUtil.filter(sw.toString()); - s = s.replaceAll(" ", " "); - s = s.replaceAll("\t", "       "); - return HtmlString.unsafe("

\n" + s + "
\n"); - } - - public static HtmlString getUnauthorizedMessage(ViewContext context) - { - return HtmlString.unsafe("
" + - (context.getUser().isGuest() ? "Please sign in to see this data." : "You do not have permission to see this data.") + - "
"); - } - - public static WebPartView getErrorWebPartView(int responseStatus, String message, Throwable ex, - HttpServletRequest request) - { - ErrorRenderer renderer = getErrorRenderer(responseStatus, message, ex, request, true, false); - return new WebPartErrorView(renderer); - } - - - public static ErrorRenderer getErrorRenderer(int responseStatus, String message, Throwable ex, - @Nullable HttpServletRequest request, boolean isPart, boolean isStartupFailure) - { - String errorCode = null; - if (!isStartupFailure && responseStatus == HttpServletResponse.SC_INTERNAL_SERVER_ERROR) - { - errorCode = logExceptionToMothership(request, ex); - } - - if (isPart) - return new WebPartErrorRenderer(responseStatus, errorCode, message, ex, isStartupFailure); - else - return new ErrorRenderer(responseStatus, errorCode, message, ex, isStartupFailure); - } - - private static ExceptionReportingLevel getExceptionReportingLevel() - { - // Assume reporting level HIGH during initial install. Admin hasn't made a choice yet plus early exceptions - // (e.g., before root container is created) will cause AppProps to throw. - boolean installing = ModuleLoader.getInstance().isUpgradeRequired() && ModuleLoader.getInstance().isNewInstall(); - return installing ? ExceptionReportingLevel.HIGH : AppProps.getInstance().getExceptionReportingLevel(); - } - - private static boolean isSelfReportExceptions() - { - // Assume false during initial install, as we likely not far enough along to be able to store the exception - boolean installing = ModuleLoader.getInstance().isUpgradeRequired() && ModuleLoader.getInstance().isNewInstall(); - return !installing && AppProps.getInstance().isSelfReportExceptions(); - } - - /** @param request may be null if this is coming from a background thread or init */ - public static String logExceptionToMothership(@Nullable HttpServletRequest request, Throwable ex) - { - return logExceptionToMothership(request, ex, true); - } - - /** @param request may be null if this is coming from a background thread or init */ - public static String logExceptionToMothership(@Nullable HttpServletRequest request, Throwable ex, boolean writeToLog4J) - { - if (ContextListener.isShuttingDown()) - return null; - - ex = unwrapException(ex); - - if (isIgnorable(ex)) - return null; - - String requestURL = request == null ? null : (String) request.getAttribute(ViewServlet.ORIGINAL_URL_STRING); - // Need this extra check to make sure we're not in an infinite loop if there's - // an exception when trying to submit an exception - if (requestURL != null && MothershipReport.isMothershipExceptionReport(requestURL)) - return null; - - String extraInfo = ""; - String sqlState = null; - for (Throwable t = ex ; t != null ; t = t.getCause()) - { - if (t instanceof DataAccessResourceFailureException) - { - // Don't report exceptions from database connectivity issues - return null; - } - if (t instanceof RuntimeSQLException runtimeSQLException) - { - // Unwrap RuntimeSQLExceptions - t = runtimeSQLException.getSQLException(); - } - - if (t instanceof SQLException sqlException) - { - if (sqlException.getMessage() != null && sqlException.getMessage().contains("terminating connection due to administrator command")) - { - // Don't report exceptions from Postgres shutting down - return null; - } - sqlState = sqlException.getSQLState(); - String extraSqlInfo = CoreSchema.getInstance().getSqlDialect().getExtraInfo(sqlException); - if (extraSqlInfo != null) - { - extraInfo = extraSqlInfo; - } - } - - if (t instanceof DeadlockPreventingException) - { - String extraSqlInfo = CoreSchema.getInstance().getSqlDialect().getOtherDatabaseThreads(); - if (extraSqlInfo != null) - { - extraInfo = extraSqlInfo; - } - } - - if (sqlState != null) - break; - } - - String errorCode = null; - try - { - // Once to labkey.org, if so configured - errorCode = sendReport(createReportFromThrowable(request, ex, requestURL, MothershipReport.Target.remote, getExceptionReportingLevel(), null, sqlState, extraInfo)); - - // And once to the local server, if so configured. If submitting to both labkey.org and the local server, the errorCode will be the same. - if (isSelfReportExceptions()) - { - String newErrorCode = sendReport(createReportFromThrowable(request, ex, requestURL, MothershipReport.Target.local, ExceptionReportingLevel.HIGH, errorCode, sqlState, extraInfo)); - if (null == errorCode) - errorCode = newErrorCode; // which may still be null, if server is configured to not send reports to either location. - } - } - finally - { - if (writeToLog4J) - { - String message = "Exception detected"; - if (null != errorCode) - message += " and logged to mothership with error code " + errorCode; - String decorations = getExtendedMessage(ex); - - if (!extraInfo.isBlank() || !decorations.isBlank()) - { - message += "\nAdditional exception info:"; - if (!extraInfo.isBlank()) - { - message += "\n" + extraInfo; - } - if (!decorations.isBlank()) - { - message += "\n" + decorations; - } - if (HttpView.hasCurrentView()) - { - ViewContext viewContext = HttpView.currentContext(); - message += "\nCurrent URL: " + viewContext.getActionURL(); - if (null != viewContext.getUser()) - { - message += "\nCurrent user: " + (viewContext.getUser().isGuest() ? "Guest" : viewContext.getUser().getEmail()); - } - } - - LOG.error(message, ex); - } - } - } - - return errorCode; - } - - private static String sendReport(MothershipReport report) - { - if (null != report) - { - JOB_RUNNER.execute(report); - return report.getErrorCode(); - } - return null; - } - - /** Figure out exactly what text for the stack trace and other details we should submit */ - @Nullable - public static MothershipReport createReportFromThrowable( - @Nullable HttpServletRequest request, - Throwable ex, - String requestURL, - MothershipReport.Target target, - ExceptionReportingLevel level, - @Nullable String errorCode, - @Nullable String sqlState, - @Nullable String extraInfo - ) - { - Map, String> decorations = getExceptionDecorations(ex); - - String exceptionMessage = null; - if (!decorations.isEmpty() && (level == ExceptionReportingLevel.MEDIUM || level == ExceptionReportingLevel.HIGH)) - exceptionMessage = getExtendedMessage(ex); - - StringWriter stringWriter = new StringWriter(); - PrintWriter printWriter = new PrintWriter(stringWriter, true); - ex.printStackTrace(printWriter); - if (ex instanceof ServletException servletException && servletException.getRootCause() != null) - { - printWriter.println("Nested ServletException cause is:"); - servletException.getRootCause().printStackTrace(printWriter); - } - String browser = request == null ? null : request.getHeader("User-Agent"); - - ExceptionStackTrace stackTrace = new ExceptionStackTrace(stringWriter.getBuffer().toString(), false); - - if (!shouldSend(level, target.isLocal(), stackTrace)) - return null; - - if (extraInfo != null) - stackTrace = new ExceptionStackTrace(stackTrace.stackTrace + "\n" + extraInfo, false); - - String referrerURL = null; - String username = "NOT SET"; - if (request != null) - { - referrerURL = request.getHeader("Referer"); - if (request.getUserPrincipal() != null) - { - User user = (User)request.getUserPrincipal(); - username = user.getEmail() == null ? "Guest" : user.getEmail(); - } - } - - return createReportFromStacktrace(stackTrace, exceptionMessage, browser, sqlState, requestURL, referrerURL, username, target, level, errorCode); - } - - // Prepares a client-side stack trace for hashing - private static void prepareClientStackTrace(StringBuilder sb, BufferedReader reader) throws IOException - { - // Skip message part of the exception for hashing as this can differ easily between browsers - reader.readLine(); - String line; - - // Our client-side applications generate resource paths that are prefixed with webpack:// - // where the is configured by the application entry in entryPoints.js. This pattern - // matches against these to coalesce the same error generated by different applications. - Pattern p = Pattern.compile("webpack://[\\w-]+/"); - - while ((line = reader.readLine()) != null) - { - line = line.trim(); - - Matcher m = p.matcher(line); - if (m.find()) - line = m.replaceAll(""); - - sb.append(line); - sb.append("\n"); - } - } - - // Prepares a server-side stack trace for hashing - private static void prepareServerStackTrace(StringBuilder sb, BufferedReader reader) throws IOException - { - String[] ignoreLineNumberList = {"at java.", "at org.apache.", "at javax.", "at sun."}; - String line = reader.readLine(); - - // Strip off the message part of the exception for hashing - if (line != null) - { - int index = line.indexOf(":"); - if (index != -1) - { - sb.append(line, 0, index); - } - else - { - sb.append(line); - } - } - while ((line = reader.readLine()) != null) - { - // Don't include the other threads when de-duping stack traces - if (line.startsWith(SqlDialect.SEPARATOR_BANNER)) - break; - - // Don't include lines that vary based on reflection - if (line.trim().startsWith("at ") && - !line.trim().startsWith("at sun.reflect.") - && !(line.trim().startsWith("...")) - && !line.trim().startsWith("Position: ") // Postgres stack traces can include a third line that indicates the position of the error in the SQL - && !line.trim().startsWith("Detail:") // Postgres stack traces can include a second details line - && !line.trim().startsWith("Detalhe:")) // which is oddly sometimes prefixed by "Detalhe:" instead of "Detail:" - { - // Don't include line numbers that depend on non-labkey version install - if (line.trim().startsWith("Caused by:") && line.indexOf(":", line.indexOf(":") + 1) != -1) - { - line = line.substring(0, line.indexOf(":", line.indexOf(":") + 1)); - } - else - { - for (String ignoreLineNumber : ignoreLineNumberList) - { - if (line.trim().startsWith(ignoreLineNumber) && line.contains("(")) - { - line = line.substring(0, line.lastIndexOf("(")); - break; - } - } - } - - sb.append(line); - sb.append("\n"); - } - } - } - - private static String hashStackTrace(ExceptionStackTrace stackTrace) - { - return hashStackTrace(stackTrace.stackTrace, stackTrace.clientException); - } - - public static String hashStackTrace(String fullStackTrace, boolean isClientException) - { - BufferedReader reader = new BufferedReader(new StringReader(fullStackTrace)); - StringBuilder sb = new StringBuilder(); - - try - { - if (isClientException) - prepareClientStackTrace(sb, reader); - else - prepareServerStackTrace(sb, reader); - } - catch (IOException e) - { - // Shouldn't happen - this is an in-memory source - throw UnexpectedException.wrap(e); - } - - return HashHelpers.hash(sb.toString()); - } - - private record ExceptionStackTrace(String stackTrace, boolean clientException) {} - - /** - * This has been separated from logExceptionToMothership() in order to provide more verbose server-side logging of client context - */ - public static @Nullable String logClientExceptionToMothership( - String fullStackTrace, - String exceptionMessage, - String browser, - String sqlState, - String requestURL, - String referrerURL, - String username - ) - { - String errorCode = null; - ExceptionStackTrace stackTrace = new ExceptionStackTrace(fullStackTrace, true); - - try - { - // Once to labkey.org, if so configured - ExceptionReportingLevel level = getExceptionReportingLevel(); - if (shouldSend(level, false, stackTrace)) - { - errorCode = sendReport(createReportFromStacktrace(stackTrace, exceptionMessage, browser, sqlState, requestURL, referrerURL, username, MothershipReport.Target.remote, level, null)); - } - - // And once to the local server, if so configured - if (isSelfReportExceptions() && shouldSend(ExceptionReportingLevel.HIGH, true, stackTrace)) - { - String newErrorCode = sendReport(createReportFromStacktrace(stackTrace, exceptionMessage, browser, sqlState, requestURL, referrerURL, username, MothershipReport.Target.local, ExceptionReportingLevel.HIGH, errorCode)); - if (null == errorCode) - errorCode = newErrorCode; - } - } - finally - { - String message = "Client exception detected"; - if (null != errorCode) - message += " and logged to mothership with error code " + errorCode + " "; - - message += "\nrequestURL: " + requestURL; - if (null != referrerURL && !referrerURL.equals(requestURL)) - message += "\nreferrerURL: " + referrerURL; - message += "\nbrowser: " + browser; - message += "\n" + stackTrace.stackTrace; - - LOG.error(message); - } - - return errorCode; - } - - private static boolean shouldSend(ExceptionReportingLevel level, boolean local, ExceptionStackTrace stackTrace) - { - boolean send = true; - - if (level == ExceptionReportingLevel.NONE) - { - send = false; - } - else if (local && ModuleLoader.getInstance().isUpgradeInProgress()) - { - LOG.error("Not logging exception to local mothership because upgrade is in progress"); - send = false; - } - // In dev mode, don't report to labkey.org if the Mothership module is installed. - else if (!local && AppProps.getInstance().isDevMode() && MothershipReport.isShowSelfReportExceptions()) - { - send = false; - } - else if (!local) - { - String hash = hashStackTrace(stackTrace); - ExceptionTally tally = EXCEPTION_TALLIES.computeIfAbsent(hash, k -> new ExceptionTally()); - // Once we've reported an exception 5 times within this server session, don't report it more than every 5 minutes - if (tally._count.incrementAndGet() > 5 && System.currentTimeMillis() - tally._lastReported < 1000 * 60 * 5) - { - LOG.debug("Not logging exception to mothership because it has been reported too many times recently"); - MothershipReport.incrementDroppedExceptionCount(); - send = false; - } - else - { - tally._lastReported = System.currentTimeMillis(); - } - } - - return send; - } - - @NotNull - private static MothershipReport createReportFromStacktrace( - ExceptionStackTrace exceptionStackTrace, - String exceptionMessage, - String browser, - String sqlState, - String requestURL, - String referrerURL, - String username, - MothershipReport.Target target, - ExceptionReportingLevel level, - @Nullable String errorCode - ) - { - try - { - MothershipReport report = new MothershipReport(MothershipReport.Type.ReportException, target, errorCode); - report.addServerSessionParams(); - report.addParam("stackTrace", exceptionStackTrace.stackTrace); - report.addParam("clientException", exceptionStackTrace.clientException); - report.addParam("sqlState", sqlState); - report.addParam("browser", browser); - - Map> modulesMap = new HashMap<>(); - UsageReportingLevel.putModulesBuildInfo(modulesMap); - report.setMetrics(Collections.singletonMap("modules", modulesMap)); - - if (requestURL != null) - { - try - { - ActionURL url = new ActionURL(requestURL); - report.addParam("pageflowName", url.getController()); - report.addParam("pageflowAction", url.getAction()); - } - catch (IllegalArgumentException x) - { - // fall through - } - } - - if (level == ExceptionReportingLevel.MEDIUM || level == ExceptionReportingLevel.HIGH) - { - report.addParam("exceptionMessage", exceptionMessage); - report.addParam("requestURL", requestURL); - report.addParam("referrerURL", referrerURL); - report.addHostName(); - - if (level == ExceptionReportingLevel.HIGH) - { - if (username == null) - username = "NOT SET"; - report.addParam("username", username); - } - } - - return report; - } - catch (MalformedURLException | URISyntaxException e) - { - throw new RuntimeException(e); - } - } - - public static boolean isIgnorable(Throwable ex) - { - Map, String> decorations = getExceptionDecorations(ex); - - return ex == null || - null != decorations.get(ExceptionInfo.SkipMothershipLogging) || - ex instanceof SkipMothershipLogging || - isClientAbortException(ex) || - (ex instanceof IllegalStateException && "Page needs a session and none is available".equalsIgnoreCase(ex.getMessage())); - } - - static class WebPartErrorView extends WebPartView - { - private final ErrorRenderer _renderer; - - WebPartErrorView(ErrorRenderer renderer) - { - super(FrameType.DIV); - _renderer = renderer; - } - - @Override - protected void renderView(Object model, HttpServletRequest request, HttpServletResponse response) throws Exception - { - PrintWriter out = response.getWriter(); - _renderer.renderStart(out); - _renderer.renderContent(out, request, null); - _renderer.renderEnd(out); - } - } - - public static boolean isClientAbortException(Throwable ex) - { - if (ex != null) - { - String className = ex.getClass().getName(); - if (className.endsWith("SocketTimeoutException") || - className.endsWith("CancelledException") || - className.endsWith("ClientAbortException") || - className.endsWith("FileUploadException")) - { - LOG.trace("Client abort exception", ex); - return true; - } - if (ex.getClass().equals(IllegalStateException.class) && ex.getMessage() != null && - ("Cannot create a session after the response has been committed".equals(ex.getMessage()) || - "Cannot call sendError() after the response has been committed".equals(ex.getMessage()) || - ex.getMessage().contains("Session already invalidated"))) - - { - LOG.trace("Client abort exception", ex); - return true; - } - if (ex.getClass().equals(SocketException.class) && "Connection reset".equalsIgnoreCase(ex.getMessage())) - { - LOG.trace("Client abort exception", ex); - return true; - } - // Bug 15371 and 34605 - if (ex.getClass().equals(IOException.class) && ex.getMessage() != null && (ex.getMessage().contains("disconnected client") || ex.getMessage().contains("Socket read failed"))) - { - LOG.trace("Client abort exception", ex); - return true; - } - // Bug 32056 - if (ex.getClass().equals(EOFException.class)) - { - LOG.trace("Client abort exception", ex); - return true; - } - - if (ex instanceof AbortedRequestException || ex instanceof DbScope.ConnectionAlreadyReleasedException) - { - return true; - } - - // Recurse to see if the root exception is a client abort exception - if (ex.getCause() != ex) - { - return isClientAbortException(ex.getCause()); - } - } - return false; - } - - public static ActionURL handleException(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, Throwable ex, @Nullable String message, boolean startupFailure) - { - return handleException(request, response, ex, message, startupFailure, SearchService.get(), LOG, null, null); - } - - // This is called by SpringActionController (to display unhandled exceptions) and called directly by AuthFilter.doFilter() (to display startup errors and bypass normal request handling) - public static ActionURL handleException(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, Throwable ex, @Nullable String message, boolean startupFailure, ViewContext context, @Nullable PageConfig pageConfig) - { - return handleException(request, response, ex, message, startupFailure, SearchService.get(), LOG, context, pageConfig); - } - - static ActionURL handleException(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, Throwable ex, @Nullable String message, boolean startupFailure, - SearchService ss, Logger log, ViewContext context, @Nullable PageConfig pageConfig) - { - try - { - DbScope.closeAllConnectionsForCurrentThread(); - } - catch (Throwable t) - { - // This will fail if, for example, we're dealing with a startup exception - } - - // First, get rid of RuntimeException, InvocationTargetException, etc. wrappers - ex = unwrapException(ex); - request.setAttribute(REQUEST_EXCEPTION_ATTRIBUTE, ex); - - // unhandledException indicates whether the exception is expected or not - // assume it is unhandled and clear for Unauthorized, NotFound, etc - Throwable unhandledException = ex; - String responseStatusMessage = null; // set to use non default status message - - if (isClientAbortException(ex)) - { - // The client dropped the connection. We don't care about this case, - // and don't need to send an error back to the browser either. - return null; - } - - int responseStatus = HttpServletResponse.SC_OK; - ErrorRenderer.ErrorType errorType = ErrorRenderer.ErrorType.execution; - - if (response.isCommitted()) - { - // if we can't reset(), flushing might make it slightly less likely to send half-written attributes etc - try {response.getOutputStream().flush();} catch (Exception ignored) {} - try {response.getWriter().flush();} catch (Exception ignored) {} - try {response.flushBuffer();} catch (Exception ignored) {} - } - - // Do redirects before response.reset() otherwise we'll lose cookies (e.g., login page) - if (ex instanceof RedirectException rex) - { - String url = rex.getURL(); - doErrorRedirect(response, url, rex.getHttpStatusCode()); - return null; - } - - if (!response.isCommitted()) - { - try - { - response.reset(); - // mostly to make security scanners happy - if (ModuleLoader.getInstance().isStartupComplete()) - { - if (!"ALLOW".equals(AppProps.getInstance().getXFrameOption())) - response.setHeader("X-Frame-Options", AppProps.getInstance().getXFrameOption()); - response.setHeader("X-Content-Type-Options", "nosniff"); - } - } - catch (IllegalStateException x) - { - // This is fine, just can't clear the existing response as its - // been at least partially written back to the client - } - } - - User user = (User) request.getUserPrincipal(); - boolean isGET = "GET".equals(request.getMethod()); - Map headers = new TreeMap<>(); - - if (ContextListener.isShuttingDown()) - { - try - { - response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, "The server is shutting down"); - } - catch (IOException e) - { - // Do nothing - } - return null; - } - else if (ex instanceof ApiUsageException) - { - responseStatus = HttpServletResponse.SC_BAD_REQUEST; - errorType = ErrorRenderer.ErrorType.notFound; - if (ex.getMessage() != null) - { - message = ex.getMessage(); - responseStatusMessage = message; - } - else - message = responseStatus + ": API usage error - bad request"; - } - else if (ex instanceof BadRequestException) - { - responseStatus = ((BadRequestException) ex).getStatus(); - errorType = ErrorRenderer.ErrorType.notFound; - message = ex.getMessage(); - unhandledException = null; - } - else if (ex instanceof NotFoundException) - { - responseStatus = HttpServletResponse.SC_NOT_FOUND; - errorType = ErrorRenderer.ErrorType.notFound; - if (ex.getMessage() != null) - { - message = ex.getMessage(); - responseStatusMessage = message; - } - else - message = responseStatus + ": Page not Found"; - - URLHelper url = (URLHelper)request.getAttribute(ViewServlet.ORIGINAL_URL_URLHELPER); - if (null != url && null != url.getParameter(ActionURL.Param._docid.name())) - { - if (null != ss) - ss.notFound(url); - } - unhandledException = null; - } - else if (ex instanceof UnauthorizedException uae) - { - // This header allows for requests to explicitly ask to not get basic auth headers back - // useful for when the page wants to handle 401s itself - String headerHint = request.getHeader("X-ONUNAUTHORIZED"); - - boolean isGuest = user == null || user.isGuest(); - UnauthorizedException.Type type = uae.getType(); - boolean overrideBasicAuth = "UNAUTHORIZED".equals(headerHint); - boolean isCSRFViolation = uae instanceof CSRFException; - - // check for redirect to login page -- unauthorized guest - if (isGET) - { - // If user has not logged in or agreed to terms, not really unauthorized yet... - if (!isCSRFViolation && isGuest && type == UnauthorizedException.Type.redirectToLogin && !overrideBasicAuth && HttpView.hasCurrentView()) - { - // Issue 43307: If this is a locked project then just show the login page in the root. Register, - // forgot my password, profile update, password reset, etc. aren't going to work right in a locked - // project. - Container c = HttpView.getContextContainer(); - if (c.getLockState().isLocked()) - c = ContainerManager.getRoot(); - - // Issue 43387 - Retain original container info on login redirect URL, even if it resolved to something else - ActionURL loginURL = PageFlowUtil.urlProvider(LoginUrls.class).getLoginURL(c, HttpView.getContextURLHelper()); - Path originalContainerPath = (Path)request.getAttribute(ViewServlet.ORIGINAL_URL_CONTAINER_PATH); - if (originalContainerPath != null) - { - loginURL.setExtraPath(originalContainerPath.toString("/", "")); - } - return loginURL; - } - } - - // we know who you are, you're just forbidden from seeing it (unless bad CSRF, silly kids) - responseStatus = isGuest || isCSRFViolation ? HttpServletResponse.SC_UNAUTHORIZED : HttpServletResponse.SC_FORBIDDEN; - errorType = ErrorRenderer.ErrorType.permission; - - message = ex.getMessage(); - responseStatusMessage = message; - - if (isGuest && type == UnauthorizedException.Type.sendBasicAuth && !overrideBasicAuth) - { - headers.put("WWW-Authenticate", "Basic realm=\"" + LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getDescription() + "\""); - - if (isGET) - message = "You must log in to view this content."; - } - - unhandledException = null; - } - else if (ex instanceof SQLException) - { - responseStatus = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; - message = SqlDialect.GENERIC_ERROR_MESSAGE; - - if (ex instanceof BatchUpdateException) - { - if (null != ((BatchUpdateException)ex).getNextException()) - ex = ((BatchUpdateException)ex).getNextException(); - unhandledException = ex; - } - } - else if (ex instanceof ConfigurationException) - { - errorType = ErrorRenderer.ErrorType.configuration; - } - - if (null == message && null != unhandledException) - { - responseStatus = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; - message = ex.getMessage(); - } - - //don't log unauthorized (basic-auth challenge), forbiddens, or simple not found (404s) - if (null != unhandledException && responseStatus != HttpServletResponse.SC_BAD_REQUEST) - { - log.error("Unhandled exception: " + message, unhandledException); - } - - ApiResponseWriter.Format responseFormat = ApiResponseWriter.getResponseFormat(request, null); - - if (response.isCommitted()) - { - // This is fine, just can't clear the existing response as it has - // been at least partially written back to the client - - if (ex != null) - { - try - { - response.getWriter().println("\"> -->"); - response.getWriter().println(); - response.getWriter().println(); - response.getWriter().println("
");
-                    ex.printStackTrace(response.getWriter());
-                    response.getWriter().println("
"); - } - catch (IOException | IllegalStateException e) - { - // Give up at this point - } - } - } - else if (responseFormat != null) - { - try - { - response.setContentType(request.getContentType()); - response.setCharacterEncoding("utf-8"); - - response.setStatus(responseStatus); - - for (Map.Entry entry : headers.entrySet()) - response.addHeader(entry.getKey(), entry.getValue()); - - ApiSimpleResponse errorResponse = new ApiSimpleResponse("success", false); - - if (responseStatusMessage != null || message != null) - errorResponse.put("exception", Objects.toString(message, responseStatusMessage)); - - ApiResponseWriter writer = responseFormat.createWriter(response, null, null); - - errorResponse.render(writer); - writer.close(); - } - catch (Exception x) - { - log.error("Global.handleException", x); - } - } - else - { - response.setContentType("text/html"); - response.setStatus(responseStatus); - for (Map.Entry entry : headers.entrySet()) - response.addHeader(entry.getKey(), entry.getValue()); - renderErrorPage(ex, responseStatus, message, request, response, context, pageConfig, errorType, log, startupFailure); - } - - return null; - } - - private static void renderErrorPage(Throwable ex, int responseStatus, String message, HttpServletRequest request, HttpServletResponse response, ViewContext context, PageConfig originalConfig, ErrorRenderer.ErrorType errorType, Logger log, boolean startupFailure) - { - ErrorRenderer renderer = getErrorRenderer(responseStatus, message, ex, request, false, startupFailure); - - if (ex instanceof UnauthorizedException) - { - errorType = ErrorRenderer.ErrorType.permission; - } - - if (ex instanceof DavException) - { - errorType = ErrorRenderer.ErrorType.notFound; - } - - if (context == null) - { - // context is null in cases of garbage urls, this helps in rendering error page in App template - context = new ViewContext(); - context.setRequest(request); - context.setResponse(response); - } - - // Setup a fresh PageConfig, as we don't want to pull in ClientDependencies or other config that might have - // been initialized from the "real" page that we ultimately didn't render due to an error - try (var ignored = HttpView.initForRequest(context, request, response)) - { - PageConfig pageConfig = HttpView.currentPageConfig(); - - // Issue 41891: do not add google analytics on error pages. - pageConfig.setAllowTrackingScript(PageConfig.TrueFalse.False); - - if (originalConfig == null) - { - pageConfig.setTemplate(PageConfig.Template.Home); - } - else - { - pageConfig.setTemplate(originalConfig.getTemplate()); - pageConfig.setFrameOption(originalConfig.getFrameOption()); - } - - HttpView errorView = null; - try - { - renderer.setErrorType(errorType); - - Path originalContainerPath = (Path) request.getAttribute(ViewServlet.ORIGINAL_URL_CONTAINER_PATH); - - // Issue 43387 - don't render the full header if the container referenced in the request isn't the same - if (ex instanceof UnauthorizedException && (context.getContainer() == null || !context.getContainer().getParsedPath().equals(originalContainerPath))) - { - renderHeaderlessReactErrorPage(ex, responseStatus, request, response, pageConfig, log, renderer, context, null); - } - else - { - if (HttpView.hasCurrentView()) - { - errorView = pageConfig.getTemplate().getTemplate(context, new ErrorView(renderer), pageConfig); - } - - if (errorView == null) - { - // context can be null for configuration exceptions depending on how far server got through initialization - errorView = PageConfig.Template.Body.getTemplate(new ViewContext(request, response, new ActionURL(ActionURL.getBaseServerURL())), new ErrorView(renderer), pageConfig); - } - - addDependenciesAndRender(responseStatus, pageConfig, errorView, ex, request, response); - } - } - catch (ConfigurationException ce) - { - throw ce; - } - catch (Exception e) - { - // non config exceptions like SqlScriptException that occur during startup - if (null != ModuleLoader.getInstance().getStartupFailure()) - { - throw new ConfigurationException(ex.getMessage(), ex); - } - - renderHeaderlessReactErrorPage(ex, responseStatus, request, response, pageConfig, log, renderer, context, e); - } - } - } - - private static void renderHeaderlessReactErrorPage(Throwable ex, int responseStatus, HttpServletRequest request, HttpServletResponse response, PageConfig pageConfig, Logger log, ErrorRenderer renderer, ViewContext context, @Nullable Exception errorRenderException) - { - HttpView errorView; - // try to render just the react app - try - { - errorView = PageConfig.Template.App.getTemplate(context, new ErrorView(renderer), pageConfig); - addDependenciesAndRender(responseStatus, pageConfig, errorView, ex, request, response); - } - catch (Exception exc) - { - if (errorRenderException != null) - { - log.error("Global.handleException", errorRenderException); - } - log.error("Failed to create App template", exc); - } - } - - public static void renderErrorView(ViewContext context, PageConfig pageConfig, ErrorRenderer.ErrorType errorType, int responseStatus, String message, @Nullable Throwable ex, boolean isPart, boolean isStartupFailure) throws Exception - { - ErrorRenderer renderer = ExceptionUtil.getErrorRenderer(responseStatus, message, ex, context.getRequest(), isPart, isStartupFailure); - renderer.setErrorType(errorType); - HttpView errorView = PageConfig.Template.App.getTemplate(context, new ErrorView(renderer), pageConfig); - if (null != errorView) - { - addDependenciesAndRender(responseStatus, pageConfig, errorView, ex, context.getRequest(), context.getResponse()); - } - } - - private static void addDependenciesAndRender(int responseStatus, PageConfig pageConfig, HttpView errorView, @Nullable Throwable ex, HttpServletRequest request, HttpServletResponse response) throws Exception - { - if (null == errorView) - { - LOG.error("Failed to create errorView in response to exception", ex); - return; - } - pageConfig.addClientDependencies(errorView.getClientDependencies()); - - var title = responseStatus + ": " + ErrorView.ERROR_PAGE_TITLE; - if (null != ex) - { - if (null != ex.getMessage()) - { - title += " -- " + ex.getMessage(); - } - else - { - title += " -- " + ex; - } - } - pageConfig.setTitle(title, false); - response.setStatus(responseStatus); - errorView.getView().render(errorView.getModel(), request, response); - } - - // Temporary redirect - public static void doErrorRedirect(HttpServletResponse response, String url) - { - doErrorRedirect(response, url, HttpServletResponse.SC_MOVED_TEMPORARILY); - } - - // Pass in HTTP status code to designate temporary vs. permanent redirect - private static void doErrorRedirect(HttpServletResponse response, String url, int httpStatusCode) - { - response.setStatus(httpStatusCode); - response.setDateHeader("Expires", 0); - response.setHeader("Location", url); - response.setContentType("text/html; charset=UTF-8"); - - // backup strategy! - try - { - if (response.isCommitted()) - { - PrintWriter out = response.getWriter(); - out.println("\"'>-->"); - } - } - catch (IOException x) - { - LOG.error("doErrorRedirect", x); - } - } - - public enum ExceptionInfo - { - ResolveURL, // suggestion for where to fix this e.g. sourceQuery.view - ResolveText, // text to go with the ResolveURL - HelpURL, - DialectSQL, - LabkeySQL, - QueryName, - QuerySchema, - SkipMothershipLogging, - ExtraMessage - } - - - private final static WeakHashMap, String>> _exceptionDecorations = new WeakHashMap<>(); - - public static boolean decorateException(Throwable t, Enum key, String value, boolean overwrite) - { - t = unwrapException(t); - synchronized (_exceptionDecorations) - { - HashMap, String> m = _exceptionDecorations.computeIfAbsent(t, k -> new HashMap<>()); - if (overwrite || !m.containsKey(key)) - { - LOG.debug("add decoration to " + t.getClass() + "@" + System.identityHashCode(t) + " " + key + "=" + value); - m.put(key,value); - return true; - } - } - return false; - } - - - @NotNull - public static Map, String> getExceptionDecorations(Throwable start) - { - HashMap, String> collect = new HashMap<>(); - LinkedList list = new LinkedList<>(); - - Throwable next = unwrapException(start); - while (null != next) - { - list.addFirst(next); - next = getCause(next); - } - - synchronized (_exceptionDecorations) - { - for (Throwable th : list) - { - HashMap, String> m = _exceptionDecorations.get(th); - if (null != m) - collect.putAll(m); - } - } - return collect; - } - - - @Nullable - public static String getExceptionDecoration(Throwable t, Enum e) - { - // could optimize... - return getExceptionDecorations(t).get(e); - } - - - @NotNull - public static String getExtendedMessage(Throwable t) - { - StringBuilder sb = new StringBuilder(t.toString()); - for (Map.Entry, String> e : getExceptionDecorations(t).entrySet()) - sb.append("\n").append(e.getKey()).append("=").append(e.getValue()); - return sb.toString(); - } - - - @Nullable - public static Throwable getCause(Throwable t) - { - Throwable cause; - if (t instanceof RuntimeSQLException) - cause = ((RuntimeSQLException)t).getSQLException(); - else if (t instanceof ServletException) - cause = ((ServletException)t).getRootCause(); - else if (t instanceof BatchUpdateException) - cause = ((BatchUpdateException)t).getNextException(); - else - cause = t.getCause(); - return cause==t ? null : cause; - } - - - static class ExceptionResponse - { - ActionURL redirect; - MockServletResponse response; - String body; - } - - public static class TestCase extends Assert - { - ExceptionResponse handleIt(final User user, Exception ex) - { - final MockServletResponse res = new MockServletResponse(); - InvocationHandler h = (o, method, objects) -> { - // still calls in 'headers' for validation - res.addHeader(method.getDeclaringClass().getSimpleName() + "." + method.getName(), objects.length==0 ? "" : objects.length==1 ? String.valueOf(objects[0]) : Arrays.toString(objects)); - return null; - }; - SearchService dummySearch = (SearchService) Proxy.newProxyInstance(this.getClass().getClassLoader(), new Class[]{SearchService.class}, h); - Logger dummyLog = new org.apache.logging.log4j.core.Logger((LoggerContext) LogManager.getContext(), "mock logger", LogManager.getLogger("mock logger").getMessageFactory()) - { - @Override - public void debug(Object message) - { - } - @Override - public void debug(Object message, Throwable t) - { - } - @Override - public void error(Object message) - { - } - @Override - public void error(String message, Throwable t) - { - res.addHeader("Logger.error", null!=message? message :null!=t?t.getMessage():""); - } - @Override - public void fatal(Object message) - { - } - @Override - public void fatal(Object message, Throwable t) - { - } - @Override - public void warn(Object message) - { - } - @Override - public void warn(Object message, Throwable t) - { - } - }; - HttpServletRequestWrapper req = new HttpServletRequestWrapper(TestContext.get().getRequest()) - { - @Override - public Principal getUserPrincipal() - { - return user; - } - - @Override - public String getMethod() - { - return "GET"; - } - }; - ExceptionUtil.decorateException(ex, ExceptionInfo.SkipMothershipLogging, "true", true); - ActionURL url = ExceptionUtil.handleException(req, res, ex, null, false, dummySearch, dummyLog, null, null); - ExceptionResponse ret = new ExceptionResponse(); - ret.redirect = url; - ret.response = res; - ret.body = res.getBodyAsText(); - return ret; - } - - - @Test - public void testUnauthorized() - { - User guest = UserManager.getGuestUser(); - User me = TestContext.get().getUser(); - ExceptionResponse answer; - - // Guest Unauthorized - answer = handleIt(guest, new UnauthorizedException("Not on my watch")); - assertNotNull("expect return url for login redirect", answer.redirect); - assertEquals(0, answer.response.status); // status not set - - // Non-Guest Unauthorized - answer = handleIt(me, new UnauthorizedException("Not on my watch")); - assertNull(answer.redirect); - assertEquals(HttpServletResponse.SC_FORBIDDEN, answer.response.status); - - // Guest Basic Unauthorized - answer = handleIt(guest, new RequestBasicAuthException()); - assertNull("BasicAuth should not redirect", answer.redirect); - assertEquals(HttpServletResponse.SC_UNAUTHORIZED, answer.response.status); - assertTrue(answer.response.headers.containsKey("WWW-Authenticate")); - - // Non-Guest Basic Unauthorized - answer = handleIt(me, new RequestBasicAuthException()); - assertNull("BasicAuth should not redirect", answer.redirect); - assertEquals(HttpServletResponse.SC_FORBIDDEN, answer.response.status); - assertFalse(answer.response.headers.containsKey("WWW-Authenticate")); - - // Guest CSRF - answer = handleIt(guest, new CSRFException(TestContext.get().getRequest())); - assertNull(answer.redirect); - assertEquals(HttpServletResponse.SC_UNAUTHORIZED, answer.response.status); - - // Non-Guest CSRF - answer = handleIt(me, new CSRFException(TestContext.get().getRequest())); - assertNull(answer.redirect); - assertEquals(HttpServletResponse.SC_UNAUTHORIZED, answer.response.status); - } - - @Test - public void testRedirect() - { - User guest = UserManager.getGuestUser(); - ExceptionResponse answer; - - ActionURL url = new ActionURL("controller", "action", JunitUtil.getTestContainer()); - answer = handleIt(guest, new RedirectException(url)); - assertNull(answer.redirect); - assertEquals(HttpServletResponse.SC_MOVED_TEMPORARILY, answer.response.status); - assertTrue(answer.response.headers.containsKey("Location")); - } - - @Test - public void testNotFound() - { - User guest = UserManager.getGuestUser(); - ExceptionResponse answer; - - answer = handleIt(guest, new NotFoundException("Not here")); - assertNull("not found does not redirect", answer.redirect); - assertEquals(HttpServletResponse.SC_NOT_FOUND, answer.response.status); - assertTrue(answer.body.contains("Not here")); - - // simulate a search result not found - HttpServletRequest req = TestContext.get().getRequest(); - ActionURL orig = new ActionURL("controller", "action", JunitUtil.getTestContainer()); - orig.addParameter(ActionURL.Param._docid, "fred"); - req.setAttribute(ViewServlet.ORIGINAL_URL_URLHELPER, orig); - - answer = handleIt(guest, new NotFoundException("Not here")); - assertNull("not found does not redirect", answer.redirect); - assertEquals(HttpServletResponse.SC_NOT_FOUND, answer.response.status); - assertTrue(answer.body.contains("Not here")); - assertTrue(answer.response.headers.containsKey("SearchService.notFound")); - } - - @Test - public void testServerError() - { - User me = TestContext.get().getUser(); - ExceptionResponse answer; - - answer = handleIt(me, new NullPointerException()); - assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, answer.response.status); - assertTrue(answer.response.headers.containsKey("Logger.error")); - } - - @Test - public void testUnwrap() - { - - } - } - - /** - * Mock response for testing purposes. - */ - static class MockServletResponse implements HttpServletResponse - { - Map headers = new TreeMap<>(); - int status = 0; - String message = null; - String redirect = null; - String contentType = null; - String characterEncoding = null; - long contentLength = 0; - ByteArrayOutputStream os = new ByteArrayOutputStream(); - Locale locale = null; - PrintWriter printWriter = PrintWriters.getPrintWriter(os); - - - ServletOutputStream servletOutputStream = new ServletOutputStream() - { - @Override - public void write(int i) - { - os.write(i); - } - - @Override - public boolean isReady() - { - return true; - } - - @Override - public void setWriteListener(WriteListener writeListener) - { - throw new UnsupportedOperationException(); - } - }; - - - @Override - public void addCookie(Cookie cookie) - { - } - - @Override - public boolean containsHeader(String s) - { - return headers.containsKey(s); - } - - @Override - public String encodeURL(String s) - { - throw new IllegalStateException(); - } - - @Override - public String encodeRedirectURL(String s) - { - throw new IllegalStateException(); - } - - @Override - public void sendError(int i, String s) - { - status = i; - message = s; - } - - @Override - public void sendError(int i) - { - status = i; - } - - @Override - public void sendRedirect(String s) - { - redirect = s; - } - - @Override - public void setDateHeader(String s, long l) - { - headers.put(s, DateUtil.toISO(l)); - } - - @Override - public void addDateHeader(String s, long l) - { - headers.put(s, DateUtil.toISO(l)); - } - - @Override - public void setHeader(String s, String s1) - { - headers.put(s,s1); - } - - @Override - public void addHeader(String s, String s1) - { - headers.put(s,s1); - } - - @Override - public void setIntHeader(String s, int i) - { - headers.put(s,String.valueOf(i)); - } - - @Override - public void addIntHeader(String s, int i) - { - headers.put(s,String.valueOf(i)); - } - - @Override - public void setStatus(int i) - { - status = i; - } - - @Override - public String getCharacterEncoding() - { - return characterEncoding; - } - - @Override - public String getContentType() - { - return contentType; - } - - @Override - public ServletOutputStream getOutputStream() - { - return servletOutputStream; - } - - @Override - public PrintWriter getWriter() - { - return printWriter; - } - - @Override - public void setCharacterEncoding(String s) - { - characterEncoding = s; - } - - @Override - public void setContentLength(int i) - { - contentLength = i; - } - - @Override - public void setContentLengthLong(long len) - { - contentLength = len; - } - - @Override - public void setContentType(String s) - { - contentType = s; - } - - @Override - public void setBufferSize(int i) - { - } - - @Override - public int getBufferSize() - { - return 0; - } - - @Override - public void flushBuffer() - { - } - - @Override - public void resetBuffer() - { - printWriter.flush(); - os.reset(); - } - - @Override - public boolean isCommitted() - { - return false; - } - - @Override - public void reset() - { - resetBuffer(); - status = 0; - message = null; - // headers.clear(); - } - - @Override - public void setLocale(Locale locale) - { - this.locale = locale; - } - - @Override - public Locale getLocale() - { - return locale; - } - - public String getBodyAsText() - { - printWriter.flush(); - return new String(os.toByteArray(), 0, os.size()); - } - - @Override - public int getStatus() - { - return status; - } - - @Override - public String getHeader(String s) - { - return headers.get(s); - } - - @Override - public Collection getHeaders(String s) - { - return null; // TODO - } - - @Override - public Collection getHeaderNames() - { - return headers.keySet(); - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.util; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.action.ApiResponseWriter; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.miniprofiler.MiniProfiler; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.LoginUrls; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.BadRequestException; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.RedirectException; +import org.labkey.api.view.RequestBasicAuthException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.ViewServlet; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.PageConfig; +import org.labkey.api.webdav.DavException; +import org.labkey.api.writer.PrintWriters; +import org.springframework.dao.DataAccessResourceFailureException; + +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.WriteListener; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; +import jakarta.servlet.http.HttpServletResponse; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; +import java.io.StringWriter; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.net.MalformedURLException; +import java.net.SocketException; +import java.net.URISyntaxException; +import java.security.Principal; +import java.sql.BatchUpdateException; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; +import java.util.WeakHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ExceptionUtil +{ + public static final String REQUEST_EXCEPTION_ATTRIBUTE = ExceptionUtil.class.getName() + "$exception"; + public static final String CALCULATED_COLUMN_SQL_TAG = "/* CALCULATED-EXPRESSION-COLUMN-QUERY */"; + + private static final JobRunner JOB_RUNNER = new JobRunner("Mothership Reporting", 1); + private static final Logger LOG = LogHelper.getLogger(ExceptionUtil.class, "Handles rendering of errors during requests"); + /** + * Remember all the exceptions we've seen since the server started up. + * Key is the exception's hash as calculated by {@link #hashStackTrace(ExceptionStackTrace)}. + */ + private static final Map EXCEPTION_TALLIES = Collections.synchronizedMap(new HashMap<>()); + + private static class ExceptionTally + { + /** Total number of times the exception has happened */ + private final AtomicInteger _count = new AtomicInteger(0); + /** Timestamp of the last time we reported it */ + private long _lastReported = System.currentTimeMillis(); + } + + private ExceptionUtil() + { + } + + public static String renderStackTrace(@Nullable StackTraceElement[] stackTrace) + { + return renderStackTrace(stackTrace, 2); + } + + public static String renderStackTrace(@Nullable StackTraceElement[] stackTrace, int linesToSkip) + { + if (stackTrace == null) + { + return MiniProfiler.NO_STACK_TRACE_AVAILABLE; + } + StringBuilder trace = new StringBuilder(); + + for (int i = linesToSkip; i < stackTrace.length; i++) + { + String line = String.valueOf(stackTrace[i]); + if (line.startsWith("javax.servlet.http.HttpServlet.service(")) + break; + trace.append("\n\tat "); + trace.append(line); + } + + return trace.toString(); + } + + @NotNull + public static Throwable unwrapException(@NotNull Throwable ex) + { + Throwable cause=ex; + + while (null != cause) + { + ex = cause; + cause = null; + + if (ex.getClass() == RuntimeException.class || ex.getClass() == UnexpectedException.class || ex.getClass() == RuntimeSQLException.class || ex instanceof InvocationTargetException || ex instanceof com.google.gwt.user.server.rpc.UnexpectedException) + { + cause = ex.getCause(); + } + else if (ex.getClass() == ServletException.class && ((ServletException)ex).getRootCause() != null) + { + ex = ((ServletException)ex).getRootCause(); + } + else if (ex instanceof BatchUpdateException) + { + cause = ((BatchUpdateException)ex).getNextException(); + } + } + + return ex; + } + + public static HtmlString renderException(Throwable e) + { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String s = PageFlowUtil.filter(sw.toString()); + s = s.replaceAll(" ", " "); + s = s.replaceAll("\t", "       "); + return HtmlString.unsafe("
\n" + s + "
\n"); + } + + public static HtmlString getUnauthorizedMessage(ViewContext context) + { + return HtmlString.unsafe("
" + + (context.getUser().isGuest() ? "Please sign in to see this data." : "You do not have permission to see this data.") + + "
"); + } + + public static WebPartView getErrorWebPartView(int responseStatus, String message, Throwable ex, + HttpServletRequest request) + { + ErrorRenderer renderer = getErrorRenderer(responseStatus, message, ex, request, true, false); + return new WebPartErrorView(renderer); + } + + + public static ErrorRenderer getErrorRenderer(int responseStatus, String message, Throwable ex, + @Nullable HttpServletRequest request, boolean isPart, boolean isStartupFailure) + { + String errorCode = null; + if (!isStartupFailure && responseStatus == HttpServletResponse.SC_INTERNAL_SERVER_ERROR) + { + errorCode = logExceptionToMothership(request, ex); + } + + if (isPart) + return new WebPartErrorRenderer(responseStatus, errorCode, message, ex, isStartupFailure); + else + return new ErrorRenderer(responseStatus, errorCode, message, ex, isStartupFailure); + } + + private static ExceptionReportingLevel getExceptionReportingLevel() + { + // Assume reporting level HIGH during initial install. Admin hasn't made a choice yet plus early exceptions + // (e.g., before root container is created) will cause AppProps to throw. + boolean installing = ModuleLoader.getInstance().isUpgradeRequired() && ModuleLoader.getInstance().isNewInstall(); + return installing ? ExceptionReportingLevel.HIGH : AppProps.getInstance().getExceptionReportingLevel(); + } + + private static boolean isSelfReportExceptions() + { + // Assume false during initial install, as we likely not far enough along to be able to store the exception + boolean installing = ModuleLoader.getInstance().isUpgradeRequired() && ModuleLoader.getInstance().isNewInstall(); + return !installing && AppProps.getInstance().isSelfReportExceptions(); + } + + /** @param request may be null if this is coming from a background thread or init */ + public static String logExceptionToMothership(@Nullable HttpServletRequest request, Throwable ex) + { + return logExceptionToMothership(request, ex, true); + } + + /** @param request may be null if this is coming from a background thread or init */ + public static String logExceptionToMothership(@Nullable HttpServletRequest request, Throwable ex, boolean writeToLog4J) + { + if (ContextListener.isShuttingDown()) + return null; + + ex = unwrapException(ex); + + if (isIgnorable(ex)) + return null; + + String requestURL = request == null ? null : (String) request.getAttribute(ViewServlet.ORIGINAL_URL_STRING); + // Need this extra check to make sure we're not in an infinite loop if there's + // an exception when trying to submit an exception + if (requestURL != null && MothershipReport.isMothershipExceptionReport(requestURL)) + return null; + + String extraInfo = ""; + String sqlState = null; + for (Throwable t = ex ; t != null ; t = t.getCause()) + { + if (t instanceof DataAccessResourceFailureException) + { + // Don't report exceptions from database connectivity issues + return null; + } + if (t instanceof RuntimeSQLException runtimeSQLException) + { + // Unwrap RuntimeSQLExceptions + t = runtimeSQLException.getSQLException(); + } + + if (t instanceof SQLException sqlException) + { + if (sqlException.getMessage() != null && sqlException.getMessage().contains("terminating connection due to administrator command")) + { + // Don't report exceptions from Postgres shutting down + return null; + } + sqlState = sqlException.getSQLState(); + String extraSqlInfo = CoreSchema.getInstance().getSqlDialect().getExtraInfo(sqlException); + if (extraSqlInfo != null) + { + extraInfo = extraSqlInfo; + } + } + + if (t instanceof DeadlockPreventingException) + { + String extraSqlInfo = CoreSchema.getInstance().getSqlDialect().getOtherDatabaseThreads(); + if (extraSqlInfo != null) + { + extraInfo = extraSqlInfo; + } + } + + if (sqlState != null) + break; + } + + String errorCode = null; + try + { + // Once to labkey.org, if so configured + errorCode = sendReport(createReportFromThrowable(request, ex, requestURL, MothershipReport.Target.remote, getExceptionReportingLevel(), null, sqlState, extraInfo)); + + // And once to the local server, if so configured. If submitting to both labkey.org and the local server, the errorCode will be the same. + if (isSelfReportExceptions()) + { + String newErrorCode = sendReport(createReportFromThrowable(request, ex, requestURL, MothershipReport.Target.local, ExceptionReportingLevel.HIGH, errorCode, sqlState, extraInfo)); + if (null == errorCode) + errorCode = newErrorCode; // which may still be null, if server is configured to not send reports to either location. + } + } + finally + { + if (writeToLog4J) + { + String message = "Exception detected"; + if (null != errorCode) + message += " and logged to mothership with error code " + errorCode; + String decorations = getExtendedMessage(ex); + + if (!extraInfo.isBlank() || !decorations.isBlank()) + { + message += "\nAdditional exception info:"; + if (!extraInfo.isBlank()) + { + message += "\n" + extraInfo; + } + if (!decorations.isBlank()) + { + message += "\n" + decorations; + } + if (HttpView.hasCurrentView()) + { + ViewContext viewContext = HttpView.currentContext(); + message += "\nCurrent URL: " + viewContext.getActionURL(); + if (null != viewContext.getUser()) + { + message += "\nCurrent user: " + (viewContext.getUser().isGuest() ? "Guest" : viewContext.getUser().getEmail()); + } + } + + LOG.error(message, ex); + } + } + } + + return errorCode; + } + + private static String sendReport(MothershipReport report) + { + if (null != report) + { + JOB_RUNNER.execute(report); + return report.getErrorCode(); + } + return null; + } + + /** Figure out exactly what text for the stack trace and other details we should submit */ + @Nullable + public static MothershipReport createReportFromThrowable( + @Nullable HttpServletRequest request, + Throwable ex, + String requestURL, + MothershipReport.Target target, + ExceptionReportingLevel level, + @Nullable String errorCode, + @Nullable String sqlState, + @Nullable String extraInfo + ) + { + Map, String> decorations = getExceptionDecorations(ex); + + String exceptionMessage = null; + if (!decorations.isEmpty() && (level == ExceptionReportingLevel.MEDIUM || level == ExceptionReportingLevel.HIGH)) + exceptionMessage = getExtendedMessage(ex); + + StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter, true); + ex.printStackTrace(printWriter); + if (ex instanceof ServletException servletException && servletException.getRootCause() != null) + { + printWriter.println("Nested ServletException cause is:"); + servletException.getRootCause().printStackTrace(printWriter); + } + String browser = request == null ? null : request.getHeader("User-Agent"); + + ExceptionStackTrace stackTrace = new ExceptionStackTrace(stringWriter.getBuffer().toString(), false); + + if (!shouldSend(level, target.isLocal(), stackTrace)) + return null; + + if (extraInfo != null) + stackTrace = new ExceptionStackTrace(stackTrace.stackTrace + "\n" + extraInfo, false); + + String referrerURL = null; + String username = "NOT SET"; + if (request != null) + { + referrerURL = request.getHeader("Referer"); + if (request.getUserPrincipal() != null) + { + User user = (User)request.getUserPrincipal(); + username = user.getEmail() == null ? "Guest" : user.getEmail(); + } + } + + return createReportFromStacktrace(stackTrace, exceptionMessage, browser, sqlState, requestURL, referrerURL, username, target, level, errorCode); + } + + // Prepares a client-side stack trace for hashing + private static void prepareClientStackTrace(StringBuilder sb, BufferedReader reader) throws IOException + { + // Skip message part of the exception for hashing as this can differ easily between browsers + reader.readLine(); + String line; + + // Our client-side applications generate resource paths that are prefixed with webpack:// + // where the is configured by the application entry in entryPoints.js. This pattern + // matches against these to coalesce the same error generated by different applications. + Pattern p = Pattern.compile("webpack://[\\w-]+/"); + + while ((line = reader.readLine()) != null) + { + line = line.trim(); + + Matcher m = p.matcher(line); + if (m.find()) + line = m.replaceAll(""); + + sb.append(line); + sb.append("\n"); + } + } + + // Prepares a server-side stack trace for hashing + private static void prepareServerStackTrace(StringBuilder sb, BufferedReader reader) throws IOException + { + String[] ignoreLineNumberList = {"at java.", "at org.apache.", "at javax.", "at sun."}; + String line = reader.readLine(); + + // Strip off the message part of the exception for hashing + if (line != null) + { + int index = line.indexOf(":"); + if (index != -1) + { + sb.append(line, 0, index); + } + else + { + sb.append(line); + } + } + while ((line = reader.readLine()) != null) + { + // Don't include the other threads when de-duping stack traces + if (line.startsWith(SqlDialect.SEPARATOR_BANNER)) + break; + + // Don't include lines that vary based on reflection + if (line.trim().startsWith("at ") && + !line.trim().startsWith("at sun.reflect.") + && !(line.trim().startsWith("...")) + && !line.trim().startsWith("Position: ") // Postgres stack traces can include a third line that indicates the position of the error in the SQL + && !line.trim().startsWith("Detail:") // Postgres stack traces can include a second details line + && !line.trim().startsWith("Detalhe:")) // which is oddly sometimes prefixed by "Detalhe:" instead of "Detail:" + { + // Don't include line numbers that depend on non-labkey version install + if (line.trim().startsWith("Caused by:") && line.indexOf(":", line.indexOf(":") + 1) != -1) + { + line = line.substring(0, line.indexOf(":", line.indexOf(":") + 1)); + } + else + { + for (String ignoreLineNumber : ignoreLineNumberList) + { + if (line.trim().startsWith(ignoreLineNumber) && line.contains("(")) + { + line = line.substring(0, line.lastIndexOf("(")); + break; + } + } + } + + sb.append(line); + sb.append("\n"); + } + } + } + + private static String hashStackTrace(ExceptionStackTrace stackTrace) + { + return hashStackTrace(stackTrace.stackTrace, stackTrace.clientException); + } + + public static String hashStackTrace(String fullStackTrace, boolean isClientException) + { + BufferedReader reader = new BufferedReader(new StringReader(fullStackTrace)); + StringBuilder sb = new StringBuilder(); + + try + { + if (isClientException) + prepareClientStackTrace(sb, reader); + else + prepareServerStackTrace(sb, reader); + } + catch (IOException e) + { + // Shouldn't happen - this is an in-memory source + throw UnexpectedException.wrap(e); + } + + return HashHelpers.hash(sb.toString()); + } + + private record ExceptionStackTrace(String stackTrace, boolean clientException) {} + + /** + * This has been separated from logExceptionToMothership() in order to provide more verbose server-side logging of client context + */ + public static @Nullable String logClientExceptionToMothership( + String fullStackTrace, + String exceptionMessage, + String browser, + String sqlState, + String requestURL, + String referrerURL, + String username + ) + { + String errorCode = null; + ExceptionStackTrace stackTrace = new ExceptionStackTrace(fullStackTrace, true); + + try + { + // Once to labkey.org, if so configured + ExceptionReportingLevel level = getExceptionReportingLevel(); + if (shouldSend(level, false, stackTrace)) + { + errorCode = sendReport(createReportFromStacktrace(stackTrace, exceptionMessage, browser, sqlState, requestURL, referrerURL, username, MothershipReport.Target.remote, level, null)); + } + + // And once to the local server, if so configured + if (isSelfReportExceptions() && shouldSend(ExceptionReportingLevel.HIGH, true, stackTrace)) + { + String newErrorCode = sendReport(createReportFromStacktrace(stackTrace, exceptionMessage, browser, sqlState, requestURL, referrerURL, username, MothershipReport.Target.local, ExceptionReportingLevel.HIGH, errorCode)); + if (null == errorCode) + errorCode = newErrorCode; + } + } + finally + { + String message = "Client exception detected"; + if (null != errorCode) + message += " and logged to mothership with error code " + errorCode + " "; + + message += "\nrequestURL: " + requestURL; + if (null != referrerURL && !referrerURL.equals(requestURL)) + message += "\nreferrerURL: " + referrerURL; + message += "\nbrowser: " + browser; + message += "\n" + stackTrace.stackTrace; + + LOG.error(message); + } + + return errorCode; + } + + private static boolean shouldSend(ExceptionReportingLevel level, boolean local, ExceptionStackTrace stackTrace) + { + boolean send = true; + + if (level == ExceptionReportingLevel.NONE) + { + send = false; + } + else if (local && ModuleLoader.getInstance().isUpgradeInProgress()) + { + LOG.error("Not logging exception to local mothership because upgrade is in progress"); + send = false; + } + // In dev mode, don't report to labkey.org if the Mothership module is installed. + else if (!local && AppProps.getInstance().isDevMode() && MothershipReport.isShowSelfReportExceptions()) + { + send = false; + } + else if (!local) + { + String hash = hashStackTrace(stackTrace); + ExceptionTally tally = EXCEPTION_TALLIES.computeIfAbsent(hash, k -> new ExceptionTally()); + // Once we've reported an exception 5 times within this server session, don't report it more than every 5 minutes + if (tally._count.incrementAndGet() > 5 && System.currentTimeMillis() - tally._lastReported < 1000 * 60 * 5) + { + LOG.debug("Not logging exception to mothership because it has been reported too many times recently"); + MothershipReport.incrementDroppedExceptionCount(); + send = false; + } + else + { + tally._lastReported = System.currentTimeMillis(); + } + } + + return send; + } + + @NotNull + private static MothershipReport createReportFromStacktrace( + ExceptionStackTrace exceptionStackTrace, + String exceptionMessage, + String browser, + String sqlState, + String requestURL, + String referrerURL, + String username, + MothershipReport.Target target, + ExceptionReportingLevel level, + @Nullable String errorCode + ) + { + try + { + MothershipReport report = new MothershipReport(MothershipReport.Type.ReportException, target, errorCode); + report.addServerSessionParams(); + report.addParam("stackTrace", exceptionStackTrace.stackTrace); + report.addParam("clientException", exceptionStackTrace.clientException); + report.addParam("sqlState", sqlState); + report.addParam("browser", browser); + + Map> modulesMap = new HashMap<>(); + UsageReportingLevel.putModulesBuildInfo(modulesMap); + report.setMetrics(Collections.singletonMap("modules", modulesMap)); + + if (requestURL != null) + { + try + { + ActionURL url = new ActionURL(requestURL); + report.addParam("pageflowName", url.getController()); + report.addParam("pageflowAction", url.getAction()); + } + catch (IllegalArgumentException x) + { + // fall through + } + } + + if (level == ExceptionReportingLevel.MEDIUM || level == ExceptionReportingLevel.HIGH) + { + report.addParam("exceptionMessage", exceptionMessage); + report.addParam("requestURL", requestURL); + report.addParam("referrerURL", referrerURL); + report.addHostName(); + + if (level == ExceptionReportingLevel.HIGH) + { + if (username == null) + username = "NOT SET"; + report.addParam("username", username); + } + } + + return report; + } + catch (MalformedURLException | URISyntaxException e) + { + throw new RuntimeException(e); + } + } + + public static boolean isIgnorable(Throwable ex) + { + Map, String> decorations = getExceptionDecorations(ex); + + return ex == null || + null != decorations.get(ExceptionInfo.SkipMothershipLogging) || + ex instanceof SkipMothershipLogging || + isClientAbortException(ex) || + (ex instanceof IllegalStateException && "Page needs a session and none is available".equalsIgnoreCase(ex.getMessage())); + } + + static class WebPartErrorView extends WebPartView + { + private final ErrorRenderer _renderer; + + WebPartErrorView(ErrorRenderer renderer) + { + super(FrameType.DIV); + _renderer = renderer; + } + + @Override + protected void renderView(Object model, HttpServletRequest request, HttpServletResponse response) throws Exception + { + PrintWriter out = response.getWriter(); + _renderer.renderStart(out); + _renderer.renderContent(out, request, null); + _renderer.renderEnd(out); + } + } + + public static boolean isClientAbortException(Throwable ex) + { + if (ex != null) + { + String className = ex.getClass().getName(); + if (className.endsWith("SocketTimeoutException") || + className.endsWith("CancelledException") || + className.endsWith("ClientAbortException") || + className.endsWith("FileUploadException")) + { + LOG.trace("Client abort exception", ex); + return true; + } + if (ex.getClass().equals(IllegalStateException.class) && ex.getMessage() != null && + ("Cannot create a session after the response has been committed".equals(ex.getMessage()) || + "Cannot call sendError() after the response has been committed".equals(ex.getMessage()) || + ex.getMessage().contains("Session already invalidated"))) + + { + LOG.trace("Client abort exception", ex); + return true; + } + if (ex.getClass().equals(SocketException.class) && "Connection reset".equalsIgnoreCase(ex.getMessage())) + { + LOG.trace("Client abort exception", ex); + return true; + } + // Bug 15371 and 34605 + if (ex.getClass().equals(IOException.class) && ex.getMessage() != null && (ex.getMessage().contains("disconnected client") || ex.getMessage().contains("Socket read failed"))) + { + LOG.trace("Client abort exception", ex); + return true; + } + // Bug 32056 + if (ex.getClass().equals(EOFException.class)) + { + LOG.trace("Client abort exception", ex); + return true; + } + + if (ex instanceof AbortedRequestException || ex instanceof DbScope.ConnectionAlreadyReleasedException) + { + return true; + } + + // Recurse to see if the root exception is a client abort exception + if (ex.getCause() != ex) + { + return isClientAbortException(ex.getCause()); + } + } + return false; + } + + public static ActionURL handleException(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, Throwable ex, @Nullable String message, boolean startupFailure) + { + return handleException(request, response, ex, message, startupFailure, SearchService.get(), LOG, null, null); + } + + // This is called by SpringActionController (to display unhandled exceptions) and called directly by AuthFilter.doFilter() (to display startup errors and bypass normal request handling) + public static ActionURL handleException(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, Throwable ex, @Nullable String message, boolean startupFailure, ViewContext context, @Nullable PageConfig pageConfig) + { + return handleException(request, response, ex, message, startupFailure, SearchService.get(), LOG, context, pageConfig); + } + + static ActionURL handleException(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, Throwable ex, @Nullable String message, boolean startupFailure, + @NotNull SearchService ss, Logger log, ViewContext context, @Nullable PageConfig pageConfig) + { + try + { + DbScope.closeAllConnectionsForCurrentThread(); + } + catch (Throwable t) + { + // This will fail if, for example, we're dealing with a startup exception + } + + // First, get rid of RuntimeException, InvocationTargetException, etc. wrappers + ex = unwrapException(ex); + request.setAttribute(REQUEST_EXCEPTION_ATTRIBUTE, ex); + + // unhandledException indicates whether the exception is expected or not + // assume it is unhandled and clear for Unauthorized, NotFound, etc + Throwable unhandledException = ex; + String responseStatusMessage = null; // set to use non default status message + + if (isClientAbortException(ex)) + { + // The client dropped the connection. We don't care about this case, + // and don't need to send an error back to the browser either. + return null; + } + + int responseStatus = HttpServletResponse.SC_OK; + ErrorRenderer.ErrorType errorType = ErrorRenderer.ErrorType.execution; + + if (response.isCommitted()) + { + // if we can't reset(), flushing might make it slightly less likely to send half-written attributes etc + try {response.getOutputStream().flush();} catch (Exception ignored) {} + try {response.getWriter().flush();} catch (Exception ignored) {} + try {response.flushBuffer();} catch (Exception ignored) {} + } + + // Do redirects before response.reset() otherwise we'll lose cookies (e.g., login page) + if (ex instanceof RedirectException rex) + { + String url = rex.getURL(); + doErrorRedirect(response, url, rex.getHttpStatusCode()); + return null; + } + + if (!response.isCommitted()) + { + try + { + response.reset(); + // mostly to make security scanners happy + if (ModuleLoader.getInstance().isStartupComplete()) + { + if (!"ALLOW".equals(AppProps.getInstance().getXFrameOption())) + response.setHeader("X-Frame-Options", AppProps.getInstance().getXFrameOption()); + response.setHeader("X-Content-Type-Options", "nosniff"); + } + } + catch (IllegalStateException x) + { + // This is fine, just can't clear the existing response as its + // been at least partially written back to the client + } + } + + User user = (User) request.getUserPrincipal(); + boolean isGET = "GET".equals(request.getMethod()); + Map headers = new TreeMap<>(); + + if (ContextListener.isShuttingDown()) + { + try + { + response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, "The server is shutting down"); + } + catch (IOException e) + { + // Do nothing + } + return null; + } + else if (ex instanceof ApiUsageException) + { + responseStatus = HttpServletResponse.SC_BAD_REQUEST; + errorType = ErrorRenderer.ErrorType.notFound; + if (ex.getMessage() != null) + { + message = ex.getMessage(); + responseStatusMessage = message; + } + else + message = responseStatus + ": API usage error - bad request"; + } + else if (ex instanceof BadRequestException) + { + responseStatus = ((BadRequestException) ex).getStatus(); + errorType = ErrorRenderer.ErrorType.notFound; + message = ex.getMessage(); + unhandledException = null; + } + else if (ex instanceof NotFoundException) + { + responseStatus = HttpServletResponse.SC_NOT_FOUND; + errorType = ErrorRenderer.ErrorType.notFound; + if (ex.getMessage() != null) + { + message = ex.getMessage(); + responseStatusMessage = message; + } + else + message = responseStatus + ": Page not Found"; + + URLHelper url = (URLHelper)request.getAttribute(ViewServlet.ORIGINAL_URL_URLHELPER); + if (null != url && null != url.getParameter(ActionURL.Param._docid.name())) + { + ss.notFound(url); + } + unhandledException = null; + } + else if (ex instanceof UnauthorizedException uae) + { + // This header allows for requests to explicitly ask to not get basic auth headers back + // useful for when the page wants to handle 401s itself + String headerHint = request.getHeader("X-ONUNAUTHORIZED"); + + boolean isGuest = user == null || user.isGuest(); + UnauthorizedException.Type type = uae.getType(); + boolean overrideBasicAuth = "UNAUTHORIZED".equals(headerHint); + boolean isCSRFViolation = uae instanceof CSRFException; + + // check for redirect to login page -- unauthorized guest + if (isGET) + { + // If user has not logged in or agreed to terms, not really unauthorized yet... + if (!isCSRFViolation && isGuest && type == UnauthorizedException.Type.redirectToLogin && !overrideBasicAuth && HttpView.hasCurrentView()) + { + // Issue 43307: If this is a locked project then just show the login page in the root. Register, + // forgot my password, profile update, password reset, etc. aren't going to work right in a locked + // project. + Container c = HttpView.getContextContainer(); + if (c.getLockState().isLocked()) + c = ContainerManager.getRoot(); + + // Issue 43387 - Retain original container info on login redirect URL, even if it resolved to something else + ActionURL loginURL = PageFlowUtil.urlProvider(LoginUrls.class).getLoginURL(c, HttpView.getContextURLHelper()); + Path originalContainerPath = (Path)request.getAttribute(ViewServlet.ORIGINAL_URL_CONTAINER_PATH); + if (originalContainerPath != null) + { + loginURL.setExtraPath(originalContainerPath.toString("/", "")); + } + return loginURL; + } + } + + // we know who you are, you're just forbidden from seeing it (unless bad CSRF, silly kids) + responseStatus = isGuest || isCSRFViolation ? HttpServletResponse.SC_UNAUTHORIZED : HttpServletResponse.SC_FORBIDDEN; + errorType = ErrorRenderer.ErrorType.permission; + + message = ex.getMessage(); + responseStatusMessage = message; + + if (isGuest && type == UnauthorizedException.Type.sendBasicAuth && !overrideBasicAuth) + { + headers.put("WWW-Authenticate", "Basic realm=\"" + LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getDescription() + "\""); + + if (isGET) + message = "You must log in to view this content."; + } + + unhandledException = null; + } + else if (ex instanceof SQLException) + { + responseStatus = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; + message = SqlDialect.GENERIC_ERROR_MESSAGE; + + if (ex instanceof BatchUpdateException) + { + if (null != ((BatchUpdateException)ex).getNextException()) + ex = ((BatchUpdateException)ex).getNextException(); + unhandledException = ex; + } + } + else if (ex instanceof ConfigurationException) + { + errorType = ErrorRenderer.ErrorType.configuration; + } + + if (null == message && null != unhandledException) + { + responseStatus = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; + message = ex.getMessage(); + } + + //don't log unauthorized (basic-auth challenge), forbiddens, or simple not found (404s) + if (null != unhandledException && responseStatus != HttpServletResponse.SC_BAD_REQUEST) + { + log.error("Unhandled exception: " + message, unhandledException); + } + + ApiResponseWriter.Format responseFormat = ApiResponseWriter.getResponseFormat(request, null); + + if (response.isCommitted()) + { + // This is fine, just can't clear the existing response as it has + // been at least partially written back to the client + + if (ex != null) + { + try + { + response.getWriter().println("\"> -->"); + response.getWriter().println(); + response.getWriter().println(); + response.getWriter().println("
");
+                    ex.printStackTrace(response.getWriter());
+                    response.getWriter().println("
"); + } + catch (IOException | IllegalStateException e) + { + // Give up at this point + } + } + } + else if (responseFormat != null) + { + try + { + response.setContentType(request.getContentType()); + response.setCharacterEncoding("utf-8"); + + response.setStatus(responseStatus); + + for (Map.Entry entry : headers.entrySet()) + response.addHeader(entry.getKey(), entry.getValue()); + + ApiSimpleResponse errorResponse = new ApiSimpleResponse("success", false); + + if (responseStatusMessage != null || message != null) + errorResponse.put("exception", Objects.toString(message, responseStatusMessage)); + + ApiResponseWriter writer = responseFormat.createWriter(response, null, null); + + errorResponse.render(writer); + writer.close(); + } + catch (Exception x) + { + log.error("Global.handleException", x); + } + } + else + { + response.setContentType("text/html"); + response.setStatus(responseStatus); + for (Map.Entry entry : headers.entrySet()) + response.addHeader(entry.getKey(), entry.getValue()); + renderErrorPage(ex, responseStatus, message, request, response, context, pageConfig, errorType, log, startupFailure); + } + + return null; + } + + private static void renderErrorPage(Throwable ex, int responseStatus, String message, HttpServletRequest request, HttpServletResponse response, ViewContext context, PageConfig originalConfig, ErrorRenderer.ErrorType errorType, Logger log, boolean startupFailure) + { + ErrorRenderer renderer = getErrorRenderer(responseStatus, message, ex, request, false, startupFailure); + + if (ex instanceof UnauthorizedException) + { + errorType = ErrorRenderer.ErrorType.permission; + } + + if (ex instanceof DavException) + { + errorType = ErrorRenderer.ErrorType.notFound; + } + + if (context == null) + { + // context is null in cases of garbage urls, this helps in rendering error page in App template + context = new ViewContext(); + context.setRequest(request); + context.setResponse(response); + } + + // Setup a fresh PageConfig, as we don't want to pull in ClientDependencies or other config that might have + // been initialized from the "real" page that we ultimately didn't render due to an error + try (var ignored = HttpView.initForRequest(context, request, response)) + { + PageConfig pageConfig = HttpView.currentPageConfig(); + + // Issue 41891: do not add google analytics on error pages. + pageConfig.setAllowTrackingScript(PageConfig.TrueFalse.False); + + if (originalConfig == null) + { + pageConfig.setTemplate(PageConfig.Template.Home); + } + else + { + pageConfig.setTemplate(originalConfig.getTemplate()); + pageConfig.setFrameOption(originalConfig.getFrameOption()); + } + + HttpView errorView = null; + try + { + renderer.setErrorType(errorType); + + Path originalContainerPath = (Path) request.getAttribute(ViewServlet.ORIGINAL_URL_CONTAINER_PATH); + + // Issue 43387 - don't render the full header if the container referenced in the request isn't the same + if (ex instanceof UnauthorizedException && (context.getContainer() == null || !context.getContainer().getParsedPath().equals(originalContainerPath))) + { + renderHeaderlessReactErrorPage(ex, responseStatus, request, response, pageConfig, log, renderer, context, null); + } + else + { + if (HttpView.hasCurrentView()) + { + errorView = pageConfig.getTemplate().getTemplate(context, new ErrorView(renderer), pageConfig); + } + + if (errorView == null) + { + // context can be null for configuration exceptions depending on how far server got through initialization + errorView = PageConfig.Template.Body.getTemplate(new ViewContext(request, response, new ActionURL(ActionURL.getBaseServerURL())), new ErrorView(renderer), pageConfig); + } + + addDependenciesAndRender(responseStatus, pageConfig, errorView, ex, request, response); + } + } + catch (ConfigurationException ce) + { + throw ce; + } + catch (Exception e) + { + // non config exceptions like SqlScriptException that occur during startup + if (null != ModuleLoader.getInstance().getStartupFailure()) + { + throw new ConfigurationException(ex.getMessage(), ex); + } + + renderHeaderlessReactErrorPage(ex, responseStatus, request, response, pageConfig, log, renderer, context, e); + } + } + } + + private static void renderHeaderlessReactErrorPage(Throwable ex, int responseStatus, HttpServletRequest request, HttpServletResponse response, PageConfig pageConfig, Logger log, ErrorRenderer renderer, ViewContext context, @Nullable Exception errorRenderException) + { + HttpView errorView; + // try to render just the react app + try + { + errorView = PageConfig.Template.App.getTemplate(context, new ErrorView(renderer), pageConfig); + addDependenciesAndRender(responseStatus, pageConfig, errorView, ex, request, response); + } + catch (Exception exc) + { + if (errorRenderException != null) + { + log.error("Global.handleException", errorRenderException); + } + log.error("Failed to create App template", exc); + } + } + + public static void renderErrorView(ViewContext context, PageConfig pageConfig, ErrorRenderer.ErrorType errorType, int responseStatus, String message, @Nullable Throwable ex, boolean isPart, boolean isStartupFailure) throws Exception + { + ErrorRenderer renderer = ExceptionUtil.getErrorRenderer(responseStatus, message, ex, context.getRequest(), isPart, isStartupFailure); + renderer.setErrorType(errorType); + HttpView errorView = PageConfig.Template.App.getTemplate(context, new ErrorView(renderer), pageConfig); + if (null != errorView) + { + addDependenciesAndRender(responseStatus, pageConfig, errorView, ex, context.getRequest(), context.getResponse()); + } + } + + private static void addDependenciesAndRender(int responseStatus, PageConfig pageConfig, HttpView errorView, @Nullable Throwable ex, HttpServletRequest request, HttpServletResponse response) throws Exception + { + if (null == errorView) + { + LOG.error("Failed to create errorView in response to exception", ex); + return; + } + pageConfig.addClientDependencies(errorView.getClientDependencies()); + + var title = responseStatus + ": " + ErrorView.ERROR_PAGE_TITLE; + if (null != ex) + { + if (null != ex.getMessage()) + { + title += " -- " + ex.getMessage(); + } + else + { + title += " -- " + ex; + } + } + pageConfig.setTitle(title, false); + response.setStatus(responseStatus); + errorView.getView().render(errorView.getModel(), request, response); + } + + // Temporary redirect + public static void doErrorRedirect(HttpServletResponse response, String url) + { + doErrorRedirect(response, url, HttpServletResponse.SC_MOVED_TEMPORARILY); + } + + // Pass in HTTP status code to designate temporary vs. permanent redirect + private static void doErrorRedirect(HttpServletResponse response, String url, int httpStatusCode) + { + response.setStatus(httpStatusCode); + response.setDateHeader("Expires", 0); + response.setHeader("Location", url); + response.setContentType("text/html; charset=UTF-8"); + + // backup strategy! + try + { + if (response.isCommitted()) + { + PrintWriter out = response.getWriter(); + out.println("\"'>-->"); + } + } + catch (IOException x) + { + LOG.error("doErrorRedirect", x); + } + } + + public enum ExceptionInfo + { + ResolveURL, // suggestion for where to fix this e.g. sourceQuery.view + ResolveText, // text to go with the ResolveURL + HelpURL, + DialectSQL, + LabkeySQL, + QueryName, + QuerySchema, + SkipMothershipLogging, + ExtraMessage + } + + + private final static WeakHashMap, String>> _exceptionDecorations = new WeakHashMap<>(); + + public static boolean decorateException(Throwable t, Enum key, String value, boolean overwrite) + { + t = unwrapException(t); + synchronized (_exceptionDecorations) + { + HashMap, String> m = _exceptionDecorations.computeIfAbsent(t, k -> new HashMap<>()); + if (overwrite || !m.containsKey(key)) + { + LOG.debug("add decoration to " + t.getClass() + "@" + System.identityHashCode(t) + " " + key + "=" + value); + m.put(key,value); + return true; + } + } + return false; + } + + + @NotNull + public static Map, String> getExceptionDecorations(Throwable start) + { + HashMap, String> collect = new HashMap<>(); + LinkedList list = new LinkedList<>(); + + Throwable next = unwrapException(start); + while (null != next) + { + list.addFirst(next); + next = getCause(next); + } + + synchronized (_exceptionDecorations) + { + for (Throwable th : list) + { + HashMap, String> m = _exceptionDecorations.get(th); + if (null != m) + collect.putAll(m); + } + } + return collect; + } + + + @Nullable + public static String getExceptionDecoration(Throwable t, Enum e) + { + // could optimize... + return getExceptionDecorations(t).get(e); + } + + + @NotNull + public static String getExtendedMessage(Throwable t) + { + StringBuilder sb = new StringBuilder(t.toString()); + for (Map.Entry, String> e : getExceptionDecorations(t).entrySet()) + sb.append("\n").append(e.getKey()).append("=").append(e.getValue()); + return sb.toString(); + } + + + @Nullable + public static Throwable getCause(Throwable t) + { + Throwable cause; + if (t instanceof RuntimeSQLException) + cause = ((RuntimeSQLException)t).getSQLException(); + else if (t instanceof ServletException) + cause = ((ServletException)t).getRootCause(); + else if (t instanceof BatchUpdateException) + cause = ((BatchUpdateException)t).getNextException(); + else + cause = t.getCause(); + return cause==t ? null : cause; + } + + + static class ExceptionResponse + { + ActionURL redirect; + MockServletResponse response; + String body; + } + + public static class TestCase extends Assert + { + ExceptionResponse handleIt(final User user, Exception ex) + { + final MockServletResponse res = new MockServletResponse(); + InvocationHandler h = (o, method, objects) -> { + // still calls in 'headers' for validation + res.addHeader(method.getDeclaringClass().getSimpleName() + "." + method.getName(), objects.length==0 ? "" : objects.length==1 ? String.valueOf(objects[0]) : Arrays.toString(objects)); + return null; + }; + SearchService dummySearch = (SearchService) Proxy.newProxyInstance(this.getClass().getClassLoader(), new Class[]{SearchService.class}, h); + Logger dummyLog = new org.apache.logging.log4j.core.Logger((LoggerContext) LogManager.getContext(), "mock logger", LogManager.getLogger("mock logger").getMessageFactory()) + { + @Override + public void debug(Object message) + { + } + @Override + public void debug(Object message, Throwable t) + { + } + @Override + public void error(Object message) + { + } + @Override + public void error(String message, Throwable t) + { + res.addHeader("Logger.error", null!=message? message :null!=t?t.getMessage():""); + } + @Override + public void fatal(Object message) + { + } + @Override + public void fatal(Object message, Throwable t) + { + } + @Override + public void warn(Object message) + { + } + @Override + public void warn(Object message, Throwable t) + { + } + }; + HttpServletRequestWrapper req = new HttpServletRequestWrapper(TestContext.get().getRequest()) + { + @Override + public Principal getUserPrincipal() + { + return user; + } + + @Override + public String getMethod() + { + return "GET"; + } + }; + ExceptionUtil.decorateException(ex, ExceptionInfo.SkipMothershipLogging, "true", true); + ActionURL url = ExceptionUtil.handleException(req, res, ex, null, false, dummySearch, dummyLog, null, null); + ExceptionResponse ret = new ExceptionResponse(); + ret.redirect = url; + ret.response = res; + ret.body = res.getBodyAsText(); + return ret; + } + + + @Test + public void testUnauthorized() + { + User guest = UserManager.getGuestUser(); + User me = TestContext.get().getUser(); + ExceptionResponse answer; + + // Guest Unauthorized + answer = handleIt(guest, new UnauthorizedException("Not on my watch")); + assertNotNull("expect return url for login redirect", answer.redirect); + assertEquals(0, answer.response.status); // status not set + + // Non-Guest Unauthorized + answer = handleIt(me, new UnauthorizedException("Not on my watch")); + assertNull(answer.redirect); + assertEquals(HttpServletResponse.SC_FORBIDDEN, answer.response.status); + + // Guest Basic Unauthorized + answer = handleIt(guest, new RequestBasicAuthException()); + assertNull("BasicAuth should not redirect", answer.redirect); + assertEquals(HttpServletResponse.SC_UNAUTHORIZED, answer.response.status); + assertTrue(answer.response.headers.containsKey("WWW-Authenticate")); + + // Non-Guest Basic Unauthorized + answer = handleIt(me, new RequestBasicAuthException()); + assertNull("BasicAuth should not redirect", answer.redirect); + assertEquals(HttpServletResponse.SC_FORBIDDEN, answer.response.status); + assertFalse(answer.response.headers.containsKey("WWW-Authenticate")); + + // Guest CSRF + answer = handleIt(guest, new CSRFException(TestContext.get().getRequest())); + assertNull(answer.redirect); + assertEquals(HttpServletResponse.SC_UNAUTHORIZED, answer.response.status); + + // Non-Guest CSRF + answer = handleIt(me, new CSRFException(TestContext.get().getRequest())); + assertNull(answer.redirect); + assertEquals(HttpServletResponse.SC_UNAUTHORIZED, answer.response.status); + } + + @Test + public void testRedirect() + { + User guest = UserManager.getGuestUser(); + ExceptionResponse answer; + + ActionURL url = new ActionURL("controller", "action", JunitUtil.getTestContainer()); + answer = handleIt(guest, new RedirectException(url)); + assertNull(answer.redirect); + assertEquals(HttpServletResponse.SC_MOVED_TEMPORARILY, answer.response.status); + assertTrue(answer.response.headers.containsKey("Location")); + } + + @Test + public void testNotFound() + { + User guest = UserManager.getGuestUser(); + ExceptionResponse answer; + + answer = handleIt(guest, new NotFoundException("Not here")); + assertNull("not found does not redirect", answer.redirect); + assertEquals(HttpServletResponse.SC_NOT_FOUND, answer.response.status); + assertTrue(answer.body.contains("Not here")); + + // simulate a search result not found + HttpServletRequest req = TestContext.get().getRequest(); + ActionURL orig = new ActionURL("controller", "action", JunitUtil.getTestContainer()); + orig.addParameter(ActionURL.Param._docid, "fred"); + req.setAttribute(ViewServlet.ORIGINAL_URL_URLHELPER, orig); + + answer = handleIt(guest, new NotFoundException("Not here")); + assertNull("not found does not redirect", answer.redirect); + assertEquals(HttpServletResponse.SC_NOT_FOUND, answer.response.status); + assertTrue(answer.body.contains("Not here")); + assertTrue(answer.response.headers.containsKey("SearchService.notFound")); + } + + @Test + public void testServerError() + { + User me = TestContext.get().getUser(); + ExceptionResponse answer; + + answer = handleIt(me, new NullPointerException()); + assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, answer.response.status); + assertTrue(answer.response.headers.containsKey("Logger.error")); + } + + @Test + public void testUnwrap() + { + + } + } + + /** + * Mock response for testing purposes. + */ + static class MockServletResponse implements HttpServletResponse + { + Map headers = new TreeMap<>(); + int status = 0; + String message = null; + String redirect = null; + String contentType = null; + String characterEncoding = null; + long contentLength = 0; + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Locale locale = null; + PrintWriter printWriter = PrintWriters.getPrintWriter(os); + + + ServletOutputStream servletOutputStream = new ServletOutputStream() + { + @Override + public void write(int i) + { + os.write(i); + } + + @Override + public boolean isReady() + { + return true; + } + + @Override + public void setWriteListener(WriteListener writeListener) + { + throw new UnsupportedOperationException(); + } + }; + + + @Override + public void addCookie(Cookie cookie) + { + } + + @Override + public boolean containsHeader(String s) + { + return headers.containsKey(s); + } + + @Override + public String encodeURL(String s) + { + throw new IllegalStateException(); + } + + @Override + public String encodeRedirectURL(String s) + { + throw new IllegalStateException(); + } + + @Override + public void sendError(int i, String s) + { + status = i; + message = s; + } + + @Override + public void sendError(int i) + { + status = i; + } + + @Override + public void sendRedirect(String s) + { + redirect = s; + } + + @Override + public void setDateHeader(String s, long l) + { + headers.put(s, DateUtil.toISO(l)); + } + + @Override + public void addDateHeader(String s, long l) + { + headers.put(s, DateUtil.toISO(l)); + } + + @Override + public void setHeader(String s, String s1) + { + headers.put(s,s1); + } + + @Override + public void addHeader(String s, String s1) + { + headers.put(s,s1); + } + + @Override + public void setIntHeader(String s, int i) + { + headers.put(s,String.valueOf(i)); + } + + @Override + public void addIntHeader(String s, int i) + { + headers.put(s,String.valueOf(i)); + } + + @Override + public void setStatus(int i) + { + status = i; + } + + @Override + public String getCharacterEncoding() + { + return characterEncoding; + } + + @Override + public String getContentType() + { + return contentType; + } + + @Override + public ServletOutputStream getOutputStream() + { + return servletOutputStream; + } + + @Override + public PrintWriter getWriter() + { + return printWriter; + } + + @Override + public void setCharacterEncoding(String s) + { + characterEncoding = s; + } + + @Override + public void setContentLength(int i) + { + contentLength = i; + } + + @Override + public void setContentLengthLong(long len) + { + contentLength = len; + } + + @Override + public void setContentType(String s) + { + contentType = s; + } + + @Override + public void setBufferSize(int i) + { + } + + @Override + public int getBufferSize() + { + return 0; + } + + @Override + public void flushBuffer() + { + } + + @Override + public void resetBuffer() + { + printWriter.flush(); + os.reset(); + } + + @Override + public boolean isCommitted() + { + return false; + } + + @Override + public void reset() + { + resetBuffer(); + status = 0; + message = null; + // headers.clear(); + } + + @Override + public void setLocale(Locale locale) + { + this.locale = locale; + } + + @Override + public Locale getLocale() + { + return locale; + } + + public String getBodyAsText() + { + printWriter.flush(); + return new String(os.toByteArray(), 0, os.size()); + } + + @Override + public int getStatus() + { + return status; + } + + @Override + public String getHeader(String s) + { + return headers.get(s); + } + + @Override + public Collection getHeaders(String s) + { + return null; // TODO + } + + @Override + public Collection getHeaderNames() + { + return headers.keySet(); + } + } +} diff --git a/api/src/org/labkey/api/webdav/AbstractWebdavResource.java b/api/src/org/labkey/api/webdav/AbstractWebdavResource.java index e9366d454db..223414bce3f 100644 --- a/api/src/org/labkey/api/webdav/AbstractWebdavResource.java +++ b/api/src/org/labkey/api/webdav/AbstractWebdavResource.java @@ -1,728 +1,727 @@ -/* - * Copyright (c) 2010-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.webdav; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.attachments.Attachment; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.exp.LsidManager; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpObject; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.resource.AbstractResource; -import org.labkey.api.resource.Resource; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.SecurableResource; -import org.labkey.api.security.SecurityLogger; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.security.roles.OwnerRole; -import org.labkey.api.security.roles.Role; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.FileStream; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Path; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.ViewContext; -import org.labkey.api.writer.ContainerUser; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public abstract class AbstractWebdavResource extends AbstractResource implements WebdavResource -{ - private static final String FOLDER_FONT_CLS = "fa fa-folder-o"; - - private SecurableResource _resource; - private List _data = null; - - protected GUID _containerId; - protected String _etag = null; - protected Map _properties = null; - - protected AbstractWebdavResource(Path path) - { - this(path, WebdavService.get().getResolver()); - } - - protected AbstractWebdavResource(Path path, WebdavResolver resolver) - { - super(path, resolver); - } - - protected AbstractWebdavResource(Path folder, String name) - { - this(folder, name, WebdavService.get().getResolver()); - } - - protected AbstractWebdavResource(Path folder, String name, WebdavResolver resolver) - { - super(folder, name, resolver); - } - - protected AbstractWebdavResource(Resource folder, String name) - { - super(folder, name, WebdavService.get().getResolver()); - } - - @Override - public WebdavResolver getResolver() - { - return (WebdavResolver)super.getResolver(); - } - - @Override - public boolean createCollection(User user) throws DavException - { - try - { - return this.getFile() != null && FileUtil.mkdirs(this.getFile(), AppProps.getInstance().isInvalidFilenameUploadBlocked()); - } - catch (IOException e) - { - throw new DavException(e.getCause()); - } - } - - @Override - public WebdavResource parent() - { - Path p = getPath(); - if (p.getNameCount()==0) - return null; - Path parent = p.getParent(); - return WebdavService.get().lookup(parent); - } - - @Override - public WebdavResource find(Path.Part name) - { - return null; - } - - @Override - public Collection list() - { - return Collections.emptyList(); - } - - @Override - public long getCreated() - { - return getLastModified(); - } - - @Override - public User getCreatedBy() - { - List data = getExpData(); - if (data == null || data.isEmpty()) - return null; - - return data.get(0).getCreatedBy(); - } - - @Override - public String getDescription() - { - List data = getExpData(); - if (data == null || data.isEmpty()) - return null; - - return data.get(0).getComment(); - } - - @Override - public User getModifiedBy() - { - List data = getExpData(); - if (data == null || data.isEmpty()) - return null; - - return data.get(0).getModifiedBy(); - } - - @Override - public void setLastIndexed(long indexed, long modified) - { - SearchService ss = SearchService.get(); - if (isFile() && ss != null) - ss.setLastIndexedForPath(getPath(), indexed, modified); - } - - @Override - public String getContentType() - { - if (isCollection()) - return "text/html"; - return PageFlowUtil.getContentTypeFor(getName()); - } - - @Override - public String getAbsolutePath(User user) - { - return null; - } - - @Override - @NotNull - public String getHref(ViewContext context) - { - ActionURL url = null==context ? null : context.getActionURL(); - int port = null==url ? AppProps.getInstance().getServerPort() : url.getPort(); - String host = null==url ? AppProps.getInstance().getServerName() : url.getHost(); - String scheme = null==context ? AppProps.getInstance().getScheme() : url.getScheme(); - boolean defaultPort = "http".equals(scheme) && 80 == port || "https".equals(scheme) && 443 == port; - String portStr = defaultPort ? "" : ":" + port; - return c(scheme + "://" + host + portStr, getLocalHref(context)); - } - - @Override - @NotNull - public String getLocalHref(ViewContext context) - { - String contextPath = null==context ? AppProps.getInstance().getContextPath() : context.getContextPath(); - String href = c(contextPath, getPath().encode()); - if (isCollection() && !href.endsWith("/")) - href += "/"; - return href; - } - - @Override - public String getExecuteHref(ViewContext context) - { - String path = parent().getExecuteHref(context); - path += (path.endsWith("/")?"":"/") + PageFlowUtil.encode(getPath().getName()); - return path; - } - - @Override - public String getIconHref() - { - if (isCollection()) - return AppProps.getInstance().getContextPath() + "/_icons/folder.gif"; - return AppProps.getInstance().getContextPath() + Attachment.getFileIcon(getName()); - } - - @Nullable - @Override - public String getIconFontCls() - { - if (isCollection()) - return FOLDER_FONT_CLS; - return Attachment.getFileIconFontCls(getName()); - } - - @Nullable - @Override - public DirectRequest getDirectGetRequest(ViewContext context, String contentDisposition) - { - return null; - } - - @Nullable - @Override - public DirectRequest getDirectPutRequest(ViewContext context) - { - return null; - } - - @Override - public String getETag(boolean force) - { - long len = 0; - if (null == _etag) - { - try - { - len = getContentLength(); - } - catch (IOException x) - { - /* */ - } - _etag = "W/\"" + len + "-" + getLastModified() + "\""; - } - return _etag; - } - - @Override - public String getETag() - { - return getETag(false); - } - - @Override - public String getMD5(User user) throws IOException - { - return FileUtil.md5sum(getInputStream(user)); - } - - @Override - public Map getProperties() - { - if (null == _properties) - return Collections.emptyMap(); - Map ret = _properties; - assert null != (ret = Collections.unmodifiableMap(ret)); - return ret; - } - - @Override - public Map getMutableProperties() - { - if (null == _properties) - _properties = new HashMap<>(); - return _properties; - } - - @Override - public InputStream getInputStream() throws IOException - { - return getInputStream(null); - } - - /** provides one place to completely block access to the resource */ - protected boolean hasAccess(User user) - { - return true; - } - - protected SecurableResource getSecurableResource() - { - return _resource; - } - - protected void setSecurableResource(SecurableResource resource) - { - _resource = resource; - } - - /** permissions */ - - @Override - public boolean canList(User user, boolean forRead) - { - return canRead(user, forRead); - } - - @Override - public boolean canRead(User user, boolean forRead) - { - // TODO: This looks wrong - if ("/".equals(getPath())) - return true; - try - { - SecurityLogger.indent(getPath() + " AbstractWebdavResource.canRead()"); - if (!hasAccess(user)) - { - SecurityLogger.log("hasAccess()==false", user, null, false); - return false; - } - return getPermissions(user).contains(ReadPermission.class); - } - finally - { - SecurityLogger.outdent(); - } - } - - @Override - public boolean canWrite(User user, boolean forWrite) - { - Set roles = user.equals(getCreatedBy()) ? RoleManager.roleSet(OwnerRole.class) : Set.of(); - return hasAccess(user) && !user.isGuest() && - SecurityManager.hasAllPermissions(null, getSecurableResource(), user, Set.of(UpdatePermission.class), roles); - } - - @Override - public boolean canCreate(User user, boolean forCreate) - { - return hasAccess(user) && !user.isGuest() && - SecurityManager.hasAllPermissions(null, getSecurableResource(), user, Set.of(InsertPermission.class), Set.of()); - } - - @Override - public boolean canCreateCollection(User user, boolean forCreate) - { - return canCreate(user, forCreate); - } - - @Override - public boolean canDelete(User user, boolean forDelete) - { - return canDelete(user, forDelete, null); - } - - @Override - public boolean canDelete(User user, boolean forDelete, /* OUT */ @Nullable List message) - { - if (user.isGuest() || !hasAccess(user)) - return false; - Set> perms = getPermissions(user); - return perms.contains(DeletePermission.class); - } - - @Override - public boolean canRename(User user, boolean forRename) - { - return hasAccess(user) && !user.isGuest() && canCreate(user, forRename) && canDelete(user, forRename, null); - } - - public Set> getPermissions(User user) - { - return SecurityManager.getPermissions(getSecurableResource(), user, Set.of()); - } - - @Override - public boolean delete(User user) throws IOException - { - assert null == user || canDelete(user, true, null); - return false; - } - - @Override - public File getFile() - { - return null; - } - - @Override - public long copyFrom(User user, WebdavResource r) throws IOException, DavException - { - return copyFrom(user, r.getFileStream(user)); - } - - @Override - public void moveFrom(User user, WebdavResource src) throws IOException, DavException - { - copyFrom(user, src); - src.delete(user); - } - - @Override - @NotNull - public Collection getHistory() - { - return Collections.emptyList(); - } - - @Override - @NotNull - public Collection getActions(User user) - { - return Collections.emptyList(); - } - - protected Collection getActionsHelper(User user, List expDatas) - { - List result = new ArrayList<>(); - Set runIDs = new HashSet<>(); - - for (ExpData data : expDatas) - { - if (data == null || !data.getContainer().hasPermission(user, ReadPermission.class)) - continue; - - ActionURL dataURL = data.findDataHandler().getContentURL(data); - List runs = ExperimentService.get().getRunsUsingDatas(Collections.singletonList(data)); - - for (ExpRun run : runs) - { - if (!run.getContainer().hasPermission(user, ReadPermission.class)) - continue; - if (!runIDs.add(run.getRowId())) - continue; - - ActionURL runURL = dataURL == null ? LsidManager.get().getDisplayURL(run.getLSID()) : dataURL; - String actionName; - - if (!run.getName().equals(data.getName())) - { - actionName = run.getName() + " (" + run.getProtocol().getName() + ")"; - } - else - { - actionName = run.getProtocol().getName(); - } - - result.add(new NavTree(actionName, runURL)); - } - } - return result; - } - - public static String c(String path, String... names) - { - StringBuilder s = new StringBuilder(); - s.append(StringUtils.stripEnd(path,"/")); - for (String name : names) - { - String bare = StringUtils.strip(name, "/"); - if (!bare.isEmpty()) - s.append("/").append(bare); - } - return s.toString(); - } - - @Override - public FileStream getFileStream(User user) throws IOException - { - return new _FileStream(user); - } - - private class _FileStream implements FileStream - { - User _user; - InputStream _is = null; - - _FileStream(User user) - { - _user = user; - } - - @Override - public long getSize() - { - try - { - return getContentLength(); - } - catch (IOException x) - { - throw new RuntimeException(x); - } - } - - @Override - public InputStream openInputStream() throws IOException - { - if (null == _is) - _is = getInputStream(_user); - return _is; - } - - @Override - public void closeInputStream() - { - IOUtils.closeQuietly(_is); - } - } - - public void createLink(String name, Path target, @Nullable String indexPage) - { - throw new UnsupportedOperationException(); - } - - public void removeLink(String name) - { - throw new UnsupportedOperationException(); - } - - // - // SearchService - // - @Override - public String getDocumentId() - { - if (null == parent()) - return "dav:" + getPath(); - StringBuilder docid = new StringBuilder(parent().getDocumentId()); - if (docid.charAt(docid.length()-1)!='/') - docid.append("/"); - docid.append(getName()); - if (isCollection()) - docid.append('/'); - return docid.toString(); - } - - @Override - public GUID getContainerId() - { - return _containerId; - } - - @Override - public boolean shouldIndex() - { - // TODO would be nice to call DavController.isTempFile() - String name = getName(); - if (name.startsWith(".part")) // applet uploader temp files - return false; - if (name.startsWith("._")) // mac finder temporary files - return false; - if (name.equals(".DS_Store")) // mac - return false; - if (name.startsWith("~") || name.startsWith(".~")) // Office working files - return false; - return true; - } - - @Override - public Map getCustomProperties(User user) - { - return Collections.emptyMap(); - } - - @Override - public void notify(ContainerUser context, String message) - { - } - - protected void addAuditEvent(ContainerUser context, String message) - { - String dir; - String name; - File f = getFile(); - if (f != null) - { - dir = f.getParent(); - name = f.getName(); - } - else - { - Resource parent = parent(); - dir = parent == null ? "" : parent.getPath().toString(); - name = getName(); - } - - Container c = context.getContainer(); - - // translate the actions into a more meaningful message - if ("created".equalsIgnoreCase(message)) - { - message = "File uploaded to " + c.getContainerNoun() + ": " + c.getPath(); - } - else if ("deleted".equalsIgnoreCase(message)) - { - message = "File deleted from " + c.getContainerNoun() + ": " + c.getPath(); - } - else if ("replaced".equalsIgnoreCase(message)) - { - String path = ("/".equals(c.getPath())) ? c.getPath() : this.getPath().toString(); - message = "File replaced in " + c.getContainerNoun() + ": " + path; - } - else if ("fileDeleteFailed".equalsIgnoreCase(message)) - { - message = "File delete failed from " + c.getContainerNoun() + ": " + c.getPath(); - } - else if ("dirDeleteFailed".equalsIgnoreCase(message)) - { - message = "Directory delete failed from " + c.getContainerNoun() + ": " + c.getPath(); - } - -// String subject = "File Management Tool notification: " + message; - - FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(c, message); - - event.setDirectory(dir); - event.setFile(name); - event.setProvidedFileName(name); - event.setResourcePath(getPath().toString()); - - AuditLogService.get().addEvent(context.getUser(), event); - } - - protected void setProperty(String key, String value) - { - if (_properties == null) - _properties = new HashMap<>(); - _properties.put(key,value); - } - - protected void setSearchProperty(SearchService.PROPERTY searchProperty, String value) - { - setProperty(searchProperty.toString(), value); - } - - protected void setSearchCategory(SearchService.SearchCategory category) - { - setSearchProperty(SearchService.PROPERTY.categories,category.toString()); - } - - protected List getExpData() - { - if (null == _data) - { - java.nio.file.Path file = getNioPath(); - return getExpDatasHelper(file, getContainer()); - } - return _data; - } - - @NotNull - protected List getExpDatasHelper(@Nullable java.nio.file.Path path, Container container) - { - if (null == _data) - { - List list = new LinkedList<>(); - - if (null != path) - { - for (WebdavResourceExpDataProvider provider : WebdavService.get().getExpDataProviders()) - { - list.addAll(provider.getExpDataByPath(path, container)); - } - } - - //Sort the results by creation date so the original is used for metadata display - _data = list.stream().sorted(Comparator.comparing(ExpObject::getCreated)).toList(); - } - return _data; - } - - Container getContainer() - { - GUID id = getContainerId(); - if (null == id) - return null; - return ContainerManager.getForId(id); - } - - @Override - public void setLastModified(long time) throws IOException - { - // No-op - } -} +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.webdav; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.Attachment; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.exp.LsidManager; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpObject; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.resource.AbstractResource; +import org.labkey.api.resource.Resource; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.SecurableResource; +import org.labkey.api.security.SecurityLogger; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.security.roles.OwnerRole; +import org.labkey.api.security.roles.Role; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.FileStream; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.ViewContext; +import org.labkey.api.writer.ContainerUser; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public abstract class AbstractWebdavResource extends AbstractResource implements WebdavResource +{ + private static final String FOLDER_FONT_CLS = "fa fa-folder-o"; + + private SecurableResource _resource; + private List _data = null; + + protected GUID _containerId; + protected String _etag = null; + protected Map _properties = null; + + protected AbstractWebdavResource(Path path) + { + this(path, WebdavService.get().getResolver()); + } + + protected AbstractWebdavResource(Path path, WebdavResolver resolver) + { + super(path, resolver); + } + + protected AbstractWebdavResource(Path folder, String name) + { + this(folder, name, WebdavService.get().getResolver()); + } + + protected AbstractWebdavResource(Path folder, String name, WebdavResolver resolver) + { + super(folder, name, resolver); + } + + protected AbstractWebdavResource(Resource folder, String name) + { + super(folder, name, WebdavService.get().getResolver()); + } + + @Override + public WebdavResolver getResolver() + { + return (WebdavResolver)super.getResolver(); + } + + @Override + public boolean createCollection(User user) throws DavException + { + try + { + return this.getFile() != null && FileUtil.mkdirs(this.getFile(), AppProps.getInstance().isInvalidFilenameUploadBlocked()); + } + catch (IOException e) + { + throw new DavException(e.getCause()); + } + } + + @Override + public WebdavResource parent() + { + Path p = getPath(); + if (p.getNameCount()==0) + return null; + Path parent = p.getParent(); + return WebdavService.get().lookup(parent); + } + + @Override + public WebdavResource find(Path.Part name) + { + return null; + } + + @Override + public Collection list() + { + return Collections.emptyList(); + } + + @Override + public long getCreated() + { + return getLastModified(); + } + + @Override + public User getCreatedBy() + { + List data = getExpData(); + if (data == null || data.isEmpty()) + return null; + + return data.get(0).getCreatedBy(); + } + + @Override + public String getDescription() + { + List data = getExpData(); + if (data == null || data.isEmpty()) + return null; + + return data.get(0).getComment(); + } + + @Override + public User getModifiedBy() + { + List data = getExpData(); + if (data == null || data.isEmpty()) + return null; + + return data.get(0).getModifiedBy(); + } + + @Override + public void setLastIndexed(long indexed, long modified) + { + if (isFile()) + SearchService.get().setLastIndexedForPath(getPath(), indexed, modified); + } + + @Override + public String getContentType() + { + if (isCollection()) + return "text/html"; + return PageFlowUtil.getContentTypeFor(getName()); + } + + @Override + public String getAbsolutePath(User user) + { + return null; + } + + @Override + @NotNull + public String getHref(ViewContext context) + { + ActionURL url = null==context ? null : context.getActionURL(); + int port = null==url ? AppProps.getInstance().getServerPort() : url.getPort(); + String host = null==url ? AppProps.getInstance().getServerName() : url.getHost(); + String scheme = null==context ? AppProps.getInstance().getScheme() : url.getScheme(); + boolean defaultPort = "http".equals(scheme) && 80 == port || "https".equals(scheme) && 443 == port; + String portStr = defaultPort ? "" : ":" + port; + return c(scheme + "://" + host + portStr, getLocalHref(context)); + } + + @Override + @NotNull + public String getLocalHref(ViewContext context) + { + String contextPath = null==context ? AppProps.getInstance().getContextPath() : context.getContextPath(); + String href = c(contextPath, getPath().encode()); + if (isCollection() && !href.endsWith("/")) + href += "/"; + return href; + } + + @Override + public String getExecuteHref(ViewContext context) + { + String path = parent().getExecuteHref(context); + path += (path.endsWith("/")?"":"/") + PageFlowUtil.encode(getPath().getName()); + return path; + } + + @Override + public String getIconHref() + { + if (isCollection()) + return AppProps.getInstance().getContextPath() + "/_icons/folder.gif"; + return AppProps.getInstance().getContextPath() + Attachment.getFileIcon(getName()); + } + + @Nullable + @Override + public String getIconFontCls() + { + if (isCollection()) + return FOLDER_FONT_CLS; + return Attachment.getFileIconFontCls(getName()); + } + + @Nullable + @Override + public DirectRequest getDirectGetRequest(ViewContext context, String contentDisposition) + { + return null; + } + + @Nullable + @Override + public DirectRequest getDirectPutRequest(ViewContext context) + { + return null; + } + + @Override + public String getETag(boolean force) + { + long len = 0; + if (null == _etag) + { + try + { + len = getContentLength(); + } + catch (IOException x) + { + /* */ + } + _etag = "W/\"" + len + "-" + getLastModified() + "\""; + } + return _etag; + } + + @Override + public String getETag() + { + return getETag(false); + } + + @Override + public String getMD5(User user) throws IOException + { + return FileUtil.md5sum(getInputStream(user)); + } + + @Override + public Map getProperties() + { + if (null == _properties) + return Collections.emptyMap(); + Map ret = _properties; + assert null != (ret = Collections.unmodifiableMap(ret)); + return ret; + } + + @Override + public Map getMutableProperties() + { + if (null == _properties) + _properties = new HashMap<>(); + return _properties; + } + + @Override + public InputStream getInputStream() throws IOException + { + return getInputStream(null); + } + + /** provides one place to completely block access to the resource */ + protected boolean hasAccess(User user) + { + return true; + } + + protected SecurableResource getSecurableResource() + { + return _resource; + } + + protected void setSecurableResource(SecurableResource resource) + { + _resource = resource; + } + + /** permissions */ + + @Override + public boolean canList(User user, boolean forRead) + { + return canRead(user, forRead); + } + + @Override + public boolean canRead(User user, boolean forRead) + { + // TODO: This looks wrong + if ("/".equals(getPath())) + return true; + try + { + SecurityLogger.indent(getPath() + " AbstractWebdavResource.canRead()"); + if (!hasAccess(user)) + { + SecurityLogger.log("hasAccess()==false", user, null, false); + return false; + } + return getPermissions(user).contains(ReadPermission.class); + } + finally + { + SecurityLogger.outdent(); + } + } + + @Override + public boolean canWrite(User user, boolean forWrite) + { + Set roles = user.equals(getCreatedBy()) ? RoleManager.roleSet(OwnerRole.class) : Set.of(); + return hasAccess(user) && !user.isGuest() && + SecurityManager.hasAllPermissions(null, getSecurableResource(), user, Set.of(UpdatePermission.class), roles); + } + + @Override + public boolean canCreate(User user, boolean forCreate) + { + return hasAccess(user) && !user.isGuest() && + SecurityManager.hasAllPermissions(null, getSecurableResource(), user, Set.of(InsertPermission.class), Set.of()); + } + + @Override + public boolean canCreateCollection(User user, boolean forCreate) + { + return canCreate(user, forCreate); + } + + @Override + public boolean canDelete(User user, boolean forDelete) + { + return canDelete(user, forDelete, null); + } + + @Override + public boolean canDelete(User user, boolean forDelete, /* OUT */ @Nullable List message) + { + if (user.isGuest() || !hasAccess(user)) + return false; + Set> perms = getPermissions(user); + return perms.contains(DeletePermission.class); + } + + @Override + public boolean canRename(User user, boolean forRename) + { + return hasAccess(user) && !user.isGuest() && canCreate(user, forRename) && canDelete(user, forRename, null); + } + + public Set> getPermissions(User user) + { + return SecurityManager.getPermissions(getSecurableResource(), user, Set.of()); + } + + @Override + public boolean delete(User user) throws IOException + { + assert null == user || canDelete(user, true, null); + return false; + } + + @Override + public File getFile() + { + return null; + } + + @Override + public long copyFrom(User user, WebdavResource r) throws IOException, DavException + { + return copyFrom(user, r.getFileStream(user)); + } + + @Override + public void moveFrom(User user, WebdavResource src) throws IOException, DavException + { + copyFrom(user, src); + src.delete(user); + } + + @Override + @NotNull + public Collection getHistory() + { + return Collections.emptyList(); + } + + @Override + @NotNull + public Collection getActions(User user) + { + return Collections.emptyList(); + } + + protected Collection getActionsHelper(User user, List expDatas) + { + List result = new ArrayList<>(); + Set runIDs = new HashSet<>(); + + for (ExpData data : expDatas) + { + if (data == null || !data.getContainer().hasPermission(user, ReadPermission.class)) + continue; + + ActionURL dataURL = data.findDataHandler().getContentURL(data); + List runs = ExperimentService.get().getRunsUsingDatas(Collections.singletonList(data)); + + for (ExpRun run : runs) + { + if (!run.getContainer().hasPermission(user, ReadPermission.class)) + continue; + if (!runIDs.add(run.getRowId())) + continue; + + ActionURL runURL = dataURL == null ? LsidManager.get().getDisplayURL(run.getLSID()) : dataURL; + String actionName; + + if (!run.getName().equals(data.getName())) + { + actionName = run.getName() + " (" + run.getProtocol().getName() + ")"; + } + else + { + actionName = run.getProtocol().getName(); + } + + result.add(new NavTree(actionName, runURL)); + } + } + return result; + } + + public static String c(String path, String... names) + { + StringBuilder s = new StringBuilder(); + s.append(StringUtils.stripEnd(path,"/")); + for (String name : names) + { + String bare = StringUtils.strip(name, "/"); + if (!bare.isEmpty()) + s.append("/").append(bare); + } + return s.toString(); + } + + @Override + public FileStream getFileStream(User user) throws IOException + { + return new _FileStream(user); + } + + private class _FileStream implements FileStream + { + User _user; + InputStream _is = null; + + _FileStream(User user) + { + _user = user; + } + + @Override + public long getSize() + { + try + { + return getContentLength(); + } + catch (IOException x) + { + throw new RuntimeException(x); + } + } + + @Override + public InputStream openInputStream() throws IOException + { + if (null == _is) + _is = getInputStream(_user); + return _is; + } + + @Override + public void closeInputStream() + { + IOUtils.closeQuietly(_is); + } + } + + public void createLink(String name, Path target, @Nullable String indexPage) + { + throw new UnsupportedOperationException(); + } + + public void removeLink(String name) + { + throw new UnsupportedOperationException(); + } + + // + // SearchService + // + @Override + public String getDocumentId() + { + if (null == parent()) + return "dav:" + getPath(); + StringBuilder docid = new StringBuilder(parent().getDocumentId()); + if (docid.charAt(docid.length()-1)!='/') + docid.append("/"); + docid.append(getName()); + if (isCollection()) + docid.append('/'); + return docid.toString(); + } + + @Override + public GUID getContainerId() + { + return _containerId; + } + + @Override + public boolean shouldIndex() + { + // TODO would be nice to call DavController.isTempFile() + String name = getName(); + if (name.startsWith(".part")) // applet uploader temp files + return false; + if (name.startsWith("._")) // mac finder temporary files + return false; + if (name.equals(".DS_Store")) // mac + return false; + if (name.startsWith("~") || name.startsWith(".~")) // Office working files + return false; + return true; + } + + @Override + public Map getCustomProperties(User user) + { + return Collections.emptyMap(); + } + + @Override + public void notify(ContainerUser context, String message) + { + } + + protected void addAuditEvent(ContainerUser context, String message) + { + String dir; + String name; + File f = getFile(); + if (f != null) + { + dir = f.getParent(); + name = f.getName(); + } + else + { + Resource parent = parent(); + dir = parent == null ? "" : parent.getPath().toString(); + name = getName(); + } + + Container c = context.getContainer(); + + // translate the actions into a more meaningful message + if ("created".equalsIgnoreCase(message)) + { + message = "File uploaded to " + c.getContainerNoun() + ": " + c.getPath(); + } + else if ("deleted".equalsIgnoreCase(message)) + { + message = "File deleted from " + c.getContainerNoun() + ": " + c.getPath(); + } + else if ("replaced".equalsIgnoreCase(message)) + { + String path = ("/".equals(c.getPath())) ? c.getPath() : this.getPath().toString(); + message = "File replaced in " + c.getContainerNoun() + ": " + path; + } + else if ("fileDeleteFailed".equalsIgnoreCase(message)) + { + message = "File delete failed from " + c.getContainerNoun() + ": " + c.getPath(); + } + else if ("dirDeleteFailed".equalsIgnoreCase(message)) + { + message = "Directory delete failed from " + c.getContainerNoun() + ": " + c.getPath(); + } + +// String subject = "File Management Tool notification: " + message; + + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(c, message); + + event.setDirectory(dir); + event.setFile(name); + event.setProvidedFileName(name); + event.setResourcePath(getPath().toString()); + + AuditLogService.get().addEvent(context.getUser(), event); + } + + protected void setProperty(String key, String value) + { + if (_properties == null) + _properties = new HashMap<>(); + _properties.put(key,value); + } + + protected void setSearchProperty(SearchService.PROPERTY searchProperty, String value) + { + setProperty(searchProperty.toString(), value); + } + + protected void setSearchCategory(SearchService.SearchCategory category) + { + setSearchProperty(SearchService.PROPERTY.categories,category.toString()); + } + + protected List getExpData() + { + if (null == _data) + { + java.nio.file.Path file = getNioPath(); + return getExpDatasHelper(file, getContainer()); + } + return _data; + } + + @NotNull + protected List getExpDatasHelper(@Nullable java.nio.file.Path path, Container container) + { + if (null == _data) + { + List list = new LinkedList<>(); + + if (null != path) + { + for (WebdavResourceExpDataProvider provider : WebdavService.get().getExpDataProviders()) + { + list.addAll(provider.getExpDataByPath(path, container)); + } + } + + //Sort the results by creation date so the original is used for metadata display + _data = list.stream().sorted(Comparator.comparing(ExpObject::getCreated)).toList(); + } + return _data; + } + + Container getContainer() + { + GUID id = getContainerId(); + if (null == id) + return null; + return ContainerManager.getForId(id); + } + + @Override + public void setLastModified(long time) throws IOException + { + // No-op + } +} diff --git a/assay/src/org/labkey/assay/AssayModule.java b/assay/src/org/labkey/assay/AssayModule.java index 4e8bb7e82c9..171e1730ae1 100644 --- a/assay/src/org/labkey/assay/AssayModule.java +++ b/assay/src/org/labkey/assay/AssayModule.java @@ -202,33 +202,30 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) PlateManager.get().registerLsidHandlers(); SearchService ss = SearchService.get(); - if (null != ss) - { - // ASSAY_CATEGORY - ss.addSearchCategory(AssayManager.get().ASSAY_CATEGORY); - ss.addResourceResolver(AssayManager.get().ASSAY_CATEGORY.getName(), AssayDocumentProvider.getSearchResolver()); - ss.addDocumentProvider(new AssayDocumentProvider()); - - // ASSAY_RUN_CATEGORY - ss.addSearchCategory(AssayManager.get().ASSAY_RUN_CATEGORY); - ss.addResourceResolver(AssayManager.get().ASSAY_RUN_CATEGORY.getName(), AssayRunDocumentProvider.getResourceResolver()); - ss.addDocumentProvider(new AssayRunDocumentProvider()); - - // ASSAY_BATCH_CATEGORY - ss.addSearchCategory(AssayManager.get().ASSAY_BATCH_CATEGORY); - ss.addResourceResolver(AssayManager.get().ASSAY_BATCH_CATEGORY.getName(), AssayBatchDocumentProvider.getResourceResolver()); - ss.addDocumentProvider(new AssayBatchDocumentProvider()); - - // PLATE_CATEGORY - ss.addSearchCategory(PlateManager.get().PLATE_CATEGORY); - ss.addResourceResolver(PlateManager.get().PLATE_CATEGORY.getName(), PlateDocumentProvider.getResourceResolver()); - ss.addDocumentProvider(new PlateDocumentProvider()); - - // PLATE_SET_CATEGORY - ss.addSearchCategory(PlateManager.get().PLATE_SET_CATEGORY); - ss.addResourceResolver(PlateManager.get().PLATE_SET_CATEGORY.getName(), PlateSetDocumentProvider.getResourceResolver()); - ss.addDocumentProvider(new PlateSetDocumentProvider()); - } + // ASSAY_CATEGORY + ss.addSearchCategory(AssayManager.get().ASSAY_CATEGORY); + ss.addResourceResolver(AssayManager.get().ASSAY_CATEGORY.getName(), AssayDocumentProvider.getSearchResolver()); + ss.addDocumentProvider(new AssayDocumentProvider()); + + // ASSAY_RUN_CATEGORY + ss.addSearchCategory(AssayManager.get().ASSAY_RUN_CATEGORY); + ss.addResourceResolver(AssayManager.get().ASSAY_RUN_CATEGORY.getName(), AssayRunDocumentProvider.getResourceResolver()); + ss.addDocumentProvider(new AssayRunDocumentProvider()); + + // ASSAY_BATCH_CATEGORY + ss.addSearchCategory(AssayManager.get().ASSAY_BATCH_CATEGORY); + ss.addResourceResolver(AssayManager.get().ASSAY_BATCH_CATEGORY.getName(), AssayBatchDocumentProvider.getResourceResolver()); + ss.addDocumentProvider(new AssayBatchDocumentProvider()); + + // PLATE_CATEGORY + ss.addSearchCategory(PlateManager.get().PLATE_CATEGORY); + ss.addResourceResolver(PlateManager.get().PLATE_CATEGORY.getName(), PlateDocumentProvider.getResourceResolver()); + ss.addDocumentProvider(new PlateDocumentProvider()); + + // PLATE_SET_CATEGORY + ss.addSearchCategory(PlateManager.get().PLATE_SET_CATEGORY); + ss.addResourceResolver(PlateManager.get().PLATE_SET_CATEGORY.getName(), PlateSetDocumentProvider.getResourceResolver()); + ss.addDocumentProvider(new PlateSetDocumentProvider()); // add a container listener so we'll know when our container is deleted: ContainerManager.addContainerListener(new AssayContainerListener()); diff --git a/core/src/org/labkey/core/CoreContainerListener.java b/core/src/org/labkey/core/CoreContainerListener.java index 682c8f5ea31..2f9aaa9de0f 100644 --- a/core/src/org/labkey/core/CoreContainerListener.java +++ b/core/src/org/labkey/core/CoreContainerListener.java @@ -1,125 +1,120 @@ -/* - * Copyright (c) 2008-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.core; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.provider.ContainerAuditProvider; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.MvUtil; -import org.labkey.api.data.PropertyManager; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TestSchema; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.view.Portal; - -import java.beans.PropertyChangeEvent; -import java.util.Collection; -import java.util.Collections; - -public class CoreContainerListener implements ContainerManager.ContainerListener -{ - private static final Logger _log = LogManager.getLogger(CoreContainerListener.class); - - @Override - public void containerCreated(Container c, User user) - { - containerCreated(c, user, null); - } - - @Override - public void containerCreated(Container c, User user, @Nullable String auditMsg) - { - String message = auditMsg == null ? c.getContainerNoun(true) + " " + c.getName() + " was created" : auditMsg; - addAuditEvent(user, c, message); - SearchService ss = SearchService.get(); - if (ss != null) - ((CoreModule)ModuleLoader.getInstance().getCoreModule()).enumerateDocuments(ss.defaultTask().getQueue(c, SearchService.PRIORITY.modified), null); - } - - @Override - public void containerDeleted(Container c, User user) - { - PropertyManager.purgeObjectProperties(c); - MvUtil.containerDeleted(c); - - // Delete any rows in test.TestTable associated with this container - SimpleFilter containerFilter = SimpleFilter.createContainerFilter(c); - Table.delete(TestSchema.getInstance().getTableInfoTestTable(), containerFilter); - - // Data States - Table.delete(CoreSchema.getInstance().getTableInfoDataStates(), containerFilter); - - // report engine folder mapping - Table.delete(CoreSchema.getInstance().getTableInfoReportEngineMap(), containerFilter); - - // Let containerManager delete ACLs, we want that to happen last - Portal.containerDeleted(c); - } - - @Override - public void containerMoved(Container c, Container oldParent, User user) - { - String message = c.getName() + " was moved from " + oldParent.getPath() + " to " + c.getParent().getPath(); - addAuditEvent(user, c, message); - // re-index is handled when the propertyChange() event fires - } - - @NotNull - @Override - public Collection canMove(Container c, Container newParent, User user) - { - return Collections.emptyList(); - } - - private void addAuditEvent(User user, Container c, String comment) - { - if (user != null) - { - AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, c, comment); - AuditLogService.get().addEvent(user, event); - } - } - - @Override - public void propertyChange(PropertyChangeEvent propertyChangeEvent) - { - ContainerManager.ContainerPropertyChangeEvent evt = (ContainerManager.ContainerPropertyChangeEvent)propertyChangeEvent; - Container c = evt.container; - ((CoreModule) ModuleLoader.getInstance().getCoreModule()).enumerateDocuments(SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified), null); - - switch (evt.property) - { - case Name: - { - String oldValue = (String) evt.getOldValue(); - String newValue = (String) evt.getNewValue(); - String message = c.getName() + " was renamed from " + oldValue + " to " + newValue; - addAuditEvent(evt.user, c, message); - break; - } - } - } -} +/* + * Copyright (c) 2008-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.core; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.provider.ContainerAuditProvider; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.MvUtil; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TestSchema; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.view.Portal; + +import java.beans.PropertyChangeEvent; +import java.util.Collection; +import java.util.Collections; +import java.util.Objects; + +public class CoreContainerListener implements ContainerManager.ContainerListener +{ + private static final Logger _log = LogManager.getLogger(CoreContainerListener.class); + + @Override + public void containerCreated(Container c, User user) + { + containerCreated(c, user, null); + } + + @Override + public void containerCreated(Container c, User user, @Nullable String auditMsg) + { + String message = auditMsg == null ? c.getContainerNoun(true) + " " + c.getName() + " was created" : auditMsg; + addAuditEvent(user, c, message); + ((CoreModule)ModuleLoader.getInstance().getCoreModule()).enumerateDocuments(SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified), null); + } + + @Override + public void containerDeleted(Container c, User user) + { + PropertyManager.purgeObjectProperties(c); + MvUtil.containerDeleted(c); + + // Delete any rows in test.TestTable associated with this container + SimpleFilter containerFilter = SimpleFilter.createContainerFilter(c); + Table.delete(TestSchema.getInstance().getTableInfoTestTable(), containerFilter); + + // Data States + Table.delete(CoreSchema.getInstance().getTableInfoDataStates(), containerFilter); + + // report engine folder mapping + Table.delete(CoreSchema.getInstance().getTableInfoReportEngineMap(), containerFilter); + + // Let containerManager delete ACLs, we want that to happen last + Portal.containerDeleted(c); + } + + @Override + public void containerMoved(Container c, Container oldParent, User user) + { + String message = c.getName() + " was moved from " + oldParent.getPath() + " to " + c.getParent().getPath(); + addAuditEvent(user, c, message); + // re-index is handled when the propertyChange() event fires + } + + @NotNull + @Override + public Collection canMove(Container c, Container newParent, User user) + { + return Collections.emptyList(); + } + + private void addAuditEvent(User user, Container c, String comment) + { + if (user != null) + { + AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, c, comment); + AuditLogService.get().addEvent(user, event); + } + } + + @Override + public void propertyChange(PropertyChangeEvent propertyChangeEvent) + { + ContainerManager.ContainerPropertyChangeEvent evt = (ContainerManager.ContainerPropertyChangeEvent)propertyChangeEvent; + Container c = evt.container; + ((CoreModule) ModuleLoader.getInstance().getCoreModule()).enumerateDocuments(SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified), null); + + if (Objects.requireNonNull(evt.property) == ContainerManager.Property.Name) + { + String oldValue = (String) evt.getOldValue(); + String newValue = (String) evt.getNewValue(); + String message = c.getName() + " was renamed from " + oldValue + " to " + newValue; + addAuditEvent(evt.user, c, message); + } + } +} diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index 8f128c8df79..750d96c0166 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -1,1775 +1,1772 @@ -/* - * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.core; - -import com.fasterxml.jackson.core.io.CharTypes; -import com.google.common.collect.Sets; -import jakarta.servlet.ServletContext; -import jakarta.servlet.ServletRegistration; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pdfbox.pdmodel.font.FontMapper; -import org.apache.pdfbox.pdmodel.font.FontMappers; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.action.ApiJsonWriter; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AdminConsoleService; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.admin.HealthCheck; -import org.labkey.api.admin.HealthCheckRegistry; -import org.labkey.api.admin.TableXmlUtils; -import org.labkey.api.admin.notification.NotificationService; -import org.labkey.api.admin.sitevalidation.SiteValidationService; -import org.labkey.api.analytics.AnalyticsService; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.attachments.DocumentConversionService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.ClientApiAuditProvider; -import org.labkey.api.audit.DefaultAuditProvider; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.ContainerAuditProvider; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.audit.provider.GroupAuditProvider; -import org.labkey.api.audit.provider.ModulePropertiesAuditProvider; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerService; -import org.labkey.api.data.ContainerServiceImpl; -import org.labkey.api.data.ContainerTypeRegistry; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DataColumn; -import org.labkey.api.data.DataRegion; -import org.labkey.api.data.DatabaseMigrationService; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationHandler; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.FileSqlScriptProvider; -import org.labkey.api.data.MvUtil; -import org.labkey.api.data.NormalContainerType; -import org.labkey.api.data.OutOfRangeDisplayColumn; -import org.labkey.api.data.PropertySchema; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SchemaTableInfoFactory; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TSVWriter; -import org.labkey.api.data.TabContainerType; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TempTableTracker; -import org.labkey.api.data.TestSchema; -import org.labkey.api.data.WorkbookContainerType; -import org.labkey.api.data.dialect.BasePostgreSqlDialect; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.data.dialect.SqlDialectManager; -import org.labkey.api.data.dialect.SqlDialectRegistry; -import org.labkey.api.data.statistics.StatsService; -import org.labkey.api.dataiterator.SimpleTranslator; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.TestDomainKind; -import org.labkey.api.external.tools.ExternalToolsViewService; -import org.labkey.api.files.FileBrowserConfigImporter; -import org.labkey.api.files.FileBrowserConfigWriter; -import org.labkey.api.files.FileContentService; -import org.labkey.api.markdown.MarkdownService; -import org.labkey.api.message.settings.MessageConfigService; -import org.labkey.api.module.FolderType; -import org.labkey.api.module.FolderTypeManager; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.module.SchemaUpdateType; -import org.labkey.api.module.SpringModule; -import org.labkey.api.module.Summary; -import org.labkey.api.notification.EmailMessage; -import org.labkey.api.notification.EmailService; -import org.labkey.api.notification.NotificationMenuView; -import org.labkey.api.portal.ProjectUrls; -import org.labkey.api.premium.AntiVirusProviderRegistry; -import org.labkey.api.products.ProductRegistry; -import org.labkey.api.qc.DataStateManager; -import org.labkey.api.query.DefaultSchema; -import org.labkey.api.query.QuerySchema; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.reader.DataLoaderFactory; -import org.labkey.api.reader.DataLoaderService; -import org.labkey.api.reader.ExcelLoader; -import org.labkey.api.reader.FastaDataLoader; -import org.labkey.api.reader.HTMLDataLoader; -import org.labkey.api.reader.JSONDataLoader; -import org.labkey.api.reader.TabLoader; -import org.labkey.api.reports.LabKeyScriptEngineManager; -import org.labkey.api.resource.Resource; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.AuthenticationManager; -import org.labkey.api.security.AuthenticationManager.Priority; -import org.labkey.api.security.AuthenticationSettingsAuditTypeProvider; -import org.labkey.api.security.DbLoginService; -import org.labkey.api.security.DummyAntiVirusService; -import org.labkey.api.security.Group; -import org.labkey.api.security.GroupManager; -import org.labkey.api.security.LimitActiveUsersService; -import org.labkey.api.security.MutableSecurityPolicy; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.SecurityPointcutService; -import org.labkey.api.security.SecurityPolicyManager; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.security.WikiTermsOfUseProvider; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.QCAnalystPermission; -import org.labkey.api.security.permissions.TroubleshooterPermission; -import org.labkey.api.security.roles.NoPermissionsRole; -import org.labkey.api.security.roles.ReaderRole; -import org.labkey.api.security.roles.Role; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.CustomLabelService; -import org.labkey.api.settings.CustomLabelService.CustomLabelServiceImpl; -import org.labkey.api.settings.FolderSettingsCache; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.settings.LookAndFeelPropertiesManager; -import org.labkey.api.settings.LookAndFeelPropertiesManager.ResourceType; -import org.labkey.api.settings.LookAndFeelPropertiesManager.SiteResourceHandler; -import org.labkey.api.settings.OptionalFeatureFlag; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.settings.OptionalFeatureService.FeatureType; -import org.labkey.api.settings.ProductConfiguration; -import org.labkey.api.settings.StandardStartupPropertyHandler; -import org.labkey.api.settings.StartupPropertyEntry; -import org.labkey.api.settings.StashedStartupProperties; -import org.labkey.api.settings.WriteableAppProps; -import org.labkey.api.settings.WriteableLookAndFeelProperties; -import org.labkey.api.stats.AnalyticsProviderRegistry; -import org.labkey.api.stats.SummaryStatisticRegistry; -import org.labkey.api.study.Study; -import org.labkey.api.study.StudyService; -import org.labkey.api.thumbnail.ThumbnailService; -import org.labkey.api.usageMetrics.SimpleMetricsService; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.ContextListener; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JobRunner; -import org.labkey.api.util.MimeMap; -import org.labkey.api.util.MothershipReport; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.ShutdownListener; -import org.labkey.api.util.StartupListener; -import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.util.UsageReportingLevel; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.vcs.VcsService; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.AlwaysAvailableWebPartFactory; -import org.labkey.api.view.BaseWebPartFactory; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.Portal; -import org.labkey.api.view.Portal.WebPart; -import org.labkey.api.view.ShortURLService; -import org.labkey.api.view.VBox; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.ViewService; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.menu.FolderMenu; -import org.labkey.api.view.template.WarningService; -import org.labkey.api.webdav.SimpleDocumentResource; -import org.labkey.api.webdav.WebdavResolverImpl; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.api.webdav.WebdavService; -import org.labkey.api.wiki.WikiRenderingService; -import org.labkey.api.writer.ContainerUser; -import org.labkey.core.admin.ActionsTsvWriter; -import org.labkey.core.admin.AdminConsoleServiceImpl; -import org.labkey.core.admin.AdminController; -import org.labkey.core.admin.AllowListType; -import org.labkey.core.admin.CopyFileRootPipelineJob; -import org.labkey.core.admin.CustomizeMenuForm; -import org.labkey.core.admin.DisplayFormatAnalyzer; -import org.labkey.core.admin.DisplayFormatValidationProviderFactory; -import org.labkey.core.admin.FilesSiteSettingsAction; -import org.labkey.core.admin.MenuViewFactory; -import org.labkey.core.admin.OptionalFeatureServiceImpl; -import org.labkey.core.admin.OptionalFeatureStartupListener; -import org.labkey.core.admin.importer.FolderTypeImporterFactory; -import org.labkey.core.admin.importer.MissingValueImporterFactory; -import org.labkey.core.admin.importer.ModulePropertiesImporterFactory; -import org.labkey.core.admin.importer.PageImporterFactory; -import org.labkey.core.admin.importer.RoleAssignmentsImporterFactory; -import org.labkey.core.admin.importer.SearchSettingsImporterFactory; -import org.labkey.core.admin.importer.SecurityGroupImporterFactory; -import org.labkey.core.admin.importer.SubfolderImporterFactory; -import org.labkey.core.admin.logger.LoggerController; -import org.labkey.core.admin.logger.LoggingTestCase; -import org.labkey.core.admin.miniprofiler.MiniProfilerController; -import org.labkey.core.admin.sitevalidation.SiteValidationServiceImpl; -import org.labkey.core.admin.sql.SqlScriptController; -import org.labkey.core.admin.test.SchemaXMLTestCase; -import org.labkey.core.admin.test.UnknownSchemasTest; -import org.labkey.core.admin.usageMetrics.UsageMetricsServiceImpl; -import org.labkey.core.admin.writer.FolderSerializationRegistryImpl; -import org.labkey.core.admin.writer.FolderTypeWriterFactory; -import org.labkey.core.admin.writer.MissingValueWriterFactory; -import org.labkey.core.admin.writer.ModulePropertiesWriterFactory; -import org.labkey.core.admin.writer.PageWriterFactory; -import org.labkey.core.admin.writer.RoleAssignmentsWriterFactory; -import org.labkey.core.admin.writer.SearchSettingsWriterFactory; -import org.labkey.core.admin.writer.SecurityGroupWriterFactory; -import org.labkey.core.analytics.AnalyticsController; -import org.labkey.core.analytics.AnalyticsServiceImpl; -import org.labkey.core.attachment.AttachmentServiceImpl; -import org.labkey.core.dialect.PostgreSqlDialectFactory; -import org.labkey.core.dialect.PostgreSqlInClauseTest; -import org.labkey.core.dialect.PostgreSqlVersion; -import org.labkey.core.junit.JunitController; -import org.labkey.core.login.DbLoginAuthenticationProvider; -import org.labkey.core.login.DbLoginManager; -import org.labkey.core.login.LoginController; -import org.labkey.core.metrics.SimpleMetricsServiceImpl; -import org.labkey.core.metrics.WebSocketConnectionManager; -import org.labkey.core.notification.EmailPreferenceConfigServiceImpl; -import org.labkey.core.notification.EmailPreferenceContainerListener; -import org.labkey.core.notification.EmailPreferenceUserListener; -import org.labkey.core.notification.EmailServiceImpl; -import org.labkey.core.notification.NotificationController; -import org.labkey.core.notification.NotificationServiceImpl; -import org.labkey.core.portal.CollaborationFolderType; -import org.labkey.core.portal.PortalJUnitTest; -import org.labkey.core.portal.ProjectController; -import org.labkey.core.portal.UtilController; -import org.labkey.core.products.ProductController; -import org.labkey.core.project.FolderNavigationForm; -import org.labkey.core.qc.CoreQCStateHandler; -import org.labkey.core.qc.DataStateImporter; -import org.labkey.core.qc.DataStateWriter; -import org.labkey.core.query.AttachmentAuditProvider; -import org.labkey.core.query.CoreQuerySchema; -import org.labkey.core.query.PostgresUserSchema; -import org.labkey.core.query.UserAuditProvider; -import org.labkey.core.query.UsersDomainKind; -import org.labkey.core.reader.DataLoaderServiceImpl; -import org.labkey.core.reports.DocumentConversionServiceImpl; -import org.labkey.core.reports.ScriptEngineManagerImpl; -import org.labkey.core.script.RhinoService; -import org.labkey.core.security.AllowedExternalResourceHosts; -import org.labkey.core.security.ApiKeyViewProvider; -import org.labkey.core.security.SecurityApiActions; -import org.labkey.core.security.SecurityController; -import org.labkey.core.security.SecurityPointcutServiceImpl; -import org.labkey.core.security.validators.PermissionsValidatorFactory; -import org.labkey.core.statistics.AnalyticsProviderRegistryImpl; -import org.labkey.core.statistics.StatsServiceImpl; -import org.labkey.core.statistics.SummaryStatisticRegistryImpl; -import org.labkey.core.thumbnail.ThumbnailServiceImpl; -import org.labkey.core.user.LimitActiveUsersSettings; -import org.labkey.core.user.UserController; -import org.labkey.core.vcs.VcsServiceImpl; -import org.labkey.core.view.ShortURLServiceImpl; -import org.labkey.core.view.TableViewFormTestCase; -import org.labkey.core.view.external.tools.ExternalToolsViewServiceImpl; -import org.labkey.core.view.template.bootstrap.CoreWarningProvider; -import org.labkey.core.view.template.bootstrap.PageTemplate; -import org.labkey.core.view.template.bootstrap.ViewServiceImpl; -import org.labkey.core.view.template.bootstrap.WarningServiceImpl; -import org.labkey.core.webdav.DavController; -import org.labkey.core.webdav.ModuleStaticResolverImpl; -import org.labkey.core.webdav.WebFilesResolverImpl; -import org.labkey.core.webdav.WebdavServlet; -import org.labkey.core.wiki.MarkdownServiceImpl; -import org.labkey.core.wiki.MarkdownTestCase; -import org.labkey.core.wiki.RadeoxRenderer; -import org.labkey.core.wiki.WikiRenderingServiceImpl; -import org.labkey.core.workbook.WorkbookFolderType; -import org.labkey.core.workbook.WorkbookQueryView; -import org.labkey.core.workbook.WorkbookSearchView; -import org.labkey.filters.ContentSecurityPolicyFilter; -import org.quartz.Scheduler; -import org.quartz.SchedulerException; -import org.quartz.impl.StdSchedulerFactory; -import org.radeox.test.BaseRenderEngineTest; -import org.radeox.test.filter.BasicRegexTest; -import org.radeox.test.filter.BoldFilterTest; -import org.radeox.test.filter.EscapeFilterTest; -import org.radeox.test.filter.FilterPipeTest; -import org.radeox.test.filter.HeadingFilterTest; -import org.radeox.test.filter.HtmlRemoveFilterTest; -import org.radeox.test.filter.ItalicFilterTest; -import org.radeox.test.filter.KeyFilterTest; -import org.radeox.test.filter.LineFilterTest; -import org.radeox.test.filter.LinkTestFilterTest; -import org.radeox.test.filter.ListFilterTest; -import org.radeox.test.filter.NewlineFilterTest; -import org.radeox.test.filter.ParamFilterTest; -import org.radeox.test.filter.SmileyFilterTest; -import org.radeox.test.filter.StrikeThroughFilterTest; -import org.radeox.test.filter.TypographyFilterTest; -import org.radeox.test.filter.UrlFilterTest; -import org.radeox.test.filter.WikiLinkFilterTest; -import org.radeox.test.macro.list.AtoZListFormatterTest; -import org.radeox.test.macro.list.ExampleListFormatterTest; -import org.radeox.test.macro.list.SimpleListTest; - -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.sql.Connection; -import java.sql.SQLException; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.labkey.api.settings.StashedStartupProperties.homeProjectFolderType; -import static org.labkey.api.settings.StashedStartupProperties.homeProjectResetPermissions; -import static org.labkey.api.settings.StashedStartupProperties.homeProjectWebparts; -import static org.labkey.api.settings.StashedStartupProperties.siteAvailableEmailFrom; -import static org.labkey.api.settings.StashedStartupProperties.siteAvailableEmailMessage; -import static org.labkey.api.settings.StashedStartupProperties.siteAvailableEmailSubject; -import static org.labkey.api.util.MothershipReport.EXPERIMENTAL_LOCAL_MARKETING_UPDATE; -import static org.labkey.api.util.MothershipReport.FEATURE_FLAG_EXTENDED_METRICS; -import static org.labkey.filters.ContentSecurityPolicyFilter.FEATURE_FLAG_DISABLE_ENFORCE_CSP; - -public class CoreModule extends SpringModule implements SearchService.DocumentProvider -{ - private static final Logger LOG = LogHelper.getLogger(CoreModule.class, "Errors during server startup and shut down"); - public static final String PROJECTS_WEB_PART_NAME = "Projects"; - - static - { - // Accept most of the standard Quartz properties, but set the misfire threshold to five minutes. This prevents - // Quartz from dropping scheduled work if a lot of items fire at the same time, like a lot of ETLs triggering at 2AM. - // This can overwhelm the thread pool running them so they don't complete in the default 1 minute window. Set it early so - // if any other module touches Quartz in its setup, it initializes with this setting. - Properties props = System.getProperties(); - props.setProperty("org.quartz.jobStore.misfireThreshold", "300000"); - - // Register dialect extra early, since we need to initialize the data sources before calling DefaultModule.initialize() - SqlDialectRegistry.register(new PostgreSqlDialectFactory()); - - try - { - var field = CharTypes.class.getDeclaredField("sOutputEscapes128"); - field.setAccessible(true); - ((int[])field.get(null))['/'] = '/'; - field.setAccessible(false); - } - catch (NoSuchFieldException|IllegalArgumentException|IllegalAccessException x) - { - // pass - } - } - - private CoreWarningProvider _warningProvider; - private ServletRegistration.Dynamic _webdavServletDynamic; - - @Override - public boolean hasScripts() - { - return true; - } - - @Override - protected void init() - { - ContainerService.setInstance(new ContainerServiceImpl()); - FolderSerializationRegistry.setInstance(new FolderSerializationRegistryImpl()); - ExternalToolsViewService.setInstance(new ExternalToolsViewServiceImpl()); - ExternalToolsViewService.get().registerExternalAccessViewProvider(new ApiKeyViewProvider()); - LimitActiveUsersService.setInstance(() -> new LimitActiveUsersSettings().isUserLimitReached()); - - // Register the default DataLoaders during init so they are available to sql upgrade scripts - DataLoaderServiceImpl dls = new DataLoaderServiceImpl(); - dls.registerFactory(new ExcelLoader.Factory()); - dls.registerFactory(new TabLoader.TsvFactory()); - dls.registerFactory(new TabLoader.CsvFactory()); - dls.registerFactory(new HTMLDataLoader.Factory()); - dls.registerFactory(new JSONDataLoader.Factory()); - dls.registerFactory(new FastaDataLoader.Factory()); - DataLoaderService.setInstance(dls); - - addController("admin", AdminController.class); - addController("admin-sql", SqlScriptController.class); - addController("security", SecurityController.class); - addController("user", UserController.class); - addController("login", LoginController.class); - addController("junit", JunitController.class); - addController("core", CoreController.class); - addController("analytics", AnalyticsController.class); - addController("project", ProjectController.class); - addController("util", UtilController.class); - addController("logger", LoggerController.class); - addController("mini-profiler", MiniProfilerController.class); - addController("notification", NotificationController.class); - addController("product", ProductController.class); - - AuthenticationManager.registerProvider(new DbLoginAuthenticationProvider(), Priority.Low); - AttachmentService.setInstance(new AttachmentServiceImpl()); - AnalyticsService.setInstance(new AnalyticsServiceImpl()); - RhinoService.register(); - CacheManager.addListener(RhinoService::clearCaches); - NotificationService.setInstance(NotificationServiceImpl.getInstance()); - - WarningService.setInstance(new WarningServiceImpl()); - - ViewService.setInstance(ViewServiceImpl.getInstance()); - OptionalFeatureService.setInstance(new OptionalFeatureServiceImpl()); - ThumbnailService.setInstance(new ThumbnailServiceImpl()); - ShortURLService.setInstance(new ShortURLServiceImpl()); - StatsService.setInstance(new StatsServiceImpl()); - SiteValidationService.setInstance(new SiteValidationServiceImpl()); - AnalyticsProviderRegistry.setInstance(new AnalyticsProviderRegistryImpl()); - SummaryStatisticRegistry.setInstance(new SummaryStatisticRegistryImpl()); - UsageMetricsService.setInstance(new UsageMetricsServiceImpl()); - CustomLabelService.setInstance(new CustomLabelServiceImpl()); - SecurityPointcutService.setInstance(new SecurityPointcutServiceImpl()); - AdminConsoleService.setInstance(new AdminConsoleServiceImpl()); - WikiRenderingService.setInstance(new WikiRenderingServiceImpl()); - VcsService.setInstance(new VcsServiceImpl()); - LabKeyScriptEngineManager.setInstance(new ScriptEngineManagerImpl()); - DocumentConversionService.setInstance(new DocumentConversionServiceImpl()); - DbLoginService.setInstance(new DbLoginManager()); - - try - { - ContainerTypeRegistry.get().register("normal", new NormalContainerType()); - ContainerTypeRegistry.get().register("tab", new TabContainerType()); - ContainerTypeRegistry.get().register("workbook", new WorkbookContainerType()); - } - catch (Exception e) - { - throw UnexpectedException.wrap(e); - } - - _warningProvider = new CoreWarningProvider(); - WarningService.get().register(_warningProvider); - - WebdavService.get().setResolver(ModuleStaticResolverImpl.get()); - // need to register webdav resolvers in init() instead of startupAfterSpringConfig since static module files are loaded during module startup - WebdavService.get().registerRootResolver(WebdavResolverImpl.get()); - WebdavService.get().registerRootResolver(WebFilesResolverImpl.get()); - - DefaultSchema.registerProvider(CoreQuerySchema.NAME, new DefaultSchema.SchemaProvider(this) - { - @Override - public boolean isAvailable(DefaultSchema schema, Module module) - { - return true; - } - - @Override - public QuerySchema createSchema(DefaultSchema schema, Module module) - { - return new CoreQuerySchema(schema.getUser(), schema.getContainer()); - } - }); - - if (CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) - { - DefaultSchema.registerProvider(BasePostgreSqlDialect.POSTGRES_SCHEMA_NAME, new DefaultSchema.SchemaProvider(this) - { - @Override - public boolean isAvailable(DefaultSchema schema, Module module) - { - return schema.getContainer().isRoot() && schema.getContainer().hasPermission(schema.getUser(), TroubleshooterPermission.class); - } - - @Override - public QuerySchema createSchema(DefaultSchema schema, Module module) - { - return new PostgresUserSchema(schema.getUser(), schema.getContainer()); - } - }); - } - - OptionalFeatureService.get().addExperimentalFeatureFlag(NotificationMenuView.EXPERIMENTAL_NOTIFICATION_MENU, "Notifications Menu", - "Notifications 'inbox' count display in the header bar with click to show the notifications panel of unread notifications.", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(DataColumn.EXPERIMENTAL_USE_QUERYSELECT_COMPONENT, "Use QuerySelect for row insert/update form", - "This feature will switch the query based select inputs on the row insert/update form to use the React QuerySelect" + - "component. This will allow for a user to view the first 100 options in the select but then use type ahead" + - "search to find the other select values.", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(SQLFragment.FEATUREFLAG_DISABLE_STRICT_CHECKS, "Disable SQLFragment strict checks", - "SQLFragment now has very strict usage validation, these checks may cause errors in code that has not been updated. Turn on this feature to disable checks.", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(LoginController.FEATUREFLAG_DISABLE_LOGIN_XFRAME, "Disable Login X-FRAME-OPTIONS=DENY", - "By default LabKey disables all framing of login related actions. Disabling this feature will revert to using the standard site settings.", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(PageTemplate.EXPERIMENTAL_SHORT_CIRCUIT_ROBOTS, - "Short-circuit robots", - "Save resources by not rendering pages marked as 'noindex' for robots. This is experimental as not all robots are search engines.", - false); - OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(AppProps.DEPRECATED_OBJECT_LEVEL_DISCUSSIONS, - "Restore Object-Level Discussions", - "This option and all support for Object-Level Discussions will be removed in LabKey Server v25.11.", - false, false, FeatureType.Deprecated)); - OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(SimpleTranslator.DEPRECATED_NULL_MISSING_VALUE_RESOLUTION, - "Resolve Missing Lookup Values to Null", - "When Lookup Validation for a field is not selected and lookup by alternate key is enabled, resolves missing lookup values to null instead of throwing an error. This option will be removed in LabKey Server v25.11.", - false, false, OptionalFeatureService.FeatureType.Deprecated)); - OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(TabLoader.FEATUREFLAG_UNESCAPE_BACKSLASH, - "Unescape backslash character on import", - "Treat backslash '\\' character as an escape character when loading data from file.", - false, false, OptionalFeatureService.FeatureType.Deprecated - )); - OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(AppProps.GENERATE_CONTROLLER_FIRST_URLS, - "Restore controller-first URLS", - "Generate URLs in a legacy format that puts the controller name before the folder path. This option will be removed in LabKey Server 26.3.", - false, false, OptionalFeatureService.FeatureType.Deprecated - )); - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.REJECT_CONTROLLER_FIRST_URLS, - "Reject controller-first URLs", - "Require standard path-first URLs. Note: This option will be ignored if the deprecated feature for generating controller-first URLs is enabled.", - false - ); - - SiteValidationService svc = SiteValidationService.get(); - if (null != svc) - { - svc.registerProviderFactory(getName(), new PermissionsValidatorFactory()); - svc.registerProviderFactory(getName(), new DisplayFormatValidationProviderFactory()); - } - - registerHealthChecks(); - - ContextListener.addNewInstallCompleteListener(() -> sendSystemReadyEmail(UserManager.getAppAdmins())); - - ScriptEngineManagerImpl.registerEncryptionMigrationHandler(); - - deleteTempFiles(); - } - - private void deleteTempFiles() - { - try - { - // Issue 46598 - clean up previously created temp files from file uploads - FileUtil.deleteDirectoryContents(SpringActionController.getTempUploadDir()); - } - catch (IOException e) - { - LOG.warn("Failed to clean up previously uploaded files from {}", SpringActionController.getTempUploadDir(), e); - } - } - - private void registerHealthChecks() - { - HealthCheckRegistry.get().registerHealthCheck("database", HealthCheckRegistry.DEFAULT_CATEGORY, () -> - { - Map healthValues = new HashMap<>(); - boolean allConnected = true; - for (DbScope dbScope : DbScope.getDbScopes()) - { - boolean dbConnected; - try (Connection conn = dbScope.getConnection()) - { - dbConnected = conn != null; - } - catch (SQLException e) - { - dbConnected = false; - } - - healthValues.put(dbScope.getDatabaseName(), dbConnected); - allConnected &= dbConnected; - } - - return new HealthCheck.Result(allConnected, healthValues); - } - ); - - HealthCheckRegistry.get().registerHealthCheck("modules", HealthCheckRegistry.TRIAL_INSTANCES_CATEGORY, () -> { - Map failures = ModuleLoader.getInstance().getModuleFailures(); - Map failureDetails = new HashMap<>(); - for (Map.Entry failure : failures.entrySet()) - { - failureDetails.put(failure.getKey(), failure.getValue().getMessage()); - } - return new HealthCheck.Result(failures.isEmpty(), failureDetails); - }); - - HealthCheckRegistry.get().registerHealthCheck("users", HealthCheckRegistry.TRIAL_INSTANCES_CATEGORY, () -> { - Map userHealth = new HashMap<>(); - ZonedDateTime now = ZonedDateTime.now(); - int userCount = UserManager.getUserCount(Date.from(now.toInstant())); - userHealth.put("registeredUsers", userCount); - return new HealthCheck.Result(userCount > 0, userHealth); - }); - } - - private void sendSystemReadyEmail(List users) - { - if (users.isEmpty()) - return; - - Map map = AppProps.getInstance().getStashedStartupProperties(); - String fromEmail = getValue(map, siteAvailableEmailFrom); - String subject = getValue(map, siteAvailableEmailSubject); - String body = getValue(map, siteAvailableEmailMessage); - - if (fromEmail == null || subject == null || body == null) - return; - - EmailService svc = EmailService.get(); - List messages = new ArrayList<>(); - for (User user: users) - { - EmailMessage message = svc.createMessage(fromEmail, Collections.singletonList(user.getEmail()), subject); - message.addContent(MimeMap.MimeType.HTML, body); - messages.add(message); - } - // For audit purposes, we use the first user as the originator of the message. - // Would be better to have this be a site admin, but we aren't guaranteed to have such a user - // for hosted sites. Another option is to use the guest user here, but that's strange. - svc.sendMessages(messages, users.get(0), ContainerManager.getRoot()); - } - - private @Nullable String getValue(Map map, StashedStartupProperties prop) - { - StartupPropertyEntry entry = map.get(prop); - return null != entry ? StringUtils.trimToNull(entry.getValue()) : null; - } - - @NotNull - @Override - protected Collection createWebPartFactories() - { - return List.of( - new AlwaysAvailableWebPartFactory("Contacts") - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext ctx, @NotNull WebPart webPart) - { - UserSchema schema = QueryService.get().getUserSchema(ctx.getUser(), ctx.getContainer(), CoreQuerySchema.NAME); - QuerySettings settings = new QuerySettings(ctx, QueryView.DATAREGIONNAME_DEFAULT); - - settings.setQueryName(CoreQuerySchema.USERS_TABLE_NAME); - - QueryView view = schema.createView(ctx, settings, null); - view.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); - view.setFrame(WebPartView.FrameType.PORTAL); - view.setTitle("Project Contacts"); - - return view; - } - }, - new BaseWebPartFactory("FolderNav") - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) - { - FolderNavigationForm form = getForm(portalCtx); - - form.setFolderMenu(new FolderMenu(portalCtx)); - - JspView view = new JspView<>("/org/labkey/core/project/folderNav.jsp", form); - view.setTitle("Folder Navigation"); - view.setFrame(WebPartView.FrameType.NONE); - return view; - } - - @Override - public boolean isAvailable(Container c, String scope, String location) - { - return false; - } - - private FolderNavigationForm getForm(ViewContext context) - { - FolderNavigationForm form = new FolderNavigationForm(); - form.setPortalContext(context); - return form; - } - }, - new BaseWebPartFactory("Workbooks") - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) - { - UserSchema schema = QueryService.get().getUserSchema(portalCtx.getUser(), portalCtx.getContainer(), SchemaKey.fromParts(CoreQuerySchema.NAME)); - WorkbookQueryView wbqview = new WorkbookQueryView(portalCtx, schema); - VBox box = new VBox(new WorkbookSearchView(wbqview), wbqview); - box.setFrame(WebPartView.FrameType.PORTAL); - box.setTitle("Workbooks"); - return box; - } - - @Override - public boolean isAvailable(Container c, String scope, String location) - { - return !c.isWorkbook() && "folder".equals(scope) && location.equalsIgnoreCase(HttpView.BODY); - } - }, - new BaseWebPartFactory("Workbook Description") - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) - { - JspView view = new JspView<>("/org/labkey/core/workbook/workbookDescription.jsp"); - view.setTitle("Workbook Description"); - view.setFrame(WebPartView.FrameType.NONE); - return view; - } - - @Override - public boolean isAvailable(Container c, String scope, String location) - { - return false; - } - }, - new AlwaysAvailableWebPartFactory(PROJECTS_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) - { - JspView view = new JspView<>("/org/labkey/core/project/projects.jsp", webPart); - - String title = webPart.getPropertyMap().getOrDefault("title", "Projects"); - view.setTitle(title); - - if (portalCtx.hasPermission(getClass().getName(), AdminPermission.class)) - { - NavTree customize = new NavTree(""); - customize.setScript("customizeProjectWebpart" + webPart.getRowId() + "();"); - view.setCustomize(customize); - } - return view; - } - }, - new AlwaysAvailableWebPartFactory("Subfolders", WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPart createWebPart() - { - // Issue 44913: Set the default properties for all new instances of the Subfolders webpart - WebPart webPart = super.createWebPart(); - return setDefaultProperties(webPart); - } - - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) - { - if (webPart.getPropertyMap().isEmpty()) - { - // Configure to show subfolders if not previously configured - webPart = setDefaultProperties(new WebPart(webPart)); - } - - JspView view = new JspView<>("/org/labkey/core/project/projects.jsp", webPart); - view.setTitle(webPart.getPropertyMap().get("title")); - - if (portalCtx.hasPermission(getClass().getName(), AdminPermission.class)) - { - NavTree customize = new NavTree(""); - customize.setScript("customizeProjectWebpart" + webPart.getRowId() + "(" + webPart.getRowId() + ", " + PageFlowUtil.jsString(webPart.getPageId()) + ", " + webPart.getIndex() + ");"); - view.setCustomize(customize); - } - - return view; - } - - private WebPart setDefaultProperties(WebPart webPart) - { - webPart.setProperty("title", "Subfolders"); - webPart.setProperty("containerFilter", ContainerFilter.Type.CurrentAndFirstChildren.name()); - webPart.setProperty("containerTypes", "folder"); - return webPart; - } - }, - new AlwaysAvailableWebPartFactory("Custom Menu", true, true, WebPartFactory.LOCATION_MENUBAR) - { - @Override - public WebPartView getWebPartView(@NotNull final ViewContext portalCtx, @NotNull WebPart webPart) - { - final CustomizeMenuForm form = AdminController.getCustomizeMenuForm(webPart); - String title = "My Menu"; - if (form.getTitle() != null && !form.getTitle().isEmpty()) - title = form.getTitle(); - - WebPartView view; - if (form.isChoiceListQuery()) - { - view = MenuViewFactory.createMenuQueryView(portalCtx, title, form); - } - else - { - view = MenuViewFactory.createMenuFolderView(portalCtx, title, form); - } - view.setFrame(WebPartView.FrameType.PORTAL); - return view; - } - - @Override - public HttpView getEditView(WebPart webPart, ViewContext context) - { - CustomizeMenuForm form = AdminController.getCustomizeMenuForm(webPart); - JspView view = new JspView<>("/org/labkey/core/admin/customizeMenu.jsp", form); - view.setTitle(form.getTitle()); - view.setFrame(WebPartView.FrameType.PORTAL); - return view; - } - } - ); - } - - @Override - public void afterUpdate(ModuleContext moduleContext) - { - if (moduleContext.isNewInstall()) - { - bootstrap(); - } - - // Increment on every core module upgrade to defeat browser caching of static resources. - if (ModuleLoader.getInstance().shouldInsertData()) - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - - // Allow dialect to make adjustments to the just upgraded core database (e.g., install aggregate functions, etc.) - CoreSchema.getInstance().getSqlDialect().afterCoreUpgrade(moduleContext); - - // The core SQL scripts install aggregate functions and other objects that dialects need to know about. Prepare - // the previously initialized dialects again to make sure they're aware of all the changes. Prepare all the - // initialized scopes because we could have more than one scope pointed at the core database (e.g., external - // schemas). See #17077 (pg example) and #19177 (ss example). - for (DbScope scope : DbScope.getInitializedDbScopes()) - scope.getSqlDialect().prepare(scope); - - // Now that we know the standard containers have been created, add a listener that warms the just-cleared caches with - // core.Containers metadata and a few common containers. This may prevent some deadlocks during upgrade, #33550. - CacheManager.addListener(() -> { - ContainerManager.getRoot(); - ContainerManager.getSharedContainer(); - if (ModuleLoader.getInstance().shouldInsertData()) - { - ContainerManager.getHomeContainer(); - } - }); - } - - private void bootstrap() - { - if (ModuleLoader.getInstance().shouldInsertData()) - { - // Create the initial groups - GroupManager.bootstrapGroup(Group.groupUsers, "Users"); - GroupManager.bootstrapGroup(Group.groupGuests, "Guests"); - - // Other containers inherit permissions from root; admins get all permissions, users & guests none - Role noPermsRole = RoleManager.getRole(NoPermissionsRole.class); - Role readerRole = RoleManager.getRole(ReaderRole.class); - - ContainerManager.bootstrapContainer("/", noPermsRole, noPermsRole); - Container rootContainer = ContainerManager.getRoot(); - - // Create all the standard containers (Home, Home/support, Shared) using an empty Collaboration folder type - FolderType collaborationType = new CollaborationFolderType(Collections.emptyList()); - - // Users & guests can read from /home - Container home = ContainerManager.bootstrapContainer(ContainerManager.HOME_PROJECT_PATH, readerRole, readerRole); - home.setFolderType(collaborationType, null); - - ContainerManager.createDefaultSupportContainer().setFolderType(collaborationType, null); - - // Only users can read from /Shared - ContainerManager.bootstrapContainer(ContainerManager.SHARED_CONTAINER_PATH, readerRole, null).setFolderType(collaborationType, null); - - try - { - // Need to insert standard MV indicators for the root -- okay to call getRoot() since we just created it. - String rootContainerId = rootContainer.getId(); - TableInfo mvTable = CoreSchema.getInstance().getTableInfoMvIndicators(); - - for (Map.Entry qcEntry : MvUtil.getDefaultMvIndicators().entrySet()) - { - Map params = new HashMap<>(); - params.put("Container", rootContainerId); - params.put("MvIndicator", qcEntry.getKey()); - params.put("Label", qcEntry.getValue()); - - Table.insert(null, mvTable, params); - } - } - catch (Throwable t) - { - ExceptionUtil.logExceptionToMothership(null, t); - } - - List guids = Stream.of(ContainerManager.HOME_PROJECT_PATH, ContainerManager.SHARED_CONTAINER_PATH) - .map(ContainerManager::getForPath) - .filter(Objects::nonNull) - .map(Container::getEntityId) - .collect(Collectors.toList()); - ContainerManager.setExcludedProjects(guids, () -> {}); - } - else - { - // It's very difficult to bootstrap without the root or shared containers in place; create them now and - // we'll delete them later - Container root = ContainerManager.ensureContainer("/", User.getAdminServiceUser()); - Table.insert(null, CoreSchema.getInstance().getTableInfoContainers(), Map.of("Parent", root.getId(), "Name", "Shared")); - } - } - - - @Override - public CoreUpgradeCode getUpgradeCode() - { - return new CoreUpgradeCode(); - } - - - @Override - public void destroy() - { - super.destroy(); - UsageReportingLevel.shutdown(); - } - - - @Override - public void startupAfterSpringConfig(ModuleContext moduleContext) - { - // Any containers in the cache have bogus folder types since they aren't registered until startup(). See #10310 - ContainerManager.clearCache(); - - checkForMissingDbViews(); - - ProductConfiguration.handleStartupProperties(); - // This listener deletes all properties; make sure it executes after most of the other listeners - ContainerManager.addContainerListener(new CoreContainerListener(), ContainerManager.ContainerListener.Order.Last); - ContainerManager.addContainerListener(new FolderSettingsCache.FolderSettingsCacheListener()); - SecurityManager.init(); - FolderTypeManager.get().registerFolderType(this, FolderType.NONE); - FolderTypeManager.get().registerFolderType(this, new CollaborationFolderType()); - - AnalyticsServiceImpl.get().resetCSP(); - - if (moduleContext.isNewInstall() && ModuleLoader.getInstance().shouldInsertData()) - { - // To initialize the portal layout correctly, we need to add the web parts after the folder types have been - // registered. Thus, it needs to be here in startupAfterSpringConfig() instead of grouped in bootstrap(). - Container homeContainer = ContainerManager.getHomeContainer(); - int count = Portal.getParts(homeContainer, homeContainer.getFolderType().getDefaultPageId(homeContainer)).size(); - addWebPart(PROJECTS_WEB_PART_NAME, homeContainer, HttpView.BODY, count); - } - - EmailService.setInstance(new EmailServiceImpl()); - - if (null != AuditLogService.get() && AuditLogService.get().getClass() != DefaultAuditProvider.class) - { - AuditLogService.get().registerAuditType(new UserAuditProvider()); - AuditLogService.get().registerAuditType(new GroupAuditProvider()); - AuditLogService.get().registerAuditType(new AttachmentAuditProvider()); - AuditLogService.get().registerAuditType(new ContainerAuditProvider()); - AuditLogService.get().registerAuditType(new FileSystemAuditProvider()); - AuditLogService.get().registerAuditType(new ClientApiAuditProvider()); - AuditLogService.get().registerAuditType(new AuthenticationSettingsAuditTypeProvider()); - AuditLogService.get().registerAuditType(new TransactionAuditProvider()); - AuditLogService.get().registerAuditType(new ModulePropertiesAuditProvider()); - - DataStateManager.getInstance().registerDataStateHandler(new CoreQCStateHandler()); - } - ContextListener.addShutdownListener(TempTableTracker.getShutdownListener()); - ContextListener.addShutdownListener(DavController.getShutdownListener()); - ContextListener.addShutdownListener(ShutdownListener.of("Temp file cleanup", null, this::deleteTempFiles)); - - SimpleMetricsService.setInstance(new SimpleMetricsServiceImpl()); - - // Export action stats on graceful shutdown - ContextListener.addShutdownListener(new ShutdownListener() { - @Override - public String getName() - { - return "Action stats export"; - } - - @Override - public void shutdownPre() - { - try - { - // Halt firing of Quartz triggers - Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler(); - scheduler.standby(); - } - catch (SchedulerException ignored) - { - } - - Logger logger = LogManager.getLogger(ActionsTsvWriter.class); - - if (null != logger) - { - StringBuilder buf = new StringBuilder(); - - try (TSVWriter writer = new ActionsTsvWriter()) - { - writer.write(buf); - } - catch (IOException e) - { - LOG.error("Exception exporting action stats", e); - } - - logger.info(buf.toString()); - LOG.info("Completed logging statistics for actions prior to web application shut down"); - } - } - - @Override - public void shutdownStarted() - { - try - { - // Clean up Quartz resources and wait for jobs to complete - Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler(); - scheduler.shutdown(true); - } - catch (SchedulerException ignored) - { - } - } - }); - - // populate look and feel settings and site settings with values read from startup properties as appropriate for not bootstrap - populateLookAndFeelResourcesWithStartupProps(); - AllowedExternalResourceHosts.registerStartupProperties(); - AllowedExternalResourceHosts.registerHosts(); - WriteableLookAndFeelProperties.populateLookAndFeelWithStartupProps(); - WriteableAppProps.populateSiteSettingsWithStartupProps(); - // create users and groups and assign roles with values read from startup properties as appropriate for not bootstrap - SecurityManager.populateStartupProperties(); - // This method depends on resources (FolderType) from other modules, so invoke after startup - ContextListener.addStartupListener(new StartupListener() - { - @Override - public String getName() - { - return "CoreModule.populateSiteSettingsWithStartupProps"; - } - - @Override - public void moduleStartupComplete(ServletContext servletContext) - { - populateSiteSettingsWithStartupProps(); - } - }); - - // Handle optional feature startup properties as late as possible; we want all optional features to be registered first - ContextListener.addStartupListener(new OptionalFeatureStartupListener()); - - LabKeyScriptEngineManager svc = LabKeyScriptEngineManager.get(); - // populate script engine definitions values read from startup properties - if (svc instanceof ScriptEngineManagerImpl) - ((ScriptEngineManagerImpl)svc).populateScriptEngineDefinitionsWithStartupProps(); - - // populate folder types from startup properties as appropriate for not bootstrap - FolderTypeManager.get().populateWithStartupProps(); - LimitActiveUsersSettings.populateStartupProperties(); - - AdminController.registerAdminConsoleLinks(); - AdminController.registerManagementTabs(); - AnalyticsController.registerAdminConsoleLinks(); - UserController.registerAdminConsoleLinks(); - LoggerController.registerAdminConsoleLinks(); - CoreController.registerAdminConsoleLinks(); - - FolderTypeManager.get().registerFolderType(this, new WorkbookFolderType()); - - SecurityManager.addViewFactory(new SecurityController.GroupDiagramViewFactory()); - - FolderSerializationRegistry fsr = FolderSerializationRegistry.get(); - if (null != fsr) - { - fsr.addFactories(new FolderTypeWriterFactory(), new FolderTypeImporterFactory()); - fsr.addFactories(new MissingValueWriterFactory(), new MissingValueImporterFactory()); - fsr.addFactories(new SearchSettingsWriterFactory(), new SearchSettingsImporterFactory()); - fsr.addFactories(new PageWriterFactory(), new PageImporterFactory()); - fsr.addFactories(new ModulePropertiesWriterFactory(), new ModulePropertiesImporterFactory()); - fsr.addFactories(new SecurityGroupWriterFactory(), new SecurityGroupImporterFactory()); - fsr.addFactories(new RoleAssignmentsWriterFactory(), new RoleAssignmentsImporterFactory()); - fsr.addFactories(new DataStateWriter.Factory(), new DataStateImporter.Factory()); - fsr.addFactories(new FileBrowserConfigWriter.Factory(), new FileBrowserConfigImporter.Factory()); - fsr.addImportFactory(new SubfolderImporterFactory()); - } - - SearchService ss = SearchService.get(); - if (null != ss) - { - ss.addDocumentParser(new TabLoader.CsvFactoryNoConversions()); - ss.addDocumentProvider(this); - - // Register indexable DataLoaders with the search service - DataLoaderServiceImpl.get().getFactories() - .stream() - .filter(DataLoaderFactory::indexable) - .forEach(ss::addDocumentParser); - } - - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_NO_GUESTS, - "No Guest Account", - "Disable the guest account", - false); - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_BLOCKER, - "Block malicious clients", - "Reject requests from clients that appear malicious. Turn this feature off if you want to run a security scanner.", - false); - OptionalFeatureService.get().addExperimentalFeatureFlag(FEATURE_FLAG_DISABLE_ENFORCE_CSP, - "Disable enforce Content Security Policy", - "Stop sending the " + ContentSecurityPolicyFilter.ContentSecurityPolicyType.Enforce.getHeaderName() + " header to browsers, " + - "but continue sending the " + ContentSecurityPolicyFilter.ContentSecurityPolicyType.Report.getHeaderName() + " header. " + - "This turns off an important layer of security for the entire site, so use it as a last resort only on a temporary basis " + - "(e.g., if an enforce CSP breaks critical functionality).", - false); - OptionalFeatureService.get().addExperimentalFeatureFlag(DataRegion.EXPERIMENTAL_DATA_REGION_ASYNC_TOTAL_ROWS, - "Data Region Async Total Rows", - "Enable asynchronous calculation of total rows for data regions. This can improve performance for large datasets.", - false); - - OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(EXPERIMENTAL_LOCAL_MARKETING_UPDATE, - "Self test marketing updates", "Test marketing updates from this local server (requires the mothership module).", false, true, FeatureType.Experimental)); - OptionalFeatureService.get().addFeatureListener(EXPERIMENTAL_LOCAL_MARKETING_UPDATE, (feature, enabled) -> { - // update the timer task when this setting changes - MothershipReport.setSelfTestMarketingUpdates(enabled); - UsageReportingLevel.reportNow(); - }); - - if (null != PropertyService.get()) - { - PropertyService.get().registerDomainKind(new UsersDomainKind()); - if (ModuleLoader.getInstance().shouldInsertData()) - UsersDomainKind.ensureDomain(moduleContext); - } - - // Register the standard, wiki-based terms-of-use provider - SecurityManager.addTermsOfUseProvider(new WikiTermsOfUseProvider()); - - if (null != PropertyService.get()) - PropertyService.get().registerDomainKind(new TestDomainKind()); - - AuthenticationManager.populateSettingsWithStartupProps(); - AnalyticsServiceImpl.populateSettingsWithStartupProps(); - - UsageMetricsService.get().registerUsageMetrics(getName(), () -> { - Map results = new HashMap<>(); - Map javaInfo = new HashMap<>(); - javaInfo.put("java.vendor", System.getProperty("java.vendor")); - javaInfo.put("java.vm.name", System.getProperty("java.vm.name")); - results.put("javaRuntime", javaInfo); - results.put("distributionFilename", AppProps.getInstance().getDistributionFilename()); - results.put("applicationMenuDisplayMode", LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getApplicationMenuDisplayMode()); - results.put("optionalFeatures", OptionalFeatureService.get().getOptionalFeatureFlags().stream() - .collect(Collectors.groupingBy(optionalFeatureFlag -> optionalFeatureFlag.getType().name().toLowerCase(), - Collectors.mapping(flag -> flag, Collectors.toMap(OptionalFeatureFlag::getFlag, OptionalFeatureFlag::isEnabled)) - )) - ); - results.put("productFeaturesEnabled", ProductRegistry.getProductFeatureSet()); - results.put("analyticsTrackingStatus", AnalyticsServiceImpl.get().getTrackingStatus().toString()); - String labkeyContextPath = AppProps.getInstance().getContextPath(); - results.put("webappContextPath", labkeyContextPath); - results.put("embeddedTomcat", true); - boolean customLog4JConfig = false; - if (ModuleLoader.getServletContext() != null) - { - customLog4JConfig = Boolean.parseBoolean(ModuleLoader.getServletContext().getInitParameter("org.labkey.customLog4JConfig")); - } - results.put("customLog4JConfig", customLog4JConfig); - results.put("containerRelativeURL", AppProps.getInstance().getUseContainerRelativeURL()); - results.put("runtimeMode", AppProps.getInstance().isDevMode() ? "development" : "production"); - Set deployedApps = new HashSet<>(CoreWarningProvider.collectAllDeployedApps()); - deployedApps.remove(labkeyContextPath); - if (labkeyContextPath.startsWith("/")) - { - deployedApps.remove(labkeyContextPath.substring(1)); - } - results.put("otherDeployedWebapps", StringUtils.join(deployedApps, ",")); - - // Report the total number of login entries in the audit log - results.put("totalLogins", UserManager.getAuthCount(null, false, false, false)); - results.put("apiKeyLogins", UserManager.getAuthCount(null, false, true, false)); - results.put("sessionTimeout", ModuleLoader.getServletContext().getSessionTimeout()); - results.put("userLimits", new LimitActiveUsersSettings().getMetricsMap()); - results.put("systemUserCount", UserManager.getSystemUserCount()); - Calendar cal = new GregorianCalendar(); - cal.add(Calendar.DATE, -30); - results.put("uniqueRecentUserCount", UserManager.getAuthCount(cal.getTime(), false, false, true)); - results.put("uniqueRecentNonSystemUserCount", UserManager.getAuthCount(cal.getTime(), true, false, true)); - if (OptionalFeatureService.get().isFeatureEnabled(FEATURE_FLAG_EXTENDED_METRICS)) - { - // Optionally include a list of active users, Issue #53050 - results.put("activeUsers", UserManager.getActiveUsers().stream() - .filter(u -> !u.isSystem()) - .map(User::getEmail) - .toList() - ); - } - - results.put("workbookCount", ContainerManager.getWorkbookCount()); - results.put("archivedFolderCount", ContainerManager.getArchivedContainerCount()); - results.put("databaseSize", CoreSchema.getInstance().getSchema().getScope().getDatabaseSize()); - results.put("scriptEngines", LabKeyScriptEngineManager.get().getScriptEngineMetrics()); - results.put("customLabels", CustomLabelService.get().getCustomLabelMetrics()); - Map roleAssignments = new HashMap<>(); - final String roleCountSql = "SELECT COUNT(*) FROM core.RoleAssignments WHERE userid > 0 AND role = ?"; - roleAssignments.put("assayDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.assay.security.AssayDesignerRole").getObject(Long.class)); - roleAssignments.put("dataClassDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.DataClassDesignerRole").getObject(Long.class)); - roleAssignments.put("sampleTypeDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.SampleTypeDesignerRole").getObject(Long.class)); - results.put("roleAssignments", roleAssignments); - - Map allowListCounts = new HashMap<>(); - for (AllowListType type : AllowListType.values()) - { - allowListCounts.put(type.name(), type.getValues().size()); - } - results.put("allowListCounts", allowListCounts); - - return results; - }); - - UsageMetricsService.get().registerUsageMetrics(getName(), WebSocketConnectionManager.getInstance()); - UsageMetricsService.get().registerUsageMetrics(getName(), DbLoginManager.getMetricsProvider()); - UsageMetricsService.get().registerUsageMetrics(getName(), SecurityManager.getMetricsProvider()); - UsageMetricsService.get().registerUsageMetrics(getName(), DisplayFormatAnalyzer.getMetricsProvider()); - UsageMetricsService.get().registerUsageMetrics(getName(), Portal.getMetricsProvider()); - - if (AppProps.getInstance().isDevMode()) - AntiVirusProviderRegistry.get().registerAntiVirusProvider(new DummyAntiVirusService.Provider()); - - FileContentService fileContentService = FileContentService.get(); - if (fileContentService != null) - fileContentService.addFileListener(WebFilesResolverImpl.get()); - - RoleManager.registerPermission(new QCAnalystPermission()); - MarkdownService.setInstance(new MarkdownServiceImpl()); - - // initialize email preference service and listeners - MessageConfigService.setInstance(new EmailPreferenceConfigServiceImpl()); - ContainerManager.addContainerListener(new EmailPreferenceContainerListener()); - UserManager.addUserListener(new EmailPreferenceUserListener()); - - DatabaseMigrationService.get().registerHandler(CoreSchema.getInstance().getSchema(), new DefaultMigrationHandler() - { - @Override - public void beforeVerification(DbSchema targetSchema) - { - super.beforeVerification(targetSchema); - - // Delete root and shared containers that were needed for bootstrapping - TableInfo containers = CoreSchema.getInstance().getTableInfoContainers(); - Table.delete(containers); - DbScope targetScope = DbScope.getLabKeyScope(); - new SqlExecutor(targetScope).execute("ALTER SEQUENCE core.containers_rowid_seq RESTART"); // Reset Containers sequence - } - - @Override - public List getTablesToCopy(DbSchema targetSchema) - { - List tablesToCopy = super.getTablesToCopy(targetSchema); - tablesToCopy.remove(targetSchema.getTable("Modules")); - tablesToCopy.remove(targetSchema.getTable("SqlScripts")); - tablesToCopy.remove(targetSchema.getTable("UpgradeSteps")); - - return tablesToCopy; - } - }); - } - - // Issue 7527: Auto-detect missing SQL views and attempt to recreate - private void checkForMissingDbViews() - { - ModuleLoader.getInstance().getModules().stream() - .map(FileSqlScriptProvider::new) - .flatMap(p -> p.getSchemas().stream() - .filter(schema-> SchemaUpdateType.Before.getScript(p, schema) != null || SchemaUpdateType.After.getScript(p, schema) != null) - ) - .filter(schema -> TableXmlUtils.compareXmlToMetaData(schema, false, false, true).hasViewProblem()) - .findAny() - .ifPresent(schema -> - { - LOG.warn("At least one database view was not as expected in the {} schema. Attempting to recreate views automatically", schema.getName()); - ModuleLoader.getInstance().recreateViews(); - }); - } - - @Override - public void registerServlets(ServletContext servletCtx) - { -// even though there is one webdav tree rooted at "/" we still use two servlet bindings. -// This is because we want /_webdav/* to be resolved BEFORE all other servlet-mappings -// and /* to resolve AFTER all other servlet-mappings - _webdavServletDynamic = servletCtx.addServlet("static", new WebdavServlet(true)); - _webdavServletDynamic.setMultipartConfig(SpringActionController.getMultiPartConfigElement()); - _webdavServletDynamic.addMapping("/_webdav/*"); - } - - @Override - public void registerFinalServlets(ServletContext servletCtx) - { - _webdavServletDynamic.addMapping("/"); - } - - @Override - public void startBackgroundThreads() - { - SystemMaintenance.setTimer(); - ThumbnailServiceImpl.startThread(); - // Launch in the background, but delay by 10 seconds to reduce impact on other startup tasks - _warningProvider.startSchemaCheck(10); - - // Start up the default Quartz scheduler, used in many places - try - { - StdSchedulerFactory.getDefaultScheduler().start(); - } - catch (SchedulerException e) - { - throw UnexpectedException.wrap(e); - } - - if (MothershipReport.shouldReceiveMarketingUpdates()) - { - if (AppProps.getInstance().getUsageReportingLevel() == UsageReportingLevel.NONE) - { - // force the usage reporting level to on for community edition distributions - WriteableAppProps appProps = AppProps.getWriteableInstance(); - appProps.setUsageReportingLevel(UsageReportingLevel.ON); - appProps.save(User.getAdminServiceUser()); - } - } - // On bootstrap in production mode, this will send an initial ping with very little information, as the admin will - // not have set up their account yet. On later startups, depending on the reporting level, this will send an immediate - // ping, and then once every 24 hours. - UsageReportingLevel.reportNow(); - TempTableTracker.init(); - - // Loading the PDFBox font cache can be very slow on some agents; fill it proactively. Issue 50601 - JobRunner.getDefault().execute(() -> { - try - { - long start = System.currentTimeMillis(); - FontMapper mapper = FontMappers.instance(); - Method method = mapper.getClass().getMethod("getProvider"); - method.setAccessible(true); - method.invoke(mapper); - long duration = System.currentTimeMillis() - start; - LOG.info("Ensuring PDFBox on-disk font cache took {} seconds", Math.round(duration / 100.0) / 10.0); - } - catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) - { - LOG.warn("Unable to initialize PDFBox font cache", e); - } - }); - } - - @Override - public @NotNull List getDetailedSummary(Container c, User user) - { - long childContainerCount = ContainerManager.getChildren(c).stream().filter(Container::isInFolderNav).count(); - if (childContainerCount == 0) - return Collections.emptyList(); - return List.of(new Summary(childContainerCount, "Subfolder")); - } - - @Override - public JSONObject getPageContextJson(ContainerUser context) - { - JSONObject json = new JSONObject(getDefaultPageContextJson(context.getContainer())); - json.put("productFeatures", ProductRegistry.getProductFeatureSet()); - json.put("primaryApplicationId", ProductRegistry.get().getPrimaryApplicationId(context.getContainer())); - json.put(AppProps.DEPRECATED_OBJECT_LEVEL_DISCUSSIONS, AppProps.getInstance().isOptionalFeatureEnabled(AppProps.DEPRECATED_OBJECT_LEVEL_DISCUSSIONS)); - return json; - } - - @Override - public String getTabName(ViewContext context) - { - return "Portal"; - } - - - @Override - public ActionURL getTabURL(Container c, User user) - { - if (user == null) - return AppProps.getInstance().getHomePageActionURL(); - - return PageFlowUtil.urlProvider(ProjectUrls.class).getBeginURL(c); - } - - @Override - public TabDisplayMode getTabDisplayMode() - { - return TabDisplayMode.DISPLAY_USER_PREFERENCE_DEFAULT; - } - - @Override - @NotNull - public Set getIntegrationTests() - { - // Must be mutable since we add the dialect tests below - Set testClasses = Sets.newHashSet - ( - AdminController.SchemaVersionTestCase.class, - AdminController.SerializationTest.class, - AdminController.TestCase.class, - AdminController.WorkbookDeleteTestCase.class, - AllowListType.TestCase.class, - AttachmentServiceImpl.TestCase.class, - CoreController.TestCase.class, - DataRegion.TestCase.class, - DavController.TestCase.class, - EmailServiceImpl.TestCase.class, - FilesSiteSettingsAction.TestCase.class, - LoggerController.TestCase.class, - LoggingTestCase.class, - LoginController.TestCase.class, - MarkdownTestCase.class, - ModuleInfoTestCase.class, - ModulePropertiesTestCase.class, - ModuleStaticResolverImpl.TestCase.class, - NotificationServiceImpl.TestCase.class, - PortalJUnitTest.class, - PostgreSqlInClauseTest.class, - ProductRegistry.TestCase.class, - RadeoxRenderer.RadeoxRenderTest.class, - RhinoService.TestCase.class, - SchemaXMLTestCase.class, - SecurityApiActions.TestCase.class, - SecurityController.TestCase.class, - SqlDialect.DialectTestCase.class, - SqlScriptController.TestCase.class, - TableViewFormTestCase.class, - UnknownSchemasTest.class, - UserController.TestCase.class - ); - - testClasses.addAll(SqlDialectManager.getAllJUnitTests()); - - return testClasses; - } - - @NotNull - @Override - public Set getUnitTests() - { - return Set.of( - ApiJsonWriter.TestCase.class, - ClassLoaderTestCase.class, - CopyFileRootPipelineJob.TestCase.class, - OutOfRangeDisplayColumn.TestCase.class, - PostgreSqlVersion.TestCase.class, - ScriptEngineManagerImpl.TestCase.class, - StatsServiceImpl.TestCase.class, - - - // Radeox tests - SimpleListTest.class, - ExampleListFormatterTest.class, - AtoZListFormatterTest.class, - BaseRenderEngineTest.class, - BasicRegexTest.class, - ItalicFilterTest.class, - BoldFilterTest.class, - KeyFilterTest.class, - NewlineFilterTest.class, - LineFilterTest.class, - TypographyFilterTest.class, - HtmlRemoveFilterTest.class, - StrikeThroughFilterTest.class, - UrlFilterTest.class, - ParamFilterTest.class, - FilterPipeTest.class, - EscapeFilterTest.class, - LinkTestFilterTest.class, - WikiLinkFilterTest.class, - SmileyFilterTest.class, - ListFilterTest.class, - HeadingFilterTest.class - ); - } - - @Override - public DbSchema createModuleDbSchema(DbScope scope, String metaDataName, Map tableInfoFactoryMap) - { - // Special case for the "labkey" schema we create in every module data source - if ("labkey".equals(metaDataName)) - return new LabKeyDbSchema(scope, tableInfoFactoryMap); - - return super.createModuleDbSchema(scope, metaDataName, tableInfoFactoryMap); - } - - @Override - @NotNull - public Collection getSchemaNames() - { - return List.of - ( - CoreSchema.getInstance().getSchemaName(), // core - PropertySchema.getInstance().getSchemaName(), // prop - TestSchema.getInstance().getSchemaName(), // test - DbSchema.TEMP_SCHEMA_NAME // temp - ); - } - - @NotNull - @Override - public Collection getProvisionedSchemaNames() - { - return Collections.singleton(DbSchema.TEMP_SCHEMA_NAME); - } - - @NotNull - @Override - public Set getSchemasToTest() - { - Set result = new LinkedHashSet<>(super.getSchemasToTest()); - - // Add the "labkey" schema in all module data sources as well... should match application.properties - for (String dataSourceName : ModuleLoader.getInstance().getAllModuleDataSourceNames()) - { - DbScope scope = DbScope.getDbScope(dataSourceName); - if (scope != null) - { - result.add(scope.getLabKeySchema()); - } - } - - return result; - } - - @Override - public void enumerateDocuments(SearchService.TaskIndexingQueue queue, Date since) - { - Container c = queue.getContainer(); - if (c.isRoot()) - return; - - Runnable r = () -> { - Container p = c.getProject(); - if (null == p) - return; - String title; - String keywords; - String body; - - // UNDONE: generalize to other folder types - StudyService svc = StudyService.get(); - Study study = svc != null ? svc.getStudy(c) : null; - - if (null != study) - { - title = study.getSearchDisplayTitle(); - keywords = study.getSearchKeywords(); - body = study.getSearchBody(); - } - else - { - String type = c.getContainerNoun(true); - - String containerTitle = c.getTitle(); - - String description = StringUtils.trimToEmpty(c.getDescription()); - title = type + " -- " + containerTitle; - User u_user = UserManager.getUser(c.getCreatedBy()); - String user = (u_user == null) ? "" : u_user.getDisplayName(User.getSearchUser()); - keywords = description + " " + type + " " + user; - body = type + " " + containerTitle + (c.isProject() ? "" : " in Project " + p.getName()); - body += "\n" + description; - } - - String identifiers = c.getName(); - - Map properties = new HashMap<>(); - - assert (null != keywords); - properties.put(SearchService.PROPERTY.identifiersMed.toString(), identifiers); - properties.put(SearchService.PROPERTY.keywordsMed.toString(), keywords); - properties.put(SearchService.PROPERTY.title.toString(), title); - properties.put(SearchService.PROPERTY.categories.toString(), SearchService.navigationCategory.getName()); - ActionURL startURL = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(c); - startURL.setExtraPath(c.getId()); - WebdavResource doc = new SimpleDocumentResource(c.getParsedPath(), - "link:" + c.getId(), - c.getEntityId(), - "text/plain", - body, - startURL, - UserManager.getUser(c.getCreatedBy()), c.getCreated(), - null, null, - properties); - queue.addResource(doc); - }; - r.run(); - } - - @Override - public void indexDeleted() - { - new SqlExecutor(CoreSchema.getInstance().getSchema()).execute("UPDATE core.Documents SET LastIndexed = NULL"); - } - - /** - * Handles startup props for LookAndFeelSettings resources - */ - private void populateLookAndFeelResourcesWithStartupProps() - { - ModuleLoader.getInstance().handleStartupProperties(new StandardStartupPropertyHandler<>(WriteableLookAndFeelProperties.SCOPE_LOOK_AND_FEEL_SETTINGS, ResourceType.class) - { - @Override - public void handle(Map map) - { - boolean incrementRevision = false; - - for (Map.Entry entry : map.entrySet()) - { - SiteResourceHandler handler = getResourceHandler(entry.getKey()); - if (handler != null) - incrementRevision |= setSiteResource(handler, entry.getValue(), User.guest); - } - - // Bump the look & feel revision so browsers retrieve the new logo, custom stylesheet, etc. - if (incrementRevision) - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - } - }); - } - - /** - * This method handles the home project settings - */ - private void populateSiteSettingsWithStartupProps() - { - Map props = AppProps.getInstance().getStashedStartupProperties(); - - StartupPropertyEntry folderTypeEntry = props.get(homeProjectFolderType); - if (null != folderTypeEntry) - { - FolderType folderType = FolderTypeManager.get().getFolderType(folderTypeEntry.getValue()); - if (folderType != null) - // using guest user since the server startup doesn't have a true user (this will be used for audit events) - ContainerManager.getHomeContainer().setFolderType(folderType, User.guest); - else - LOG.error("Unable to find folder type for home project during server startup: " + folderTypeEntry.getValue()); - } - - StartupPropertyEntry resetPermissionsEntry = props.get(homeProjectResetPermissions); - if (null != resetPermissionsEntry && Boolean.valueOf(resetPermissionsEntry.getValue())) - { - // reset the home project permissions to remove the default assignments given at server install - MutableSecurityPolicy homePolicy = new MutableSecurityPolicy(ContainerManager.getHomeContainer()); - SecurityPolicyManager.savePolicy(homePolicy, User.getAdminServiceUser()); - // remove the guest role assignment from the support subfolder - Group guests = SecurityManager.getGroup(Group.groupGuests); - if (null != guests) - { - Container supportFolder = ContainerManager.getDefaultSupportContainer(); - if (supportFolder != null) - { - MutableSecurityPolicy supportPolicy = new MutableSecurityPolicy(supportFolder.getPolicy()); - for (Role assignedRole : supportPolicy.getAssignedRoles(guests)) - supportPolicy.removeRoleAssignment(guests, assignedRole); - SecurityPolicyManager.savePolicy(supportPolicy, User.getAdminServiceUser()); - } - } - } - - StartupPropertyEntry webparts = props.get(homeProjectWebparts); - if (null != webparts) - { - // Clear existing webparts added by core and wiki modules - Container homeContainer = ContainerManager.getHomeContainer(); - Portal.saveParts(homeContainer, Collections.emptyList()); - - for (String webpartName : StringUtils.split(webparts.getValue(), ';')) - { - WebPartFactory webPartFactory = Portal.getPortalPart(webpartName); - if (webPartFactory != null) - addWebPart(webPartFactory.getName(), homeContainer, HttpView.BODY); - } - } - } - - private @Nullable SiteResourceHandler getResourceHandler(@NotNull ResourceType type) - { - return LookAndFeelPropertiesManager.get().getResourceHandler(type); - } - - private boolean setSiteResource(SiteResourceHandler resourceHandler, StartupPropertyEntry prop, User user) - { - Resource resource = getModuleResourceFromPropValue(prop.getValue()); - if (resource != null) - { - try - { - resourceHandler.accept(resource, ContainerManager.getRoot(), user); - return true; - } - catch(Exception e) - { - LOG.error("Exception setting {} during server startup.", prop.getName(), e); - } - } - - LOG.error("Unable to find {} resource during server startup: {}", prop.getName(), prop.getValue()); - return false; - } - - private Resource getModuleResourceFromPropValue(String propValue) - { - if (propValue != null) - { - // split the prop value on the separator char to get the module name and resource path in that module - String moduleName = propValue.substring(0, propValue.indexOf(":")); - String resourcePath = propValue.substring(propValue.indexOf(":") + 1); - - Module module = ModuleLoader.getInstance().getModule(moduleName); - if (module != null) - return module.getModuleResource(resourcePath); - } - - return null; - } - - public void rerunSchemaCheck() - { - // Queue a job without delay. This avoids executing multiple overlapping schema checks. Not bothering with a - // more surgical approach since this variant is likely being called during development. - _warningProvider.startSchemaCheck(0); - } -} +/* + * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.core; + +import com.fasterxml.jackson.core.io.CharTypes; +import com.google.common.collect.Sets; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletRegistration; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pdfbox.pdmodel.font.FontMapper; +import org.apache.pdfbox.pdmodel.font.FontMappers; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.action.ApiJsonWriter; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AdminConsoleService; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.admin.HealthCheck; +import org.labkey.api.admin.HealthCheckRegistry; +import org.labkey.api.admin.TableXmlUtils; +import org.labkey.api.admin.notification.NotificationService; +import org.labkey.api.admin.sitevalidation.SiteValidationService; +import org.labkey.api.analytics.AnalyticsService; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.attachments.DocumentConversionService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.ClientApiAuditProvider; +import org.labkey.api.audit.DefaultAuditProvider; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.ContainerAuditProvider; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.audit.provider.GroupAuditProvider; +import org.labkey.api.audit.provider.ModulePropertiesAuditProvider; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ContainerService; +import org.labkey.api.data.ContainerServiceImpl; +import org.labkey.api.data.ContainerTypeRegistry; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DataColumn; +import org.labkey.api.data.DataRegion; +import org.labkey.api.data.DatabaseMigrationService; +import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationHandler; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.FileSqlScriptProvider; +import org.labkey.api.data.MvUtil; +import org.labkey.api.data.NormalContainerType; +import org.labkey.api.data.OutOfRangeDisplayColumn; +import org.labkey.api.data.PropertySchema; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SchemaTableInfoFactory; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TSVWriter; +import org.labkey.api.data.TabContainerType; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TempTableTracker; +import org.labkey.api.data.TestSchema; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.data.dialect.BasePostgreSqlDialect; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.data.dialect.SqlDialectManager; +import org.labkey.api.data.dialect.SqlDialectRegistry; +import org.labkey.api.data.statistics.StatsService; +import org.labkey.api.dataiterator.SimpleTranslator; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.TestDomainKind; +import org.labkey.api.external.tools.ExternalToolsViewService; +import org.labkey.api.files.FileBrowserConfigImporter; +import org.labkey.api.files.FileBrowserConfigWriter; +import org.labkey.api.files.FileContentService; +import org.labkey.api.markdown.MarkdownService; +import org.labkey.api.message.settings.MessageConfigService; +import org.labkey.api.module.FolderType; +import org.labkey.api.module.FolderTypeManager; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.SchemaUpdateType; +import org.labkey.api.module.SpringModule; +import org.labkey.api.module.Summary; +import org.labkey.api.notification.EmailMessage; +import org.labkey.api.notification.EmailService; +import org.labkey.api.notification.NotificationMenuView; +import org.labkey.api.portal.ProjectUrls; +import org.labkey.api.premium.AntiVirusProviderRegistry; +import org.labkey.api.products.ProductRegistry; +import org.labkey.api.qc.DataStateManager; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.QuerySchema; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.reader.DataLoaderFactory; +import org.labkey.api.reader.DataLoaderService; +import org.labkey.api.reader.ExcelLoader; +import org.labkey.api.reader.FastaDataLoader; +import org.labkey.api.reader.HTMLDataLoader; +import org.labkey.api.reader.JSONDataLoader; +import org.labkey.api.reader.TabLoader; +import org.labkey.api.reports.LabKeyScriptEngineManager; +import org.labkey.api.resource.Resource; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.AuthenticationManager; +import org.labkey.api.security.AuthenticationManager.Priority; +import org.labkey.api.security.AuthenticationSettingsAuditTypeProvider; +import org.labkey.api.security.DbLoginService; +import org.labkey.api.security.DummyAntiVirusService; +import org.labkey.api.security.Group; +import org.labkey.api.security.GroupManager; +import org.labkey.api.security.LimitActiveUsersService; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPointcutService; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.WikiTermsOfUseProvider; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.QCAnalystPermission; +import org.labkey.api.security.permissions.TroubleshooterPermission; +import org.labkey.api.security.roles.NoPermissionsRole; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.security.roles.Role; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.CustomLabelService; +import org.labkey.api.settings.CustomLabelService.CustomLabelServiceImpl; +import org.labkey.api.settings.FolderSettingsCache; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.settings.LookAndFeelPropertiesManager; +import org.labkey.api.settings.LookAndFeelPropertiesManager.ResourceType; +import org.labkey.api.settings.LookAndFeelPropertiesManager.SiteResourceHandler; +import org.labkey.api.settings.OptionalFeatureFlag; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.settings.OptionalFeatureService.FeatureType; +import org.labkey.api.settings.ProductConfiguration; +import org.labkey.api.settings.StandardStartupPropertyHandler; +import org.labkey.api.settings.StartupPropertyEntry; +import org.labkey.api.settings.StashedStartupProperties; +import org.labkey.api.settings.WriteableAppProps; +import org.labkey.api.settings.WriteableLookAndFeelProperties; +import org.labkey.api.stats.AnalyticsProviderRegistry; +import org.labkey.api.stats.SummaryStatisticRegistry; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; +import org.labkey.api.thumbnail.ThumbnailService; +import org.labkey.api.usageMetrics.SimpleMetricsService; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.ContextListener; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JobRunner; +import org.labkey.api.util.MimeMap; +import org.labkey.api.util.MothershipReport; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.ShutdownListener; +import org.labkey.api.util.StartupListener; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.util.UsageReportingLevel; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.vcs.VcsService; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.AlwaysAvailableWebPartFactory; +import org.labkey.api.view.BaseWebPartFactory; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.Portal; +import org.labkey.api.view.Portal.WebPart; +import org.labkey.api.view.ShortURLService; +import org.labkey.api.view.VBox; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.ViewService; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.menu.FolderMenu; +import org.labkey.api.view.template.WarningService; +import org.labkey.api.webdav.SimpleDocumentResource; +import org.labkey.api.webdav.WebdavResolverImpl; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; +import org.labkey.api.wiki.WikiRenderingService; +import org.labkey.api.writer.ContainerUser; +import org.labkey.core.admin.ActionsTsvWriter; +import org.labkey.core.admin.AdminConsoleServiceImpl; +import org.labkey.core.admin.AdminController; +import org.labkey.core.admin.AllowListType; +import org.labkey.core.admin.CopyFileRootPipelineJob; +import org.labkey.core.admin.CustomizeMenuForm; +import org.labkey.core.admin.DisplayFormatAnalyzer; +import org.labkey.core.admin.DisplayFormatValidationProviderFactory; +import org.labkey.core.admin.FilesSiteSettingsAction; +import org.labkey.core.admin.MenuViewFactory; +import org.labkey.core.admin.OptionalFeatureServiceImpl; +import org.labkey.core.admin.OptionalFeatureStartupListener; +import org.labkey.core.admin.importer.FolderTypeImporterFactory; +import org.labkey.core.admin.importer.MissingValueImporterFactory; +import org.labkey.core.admin.importer.ModulePropertiesImporterFactory; +import org.labkey.core.admin.importer.PageImporterFactory; +import org.labkey.core.admin.importer.RoleAssignmentsImporterFactory; +import org.labkey.core.admin.importer.SearchSettingsImporterFactory; +import org.labkey.core.admin.importer.SecurityGroupImporterFactory; +import org.labkey.core.admin.importer.SubfolderImporterFactory; +import org.labkey.core.admin.logger.LoggerController; +import org.labkey.core.admin.logger.LoggingTestCase; +import org.labkey.core.admin.miniprofiler.MiniProfilerController; +import org.labkey.core.admin.sitevalidation.SiteValidationServiceImpl; +import org.labkey.core.admin.sql.SqlScriptController; +import org.labkey.core.admin.test.SchemaXMLTestCase; +import org.labkey.core.admin.test.UnknownSchemasTest; +import org.labkey.core.admin.usageMetrics.UsageMetricsServiceImpl; +import org.labkey.core.admin.writer.FolderSerializationRegistryImpl; +import org.labkey.core.admin.writer.FolderTypeWriterFactory; +import org.labkey.core.admin.writer.MissingValueWriterFactory; +import org.labkey.core.admin.writer.ModulePropertiesWriterFactory; +import org.labkey.core.admin.writer.PageWriterFactory; +import org.labkey.core.admin.writer.RoleAssignmentsWriterFactory; +import org.labkey.core.admin.writer.SearchSettingsWriterFactory; +import org.labkey.core.admin.writer.SecurityGroupWriterFactory; +import org.labkey.core.analytics.AnalyticsController; +import org.labkey.core.analytics.AnalyticsServiceImpl; +import org.labkey.core.attachment.AttachmentServiceImpl; +import org.labkey.core.dialect.PostgreSqlDialectFactory; +import org.labkey.core.dialect.PostgreSqlInClauseTest; +import org.labkey.core.dialect.PostgreSqlVersion; +import org.labkey.core.junit.JunitController; +import org.labkey.core.login.DbLoginAuthenticationProvider; +import org.labkey.core.login.DbLoginManager; +import org.labkey.core.login.LoginController; +import org.labkey.core.metrics.SimpleMetricsServiceImpl; +import org.labkey.core.metrics.WebSocketConnectionManager; +import org.labkey.core.notification.EmailPreferenceConfigServiceImpl; +import org.labkey.core.notification.EmailPreferenceContainerListener; +import org.labkey.core.notification.EmailPreferenceUserListener; +import org.labkey.core.notification.EmailServiceImpl; +import org.labkey.core.notification.NotificationController; +import org.labkey.core.notification.NotificationServiceImpl; +import org.labkey.core.portal.CollaborationFolderType; +import org.labkey.core.portal.PortalJUnitTest; +import org.labkey.core.portal.ProjectController; +import org.labkey.core.portal.UtilController; +import org.labkey.core.products.ProductController; +import org.labkey.core.project.FolderNavigationForm; +import org.labkey.core.qc.CoreQCStateHandler; +import org.labkey.core.qc.DataStateImporter; +import org.labkey.core.qc.DataStateWriter; +import org.labkey.core.query.AttachmentAuditProvider; +import org.labkey.core.query.CoreQuerySchema; +import org.labkey.core.query.PostgresUserSchema; +import org.labkey.core.query.UserAuditProvider; +import org.labkey.core.query.UsersDomainKind; +import org.labkey.core.reader.DataLoaderServiceImpl; +import org.labkey.core.reports.DocumentConversionServiceImpl; +import org.labkey.core.reports.ScriptEngineManagerImpl; +import org.labkey.core.script.RhinoService; +import org.labkey.core.security.AllowedExternalResourceHosts; +import org.labkey.core.security.ApiKeyViewProvider; +import org.labkey.core.security.SecurityApiActions; +import org.labkey.core.security.SecurityController; +import org.labkey.core.security.SecurityPointcutServiceImpl; +import org.labkey.core.security.validators.PermissionsValidatorFactory; +import org.labkey.core.statistics.AnalyticsProviderRegistryImpl; +import org.labkey.core.statistics.StatsServiceImpl; +import org.labkey.core.statistics.SummaryStatisticRegistryImpl; +import org.labkey.core.thumbnail.ThumbnailServiceImpl; +import org.labkey.core.user.LimitActiveUsersSettings; +import org.labkey.core.user.UserController; +import org.labkey.core.vcs.VcsServiceImpl; +import org.labkey.core.view.ShortURLServiceImpl; +import org.labkey.core.view.TableViewFormTestCase; +import org.labkey.core.view.external.tools.ExternalToolsViewServiceImpl; +import org.labkey.core.view.template.bootstrap.CoreWarningProvider; +import org.labkey.core.view.template.bootstrap.PageTemplate; +import org.labkey.core.view.template.bootstrap.ViewServiceImpl; +import org.labkey.core.view.template.bootstrap.WarningServiceImpl; +import org.labkey.core.webdav.DavController; +import org.labkey.core.webdav.ModuleStaticResolverImpl; +import org.labkey.core.webdav.WebFilesResolverImpl; +import org.labkey.core.webdav.WebdavServlet; +import org.labkey.core.wiki.MarkdownServiceImpl; +import org.labkey.core.wiki.MarkdownTestCase; +import org.labkey.core.wiki.RadeoxRenderer; +import org.labkey.core.wiki.WikiRenderingServiceImpl; +import org.labkey.core.workbook.WorkbookFolderType; +import org.labkey.core.workbook.WorkbookQueryView; +import org.labkey.core.workbook.WorkbookSearchView; +import org.labkey.filters.ContentSecurityPolicyFilter; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.impl.StdSchedulerFactory; +import org.radeox.test.BaseRenderEngineTest; +import org.radeox.test.filter.BasicRegexTest; +import org.radeox.test.filter.BoldFilterTest; +import org.radeox.test.filter.EscapeFilterTest; +import org.radeox.test.filter.FilterPipeTest; +import org.radeox.test.filter.HeadingFilterTest; +import org.radeox.test.filter.HtmlRemoveFilterTest; +import org.radeox.test.filter.ItalicFilterTest; +import org.radeox.test.filter.KeyFilterTest; +import org.radeox.test.filter.LineFilterTest; +import org.radeox.test.filter.LinkTestFilterTest; +import org.radeox.test.filter.ListFilterTest; +import org.radeox.test.filter.NewlineFilterTest; +import org.radeox.test.filter.ParamFilterTest; +import org.radeox.test.filter.SmileyFilterTest; +import org.radeox.test.filter.StrikeThroughFilterTest; +import org.radeox.test.filter.TypographyFilterTest; +import org.radeox.test.filter.UrlFilterTest; +import org.radeox.test.filter.WikiLinkFilterTest; +import org.radeox.test.macro.list.AtoZListFormatterTest; +import org.radeox.test.macro.list.ExampleListFormatterTest; +import org.radeox.test.macro.list.SimpleListTest; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.sql.Connection; +import java.sql.SQLException; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.labkey.api.settings.StashedStartupProperties.homeProjectFolderType; +import static org.labkey.api.settings.StashedStartupProperties.homeProjectResetPermissions; +import static org.labkey.api.settings.StashedStartupProperties.homeProjectWebparts; +import static org.labkey.api.settings.StashedStartupProperties.siteAvailableEmailFrom; +import static org.labkey.api.settings.StashedStartupProperties.siteAvailableEmailMessage; +import static org.labkey.api.settings.StashedStartupProperties.siteAvailableEmailSubject; +import static org.labkey.api.util.MothershipReport.EXPERIMENTAL_LOCAL_MARKETING_UPDATE; +import static org.labkey.api.util.MothershipReport.FEATURE_FLAG_EXTENDED_METRICS; +import static org.labkey.filters.ContentSecurityPolicyFilter.FEATURE_FLAG_DISABLE_ENFORCE_CSP; + +public class CoreModule extends SpringModule implements SearchService.DocumentProvider +{ + private static final Logger LOG = LogHelper.getLogger(CoreModule.class, "Errors during server startup and shut down"); + public static final String PROJECTS_WEB_PART_NAME = "Projects"; + + static + { + // Accept most of the standard Quartz properties, but set the misfire threshold to five minutes. This prevents + // Quartz from dropping scheduled work if a lot of items fire at the same time, like a lot of ETLs triggering at 2AM. + // This can overwhelm the thread pool running them so they don't complete in the default 1 minute window. Set it early so + // if any other module touches Quartz in its setup, it initializes with this setting. + Properties props = System.getProperties(); + props.setProperty("org.quartz.jobStore.misfireThreshold", "300000"); + + // Register dialect extra early, since we need to initialize the data sources before calling DefaultModule.initialize() + SqlDialectRegistry.register(new PostgreSqlDialectFactory()); + + try + { + var field = CharTypes.class.getDeclaredField("sOutputEscapes128"); + field.setAccessible(true); + ((int[])field.get(null))['/'] = '/'; + field.setAccessible(false); + } + catch (NoSuchFieldException|IllegalArgumentException|IllegalAccessException x) + { + // pass + } + } + + private CoreWarningProvider _warningProvider; + private ServletRegistration.Dynamic _webdavServletDynamic; + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + protected void init() + { + ContainerService.setInstance(new ContainerServiceImpl()); + FolderSerializationRegistry.setInstance(new FolderSerializationRegistryImpl()); + ExternalToolsViewService.setInstance(new ExternalToolsViewServiceImpl()); + ExternalToolsViewService.get().registerExternalAccessViewProvider(new ApiKeyViewProvider()); + LimitActiveUsersService.setInstance(() -> new LimitActiveUsersSettings().isUserLimitReached()); + + // Register the default DataLoaders during init so they are available to sql upgrade scripts + DataLoaderServiceImpl dls = new DataLoaderServiceImpl(); + dls.registerFactory(new ExcelLoader.Factory()); + dls.registerFactory(new TabLoader.TsvFactory()); + dls.registerFactory(new TabLoader.CsvFactory()); + dls.registerFactory(new HTMLDataLoader.Factory()); + dls.registerFactory(new JSONDataLoader.Factory()); + dls.registerFactory(new FastaDataLoader.Factory()); + DataLoaderService.setInstance(dls); + + addController("admin", AdminController.class); + addController("admin-sql", SqlScriptController.class); + addController("security", SecurityController.class); + addController("user", UserController.class); + addController("login", LoginController.class); + addController("junit", JunitController.class); + addController("core", CoreController.class); + addController("analytics", AnalyticsController.class); + addController("project", ProjectController.class); + addController("util", UtilController.class); + addController("logger", LoggerController.class); + addController("mini-profiler", MiniProfilerController.class); + addController("notification", NotificationController.class); + addController("product", ProductController.class); + + AuthenticationManager.registerProvider(new DbLoginAuthenticationProvider(), Priority.Low); + AttachmentService.setInstance(new AttachmentServiceImpl()); + AnalyticsService.setInstance(new AnalyticsServiceImpl()); + RhinoService.register(); + CacheManager.addListener(RhinoService::clearCaches); + NotificationService.setInstance(NotificationServiceImpl.getInstance()); + + WarningService.setInstance(new WarningServiceImpl()); + + ViewService.setInstance(ViewServiceImpl.getInstance()); + OptionalFeatureService.setInstance(new OptionalFeatureServiceImpl()); + ThumbnailService.setInstance(new ThumbnailServiceImpl()); + ShortURLService.setInstance(new ShortURLServiceImpl()); + StatsService.setInstance(new StatsServiceImpl()); + SiteValidationService.setInstance(new SiteValidationServiceImpl()); + AnalyticsProviderRegistry.setInstance(new AnalyticsProviderRegistryImpl()); + SummaryStatisticRegistry.setInstance(new SummaryStatisticRegistryImpl()); + UsageMetricsService.setInstance(new UsageMetricsServiceImpl()); + CustomLabelService.setInstance(new CustomLabelServiceImpl()); + SecurityPointcutService.setInstance(new SecurityPointcutServiceImpl()); + AdminConsoleService.setInstance(new AdminConsoleServiceImpl()); + WikiRenderingService.setInstance(new WikiRenderingServiceImpl()); + VcsService.setInstance(new VcsServiceImpl()); + LabKeyScriptEngineManager.setInstance(new ScriptEngineManagerImpl()); + DocumentConversionService.setInstance(new DocumentConversionServiceImpl()); + DbLoginService.setInstance(new DbLoginManager()); + + try + { + ContainerTypeRegistry.get().register("normal", new NormalContainerType()); + ContainerTypeRegistry.get().register("tab", new TabContainerType()); + ContainerTypeRegistry.get().register("workbook", new WorkbookContainerType()); + } + catch (Exception e) + { + throw UnexpectedException.wrap(e); + } + + _warningProvider = new CoreWarningProvider(); + WarningService.get().register(_warningProvider); + + WebdavService.get().setResolver(ModuleStaticResolverImpl.get()); + // need to register webdav resolvers in init() instead of startupAfterSpringConfig since static module files are loaded during module startup + WebdavService.get().registerRootResolver(WebdavResolverImpl.get()); + WebdavService.get().registerRootResolver(WebFilesResolverImpl.get()); + + DefaultSchema.registerProvider(CoreQuerySchema.NAME, new DefaultSchema.SchemaProvider(this) + { + @Override + public boolean isAvailable(DefaultSchema schema, Module module) + { + return true; + } + + @Override + public QuerySchema createSchema(DefaultSchema schema, Module module) + { + return new CoreQuerySchema(schema.getUser(), schema.getContainer()); + } + }); + + if (CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) + { + DefaultSchema.registerProvider(BasePostgreSqlDialect.POSTGRES_SCHEMA_NAME, new DefaultSchema.SchemaProvider(this) + { + @Override + public boolean isAvailable(DefaultSchema schema, Module module) + { + return schema.getContainer().isRoot() && schema.getContainer().hasPermission(schema.getUser(), TroubleshooterPermission.class); + } + + @Override + public QuerySchema createSchema(DefaultSchema schema, Module module) + { + return new PostgresUserSchema(schema.getUser(), schema.getContainer()); + } + }); + } + + OptionalFeatureService.get().addExperimentalFeatureFlag(NotificationMenuView.EXPERIMENTAL_NOTIFICATION_MENU, "Notifications Menu", + "Notifications 'inbox' count display in the header bar with click to show the notifications panel of unread notifications.", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(DataColumn.EXPERIMENTAL_USE_QUERYSELECT_COMPONENT, "Use QuerySelect for row insert/update form", + "This feature will switch the query based select inputs on the row insert/update form to use the React QuerySelect" + + "component. This will allow for a user to view the first 100 options in the select but then use type ahead" + + "search to find the other select values.", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(SQLFragment.FEATUREFLAG_DISABLE_STRICT_CHECKS, "Disable SQLFragment strict checks", + "SQLFragment now has very strict usage validation, these checks may cause errors in code that has not been updated. Turn on this feature to disable checks.", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(LoginController.FEATUREFLAG_DISABLE_LOGIN_XFRAME, "Disable Login X-FRAME-OPTIONS=DENY", + "By default LabKey disables all framing of login related actions. Disabling this feature will revert to using the standard site settings.", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(PageTemplate.EXPERIMENTAL_SHORT_CIRCUIT_ROBOTS, + "Short-circuit robots", + "Save resources by not rendering pages marked as 'noindex' for robots. This is experimental as not all robots are search engines.", + false); + OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(AppProps.DEPRECATED_OBJECT_LEVEL_DISCUSSIONS, + "Restore Object-Level Discussions", + "This option and all support for Object-Level Discussions will be removed in LabKey Server v25.11.", + false, false, FeatureType.Deprecated)); + OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(SimpleTranslator.DEPRECATED_NULL_MISSING_VALUE_RESOLUTION, + "Resolve Missing Lookup Values to Null", + "When Lookup Validation for a field is not selected and lookup by alternate key is enabled, resolves missing lookup values to null instead of throwing an error. This option will be removed in LabKey Server v25.11.", + false, false, OptionalFeatureService.FeatureType.Deprecated)); + OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(TabLoader.FEATUREFLAG_UNESCAPE_BACKSLASH, + "Unescape backslash character on import", + "Treat backslash '\\' character as an escape character when loading data from file.", + false, false, OptionalFeatureService.FeatureType.Deprecated + )); + OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(AppProps.GENERATE_CONTROLLER_FIRST_URLS, + "Restore controller-first URLS", + "Generate URLs in a legacy format that puts the controller name before the folder path. This option will be removed in LabKey Server 26.3.", + false, false, OptionalFeatureService.FeatureType.Deprecated + )); + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.REJECT_CONTROLLER_FIRST_URLS, + "Reject controller-first URLs", + "Require standard path-first URLs. Note: This option will be ignored if the deprecated feature for generating controller-first URLs is enabled.", + false + ); + + SiteValidationService svc = SiteValidationService.get(); + if (null != svc) + { + svc.registerProviderFactory(getName(), new PermissionsValidatorFactory()); + svc.registerProviderFactory(getName(), new DisplayFormatValidationProviderFactory()); + } + + registerHealthChecks(); + + ContextListener.addNewInstallCompleteListener(() -> sendSystemReadyEmail(UserManager.getAppAdmins())); + + ScriptEngineManagerImpl.registerEncryptionMigrationHandler(); + + deleteTempFiles(); + } + + private void deleteTempFiles() + { + try + { + // Issue 46598 - clean up previously created temp files from file uploads + FileUtil.deleteDirectoryContents(SpringActionController.getTempUploadDir()); + } + catch (IOException e) + { + LOG.warn("Failed to clean up previously uploaded files from {}", SpringActionController.getTempUploadDir(), e); + } + } + + private void registerHealthChecks() + { + HealthCheckRegistry.get().registerHealthCheck("database", HealthCheckRegistry.DEFAULT_CATEGORY, () -> + { + Map healthValues = new HashMap<>(); + boolean allConnected = true; + for (DbScope dbScope : DbScope.getDbScopes()) + { + boolean dbConnected; + try (Connection conn = dbScope.getConnection()) + { + dbConnected = conn != null; + } + catch (SQLException e) + { + dbConnected = false; + } + + healthValues.put(dbScope.getDatabaseName(), dbConnected); + allConnected &= dbConnected; + } + + return new HealthCheck.Result(allConnected, healthValues); + } + ); + + HealthCheckRegistry.get().registerHealthCheck("modules", HealthCheckRegistry.TRIAL_INSTANCES_CATEGORY, () -> { + Map failures = ModuleLoader.getInstance().getModuleFailures(); + Map failureDetails = new HashMap<>(); + for (Map.Entry failure : failures.entrySet()) + { + failureDetails.put(failure.getKey(), failure.getValue().getMessage()); + } + return new HealthCheck.Result(failures.isEmpty(), failureDetails); + }); + + HealthCheckRegistry.get().registerHealthCheck("users", HealthCheckRegistry.TRIAL_INSTANCES_CATEGORY, () -> { + Map userHealth = new HashMap<>(); + ZonedDateTime now = ZonedDateTime.now(); + int userCount = UserManager.getUserCount(Date.from(now.toInstant())); + userHealth.put("registeredUsers", userCount); + return new HealthCheck.Result(userCount > 0, userHealth); + }); + } + + private void sendSystemReadyEmail(List users) + { + if (users.isEmpty()) + return; + + Map map = AppProps.getInstance().getStashedStartupProperties(); + String fromEmail = getValue(map, siteAvailableEmailFrom); + String subject = getValue(map, siteAvailableEmailSubject); + String body = getValue(map, siteAvailableEmailMessage); + + if (fromEmail == null || subject == null || body == null) + return; + + EmailService svc = EmailService.get(); + List messages = new ArrayList<>(); + for (User user: users) + { + EmailMessage message = svc.createMessage(fromEmail, Collections.singletonList(user.getEmail()), subject); + message.addContent(MimeMap.MimeType.HTML, body); + messages.add(message); + } + // For audit purposes, we use the first user as the originator of the message. + // Would be better to have this be a site admin, but we aren't guaranteed to have such a user + // for hosted sites. Another option is to use the guest user here, but that's strange. + svc.sendMessages(messages, users.get(0), ContainerManager.getRoot()); + } + + private @Nullable String getValue(Map map, StashedStartupProperties prop) + { + StartupPropertyEntry entry = map.get(prop); + return null != entry ? StringUtils.trimToNull(entry.getValue()) : null; + } + + @NotNull + @Override + protected Collection createWebPartFactories() + { + return List.of( + new AlwaysAvailableWebPartFactory("Contacts") + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext ctx, @NotNull WebPart webPart) + { + UserSchema schema = QueryService.get().getUserSchema(ctx.getUser(), ctx.getContainer(), CoreQuerySchema.NAME); + QuerySettings settings = new QuerySettings(ctx, QueryView.DATAREGIONNAME_DEFAULT); + + settings.setQueryName(CoreQuerySchema.USERS_TABLE_NAME); + + QueryView view = schema.createView(ctx, settings, null); + view.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); + view.setFrame(WebPartView.FrameType.PORTAL); + view.setTitle("Project Contacts"); + + return view; + } + }, + new BaseWebPartFactory("FolderNav") + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) + { + FolderNavigationForm form = getForm(portalCtx); + + form.setFolderMenu(new FolderMenu(portalCtx)); + + JspView view = new JspView<>("/org/labkey/core/project/folderNav.jsp", form); + view.setTitle("Folder Navigation"); + view.setFrame(WebPartView.FrameType.NONE); + return view; + } + + @Override + public boolean isAvailable(Container c, String scope, String location) + { + return false; + } + + private FolderNavigationForm getForm(ViewContext context) + { + FolderNavigationForm form = new FolderNavigationForm(); + form.setPortalContext(context); + return form; + } + }, + new BaseWebPartFactory("Workbooks") + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) + { + UserSchema schema = QueryService.get().getUserSchema(portalCtx.getUser(), portalCtx.getContainer(), SchemaKey.fromParts(CoreQuerySchema.NAME)); + WorkbookQueryView wbqview = new WorkbookQueryView(portalCtx, schema); + VBox box = new VBox(new WorkbookSearchView(wbqview), wbqview); + box.setFrame(WebPartView.FrameType.PORTAL); + box.setTitle("Workbooks"); + return box; + } + + @Override + public boolean isAvailable(Container c, String scope, String location) + { + return !c.isWorkbook() && "folder".equals(scope) && location.equalsIgnoreCase(HttpView.BODY); + } + }, + new BaseWebPartFactory("Workbook Description") + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) + { + JspView view = new JspView<>("/org/labkey/core/workbook/workbookDescription.jsp"); + view.setTitle("Workbook Description"); + view.setFrame(WebPartView.FrameType.NONE); + return view; + } + + @Override + public boolean isAvailable(Container c, String scope, String location) + { + return false; + } + }, + new AlwaysAvailableWebPartFactory(PROJECTS_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) + { + JspView view = new JspView<>("/org/labkey/core/project/projects.jsp", webPart); + + String title = webPart.getPropertyMap().getOrDefault("title", "Projects"); + view.setTitle(title); + + if (portalCtx.hasPermission(getClass().getName(), AdminPermission.class)) + { + NavTree customize = new NavTree(""); + customize.setScript("customizeProjectWebpart" + webPart.getRowId() + "();"); + view.setCustomize(customize); + } + return view; + } + }, + new AlwaysAvailableWebPartFactory("Subfolders", WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPart createWebPart() + { + // Issue 44913: Set the default properties for all new instances of the Subfolders webpart + WebPart webPart = super.createWebPart(); + return setDefaultProperties(webPart); + } + + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) + { + if (webPart.getPropertyMap().isEmpty()) + { + // Configure to show subfolders if not previously configured + webPart = setDefaultProperties(new WebPart(webPart)); + } + + JspView view = new JspView<>("/org/labkey/core/project/projects.jsp", webPart); + view.setTitle(webPart.getPropertyMap().get("title")); + + if (portalCtx.hasPermission(getClass().getName(), AdminPermission.class)) + { + NavTree customize = new NavTree(""); + customize.setScript("customizeProjectWebpart" + webPart.getRowId() + "(" + webPart.getRowId() + ", " + PageFlowUtil.jsString(webPart.getPageId()) + ", " + webPart.getIndex() + ");"); + view.setCustomize(customize); + } + + return view; + } + + private WebPart setDefaultProperties(WebPart webPart) + { + webPart.setProperty("title", "Subfolders"); + webPart.setProperty("containerFilter", ContainerFilter.Type.CurrentAndFirstChildren.name()); + webPart.setProperty("containerTypes", "folder"); + return webPart; + } + }, + new AlwaysAvailableWebPartFactory("Custom Menu", true, true, WebPartFactory.LOCATION_MENUBAR) + { + @Override + public WebPartView getWebPartView(@NotNull final ViewContext portalCtx, @NotNull WebPart webPart) + { + final CustomizeMenuForm form = AdminController.getCustomizeMenuForm(webPart); + String title = "My Menu"; + if (form.getTitle() != null && !form.getTitle().isEmpty()) + title = form.getTitle(); + + WebPartView view; + if (form.isChoiceListQuery()) + { + view = MenuViewFactory.createMenuQueryView(portalCtx, title, form); + } + else + { + view = MenuViewFactory.createMenuFolderView(portalCtx, title, form); + } + view.setFrame(WebPartView.FrameType.PORTAL); + return view; + } + + @Override + public HttpView getEditView(WebPart webPart, ViewContext context) + { + CustomizeMenuForm form = AdminController.getCustomizeMenuForm(webPart); + JspView view = new JspView<>("/org/labkey/core/admin/customizeMenu.jsp", form); + view.setTitle(form.getTitle()); + view.setFrame(WebPartView.FrameType.PORTAL); + return view; + } + } + ); + } + + @Override + public void afterUpdate(ModuleContext moduleContext) + { + if (moduleContext.isNewInstall()) + { + bootstrap(); + } + + // Increment on every core module upgrade to defeat browser caching of static resources. + if (ModuleLoader.getInstance().shouldInsertData()) + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + + // Allow dialect to make adjustments to the just upgraded core database (e.g., install aggregate functions, etc.) + CoreSchema.getInstance().getSqlDialect().afterCoreUpgrade(moduleContext); + + // The core SQL scripts install aggregate functions and other objects that dialects need to know about. Prepare + // the previously initialized dialects again to make sure they're aware of all the changes. Prepare all the + // initialized scopes because we could have more than one scope pointed at the core database (e.g., external + // schemas). See #17077 (pg example) and #19177 (ss example). + for (DbScope scope : DbScope.getInitializedDbScopes()) + scope.getSqlDialect().prepare(scope); + + // Now that we know the standard containers have been created, add a listener that warms the just-cleared caches with + // core.Containers metadata and a few common containers. This may prevent some deadlocks during upgrade, #33550. + CacheManager.addListener(() -> { + ContainerManager.getRoot(); + ContainerManager.getSharedContainer(); + if (ModuleLoader.getInstance().shouldInsertData()) + { + ContainerManager.getHomeContainer(); + } + }); + } + + private void bootstrap() + { + if (ModuleLoader.getInstance().shouldInsertData()) + { + // Create the initial groups + GroupManager.bootstrapGroup(Group.groupUsers, "Users"); + GroupManager.bootstrapGroup(Group.groupGuests, "Guests"); + + // Other containers inherit permissions from root; admins get all permissions, users & guests none + Role noPermsRole = RoleManager.getRole(NoPermissionsRole.class); + Role readerRole = RoleManager.getRole(ReaderRole.class); + + ContainerManager.bootstrapContainer("/", noPermsRole, noPermsRole); + Container rootContainer = ContainerManager.getRoot(); + + // Create all the standard containers (Home, Home/support, Shared) using an empty Collaboration folder type + FolderType collaborationType = new CollaborationFolderType(Collections.emptyList()); + + // Users & guests can read from /home + Container home = ContainerManager.bootstrapContainer(ContainerManager.HOME_PROJECT_PATH, readerRole, readerRole); + home.setFolderType(collaborationType, null); + + ContainerManager.createDefaultSupportContainer().setFolderType(collaborationType, null); + + // Only users can read from /Shared + ContainerManager.bootstrapContainer(ContainerManager.SHARED_CONTAINER_PATH, readerRole, null).setFolderType(collaborationType, null); + + try + { + // Need to insert standard MV indicators for the root -- okay to call getRoot() since we just created it. + String rootContainerId = rootContainer.getId(); + TableInfo mvTable = CoreSchema.getInstance().getTableInfoMvIndicators(); + + for (Map.Entry qcEntry : MvUtil.getDefaultMvIndicators().entrySet()) + { + Map params = new HashMap<>(); + params.put("Container", rootContainerId); + params.put("MvIndicator", qcEntry.getKey()); + params.put("Label", qcEntry.getValue()); + + Table.insert(null, mvTable, params); + } + } + catch (Throwable t) + { + ExceptionUtil.logExceptionToMothership(null, t); + } + + List guids = Stream.of(ContainerManager.HOME_PROJECT_PATH, ContainerManager.SHARED_CONTAINER_PATH) + .map(ContainerManager::getForPath) + .filter(Objects::nonNull) + .map(Container::getEntityId) + .collect(Collectors.toList()); + ContainerManager.setExcludedProjects(guids, () -> {}); + } + else + { + // It's very difficult to bootstrap without the root or shared containers in place; create them now and + // we'll delete them later + Container root = ContainerManager.ensureContainer("/", User.getAdminServiceUser()); + Table.insert(null, CoreSchema.getInstance().getTableInfoContainers(), Map.of("Parent", root.getId(), "Name", "Shared")); + } + } + + + @Override + public CoreUpgradeCode getUpgradeCode() + { + return new CoreUpgradeCode(); + } + + + @Override + public void destroy() + { + super.destroy(); + UsageReportingLevel.shutdown(); + } + + + @Override + public void startupAfterSpringConfig(ModuleContext moduleContext) + { + // Any containers in the cache have bogus folder types since they aren't registered until startup(). See #10310 + ContainerManager.clearCache(); + + checkForMissingDbViews(); + + ProductConfiguration.handleStartupProperties(); + // This listener deletes all properties; make sure it executes after most of the other listeners + ContainerManager.addContainerListener(new CoreContainerListener(), ContainerManager.ContainerListener.Order.Last); + ContainerManager.addContainerListener(new FolderSettingsCache.FolderSettingsCacheListener()); + SecurityManager.init(); + FolderTypeManager.get().registerFolderType(this, FolderType.NONE); + FolderTypeManager.get().registerFolderType(this, new CollaborationFolderType()); + + AnalyticsServiceImpl.get().resetCSP(); + + if (moduleContext.isNewInstall() && ModuleLoader.getInstance().shouldInsertData()) + { + // To initialize the portal layout correctly, we need to add the web parts after the folder types have been + // registered. Thus, it needs to be here in startupAfterSpringConfig() instead of grouped in bootstrap(). + Container homeContainer = ContainerManager.getHomeContainer(); + int count = Portal.getParts(homeContainer, homeContainer.getFolderType().getDefaultPageId(homeContainer)).size(); + addWebPart(PROJECTS_WEB_PART_NAME, homeContainer, HttpView.BODY, count); + } + + EmailService.setInstance(new EmailServiceImpl()); + + if (null != AuditLogService.get() && AuditLogService.get().getClass() != DefaultAuditProvider.class) + { + AuditLogService.get().registerAuditType(new UserAuditProvider()); + AuditLogService.get().registerAuditType(new GroupAuditProvider()); + AuditLogService.get().registerAuditType(new AttachmentAuditProvider()); + AuditLogService.get().registerAuditType(new ContainerAuditProvider()); + AuditLogService.get().registerAuditType(new FileSystemAuditProvider()); + AuditLogService.get().registerAuditType(new ClientApiAuditProvider()); + AuditLogService.get().registerAuditType(new AuthenticationSettingsAuditTypeProvider()); + AuditLogService.get().registerAuditType(new TransactionAuditProvider()); + AuditLogService.get().registerAuditType(new ModulePropertiesAuditProvider()); + + DataStateManager.getInstance().registerDataStateHandler(new CoreQCStateHandler()); + } + ContextListener.addShutdownListener(TempTableTracker.getShutdownListener()); + ContextListener.addShutdownListener(DavController.getShutdownListener()); + ContextListener.addShutdownListener(ShutdownListener.of("Temp file cleanup", null, this::deleteTempFiles)); + + SimpleMetricsService.setInstance(new SimpleMetricsServiceImpl()); + + // Export action stats on graceful shutdown + ContextListener.addShutdownListener(new ShutdownListener() { + @Override + public String getName() + { + return "Action stats export"; + } + + @Override + public void shutdownPre() + { + try + { + // Halt firing of Quartz triggers + Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler(); + scheduler.standby(); + } + catch (SchedulerException ignored) + { + } + + Logger logger = LogManager.getLogger(ActionsTsvWriter.class); + + if (null != logger) + { + StringBuilder buf = new StringBuilder(); + + try (TSVWriter writer = new ActionsTsvWriter()) + { + writer.write(buf); + } + catch (IOException e) + { + LOG.error("Exception exporting action stats", e); + } + + logger.info(buf.toString()); + LOG.info("Completed logging statistics for actions prior to web application shut down"); + } + } + + @Override + public void shutdownStarted() + { + try + { + // Clean up Quartz resources and wait for jobs to complete + Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler(); + scheduler.shutdown(true); + } + catch (SchedulerException ignored) + { + } + } + }); + + // populate look and feel settings and site settings with values read from startup properties as appropriate for not bootstrap + populateLookAndFeelResourcesWithStartupProps(); + AllowedExternalResourceHosts.registerStartupProperties(); + AllowedExternalResourceHosts.registerHosts(); + WriteableLookAndFeelProperties.populateLookAndFeelWithStartupProps(); + WriteableAppProps.populateSiteSettingsWithStartupProps(); + // create users and groups and assign roles with values read from startup properties as appropriate for not bootstrap + SecurityManager.populateStartupProperties(); + // This method depends on resources (FolderType) from other modules, so invoke after startup + ContextListener.addStartupListener(new StartupListener() + { + @Override + public String getName() + { + return "CoreModule.populateSiteSettingsWithStartupProps"; + } + + @Override + public void moduleStartupComplete(ServletContext servletContext) + { + populateSiteSettingsWithStartupProps(); + } + }); + + // Handle optional feature startup properties as late as possible; we want all optional features to be registered first + ContextListener.addStartupListener(new OptionalFeatureStartupListener()); + + LabKeyScriptEngineManager svc = LabKeyScriptEngineManager.get(); + // populate script engine definitions values read from startup properties + if (svc instanceof ScriptEngineManagerImpl) + ((ScriptEngineManagerImpl)svc).populateScriptEngineDefinitionsWithStartupProps(); + + // populate folder types from startup properties as appropriate for not bootstrap + FolderTypeManager.get().populateWithStartupProps(); + LimitActiveUsersSettings.populateStartupProperties(); + + AdminController.registerAdminConsoleLinks(); + AdminController.registerManagementTabs(); + AnalyticsController.registerAdminConsoleLinks(); + UserController.registerAdminConsoleLinks(); + LoggerController.registerAdminConsoleLinks(); + CoreController.registerAdminConsoleLinks(); + + FolderTypeManager.get().registerFolderType(this, new WorkbookFolderType()); + + SecurityManager.addViewFactory(new SecurityController.GroupDiagramViewFactory()); + + FolderSerializationRegistry fsr = FolderSerializationRegistry.get(); + if (null != fsr) + { + fsr.addFactories(new FolderTypeWriterFactory(), new FolderTypeImporterFactory()); + fsr.addFactories(new MissingValueWriterFactory(), new MissingValueImporterFactory()); + fsr.addFactories(new SearchSettingsWriterFactory(), new SearchSettingsImporterFactory()); + fsr.addFactories(new PageWriterFactory(), new PageImporterFactory()); + fsr.addFactories(new ModulePropertiesWriterFactory(), new ModulePropertiesImporterFactory()); + fsr.addFactories(new SecurityGroupWriterFactory(), new SecurityGroupImporterFactory()); + fsr.addFactories(new RoleAssignmentsWriterFactory(), new RoleAssignmentsImporterFactory()); + fsr.addFactories(new DataStateWriter.Factory(), new DataStateImporter.Factory()); + fsr.addFactories(new FileBrowserConfigWriter.Factory(), new FileBrowserConfigImporter.Factory()); + fsr.addImportFactory(new SubfolderImporterFactory()); + } + + SearchService ss = SearchService.get(); + ss.addDocumentParser(new TabLoader.CsvFactoryNoConversions()); + ss.addDocumentProvider(this); + + // Register indexable DataLoaders with the search service + DataLoaderServiceImpl.get().getFactories() + .stream() + .filter(DataLoaderFactory::indexable) + .forEach(ss::addDocumentParser); + + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_NO_GUESTS, + "No Guest Account", + "Disable the guest account", + false); + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_BLOCKER, + "Block malicious clients", + "Reject requests from clients that appear malicious. Turn this feature off if you want to run a security scanner.", + false); + OptionalFeatureService.get().addExperimentalFeatureFlag(FEATURE_FLAG_DISABLE_ENFORCE_CSP, + "Disable enforce Content Security Policy", + "Stop sending the " + ContentSecurityPolicyFilter.ContentSecurityPolicyType.Enforce.getHeaderName() + " header to browsers, " + + "but continue sending the " + ContentSecurityPolicyFilter.ContentSecurityPolicyType.Report.getHeaderName() + " header. " + + "This turns off an important layer of security for the entire site, so use it as a last resort only on a temporary basis " + + "(e.g., if an enforce CSP breaks critical functionality).", + false); + OptionalFeatureService.get().addExperimentalFeatureFlag(DataRegion.EXPERIMENTAL_DATA_REGION_ASYNC_TOTAL_ROWS, + "Data Region Async Total Rows", + "Enable asynchronous calculation of total rows for data regions. This can improve performance for large datasets.", + false); + + OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(EXPERIMENTAL_LOCAL_MARKETING_UPDATE, + "Self test marketing updates", "Test marketing updates from this local server (requires the mothership module).", false, true, FeatureType.Experimental)); + OptionalFeatureService.get().addFeatureListener(EXPERIMENTAL_LOCAL_MARKETING_UPDATE, (feature, enabled) -> { + // update the timer task when this setting changes + MothershipReport.setSelfTestMarketingUpdates(enabled); + UsageReportingLevel.reportNow(); + }); + + if (null != PropertyService.get()) + { + PropertyService.get().registerDomainKind(new UsersDomainKind()); + if (ModuleLoader.getInstance().shouldInsertData()) + UsersDomainKind.ensureDomain(moduleContext); + } + + // Register the standard, wiki-based terms-of-use provider + SecurityManager.addTermsOfUseProvider(new WikiTermsOfUseProvider()); + + if (null != PropertyService.get()) + PropertyService.get().registerDomainKind(new TestDomainKind()); + + AuthenticationManager.populateSettingsWithStartupProps(); + AnalyticsServiceImpl.populateSettingsWithStartupProps(); + + UsageMetricsService.get().registerUsageMetrics(getName(), () -> { + Map results = new HashMap<>(); + Map javaInfo = new HashMap<>(); + javaInfo.put("java.vendor", System.getProperty("java.vendor")); + javaInfo.put("java.vm.name", System.getProperty("java.vm.name")); + results.put("javaRuntime", javaInfo); + results.put("distributionFilename", AppProps.getInstance().getDistributionFilename()); + results.put("applicationMenuDisplayMode", LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getApplicationMenuDisplayMode()); + results.put("optionalFeatures", OptionalFeatureService.get().getOptionalFeatureFlags().stream() + .collect(Collectors.groupingBy(optionalFeatureFlag -> optionalFeatureFlag.getType().name().toLowerCase(), + Collectors.mapping(flag -> flag, Collectors.toMap(OptionalFeatureFlag::getFlag, OptionalFeatureFlag::isEnabled)) + )) + ); + results.put("productFeaturesEnabled", ProductRegistry.getProductFeatureSet()); + results.put("analyticsTrackingStatus", AnalyticsServiceImpl.get().getTrackingStatus().toString()); + String labkeyContextPath = AppProps.getInstance().getContextPath(); + results.put("webappContextPath", labkeyContextPath); + results.put("embeddedTomcat", true); + boolean customLog4JConfig = false; + if (ModuleLoader.getServletContext() != null) + { + customLog4JConfig = Boolean.parseBoolean(ModuleLoader.getServletContext().getInitParameter("org.labkey.customLog4JConfig")); + } + results.put("customLog4JConfig", customLog4JConfig); + results.put("containerRelativeURL", AppProps.getInstance().getUseContainerRelativeURL()); + results.put("runtimeMode", AppProps.getInstance().isDevMode() ? "development" : "production"); + Set deployedApps = new HashSet<>(CoreWarningProvider.collectAllDeployedApps()); + deployedApps.remove(labkeyContextPath); + if (labkeyContextPath.startsWith("/")) + { + deployedApps.remove(labkeyContextPath.substring(1)); + } + results.put("otherDeployedWebapps", StringUtils.join(deployedApps, ",")); + + // Report the total number of login entries in the audit log + results.put("totalLogins", UserManager.getAuthCount(null, false, false, false)); + results.put("apiKeyLogins", UserManager.getAuthCount(null, false, true, false)); + results.put("sessionTimeout", ModuleLoader.getServletContext().getSessionTimeout()); + results.put("userLimits", new LimitActiveUsersSettings().getMetricsMap()); + results.put("systemUserCount", UserManager.getSystemUserCount()); + Calendar cal = new GregorianCalendar(); + cal.add(Calendar.DATE, -30); + results.put("uniqueRecentUserCount", UserManager.getAuthCount(cal.getTime(), false, false, true)); + results.put("uniqueRecentNonSystemUserCount", UserManager.getAuthCount(cal.getTime(), true, false, true)); + if (OptionalFeatureService.get().isFeatureEnabled(FEATURE_FLAG_EXTENDED_METRICS)) + { + // Optionally include a list of active users, Issue #53050 + results.put("activeUsers", UserManager.getActiveUsers().stream() + .filter(u -> !u.isSystem()) + .map(User::getEmail) + .toList() + ); + } + + results.put("workbookCount", ContainerManager.getWorkbookCount()); + results.put("archivedFolderCount", ContainerManager.getArchivedContainerCount()); + results.put("databaseSize", CoreSchema.getInstance().getSchema().getScope().getDatabaseSize()); + results.put("scriptEngines", LabKeyScriptEngineManager.get().getScriptEngineMetrics()); + results.put("customLabels", CustomLabelService.get().getCustomLabelMetrics()); + Map roleAssignments = new HashMap<>(); + final String roleCountSql = "SELECT COUNT(*) FROM core.RoleAssignments WHERE userid > 0 AND role = ?"; + roleAssignments.put("assayDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.assay.security.AssayDesignerRole").getObject(Long.class)); + roleAssignments.put("dataClassDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.DataClassDesignerRole").getObject(Long.class)); + roleAssignments.put("sampleTypeDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.SampleTypeDesignerRole").getObject(Long.class)); + results.put("roleAssignments", roleAssignments); + + Map allowListCounts = new HashMap<>(); + for (AllowListType type : AllowListType.values()) + { + allowListCounts.put(type.name(), type.getValues().size()); + } + results.put("allowListCounts", allowListCounts); + + return results; + }); + + UsageMetricsService.get().registerUsageMetrics(getName(), WebSocketConnectionManager.getInstance()); + UsageMetricsService.get().registerUsageMetrics(getName(), DbLoginManager.getMetricsProvider()); + UsageMetricsService.get().registerUsageMetrics(getName(), SecurityManager.getMetricsProvider()); + UsageMetricsService.get().registerUsageMetrics(getName(), DisplayFormatAnalyzer.getMetricsProvider()); + UsageMetricsService.get().registerUsageMetrics(getName(), Portal.getMetricsProvider()); + + if (AppProps.getInstance().isDevMode()) + AntiVirusProviderRegistry.get().registerAntiVirusProvider(new DummyAntiVirusService.Provider()); + + FileContentService fileContentService = FileContentService.get(); + if (fileContentService != null) + fileContentService.addFileListener(WebFilesResolverImpl.get()); + + RoleManager.registerPermission(new QCAnalystPermission()); + MarkdownService.setInstance(new MarkdownServiceImpl()); + + // initialize email preference service and listeners + MessageConfigService.setInstance(new EmailPreferenceConfigServiceImpl()); + ContainerManager.addContainerListener(new EmailPreferenceContainerListener()); + UserManager.addUserListener(new EmailPreferenceUserListener()); + + DatabaseMigrationService.get().registerHandler(CoreSchema.getInstance().getSchema(), new DefaultMigrationHandler() + { + @Override + public void beforeVerification(DbSchema targetSchema) + { + super.beforeVerification(targetSchema); + + // Delete root and shared containers that were needed for bootstrapping + TableInfo containers = CoreSchema.getInstance().getTableInfoContainers(); + Table.delete(containers); + DbScope targetScope = DbScope.getLabKeyScope(); + new SqlExecutor(targetScope).execute("ALTER SEQUENCE core.containers_rowid_seq RESTART"); // Reset Containers sequence + } + + @Override + public List getTablesToCopy(DbSchema targetSchema) + { + List tablesToCopy = super.getTablesToCopy(targetSchema); + tablesToCopy.remove(targetSchema.getTable("Modules")); + tablesToCopy.remove(targetSchema.getTable("SqlScripts")); + tablesToCopy.remove(targetSchema.getTable("UpgradeSteps")); + + return tablesToCopy; + } + }); + } + + // Issue 7527: Auto-detect missing SQL views and attempt to recreate + private void checkForMissingDbViews() + { + ModuleLoader.getInstance().getModules().stream() + .map(FileSqlScriptProvider::new) + .flatMap(p -> p.getSchemas().stream() + .filter(schema-> SchemaUpdateType.Before.getScript(p, schema) != null || SchemaUpdateType.After.getScript(p, schema) != null) + ) + .filter(schema -> TableXmlUtils.compareXmlToMetaData(schema, false, false, true).hasViewProblem()) + .findAny() + .ifPresent(schema -> + { + LOG.warn("At least one database view was not as expected in the {} schema. Attempting to recreate views automatically", schema.getName()); + ModuleLoader.getInstance().recreateViews(); + }); + } + + @Override + public void registerServlets(ServletContext servletCtx) + { +// even though there is one webdav tree rooted at "/" we still use two servlet bindings. +// This is because we want /_webdav/* to be resolved BEFORE all other servlet-mappings +// and /* to resolve AFTER all other servlet-mappings + _webdavServletDynamic = servletCtx.addServlet("static", new WebdavServlet(true)); + _webdavServletDynamic.setMultipartConfig(SpringActionController.getMultiPartConfigElement()); + _webdavServletDynamic.addMapping("/_webdav/*"); + } + + @Override + public void registerFinalServlets(ServletContext servletCtx) + { + _webdavServletDynamic.addMapping("/"); + } + + @Override + public void startBackgroundThreads() + { + SystemMaintenance.setTimer(); + ThumbnailServiceImpl.startThread(); + // Launch in the background, but delay by 10 seconds to reduce impact on other startup tasks + _warningProvider.startSchemaCheck(10); + + // Start up the default Quartz scheduler, used in many places + try + { + StdSchedulerFactory.getDefaultScheduler().start(); + } + catch (SchedulerException e) + { + throw UnexpectedException.wrap(e); + } + + if (MothershipReport.shouldReceiveMarketingUpdates()) + { + if (AppProps.getInstance().getUsageReportingLevel() == UsageReportingLevel.NONE) + { + // force the usage reporting level to on for community edition distributions + WriteableAppProps appProps = AppProps.getWriteableInstance(); + appProps.setUsageReportingLevel(UsageReportingLevel.ON); + appProps.save(User.getAdminServiceUser()); + } + } + // On bootstrap in production mode, this will send an initial ping with very little information, as the admin will + // not have set up their account yet. On later startups, depending on the reporting level, this will send an immediate + // ping, and then once every 24 hours. + UsageReportingLevel.reportNow(); + TempTableTracker.init(); + + // Loading the PDFBox font cache can be very slow on some agents; fill it proactively. Issue 50601 + JobRunner.getDefault().execute(() -> { + try + { + long start = System.currentTimeMillis(); + FontMapper mapper = FontMappers.instance(); + Method method = mapper.getClass().getMethod("getProvider"); + method.setAccessible(true); + method.invoke(mapper); + long duration = System.currentTimeMillis() - start; + LOG.info("Ensuring PDFBox on-disk font cache took {} seconds", Math.round(duration / 100.0) / 10.0); + } + catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) + { + LOG.warn("Unable to initialize PDFBox font cache", e); + } + }); + } + + @Override + public @NotNull List getDetailedSummary(Container c, User user) + { + long childContainerCount = ContainerManager.getChildren(c).stream().filter(Container::isInFolderNav).count(); + if (childContainerCount == 0) + return Collections.emptyList(); + return List.of(new Summary(childContainerCount, "Subfolder")); + } + + @Override + public JSONObject getPageContextJson(ContainerUser context) + { + JSONObject json = new JSONObject(getDefaultPageContextJson(context.getContainer())); + json.put("productFeatures", ProductRegistry.getProductFeatureSet()); + json.put("primaryApplicationId", ProductRegistry.get().getPrimaryApplicationId(context.getContainer())); + json.put(AppProps.DEPRECATED_OBJECT_LEVEL_DISCUSSIONS, AppProps.getInstance().isOptionalFeatureEnabled(AppProps.DEPRECATED_OBJECT_LEVEL_DISCUSSIONS)); + return json; + } + + @Override + public String getTabName(ViewContext context) + { + return "Portal"; + } + + + @Override + public ActionURL getTabURL(Container c, User user) + { + if (user == null) + return AppProps.getInstance().getHomePageActionURL(); + + return PageFlowUtil.urlProvider(ProjectUrls.class).getBeginURL(c); + } + + @Override + public TabDisplayMode getTabDisplayMode() + { + return TabDisplayMode.DISPLAY_USER_PREFERENCE_DEFAULT; + } + + @Override + @NotNull + public Set getIntegrationTests() + { + // Must be mutable since we add the dialect tests below + Set testClasses = Sets.newHashSet + ( + AdminController.SchemaVersionTestCase.class, + AdminController.SerializationTest.class, + AdminController.TestCase.class, + AdminController.WorkbookDeleteTestCase.class, + AllowListType.TestCase.class, + AttachmentServiceImpl.TestCase.class, + CoreController.TestCase.class, + DataRegion.TestCase.class, + DavController.TestCase.class, + EmailServiceImpl.TestCase.class, + FilesSiteSettingsAction.TestCase.class, + LoggerController.TestCase.class, + LoggingTestCase.class, + LoginController.TestCase.class, + MarkdownTestCase.class, + ModuleInfoTestCase.class, + ModulePropertiesTestCase.class, + ModuleStaticResolverImpl.TestCase.class, + NotificationServiceImpl.TestCase.class, + PortalJUnitTest.class, + PostgreSqlInClauseTest.class, + ProductRegistry.TestCase.class, + RadeoxRenderer.RadeoxRenderTest.class, + RhinoService.TestCase.class, + SchemaXMLTestCase.class, + SecurityApiActions.TestCase.class, + SecurityController.TestCase.class, + SqlDialect.DialectTestCase.class, + SqlScriptController.TestCase.class, + TableViewFormTestCase.class, + UnknownSchemasTest.class, + UserController.TestCase.class + ); + + testClasses.addAll(SqlDialectManager.getAllJUnitTests()); + + return testClasses; + } + + @NotNull + @Override + public Set getUnitTests() + { + return Set.of( + ApiJsonWriter.TestCase.class, + ClassLoaderTestCase.class, + CopyFileRootPipelineJob.TestCase.class, + OutOfRangeDisplayColumn.TestCase.class, + PostgreSqlVersion.TestCase.class, + ScriptEngineManagerImpl.TestCase.class, + StatsServiceImpl.TestCase.class, + + + // Radeox tests + SimpleListTest.class, + ExampleListFormatterTest.class, + AtoZListFormatterTest.class, + BaseRenderEngineTest.class, + BasicRegexTest.class, + ItalicFilterTest.class, + BoldFilterTest.class, + KeyFilterTest.class, + NewlineFilterTest.class, + LineFilterTest.class, + TypographyFilterTest.class, + HtmlRemoveFilterTest.class, + StrikeThroughFilterTest.class, + UrlFilterTest.class, + ParamFilterTest.class, + FilterPipeTest.class, + EscapeFilterTest.class, + LinkTestFilterTest.class, + WikiLinkFilterTest.class, + SmileyFilterTest.class, + ListFilterTest.class, + HeadingFilterTest.class + ); + } + + @Override + public DbSchema createModuleDbSchema(DbScope scope, String metaDataName, Map tableInfoFactoryMap) + { + // Special case for the "labkey" schema we create in every module data source + if ("labkey".equals(metaDataName)) + return new LabKeyDbSchema(scope, tableInfoFactoryMap); + + return super.createModuleDbSchema(scope, metaDataName, tableInfoFactoryMap); + } + + @Override + @NotNull + public Collection getSchemaNames() + { + return List.of + ( + CoreSchema.getInstance().getSchemaName(), // core + PropertySchema.getInstance().getSchemaName(), // prop + TestSchema.getInstance().getSchemaName(), // test + DbSchema.TEMP_SCHEMA_NAME // temp + ); + } + + @NotNull + @Override + public Collection getProvisionedSchemaNames() + { + return Collections.singleton(DbSchema.TEMP_SCHEMA_NAME); + } + + @NotNull + @Override + public Set getSchemasToTest() + { + Set result = new LinkedHashSet<>(super.getSchemasToTest()); + + // Add the "labkey" schema in all module data sources as well... should match application.properties + for (String dataSourceName : ModuleLoader.getInstance().getAllModuleDataSourceNames()) + { + DbScope scope = DbScope.getDbScope(dataSourceName); + if (scope != null) + { + result.add(scope.getLabKeySchema()); + } + } + + return result; + } + + @Override + public void enumerateDocuments(SearchService.TaskIndexingQueue queue, Date since) + { + Container c = queue.getContainer(); + if (c.isRoot()) + return; + + Runnable r = () -> { + Container p = c.getProject(); + if (null == p) + return; + String title; + String keywords; + String body; + + // UNDONE: generalize to other folder types + StudyService svc = StudyService.get(); + Study study = svc != null ? svc.getStudy(c) : null; + + if (null != study) + { + title = study.getSearchDisplayTitle(); + keywords = study.getSearchKeywords(); + body = study.getSearchBody(); + } + else + { + String type = c.getContainerNoun(true); + + String containerTitle = c.getTitle(); + + String description = StringUtils.trimToEmpty(c.getDescription()); + title = type + " -- " + containerTitle; + User u_user = UserManager.getUser(c.getCreatedBy()); + String user = (u_user == null) ? "" : u_user.getDisplayName(User.getSearchUser()); + keywords = description + " " + type + " " + user; + body = type + " " + containerTitle + (c.isProject() ? "" : " in Project " + p.getName()); + body += "\n" + description; + } + + String identifiers = c.getName(); + + Map properties = new HashMap<>(); + + assert (null != keywords); + properties.put(SearchService.PROPERTY.identifiersMed.toString(), identifiers); + properties.put(SearchService.PROPERTY.keywordsMed.toString(), keywords); + properties.put(SearchService.PROPERTY.title.toString(), title); + properties.put(SearchService.PROPERTY.categories.toString(), SearchService.navigationCategory.getName()); + ActionURL startURL = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(c); + startURL.setExtraPath(c.getId()); + WebdavResource doc = new SimpleDocumentResource(c.getParsedPath(), + "link:" + c.getId(), + c.getEntityId(), + "text/plain", + body, + startURL, + UserManager.getUser(c.getCreatedBy()), c.getCreated(), + null, null, + properties); + queue.addResource(doc); + }; + r.run(); + } + + @Override + public void indexDeleted() + { + new SqlExecutor(CoreSchema.getInstance().getSchema()).execute("UPDATE core.Documents SET LastIndexed = NULL"); + } + + /** + * Handles startup props for LookAndFeelSettings resources + */ + private void populateLookAndFeelResourcesWithStartupProps() + { + ModuleLoader.getInstance().handleStartupProperties(new StandardStartupPropertyHandler<>(WriteableLookAndFeelProperties.SCOPE_LOOK_AND_FEEL_SETTINGS, ResourceType.class) + { + @Override + public void handle(Map map) + { + boolean incrementRevision = false; + + for (Map.Entry entry : map.entrySet()) + { + SiteResourceHandler handler = getResourceHandler(entry.getKey()); + if (handler != null) + incrementRevision |= setSiteResource(handler, entry.getValue(), User.guest); + } + + // Bump the look & feel revision so browsers retrieve the new logo, custom stylesheet, etc. + if (incrementRevision) + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + } + }); + } + + /** + * This method handles the home project settings + */ + private void populateSiteSettingsWithStartupProps() + { + Map props = AppProps.getInstance().getStashedStartupProperties(); + + StartupPropertyEntry folderTypeEntry = props.get(homeProjectFolderType); + if (null != folderTypeEntry) + { + FolderType folderType = FolderTypeManager.get().getFolderType(folderTypeEntry.getValue()); + if (folderType != null) + // using guest user since the server startup doesn't have a true user (this will be used for audit events) + ContainerManager.getHomeContainer().setFolderType(folderType, User.guest); + else + LOG.error("Unable to find folder type for home project during server startup: " + folderTypeEntry.getValue()); + } + + StartupPropertyEntry resetPermissionsEntry = props.get(homeProjectResetPermissions); + if (null != resetPermissionsEntry && Boolean.valueOf(resetPermissionsEntry.getValue())) + { + // reset the home project permissions to remove the default assignments given at server install + MutableSecurityPolicy homePolicy = new MutableSecurityPolicy(ContainerManager.getHomeContainer()); + SecurityPolicyManager.savePolicy(homePolicy, User.getAdminServiceUser()); + // remove the guest role assignment from the support subfolder + Group guests = SecurityManager.getGroup(Group.groupGuests); + if (null != guests) + { + Container supportFolder = ContainerManager.getDefaultSupportContainer(); + if (supportFolder != null) + { + MutableSecurityPolicy supportPolicy = new MutableSecurityPolicy(supportFolder.getPolicy()); + for (Role assignedRole : supportPolicy.getAssignedRoles(guests)) + supportPolicy.removeRoleAssignment(guests, assignedRole); + SecurityPolicyManager.savePolicy(supportPolicy, User.getAdminServiceUser()); + } + } + } + + StartupPropertyEntry webparts = props.get(homeProjectWebparts); + if (null != webparts) + { + // Clear existing webparts added by core and wiki modules + Container homeContainer = ContainerManager.getHomeContainer(); + Portal.saveParts(homeContainer, Collections.emptyList()); + + for (String webpartName : StringUtils.split(webparts.getValue(), ';')) + { + WebPartFactory webPartFactory = Portal.getPortalPart(webpartName); + if (webPartFactory != null) + addWebPart(webPartFactory.getName(), homeContainer, HttpView.BODY); + } + } + } + + private @Nullable SiteResourceHandler getResourceHandler(@NotNull ResourceType type) + { + return LookAndFeelPropertiesManager.get().getResourceHandler(type); + } + + private boolean setSiteResource(SiteResourceHandler resourceHandler, StartupPropertyEntry prop, User user) + { + Resource resource = getModuleResourceFromPropValue(prop.getValue()); + if (resource != null) + { + try + { + resourceHandler.accept(resource, ContainerManager.getRoot(), user); + return true; + } + catch(Exception e) + { + LOG.error("Exception setting {} during server startup.", prop.getName(), e); + } + } + + LOG.error("Unable to find {} resource during server startup: {}", prop.getName(), prop.getValue()); + return false; + } + + private Resource getModuleResourceFromPropValue(String propValue) + { + if (propValue != null) + { + // split the prop value on the separator char to get the module name and resource path in that module + String moduleName = propValue.substring(0, propValue.indexOf(":")); + String resourcePath = propValue.substring(propValue.indexOf(":") + 1); + + Module module = ModuleLoader.getInstance().getModule(moduleName); + if (module != null) + return module.getModuleResource(resourcePath); + } + + return null; + } + + public void rerunSchemaCheck() + { + // Queue a job without delay. This avoids executing multiple overlapping schema checks. Not bothering with a + // more surgical approach since this variant is likely being called during development. + _warningProvider.startSchemaCheck(0); + } +} diff --git a/core/src/org/labkey/core/admin/AdminController.java b/core/src/org/labkey/core/admin/AdminController.java index 865bb412b9e..f22bf400c8f 100644 --- a/core/src/org/labkey/core/admin/AdminController.java +++ b/core/src/org/labkey/core/admin/AdminController.java @@ -1,12299 +1,12295 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.core.admin; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Joiner; -import com.google.common.util.concurrent.UncheckedExecutionException; -import jakarta.mail.MessagingException; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.collections4.MultiValuedMap; -import org.apache.commons.collections4.map.LRUMap; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.xmlbeans.XmlOptions; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.jfree.chart.ChartFactory; -import org.jfree.chart.ChartUtilities; -import org.jfree.chart.JFreeChart; -import org.jfree.chart.plot.PlotOrientation; -import org.jfree.data.category.DefaultCategoryDataset; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.Constants; -import org.labkey.api.action.ApiResponse; -import org.labkey.api.action.ApiSimpleResponse; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.action.BaseApiAction; -import org.labkey.api.action.BaseViewAction; -import org.labkey.api.action.ConfirmAction; -import org.labkey.api.action.ExportAction; -import org.labkey.api.action.FormHandlerAction; -import org.labkey.api.action.FormViewAction; -import org.labkey.api.action.HasViewContext; -import org.labkey.api.action.IgnoresAllocationTracking; -import org.labkey.api.action.LabKeyError; -import org.labkey.api.action.Marshal; -import org.labkey.api.action.Marshaller; -import org.labkey.api.action.MutatingApiAction; -import org.labkey.api.action.QueryViewAction; -import org.labkey.api.action.ReadOnlyApiAction; -import org.labkey.api.action.ReturnUrlForm; -import org.labkey.api.action.SimpleApiJsonForm; -import org.labkey.api.action.SimpleErrorView; -import org.labkey.api.action.SimpleRedirectAction; -import org.labkey.api.action.SimpleViewAction; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AbstractFolderContext.ExportType; -import org.labkey.api.admin.AdminBean; -import org.labkey.api.admin.AdminUrls; -import org.labkey.api.admin.FolderExportContext; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.admin.FolderWriter; -import org.labkey.api.admin.FolderWriterImpl; -import org.labkey.api.admin.HealthCheck; -import org.labkey.api.admin.HealthCheckRegistry; -import org.labkey.api.admin.ImportOptions; -import org.labkey.api.admin.StaticLoggerGetter; -import org.labkey.api.admin.TableXmlUtils; -import org.labkey.api.admin.sitevalidation.SiteValidationResult; -import org.labkey.api.admin.sitevalidation.SiteValidationResultList; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.provider.ContainerAuditProvider; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.cache.CacheStats; -import org.labkey.api.cache.TrackingCache; -import org.labkey.api.cloud.CloudStoreService; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.CaseInsensitiveHashSetValuedMap; -import org.labkey.api.collections.LabKeyCollectors; -import org.labkey.api.compliance.ComplianceFolderSettings; -import org.labkey.api.compliance.ComplianceService; -import org.labkey.api.compliance.PhiColumnBehavior; -import org.labkey.api.data.ButtonBar; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.ConnectionWrapper; -import org.labkey.api.data.Container; -import org.labkey.api.data.Container.ContainerException; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerType; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DataColumn; -import org.labkey.api.data.DataRegion; -import org.labkey.api.data.DataRegionSelection; -import org.labkey.api.data.DatabaseTableType; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DisplayColumn; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.MenuButton; -import org.labkey.api.data.MvUtil; -import org.labkey.api.data.NormalContainerType; -import org.labkey.api.data.PHI; -import org.labkey.api.data.RenderContext; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.Sort; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TransactionFilter; -import org.labkey.api.data.WorkbookContainerType; -import org.labkey.api.data.dialect.SqlDialect.ExecutionPlanType; -import org.labkey.api.data.queryprofiler.QueryProfiler; -import org.labkey.api.data.queryprofiler.QueryProfiler.QueryStatTsvWriter; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Lookup; -import org.labkey.api.files.FileContentService; -import org.labkey.api.message.settings.AbstractConfigTypeProvider.EmailConfigFormImpl; -import org.labkey.api.message.settings.MessageConfigService; -import org.labkey.api.message.settings.MessageConfigService.ConfigTypeProvider; -import org.labkey.api.message.settings.MessageConfigService.NotificationOption; -import org.labkey.api.message.settings.MessageConfigService.UserPreference; -import org.labkey.api.miniprofiler.RequestInfo; -import org.labkey.api.module.AllowedBeforeInitialUserIsSet; -import org.labkey.api.module.AllowedDuringUpgrade; -import org.labkey.api.module.DefaultModule; -import org.labkey.api.module.FolderType; -import org.labkey.api.module.FolderTypeManager; -import org.labkey.api.module.IgnoresForbiddenProjectCheck; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleHtmlView; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.module.ModuleLoader.SchemaActions; -import org.labkey.api.module.ModuleLoader.SchemaAndModule; -import org.labkey.api.module.SimpleModule; -import org.labkey.api.moduleeditor.api.ModuleEditorService; -import org.labkey.api.pipeline.DirectoryNotDeletedException; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineStatusFile; -import org.labkey.api.pipeline.PipelineStatusUrls; -import org.labkey.api.pipeline.PipelineUrls; -import org.labkey.api.pipeline.PipelineValidationException; -import org.labkey.api.pipeline.view.SetupForm; -import org.labkey.api.products.ProductRegistry; -import org.labkey.api.query.DefaultSchema; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QuerySchema; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.RuntimeValidationException; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reports.ExternalScriptEngineDefinition; -import org.labkey.api.reports.LabKeyScriptEngineManager; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.ActionNames; -import org.labkey.api.security.AdminConsoleAction; -import org.labkey.api.security.CSRF; -import org.labkey.api.security.Directive; -import org.labkey.api.security.Group; -import org.labkey.api.security.GroupManager; -import org.labkey.api.security.IgnoresTermsOfUse; -import org.labkey.api.security.LoginUrls; -import org.labkey.api.security.MutableSecurityPolicy; -import org.labkey.api.security.RequiresLogin; -import org.labkey.api.security.RequiresNoPermission; -import org.labkey.api.security.RequiresPermission; -import org.labkey.api.security.RequiresSiteAdmin; -import org.labkey.api.security.RoleAssignment; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.SecurityPolicy; -import org.labkey.api.security.SecurityPolicyManager; -import org.labkey.api.security.SecurityUrls; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.security.UserPrincipal; -import org.labkey.api.security.ValidEmail; -import org.labkey.api.security.impersonation.GroupImpersonationContextFactory; -import org.labkey.api.security.impersonation.ImpersonationContext; -import org.labkey.api.security.impersonation.RoleImpersonationContextFactory; -import org.labkey.api.security.impersonation.UserImpersonationContextFactory; -import org.labkey.api.security.permissions.AbstractActionPermissionTest; -import org.labkey.api.security.permissions.AdminOperationsPermission; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.ApplicationAdminPermission; -import org.labkey.api.security.permissions.CreateProjectPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.PlatformDeveloperPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.SiteAdminPermission; -import org.labkey.api.security.permissions.TroubleshooterPermission; -import org.labkey.api.security.permissions.UploadFileBasedModulePermission; -import org.labkey.api.security.roles.EditorRole; -import org.labkey.api.security.roles.FolderAdminRole; -import org.labkey.api.security.roles.ProjectAdminRole; -import org.labkey.api.security.roles.ReaderRole; -import org.labkey.api.security.roles.Role; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.security.roles.SharedViewEditorRole; -import org.labkey.api.services.ServiceRegistry; -import org.labkey.api.settings.AdminConsole; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.ConceptURIProperties; -import org.labkey.api.settings.DateParsingMode; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.settings.LookAndFeelPropertiesManager.ResourceType; -import org.labkey.api.settings.NetworkDriveProps; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.settings.OptionalFeatureService.FeatureType; -import org.labkey.api.settings.ProductConfiguration; -import org.labkey.api.settings.WriteableAppProps; -import org.labkey.api.settings.WriteableFolderLookAndFeelProperties; -import org.labkey.api.settings.WriteableLookAndFeelProperties; -import org.labkey.api.util.ButtonBuilder; -import org.labkey.api.util.ConfigurationException; -import org.labkey.api.util.DOM; -import org.labkey.api.util.DOM.Renderable; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.DebugInfoDumper; -import org.labkey.api.util.ExceptionReportingLevel; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.FolderDisplayMode; -import org.labkey.api.util.GUID; -import org.labkey.api.util.HelpTopic; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.HtmlStringBuilder; -import org.labkey.api.util.HttpsUtil; -import org.labkey.api.util.JsonUtil; -import org.labkey.api.util.LinkBuilder; -import org.labkey.api.util.MailHelper; -import org.labkey.api.util.MemTracker; -import org.labkey.api.util.MemTracker.HeldReference; -import org.labkey.api.util.MothershipReport; -import org.labkey.api.util.NetworkDrive; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.ResponseHelper; -import org.labkey.api.util.SafeToRenderEnum; -import org.labkey.api.util.SessionAppender; -import org.labkey.api.util.StringExpressionFactory; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.util.SystemMaintenance.SystemMaintenanceProperties; -import org.labkey.api.util.SystemMaintenanceJob; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.Tuple3; -import org.labkey.api.util.URLHelper; -import org.labkey.api.util.UniqueID; -import org.labkey.api.util.UsageReportingLevel; -import org.labkey.api.util.emailTemplate.EmailTemplate; -import org.labkey.api.util.emailTemplate.EmailTemplateService; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.DataView; -import org.labkey.api.view.FolderManagement.FolderManagementViewAction; -import org.labkey.api.view.FolderManagement.FolderManagementViewPostAction; -import org.labkey.api.view.FolderManagement.ProjectSettingsViewAction; -import org.labkey.api.view.FolderManagement.ProjectSettingsViewPostAction; -import org.labkey.api.view.FolderManagement.TYPE; -import org.labkey.api.view.FolderTab; -import org.labkey.api.view.HtmlView; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.Portal; -import org.labkey.api.view.RedirectException; -import org.labkey.api.view.ShortURLRecord; -import org.labkey.api.view.ShortURLService; -import org.labkey.api.view.TabStripView; -import org.labkey.api.view.URLException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.VBox; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.ViewServlet; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.EmptyView; -import org.labkey.api.view.template.PageConfig; -import org.labkey.api.view.template.PageConfig.Template; -import org.labkey.api.wiki.WikiRendererType; -import org.labkey.api.wiki.WikiRenderingService; -import org.labkey.api.writer.FileSystemFile; -import org.labkey.api.writer.HtmlWriter; -import org.labkey.api.writer.ZipFile; -import org.labkey.api.writer.ZipUtil; -import org.labkey.bootstrap.ExplodedModuleService; -import org.labkey.core.admin.miniprofiler.MiniProfilerController; -import org.labkey.core.admin.sitevalidation.SiteValidationJob; -import org.labkey.core.admin.sql.SqlScriptController; -import org.labkey.core.login.LoginController; -import org.labkey.core.portal.CollaborationFolderType; -import org.labkey.core.portal.ProjectController; -import org.labkey.core.query.CoreQuerySchema; -import org.labkey.core.query.PostgresUserSchema; -import org.labkey.core.reports.ExternalScriptEngineDefinitionImpl; -import org.labkey.core.security.AllowedExternalResourceHosts; -import org.labkey.core.security.AllowedExternalResourceHosts.AllowedHost; -import org.labkey.core.security.BlockListFilter; -import org.labkey.core.security.SecurityController; -import org.labkey.data.xml.TablesDocument; -import org.labkey.filters.ContentSecurityPolicyFilter; -import org.labkey.security.xml.GroupEnumType; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.servlet.ModelAndView; -import org.springframework.web.servlet.mvc.Controller; - -import java.awt.*; -import java.beans.Introspector; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.StringWriter; -import java.lang.management.BufferPoolMXBean; -import java.lang.management.ClassLoadingMXBean; -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.lang.management.MemoryPoolMXBean; -import java.lang.management.MemoryType; -import java.lang.management.MemoryUsage; -import java.lang.management.OperatingSystemMXBean; -import java.lang.management.RuntimeMXBean; -import java.lang.management.ThreadMXBean; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.sql.SQLException; -import java.text.DecimalFormat; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.StringTokenizer; -import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.commons.lang3.StringUtils.isNotBlank; -import static org.labkey.api.data.MultiValuedRenderContext.VALUE_DELIMITER_REGEX; -import static org.labkey.api.settings.AdminConsole.SettingsLinkType.Configuration; -import static org.labkey.api.settings.AdminConsole.SettingsLinkType.Diagnostics; -import static org.labkey.api.util.DOM.A; -import static org.labkey.api.util.DOM.Attribute.href; -import static org.labkey.api.util.DOM.Attribute.method; -import static org.labkey.api.util.DOM.Attribute.name; -import static org.labkey.api.util.DOM.Attribute.style; -import static org.labkey.api.util.DOM.Attribute.title; -import static org.labkey.api.util.DOM.Attribute.type; -import static org.labkey.api.util.DOM.Attribute.value; -import static org.labkey.api.util.DOM.BR; -import static org.labkey.api.util.DOM.DIV; -import static org.labkey.api.util.DOM.LI; -import static org.labkey.api.util.DOM.SPAN; -import static org.labkey.api.util.DOM.STYLE; -import static org.labkey.api.util.DOM.TABLE; -import static org.labkey.api.util.DOM.TD; -import static org.labkey.api.util.DOM.TR; -import static org.labkey.api.util.DOM.UL; -import static org.labkey.api.util.DOM.at; -import static org.labkey.api.util.DOM.cl; -import static org.labkey.api.util.DOM.createHtmlFragment; -import static org.labkey.api.util.HtmlString.NBSP; -import static org.labkey.api.util.logging.LogHelper.getLabKeyLogDir; -import static org.labkey.api.view.FolderManagement.EVERY_CONTAINER; -import static org.labkey.api.view.FolderManagement.FOLDERS_AND_PROJECTS; -import static org.labkey.api.view.FolderManagement.FOLDERS_ONLY; -import static org.labkey.api.view.FolderManagement.NOT_ROOT; -import static org.labkey.api.view.FolderManagement.PROJECTS_ONLY; -import static org.labkey.api.view.FolderManagement.ROOT; -import static org.labkey.api.view.FolderManagement.addTab; - -public class AdminController extends SpringActionController -{ - private static final DefaultActionResolver _actionResolver = new DefaultActionResolver( - AdminController.class, - FileListAction.class, - FilesSiteSettingsAction.class, - UpdateFilePathsAction.class - ); - - private static final Logger LOG = LogHelper.getLogger(AdminController.class, "Admin-related UI and APIs"); - private static final Logger CLIENT_LOG = LogHelper.getLogger(LogAction.class, "Client/browser logging submitted to server"); - private static final String HEAP_MEMORY_KEY = "Total Heap Memory"; - - private static long _errorMark = 0; - private static long _primaryLogMark = 0; - - public static void registerAdminConsoleLinks() - { - Container root = ContainerManager.getRoot(); - - // Configuration - AdminConsole.addLink(Configuration, "authentication", urlProvider(LoginUrls.class).getConfigureURL()); - AdminConsole.addLink(Configuration, "email customization", new ActionURL(CustomizeEmailAction.class, root), AdminPermission.class); - AdminConsole.addLink(Configuration, "deprecated features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Deprecated.name()), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "experimental features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Experimental.name()), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "optional features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Optional.name()), TroubleshooterPermission.class); - if (!ProductRegistry.getProducts().isEmpty()) - AdminConsole.addLink(Configuration, "product configuration", new ActionURL(ProductConfigurationAction.class, root), AdminOperationsPermission.class); - // TODO move to FileContentModule - if (ModuleLoader.getInstance().hasModule("FileContent")) - AdminConsole.addLink(Configuration, "files", new ActionURL(FilesSiteSettingsAction.class, root), AdminOperationsPermission.class); - AdminConsole.addLink(Configuration, "folder types", new ActionURL(FolderTypesAction.class, root), AdminPermission.class); - AdminConsole.addLink(Configuration, "look and feel settings", new ActionURL(LookAndFeelSettingsAction.class, root)); - AdminConsole.addLink(Configuration, "missing value indicators", new AdminUrlsImpl().getMissingValuesURL(root), AdminPermission.class); - AdminConsole.addLink(Configuration, "project display order", new ActionURL(ReorderFoldersAction.class, root), AdminPermission.class); - AdminConsole.addLink(Configuration, "short urls", new ActionURL(ShortURLAdminAction.class, root), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "site settings", new AdminUrlsImpl().getCustomizeSiteURL()); - AdminConsole.addLink(Configuration, "system maintenance", new ActionURL(ConfigureSystemMaintenanceAction.class, root)); - AdminConsole.addLink(Configuration, "allowed external redirect hosts", new ActionURL(AllowListAction.class, root).addParameter("type", AllowListType.Redirect.name()), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "allowed external resource hosts", new ActionURL(ExternalSourcesAction.class, root), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "allowed file extensions", new ActionURL(AllowListAction.class, root).addParameter("type", AllowListType.FileExtension.name()), TroubleshooterPermission.class); - - // Diagnostics - AdminConsole.addLink(Diagnostics, "actions", new ActionURL(ActionsAction.class, root)); - AdminConsole.addLink(Diagnostics, "attachments", new ActionURL(AttachmentsAction.class, root)); - AdminConsole.addLink(Diagnostics, "caches", new ActionURL(CachesAction.class, root)); - AdminConsole.addLink(Diagnostics, "check database", new ActionURL(DbCheckerAction.class, root), AdminOperationsPermission.class); - AdminConsole.addLink(Diagnostics, "credits", new ActionURL(CreditsAction.class, root)); - AdminConsole.addLink(Diagnostics, "dump heap", new ActionURL(DumpHeapAction.class, root)); - AdminConsole.addLink(Diagnostics, "environment variables", new ActionURL(EnvironmentVariablesAction.class, root), SiteAdminPermission.class); - AdminConsole.addLink(Diagnostics, "memory usage", new ActionURL(MemTrackerAction.class, root)); - - if (CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) - { - AdminConsole.addLink(Diagnostics, "postgres activity", new ActionURL(PostgresStatActivityAction.class, root)); - AdminConsole.addLink(Diagnostics, "postgres locks", new ActionURL(PostgresLocksAction.class, root)); - } - - AdminConsole.addLink(Diagnostics, "profiler", new ActionURL(MiniProfilerController.ManageAction.class, root)); - AdminConsole.addLink(Diagnostics, "queries", getQueriesURL(null)); - AdminConsole.addLink(Diagnostics, "reset site errors", new ActionURL(ResetErrorMarkAction.class, root), AdminPermission.class); - AdminConsole.addLink(Diagnostics, "running threads", new ActionURL(ShowThreadsAction.class, root)); - AdminConsole.addLink(Diagnostics, "site validation", new ActionURL(ConfigureSiteValidationAction.class, root), AdminPermission.class); - AdminConsole.addLink(Diagnostics, "sql scripts", new ActionURL(SqlScriptController.ScriptsAction.class, root), AdminOperationsPermission.class); - AdminConsole.addLink(Diagnostics, "suspicious activity", new ActionURL(SuspiciousAction.class, root)); - AdminConsole.addLink(Diagnostics, "system properties", new ActionURL(SystemPropertiesAction.class, root), SiteAdminPermission.class); - AdminConsole.addLink(Diagnostics, "test email configuration", new ActionURL(EmailTestAction.class, root), AdminOperationsPermission.class); - AdminConsole.addLink(Diagnostics, "view all site errors", new ActionURL(ShowAllErrorsAction.class, root)); - AdminConsole.addLink(Diagnostics, "view all site errors since reset", new ActionURL(ShowErrorsSinceMarkAction.class, root)); - AdminConsole.addLink(Diagnostics, "view csp report log file", new ActionURL(ShowCspReportLogAction.class, root)); - AdminConsole.addLink(Diagnostics, "view primary site log file", new ActionURL(ShowPrimaryLogAction.class, root)); - } - - public static void registerManagementTabs() - { - addTab(TYPE.FolderManagement, "Folder Tree", "folderTree", EVERY_CONTAINER, ManageFoldersAction.class); - addTab(TYPE.FolderManagement, "Folder Type", "folderType", NOT_ROOT, FolderTypeAction.class); - addTab(TYPE.FolderManagement, "Missing Values", "mvIndicators", EVERY_CONTAINER, MissingValuesAction.class); - addTab(TYPE.FolderManagement, "Module Properties", "props", c -> { - if (!c.isRoot()) - { - // Show module properties tab only if a module w/ properties to set is present for current folder - for (Module m : c.getActiveModules()) - if (!m.getModuleProperties().isEmpty()) - return true; - } - - return false; - }, ModulePropertiesAction.class); - addTab(TYPE.FolderManagement, "Concepts", "concepts", c -> { - // Show Concepts tab only if the experiment module is enabled in this container - return c.getActiveModules().contains(ModuleLoader.getInstance().getModule(ExperimentService.MODULE_NAME)); - }, AdminController.ConceptsAction.class); - // Show Notifications tab only if we have registered notification providers - addTab(TYPE.FolderManagement, "Notifications", "notifications", c -> NOT_ROOT.test(c) && !MessageConfigService.get().getConfigTypes().isEmpty(), NotificationsAction.class); - addTab(TYPE.FolderManagement, "Export", "export", NOT_ROOT, ExportFolderAction.class); - addTab(TYPE.FolderManagement, "Import", "import", NOT_ROOT, ImportFolderAction.class); - addTab(TYPE.FolderManagement, "Files", "files", FOLDERS_AND_PROJECTS, FileRootsAction.class); - addTab(TYPE.FolderManagement, "Formats", "settings", FOLDERS_ONLY, FolderSettingsAction.class); - addTab(TYPE.FolderManagement, "Information", "info", NOT_ROOT, FolderInformationAction.class); - addTab(TYPE.FolderManagement, "R Config", "rConfig", NOT_ROOT, RConfigurationAction.class); - - addTab(TYPE.ProjectSettings, "Properties", "properties", PROJECTS_ONLY, ProjectSettingsAction.class); - addTab(TYPE.ProjectSettings, "Resources", "resources", PROJECTS_ONLY, ResourcesAction.class); - addTab(TYPE.ProjectSettings, "Menu Bar", "menubar", PROJECTS_ONLY, MenuBarAction.class); - addTab(TYPE.ProjectSettings, "Files", "files", PROJECTS_ONLY, FilesAction.class); - - addTab(TYPE.LookAndFeelSettings, "Properties", "properties", ROOT, LookAndFeelSettingsAction.class); - addTab(TYPE.LookAndFeelSettings, "Resources", "resources", ROOT, AdminConsoleResourcesAction.class); - } - - public AdminController() - { - setActionResolver(_actionResolver); - } - - @RequiresNoPermission - public static class BeginAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(Object o) - { - return getShowAdminURL(); - } - } - - private void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action) - { - addAdminNavTrail(root, childTitle, action, getContainer()); - } - - private static void addAdminNavTrail(NavTree root, @NotNull Container container) - { - if (container.isRoot()) - root.addChild("Admin Console", getShowAdminURL().setFragment("links")); - } - - private static void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action, @NotNull Container container) - { - addAdminNavTrail(root, container); - root.addChild(childTitle, new ActionURL(action, container)); - } - - public static ActionURL getShowAdminURL() - { - return new ActionURL(ShowAdminAction.class, ContainerManager.getRoot()); - } - - @Override - protected void beforeAction(Controller action) throws ServletException - { - super.beforeAction(action); - if (action instanceof BaseViewAction viewaction) - viewaction.getPageConfig().setRobotsNone(); - } - - @AdminConsoleAction - public static class ShowAdminAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/admin.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - URLHelper returnUrl = getViewContext().getActionURL().getReturnUrl(); - if (null != returnUrl) - root.addChild("Return to Project", returnUrl); - root.addChild("Admin Console"); - setHelpTopic("siteManagement"); - } - } - - @RequiresPermission(TroubleshooterPermission.class) - public class ShowModuleErrorsAction extends SimpleViewAction - { - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Module Errors", this.getClass()); - } - - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/moduleErrors.jsp"); - } - } - - public static class AdminUrlsImpl implements AdminUrls - { - @Override - public ActionURL getModuleErrorsURL() - { - return new ActionURL(ShowModuleErrorsAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getAdminConsoleURL() - { - return getShowAdminURL(); - } - - @Override - public ActionURL getModuleStatusURL(URLHelper returnUrl) - { - return AdminController.getModuleStatusURL(returnUrl); - } - - @Override - public ActionURL getCustomizeSiteURL() - { - return new ActionURL(CustomizeSiteAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getCustomizeSiteURL(boolean upgradeInProgress) - { - ActionURL url = getCustomizeSiteURL(); - - if (upgradeInProgress) - url.addParameter("upgradeInProgress", "1"); - - return url; - } - - @Override - public ActionURL getProjectSettingsURL(Container c) - { - return new ActionURL(ProjectSettingsAction.class, LookAndFeelProperties.getSettingsContainer(c)); - } - - ActionURL getLookAndFeelResourcesURL(Container c) - { - return c.isRoot() ? new ActionURL(AdminConsoleResourcesAction.class, c) : new ActionURL(ResourcesAction.class, LookAndFeelProperties.getSettingsContainer(c)); - } - - @Override - public ActionURL getProjectSettingsMenuURL(Container c) - { - return new ActionURL(MenuBarAction.class, LookAndFeelProperties.getSettingsContainer(c)); - } - - @Override - public ActionURL getProjectSettingsFileURL(Container c) - { - return new ActionURL(FilesAction.class, LookAndFeelProperties.getSettingsContainer(c)); - } - - @Override - public ActionURL getCustomizeEmailURL(@NotNull Container c, @Nullable Class selectedTemplate, @Nullable URLHelper returnUrl) - { - return getCustomizeEmailURL(c, selectedTemplate == null ? null : selectedTemplate.getName(), returnUrl); - } - - public ActionURL getCustomizeEmailURL(@NotNull Container c, @Nullable String selectedTemplate, @Nullable URLHelper returnUrl) - { - ActionURL url = new ActionURL(CustomizeEmailAction.class, c); - if (selectedTemplate != null) - { - url.addParameter("templateClass", selectedTemplate); - } - if (returnUrl != null) - { - url.addReturnUrl(returnUrl); - } - return url; - } - - public ActionURL getResetLookAndFeelPropertiesURL(Container c) - { - return new ActionURL(ResetPropertiesAction.class, c); - } - - @Override - public ActionURL getMaintenanceURL(URLHelper returnUrl) - { - ActionURL url = new ActionURL(MaintenanceAction.class, ContainerManager.getRoot()); - if (returnUrl != null) - url.addReturnUrl(returnUrl); - return url; - } - - @Override - public ActionURL getModulesDetailsURL() - { - return new ActionURL(ModulesAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getDeleteModuleURL(String moduleName) - { - return new ActionURL(DeleteModuleAction.class, ContainerManager.getRoot()).addParameter("name", moduleName); - } - - @Override - public ActionURL getManageFoldersURL(Container c) - { - return new ActionURL(ManageFoldersAction.class, c); - } - - @Override - public ActionURL getFolderTypeURL(Container c) - { - return new ActionURL(FolderTypeAction.class, c); - } - - @Override - public ActionURL getExportFolderURL(Container c) - { - return new ActionURL(ExportFolderAction.class, c); - } - - @Override - public ActionURL getImportFolderURL(Container c) - { - return new ActionURL(ImportFolderAction.class, c); - } - - @Override - public ActionURL getCreateProjectURL(@Nullable ActionURL returnUrl) - { - return getCreateFolderURL(ContainerManager.getRoot(), returnUrl); - } - - @Override - public ActionURL getCreateFolderURL(Container c, @Nullable ActionURL returnUrl) - { - ActionURL result = new ActionURL(CreateFolderAction.class, c); - if (returnUrl != null) - { - result.addReturnUrl(returnUrl); - } - return result; - } - - public ActionURL getSetFolderPermissionsURL(Container c) - { - return new ActionURL(SetFolderPermissionsAction.class, c); - } - - @Override - public void addAdminNavTrail(NavTree root, @NotNull Container container) - { - AdminController.addAdminNavTrail(root, container); - } - - @Override - public void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action, @NotNull Container container) - { - AdminController.addAdminNavTrail(root, childTitle, action, container); - } - - @Override - public void addModulesNavTrail(NavTree root, String childTitle, @NotNull Container container) - { - if (container.isRoot()) - addAdminNavTrail(root, "Modules", ModulesAction.class, container); - - root.addChild(childTitle); - } - - @Override - public ActionURL getFileRootsURL(Container c) - { - return new ActionURL(FileRootsAction.class, c); - } - - @Override - public ActionURL getLookAndFeelSettingsURL(Container c) - { - if (c.isRoot()) - return getSiteLookAndFeelSettingsURL(); - else if (c.isProject()) - return getProjectSettingsURL(c); - else - return getFolderSettingsURL(c); - } - - @Override - public ActionURL getSiteLookAndFeelSettingsURL() - { - return new ActionURL(LookAndFeelSettingsAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getFolderSettingsURL(Container c) - { - return new ActionURL(FolderSettingsAction.class, c); - } - - @Override - public ActionURL getNotificationsURL(Container c) - { - return new ActionURL(NotificationsAction.class, c); - } - - @Override - public ActionURL getModulePropertiesURL(Container c) - { - return new ActionURL(ModulePropertiesAction.class, c); - } - - @Override - public ActionURL getMissingValuesURL(Container c) - { - return new ActionURL(MissingValuesAction.class, c); - } - - public ActionURL getInitialFolderSettingsURL(Container c) - { - return new ActionURL(SetInitialFolderSettingsAction.class, c); - } - - @Override - public ActionURL getMemTrackerURL() - { - return new ActionURL(MemTrackerAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getFilesSiteSettingsURL() - { - return new ActionURL(FilesSiteSettingsAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getSessionLoggingURL() - { - return new ActionURL(SessionLoggingAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getTrackedAllocationsViewerURL() - { - return new ActionURL(TrackedAllocationsViewerAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getSystemMaintenanceURL() - { - return new ActionURL(ConfigureSystemMaintenanceAction.class, ContainerManager.getRoot()); - } - - public static ActionURL getDeprecatedFeaturesURL() - { - return new ActionURL(OptionalFeaturesAction.class, ContainerManager.getRoot()).addParameter("type", FeatureType.Deprecated.name()); - } - } - - public static class MaintenanceBean - { - public HtmlString content; - public ActionURL loginURL; - } - - /** - * During upgrade, startup, or maintenance mode, the user will be redirected to - * MaintenanceAction and only admin users will be allowed to log into the server. - * The maintenance.jsp page checks startup is complete or adminOnly mode is turned off - * and will redirect to the returnUrl or the loginURL. - * See Issue 18758 for more information. - */ - @RequiresNoPermission - @AllowedDuringUpgrade - @IgnoresAllocationTracking - public static class MaintenanceAction extends SimpleViewAction - { - private String _title = "Maintenance in progress"; - - @Override - public ModelAndView getView(ReturnUrlForm form, BindException errors) - { - if (!getUser().hasSiteAdminPermission()) - { - getViewContext().getResponse().setStatus(HttpServletResponse.SC_UNAUTHORIZED); - } - getPageConfig().setTemplate(Template.Dialog); - - boolean upgradeInProgress = ModuleLoader.getInstance().isUpgradeInProgress(); - boolean startupInProgress = ModuleLoader.getInstance().isStartupInProgress(); - boolean maintenanceMode = AppProps.getInstance().isUserRequestedAdminOnlyMode(); - - HtmlString content = HtmlString.of("This site is currently undergoing maintenance, only site admins may login at this time."); - if (upgradeInProgress) - { - _title = "Upgrade in progress"; - content = HtmlString.of("Upgrade in progress: only site admins may login at this time. Your browser will be redirected when startup is complete."); - } - else if (startupInProgress) - { - _title = "Startup in progress"; - content = HtmlString.of("Startup in progress: only site admins may login at this time. Your browser will be redirected when startup is complete."); - } - else if (maintenanceMode) - { - WikiRenderingService wikiService = WikiRenderingService.get(); - content = wikiService.getFormattedHtml(WikiRendererType.RADEOX, ModuleLoader.getInstance().getAdminOnlyMessage(), "Admin only message"); - } - - if (content == null) - content = HtmlString.of(_title); - - ActionURL loginURL = null; - if (getUser().isGuest()) - { - URLHelper returnUrl = form.getReturnUrlHelper(); - if (returnUrl != null) - loginURL = urlProvider(LoginUrls.class).getLoginURL(ContainerManager.getRoot(), returnUrl); - else - loginURL = urlProvider(LoginUrls.class).getLoginURL(); - } - - MaintenanceBean bean = new MaintenanceBean(); - bean.content = content; - bean.loginURL = loginURL; - - JspView view = new JspView<>("/org/labkey/core/admin/maintenance.jsp", bean, errors); - view.setTitle(_title); - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild(_title); - } - } - - /** - * Similar to SqlScriptController.GetModuleStatusAction except that Guest is allowed to check that the startup is complete. - */ - @RequiresNoPermission - @AllowedDuringUpgrade - @IgnoresAllocationTracking - public static class StartupStatusAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(Object o, BindException errors) - { - JSONObject result = new JSONObject(); - result.put("startupComplete", ModuleLoader.getInstance().isStartupComplete()); - result.put("adminOnly", AppProps.getInstance().isUserRequestedAdminOnlyMode()); - - return new ApiSimpleResponse(result); - } - } - - @RequiresSiteAdmin - @IgnoresTermsOfUse - public static class GetPendingRequestCountAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(Object o, BindException errors) - { - JSONObject result = new JSONObject(); - result.put("pendingRequestCount", TransactionFilter.getPendingRequestCount() - 1 /* Exclude this request */); - - return new ApiSimpleResponse(result); - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetModulesAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(GetModulesForm form, BindException errors) - { - Container c = ContainerManager.getForPath(getContainer().getPath()); - - ApiSimpleResponse response = new ApiSimpleResponse(); - - List> qinfos = new ArrayList<>(); - - FolderType folderType = c.getFolderType(); - List allModules = new ArrayList<>(ModuleLoader.getInstance().getModules()); - allModules.sort(Comparator.comparing(module -> module.getTabName(getViewContext()), String.CASE_INSENSITIVE_ORDER)); - - //note: this has been altered to use Container.getRequiredModules() instead of FolderType - //this is b/c a parent container must consider child workbooks when determining the set of requiredModules - Set requiredModules = c.getRequiredModules(); //folderType.getActiveModules() != null ? folderType.getActiveModules() : new HashSet(); - Set activeModules = c.getActiveModules(getUser()); - - for (Module m : allModules) - { - Map qinfo = new HashMap<>(); - - qinfo.put("name", m.getName()); - qinfo.put("required", requiredModules.contains(m)); - qinfo.put("active", activeModules.contains(m) || requiredModules.contains(m)); - qinfo.put("enabled", (m.getTabDisplayMode() == Module.TabDisplayMode.DISPLAY_USER_PREFERENCE || - m.getTabDisplayMode() == Module.TabDisplayMode.DISPLAY_USER_PREFERENCE_DEFAULT) && !requiredModules.contains(m)); - qinfo.put("tabName", m.getTabName(getViewContext())); - qinfo.put("requireSitePermission", m.getRequireSitePermission()); - qinfos.add(qinfo); - } - - response.put("modules", qinfos); - response.put("folderType", folderType.getName()); - - return response; - } - } - - public static class GetModulesForm - { - } - - @RequiresNoPermission - @AllowedDuringUpgrade - // This action is invoked by HttpsUtil.checkSslRedirectConfiguration(), often while upgrade is in progress - public static class GuidAction extends ExportAction - { - @Override - public void export(Object o, HttpServletResponse response, BindException errors) throws Exception - { - response.getWriter().write(GUID.makeGUID()); - } - } - - /** - * Preform health checks corresponding to the given categories. - */ - @Marshal(Marshaller.Jackson) - @RequiresNoPermission - @AllowedDuringUpgrade - @AllowedBeforeInitialUserIsSet - public static class HealthCheckAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(HealthCheckForm form, BindException errors) throws Exception - { - if (!ModuleLoader.getInstance().isStartupComplete()) - return new ApiSimpleResponse("healthy", false); - - Collection categories = form.getCategories() == null ? Collections.singleton(HealthCheckRegistry.DEFAULT_CATEGORY) : Arrays.asList(form.getCategories().split(",")); - HealthCheck.Result checkResult = HealthCheckRegistry.get().checkHealth(categories); - - checkResult.getDetails().put("healthy", checkResult.isHealthy()); - - if (getUser().hasRootAdminPermission()) - { - return new ApiSimpleResponse(checkResult.getDetails()); - } - else - { - if (!checkResult.isHealthy()) - { - try (var writer = createResponseWriter()) - { - writer.writeResponse(HttpServletResponse.SC_SERVICE_UNAVAILABLE, "Server isn't ready yet"); - } - return null; - } - - return new ApiSimpleResponse("healthy", checkResult.isHealthy()); - } - } - } - - public static class HealthCheckForm - { - private String _categories; // if null, all categories will be checked. - - public String getCategories() - { - return _categories; - } - - @SuppressWarnings("unused") - public void setCategories(String categories) - { - _categories = categories; - } - } - - // No security checks... anyone (even guests) can view the credits page - @RequiresNoPermission - public class CreditsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - VBox views = new VBox(); - List modules = new ArrayList<>(ModuleLoader.getInstance().getModules()); - modules.sort(Comparator.comparing(Module::getName, String.CASE_INSENSITIVE_ORDER)); - - addCreditsViews(views, modules, "jars.txt", "JAR"); - addCreditsViews(views, modules, "scripts.txt", "Script, Icon and Font"); - addCreditsViews(views, modules, "source.txt", "Java Source Code"); - addCreditsViews(views, modules, "executables.txt", "Executable"); - - return views; - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Credits", this.getClass()); - } - } - - private void addCreditsViews(VBox views, List modules, String creditsFile, String fileType) throws IOException - { - for (Module module : modules) - { - String wikiSource = getCreditsFile(module, creditsFile); - - if (null != wikiSource) - { - String title = fileType + " Files Distributed with the " + module.getName() + " Module"; - CreditsView credits = new CreditsView(wikiSource, title); - views.addView(credits); - } - } - } - - private static class CreditsView extends WebPartView - { - private Renderable _html; - - CreditsView(@Nullable String wikiSource, String title) - { - super(title); - - wikiSource = StringUtils.trimToEmpty(wikiSource); - - if (StringUtils.isNotEmpty(wikiSource)) - { - WikiRenderingService wikiService = WikiRenderingService.get(); - HtmlString html = wikiService.getFormattedHtml(WikiRendererType.RADEOX, wikiSource, "Credits page"); - _html = DOM.createHtmlFragment(STYLE(at(type, "text/css"), "tr.table-odd td { background-color: #EEEEEE; }"), html); - } - } - - @Override - public void renderView(Object model, HtmlWriter out) - { - out.write(_html); - } - } - - private static String getCreditsFile(Module module, String filename) throws IOException - { - // credits files are in /resources/credits - InputStream is = module.getResourceStream("credits/" + filename); - - return null == is ? null : PageFlowUtil.getStreamContentsAsString(is); - } - - private void validateNetworkDrive(NetworkDriveForm form, Errors errors) - { - if (isBlank(form.getNetworkDriveUser()) || isBlank(form.getNetworkDrivePath()) || - isBlank(form.getNetworkDrivePassword()) || isBlank(form.getNetworkDriveLetter())) - { - errors.reject(ERROR_MSG, "All fields are required"); - } - else if (form.getNetworkDriveLetter().trim().length() > 1) - { - errors.reject(ERROR_MSG, "Network drive letter must be a single character"); - } - else - { - char letter = form.getNetworkDriveLetter().trim().toLowerCase().charAt(0); - - if (letter < 'a' || letter > 'z') - { - errors.reject(ERROR_MSG, "Network drive letter must be a letter"); - } - } - } - - public static class ResourceForm - { - private String _resource; - - public String getResource() - { - return _resource; - } - - public void setResource(String resource) - { - _resource = resource; - } - - public ResourceType getResourceType() - { - return ResourceType.valueOf(_resource); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ResetResourceAction extends FormHandlerAction - { - @Override - public void validateCommand(ResourceForm target, Errors errors) - { - } - - @Override - public boolean handlePost(ResourceForm form, BindException errors) throws Exception - { - form.getResourceType().delete(getContainer(), getUser()); - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - return true; - } - - @Override - public URLHelper getSuccessURL(ResourceForm form) - { - return new AdminUrlsImpl().getLookAndFeelResourcesURL(getContainer()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ResetPropertiesAction extends FormHandlerAction - { - private URLHelper _returnUrl; - - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - Container c = getContainer(); - boolean folder = !(c.isRoot() || c.isProject()); - boolean hasAdminOpsPerm = c.hasPermission(getUser(), AdminOperationsPermission.class); - - WriteableFolderLookAndFeelProperties props = folder ? LookAndFeelProperties.getWriteableFolderInstance(c) : LookAndFeelProperties.getWriteableInstance(c); - props.clear(hasAdminOpsPerm); - props.save(); - // TODO: Audit log? - - AdminUrls urls = new AdminUrlsImpl(); - - // Folder-level settings are just display formats and measure/dimension flags -- no need to increment L&F revision - if (!folder) - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - - _returnUrl = urls.getLookAndFeelSettingsURL(c); - - return true; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return _returnUrl; - } - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public class CustomizeSiteAction extends FormViewAction - { - @Override - public ModelAndView getView(SiteSettingsForm form, boolean reshow, BindException errors) - { - if (form.isUpgradeInProgress()) - getPageConfig().setTemplate(Template.Dialog); - - SiteSettingsBean bean = new SiteSettingsBean(form.isUpgradeInProgress()); - setHelpTopic("configAdmin"); - return new JspView<>("/org/labkey/core/admin/customizeSite.jsp", bean, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Customize Site", this.getClass()); - } - - @Override - public void validateCommand(SiteSettingsForm form, Errors errors) - { - if (form.isShowRibbonMessage() && StringUtils.isEmpty(form.getRibbonMessage())) - { - errors.reject(ERROR_MSG, "Cannot enable the ribbon message without providing a message to show"); - } - if (form.getMaxBLOBSize() < 0) - { - errors.reject(ERROR_MSG, "Maximum BLOB size cannot be negative"); - } - int hardCap = Math.max(WriteableAppProps.SOFT_MAX_BLOB_SIZE, AppProps.getInstance().getMaxBLOBSize()); - if (form.getMaxBLOBSize() > hardCap) - { - errors.reject(ERROR_MSG, "Maximum BLOB size cannot be set higher than " + hardCap + " bytes"); - } - if (form.getSslPort() < 1 || form.getSslPort() > 65535) - { - errors.reject(ERROR_MSG, "HTTPS port must be between 1 and 65,535"); - } - if (form.getReadOnlyHttpRequestTimeout() < 0) - { - errors.reject(ERROR_MSG, "HTTP timeout must be non-negative"); - } - if (form.getMemoryUsageDumpInterval() < 0) - { - errors.reject(ERROR_MSG, "Memory logging frequency must be non-negative"); - } - } - - @Override - public boolean handlePost(SiteSettingsForm form, BindException errors) throws Exception - { - HttpServletRequest request = getViewContext().getRequest(); - - // We only need to check that SSL is running if the user isn't already using SSL - if (form.isSslRequired() && !(request.isSecure() && (form.getSslPort() == request.getServerPort()))) - { - URL testURL = new URL("https", request.getServerName(), form.getSslPort(), AppProps.getInstance().getContextPath()); - Pair sslResponse = HttpsUtil.testHttpsUrl(testURL, "Ensure that the web server is configured for SSL and the port is correct. If SSL is enabled, try saving these settings while connected via SSL."); - - if (sslResponse != null) - { - errors.reject(ERROR_MSG, sslResponse.first); - return false; - } - } - - if (form.getReadOnlyHttpRequestTimeout() < 0) - { - errors.reject(ERROR_MSG, "Read only HTTP request timeout must be non-negative"); - } - - WriteableAppProps props = AppProps.getWriteableInstance(); - - props.setPipelineToolsDir(form.getPipelineToolsDirectory()); - props.setNavAccessOpen(form.isNavAccessOpen()); - props.setSSLRequired(form.isSslRequired()); - boolean sslSettingChanged = AppProps.getInstance().isSSLRequired() != form.isSslRequired(); - props.setSSLPort(form.getSslPort()); - props.setMemoryUsageDumpInterval(form.getMemoryUsageDumpInterval()); - props.setReadOnlyHttpRequestTimeout(form.getReadOnlyHttpRequestTimeout()); - props.setMaxBLOBSize(form.getMaxBLOBSize()); - props.setExt3Required(form.isExt3Required()); - props.setExt3APIRequired(form.isExt3APIRequired()); - props.setSelfReportExceptions(form.isSelfReportExceptions()); - - props.setAdminOnlyMessage(form.getAdminOnlyMessage()); - props.setShowRibbonMessage(form.isShowRibbonMessage()); - props.setRibbonMessage(form.getRibbonMessage()); - props.setUserRequestedAdminOnlyMode(form.isAdminOnlyMode()); - - props.setAllowApiKeys(form.isAllowApiKeys()); - props.setApiKeyExpirationSeconds(form.getApiKeyExpirationSeconds()); - props.setAllowSessionKeys(form.isAllowSessionKeys()); - - try - { - ExceptionReportingLevel level = ExceptionReportingLevel.valueOf(form.getExceptionReportingLevel()); - props.setExceptionReportingLevel(level); - } - catch (IllegalArgumentException ignored) - { - } - - try - { - if (form.getUsageReportingLevel() != null) - { - UsageReportingLevel level = UsageReportingLevel.valueOf(form.getUsageReportingLevel()); - props.setUsageReportingLevel(level); - } - } - catch (IllegalArgumentException ignored) - { - } - - props.setAdministratorContactEmail(form.getAdministratorContactEmail() == null ? null : form.getAdministratorContactEmail().trim()); - - if (null != form.getBaseServerURL()) - { - if (form.isSslRequired() && !form.getBaseServerURL().startsWith("https")) - { - errors.reject(ERROR_MSG, "Invalid Base Server URL. SSL connection is required. Consider https://."); - return false; - } - - try - { - props.setBaseServerUrl(form.getBaseServerURL()); - } - catch (URISyntaxException e) - { - errors.reject(ERROR_MSG, "Invalid Base Server URL, \"" + e.getMessage() + "\"." + - "Please enter a valid base URL containing the protocol, hostname, and port if required. " + - "The webapp context path should not be included. " + - "For example: \"https://www.example.com\" or \"http://www.labkey.org:8080\" and not \"http://www.example.com/labkey/\""); - return false; - } - } - - String frameOption = StringUtils.trimToEmpty(form.getXFrameOption()); - if (!frameOption.equals("DENY") && !frameOption.equals("SAMEORIGIN") && !frameOption.equals("ALLOW")) - { - errors.reject(ERROR_MSG, "XFrameOption must equal DENY, or SAMEORIGIN, or ALLOW"); - return false; - } - props.setXFrameOption(frameOption); - props.setIncludeServerHttpHeader(form.isIncludeServerHttpHeader()); - - props.save(getViewContext().getUser()); - UsageReportingLevel.reportNow(); - if (sslSettingChanged) - ContentSecurityPolicyFilter.regenerateSubstitutionMap(); - - return true; - } - - @Override - public ActionURL getSuccessURL(SiteSettingsForm form) - { - if (form.isUpgradeInProgress()) - { - return AppProps.getInstance().getHomePageActionURL(); - } - else - { - return new AdminUrlsImpl().getAdminConsoleURL(); - } - } - } - - public static class NetworkDriveForm - { - private String _networkDriveLetter; - private String _networkDrivePath; - private String _networkDriveUser; - private String _networkDrivePassword; - - public String getNetworkDriveLetter() - { - return _networkDriveLetter; - } - - public void setNetworkDriveLetter(String networkDriveLetter) - { - _networkDriveLetter = networkDriveLetter; - } - - public String getNetworkDrivePassword() - { - return _networkDrivePassword; - } - - public void setNetworkDrivePassword(String networkDrivePassword) - { - _networkDrivePassword = networkDrivePassword; - } - - public String getNetworkDrivePath() - { - return _networkDrivePath; - } - - public void setNetworkDrivePath(String networkDrivePath) - { - _networkDrivePath = networkDrivePath; - } - - public String getNetworkDriveUser() - { - return _networkDriveUser; - } - - public void setNetworkDriveUser(String networkDriveUser) - { - _networkDriveUser = networkDriveUser; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - @AdminConsoleAction - public class MapNetworkDriveAction extends FormViewAction - { - @Override - public void validateCommand(NetworkDriveForm form, Errors errors) - { - validateNetworkDrive(form, errors); - } - - @Override - public ModelAndView getView(NetworkDriveForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/mapNetworkDrive.jsp", null, errors); - } - - @Override - public boolean handlePost(NetworkDriveForm form, BindException errors) throws Exception - { - NetworkDriveProps.setNetworkDriveLetter(form.getNetworkDriveLetter().trim()); - NetworkDriveProps.setNetworkDrivePath(form.getNetworkDrivePath().trim()); - NetworkDriveProps.setNetworkDriveUser(form.getNetworkDriveUser().trim()); - NetworkDriveProps.setNetworkDrivePassword(form.getNetworkDrivePassword().trim()); - - return true; - } - - @Override - public URLHelper getSuccessURL(NetworkDriveForm siteSettingsForm) - { - return new ActionURL(FilesSiteSettingsAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("setRoots#map"); - addAdminNavTrail(root, "Map Network Drive", this.getClass()); - } - } - - public static class SiteSettingsBean - { - public final boolean _upgradeInProgress; - public final boolean _showSelfReportExceptions; - - private SiteSettingsBean(boolean upgradeInProgress) - { - _upgradeInProgress = upgradeInProgress; - _showSelfReportExceptions = MothershipReport.isShowSelfReportExceptions(); - } - - public HtmlString getSiteSettingsHelpLink(String fragment) - { - return new HelpTopic("configAdmin", fragment).getSimpleLinkHtml("more info..."); - } - } - - public static class SetRibbonMessageForm - { - private Boolean _show = null; - private String _message = null; - - public Boolean isShow() - { - return _show; - } - - public void setShow(Boolean show) - { - _show = show; - } - - public String getMessage() - { - return _message; - } - - public void setMessage(String message) - { - _message = message; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class SetRibbonMessageAction extends MutatingApiAction - { - @Override - public Object execute(SetRibbonMessageForm form, BindException errors) throws Exception - { - if (form.isShow() != null || form.getMessage() != null) - { - WriteableAppProps props = AppProps.getWriteableInstance(); - - if (form.isShow() != null) - props.setShowRibbonMessage(form.isShow()); - - if (form.getMessage() != null) - props.setRibbonMessage(form.getMessage()); - - props.save(getViewContext().getUser()); - } - - return null; - } - } - - @RequiresPermission(AdminPermission.class) - public class ConfigureSiteValidationAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - return new JspView<>("/org/labkey/core/admin/sitevalidation/configureSiteValidation.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("siteValidation"); - addAdminNavTrail(root, "Configure " + (getContainer().isRoot() ? "Site" : "Folder") + " Validation", getClass()); - } - } - - public static class SiteValidationForm - { - private List _providers; - private boolean _includeSubfolders = false; - private transient Consumer _logger = s -> { - }; // No-op by default - - public List getProviders() - { - return _providers; - } - - public void setProviders(List providers) - { - _providers = providers; - } - - public boolean isIncludeSubfolders() - { - return _includeSubfolders; - } - - public void setIncludeSubfolders(boolean includeSubfolders) - { - _includeSubfolders = includeSubfolders; - } - - public Consumer getLogger() - { - return _logger; - } - - public void setLogger(Consumer logger) - { - _logger = logger; - } - } - - @RequiresPermission(AdminPermission.class) - public class SiteValidationAction extends SimpleViewAction - { - @Override - public ModelAndView getView(SiteValidationForm form, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/sitevalidation/siteValidation.jsp", form); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("siteValidation"); - addAdminNavTrail(root, (getContainer().isRoot() ? "Site" : "Folder") + " Validation", getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class SiteValidationBackgroundAction extends FormHandlerAction - { - private ActionURL _redirectUrl; - - @Override - public void validateCommand(SiteValidationForm form, Errors errors) - { - } - - @Override - public boolean handlePost(SiteValidationForm form, BindException errors) throws PipelineValidationException - { - ViewBackgroundInfo vbi = new ViewBackgroundInfo(getContainer(), getUser(), null); - PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); - SiteValidationJob job = new SiteValidationJob(vbi, root, form); - PipelineService.get().queueJob(job); - String jobGuid = job.getJobGUID(); - - if (null == jobGuid) - throw new NotFoundException("Unable to determine pipeline job GUID"); - - Long jobId = PipelineService.get().getJobId(getUser(), getContainer(), jobGuid); - - if (null == jobId) - throw new NotFoundException("Unable to determine pipeline job ID"); - - PipelineStatusUrls urls = urlProvider(PipelineStatusUrls.class); - _redirectUrl = urls.urlDetails(getContainer(), jobId); - - return true; - } - - @Override - public URLHelper getSuccessURL(SiteValidationForm form) - { - return _redirectUrl; - } - } - - public static class ViewValidationResultsForm - { - private int _rowId; - - public int getRowId() - { - return _rowId; - } - - public void setRowId(int rowId) - { - _rowId = rowId; - } - } - - @RequiresPermission(AdminPermission.class) - public class ViewValidationResultsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ViewValidationResultsForm form, BindException errors) throws Exception - { - PipelineStatusFile statusFile = PipelineService.get().getStatusFile(form.getRowId()); - if (null == statusFile) - throw new NotFoundException("Status file not found"); - if (!getContainer().equals(statusFile.lookupContainer())) - throw new UnauthorizedException("Wrong container"); - - String logFilePath = statusFile.getFilePath(); - String htmlFilePath = FileUtil.getBaseName(logFilePath) + ".html"; - File htmlFile = new File(htmlFilePath); - - if (!htmlFile.exists()) - throw new NotFoundException("Results file not found"); - return new HtmlView(HtmlString.unsafe(PageFlowUtil.getFileContentsAsString(htmlFile))); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("siteValidation"); - addAdminNavTrail(root, "View " + (getContainer().isRoot() ? "Site" : "Folder") + " Validation Results", getClass()); - } - } - - public interface FileManagementForm - { - String getFolderRootPath(); - - void setFolderRootPath(String folderRootPath); - - String getFileRootOption(); - - void setFileRootOption(String fileRootOption); - - String getConfirmMessage(); - - void setConfirmMessage(String confirmMessage); - - boolean isDisableFileSharing(); - - boolean hasSiteDefaultRoot(); - - String[] getEnabledCloudStore(); - - @SuppressWarnings("unused") - void setEnabledCloudStore(String[] enabledCloudStore); - - boolean isCloudFileRoot(); - - @Nullable - String getCloudRootName(); - - void setCloudRootName(String cloudRootName); - - void setFileRootChanged(boolean changed); - - void setEnabledCloudStoresChanged(boolean changed); - - String getMigrateFilesOption(); - - void setMigrateFilesOption(String migrateFilesOption); - - default boolean isFolderSetup() - { - return false; - } - } - - public enum MigrateFilesOption implements SafeToRenderEnum - { - leave - { - @Override - public String description() - { - return "Source files not copied or moved"; - } - }, - copy - { - @Override - public String description() - { - return "Copy source files to destination"; - } - }, - move - { - @Override - public String description() - { - return "Move source files to destination"; - } - }; - - public abstract String description(); - } - - public static class ProjectSettingsForm extends FolderSettingsForm - { - // Site-only properties - private String _dateParsingMode; - private String _customWelcome; - - // Site & project properties - private boolean _shouldInherit; // new subfolders should inherit parent permissions - private String _systemDescription; - private boolean _systemDescriptionInherited; - private String _systemShortName; - private boolean _systemShortNameInherited; - private String _themeName; - private boolean _themeNameInherited; - private String _folderDisplayMode; - private boolean _folderDisplayModeInherited; - private String _applicationMenuDisplayMode; - private boolean _applicationMenuDisplayModeInherited; - private boolean _helpMenuEnabled; - private boolean _helpMenuEnabledInherited; - private boolean _discussionEnabled; - private boolean _discussionEnabledInherited; - private String _logoHref; - private boolean _logoHrefInherited; - private String _companyName; - private boolean _companyNameInherited; - private String _systemEmailAddress; - private boolean _systemEmailAddressInherited; - private String _reportAProblemPath; - private boolean _reportAProblemPathInherited; - private String _supportEmail; - private boolean _supportEmailInherited; - private String _customLogin; - private boolean _customLoginInherited; - - // Site-only properties - - public String getDateParsingMode() - { - return _dateParsingMode; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setDateParsingMode(String dateParsingMode) - { - _dateParsingMode = dateParsingMode; - } - - public String getCustomWelcome() - { - return _customWelcome; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCustomWelcome(String customWelcome) - { - _customWelcome = customWelcome; - } - - // Site & project properties - - public boolean getShouldInherit() - { - return _shouldInherit; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setShouldInherit(boolean b) - { - _shouldInherit = b; - } - - public String getSystemDescription() - { - return _systemDescription; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemDescription(String systemDescription) - { - _systemDescription = systemDescription; - } - - public boolean isSystemDescriptionInherited() - { - return _systemDescriptionInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemDescriptionInherited(boolean systemDescriptionInherited) - { - _systemDescriptionInherited = systemDescriptionInherited; - } - - public String getSystemShortName() - { - return _systemShortName; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemShortName(String systemShortName) - { - _systemShortName = systemShortName; - } - - public boolean isSystemShortNameInherited() - { - return _systemShortNameInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemShortNameInherited(boolean systemShortNameInherited) - { - _systemShortNameInherited = systemShortNameInherited; - } - - public String getThemeName() - { - return _themeName; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setThemeName(String themeName) - { - _themeName = themeName; - } - - public boolean isThemeNameInherited() - { - return _themeNameInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setThemeNameInherited(boolean themeNameInherited) - { - _themeNameInherited = themeNameInherited; - } - - public String getFolderDisplayMode() - { - return _folderDisplayMode; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setFolderDisplayMode(String folderDisplayMode) - { - _folderDisplayMode = folderDisplayMode; - } - - public boolean isFolderDisplayModeInherited() - { - return _folderDisplayModeInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setFolderDisplayModeInherited(boolean folderDisplayModeInherited) - { - _folderDisplayModeInherited = folderDisplayModeInherited; - } - - public String getApplicationMenuDisplayMode() - { - return _applicationMenuDisplayMode; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setApplicationMenuDisplayMode(String displayMode) - { - _applicationMenuDisplayMode = displayMode; - } - - public boolean isApplicationMenuDisplayModeInherited() - { - return _applicationMenuDisplayModeInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setApplicationMenuDisplayModeInherited(boolean applicationMenuDisplayModeInherited) - { - _applicationMenuDisplayModeInherited = applicationMenuDisplayModeInherited; - } - - public boolean isHelpMenuEnabled() - { - return _helpMenuEnabled; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setHelpMenuEnabled(boolean helpMenuEnabled) - { - _helpMenuEnabled = helpMenuEnabled; - } - - public boolean isHelpMenuEnabledInherited() - { - return _helpMenuEnabledInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setHelpMenuEnabledInherited(boolean helpMenuEnabledInherited) - { - _helpMenuEnabledInherited = helpMenuEnabledInherited; - } - - public boolean isDiscussionEnabled() - { - return _discussionEnabled; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setDiscussionEnabled(boolean discussionEnabled) - { - _discussionEnabled = discussionEnabled; - } - - public boolean isDiscussionEnabledInherited() - { - return _discussionEnabledInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setDiscussionEnabledInherited(boolean discussionEnabledInherited) - { - _discussionEnabledInherited = discussionEnabledInherited; - } - - public String getLogoHref() - { - return _logoHref; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setLogoHref(String logoHref) - { - _logoHref = logoHref; - } - - public boolean isLogoHrefInherited() - { - return _logoHrefInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setLogoHrefInherited(boolean logoHrefInherited) - { - _logoHrefInherited = logoHrefInherited; - } - - public String getReportAProblemPath() - { - return _reportAProblemPath; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setReportAProblemPath(String reportAProblemPath) - { - _reportAProblemPath = reportAProblemPath; - } - - public boolean isReportAProblemPathInherited() - { - return _reportAProblemPathInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setReportAProblemPathInherited(boolean reportAProblemPathInherited) - { - _reportAProblemPathInherited = reportAProblemPathInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSupportEmail(String supportEmail) - { - _supportEmail = supportEmail; - } - - public String getSupportEmail() - { - return _supportEmail; - } - - public boolean isSupportEmailInherited() - { - return _supportEmailInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSupportEmailInherited(boolean supportEmailInherited) - { - _supportEmailInherited = supportEmailInherited; - } - - public String getSystemEmailAddress() - { - return _systemEmailAddress; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemEmailAddress(String systemEmailAddress) - { - _systemEmailAddress = systemEmailAddress; - } - - public boolean isSystemEmailAddressInherited() - { - return _systemEmailAddressInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemEmailAddressInherited(boolean systemEmailAddressInherited) - { - _systemEmailAddressInherited = systemEmailAddressInherited; - } - - public String getCompanyName() - { - return _companyName; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCompanyName(String companyName) - { - _companyName = companyName; - } - - public boolean isCompanyNameInherited() - { - return _companyNameInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCompanyNameInherited(boolean companyNameInherited) - { - _companyNameInherited = companyNameInherited; - } - - public String getCustomLogin() - { - return _customLogin; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCustomLogin(String customLogin) - { - _customLogin = customLogin; - } - - public boolean isCustomLoginInherited() - { - return _customLoginInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCustomLoginInherited(boolean customLoginInherited) - { - _customLoginInherited = customLoginInherited; - } - } - - public enum FileRootProp implements SafeToRenderEnum - { - disable, - siteDefault, - folderOverride, - cloudRoot - } - - public static class FilesForm extends SetupForm implements FileManagementForm - { - private boolean _fileRootChanged; - private boolean _enabledCloudStoresChanged; - private String _cloudRootName; - private String _migrateFilesOption; - private String[] _enabledCloudStore; - private String _fileRootOption; - private String _folderRootPath; - - public boolean isFileRootChanged() - { - return _fileRootChanged; - } - - @Override - public void setFileRootChanged(boolean changed) - { - _fileRootChanged = changed; - } - - public boolean isEnabledCloudStoresChanged() - { - return _enabledCloudStoresChanged; - } - - @Override - public void setEnabledCloudStoresChanged(boolean enabledCloudStoresChanged) - { - _enabledCloudStoresChanged = enabledCloudStoresChanged; - } - - @Override - public boolean isDisableFileSharing() - { - return FileRootProp.disable.name().equals(getFileRootOption()); - } - - @Override - public boolean hasSiteDefaultRoot() - { - return FileRootProp.siteDefault.name().equals(getFileRootOption()); - } - - @Override - public String[] getEnabledCloudStore() - { - return _enabledCloudStore; - } - - @Override - public void setEnabledCloudStore(String[] enabledCloudStore) - { - _enabledCloudStore = enabledCloudStore; - } - - @Override - public boolean isCloudFileRoot() - { - return FileRootProp.cloudRoot.name().equals(getFileRootOption()); - } - - @Override - @Nullable - public String getCloudRootName() - { - return _cloudRootName; - } - - @Override - public void setCloudRootName(String cloudRootName) - { - _cloudRootName = cloudRootName; - } - - @Override - public String getMigrateFilesOption() - { - return _migrateFilesOption; - } - - @Override - public void setMigrateFilesOption(String migrateFilesOption) - { - _migrateFilesOption = migrateFilesOption; - } - - @Override - public String getFolderRootPath() - { - return _folderRootPath; - } - - @Override - public void setFolderRootPath(String folderRootPath) - { - _folderRootPath = folderRootPath; - } - - @Override - public String getFileRootOption() - { - return _fileRootOption; - } - - @Override - public void setFileRootOption(String fileRootOption) - { - _fileRootOption = fileRootOption; - } - } - - @SuppressWarnings("unused") - public static class SiteSettingsForm - { - private boolean _upgradeInProgress = false; - - private String _pipelineToolsDirectory; - private boolean _sslRequired; - private boolean _adminOnlyMode; - private boolean _showRibbonMessage; - private boolean _ext3Required; - private boolean _ext3APIRequired; - private boolean _selfReportExceptions; - private String _adminOnlyMessage; - private String _ribbonMessage; - private int _sslPort; - private int _memoryUsageDumpInterval; - private int _readOnlyHttpRequestTimeout; - private int _maxBLOBSize; - private String _exceptionReportingLevel; - private String _usageReportingLevel; - private String _administratorContactEmail; - - private String _baseServerURL; - private String _callbackPassword; - private boolean _allowApiKeys; - private int _apiKeyExpirationSeconds; - private boolean _allowSessionKeys; - private boolean _navAccessOpen; - - private String _XFrameOption; - private boolean _includeServerHttpHeader; - - public String getPipelineToolsDirectory() - { - return _pipelineToolsDirectory; - } - - public void setPipelineToolsDirectory(String pipelineToolsDirectory) - { - _pipelineToolsDirectory = pipelineToolsDirectory; - } - - public boolean isNavAccessOpen() - { - return _navAccessOpen; - } - - public void setNavAccessOpen(boolean navAccessOpen) - { - _navAccessOpen = navAccessOpen; - } - - public boolean isSslRequired() - { - return _sslRequired; - } - - public void setSslRequired(boolean sslRequired) - { - _sslRequired = sslRequired; - } - - public boolean isExt3Required() - { - return _ext3Required; - } - - public void setExt3Required(boolean ext3Required) - { - _ext3Required = ext3Required; - } - - public boolean isExt3APIRequired() - { - return _ext3APIRequired; - } - - public void setExt3APIRequired(boolean ext3APIRequired) - { - _ext3APIRequired = ext3APIRequired; - } - - public int getSslPort() - { - return _sslPort; - } - - public void setSslPort(int sslPort) - { - _sslPort = sslPort; - } - - public boolean isAdminOnlyMode() - { - return _adminOnlyMode; - } - - public void setAdminOnlyMode(boolean adminOnlyMode) - { - _adminOnlyMode = adminOnlyMode; - } - - public String getAdminOnlyMessage() - { - return _adminOnlyMessage; - } - - public void setAdminOnlyMessage(String adminOnlyMessage) - { - _adminOnlyMessage = adminOnlyMessage; - } - - public boolean isSelfReportExceptions() - { - return _selfReportExceptions; - } - - public void setSelfReportExceptions(boolean selfReportExceptions) - { - _selfReportExceptions = selfReportExceptions; - } - - public String getExceptionReportingLevel() - { - return _exceptionReportingLevel; - } - - public void setExceptionReportingLevel(String exceptionReportingLevel) - { - _exceptionReportingLevel = exceptionReportingLevel; - } - - public String getUsageReportingLevel() - { - return _usageReportingLevel; - } - - public void setUsageReportingLevel(String usageReportingLevel) - { - _usageReportingLevel = usageReportingLevel; - } - - public String getAdministratorContactEmail() - { - return _administratorContactEmail; - } - - public void setAdministratorContactEmail(String administratorContactEmail) - { - _administratorContactEmail = administratorContactEmail; - } - - public boolean isUpgradeInProgress() - { - return _upgradeInProgress; - } - - public void setUpgradeInProgress(boolean upgradeInProgress) - { - _upgradeInProgress = upgradeInProgress; - } - - public int getMemoryUsageDumpInterval() - { - return _memoryUsageDumpInterval; - } - - public void setMemoryUsageDumpInterval(int memoryUsageDumpInterval) - { - _memoryUsageDumpInterval = memoryUsageDumpInterval; - } - - public int getReadOnlyHttpRequestTimeout() - { - return _readOnlyHttpRequestTimeout; - } - - public void setReadOnlyHttpRequestTimeout(int timeout) - { - _readOnlyHttpRequestTimeout = timeout; - } - - public int getMaxBLOBSize() - { - return _maxBLOBSize; - } - - public void setMaxBLOBSize(int maxBLOBSize) - { - _maxBLOBSize = maxBLOBSize; - } - - public String getBaseServerURL() - { - return _baseServerURL; - } - - public void setBaseServerURL(String baseServerURL) - { - _baseServerURL = baseServerURL; - } - - public String getCallbackPassword() - { - return _callbackPassword; - } - - public void setCallbackPassword(String callbackPassword) - { - _callbackPassword = callbackPassword; - } - - public boolean isShowRibbonMessage() - { - return _showRibbonMessage; - } - - public void setShowRibbonMessage(boolean showRibbonMessage) - { - _showRibbonMessage = showRibbonMessage; - } - - public String getRibbonMessage() - { - return _ribbonMessage; - } - - public void setRibbonMessage(String ribbonMessage) - { - _ribbonMessage = ribbonMessage; - } - - public boolean isAllowApiKeys() - { - return _allowApiKeys; - } - - public void setAllowApiKeys(boolean allowApiKeys) - { - _allowApiKeys = allowApiKeys; - } - - public int getApiKeyExpirationSeconds() - { - return _apiKeyExpirationSeconds; - } - - public void setApiKeyExpirationSeconds(int apiKeyExpirationSeconds) - { - _apiKeyExpirationSeconds = apiKeyExpirationSeconds; - } - - public boolean isAllowSessionKeys() - { - return _allowSessionKeys; - } - - public void setAllowSessionKeys(boolean allowSessionKeys) - { - _allowSessionKeys = allowSessionKeys; - } - - public String getXFrameOption() - { - return _XFrameOption; - } - - public void setXFrameOption(String XFrameOption) - { - _XFrameOption = XFrameOption; - } - - public boolean isIncludeServerHttpHeader() - { - return _includeServerHttpHeader; - } - - public void setIncludeServerHttpHeader(boolean includeServerHttpHeader) - { - _includeServerHttpHeader = includeServerHttpHeader; - } - } - - - @AdminConsoleAction - public class ShowThreadsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - // Log to labkey.log as well as showing through the browser - DebugInfoDumper.dumpThreads(3); - return new JspView<>("/org/labkey/core/admin/threads.jsp", new ThreadsBean()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("dumpDebugging#threads"); - addAdminNavTrail(root, "Current Threads", this.getClass()); - } - } - - private abstract class AbstractPostgresAction extends QueryViewAction - { - private final String _queryName; - - protected AbstractPostgresAction(String queryName) - { - super(QueryExportForm.class); - _queryName = queryName; - } - - @Override - protected QueryView createQueryView(QueryExportForm form, BindException errors, boolean forExport, @Nullable String dataRegion) throws Exception - { - if (!CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) - { - throw new NotFoundException("Only available with Postgres as the primary database"); - } - - QuerySettings qSettings = new QuerySettings(getViewContext(), "query", _queryName); - QueryView result = new QueryView(new PostgresUserSchema(getUser(), getContainer()), qSettings, errors) - { - @Override - public DataView createDataView() - { - // Troubleshooters don't have normal read access to the root container so grant them special access - // for these queries - DataView view = super.createDataView(); - view.getRenderContext().getViewContext().addContextualRole(ReaderRole.class); - return view; - } - }; - result.setTitle(_queryName); - result.setFrame(WebPartView.FrameType.PORTAL); - return result; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("postgresActivity"); - addAdminNavTrail(root, "Postgres " + _queryName, this.getClass()); - } - - } - - @AdminConsoleAction - public class PostgresStatActivityAction extends AbstractPostgresAction - { - public PostgresStatActivityAction() - { - super(PostgresUserSchema.POSTGRES_STAT_ACTIVITY_TABLE_NAME); - } - } - - @AdminConsoleAction - public class PostgresLocksAction extends AbstractPostgresAction - { - public PostgresLocksAction() - { - super(PostgresUserSchema.POSTGRES_LOCKS_TABLE_NAME); - } - } - - @AdminConsoleAction - public class DumpHeapAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - File destination = DebugInfoDumper.dumpHeap(); - return new HtmlView(HtmlString.of("Heap dumped to " + destination.getAbsolutePath())); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("dumpHeap"); - addAdminNavTrail(root, "Heap dump", getClass()); - } - } - - - public static class ThreadsBean - { - public Map> spids; - public List threads; - public Map stackTraces; - - ThreadsBean() - { - stackTraces = Thread.getAllStackTraces(); - threads = new ArrayList<>(stackTraces.keySet()); - threads.sort(Comparator.comparing(Thread::getName, String.CASE_INSENSITIVE_ORDER)); - - spids = new HashMap<>(); - - for (Thread t : threads) - { - spids.put(t, ConnectionWrapper.getSPIDsForThread(t)); - } - } - } - - - @RequiresPermission(AdminOperationsPermission.class) - public class ShowNetworkDriveTestAction extends SimpleViewAction - { - @Override - public void validate(NetworkDriveForm form, BindException errors) - { - validateNetworkDrive(form, errors); - } - - @Override - public ModelAndView getView(NetworkDriveForm form, BindException errors) - { - NetworkDrive testDrive = new NetworkDrive(); - testDrive.setPassword(form.getNetworkDrivePassword()); - testDrive.setPath(form.getNetworkDrivePath()); - testDrive.setUser(form.getNetworkDriveUser()); - TestNetworkDriveBean bean = new TestNetworkDriveBean(); - - if (!errors.hasErrors()) - { - char driveLetter = form.getNetworkDriveLetter().trim().charAt(0); - try - { - String mountError = testDrive.mount(driveLetter); - if (mountError != null) - { - errors.reject(ERROR_MSG, mountError); - } - else - { - File f = new File(driveLetter + ":\\"); - if (!f.exists()) - { - errors.reject(ERROR_MSG, "Could not access network drive"); - } - else - { - String[] fileNames = f.list(); - if (fileNames == null) - fileNames = new String[0]; - Arrays.sort(fileNames); - bean.setFiles(fileNames); - } - } - } - catch (IOException | InterruptedException e) - { - errors.reject(ERROR_MSG, "Error mounting drive: " + e); - } - try - { - testDrive.unmount(driveLetter); - } - catch (IOException | InterruptedException e) - { - errors.reject(ERROR_MSG, "Error mounting drive: " + e); - } - } - - getPageConfig().setTemplate(Template.Dialog); - return new JspView<>("/org/labkey/core/admin/testNetworkDrive.jsp", bean, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Test Mapping Network Drive"); - } - } - - - @AdminConsoleAction(ApplicationAdminPermission.class) - public class ResetErrorMarkAction extends ConfirmAction - { - @Override - public ModelAndView getConfirmView(Object o, BindException errors) - { - return HtmlView.of("Are you sure you want to reset the site errors?"); - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - File errorLogFile = getErrorLogFile(); - _errorMark = errorLogFile.length(); - - return true; - } - - @Override - public void validateCommand(Object o, Errors errors) - { - } - - @Override - public @NotNull URLHelper getSuccessURL(Object o) - { - return getShowAdminURL(); - } - } - - abstract public static class ShowLogAction extends ExportAction - { - @Override - public final void export(Object o, HttpServletResponse response, BindException errors) throws IOException - { - getPageConfig().setNoIndex(); - export(response); - } - - protected abstract void export(HttpServletResponse response) throws IOException; - } - - @AdminConsoleAction - public class ShowErrorsSinceMarkAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, _errorMark, getErrorLogFile()); - } - } - - @AdminConsoleAction - public class ShowAllErrorsAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, 0, getErrorLogFile()); - } - } - - @AdminConsoleAction(ApplicationAdminPermission.class) - public class ResetPrimaryLogMarkAction extends MutatingApiAction - { - @Override - public Object execute(Object o, BindException errors) throws Exception - { - File logFile = getPrimaryLogFile(); - _primaryLogMark = logFile.length(); - return null; - } - } - - @AdminConsoleAction - public class ShowPrimaryLogSinceMarkAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, _primaryLogMark, getPrimaryLogFile()); - } - } - - @AdminConsoleAction - public class ShowPrimaryLogAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, 0, getPrimaryLogFile()); - } - } - - @AdminConsoleAction - public class ShowCspReportLogAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, 0, getCspReportLogFile()); - } - } - - private File getErrorLogFile() - { - return new File(getLabKeyLogDir(), "labkey-errors.log"); - } - - private File getPrimaryLogFile() - { - return new File(getLabKeyLogDir(), "labkey.log"); - } - - private File getCspReportLogFile() - { - return new File(getLabKeyLogDir(), "csp-report.log"); - } - - private static ActionURL getActionsURL() - { - return new ActionURL(ActionsAction.class, ContainerManager.getRoot()); - } - - - @AdminConsoleAction - public class ActionsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new ActionsTabStrip(); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("actionsDiagnostics"); - addAdminNavTrail(root, "Actions", this.getClass()); - } - } - - private static class ActionsTabStrip extends TabStripView - { - @Override - public List getTabList() - { - List tabs = new ArrayList<>(3); - - tabs.add(new TabInfo("Summary", "summary", getActionsURL())); - tabs.add(new TabInfo("Details", "details", getActionsURL())); - tabs.add(new TabInfo("Exceptions", "exceptions", getActionsURL())); - - return tabs; - } - - @Override - public HttpView getTabView(String tabId) - { - if ("exceptions".equals(tabId)) - return new ActionsExceptionsView(); - return new ActionsView(!"details".equals(tabId)); - } - } - - @AdminConsoleAction - public static class ExportActionsAction extends ExportAction - { - @Override - public void export(Object form, HttpServletResponse response, BindException errors) throws Exception - { - try (ActionsTsvWriter writer = new ActionsTsvWriter()) - { - writer.write(response); - } - } - } - - private static ActionURL getQueriesURL(@Nullable String statName) - { - ActionURL url = new ActionURL(QueriesAction.class, ContainerManager.getRoot()); - - if (null != statName) - url.addParameter("stat", statName); - - return url; - } - - - @AdminConsoleAction - public class QueriesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(QueriesForm form, BindException errors) - { - String buttonHTML = ""; - if (getUser().hasRootAdminPermission()) - buttonHTML += PageFlowUtil.button("Reset All Statistics").href(getResetQueryStatisticsURL()).usePost() + " "; - buttonHTML += PageFlowUtil.button("Export").href(getExportQueriesURL()) + "

"; - - return QueryProfiler.getInstance().getReportView(form.getStat(), buttonHTML, AdminController::getQueriesURL, - AdminController::getQueryStackTracesURL); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("queryPerf"); - addAdminNavTrail(root, "Queries", this.getClass()); - } - } - - public static class QueriesForm - { - private String _stat = "Count"; - - public String getStat() - { - return _stat; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setStat(String stat) - { - _stat = stat; - } - } - - - private static ActionURL getQueryStackTracesURL(String sqlHash) - { - ActionURL url = new ActionURL(QueryStackTracesAction.class, ContainerManager.getRoot()); - url.addParameter("sqlHash", sqlHash); - return url; - } - - - @AdminConsoleAction - public class QueryStackTracesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - return QueryProfiler.getInstance().getStackTraceView(form.getSqlHash(), AdminController::getExecutionPlanURL); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Queries", QueriesAction.class); - root.addChild("Query Stack Traces"); - } - } - - - private static ActionURL getExecutionPlanURL(String sqlHash) - { - ActionURL url = new ActionURL(ExecutionPlanAction.class, ContainerManager.getRoot()); - url.addParameter("sqlHash", sqlHash); - return url; - } - - - @AdminConsoleAction - public class ExecutionPlanAction extends SimpleViewAction - { - private String _sqlHash; - private ExecutionPlanType _type; - - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - _sqlHash = form.getSqlHash(); - _type = EnumUtils.getEnum(ExecutionPlanType.class, form.getType()); - if (null == _type) - throw new NotFoundException("Unknown execution plan type"); - - return QueryProfiler.getInstance().getExecutionPlanView(form.getSqlHash(), _type, form.isLog()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Queries", QueriesAction.class); - root.addChild("Query Stack Traces", getQueryStackTracesURL(_sqlHash)); - root.addChild(_type.getDescription()); - } - } - - - public static class QueryForm - { - private String _sqlHash; - private String _type = "Estimated"; // All dialects support Estimated - private boolean _log = false; - - public String getSqlHash() - { - return _sqlHash; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSqlHash(String sqlHash) - { - _sqlHash = sqlHash; - } - - public String getType() - { - return _type; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setType(String type) - { - _type = type; - } - - public boolean isLog() - { - return _log; - } - - public void setLog(boolean log) - { - _log = log; - } - } - - - private ActionURL getExportQueriesURL() - { - return new ActionURL(ExportQueriesAction.class, ContainerManager.getRoot()); - } - - - @AdminConsoleAction - public static class ExportQueriesAction extends ExportAction - { - @Override - public void export(Object o, HttpServletResponse response, BindException errors) throws Exception - { - try (QueryStatTsvWriter writer = new QueryStatTsvWriter()) - { - writer.setFilenamePrefix("SQL_Queries"); - writer.write(response); - } - } - } - - private static ActionURL getResetQueryStatisticsURL() - { - return new ActionURL(ResetQueryStatisticsAction.class, ContainerManager.getRoot()); - } - - - @RequiresPermission(AdminPermission.class) - public static class ResetQueryStatisticsAction extends FormHandlerAction - { - @Override - public void validateCommand(QueriesForm target, Errors errors) - { - } - - @Override - public boolean handlePost(QueriesForm form, BindException errors) throws Exception - { - QueryProfiler.getInstance().resetAllStatistics(); - return true; - } - - @Override - public URLHelper getSuccessURL(QueriesForm form) - { - return getQueriesURL(form.getStat()); - } - } - - - @AdminConsoleAction - public class CachesAction extends SimpleViewAction - { - private final DecimalFormat commaf0 = new DecimalFormat("#,##0"); - private final DecimalFormat percent = new DecimalFormat("0%"); - - @Override - public ModelAndView getView(MemForm form, BindException errors) - { - if (form.isClearCaches()) - { - LOG.info("Clearing Introspector caches"); - Introspector.flushCaches(); - LOG.info("Purging all caches"); - CacheManager.clearAllKnownCaches(); - ActionURL redirect = getViewContext().cloneActionURL().deleteParameter("clearCaches"); - throw new RedirectException(redirect); - } - - List> caches = CacheManager.getKnownCaches(); - - if (form.getDebugName() != null) - { - for (TrackingCache cache : caches) - { - if (form.getDebugName().equals(cache.getDebugName())) - { - LOG.info("Purging cache: " + cache.getDebugName()); - cache.clear(); - } - } - ActionURL redirect = getViewContext().cloneActionURL().deleteParameter("debugName"); - throw new RedirectException(redirect); - } - - List cacheStats = new ArrayList<>(); - List transactionStats = new ArrayList<>(); - - for (TrackingCache cache : caches) - { - cacheStats.add(CacheManager.getCacheStats(cache)); - transactionStats.add(CacheManager.getTransactionCacheStats(cache)); - } - - HtmlStringBuilder html = HtmlStringBuilder.of(); - - html.append(LinkBuilder.labkeyLink("Clear Caches and Refresh", getCachesURL(true, false))); - html.append(LinkBuilder.labkeyLink("Refresh", getCachesURL(false, false))); - - html.unsafeAppend("

\n"); - appendStats(html, "Caches", cacheStats, false); - - html.unsafeAppend("

\n"); - appendStats(html, "Transaction Caches", transactionStats, true); - - return new HtmlView(html); - } - - private void appendStats(HtmlStringBuilder html, String title, List allStats, boolean skipUnusedCaches) - { - List stats = skipUnusedCaches ? - allStats.stream() - .filter(stat->stat.getMaxSize() > 0) - .collect(Collectors.toCollection((Supplier>) ArrayList::new)) : - allStats; - - Collections.sort(stats); - - html.unsafeAppend("

"); - html.append(title); - html.append(" (").append(stats.size()).unsafeAppend(")

\n"); - - html.unsafeAppend("\n"); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - - long size = 0; - long gets = 0; - long misses = 0; - long puts = 0; - long expirations = 0; - long evictions = 0; - long removes = 0; - long clears = 0; - int rowCount = 0; - - for (CacheStats stat : stats) - { - size += stat.getSize(); - gets += stat.getGets(); - misses += stat.getMisses(); - puts += stat.getPuts(); - expirations += stat.getExpirations(); - evictions += stat.getEvictions(); - removes += stat.getRemoves(); - clears += stat.getClears(); - - html.unsafeAppend(""); - - appendDescription(html, stat.getDescription(), stat.getCreationStackTrace()); - - Long limit = stat.getLimit(); - long maxSize = stat.getMaxSize(); - - appendLongs(html, limit, maxSize, stat.getSize(), stat.getGets(), stat.getMisses(), stat.getPuts(), stat.getExpirations(), stat.getEvictions(), stat.getRemoves(), stat.getClears()); - appendDoubles(html, stat.getMissRatio()); - - html.unsafeAppend("\n"); - - if (null != limit && maxSize >= limit) - html.unsafeAppend(""); - - html.unsafeAppend("\n"); - rowCount++; - } - - double ratio = 0 != gets ? misses / (double)gets : 0; - html.unsafeAppend(""); - - appendLongs(html, null, null, size, gets, misses, puts, expirations, evictions, removes, clears); - appendDoubles(html, ratio); - - html.unsafeAppend("\n"); - html.unsafeAppend("
Debug NameLimitMax SizeCurrent SizeGetsMissesPutsExpirationsEvictionsRemovesClearsMiss PercentageClear
").append(LinkBuilder.labkeyLink("Clear", getCacheURL(stat.getDescription()))).unsafeAppend("This cache has been limited
Total
\n"); - } - - private static final List PREFIXES_TO_SKIP = List.of( - "java.base/java.lang.Thread.getStackTrace", - "org.labkey.api.cache.CacheManager", - "org.labkey.api.cache.Throttle", - "org.labkey.api.data.DatabaseCache", - "org.labkey.api.module.ModuleResourceCache" - ); - - private void appendDescription(HtmlStringBuilder html, String description, @Nullable StackTraceElement[] creationStackTrace) - { - StringBuilder sb = new StringBuilder(); - - if (creationStackTrace != null) - { - boolean trimming = true; - for (StackTraceElement element : creationStackTrace) - { - // Skip the first few uninteresting stack trace elements to highlight the caller we care about - if (trimming) - { - if (PREFIXES_TO_SKIP.stream().anyMatch(prefix->element.toString().startsWith(prefix))) - continue; - - trimming = false; - } - sb.append(element); - sb.append("\n"); - } - } - - if (!sb.isEmpty()) - { - String message = PageFlowUtil.jsString(sb); - String id = "id" + UniqueID.getServerSessionScopedUID(); - html.append(DOM.createHtmlFragment(TD(A(at(href, "#").id(id), description)))); - HttpView.currentPageConfig().addHandler(id, "click", "alert(" + message + ");return false;"); - } - } - - private void appendLongs(HtmlStringBuilder html, Long... stats) - { - for (Long stat : stats) - { - if (null == stat) - html.unsafeAppend(" "); - else - html.unsafeAppend("").append(commaf0.format(stat)).unsafeAppend(""); - } - } - - private void appendDoubles(HtmlStringBuilder html, double... stats) - { - for (double stat : stats) - html.unsafeAppend("").append(percent.format(stat)).unsafeAppend(""); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("cachesDiagnostics"); - addAdminNavTrail(root, "Cache Statistics", this.getClass()); - } - } - - @RequiresSiteAdmin - public class EnvironmentVariablesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/properties.jsp", System.getenv()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Environment Variables", this.getClass()); - } - } - - @RequiresSiteAdmin - public class SystemPropertiesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView>("/org/labkey/core/admin/properties.jsp", new HashMap(System.getProperties())); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "System Properties", this.getClass()); - } - } - - - public static class ConfigureSystemMaintenanceForm - { - private String _maintenanceTime; - private Set _enable = Collections.emptySet(); - private boolean _enableSystemMaintenance = true; - - public String getMaintenanceTime() - { - return _maintenanceTime; - } - - @SuppressWarnings("unused") - public void setMaintenanceTime(String maintenanceTime) - { - _maintenanceTime = maintenanceTime; - } - - public Set getEnable() - { - return _enable; - } - - @SuppressWarnings("unused") - public void setEnable(Set enable) - { - _enable = enable; - } - - public boolean isEnableSystemMaintenance() - { - return _enableSystemMaintenance; - } - - @SuppressWarnings("unused") - public void setEnableSystemMaintenance(boolean enableSystemMaintenance) - { - _enableSystemMaintenance = enableSystemMaintenance; - } - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public class ConfigureSystemMaintenanceAction extends FormViewAction - { - @Override - public void validateCommand(ConfigureSystemMaintenanceForm form, Errors errors) - { - Date date = SystemMaintenance.parseSystemMaintenanceTime(form.getMaintenanceTime()); - - if (null == date) - errors.reject(ERROR_MSG, "Invalid format for system maintenance time"); - } - - @Override - public ModelAndView getView(ConfigureSystemMaintenanceForm form, boolean reshow, BindException errors) - { - SystemMaintenanceProperties prop = SystemMaintenance.getProperties(); - return new JspView<>("/org/labkey/core/admin/systemMaintenance.jsp", prop, errors); - } - - @Override - public boolean handlePost(ConfigureSystemMaintenanceForm form, BindException errors) - { - SystemMaintenance.setTimeDisabled(!form.isEnableSystemMaintenance()); - SystemMaintenance.setProperties(form.getEnable(), form.getMaintenanceTime()); - - return true; - } - - @Override - public URLHelper getSuccessURL(ConfigureSystemMaintenanceForm form) - { - return new AdminUrlsImpl().getAdminConsoleURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Configure System Maintenance", this.getClass()); - } - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public static class ResetSystemMaintenanceAction extends FormHandlerAction - { - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - SystemMaintenance.clearProperties(); - return true; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return new AdminUrlsImpl().getAdminConsoleURL(); - } - } - - public static class SystemMaintenanceForm - { - private String _taskName; - private boolean _test = false; - - public String getTaskName() - { - return _taskName; - } - - @SuppressWarnings("unused") - public void setTaskName(String taskName) - { - _taskName = taskName; - } - - public boolean isTest() - { - return _test; - } - - public void setTest(boolean test) - { - _test = test; - } - } - - @RequiresSiteAdmin - public class SystemMaintenanceAction extends FormHandlerAction - { - private Long _jobId = null; - private URLHelper _url = null; - - @Override - public void validateCommand(SystemMaintenanceForm form, Errors errors) - { - } - - @Override - public ModelAndView getSuccessView(SystemMaintenanceForm form) throws IOException - { - // Send the pipeline job details absolute URL back to the test - sendPlainText(_url.getURIString()); - - // Suppress templates, divs, etc. - getPageConfig().setTemplate(Template.None); - return new EmptyView(); - } - - @Override - public boolean handlePost(SystemMaintenanceForm form, BindException errors) - { - String jobGuid = new SystemMaintenanceJob(form.getTaskName(), getUser()).call(); - - if (null != jobGuid) - _jobId = PipelineService.get().getJobId(getUser(), getContainer(), jobGuid); - - PipelineStatusUrls urls = urlProvider(PipelineStatusUrls.class); - _url = null != _jobId ? urls.urlDetails(getContainer(), _jobId) : urls.urlBegin(getContainer()); - - return true; - } - - @Override - public URLHelper getSuccessURL(SystemMaintenanceForm form) - { - // In the standard case, redirect to the pipeline details URL - // If the test is invoking system maintenance then return the URL instead - return form.isTest() ? null : _url; - } - } - - @AdminConsoleAction - public class AttachmentsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return AttachmentService.get().getAdminView(getViewContext().getActionURL()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Attachments", getClass()); - } - } - - @AdminConsoleAction - public class FindAttachmentParentsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return AttachmentService.get().getFindAttachmentParentsView(); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Find Attachment Parents", getClass()); - } - } - - public static ActionURL getMemTrackerURL(boolean clearCaches, boolean gc) - { - ActionURL url = new ActionURL(MemTrackerAction.class, ContainerManager.getRoot()); - - if (clearCaches) - url.addParameter(MemForm.Params.clearCaches, "1"); - - if (gc) - url.addParameter(MemForm.Params.gc, "1"); - - return url; - } - - public static ActionURL getCachesURL(boolean clearCaches, boolean gc) - { - ActionURL url = new ActionURL(CachesAction.class, ContainerManager.getRoot()); - - if (clearCaches) - url.addParameter(MemForm.Params.clearCaches, "1"); - - if (gc) - url.addParameter(MemForm.Params.gc, "1"); - - return url; - } - - public static ActionURL getCacheURL(String debugName) - { - ActionURL url = new ActionURL(CachesAction.class, ContainerManager.getRoot()); - - url.addParameter(MemForm.Params.debugName, debugName); - - return url; - } - - private static volatile String lastCacheMemUsed = null; - - @AdminConsoleAction - public class MemTrackerAction extends SimpleViewAction - { - @Override - public ModelAndView getView(MemForm form, BindException errors) - { - Set objectsToIgnore = MemTracker.getInstance().beforeReport(); - - boolean gc = form.isGc(); - boolean cc = form.isClearCaches(); - - if (getUser().hasRootAdminPermission() && (gc || cc)) - { - // If both are requested then try to determine and record cache memory usage - if (gc && cc) - { - // gc once to get an accurate free memory read - long before = gc(); - clearCaches(); - // gc again now that we cleared caches - long cacheMemoryUsed = before - gc(); - - // Difference could be < 0 if JVM or other threads have performed gc, in which case we can't guesstimate cache memory usage - String cacheMemUsed = cacheMemoryUsed > 0 ? FileUtils.byteCountToDisplaySize(cacheMemoryUsed) : "Unknown"; - LOG.info("Estimate of cache memory used: " + cacheMemUsed); - lastCacheMemUsed = cacheMemUsed; - } - else if (cc) - { - clearCaches(); - } - else - { - gc(); - } - - LOG.info("Cache clearing and garbage collecting complete"); - } - - return new JspView<>("/org/labkey/core/admin/memTracker.jsp", new MemBean(getViewContext().getRequest(), objectsToIgnore)); - } - - /** @return estimated current memory usage, post-garbage collection */ - private long gc() - { - LOG.info("Garbage collecting"); - System.gc(); - // This is more reliable than relying on just free memory size, as the VM can grow/shrink the heap at will - return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); - } - - private void clearCaches() - { - LOG.info("Clearing Introspector caches"); - Introspector.flushCaches(); - LOG.info("Purging all caches"); - CacheManager.clearAllKnownCaches(); - SearchService ss = SearchService.get(); - if (null != ss) - { - LOG.info("Purging SearchService queues"); - ss.purgeQueues(); - } - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("memTracker"); - addAdminNavTrail(root, "Memory usage -- " + DateUtil.formatDateTime(getContainer()), this.getClass()); - } - } - - public static class MemForm - { - private enum Params {clearCaches, debugName, gc} - - private boolean _clearCaches = false; - private boolean _gc = false; - private String _debugName; - - public boolean isClearCaches() - { - return _clearCaches; - } - - @SuppressWarnings("unused") - public void setClearCaches(boolean clearCaches) - { - _clearCaches = clearCaches; - } - - public boolean isGc() - { - return _gc; - } - - @SuppressWarnings("unused") - public void setGc(boolean gc) - { - _gc = gc; - } - - public String getDebugName() - { - return _debugName; - } - - @SuppressWarnings("unused") - public void setDebugName(String debugName) - { - _debugName = debugName; - } - } - - public static class MemBean - { - public final List> memoryUsages = new ArrayList<>(); - public final List> systemProperties = new ArrayList<>(); - public final List references; - public final List graphNames = new ArrayList<>(); - public final List activeThreads = new LinkedList<>(); - - public boolean assertsEnabled = false; - - private MemBean(HttpServletRequest request, Set objectsToIgnore) - { - MemTracker memTracker = MemTracker.getInstance(); - List all = memTracker.getReferences(); - long threadId = Thread.currentThread().getId(); - - // Attempt to detect other threads running labkey code -- mem tracker page will warn if any are found - for (Thread thread : new ThreadsBean().threads) - { - if (thread.getId() == threadId) - continue; - - Thread.State state = thread.getState(); - - if (state == Thread.State.RUNNABLE || state == Thread.State.BLOCKED) - { - boolean labkeyThread = false; - - if (memTracker.shouldDisplay(thread)) - { - for (StackTraceElement element : thread.getStackTrace()) - { - String className = element.getClassName(); - - if (className.startsWith("org.labkey") || className.startsWith("org.fhcrc")) - { - labkeyThread = true; - break; - } - } - } - - if (labkeyThread) - { - String threadInfo = thread.getName(); - TransactionFilter.RequestTracker uri = TransactionFilter.getRequestSummary(thread); - if (null != uri) - threadInfo += "; processing URL " + uri; - activeThreads.add(threadInfo); - } - } - } - - // ignore recently allocated - long start = ViewServlet.getRequestStartTime(request) - 2000; - references = new ArrayList<>(all.size()); - - for (HeldReference r : all) - { - if (r.getThreadId() == threadId && r.getAllocationTime() >= start) - continue; - - if (objectsToIgnore.contains(r.getReference())) - continue; - - references.add(r); - } - - // memory: - graphNames.add("Heap"); - graphNames.add("Non Heap"); - - MemoryMXBean membean = ManagementFactory.getMemoryMXBean(); - if (membean != null) - { - memoryUsages.add(Tuple3.of(true, HEAP_MEMORY_KEY, getUsage(membean.getHeapMemoryUsage()))); - } - - List pools = ManagementFactory.getMemoryPoolMXBeans(); - for (MemoryPoolMXBean pool : pools) - { - if (pool.getType() == MemoryType.HEAP) - { - memoryUsages.add(Tuple3.of(false, pool.getName() + " " + pool.getType(), getUsage(pool))); - graphNames.add(pool.getName()); - } - } - - if (membean != null) - { - memoryUsages.add(Tuple3.of(true, "Total Non-heap Memory", getUsage(membean.getNonHeapMemoryUsage()))); - } - - for (MemoryPoolMXBean pool : pools) - { - if (pool.getType() == MemoryType.NON_HEAP) - { - memoryUsages.add(Tuple3.of(false, pool.getName() + " " + pool.getType(), getUsage(pool))); - graphNames.add(pool.getName()); - } - } - - for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) - { - memoryUsages.add(Tuple3.of(true, "Buffer pool " + pool.getName(), new MemoryUsageSummary(pool))); - graphNames.add(pool.getName()); - } - - DecimalFormat commaf0 = new DecimalFormat("#,##0"); - - - // class loader: - ClassLoadingMXBean classbean = ManagementFactory.getClassLoadingMXBean(); - if (classbean != null) - { - systemProperties.add(new Pair<>("Loaded Class Count", commaf0.format(classbean.getLoadedClassCount()))); - systemProperties.add(new Pair<>("Unloaded Class Count", commaf0.format(classbean.getUnloadedClassCount()))); - systemProperties.add(new Pair<>("Total Loaded Class Count", commaf0.format(classbean.getTotalLoadedClassCount()))); - } - - // runtime: - RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean(); - if (runtimeBean != null) - { - systemProperties.add(new Pair<>("VM Start Time", DateUtil.formatIsoDateShortTime(new Date(runtimeBean.getStartTime())))); - long upTime = runtimeBean.getUptime(); // round to sec - upTime = upTime - (upTime % 1000); - systemProperties.add(new Pair<>("VM Uptime", DateUtil.formatDuration(upTime))); - systemProperties.add(new Pair<>("VM Version", runtimeBean.getVmVersion())); - systemProperties.add(new Pair<>("VM Classpath", runtimeBean.getClassPath())); - } - - // threads: - ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); - if (threadBean != null) - { - systemProperties.add(new Pair<>("Thread Count", threadBean.getThreadCount())); - systemProperties.add(new Pair<>("Peak Thread Count", threadBean.getPeakThreadCount())); - long[] deadlockedThreads = threadBean.findMonitorDeadlockedThreads(); - systemProperties.add(new Pair<>("Deadlocked Thread Count", deadlockedThreads != null ? deadlockedThreads.length : 0)); - } - - // threads: - List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); - for (GarbageCollectorMXBean gcBean : gcBeans) - { - systemProperties.add(new Pair<>(gcBean.getName() + " GC count", gcBean.getCollectionCount())); - systemProperties.add(new Pair<>(gcBean.getName() + " GC time", DateUtil.formatDuration(gcBean.getCollectionTime()))); - } - - String cacheMem = lastCacheMemUsed; - - if (null != cacheMem) - systemProperties.add(new Pair<>("Most Recent Estimated Cache Memory Usage", cacheMem)); - - OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); - if (osBean != null) - { - systemProperties.add(new Pair<>("CPU count", osBean.getAvailableProcessors())); - - DecimalFormat f3 = new DecimalFormat("0.000"); - - if (osBean instanceof com.sun.management.OperatingSystemMXBean sunOsBean) - { - systemProperties.add(new Pair<>("Total OS memory", FileUtils.byteCountToDisplaySize(sunOsBean.getTotalMemorySize()))); - systemProperties.add(new Pair<>("Free OS memory", FileUtils.byteCountToDisplaySize(sunOsBean.getFreeMemorySize()))); - systemProperties.add(new Pair<>("OS CPU load", f3.format(sunOsBean.getCpuLoad()))); - systemProperties.add(new Pair<>("JVM CPU load", f3.format(sunOsBean.getProcessCpuLoad()))); - } - } - - //noinspection ConstantConditions - assert assertsEnabled = true; - } - } - - private static MemoryUsageSummary getUsage(MemoryPoolMXBean pool) - { - try - { - return getUsage(pool.getUsage()); - } - catch (IllegalArgumentException x) - { - // sometimes we get usage>committed exception with older versions of JRockit - return null; - } - } - - public static class MemoryUsageSummary - { - - public final long _init; - public final long _used; - public final long _committed; - public final long _max; - - public MemoryUsageSummary(MemoryUsage usage) - { - _init = usage.getInit(); - _used = usage.getUsed(); - _committed = usage.getCommitted(); - _max = usage.getMax(); - } - - public MemoryUsageSummary(BufferPoolMXBean pool) - { - _init = -1; - _used = pool.getMemoryUsed(); - _committed = _used; - _max = pool.getTotalCapacity(); - } - } - - private static MemoryUsageSummary getUsage(MemoryUsage usage) - { - if (null == usage) - return null; - - try - { - return new MemoryUsageSummary(usage); - } - catch (IllegalArgumentException x) - { - // sometime we get usage>committed exception with older verions of JRockit - return null; - } - } - - public static class ChartForm - { - private String _type; - - public String getType() - { - return _type; - } - - public void setType(String type) - { - _type = type; - } - } - - private static class MemoryCategory implements Comparable - { - private final String _type; - private final double _mb; - - public MemoryCategory(String type, double mb) - { - _type = type; - _mb = mb; - } - - @Override - public int compareTo(@NotNull MemoryCategory o) - { - return Double.compare(getMb(), o.getMb()); - } - - public String getType() - { - return _type; - } - - public double getMb() - { - return _mb; - } - } - - @AdminConsoleAction - public static class MemoryChartAction extends ExportAction - { - @Override - public void export(ChartForm form, HttpServletResponse response, BindException errors) throws Exception - { - MemoryUsage usage = null; - boolean showLegend = false; - String title = form.getType(); - if ("Heap".equals(form.getType())) - { - usage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); - showLegend = true; - } - else if ("Non Heap".equals(form.getType())) - usage = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage(); - else - { - List pools = ManagementFactory.getMemoryPoolMXBeans(); - for (Iterator it = pools.iterator(); it.hasNext() && usage == null;) - { - MemoryPoolMXBean pool = it.next(); - if (form.getType().equals(pool.getName())) - usage = pool.getUsage(); - } - } - - Pair divisor = null; - - List types = new ArrayList<>(4); - - if (usage == null) - { - boolean found = false; - for (Iterator it = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class).iterator(); it.hasNext() && !found;) - { - BufferPoolMXBean pool = it.next(); - if (form.getType().equals(pool.getName())) - { - long total = pool.getTotalCapacity(); - long used = pool.getMemoryUsed(); - - divisor = getDivisor(total); - - title = "Buffer pool " + title; - - if (total > 0 || used > 0) - { - types.add(new MemoryCategory("Used", used / divisor.first)); - types.add(new MemoryCategory("Max", total / divisor.first)); - } - found = true; - } - } - if (!found) - { - throw new NotFoundException(); - } - } - else - { - if (usage.getInit() > 0 || usage.getUsed() > 0 || usage.getCommitted() > 0 || usage.getMax() > 0) - { - divisor = getDivisor(Math.max(usage.getInit(), Math.max(usage.getUsed(), Math.max(usage.getCommitted(), usage.getMax())))); - - types.add(new MemoryCategory("Init", (double) usage.getInit() / divisor.first)); - types.add(new MemoryCategory("Used", (double) usage.getUsed() / divisor.first)); - types.add(new MemoryCategory("Committed", (double) usage.getCommitted() / divisor.first)); - types.add(new MemoryCategory("Max", (double) usage.getMax() / divisor.first)); - } - } - - if (divisor != null) - { - title += " (" + divisor.second + ")"; - } - - DefaultCategoryDataset dataset = new DefaultCategoryDataset(); - - Collections.sort(types); - - for (int i = 0; i < types.size(); i++) - { - double mbPastPrevious = i > 0 ? types.get(i).getMb() - types.get(i - 1).getMb() : types.get(i).getMb(); - dataset.addValue(mbPastPrevious, types.get(i).getType(), ""); - } - - JFreeChart chart = ChartFactory.createStackedBarChart(title, null, null, dataset, PlotOrientation.HORIZONTAL, showLegend, false, false); - chart.getTitle().setFont(new Font("SansSerif", Font.BOLD, 14)); - response.setContentType("image/png"); - - ChartUtilities.writeChartAsPNG(response.getOutputStream(), chart, showLegend ? 800 : 398, showLegend ? 100 : 70); - } - - private Pair getDivisor(long l) - { - if (l > 4096L * 1024L * 1024L) - { - return Pair.of(1024L * 1024L * 1024L, "GB"); - } - if (l > 4096L * 1024L) - { - return Pair.of(1024L * 1024L, "MB"); - } - if (l > 4096L) - { - return Pair.of(1024L, "KB"); - } - - return Pair.of(1L, "bytes"); - - } - } - - public static class MemoryStressForm - { - private int _threads = 3; - private int _arraySize = 20_000; - private int _arrayCount = 10_000; - private float _percentChurn = 0.50f; - private int _delay = 20; - private int _iterations = 500; - - public int getThreads() - { - return _threads; - } - - public void setThreads(int threads) - { - _threads = threads; - } - - public int getArraySize() - { - return _arraySize; - } - - public void setArraySize(int arraySize) - { - _arraySize = arraySize; - } - - public int getArrayCount() - { - return _arrayCount; - } - - public void setArrayCount(int arrayCount) - { - _arrayCount = arrayCount; - } - - public float getPercentChurn() - { - return _percentChurn; - } - - public void setPercentChurn(float percentChurn) - { - _percentChurn = percentChurn; - } - - public int getDelay() - { - return _delay; - } - - public void setDelay(int delay) - { - _delay = delay; - } - - public int getIterations() - { - return _iterations; - } - - public void setIterations(int iterations) - { - _iterations = iterations; - } - } - - @RequiresSiteAdmin - public class MemoryStressTestAction extends FormViewAction - { - @Override - public void validateCommand(MemoryStressForm target, Errors errors) - { - - } - - @Override - public ModelAndView getView(MemoryStressForm memoryStressForm, boolean reshow, BindException errors) throws Exception - { - return new HtmlView( - DOM.LK.FORM(at(method, "POST"), - DOM.LK.ERRORS(errors.getBindingResult()), - DOM.BR(), DOM.BR(), - "This utility action will do a lot of memory allocation to test the memory configuration of the host.", - DOM.BR(), DOM.BR(), - "It spins up threads, all of which allocate a specified number byte arrays of specified length.", - DOM.BR(), - "The threads sleep for the delay period, and then replace the specified percent of arrays with new ones.", - DOM.BR(), - "They continue for the specified number of allocations.", - DOM.BR(), - "The memory actively held is approximately (threads * array count * array length).", - DOM.BR(), - "The memory turnover is based on the churn percentage, array length, delay, and iterations.", - DOM.BR(), DOM.BR(), - DOM.TABLE( - DOM.TR(DOM.TD("Thread count"), DOM.TD(DOM.INPUT(at(name, "threads", value, memoryStressForm._threads)))), - DOM.TR(DOM.TD("Byte array count"), DOM.TD(DOM.INPUT(at(name, "arrayCount", value, memoryStressForm._arrayCount)))), - DOM.TR(DOM.TD("Byte array size"), DOM.TD(DOM.INPUT(at(name, "arraySize", value, memoryStressForm._arraySize)))), - DOM.TR(DOM.TD("Iterations"), DOM.TD(DOM.INPUT(at(name, "iterations", value, memoryStressForm._iterations)))), - DOM.TR(DOM.TD("Delay between iterations (ms)"), DOM.TD(DOM.INPUT(at(name, "delay", value, memoryStressForm._delay)))), - DOM.TR(DOM.TD("Percent churn per iteration (0.0 - 1.0)"), DOM.TD(DOM.INPUT(at(name, "percentChurn", value, memoryStressForm._percentChurn)))) - ), - new ButtonBuilder("Perform stress test").submit(true).build()) - ); - } - - @Override - public boolean handlePost(MemoryStressForm memoryStressForm, BindException errors) throws Exception - { - List threads = new ArrayList<>(); - for (int i = 0; i < memoryStressForm._threads; i++) - { - Thread t = new Thread(() -> - { - Random r = new Random(); - byte[][] arrays = new byte[memoryStressForm._arrayCount][]; - // Initialize the arrays - for (int a = 0; a < arrays.length; a++) - { - arrays[a] = new byte[memoryStressForm._arraySize]; - } - - for (int iter = 0; iter < memoryStressForm._iterations; iter++) - { - try - { - Thread.sleep(memoryStressForm._delay); - } - catch (InterruptedException ignored) {} - - // Swap the contents based on our desired percent churn - for (int a = 0; a < arrays.length; a++) - { - if (r.nextFloat() <= memoryStressForm._percentChurn) - { - arrays[a] = new byte[memoryStressForm._arraySize]; - } - } - } - }); - t.setUncaughtExceptionHandler((t2, e) -> { - LOG.error("Stress test exception", e); - errors.reject(null, "Stress test exception: " + e); - }); - t.start(); - threads.add(t); - } - - for (Thread thread : threads) - { - thread.join(); - } - - return !errors.hasErrors(); - } - - @Override - public URLHelper getSuccessURL(MemoryStressForm memoryStressForm) - { - return new ActionURL(MemTrackerAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Memory Usage", MemTrackerAction.class); - root.addChild("Memory Stress Test"); - } - } - - public static ActionURL getModuleStatusURL(URLHelper returnUrl) - { - ActionURL url = new ActionURL(ModuleStatusAction.class, ContainerManager.getRoot()); - if (returnUrl != null) - url.addReturnUrl(returnUrl); - return url; - } - - public static class ModuleStatusBean - { - public String verb; - public String verbing; - public ActionURL nextURL; - } - - @RequiresPermission(TroubleshooterPermission.class) - @AllowedDuringUpgrade - @IgnoresAllocationTracking - public static class ModuleStatusAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ReturnUrlForm form, BindException errors) - { - ModuleLoader loader = ModuleLoader.getInstance(); - VBox vbox = new VBox(); - ModuleStatusBean bean = new ModuleStatusBean(); - - if (loader.isNewInstall()) - bean.nextURL = new ActionURL(NewInstallSiteSettingsAction.class, ContainerManager.getRoot()); - else if (form.getReturnUrl() != null) - { - try - { - bean.nextURL = form.getReturnActionURL(); - } - catch (URLException x) - { - // might not be an ActionURL e.g. /labkey/_webdav/home - } - } - if (null == bean.nextURL) - bean.nextURL = new ActionURL(InstallCompleteAction.class, ContainerManager.getRoot()); - - if (loader.isNewInstall()) - bean.verb = "Install"; - else if (loader.isUpgradeRequired() || loader.isUpgradeInProgress()) - bean.verb = "Upgrade"; - else - bean.verb = "Start"; - - if (loader.isNewInstall()) - bean.verbing = "Installing"; - else if (loader.isUpgradeRequired() || loader.isUpgradeInProgress()) - bean.verbing = "Upgrading"; - else - bean.verbing = "Starting"; - - JspView statusView = new JspView<>("/org/labkey/core/admin/moduleStatus.jsp", bean, errors); - vbox.addView(statusView); - - getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); - - getPageConfig().setTemplate(Template.Wizard); - getPageConfig().setTitle(bean.verb + " Modules"); - setHelpTopic(ModuleLoader.getInstance().isNewInstall() ? "config" : "upgrade"); - - return vbox; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - public static class NewInstallSiteSettingsForm extends FileSettingsForm - { - private String _notificationEmail; - private String _siteName; - - public String getNotificationEmail() - { - return _notificationEmail; - } - - public void setNotificationEmail(String notificationEmail) - { - _notificationEmail = notificationEmail; - } - - public String getSiteName() - { - return _siteName; - } - - public void setSiteName(String siteName) - { - _siteName = siteName; - } - } - - @RequiresSiteAdmin - public static class NewInstallSiteSettingsAction extends AbstractFileSiteSettingsAction - { - public NewInstallSiteSettingsAction() - { - super(NewInstallSiteSettingsForm.class); - } - - @Override - public void validateCommand(NewInstallSiteSettingsForm form, Errors errors) - { - super.validateCommand(form, errors); - - if (isBlank(form.getNotificationEmail())) - { - errors.reject(SpringActionController.ERROR_MSG, "Notification email address may not be blank."); - } - try - { - ValidEmail email = new ValidEmail(form.getNotificationEmail()); - } - catch (ValidEmail.InvalidEmailException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - } - - @Override - public boolean handlePost(NewInstallSiteSettingsForm form, BindException errors) throws Exception - { - boolean success = super.handlePost(form, errors); - if (success) - { - WriteableLookAndFeelProperties lafProps = LookAndFeelProperties.getWriteableInstance(ContainerManager.getRoot()); - try - { - lafProps.setSystemEmailAddress(new ValidEmail(form.getNotificationEmail())); - } - catch (ValidEmail.InvalidEmailException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - return false; - } - lafProps.setSystemShortName(form.getSiteName()); - lafProps.save(); - - // Send an immediate report now that they've set up their account and defaults, and then every 24 hours after. - UsageReportingLevel.reportNow(); - - return true; - } - return false; - } - - @Override - public ModelAndView getView(NewInstallSiteSettingsForm form, boolean reshow, BindException errors) - { - if (!reshow) - { - File root = _svc.getSiteDefaultRoot(); - - if (root.exists()) - form.setRootPath(FileUtil.getAbsoluteCaseSensitiveFile(root).getAbsolutePath()); - - LookAndFeelProperties props = LookAndFeelProperties.getInstance(ContainerManager.getRoot()); - form.setSiteName(props.getShortName()); - form.setNotificationEmail(props.getSystemEmailAddress()); - } - - JspView view = new JspView<>("/org/labkey/core/admin/newInstallSiteSettings.jsp", form, errors); - - getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); - getPageConfig().setTitle("Set Defaults"); - getPageConfig().setTemplate(Template.Wizard); - - return view; - } - - @Override - public URLHelper getSuccessURL(NewInstallSiteSettingsForm form) - { - return new ActionURL(InstallCompleteAction.class, ContainerManager.getRoot()); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresSiteAdmin - public static class InstallCompleteAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - JspView view = new JspView<>("/org/labkey/core/admin/installComplete.jsp"); - - getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); - getPageConfig().setTitle("Complete"); - getPageConfig().setTemplate(Template.Wizard); - - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - public static List getInstallUpgradeWizardSteps() - { - List navTrail = new ArrayList<>(); - if (ModuleLoader.getInstance().isNewInstall()) - { - navTrail.add(new NavTree("Account Setup")); - navTrail.add(new NavTree("Install Modules")); - navTrail.add(new NavTree("Set Defaults")); - } - else if (ModuleLoader.getInstance().isUpgradeRequired() || ModuleLoader.getInstance().isUpgradeInProgress()) - { - navTrail.add(new NavTree("Upgrade Modules")); - } - else - { - navTrail.add(new NavTree("Start Modules")); - } - navTrail.add(new NavTree("Complete")); - return navTrail; - } - - @RequiresPermission(AdminOperationsPermission.class) - public class DbCheckerAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/checkDatabase.jsp", new DataCheckForm()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Database Check Tools", this.getClass()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public class DoCheckAction extends SimpleViewAction - { - @Override - public ModelAndView getView(DataCheckForm form, BindException errors) - { - try (var ignore=SpringActionController.ignoreSqlUpdates()) - { - ActionURL currentUrl = getViewContext().cloneActionURL(); - String fixRequested = currentUrl.getParameter("_fix"); - HtmlStringBuilder contentBuilder = HtmlStringBuilder.of(HtmlString.unsafe("
")); - - if (null != fixRequested) - { - HtmlString sqlCheck = HtmlString.EMPTY_STRING; - if (fixRequested.equalsIgnoreCase("container")) - sqlCheck = DbSchema.checkAllContainerCols(getUser(), true); - else if (fixRequested.equalsIgnoreCase("descriptor")) - sqlCheck = OntologyManager.doProjectColumnCheck(true); - contentBuilder.append(sqlCheck); - } - else - { - LOG.info("Starting database check"); // Debugging test timeout - LOG.info("Checking container column references"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

") - .append("Checking Container Column References..."); - HtmlString strTemp = DbSchema.checkAllContainerCols(getUser(), false); - if (!strTemp.isEmpty()) - { - contentBuilder.append(strTemp); - currentUrl = getViewContext().cloneActionURL(); - currentUrl.addParameter("_fix", "container"); - contentBuilder.unsafeAppend("

    ") - .append(" click ") - .append(LinkBuilder.simpleLink("here", currentUrl)) - .append(" to attempt recovery."); - } - - LOG.info("Checking PropertyDescriptor and DomainDescriptor consistency"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

") - .append("Checking PropertyDescriptor and DomainDescriptor consistency..."); - strTemp = OntologyManager.doProjectColumnCheck(false); - if (!strTemp.isEmpty()) - { - contentBuilder.append(strTemp); - currentUrl = getViewContext().cloneActionURL(); - currentUrl.addParameter("_fix", "descriptor"); - contentBuilder.unsafeAppend("

    ") - .append(" click ") - .append(LinkBuilder.simpleLink("here", currentUrl)) - .append(" to attempt recovery."); - } - - LOG.info("Checking Schema consistency with tableXML"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

") - .append("Checking Schema consistency with tableXML.") - .unsafeAppend("

"); - Set schemas = DbSchema.getAllSchemasToTest(); - - for (DbSchema schema : schemas) - { - SiteValidationResultList schemaResult = TableXmlUtils.compareXmlToMetaData(schema, form.getFull(), false, true); - List results = schemaResult.getResults(null); - if (results.isEmpty()) - { - contentBuilder.unsafeAppend("") - .append(schema.getDisplayName()) - .append(": OK") - .unsafeAppend("
"); - } - else - { - contentBuilder.unsafeAppend("") - .append(schema.getDisplayName()) - .unsafeAppend(""); - for (var r : results) - { - HtmlString item = r.getMessage().isEmpty() ? NBSP : r.getMessage(); - contentBuilder.unsafeAppend("
  • ") - .append(item) - .unsafeAppend("
    • \n"); - } - contentBuilder.unsafeAppend(""); - } - } - - LOG.info("Checking consistency of provisioned storage"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

    ") - .append("Checking Consistency of Provisioned Storage...\n"); - StorageProvisioner.ProvisioningReport pr = StorageProvisioner.get().getProvisioningReport(); - contentBuilder.append(String.format("%d domains use Storage Provisioner", pr.getProvisionedDomains().size())); - for (StorageProvisioner.ProvisioningReport.DomainReport dr : pr.getProvisionedDomains()) - { - for (String error : dr.getErrors()) - { - contentBuilder.unsafeAppend("
    ") - .append(error) - .unsafeAppend("
    "); - } - } - for (String error : pr.getGlobalErrors()) - { - contentBuilder.unsafeAppend("
    ") - .append(error) - .unsafeAppend("
    "); - } - - LOG.info("Database check complete"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

    ") - .append("Database Consistency checker complete"); - } - - contentBuilder.unsafeAppend("
    "); - - return new HtmlView(contentBuilder); - } - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Database Tools", this.getClass()); - } - } - - public static class DataCheckForm - { - private String _dbSchema = ""; - private boolean _full = false; - - public List modules = ModuleLoader.getInstance().getModules(); - public DataCheckForm(){} - - public List getModules() { return modules; } - public String getDbSchema() { return _dbSchema; } - @SuppressWarnings("unused") - public void setDbSchema(String dbSchema){ _dbSchema = dbSchema; } - public boolean getFull() { return _full; } - public void setFull(boolean full) { _full = full; } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class GetSchemaXmlDocAction extends ExportAction - { - @Override - public void export(DataCheckForm form, HttpServletResponse response, BindException errors) throws Exception - { - String fullyQualifiedSchemaName = form.getDbSchema(); - if (null == fullyQualifiedSchemaName || fullyQualifiedSchemaName.isEmpty()) - { - throw new NotFoundException("Must specify dbSchema parameter"); - } - - boolean bFull = form.getFull(); - - Pair scopeAndSchemaName = DbSchema.getDbScopeAndSchemaName(fullyQualifiedSchemaName); - TablesDocument tdoc = TableXmlUtils.createXmlDocumentFromDatabaseMetaData(scopeAndSchemaName.first, scopeAndSchemaName.second, bFull); - StringWriter sw = new StringWriter(); - - XmlOptions xOpt = new XmlOptions(); - xOpt.setSavePrettyPrint(); - xOpt.setUseDefaultNamespace(); - - tdoc.save(sw, xOpt); - - sw.flush(); - PageFlowUtil.streamFileBytes(response, fullyQualifiedSchemaName + ".xml", sw.toString().getBytes(StringUtilsLabKey.DEFAULT_CHARSET), true); - } - } - - @RequiresPermission(AdminPermission.class) - public static class FolderInformationAction extends FolderManagementViewAction - { - @Override - protected HtmlView getTabView() - { - Container c = getContainer(); - User currentUser = getUser(); - - User createdBy = UserManager.getUser(c.getCreatedBy()); - Map propValueMap = new LinkedHashMap<>(); - propValueMap.put("Path", c.getPath()); - propValueMap.put("Name", c.getName()); - propValueMap.put("Displayed Title", c.getTitle()); - propValueMap.put("EntityId", c.getId()); - propValueMap.put("RowId", c.getRowId()); - propValueMap.put("Created", DateUtil.formatDateTime(c, c.getCreated())); - propValueMap.put("Created By", (createdBy != null ? createdBy.getDisplayName(currentUser) : "<" + c.getCreatedBy() + ">")); - propValueMap.put("Folder Type", c.getFolderType().getName()); - propValueMap.put("Description", c.getDescription()); - - return new HtmlView(PageFlowUtil.getDataRegionHtmlForPropertyObjects(propValueMap)); - } - } - - public static class MissingValuesForm - { - private boolean _inheritMvIndicators; - private String[] _mvIndicators; - private String[] _mvLabels; - - public boolean isInheritMvIndicators() - { - return _inheritMvIndicators; - } - - public void setInheritMvIndicators(boolean inheritMvIndicators) - { - _inheritMvIndicators = inheritMvIndicators; - } - - public String[] getMvIndicators() - { - return _mvIndicators; - } - - public void setMvIndicators(String[] mvIndicators) - { - _mvIndicators = mvIndicators; - } - - public String[] getMvLabels() - { - return _mvLabels; - } - - public void setMvLabels(String[] mvLabels) - { - _mvLabels = mvLabels; - } - } - - @RequiresPermission(AdminPermission.class) - public static class MissingValuesAction extends FolderManagementViewPostAction - { - @Override - protected JspView getTabView(MissingValuesForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/mvIndicators.jsp", form, errors); - } - - @Override - public void validateCommand(MissingValuesForm form, Errors errors) - { - } - - @Override - public boolean handlePost(MissingValuesForm form, BindException errors) - { - if (form.isInheritMvIndicators()) - { - MvUtil.inheritMvIndicators(getContainer()); - return true; - } - else - { - // Javascript should have enforced any constraints - MvUtil.assignMvIndicators(getContainer(), form.getMvIndicators(), form.getMvLabels()); - return true; - } - } - } - - @SuppressWarnings("unused") - public static class RConfigForm - { - private Integer _reportEngine; - private Integer _pipelineEngine; - private boolean _overrideDefault; - - public Integer getReportEngine() - { - return _reportEngine; - } - - public void setReportEngine(Integer reportEngine) - { - _reportEngine = reportEngine; - } - - public Integer getPipelineEngine() - { - return _pipelineEngine; - } - - public void setPipelineEngine(Integer pipelineEngine) - { - _pipelineEngine = pipelineEngine; - } - - public boolean getOverrideDefault() - { - return _overrideDefault; - } - - public void setOverrideDefault(String overrideDefault) - { - _overrideDefault = "override".equals(overrideDefault); - } - } - - @RequiresPermission(AdminPermission.class) - public static class RConfigurationAction extends FolderManagementViewPostAction - { - @Override - protected HttpView getTabView(RConfigForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/rConfiguration.jsp", form, errors); - } - - @Override - public void validateCommand(RConfigForm form, Errors errors) - { - if (form.getOverrideDefault()) - { - if (form.getReportEngine() == null) - errors.reject(ERROR_MSG, "Please select a valid report engine configuration"); - if (form.getPipelineEngine() == null) - errors.reject(ERROR_MSG, "Please select a valid pipeline engine configuration"); - } - } - - @Override - public URLHelper getSuccessURL(RConfigForm rConfigForm) - { - return getContainer().getStartURL(getUser()); - } - - @Override - public boolean handlePost(RConfigForm rConfigForm, BindException errors) throws Exception - { - LabKeyScriptEngineManager mgr = LabKeyScriptEngineManager.get(); - if (null != mgr) - { - try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) - { - if (rConfigForm.getOverrideDefault()) - { - ExternalScriptEngineDefinition reportEngine = mgr.getEngineDefinition(rConfigForm.getReportEngine(), ExternalScriptEngineDefinition.Type.R); - ExternalScriptEngineDefinition pipelineEngine = mgr.getEngineDefinition(rConfigForm.getPipelineEngine(), ExternalScriptEngineDefinition.Type.R); - - if (reportEngine != null) - mgr.setEngineScope(getContainer(), reportEngine, LabKeyScriptEngineManager.EngineContext.report); - if (pipelineEngine != null) - mgr.setEngineScope(getContainer(), pipelineEngine, LabKeyScriptEngineManager.EngineContext.pipeline); - } - else - { - // need to clear the current scope (if any) - ExternalScriptEngineDefinition reportEngine = mgr.getScopedEngine(getContainer(), "r", LabKeyScriptEngineManager.EngineContext.report, false); - ExternalScriptEngineDefinition pipelineEngine = mgr.getScopedEngine(getContainer(), "r", LabKeyScriptEngineManager.EngineContext.pipeline, false); - - if (reportEngine != null) - mgr.removeEngineScope(getContainer(), reportEngine, LabKeyScriptEngineManager.EngineContext.report); - if (pipelineEngine != null) - mgr.removeEngineScope(getContainer(), pipelineEngine, LabKeyScriptEngineManager.EngineContext.pipeline); - } - transaction.commit(); - } - return true; - } - return false; - } - } - - @SuppressWarnings("unused") - public static class ExportFolderForm - { - private String[] _types; - private int _location; - private String _format = "new"; // As of 14.3, this is the only supported format. But leave in place for the future. - private String _exportType; - private boolean _includeSubfolders; - private PHI _exportPhiLevel; // Input: max level when viewing form - private boolean _shiftDates; - private boolean _alternateIds; - private boolean _maskClinic; - - public String[] getTypes() - { - return _types; - } - - public void setTypes(String[] types) - { - _types = types; - } - - public int getLocation() - { - return _location; - } - - public void setLocation(int location) - { - _location = location; - } - - public String getFormat() - { - return _format; - } - - public void setFormat(String format) - { - _format = format; - } - - public ExportType getExportType() - { - if ("study".equals(_exportType)) - return ExportType.STUDY; - else - return ExportType.ALL; - } - - public void setExportType(String exportType) - { - _exportType = exportType; - } - - public boolean isIncludeSubfolders() - { - return _includeSubfolders; - } - - public void setIncludeSubfolders(boolean includeSubfolders) - { - _includeSubfolders = includeSubfolders; - } - - public PHI getExportPhiLevel() - { - return null != _exportPhiLevel ? _exportPhiLevel : PHI.NotPHI; - } - - public void setExportPhiLevel(PHI exportPhiLevel) - { - _exportPhiLevel = exportPhiLevel; - } - - public boolean isShiftDates() - { - return _shiftDates; - } - - public void setShiftDates(boolean shiftDates) - { - _shiftDates = shiftDates; - } - - public boolean isAlternateIds() - { - return _alternateIds; - } - - public void setAlternateIds(boolean alternateIds) - { - _alternateIds = alternateIds; - } - - public boolean isMaskClinic() - { - return _maskClinic; - } - - public void setMaskClinic(boolean maskClinic) - { - _maskClinic = maskClinic; - } - } - - public enum ExportOption - { - PipelineRootAsFiles("file root as multiple files") - { - @Override - public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception - { - PipeRoot root = PipelineService.get().findPipelineRoot(container); - if (root == null || !root.isValid()) - { - throw new NotFoundException("No valid pipeline root found"); - } - else if (root.isCloudRoot()) - { - errors.reject(ERROR_MSG, "Cannot export as individual files when root is in the cloud"); - } - else - { - File exportDir = root.resolvePath(PipelineService.EXPORT_DIR); - try - { - writer.write(container, ctx, new FileSystemFile(exportDir)); - } - catch (ContainerException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - return urlProvider(PipelineUrls.class).urlBrowse(container); - } - return null; - } - }, - - PipelineRootAsZip("file root as a single zip file") - { - @Override - public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception - { - PipeRoot root = PipelineService.get().findPipelineRoot(container); - if (root == null || !root.isValid()) - { - throw new NotFoundException("No valid pipeline root found"); - } - Path exportDir = root.resolveToNioPath(PipelineService.EXPORT_DIR); - FileUtil.createDirectories(exportDir); - exportFolderToFile(exportDir, container, writer, ctx, errors); - return urlProvider(PipelineUrls.class).urlBrowse(container); - } - }, - DownloadAsZip("browser download as a zip file") - { - @Override - public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception - { - try - { - // Export to a temporary file first so exceptions are displayed by the standard error page, Issue #44152 - // Same pattern as ExportListArchiveAction - Path tempDir = FileUtil.getTempDirectory().toPath(); - Path tempZipFile = exportFolderToFile(tempDir, container, writer, ctx, errors); - - // No exceptions, so stream the resulting zip file to the browser and delete it - try (OutputStream os = ZipFile.getOutputStream(response, tempZipFile.getFileName().toString())) - { - Files.copy(tempZipFile, os); - } - finally - { - Files.delete(tempZipFile); - } - } - catch (ContainerException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - return null; - } - }; - - private final String _description; - - ExportOption(String description) - { - _description = description; - } - - public String getDescription() - { - return _description; - } - - public abstract ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception; - - Path exportFolderToFile(Path exportDir, Container container, FolderWriterImpl writer, FolderExportContext ctx, BindException errors) throws Exception - { - String filename = FileUtil.makeFileNameWithTimestamp(container.getName(), "folder.zip"); - - try (ZipFile zip = new ZipFile(exportDir, filename)) - { - writer.write(container, ctx, zip); - } - catch (Container.ContainerException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - - return exportDir.resolve(filename); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ExportFolderAction extends FolderManagementViewPostAction - { - private ActionURL _successURL = null; - - @Override - public ModelAndView getView(ExportFolderForm exportFolderForm, boolean reshow, BindException errors) throws Exception - { - // In export-to-browser do nothing (leave the export page in place). We just exported to the response, so - // rendering a view would throw. - return reshow && !errors.hasErrors() ? null : super.getView(exportFolderForm, reshow, errors); - } - - @Override - protected HttpView getTabView(ExportFolderForm form, boolean reshow, BindException errors) - { - form.setExportType(PageFlowUtil.filter(getViewContext().getActionURL().getParameter("exportType"))); - - ComplianceFolderSettings settings = ComplianceService.get().getFolderSettings(getContainer(), User.getAdminServiceUser()); - PhiColumnBehavior columnBehavior = null==settings ? PhiColumnBehavior.show : settings.getPhiColumnBehavior(); - PHI maxAllowedPhiForExport = PhiColumnBehavior.show == columnBehavior ? PHI.Restricted : ComplianceService.get().getMaxAllowedPhi(getContainer(), getUser()); - form.setExportPhiLevel(maxAllowedPhiForExport); - - return new JspView<>("/org/labkey/core/admin/exportFolder.jsp", form, errors); - } - - @Override - public void validateCommand(ExportFolderForm form, Errors errors) - { - } - - @Override - public boolean handlePost(ExportFolderForm form, BindException errors) throws Exception - { - Container container = getContainer(); - if (container.isRoot()) - { - throw new NotFoundException(); - } - - ExportOption exportOption = null; - if (form.getLocation() >= 0 && form.getLocation() < ExportOption.values().length) - { - exportOption = ExportOption.values()[form.getLocation()]; - } - if (exportOption == null) - { - throw new NotFoundException("Invalid export location: " + form.getLocation()); - } - ContainerManager.checkContainerValidity(container); - - FolderWriterImpl writer = new FolderWriterImpl(); - FolderExportContext ctx = new FolderExportContext(getUser(), container, PageFlowUtil.set(form.getTypes()), - form.getFormat(), form.isIncludeSubfolders(), form.getExportPhiLevel(), form.isShiftDates(), - form.isAlternateIds(), form.isMaskClinic(), new StaticLoggerGetter(FolderWriterImpl.LOG)); - - AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, container, "Folder export initiated to " + exportOption.getDescription() + " " + (form.isIncludeSubfolders() ? "including" : "excluding") + " subfolders."); - AuditLogService.get().addEvent(getUser(), event); - - _successURL = exportOption.initiateExport(container, errors, writer, ctx, getViewContext().getResponse()); - - return !errors.hasErrors(); - } - - @Override - public URLHelper getSuccessURL(ExportFolderForm exportFolderForm) - { - return _successURL; - } - } - - public static class ImportFolderForm - { - private boolean _createSharedDatasets; - private boolean _validateQueries; - private boolean _failForUndefinedVisits; - private boolean _advancedImportOptions; - private String _sourceTemplateFolder; - private String _sourceTemplateFolderId; - private String _origin; - - public boolean isCreateSharedDatasets() - { - return _createSharedDatasets; - } - - public void setCreateSharedDatasets(boolean createSharedDatasets) - { - _createSharedDatasets = createSharedDatasets; - } - - public boolean isValidateQueries() - { - return _validateQueries; - } - - public boolean isFailForUndefinedVisits() - { - return _failForUndefinedVisits; - } - - public void setFailForUndefinedVisits(boolean failForUndefinedVisits) - { - _failForUndefinedVisits = failForUndefinedVisits; - } - - public void setValidateQueries(boolean validateQueries) - { - _validateQueries = validateQueries; - } - - public boolean isAdvancedImportOptions() - { - return _advancedImportOptions; - } - - public void setAdvancedImportOptions(boolean advancedImportOptions) - { - _advancedImportOptions = advancedImportOptions; - } - - public String getSourceTemplateFolder() - { - return _sourceTemplateFolder; - } - - @SuppressWarnings("unused") - public void setSourceTemplateFolder(String sourceTemplateFolder) - { - _sourceTemplateFolder = sourceTemplateFolder; - } - - public String getSourceTemplateFolderId() - { - return _sourceTemplateFolderId; - } - - @SuppressWarnings("unused") - public void setSourceTemplateFolderId(String sourceTemplateFolderId) - { - _sourceTemplateFolderId = sourceTemplateFolderId; - } - - public String getOrigin() - { - return _origin; - } - - public void setOrigin(String origin) - { - _origin = origin; - } - - public Container getSourceTemplateFolderContainer() - { - if (null == getSourceTemplateFolderId()) - return null; - return ContainerManager.getForId(getSourceTemplateFolderId().replace(',', ' ').trim()); - } - } - - @RequiresPermission(AdminPermission.class) - public class ImportFolderAction extends FolderManagementViewPostAction - { - private ActionURL _successURL; - - @Override - protected HttpView getTabView(ImportFolderForm form, boolean reshow, BindException errors) - { - // default the createSharedDatasets and validateQueries to true if this is not a form error reshow - if (!errors.hasErrors()) - { - form.setCreateSharedDatasets(true); - form.setValidateQueries(true); - } - - return new JspView<>("/org/labkey/core/admin/importFolder.jsp", form, errors); - } - - @Override - public void validateCommand(ImportFolderForm form, Errors errors) - { - // don't allow import into the root container - if (getContainer().isRoot()) - { - throw new NotFoundException(); - } - } - - @Override - public boolean handlePost(ImportFolderForm form, BindException errors) throws Exception - { - ViewContext context = getViewContext(); - ActionURL url = context.getActionURL(); - User user = getUser(); - Container container = getContainer(); - PipeRoot pipelineRoot; - Path pipelineUnzipDir; // Should be local & writable - PipelineUrls pipelineUrlProvider; - - if (form.getOrigin() == null) - { - form.setOrigin("Folder"); - } - - // make sure we have a pipeline url provider to use for the success URL redirect - pipelineUrlProvider = urlProvider(PipelineUrls.class); - if (pipelineUrlProvider == null) - { - errors.reject("folderImport", "Pipeline url provider does not exist."); - return false; - } - - // make sure that the pipeline root is valid for this container - pipelineRoot = PipelineService.get().findPipelineRoot(container); - if (!PipelineService.get().hasValidPipelineRoot(container) || pipelineRoot == null) - { - errors.reject("folderImport", "Pipeline root not set or does not exist on disk."); - return false; - } - - // make sure we are able to delete any existing unzip dir in the pipeline root - try - { - pipelineUnzipDir = pipelineRoot.deleteImportDirectory(null); - } - catch (DirectoryNotDeletedException e) - { - errors.reject("studyImport", "Import failed: Could not delete the directory \"" + PipelineService.UNZIP_DIR + "\""); - return false; - } - - FolderImportConfig fiConfig; - if (!StringUtils.isEmpty(form.getSourceTemplateFolder())) - { - fiConfig = getFolderImportConfigFromTemplateFolder(form, pipelineUnzipDir, errors); - } - else - { - fiConfig = getFolderFromZipArchive(pipelineUnzipDir, errors); - if (fiConfig == null || errors.hasErrors()) - { - return false; - } - } - - // get the folder.xml file from the unzipped import archive - Path archiveXml = pipelineUnzipDir.resolve("folder.xml"); - if (!Files.exists(archiveXml)) - { - errors.reject("folderImport", "This archive doesn't contain a folder.xml file."); - return false; - } - - ImportOptions options = new ImportOptions(getContainer().getId(), user.getUserId()); - options.setSkipQueryValidation(!form.isValidateQueries()); - options.setCreateSharedDatasets(form.isCreateSharedDatasets()); - options.setFailForUndefinedVisits(form.isFailForUndefinedVisits()); - options.setAdvancedImportOptions(form.isAdvancedImportOptions()); - options.setActivity(ComplianceService.get().getCurrentActivity(getViewContext())); - - // if the option is selected to show the advanced import options, redirect to there - if (form.isAdvancedImportOptions()) - { - // archiveFile is the zip of the source template folder located in the current container's unzip dir - _successURL = pipelineUrlProvider.urlStartFolderImport(getContainer(), fiConfig.archiveFile, options, fiConfig.fromTemplateSourceFolder); - return true; - } - - // finally, create the study or folder import pipeline job - _successURL = pipelineUrlProvider.urlBegin(container); - PipelineService.get().runFolderImportJob(container, user, url, archiveXml, fiConfig.originalFileName, pipelineRoot, options); - - return !errors.hasErrors(); - } - - private @Nullable FolderImportConfig getFolderFromZipArchive(Path pipelineUnzipDir, BindException errors) - { - // user chose to import from a zip file - Map map = getFileMap(); - - // make sure we have a single file selected for import - if (map.size() != 1) - { - errors.reject("folderImport", "You must select a valid zip archive (folder or study)."); - return null; - } - - // make sure the file is not empty and that it has a .zip extension - MultipartFile zipFile = map.values().iterator().next(); - if (0 == zipFile.getSize() || isBlank(zipFile.getOriginalFilename()) || !zipFile.getOriginalFilename().toLowerCase().endsWith(".zip")) - { - errors.reject("folderImport", "You must select a valid zip archive (folder or study)."); - return null; - } - - // copy and unzip the uploaded import archive zip file to the pipeline unzip dir - try - { - Path pipelineUnzipFile = pipelineUnzipDir.resolve(zipFile.getOriginalFilename()); - FileUtil.createDirectories(pipelineUnzipFile.getParent()); // Non-pipeline import sometimes fails here on Windows (shrug) - FileUtil.createFile(pipelineUnzipFile); - try (OutputStream os = Files.newOutputStream(pipelineUnzipFile)) - { - FileUtil.copyData(zipFile.getInputStream(), os); - } - ZipUtil.unzipToDirectory(pipelineUnzipFile, pipelineUnzipDir); - - return new FolderImportConfig( - false, - zipFile.getOriginalFilename(), - pipelineUnzipFile, - pipelineUnzipFile - ); - } - catch (FileNotFoundException e) - { - LOG.debug("Failed to import '" + zipFile.getOriginalFilename() + "'.", e); - errors.reject("folderImport", "File not found."); - return null; - } - catch (IOException e) - { - LOG.debug("Failed to import '" + zipFile.getOriginalFilename() + "'.", e); - errors.reject("folderImport", "Unable to unzip folder archive."); - return null; - } - } - - private FolderImportConfig getFolderImportConfigFromTemplateFolder(final ImportFolderForm form, final Path pipelineUnzipDir, final BindException errors) throws Exception - { - // user choose to import from a template source folder - Container sourceContainer = form.getSourceTemplateFolderContainer(); - - // In order to support the Advanced import options to import into multiple target folders we need to zip - // the source template folder so that the zip file can be passed to the pipeline processes. - FolderExportContext ctx = new FolderExportContext(getUser(), sourceContainer, - getRegisteredFolderWritersForImplicitExport(sourceContainer), "new", false, - PHI.NotPHI, false, false, false, new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); - FolderWriterImpl writer = new FolderWriterImpl(); - String zipFileName = FileUtil.makeFileNameWithTimestamp(sourceContainer.getName(), "folder.zip"); - try (ZipFile zip = new ZipFile(pipelineUnzipDir, zipFileName)) - { - writer.write(sourceContainer, ctx, zip); - } - catch (Container.ContainerException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - Path implicitZipFile = pipelineUnzipDir.resolve(zipFileName); - - // To support the simple import option unzip the zip file to the pipeline unzip dir of the current container - ZipUtil.unzipToDirectory(implicitZipFile, pipelineUnzipDir); - - return new FolderImportConfig( - StringUtils.isNotEmpty(form.getSourceTemplateFolderId()), - implicitZipFile.getFileName().toString(), - implicitZipFile, - null - ); - } - - private static class FolderImportConfig { - Path pipelineUnzipFile; - String originalFileName; - Path archiveFile; - boolean fromTemplateSourceFolder; - - public FolderImportConfig(boolean fromTemplateSourceFolder, String originalFileName, Path archiveFile, @Nullable Path pipelineUnzipFile) - { - this.originalFileName = originalFileName; - this.archiveFile = archiveFile; - this.fromTemplateSourceFolder = fromTemplateSourceFolder; - this.pipelineUnzipFile = pipelineUnzipFile; - } - } - - @Override - public URLHelper getSuccessURL(ImportFolderForm importFolderForm) - { - return _successURL; - } - } - - private Set getRegisteredFolderWritersForImplicitExport(Container sourceContainer) - { - // this method is very similar to CoreController.GetRegisteredFolderWritersAction.execute() method, but instead of - // of building up a map of Writer object names to display in the UI, we are instead adding them to the list of Writers - // to apply during the implicit export. - Set registeredFolderWriters = new HashSet<>(); - FolderSerializationRegistry registry = FolderSerializationRegistry.get(); - if (null == registry) - { - throw new RuntimeException(); - } - Collection registeredWriters = registry.getRegisteredFolderWriters(); - for (FolderWriter writer : registeredWriters) - { - String dataType = writer.getDataType(); - boolean excludeForDataspace = sourceContainer.isDataspace() && "Study".equals(dataType); - boolean excludeForTemplate = !writer.includeWithTemplate(); - - if (dataType != null && writer.show(sourceContainer) && !excludeForDataspace && !excludeForTemplate) - { - registeredFolderWriters.add(dataType); - - // for each Writer also determine if there are related children Writers, if so include them also - Collection> childWriters = writer.getChildren(true, true); - if (!childWriters.isEmpty()) - { - for (org.labkey.api.writer.Writer child : childWriters) - { - dataType = child.getDataType(); - if (dataType != null) - registeredFolderWriters.add(dataType); - } - } - } - } - return registeredFolderWriters; - } - - public static class FolderSettingsForm - { - private String _defaultDateFormat; - private boolean _defaultDateFormatInherited; - private String _defaultDateTimeFormat; - private boolean _defaultDateTimeFormatInherited; - private String _defaultTimeFormat; - private boolean _defaultTimeFormatInherited; - private String _defaultNumberFormat; - private boolean _defaultNumberFormatInherited; - private boolean _restrictedColumnsEnabled; - private boolean _restrictedColumnsEnabledInherited; - - public String getDefaultDateFormat() - { - return _defaultDateFormat; - } - - @SuppressWarnings("unused") - public void setDefaultDateFormat(String defaultDateFormat) - { - _defaultDateFormat = defaultDateFormat; - } - - public boolean isDefaultDateFormatInherited() - { - return _defaultDateFormatInherited; - } - - @SuppressWarnings("unused") - public void setDefaultDateFormatInherited(boolean defaultDateFormatInherited) - { - _defaultDateFormatInherited = defaultDateFormatInherited; - } - - public String getDefaultDateTimeFormat() - { - return _defaultDateTimeFormat; - } - - @SuppressWarnings("unused") - public void setDefaultDateTimeFormat(String defaultDateTimeFormat) - { - _defaultDateTimeFormat = defaultDateTimeFormat; - } - - public boolean isDefaultDateTimeFormatInherited() - { - return _defaultDateTimeFormatInherited; - } - - @SuppressWarnings("unused") - public void setDefaultDateTimeFormatInherited(boolean defaultDateTimeFormatInherited) - { - _defaultDateTimeFormatInherited = defaultDateTimeFormatInherited; - } - - public String getDefaultTimeFormat() - { - return _defaultTimeFormat; - } - - @SuppressWarnings("UnusedDeclaration") - public void setDefaultTimeFormat(String defaultTimeFormat) - { - _defaultTimeFormat = defaultTimeFormat; - } - - public boolean isDefaultTimeFormatInherited() - { - return _defaultTimeFormatInherited; - } - - @SuppressWarnings("unused") - public void setDefaultTimeFormatInherited(boolean defaultTimeFormatInherited) - { - _defaultTimeFormatInherited = defaultTimeFormatInherited; - } - - public String getDefaultNumberFormat() - { - return _defaultNumberFormat; - } - - @SuppressWarnings("unused") - public void setDefaultNumberFormat(String defaultNumberFormat) - { - _defaultNumberFormat = defaultNumberFormat; - } - - public boolean isDefaultNumberFormatInherited() - { - return _defaultNumberFormatInherited; - } - - @SuppressWarnings("unused") - public void setDefaultNumberFormatInherited(boolean defaultNumberFormatInherited) - { - _defaultNumberFormatInherited = defaultNumberFormatInherited; - } - - public boolean areRestrictedColumnsEnabled() - { - return _restrictedColumnsEnabled; - } - - @SuppressWarnings("unused") - public void setRestrictedColumnsEnabled(boolean restrictedColumnsEnabled) - { - _restrictedColumnsEnabled = restrictedColumnsEnabled; - } - - public boolean isRestrictedColumnsEnabledInherited() - { - return _restrictedColumnsEnabledInherited; - } - - @SuppressWarnings("unused") - public void setRestrictedColumnsEnabledInherited(boolean restrictedColumnsEnabledInherited) - { - _restrictedColumnsEnabledInherited = restrictedColumnsEnabledInherited; - } - } - - @RequiresPermission(AdminPermission.class) - public static class FolderSettingsAction extends FolderManagementViewPostAction - { - @Override - protected LookAndFeelView getTabView(FolderSettingsForm form, boolean reshow, BindException errors) - { - return new LookAndFeelView(errors); - } - - @Override - public void validateCommand(FolderSettingsForm form, Errors errors) - { - } - - @Override - public boolean handlePost(FolderSettingsForm form, BindException errors) - { - return saveFolderSettings(getContainer(), getUser(), LookAndFeelProperties.getWriteableFolderInstance(getContainer()), form, errors); - } - } - - // Validate and populate the folder settings; save & log all changes - private static boolean saveFolderSettings(Container c, User user, WriteableFolderLookAndFeelProperties props, FolderSettingsForm form, BindException errors) - { - validateAndSaveFormat(form.getDefaultDateFormat(), form.isDefaultDateFormatInherited(), props::clearDefaultDateFormat, props::setDefaultDateFormat, errors, "date display format"); - validateAndSaveFormat(form.getDefaultDateTimeFormat(), form.isDefaultDateTimeFormatInherited(), props::clearDefaultDateTimeFormat, props::setDefaultDateTimeFormat, errors, "date-time display format"); - validateAndSaveFormat(form.getDefaultTimeFormat(), form.isDefaultTimeFormatInherited(), props::clearDefaultTimeFormat, props::setDefaultTimeFormat, errors, "time display format"); - validateAndSaveFormat(form.getDefaultNumberFormat(), form.isDefaultNumberFormatInherited(), props::clearDefaultNumberFormat, props::setDefaultNumberFormat, errors, "number display format"); - - setProperty(form.isRestrictedColumnsEnabledInherited(), props::clearRestrictedColumnsEnabled, () -> props.setRestrictedColumnsEnabled(form.areRestrictedColumnsEnabled())); - - if (!errors.hasErrors()) - { - props.save(); - - //write an audit log event - props.writeAuditLogEvent(c, user); - } - - return !errors.hasErrors(); - } - - private interface FormatSaver - { - void save(String format) throws IllegalArgumentException; - } - - private static void validateAndSaveFormat(String format, boolean inherited, Runnable clearer, FormatSaver saver, BindException errors, String what) - { - String defaultFormat = StringUtils.trimToNull(format); - if (inherited) - { - clearer.run(); - } - else - { - try - { - saver.save(defaultFormat); - } - catch (IllegalArgumentException e) - { - errors.reject(ERROR_MSG, "Invalid " + what + ": " + e.getMessage()); - } - } - } - - @RequiresPermission(AdminPermission.class) - public static class ModulePropertiesAction extends FolderManagementViewAction - { - @Override - protected JspView getTabView() - { - return new JspView<>("/org/labkey/core/project/modulePropertiesAdmin.jsp"); - } - } - - @SuppressWarnings("unused") - public static class FolderTypeForm - { - private String[] _activeModules = new String[ModuleLoader.getInstance().getModules().size()]; - private String _defaultModule; - private String _folderType; - private boolean _wizard; - - public String[] getActiveModules() - { - return _activeModules; - } - - public void setActiveModules(String[] activeModules) - { - _activeModules = activeModules; - } - - public String getDefaultModule() - { - return _defaultModule; - } - - public void setDefaultModule(String defaultModule) - { - _defaultModule = defaultModule; - } - - public String getFolderType() - { - return _folderType; - } - - public void setFolderType(String folderType) - { - _folderType = folderType; - } - - public boolean isWizard() - { - return _wizard; - } - - public void setWizard(boolean wizard) - { - _wizard = wizard; - } - } - - @RequiresPermission(AdminPermission.class) - @IgnoresTermsOfUse // At the moment, compliance configuration is very sensitive to active modules, so allow those adjustments - public static class FolderTypeAction extends FolderManagementViewPostAction - { - private ActionURL _successURL = null; - - @Override - protected JspView getTabView(FolderTypeForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/folderType.jsp", form, errors); - } - - @Override - public void validateCommand(FolderTypeForm form, Errors errors) - { - boolean fEmpty = true; - for (String module : form._activeModules) - { - if (module != null) - { - fEmpty = false; - break; - } - } - if (fEmpty && "None".equals(form.getFolderType())) - { - errors.reject(SpringActionController.ERROR_MSG, "Error: Please select at least one module to display."); - } - } - - @Override - public boolean handlePost(FolderTypeForm form, BindException errors) - { - Container container = getContainer(); - if (container.isRoot()) - { - throw new NotFoundException(); - } - - String[] modules = form.getActiveModules(); - - if (modules.length == 0) - { - errors.reject(null, "At least one module must be selected"); - return false; - } - - Set activeModules = new HashSet<>(); - for (String moduleName : modules) - { - Module module = ModuleLoader.getInstance().getModule(moduleName); - if (module != null) - activeModules.add(module); - } - - if (null == StringUtils.trimToNull(form.getFolderType()) || FolderType.NONE.getName().equals(form.getFolderType())) - { - container.setFolderType(FolderType.NONE, getUser(), errors, activeModules); - Module defaultModule = ModuleLoader.getInstance().getModule(form.getDefaultModule()); - container.setDefaultModule(defaultModule); - } - else - { - FolderType folderType = FolderTypeManager.get().getFolderType(form.getFolderType()); - if (container.isContainerTab() && folderType.hasContainerTabs()) - errors.reject(null, "You cannot set a tab folder to a folder type that also has tab folders"); - else - container.setFolderType(folderType, getUser(), errors, activeModules); - } - if (errors.hasErrors()) - return false; - - if (form.isWizard()) - { - _successURL = urlProvider(SecurityUrls.class).getContainerURL(container); - _successURL.addParameter("wizard", Boolean.TRUE.toString()); - } - else - _successURL = container.getFolderType().getStartURL(container, getUser()); - - return true; - } - - @Override - public URLHelper getSuccessURL(FolderTypeForm folderTypeForm) - { - return _successURL; - } - } - - @SuppressWarnings("unused") - public static class FileRootsForm extends SetupForm implements FileManagementForm - { - private String _folderRootPath; - private String _fileRootOption; - private String _cloudRootName; - private boolean _isFolderSetup; - private boolean _fileRootChanged; - private boolean _enabledCloudStoresChanged; - private String _migrateFilesOption; - - // cloud settings - private String[] _enabledCloudStore; - //file management - @Override - public String getFolderRootPath() - { - return _folderRootPath; - } - - @Override - public void setFolderRootPath(String folderRootPath) - { - _folderRootPath = folderRootPath; - } - - @Override - public String getFileRootOption() - { - return _fileRootOption; - } - - @Override - public void setFileRootOption(String fileRootOption) - { - _fileRootOption = fileRootOption; - } - - @Override - public String[] getEnabledCloudStore() - { - return _enabledCloudStore; - } - - @Override - public void setEnabledCloudStore(String[] enabledCloudStore) - { - _enabledCloudStore = enabledCloudStore; - } - - @Override - public boolean isDisableFileSharing() - { - return FileRootProp.disable.name().equals(getFileRootOption()); - } - - @Override - public boolean hasSiteDefaultRoot() - { - return FileRootProp.siteDefault.name().equals(getFileRootOption()); - } - - @Override - public boolean isCloudFileRoot() - { - return FileRootProp.cloudRoot.name().equals(getFileRootOption()); - } - - @Override - @Nullable - public String getCloudRootName() - { - return _cloudRootName; - } - - @Override - public void setCloudRootName(String cloudRootName) - { - _cloudRootName = cloudRootName; - } - - @Override - public boolean isFolderSetup() - { - return _isFolderSetup; - } - - public void setFolderSetup(boolean folderSetup) - { - _isFolderSetup = folderSetup; - } - - public boolean isFileRootChanged() - { - return _fileRootChanged; - } - - @Override - public void setFileRootChanged(boolean changed) - { - _fileRootChanged = changed; - } - - public boolean isEnabledCloudStoresChanged() - { - return _enabledCloudStoresChanged; - } - - @Override - public void setEnabledCloudStoresChanged(boolean enabledCloudStoresChanged) - { - _enabledCloudStoresChanged = enabledCloudStoresChanged; - } - - @Override - public String getMigrateFilesOption() - { - return _migrateFilesOption; - } - - @Override - public void setMigrateFilesOption(String migrateFilesOption) - { - _migrateFilesOption = migrateFilesOption; - } - } - - @RequiresPermission(AdminPermission.class) - public class FileRootsStandAloneAction extends FormViewAction - { - @Override - public ModelAndView getView(FileRootsForm form, boolean reShow, BindException errors) - { - JspView view = getFileRootsView(form, errors, getReshow()); - view.setFrame(WebPartView.FrameType.NONE); - - getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(getContainer(), getContainer().getParent())); - getPageConfig().setTemplate(PageConfig.Template.Wizard); - getPageConfig().setTitle("Change File Root"); - return view; - } - - @Override - public void validateCommand(FileRootsForm form, Errors errors) - { - validateCloudFileRoot(form, getContainer(), errors); - } - - @Override - public boolean handlePost(FileRootsForm form, BindException errors) throws Exception - { - return handleFileRootsPost(form, errors); - } - - @Override - public ActionURL getSuccessURL(FileRootsForm form) - { - ActionURL url = new ActionURL(FileRootsStandAloneAction.class, getContainer()) - .addParameter("folderSetup", true) - .addReturnUrl(getViewContext().getActionURL().getReturnUrl()); - - if (form.isFileRootChanged()) - url.addParameter("rootSet", form.getMigrateFilesOption()); - if (form.isEnabledCloudStoresChanged()) - url.addParameter("cloudChanged", true); - return url; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - /** - * This standalone file root management action can be used on folder types that do not support - * the normal 'Manage Folder' UI. Not currently linked in the UI, but available for direct URL - * navigation when a workbook needs it. - */ - @RequiresPermission(AdminPermission.class) - public class ManageFileRootAction extends FormViewAction - { - @Override - public ModelAndView getView(FileRootsForm form, boolean reShow, BindException errors) - { - JspView view = getFileRootsView(form, errors, getReshow()); - getPageConfig().setTitle("Manage File Root"); - return view; - } - - @Override - public void validateCommand(FileRootsForm form, Errors errors) - { - validateCloudFileRoot(form, getContainer(), errors); - } - - @Override - public boolean handlePost(FileRootsForm form, BindException errors) throws Exception - { - return handleFileRootsPost(form, errors); - } - - @Override - public ActionURL getSuccessURL(FileRootsForm form) - { - ActionURL url = getContainer().getStartURL(getUser()); - - if (getViewContext().getActionURL().getReturnUrl() != null) - { - url.addReturnUrl(getViewContext().getActionURL().getReturnUrl()); - } - - return url; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresPermission(AdminPermission.class) - public class FileRootsAction extends FolderManagementViewPostAction - { - @Override - protected HttpView getTabView(FileRootsForm form, boolean reshow, BindException errors) - { - return getFileRootsView(form, errors, getReshow()); - } - - @Override - public void validateCommand(FileRootsForm form, Errors errors) - { - validateCloudFileRoot(form, getContainer(), errors); - } - - @Override - public boolean handlePost(FileRootsForm form, BindException errors) throws Exception - { - return handleFileRootsPost(form, errors); - } - - @Override - public ActionURL getSuccessURL(FileRootsForm form) - { - ActionURL url = new AdminController.AdminUrlsImpl().getFileRootsURL(getContainer()); - - if (form.isFileRootChanged()) - url.addParameter("rootSet", form.getMigrateFilesOption()); - if (form.isEnabledCloudStoresChanged()) - url.addParameter("cloudChanged", true); - return url; - } - } - - private JspView getFileRootsView(FileRootsForm form, BindException errors, boolean reshow) - { - JspView view = new JspView<>("/org/labkey/core/admin/view/filesProjectSettings.jsp", form, errors); - String title = "Configure File Root"; - if (CloudStoreService.get() != null) - title += " And Enable Cloud Stores"; - view.setTitle(title); - view.setFrame(WebPartView.FrameType.DIV); - try - { - if (!reshow) - setFormAndConfirmMessage(getViewContext(), form); - } - catch (IllegalArgumentException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - - return view; - } - - private boolean handleFileRootsPost(FileRootsForm form, BindException errors) throws Exception - { - if (form.isPipelineRootForm()) - { - return PipelineService.get().savePipelineSetup(getViewContext(), form, errors); - } - else - { - setFileRootFromForm(getViewContext(), form, errors); - setEnabledCloudStores(getViewContext(), form, errors); - return !errors.hasErrors(); - } - } - - public static void validateCloudFileRoot(FileManagementForm form, Container container, Errors errors) - { - FileContentService service = FileContentService.get(); - if (null != service) - { - boolean isOrDefaultsToCloudRoot = form.isCloudFileRoot(); - String cloudRootName = form.getCloudRootName(); - if (!isOrDefaultsToCloudRoot && form.hasSiteDefaultRoot()) - { - Path defaultRootPath = service.getDefaultRootPath(container, false); - cloudRootName = service.getDefaultRootInfo(container).getCloudName(); - isOrDefaultsToCloudRoot = (null != defaultRootPath && FileUtil.hasCloudScheme(defaultRootPath)); - } - - if (isOrDefaultsToCloudRoot && null != cloudRootName) - { - if (null != form.getEnabledCloudStore()) - { - for (String storeName : form.getEnabledCloudStore()) - { - if (Strings.CI.equals(cloudRootName, storeName)) - return; - } - } - // Didn't find cloud root in enabled list - errors.reject(ERROR_MSG, "Cannot disable cloud store used as File Root."); - } - } - } - - public static void setFileRootFromForm(ViewContext ctx, FileManagementForm form, BindException errors) - { - boolean changed = false; - boolean shouldCopyMove = false; - FileContentService service = FileContentService.get(); - if (null != service) - { - // If we need to copy/move files based on the FileRoot change, we need to check children that use the default and move them, too. - // And we need to capture the source roots for each of those, because changing this parent file root changes the child source roots. - MigrateFilesOption migrateFilesOption = null != form.getMigrateFilesOption() ? - MigrateFilesOption.valueOf(form.getMigrateFilesOption()) : - MigrateFilesOption.leave; - List> sourceInfos = - ((MigrateFilesOption.leave.equals(migrateFilesOption) && !form.isFolderSetup()) || form.isDisableFileSharing()) ? - Collections.emptyList() : - getCopySourceInfo(service, ctx.getContainer()); - - if (form.isDisableFileSharing()) - { - if (!service.isFileRootDisabled(ctx.getContainer())) - { - service.disableFileRoot(ctx.getContainer()); - changed = true; - } - } - else if (form.hasSiteDefaultRoot()) - { - if (service.isFileRootDisabled(ctx.getContainer()) || !service.isUseDefaultRoot(ctx.getContainer())) - { - service.setIsUseDefaultRoot(ctx.getContainer(), true); - changed = true; - shouldCopyMove = true; - } - } - else if (form.isCloudFileRoot()) - { - throwIfUnauthorizedFileRootChange(ctx, service, form); - String cloudRootName = form.getCloudRootName(); - if (null != cloudRootName && - (!service.isCloudRoot(ctx.getContainer()) || - !cloudRootName.equalsIgnoreCase(service.getCloudRootName(ctx.getContainer())))) - { - service.setIsUseDefaultRoot(ctx.getContainer(), false); - service.setCloudRoot(ctx.getContainer(), cloudRootName); - try - { - PipelineService.get().setPipelineRoot(ctx.getUser(), ctx.getContainer(), PipelineService.PRIMARY_ROOT, false); - if (form.isFolderSetup() && !sourceInfos.isEmpty()) - { - // File root was set to cloud storage, remove folder created - Path fromPath = FileUtil.stringToPath(sourceInfos.get(0).first, sourceInfos.get(0).second); // sourceInfos paths should be encoded - if (FileContentService.FILES_LINK.equals(FileUtil.getFileName(fromPath))) - { - try - { - Files.deleteIfExists(fromPath.getParent()); - } - catch (IOException e) - { - LOG.warn("Could not delete directory '" + FileUtil.pathToString(fromPath.getParent()) + "'"); - } - } - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - changed = true; - shouldCopyMove = true; - } - } - else - { - throwIfUnauthorizedFileRootChange(ctx, service, form); - String root = StringUtils.trimToNull(form.getFolderRootPath()); - if (root != null) - { - URI uri = FileUtil.createUri(root, false); // root is unencoded - Path path = FileUtil.getPath(ctx.getContainer(), uri); - if (null == path || !Files.exists(path)) - { - errors.reject(ERROR_MSG, "File root '" + root + "' does not appear to be a valid directory accessible to the server at " + ctx.getRequest().getServerName() + "."); - } - else - { - Path currentFileRootPath = service.getFileRootPath(ctx.getContainer()); - if (null == currentFileRootPath || !root.equalsIgnoreCase(currentFileRootPath.toAbsolutePath().toString())) - { - service.setIsUseDefaultRoot(ctx.getContainer(), false); - service.setFileRootPath(ctx.getContainer(), root); - changed = true; - shouldCopyMove = true; - } - } - } - else - { - service.setFileRootPath(ctx.getContainer(), null); - changed = true; - } - } - - if (!errors.hasErrors()) - { - if (changed && shouldCopyMove && !MigrateFilesOption.leave.equals(migrateFilesOption)) - { - // Make sure we have pipeRoot before starting jobs, even though each subfolder needs to get its own - PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(ctx.getContainer()); - if (null != pipeRoot) - { - try - { - initiateCopyFilesPipelineJobs(ctx, sourceInfos, pipeRoot, migrateFilesOption); - } - catch (PipelineValidationException e) - { - throw new RuntimeValidationException(e); - } - } - else - { - LOG.warn("Change File Root: Can't copy or move files with no pipeline root"); - } - } - - form.setFileRootChanged(changed); - if (changed && null != ctx.getUser()) - { - setFormAndConfirmMessage(ctx.getContainer(), form, true, false, migrateFilesOption.name()); - String comment = (ctx.getContainer().isProject() ? "Project " : "Folder ") + ctx.getContainer().getPath() + ": " + form.getConfirmMessage(); - AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, ctx.getContainer(), comment); - AuditLogService.get().addEvent(ctx.getUser(), event); - } - } - } - } - - private static List> getCopySourceInfo(FileContentService service, Container container) - { - - List> sourceInfo = new ArrayList<>(); - addCopySourceInfo(service, container, sourceInfo, true); - return sourceInfo; - } - - private static void addCopySourceInfo(FileContentService service, Container container, List> sourceInfo, boolean isRoot) - { - if (isRoot || service.isUseDefaultRoot(container)) - { - Path sourceFileRootDir = service.getFileRootPath(container, FileContentService.ContentType.files); - if (null != sourceFileRootDir) - { - String pathStr = FileUtil.pathToString(sourceFileRootDir); - if (null != pathStr) - sourceInfo.add(new Pair<>(container, pathStr)); - else - throw new RuntimeValidationException("Unexpected error converting path to string"); - } - } - for (Container childContainer : container.getChildren()) - addCopySourceInfo(service, childContainer, sourceInfo, false); - } - - private static void initiateCopyFilesPipelineJobs(ViewContext ctx, @NotNull List> sourceInfos, PipeRoot pipeRoot, - MigrateFilesOption migrateFilesOption) throws PipelineValidationException - { - CopyFileRootPipelineJob job = new CopyFileRootPipelineJob(ctx.getContainer(), ctx.getUser(), sourceInfos, pipeRoot, migrateFilesOption); - PipelineService.get().queueJob(job); - } - - private static void throwIfUnauthorizedFileRootChange(ViewContext ctx, FileContentService service, FileManagementForm form) - { - // test permissions. only site admins are able to turn on a custom file root for a folder - // this is only relevant if the folder is either being switched to a custom file root, - // or if the file root is changed. - if (!service.isUseDefaultRoot(ctx.getContainer())) - { - Path fileRootPath = service.getFileRootPath(ctx.getContainer()); - if (null != fileRootPath) - { - String absolutePath = FileUtil.getAbsolutePath(ctx.getContainer(), fileRootPath); - if (Strings.CI.equals(absolutePath, form.getFolderRootPath())) - { - if (!ctx.getUser().hasRootPermission(AdminOperationsPermission.class)) - throw new UnauthorizedException("Only site admins can change file roots"); - } - } - } - } - - public static void setEnabledCloudStores(ViewContext ctx, FileManagementForm form, BindException errors) - { - String[] enabledCloudStores = form.getEnabledCloudStore(); - CloudStoreService cloud = CloudStoreService.get(); - if (cloud != null) - { - Set enabled = Collections.emptySet(); - if (enabledCloudStores != null) - enabled = new HashSet<>(Arrays.asList(enabledCloudStores)); - - try - { - // Check if anything changed - boolean changed = false; - Collection storeNames = cloud.getEnabledCloudStores(ctx.getContainer()); - if (enabled.size() != storeNames.size()) - changed = true; - else - if (!enabled.containsAll(storeNames)) - changed = true; - if (changed) - cloud.setEnabledCloudStores(ctx.getContainer(), enabled); - form.setEnabledCloudStoresChanged(changed); - } - catch (UncheckedExecutionException e) - { - LOG.debug("Failed to configure cloud store(s).", e); - // UncheckedExecutionException with cause org.jclouds.blobstore.ContainerNotFoundException - // is what BlobStore hands us if bucket (S3 container) does not exist - if (null != e.getCause()) - errors.reject(ERROR_MSG, e.getCause().getMessage()); - else - throw e; - } - catch (RuntimeException e) - { - LOG.debug("Failed to configure cloud store(s).", e); - errors.reject(ERROR_MSG, e.getMessage()); - } - } - } - - - public static void setFormAndConfirmMessage(ViewContext ctx, FileManagementForm form) throws IllegalArgumentException - { - String rootSetParam = ctx.getActionURL().getParameter("rootSet"); - boolean fileRootChanged = null != rootSetParam && !"false".equalsIgnoreCase(rootSetParam); - String cloudChangedParam = ctx.getActionURL().getParameter("cloudChanged"); - boolean enabledCloudChanged = "true".equalsIgnoreCase(cloudChangedParam); - setFormAndConfirmMessage(ctx.getContainer(), form, fileRootChanged, enabledCloudChanged, rootSetParam); - } - - public static void setFormAndConfirmMessage(Container container, FileManagementForm form, boolean fileRootChanged, boolean enabledCloudChanged, - String migrateFilesOption) throws IllegalArgumentException - { - FileContentService service = FileContentService.get(); - String confirmMessage = null; - - String migrateFilesMessage = ""; - if (fileRootChanged && !form.isFolderSetup()) - { - if (MigrateFilesOption.leave.name().equals(migrateFilesOption)) - migrateFilesMessage = ". Existing files not copied or moved."; - else if (MigrateFilesOption.copy.name().equals(migrateFilesOption)) - { - migrateFilesMessage = ". Existing files copied."; - form.setMigrateFilesOption(migrateFilesOption); - } - else if (MigrateFilesOption.move.name().equals(migrateFilesOption)) - { - migrateFilesMessage = ". Existing files moved."; - form.setMigrateFilesOption(migrateFilesOption); - } - } - - if (service != null) - { - if (service.isFileRootDisabled(container)) - { - form.setFileRootOption(FileRootProp.disable.name()); - if (fileRootChanged) - confirmMessage = "File sharing has been disabled for this " + container.getContainerNoun(); - } - else if (service.isUseDefaultRoot(container)) - { - form.setFileRootOption(FileRootProp.siteDefault.name()); - Path root = service.getFileRootPath(container); - if (root != null && Files.exists(root) && fileRootChanged) - confirmMessage = "The file root is set to a default of: " + FileUtil.getAbsolutePath(container, root) + migrateFilesMessage; - } - else if (!service.isCloudRoot(container)) - { - Path root = service.getFileRootPath(container); - - form.setFileRootOption(FileRootProp.folderOverride.name()); - if (root != null) - { - String absolutePath = FileUtil.getAbsolutePath(container, root); - form.setFolderRootPath(absolutePath); - if (Files.exists(root)) - { - if (fileRootChanged) - confirmMessage = "The file root is set to: " + absolutePath + migrateFilesMessage; - } - } - } - else - { - form.setFileRootOption(FileRootProp.cloudRoot.name()); - form.setCloudRootName(service.getCloudRootName(container)); - Path root = service.getFileRootPath(container); - if (root != null && fileRootChanged) - { - confirmMessage = "The file root is set to: " + FileUtil.getCloudRootPathString(form.getCloudRootName()) + migrateFilesMessage; - } - } - } - - if (fileRootChanged && confirmMessage != null) - form.setConfirmMessage(confirmMessage); - else if (enabledCloudChanged) - form.setConfirmMessage("The enabled cloud stores changed."); - } - - @RequiresPermission(AdminPermission.class) - public static class ManageFoldersAction extends FolderManagementViewAction - { - @Override - protected HttpView getTabView() - { - return new JspView<>("/org/labkey/core/admin/manageFolders.jsp"); - } - } - - public static class NotificationsForm - { - private String _provider; - - public String getProvider() - { - return _provider; - } - - public void setProvider(String provider) - { - _provider = provider; - } - } - - private static final String DATA_REGION_NAME = "Users"; - - @RequiresPermission(AdminPermission.class) - public static class NotificationsAction extends FolderManagementViewPostAction - { - @Override - protected HttpView getTabView(NotificationsForm form, boolean reshow, BindException errors) - { - final String key = DataRegionSelection.getSelectionKey("core", CoreQuerySchema.USERS_MSG_SETTINGS_TABLE_NAME, null, DATA_REGION_NAME); - DataRegionSelection.clearAll(getViewContext(), key); - - QuerySettings settings = new QuerySettings(getViewContext(), DATA_REGION_NAME, CoreQuerySchema.USERS_MSG_SETTINGS_TABLE_NAME); - settings.setAllowChooseView(true); - settings.getBaseSort().insertSortColumn(FieldKey.fromParts("DisplayName")); - - UserSchema schema = QueryService.get().getUserSchema(getViewContext().getUser(), getViewContext().getContainer(), SchemaKey.fromParts(CoreQuerySchema.NAME)); - QueryView queryView = new QueryView(schema, settings, errors) - { - @Override - public List getDisplayColumns() - { - List columns = new ArrayList<>(); - SecurityPolicy policy = getContainer().getPolicy(); - Set assignmentSet = new HashSet<>(); - - for (RoleAssignment assignment : policy.getAssignments()) - { - Group g = SecurityManager.getGroup(assignment.getUserId()); - if (g != null) - assignmentSet.add(g.getName()); - } - - for (DisplayColumn col : super.getDisplayColumns()) - { - if (col.getName().equalsIgnoreCase("Groups")) - columns.add(new FolderGroupColumn(assignmentSet, col.getColumnInfo())); - else - columns.add(col); - } - return columns; - } - - @Override - protected void populateButtonBar(DataView dataView, ButtonBar bar) - { - try - { - // add the provider configuration menu items to the admin panel button - MenuButton adminButton = new MenuButton("Update user settings"); - adminButton.setRequiresSelection(true); - for (ConfigTypeProvider provider : MessageConfigService.get().getConfigTypes()) - adminButton.addMenuItem("For " + provider.getName().toLowerCase(), "userSettings_"+provider.getName()+"(LABKEY.DataRegions.Users.getSelectionCount())" ); - - bar.add(adminButton); - super.populateButtonBar(dataView, bar); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }; - queryView.setShadeAlternatingRows(true); - queryView.setShowBorders(true); - queryView.setShowDetailsColumn(false); - queryView.setShowRecordSelectors(true); - queryView.setFrame(WebPartView.FrameType.NONE); - queryView.disableContainerFilterSelection(); - queryView.setButtonBarPosition(DataRegion.ButtonBarPosition.TOP); - - VBox defaultsView = new VBox( - HtmlView.unsafe( - "
    Default settings
    " + - "You can change this folder's default settings for email notifications here.") - ); - - PanelConfig config = new PanelConfig(getViewContext().getActionURL().clone(), key); - for (ConfigTypeProvider provider : MessageConfigService.get().getConfigTypes()) - { - defaultsView.addView(new JspView<>("/org/labkey/core/admin/view/notifySettings.jsp", provider.createConfigForm(getViewContext(), config))); - } - - return new VBox( - new JspView<>("/org/labkey/core/admin/view/folderSettingsHeader.jsp", null, errors), - defaultsView, - new VBox( - HtmlView.unsafe( - "
    User settings
    " + - "The list below contains all users with read access to this folder who are able to receive notifications. Each user's current
    " + - "notification setting is visible in the appropriately named column.

    " + - "To bulk edit individual settings: select one or more users, click the 'Update user settings' menu, and select the notification type."), - queryView - ) - ); - } - - @Override - public void validateCommand(NotificationsForm form, Errors errors) - { - ConfigTypeProvider provider = MessageConfigService.get().getConfigType(form.getProvider()); - - if (provider != null) - provider.validateCommand(getViewContext(), errors); - } - - @Override - public boolean handlePost(NotificationsForm form, BindException errors) throws Exception - { - ConfigTypeProvider provider = MessageConfigService.get().getConfigType(form.getProvider()); - - if (provider != null) - { - return provider.handlePost(getViewContext(), errors); - } - errors.reject(SpringActionController.ERROR_MSG, "Unable to find the selected config provider"); - return false; - } - } - - public static class NotifyOptionsForm - { - private String _type; - - public String getType() - { - return _type; - } - - public void setType(String type) - { - _type = type; - } - - public ConfigTypeProvider getProvider() - { - return MessageConfigService.get().getConfigType(getType()); - } - } - - /** - * Action to populate an Ext store with email notification options for admin settings - */ - @RequiresPermission(AdminPermission.class) - public static class GetEmailOptionsAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(NotifyOptionsForm form, BindException errors) - { - ApiSimpleResponse resp = new ApiSimpleResponse(); - - ConfigTypeProvider provider = form.getProvider(); - if (provider != null) - { - List options = new ArrayList<>(); - - // if the list of options is not for the folder default, add an option to use the folder default - if (getViewContext().get("isDefault") == null) - options.add(PageFlowUtil.map("id", -1, "label", "Folder default")); - - for (NotificationOption option : provider.getOptions()) - { - options.add(PageFlowUtil.map("id", option.getEmailOptionId(), "label", option.getEmailOption())); - } - resp.put("success", true); - if (!options.isEmpty()) - resp.put("options", options); - } - else - resp.put("success", false); - - return resp; - } - } - - @RequiresPermission(AdminPermission.class) - public static class SetBulkEmailOptionsAction extends MutatingApiAction - { - @Override - public ApiResponse execute(EmailConfigFormImpl form, BindException errors) - { - ApiSimpleResponse resp = new ApiSimpleResponse(); - ConfigTypeProvider provider = form.getProvider(); - String srcIdentifier = getContainer().getId(); - - Set selections = DataRegionSelection.getSelected(getViewContext(), form.getDataRegionSelectionKey(), true); - - if (!selections.isEmpty() && provider != null) - { - int newOption = form.getIndividualEmailOption(); - - for (String user : selections) - { - User projectUser = UserManager.getUser(Integer.parseInt(user)); - UserPreference pref = provider.getPreference(getContainer(), projectUser, srcIdentifier); - - int currentEmailOption = pref != null ? pref.getEmailOptionId() : -1; - - //has this projectUser's option changed? if so, update - //creating new record in EmailPrefs table if there isn't one, or deleting if set back to folder default - if (currentEmailOption != newOption) - { - provider.savePreference(getUser(), getContainer(), projectUser, newOption, srcIdentifier); - } - } - resp.put("success", true); - } - else - { - resp.put("success", false); - resp.put("message", "There were no users selected"); - } - return resp; - } - } - - /** Renders only the groups that are assigned roles in this container */ - private static class FolderGroupColumn extends DataColumn - { - private final Set _assignmentSet; - - public FolderGroupColumn(Set assignmentSet, ColumnInfo col) - { - super(col); - _assignmentSet = assignmentSet; - } - - @Override - public void renderGridCellContents(RenderContext ctx, HtmlWriter out) - { - String value = (String)ctx.get(getBoundColumn().getDisplayField().getFieldKey()); - - if (value != null) - { - out.write(Arrays.stream(value.split(VALUE_DELIMITER_REGEX)) - .filter(_assignmentSet::contains) - .map(HtmlString::of) - .collect(LabKeyCollectors.joining(HtmlString.unsafe(",
    ")))); - } - } - } - - private static class PanelConfig implements MessageConfigService.PanelInfo - { - private final ActionURL _returnUrl; - private final String _dataRegionSelectionKey; - - public PanelConfig(ActionURL returnUrl, String selectionKey) - { - _returnUrl = returnUrl; - _dataRegionSelectionKey = selectionKey; - } - - @Override - public ActionURL getReturnUrl() - { - return _returnUrl; - } - - @Override - public String getDataRegionSelectionKey() - { - return _dataRegionSelectionKey; - } - } - - public static class ConceptsForm - { - private String _conceptURI; - private String _containerId; - private String _schemaName; - private String _queryName; - - public String getConceptURI() - { - return _conceptURI; - } - - public void setConceptURI(String conceptURI) - { - _conceptURI = conceptURI; - } - - public String getContainerId() - { - return _containerId; - } - - public void setContainerId(String containerId) - { - _containerId = containerId; - } - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public String getQueryName() - { - return _queryName; - } - - public void setQueryName(String queryName) - { - _queryName = queryName; - } - } - - @RequiresPermission(AdminPermission.class) - public static class ConceptsAction extends FolderManagementViewPostAction - { - @Override - protected HttpView getTabView(ConceptsForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/manageConcepts.jsp", form, errors); - } - - @Override - public void validateCommand(ConceptsForm form, Errors errors) - { - // validate that the required input fields are provided - String missingRequired = "", sep = ""; - if (form.getConceptURI() == null) - { - missingRequired += "conceptURI"; - sep = ", "; - } - if (form.getSchemaName() == null) - { - missingRequired += sep + "schemaName"; - sep = ", "; - } - if (form.getQueryName() == null) - missingRequired += sep + "queryName"; - if (!missingRequired.isEmpty()) - errors.reject(SpringActionController.ERROR_MSG, "Missing required field(s): " + missingRequired + "."); - - // validate that, if provided, the containerId matches an existing container - Container postContainer = null; - if (form.getContainerId() != null) - { - postContainer = ContainerManager.getForId(form.getContainerId()); - if (postContainer == null) - errors.reject(SpringActionController.ERROR_MSG, "Container does not exist for containerId provided."); - } - - // validate that the schema and query names provided exist - if (form.getSchemaName() != null && form.getQueryName() != null) - { - Container c = postContainer != null ? postContainer : getContainer(); - UserSchema schema = QueryService.get().getUserSchema(getUser(), c, form.getSchemaName()); - if (schema == null) - errors.reject(SpringActionController.ERROR_MSG, "UserSchema '" + form.getSchemaName() + "' not found."); - else if (schema.getTable(form.getQueryName()) == null) - errors.reject(SpringActionController.ERROR_MSG, "Table '" + form.getSchemaName() + "." + form.getQueryName() + "' not found."); - } - } - - @Override - public boolean handlePost(ConceptsForm form, BindException errors) - { - Lookup lookup = new Lookup(ContainerManager.getForId(form.getContainerId()), form.getSchemaName(), form.getQueryName()); - ConceptURIProperties.setLookup(getContainer(), form.getConceptURI(), lookup); - - return true; - } - } - - @RequiresPermission(AdminPermission.class) - public class FolderAliasesAction extends FormViewAction - { - @Override - public void validateCommand(FolderAliasesForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(FolderAliasesForm form, boolean reshow, BindException errors) - { - return new JspView("/org/labkey/core/admin/folderAliases.jsp"); - } - - @Override - public boolean handlePost(FolderAliasesForm form, BindException errors) - { - List aliases = new ArrayList<>(); - if (form.getAliases() != null) - { - StringTokenizer st = new StringTokenizer(form.getAliases(), "\n\r", false); - while (st.hasMoreTokens()) - { - String alias = st.nextToken().trim(); - if (!alias.startsWith("/")) - { - alias = "/" + alias; - } - while (alias.endsWith("/")) - { - alias = alias.substring(0, alias.lastIndexOf('/')); - } - aliases.add(alias); - } - } - ContainerManager.saveAliasesForContainer(getContainer(), aliases, getUser()); - - return true; - } - - @Override - public ActionURL getSuccessURL(FolderAliasesForm form) - { - return getManageFoldersURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Folder Aliases: " + getContainer().getPath(), this.getClass()); - } - } - - public static class FolderAliasesForm - { - private String _aliases; - - public String getAliases() - { - return _aliases; - } - - @SuppressWarnings("unused") - public void setAliases(String aliases) - { - _aliases = aliases; - } - } - - @RequiresPermission(AdminPermission.class) - public class CustomizeEmailAction extends FormViewAction - { - @Override - public void validateCommand(CustomEmailForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(CustomEmailForm form, boolean reshow, BindException errors) - { - JspView result = new JspView<>("/org/labkey/core/admin/customizeEmail.jsp", form, errors); - result.setTitle("Email Template"); - return result; - } - - @Override - public boolean handlePost(CustomEmailForm form, BindException errors) - { - if (form.getTemplateClass() != null) - { - EmailTemplate template = EmailTemplateService.get().createTemplate(form.getTemplateClass()); - - template.setSubject(form.getEmailSubject()); - template.setSenderName(form.getEmailSender()); - template.setReplyToEmail(form.getEmailReplyTo()); - template.setBody(form.getEmailMessage()); - - String[] errorStrings = new String[1]; - if (template.isValid(errorStrings)) // TODO: Pass in errors collection directly? Should also build a list of all validation errors and display them all. - EmailTemplateService.get().saveEmailTemplate(template, getContainer()); - else - errors.reject(ERROR_MSG, errorStrings[0]); - } - - return !errors.hasErrors(); - } - - @Override - public ActionURL getSuccessURL(CustomEmailForm form) - { - ActionURL result = new ActionURL(CustomizeEmailAction.class, getContainer()); - result.replaceParameter("templateClass", form.getTemplateClass()); - if (form.getReturnActionURL() != null) - { - result.replaceParameter(ActionURL.Param.returnUrl, form.getReturnUrl()); - } - return result; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("customEmail"); - addAdminNavTrail(root, "Customize " + (getContainer().isRoot() ? "Site-Wide" : StringUtils.capitalize(getContainer().getContainerNoun()) + "-Level") + " Email", this.getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class DeleteCustomEmailAction extends FormHandlerAction - { - @Override - public void validateCommand(CustomEmailForm target, Errors errors) - { - } - - @Override - public boolean handlePost(CustomEmailForm form, BindException errors) throws Exception - { - if (form.getTemplateClass() != null) - { - EmailTemplate template = EmailTemplateService.get().createTemplate(form.getTemplateClass()); - template.setSubject(form.getEmailSubject()); - template.setBody(form.getEmailMessage()); - - EmailTemplateService.get().deleteEmailTemplate(template, getContainer()); - } - return true; - } - - @Override - public URLHelper getSuccessURL(CustomEmailForm form) - { - return new AdminUrlsImpl().getCustomizeEmailURL(getContainer(), form.getTemplateClass(), form.getReturnUrlHelper()); - } - } - - @SuppressWarnings("unused") - public static class CustomEmailForm extends ReturnUrlForm - { - private String _templateClass; - private String _emailSubject; - private String _emailSender; - private String _emailReplyTo; - private String _emailMessage; - private String _templateDescription; - - public void setTemplateClass(String name){_templateClass = name;} - public String getTemplateClass(){return _templateClass;} - public void setEmailSubject(String subject){_emailSubject = subject;} - public String getEmailSubject(){return _emailSubject;} - public void setEmailSender(String sender){_emailSender = sender;} - public String getEmailSender(){return _emailSender;} - public void setEmailMessage(String body){_emailMessage = body;} - public String getEmailMessage(){return _emailMessage;} - public String getEmailReplyTo(){return _emailReplyTo;} - public void setEmailReplyTo(String emailReplyTo){_emailReplyTo = emailReplyTo;} - - public String getTemplateDescription() - { - return _templateDescription; - } - - public void setTemplateDescription(String templateDescription) - { - _templateDescription = templateDescription; - } - } - - private ActionURL getManageFoldersURL() - { - return new AdminUrlsImpl().getManageFoldersURL(getContainer()); - } - - public static class ManageFoldersForm extends ReturnUrlForm - { - private String name; - private String title; - private boolean titleSameAsName; - private String folder; - private String target; - private String folderType; - private String defaultModule; - private String[] activeModules; - private boolean hasLoaded = false; - private boolean showAll; - private boolean confirmed = false; - private boolean addAlias = false; - private String templateSourceId; - private String[] templateWriterTypes; - private boolean templateIncludeSubfolders = false; - private String[] targets; - private PHI _exportPhiLevel = PHI.NotPHI; - - public boolean getHasLoaded() - { - return hasLoaded; - } - - public void setHasLoaded(boolean hasLoaded) - { - this.hasLoaded = hasLoaded; - } - - public String[] getActiveModules() - { - return activeModules; - } - - public void setActiveModules(String[] activeModules) - { - this.activeModules = activeModules; - } - - public String getDefaultModule() - { - return defaultModule; - } - - public void setDefaultModule(String defaultModule) - { - this.defaultModule = defaultModule; - } - - public boolean isShowAll() - { - return showAll; - } - - public void setShowAll(boolean showAll) - { - this.showAll = showAll; - } - - public String getFolder() - { - return folder; - } - - public void setFolder(String folder) - { - this.folder = folder; - } - - public String getName() - { - return name; - } - - public String getTitle() - { - return title; - } - - public void setTitle(String title) - { - this.title = title; - } - - public boolean isTitleSameAsName() - { - return titleSameAsName; - } - - public void setTitleSameAsName(boolean updateTitle) - { - this.titleSameAsName = updateTitle; - } - public void setName(String name) - { - this.name = name; - } - - public boolean isConfirmed() - { - return confirmed; - } - - public void setConfirmed(boolean confirmed) - { - this.confirmed = confirmed; - } - - public String getFolderType() - { - return folderType; - } - - public void setFolderType(String folderType) - { - this.folderType = folderType; - } - - public boolean isAddAlias() - { - return addAlias; - } - - public void setAddAlias(boolean addAlias) - { - this.addAlias = addAlias; - } - - public String getTarget() - { - return target; - } - - public void setTarget(String target) - { - this.target = target; - } - - public void setTemplateSourceId(String templateSourceId) - { - this.templateSourceId = templateSourceId; - } - - public String getTemplateSourceId() - { - return templateSourceId; - } - - public Container getTemplateSourceContainer() - { - if (null == getTemplateSourceId()) - return null; - return ContainerManager.getForId(getTemplateSourceId()); - } - - public String[] getTemplateWriterTypes() - { - return templateWriterTypes; - } - - public void setTemplateWriterTypes(String[] templateWriterTypes) - { - this.templateWriterTypes = templateWriterTypes; - } - - public boolean getTemplateIncludeSubfolders() - { - return templateIncludeSubfolders; - } - - public void setTemplateIncludeSubfolders(boolean templateIncludeSubfolders) - { - this.templateIncludeSubfolders = templateIncludeSubfolders; - } - - public String[] getTargets() - { - return targets; - } - - public void setTargets(String[] targets) - { - this.targets = targets; - } - - public PHI getExportPhiLevel() - { - return _exportPhiLevel; - } - - public void setExportPhiLevel(PHI exportPhiLevel) - { - _exportPhiLevel = exportPhiLevel; - } - - /** - * Note: this is designed to allow code to specify a set of children to delete in bulk. The main use-case is workbooks, - * but it will work for non-workbook children as well. - */ - public List getTargetContainers(final Container currentContainer) throws IllegalArgumentException - { - if (getTargets() != null) - { - final List targets = new ArrayList<>(); - final List directChildren = ContainerManager.getChildren(currentContainer); - - Arrays.stream(getTargets()).forEach(x -> { - Container c = ContainerManager.getForId(x); - if (c == null) - { - try - { - Integer rowId = ConvertHelper.convert(x, Integer.class); - if (rowId > 0) - c = ContainerManager.getForRowId(rowId); - } - catch (ConversionException e) - { - //ignore - } - } - - if (c != null) - { - if (!c.equals(currentContainer)) - { - if (!directChildren.contains(c)) - { - throw new IllegalArgumentException("Folder " + c.getPath() + " is not a direct child of the current folder: " + currentContainer.getPath()); - } - - if (c.getContainerType().canHaveChildren()) - { - throw new IllegalArgumentException("Multi-folder delete is not supported for containers of type: " + c.getContainerType().getName()); - } - } - - targets.add(c); - } - else - { - throw new IllegalArgumentException("Unable to find folder with ID or RowId of: " + x); - } - }); - - return targets; - } - else - { - return Collections.singletonList(currentContainer); - } - } - } - - public static class RenameContainerForm - { - private String name; - private String title; - private boolean addAlias = true; - - public String getName() - { - return name; - } - - public void setName(String name) - { - this.name = name; - } - - public String getTitle() - { - return title; - } - - public void setTitle(String title) - { - this.title = title; - } - - public boolean isAddAlias() - { - return addAlias; - } - - public void setAddAlias(boolean addAlias) - { - this.addAlias = addAlias; - } - } - - // Note that validation checks occur in ContainerManager.rename() - @RequiresPermission(AdminPermission.class) - public static class RenameContainerAction extends MutatingApiAction - { - @Override - public ApiResponse execute(RenameContainerForm form, BindException errors) - { - Container container = getContainer(); - String name = StringUtils.trimToNull(form.getName()); - String title = StringUtils.trimToNull(form.getTitle()); - - String nameValue = name; - String titleValue = title; - if (name == null && title == null) - { - errors.reject(ERROR_MSG, "Please specify a name or a title."); - return new ApiSimpleResponse("success", false); - } - else if (name != null && title == null) - { - titleValue = name; - } - else if (name == null) - { - nameValue = container.getName(); - } - - boolean addAlias = form.isAddAlias(); - - try - { - Container c = ContainerManager.rename(container, getUser(), nameValue, titleValue, addAlias); - return new ApiSimpleResponse(c.toJSON(getUser())); - } - catch (Exception e) - { - errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : "Failed to rename folder. An error has occurred."); - return new ApiSimpleResponse("success", false); - } - } - } - - @RequiresPermission(AdminPermission.class) - public class RenameFolderAction extends FormViewAction - { - private ActionURL _returnUrl; - - @Override - public void validateCommand(ManageFoldersForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/renameFolder.jsp", form, errors); - } - - @Override - public boolean handlePost(ManageFoldersForm form, BindException errors) - { - try - { - String title = form.isTitleSameAsName() ? null : StringUtils.trimToNull(form.getTitle()); - Container c = ContainerManager.rename(getContainer(), getUser(), form.getName(), title, form.isAddAlias()); - _returnUrl = new AdminUrlsImpl().getManageFoldersURL(c); - return true; - } - catch (Exception e) - { - errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : "Failed to rename folder. An error has occurred."); - } - - return false; - } - - @Override - public ActionURL getSuccessURL(ManageFoldersForm form) - { - return _returnUrl; - } - - @Override - public void addNavTrail(NavTree root) - { - getPageConfig().setFocusId("name"); - String containerType = getContainer().isProject() ? "Project" : "Folder"; - addAdminNavTrail(root, "Change " + containerType + " Name Settings", this.getClass()); - } - } - - public static class MoveFolderTreeView extends JspView - { - private MoveFolderTreeView(ManageFoldersForm form, BindException errors) - { - super("/org/labkey/core/admin/moveFolder.jsp", form, errors); - } - } - - @RequiresPermission(AdminPermission.class) - @ActionNames("ShowMoveFolderTree,MoveFolder") - public class MoveFolderAction extends FormViewAction - { - boolean showConfirmPage = false; - boolean moveFailed = false; - - @Override - public void validateCommand(ManageFoldersForm form, Errors errors) - { - Container c = getContainer(); - - if (c.isRoot()) - throw new NotFoundException("Can't move the root folder."); // Don't show move tree from root - - if (c.equals(ContainerManager.getSharedContainer()) || c.equals(ContainerManager.getHomeContainer())) - errors.reject(ERROR_MSG, "Moving /Shared or /home is not possible."); - - Container newParent = isBlank(form.getTarget()) ? null : ContainerManager.getForPath(form.getTarget()); - if (null == newParent) - { - errors.reject(ERROR_MSG, "Target '" + form.getTarget() + "' folder does not exist."); - } - else if (!newParent.hasPermission(getUser(), AdminPermission.class)) - { - throw new UnauthorizedException(); - } - else if (newParent.hasChild(c.getName())) - { - errors.reject(ERROR_MSG, "Error: The selected folder already has a folder with that name. Please select a different location (or Cancel)."); - } - } - - @Override - public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) throws Exception - { - if (showConfirmPage) - return new JspView<>("/org/labkey/core/admin/confirmProjectMove.jsp", form); - if (moveFailed) - return new SimpleErrorView(errors); - else - return new MoveFolderTreeView(form, errors); - } - - @Override - public boolean handlePost(ManageFoldersForm form, BindException errors) throws Exception - { - Container c = getContainer(); - Container newParent = ContainerManager.getForPath(form.getTarget()); - Container oldProject = c.getProject(); - Container newProject = newParent.isRoot() ? c : newParent.getProject(); - - if (!oldProject.getId().equals(newProject.getId()) && !form.isConfirmed()) - { - showConfirmPage = true; - return false; // reshow - } - - try - { - ContainerManager.move(c, newParent, getUser()); - } - catch (ValidationException e) - { - moveFailed = true; - getPageConfig().setTemplate(Template.Dialog); - for (ValidationError validationError : e.getErrors()) - { - errors.addError(new LabKeyError(validationError.getMessage())); - } - if (!errors.hasErrors()) - errors.addError(new LabKeyError("Move failed")); - return false; - } - - if (form.isAddAlias()) - { - List newAliases = new ArrayList<>(ContainerManager.getAliasesForContainer(c)); - newAliases.add(c.getPath()); - ContainerManager.saveAliasesForContainer(c, newAliases, getUser()); - } - return true; - } - - @Override - public URLHelper getSuccessURL(ManageFoldersForm manageFoldersForm) - { - Container c = getContainer(); - c = ContainerManager.getForId(c.getId()); // Reload container to populate new location - return new AdminUrlsImpl().getManageFoldersURL(c); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Folder Management", getManageFoldersURL()); - root.addChild("Move Folder"); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ConfirmProjectMoveAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ManageFoldersForm form, BindException errors) - { - getPageConfig().setTemplate(Template.Dialog); - return new JspView<>("/org/labkey/core/admin/confirmProjectMove.jsp", form); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Confirm Project Move"); - } - } - - private static abstract class AbstractCreateFolderAction
    extends FormViewAction - { - private ActionURL _successURL; - - @Override - public void validateCommand(FORM target, Errors errors) - { - } - - @Override - public ModelAndView getView(FORM form, boolean reshow, BindException errors) - { - VBox vbox = new VBox(); - - if (!reshow) - { - FolderType folderType = FolderTypeManager.get().getDefaultFolderType(); - if (null != folderType) - { - // If a default folder type has been configured by a site admin set that as the default folder type choice - form.setFolderType(folderType.getName()); - } - form.setExportPhiLevel(ComplianceService.get().getMaxAllowedPhi(getContainer(), getUser())); - } - JspView statusView = new JspView<>("/org/labkey/core/admin/createFolder.jsp", form, errors); - vbox.addView(statusView); - - Container c = getViewContext().getContainerNoTab(); // Cannot create subfolder of tab folder - - setHelpTopic("createProject"); - getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(null, c)); - getPageConfig().setTemplate(Template.Wizard); - - if (c.isRoot()) - getPageConfig().setTitle("Create Project"); - else - { - String title = "Create Folder"; - - title += " in /"; - if (c == ContainerManager.getHomeContainer()) - title += "Home"; - else - title += c.getName(); - - getPageConfig().setTitle(title); - } - - return vbox; - } - - @Override - public boolean handlePost(FORM form, BindException errors) throws Exception - { - Container parent = getViewContext().getContainerNoTab(); - String folderName = StringUtils.trimToNull(form.getName()); - String folderTitle = (form.isTitleSameAsName() || folderName.equals(form.getTitle())) ? null : form.getTitle(); - StringBuilder error = new StringBuilder(); - Consumer afterCreateHandler = getAfterCreateHandler(form); - - Container container; - - if (Container.isLegalName(folderName, parent.isRoot(), error)) - { - if (parent.hasChild(folderName)) - { - if (parent.isRoot()) - { - error.append("The server already has a project with this name."); - } - else - { - error.append("The ").append(parent.isProject() ? "project " : "folder ").append(parent.getPath()).append(" already has a folder with this name."); - } - } - else - { - String folderType = form.getFolderType(); - - if (null == folderType) - { - errors.reject(null, "Folder type must be specified"); - return false; - } - - if ("Template".equals(folderType)) // Create folder from selected template - { - Container sourceContainer = form.getTemplateSourceContainer(); - if (null == sourceContainer) - { - errors.reject(null, "Source template folder not selected"); - return false; - } - else if (!sourceContainer.hasPermission(getUser(), AdminPermission.class)) - { - errors.reject(null, "User does not have administrator permissions to the source container"); - return false; - } - else if (!sourceContainer.hasEnableRestrictedModules(getUser()) && sourceContainer.hasRestrictedActiveModule(sourceContainer.getActiveModules())) - { - errors.reject(null, "The source folder has a restricted module for which you do not have permission."); - return false; - } - - FolderExportContext exportCtx = new FolderExportContext(getUser(), sourceContainer, PageFlowUtil.set(form.getTemplateWriterTypes()), "new", - form.getTemplateIncludeSubfolders(), form.getExportPhiLevel(), false, false, false, - new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); - - container = ContainerManager.createContainerFromTemplate(parent, folderName, folderTitle, sourceContainer, getUser(), exportCtx, afterCreateHandler); - } - else - { - FolderType type = FolderTypeManager.get().getFolderType(folderType); - - if (type == null) - { - errors.reject(null, "Folder type not recognized"); - return false; - } - - String[] modules = form.getActiveModules(); - - if (null == StringUtils.trimToNull(folderType) || FolderType.NONE.getName().equals(folderType)) - { - if (null == modules || modules.length == 0) - { - errors.reject(null, "At least one module must be selected"); - return false; - } - } - - // Work done in this lambda will not fire container events. Only fireCreateContainer() will be called. - Consumer configureContainer = (newContainer) -> - { - afterCreateHandler.accept(newContainer); - newContainer.setFolderType(type, getUser(), errors); - - if (null == StringUtils.trimToNull(folderType) || FolderType.NONE.getName().equals(folderType)) - { - Set activeModules = new HashSet<>(); - for (String moduleName : modules) - { - Module module = ModuleLoader.getInstance().getModule(moduleName); - if (module != null) - activeModules.add(module); - } - - newContainer.setFolderType(FolderType.NONE, getUser(), errors, activeModules); - Module defaultModule = ModuleLoader.getInstance().getModule(form.getDefaultModule()); - newContainer.setDefaultModule(defaultModule); - } - }; - container = ContainerManager.createContainer(parent, folderName, folderTitle, null, NormalContainerType.NAME, getUser(), null, configureContainer); - } - - _successURL = new AdminUrlsImpl().getSetFolderPermissionsURL(container); - _successURL.addParameter("wizard", Boolean.TRUE.toString()); - - return true; - } - } - - errors.reject(ERROR_MSG, "Error: " + error + " Please enter a different name."); - return false; - } - - /** - * Return a Consumer that provides post-creation handling on the new Container - */ - abstract public Consumer getAfterCreateHandler(FORM form); - - @Override - protected String getCommandClassMethodName() - { - return "getAfterCreateHandler"; - } - - @Override - public ActionURL getSuccessURL(FORM form) - { - return _successURL; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresPermission(AdminPermission.class) - public static class CreateFolderAction extends AbstractCreateFolderAction - { - @Override - public Consumer getAfterCreateHandler(ManageFoldersForm form) - { - // No special handling - return container -> {}; - } - } - - public static class CreateProjectForm extends ManageFoldersForm - { - private boolean _assignProjectAdmin = false; - - public boolean isAssignProjectAdmin() - { - return _assignProjectAdmin; - } - - @SuppressWarnings("unused") - public void setAssignProjectAdmin(boolean assignProjectAdmin) - { - _assignProjectAdmin = assignProjectAdmin; - } - } - - @RequiresPermission(CreateProjectPermission.class) - public static class CreateProjectAction extends AbstractCreateFolderAction - { - @Override - public void validateCommand(CreateProjectForm target, Errors errors) - { - super.validateCommand(target, errors); - if (!getContainer().isRoot()) - errors.reject(ERROR_MSG, "Must be invoked from the root"); - } - - @Override - public Consumer getAfterCreateHandler(CreateProjectForm form) - { - if (form.isAssignProjectAdmin()) - { - return c -> { - MutableSecurityPolicy policy = new MutableSecurityPolicy(c.getPolicy()); - policy.addRoleAssignment(getUser(), ProjectAdminRole.class); - User savePolicyUser = getUser(); - if (c.isProject() && !c.hasPermission(savePolicyUser, AdminPermission.class) && ContainerManager.getRoot().hasPermission(savePolicyUser, CreateProjectPermission.class)) - { - // Special case for project creators who don't necessarily yet have permission to save the policy of - // the project they just created - savePolicyUser = User.getAdminServiceUser(); - } - - SecurityPolicyManager.savePolicy(policy, savePolicyUser); - }; - } - else - { - return c -> {}; - } - } - } - - @RequiresPermission(AdminPermission.class) - public static class SetFolderPermissionsAction extends FormViewAction - { - private ActionURL _successURL; - - @Override - public void validateCommand(SetFolderPermissionsForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(SetFolderPermissionsForm form, boolean reshow, BindException errors) - { - VBox vbox = new VBox(); - - JspView statusView = new JspView<>("/org/labkey/core/admin/setFolderPermissions.jsp", form, errors); - vbox.addView(statusView); - - Container c = getContainer(); - getPageConfig().setTitle("Users / Permissions"); - getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(c, c.getParent())); - getPageConfig().setTemplate(Template.Wizard); - setHelpTopic("createProject"); - - return vbox; - } - - @Override - public boolean handlePost(SetFolderPermissionsForm form, BindException errors) - { - Container c = getContainer(); - String permissionType = form.getPermissionType(); - - if(c.isProject()){ - _successURL = new AdminUrlsImpl().getInitialFolderSettingsURL(c); - } - else - { - List extraSteps = getContainer().getFolderType().getExtraSetupSteps(getContainer()); - if (extraSteps.isEmpty()) - { - if (form.isAdvanced()) - { - _successURL = new SecurityController.SecurityUrlsImpl().getPermissionsURL(getContainer()); - } - else - { - _successURL = getContainer().getStartURL(getUser()); - } - } - else - { - _successURL = new ActionURL(extraSteps.get(0).getHref()); - } - } - - if(permissionType == null){ - errors.reject(ERROR_MSG, "You must select one of the options for permissions."); - return false; - } - - switch (permissionType) - { - case "CurrentUser" -> { - MutableSecurityPolicy policy = new MutableSecurityPolicy(c); - Role role = RoleManager.getRole(c.isProject() ? ProjectAdminRole.class : FolderAdminRole.class); - policy.addRoleAssignment(getUser(), role); - SecurityPolicyManager.savePolicy(policy, getUser()); - } - case "Inherit" -> SecurityManager.setInheritPermissions(c); - case "CopyExistingProject" -> { - String targetProject = form.getTargetProject(); - if (targetProject == null) - { - errors.reject(ERROR_MSG, "In order to copy permissions from an existing project, you must pick a project."); - return false; - } - Container source = ContainerManager.getForId(targetProject); - if (source == null) - { - source = ContainerManager.getForPath(targetProject); - } - if (source == null) - { - throw new NotFoundException("An unknown project was specified to copy permissions from: " + targetProject); - } - Map groupMap = GroupManager.copyGroupsToContainer(source, c, getUser()); - - //copy role assignments - SecurityPolicy op = SecurityPolicyManager.getPolicy(source); - MutableSecurityPolicy np = new MutableSecurityPolicy(c); - for (RoleAssignment assignment : op.getAssignments()) - { - int userId = assignment.getUserId(); - UserPrincipal p = SecurityManager.getPrincipal(userId); - Role r = assignment.getRole(); - - if (p instanceof Group g) - { - if (!g.isProjectGroup()) - { - np.addRoleAssignment(p, r, false); - } - else - { - np.addRoleAssignment(groupMap.get(p), r, false); - } - } - else - { - np.addRoleAssignment(p, r, false); - } - } - SecurityPolicyManager.savePolicy(np, getUser()); - } - default -> throw new UnsupportedOperationException("An Unknown permission type was supplied: " + permissionType); - } - _successURL.addParameter("wizard", Boolean.TRUE.toString()); - - return true; - } - - @Override - public ActionURL getSuccessURL(SetFolderPermissionsForm form) - { - return _successURL; - } - - @Override - public void addNavTrail(NavTree root) - { - getPageConfig().setFocusId("name"); - } - } - - public static class SetFolderPermissionsForm - { - private String targetProject; - private String permissionType; - private boolean advanced; - - public String getPermissionType() - { - return permissionType; - } - - @SuppressWarnings("unused") - public void setPermissionType(String permissionType) - { - this.permissionType = permissionType; - } - - public String getTargetProject() - { - return targetProject; - } - - @SuppressWarnings("unused") - public void setTargetProject(String targetProject) - { - this.targetProject = targetProject; - } - - public boolean isAdvanced() - { - return advanced; - } - - @SuppressWarnings("unused") - public void setAdvanced(boolean advanced) - { - this.advanced = advanced; - } - } - - @RequiresPermission(AdminPermission.class) - public static class SetInitialFolderSettingsAction extends FormViewAction - { - private ActionURL _successURL; - - @Override - public void validateCommand(FilesForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(FilesForm form, boolean reshow, BindException errors) - { - VBox vbox = new VBox(); - Container c = getContainer(); - - JspView statusView = new JspView<>("/org/labkey/core/admin/setInitialFolderSettings.jsp", form, errors); - vbox.addView(statusView); - - getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(c, c.getParent())); - getPageConfig().setTemplate(Template.Wizard); - - String noun = c.isProject() ? "Project": "Folder"; - getPageConfig().setTitle(noun + " Settings"); - - return vbox; - } - - @Override - public boolean handlePost(FilesForm form, BindException errors) - { - Container c = getContainer(); - String folderRootPath = StringUtils.trimToNull(form.getFolderRootPath()); - String fileRootOption = form.getFileRootOption() != null ? form.getFileRootOption() : "default"; - - if(folderRootPath == null && !fileRootOption.equals("default")) - { - errors.reject(ERROR_MSG, "Error: Must supply a default file location."); - return false; - } - - FileContentService service = FileContentService.get(); - if(fileRootOption.equals("default")) - { - service.setIsUseDefaultRoot(c, true); - } - // Requires AdminOperationsPermission to set file root - else if (c.hasPermission(getUser(), AdminOperationsPermission.class)) - { - if (!service.isValidProjectRoot(folderRootPath)) - { - errors.reject(ERROR_MSG, "File root '" + folderRootPath + "' does not appear to be a valid directory accessible to the server at " + getViewContext().getRequest().getServerName() + "."); - return false; - } - - service.setIsUseDefaultRoot(c.getProject(), false); - service.setFileRootPath(c.getProject(), folderRootPath); - } - - List extraSteps = getContainer().getFolderType().getExtraSetupSteps(getContainer()); - if (extraSteps.isEmpty()) - { - _successURL = getContainer().getStartURL(getUser()); - } - else - { - _successURL = new ActionURL(extraSteps.get(0).getHref()); - } - - return true; - } - - @Override - public ActionURL getSuccessURL(FilesForm form) - { - return _successURL; - } - - @Override - public void addNavTrail(NavTree root) - { - getPageConfig().setFocusId("name"); - setHelpTopic("createProject"); - } - } - - @RequiresPermission(DeletePermission.class) - public static class DeleteWorkbooksAction extends SimpleRedirectAction - { - public void validateCommand(ReturnUrlForm target, Errors errors) - { - Set ids = DataRegionSelection.getSelected(getViewContext(), true); - if (ids.isEmpty()) - { - errors.reject(ERROR_MSG, "No IDs provided"); - } - } - - @Override - public @Nullable URLHelper getRedirectURL(ReturnUrlForm form) throws Exception - { - Set ids = DataRegionSelection.getSelected(getViewContext(), true); - - ActionURL ret = new ActionURL(DeleteFolderAction.class, getContainer()); - ids.forEach(id -> ret.addParameter("targets", id)); - - ret.replaceParameter(ActionURL.Param.returnUrl, form.getReturnUrl()); - - return ret; - } - } - - //NOTE: some types of containers can be deleted by non-admin users, provided they have DeletePermission on the parent - @RequiresPermission(DeletePermission.class) - public static class DeleteFolderAction extends FormViewAction - { - private final List _deleted = new ArrayList<>(); - - @Override - public void validateCommand(ManageFoldersForm form, Errors errors) - { - try - { - List targets = form.getTargetContainers(getContainer()); - for (Container target : targets) - { - if (!ContainerManager.isDeletable(target)) - errors.reject(ERROR_MSG, "The path " + target.getPath() + " is not deletable."); - - if (target.isProject() && !getUser().hasRootAdminPermission()) - { - throw new UnauthorizedException(); - } - - Class permClass = target.getPermissionNeededToDelete(); - if (!target.hasPermission(getUser(), permClass)) - { - Permission perm = RoleManager.getPermission(permClass); - throw new UnauthorizedException("Cannot delete folder: " + target.getName() + ". " + perm.getName() + " permission required"); - } - - if (target.hasChildren() && !ContainerManager.hasTreePermission(target, getUser(), AdminPermission.class)) - { - throw new UnauthorizedException("Deleting the " + target.getContainerNoun() + " " + target.getName() + " requires admin permissions on that folder and all children. You do not have admin permission on all subfolders."); - } - - if (target.equals(ContainerManager.getSharedContainer()) || target.equals(ContainerManager.getHomeContainer())) - errors.reject(ERROR_MSG, "Deleting /Shared or /home is not possible."); - } - } - catch (IllegalArgumentException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - } - - @Override - public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) - { - getPageConfig().setTemplate(Template.Dialog); - return new JspView<>("/org/labkey/core/admin/deleteFolder.jsp", form); - } - - @Override - public boolean handlePost(ManageFoldersForm form, BindException errors) - { - List targets = form.getTargetContainers(getContainer()); - - // Must be site/app admin to delete a project - for (Container c : targets) - { - ContainerManager.deleteAll(c, getUser()); - } - - _deleted.addAll(targets); - - return true; - } - - @Override - public ActionURL getSuccessURL(ManageFoldersForm form) - { - // Note: because in some scenarios we might be deleting children of the current contaner, in those cases we remain in this folder: - // If we just deleted a project then redirect to the home page, otherwise back to managing the project folders - if (_deleted.size() == 1 && _deleted.get(0).equals(getContainer())) - { - Container c = getContainer(); - if (c.isProject()) - return AppProps.getInstance().getHomePageActionURL(); - else - return new AdminUrlsImpl().getManageFoldersURL(c.getParent()); - } - else - { - if (form.getReturnUrl() != null) - { - return form.getReturnActionURL(); - } - else - { - return getContainer().getStartURL(getUser()); - } - } - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Confirm " + getContainer().getContainerNoun() + " deletion"); - } - } - - @RequiresPermission(AdminPermission.class) - public class ReorderFoldersAction extends FormViewAction - { - @Override - public void validateCommand(FolderReorderForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(FolderReorderForm folderReorderForm, boolean reshow, BindException errors) - { - return new JspView("/org/labkey/core/admin/reorderFolders.jsp"); - } - - @Override - public boolean handlePost(FolderReorderForm form, BindException errors) - { - return ReorderFolders(form, errors); - } - - @Override - public ActionURL getSuccessURL(FolderReorderForm folderReorderForm) - { - if (getContainer().isRoot()) - return getShowAdminURL(); - else - return getManageFoldersURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - String title = "Reorder " + (getContainer().isRoot() || getContainer().getParent().isRoot() ? "Projects" : "Folders"); - addAdminNavTrail(root, title, this.getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public class ReorderFoldersApiAction extends MutatingApiAction - { - @Override - public ApiResponse execute(FolderReorderForm form, BindException errors) - { - return new ApiSimpleResponse("success", ReorderFolders(form, errors)); - } - } - - private boolean ReorderFolders(FolderReorderForm form, BindException errors) - { - Container parent = getContainer().isRoot() ? getContainer() : getContainer().getParent(); - if (form.isResetToAlphabetical()) - ContainerManager.setChildOrderToAlphabetical(parent); - else if (form.getOrder() != null) - { - List children = parent.getChildren(); - String[] order = form.getOrder().split(";"); - Map nameToContainer = new HashMap<>(); - for (Container child : children) - nameToContainer.put(child.getName(), child); - List sorted = new ArrayList<>(children.size()); - for (String childName : order) - { - Container child = nameToContainer.get(childName); - sorted.add(child); - } - - try - { - ContainerManager.setChildOrder(parent, sorted); - } - catch (ContainerException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - return false; - } - } - - return true; - } - - public static class FolderReorderForm - { - private String _order; - private boolean _resetToAlphabetical; - - public String getOrder() - { - return _order; - } - - @SuppressWarnings("unused") - public void setOrder(String order) - { - _order = order; - } - - public boolean isResetToAlphabetical() - { - return _resetToAlphabetical; - } - - @SuppressWarnings("unused") - public void setResetToAlphabetical(boolean resetToAlphabetical) - { - _resetToAlphabetical = resetToAlphabetical; - } - } - - @RequiresPermission(AdminPermission.class) - public static class RevertFolderAction extends MutatingApiAction - { - @Override - public ApiResponse execute(RevertFolderForm form, BindException errors) - { - if (isBlank(form.getContainerPath())) - throw new NotFoundException(); - - boolean success = false; - Container revertContainer = ContainerManager.getForPath(form.getContainerPath()); - if (null != revertContainer) - { - if (revertContainer.isContainerTab()) - { - FolderTab tab = revertContainer.getParent().getFolderType().findTab(revertContainer.getName()); - if (null != tab) - { - FolderType origFolderType = tab.getFolderType(); - if (null != origFolderType) - { - revertContainer.setFolderType(origFolderType, getUser(), errors); - if (!errors.hasErrors()) - success = true; - } - } - } - else if (revertContainer.getFolderType().hasContainerTabs()) - { - try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) - { - List children = revertContainer.getChildren(); - for (Container container : children) - { - if (container.isContainerTab()) - { - FolderTab tab = revertContainer.getFolderType().findTab(container.getName()); - if (null != tab) - { - FolderType origFolderType = tab.getFolderType(); - if (null != origFolderType) - { - container.setFolderType(origFolderType, getUser(), errors); - } - } - } - } - if (!errors.hasErrors()) - { - transaction.commit(); - success = true; - } - } - } - } - return new ApiSimpleResponse("success", success); - } - } - - public static class RevertFolderForm - { - private String _containerPath; - - public String getContainerPath() - { - return _containerPath; - } - - public void setContainerPath(String containerPath) - { - _containerPath = containerPath; - } - } - - public static class EmailTestForm - { - private String _to; - private String _body; - private ConfigurationException _exception; - - public String getTo() - { - return _to; - } - - public void setTo(String to) - { - _to = to; - } - - public String getBody() - { - return _body; - } - - public void setBody(String body) - { - _body = body; - } - - public ConfigurationException getException() - { - return _exception; - } - - public void setException(ConfigurationException exception) - { - _exception = exception; - } - - public String getFrom(Container c) - { - LookAndFeelProperties props = LookAndFeelProperties.getInstance(c); - return props.getSystemEmailAddress(); - } - } - - @AdminConsoleAction - @RequiresPermission(AdminOperationsPermission.class) - public class EmailTestAction extends FormViewAction - { - @Override - public void validateCommand(EmailTestForm form, Errors errors) - { - if(null == form.getTo() || form.getTo().isEmpty()) - { - errors.reject(ERROR_MSG, "To field cannot be blank."); - form.setException(new ConfigurationException("To field cannot be blank")); - return; - } - - try - { - ValidEmail email = new ValidEmail(form.getTo()); - } - catch(ValidEmail.InvalidEmailException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - form.setException(new ConfigurationException(e.getMessage())); - } - } - - @Override - public ModelAndView getView(EmailTestForm form, boolean reshow, BindException errors) - { - JspView testView = new JspView<>("/org/labkey/core/admin/emailTest.jsp", form); - testView.setTitle("Send a Test Email"); - - if(null != MailHelper.getSession() && null != MailHelper.getSession().getProperties()) - { - JspView emailPropsView = new JspView<>("/org/labkey/core/admin/emailProps.jsp"); - emailPropsView.setTitle("Current Email Settings"); - - return new VBox(emailPropsView, testView); - } - else - return testView; - } - - @Override - public boolean handlePost(EmailTestForm form, BindException errors) throws Exception - { - if (errors.hasErrors()) - { - return false; - } - - LookAndFeelProperties props = LookAndFeelProperties.getInstance(getContainer()); - try - { - MailHelper.ViewMessage msg = MailHelper.createMessage(props.getSystemEmailAddress(), new ValidEmail(form.getTo()).toString()); - msg.setSubject("Test email message sent from " + props.getShortName()); - msg.setText(PageFlowUtil.filter(form.getBody())); - - try - { - MailHelper.send(msg, getUser(), getContainer()); - } - catch (ConfigurationException e) - { - form.setException(e); - return false; - } - catch (Exception e) - { - form.setException(new ConfigurationException(e.getMessage())); - return false; - } - } - catch (MessagingException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - return false; - } - return true; - } - - @Override - public URLHelper getSuccessURL(EmailTestForm emailTestForm) - { - return new ActionURL(EmailTestAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Test Email Configuration", getClass()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class RecreateViewsAction extends ConfirmAction - { - @Override - public ModelAndView getConfirmView(Object o, BindException errors) - { - getPageConfig().setShowHeader(false); - getPageConfig().setTitle("Recreate Views?"); - return new HtmlView(HtmlString.of("Are you sure you want to drop and recreate all module views?")); - } - - @Override - public boolean handlePost(Object o, BindException errors) - { - ModuleLoader.getInstance().recreateViews(); - return true; - } - - @Override - public void validateCommand(Object o, Errors errors) - { - } - - @Override - public @NotNull ActionURL getSuccessURL(Object o) - { - return AppProps.getInstance().getHomePageActionURL(); - } - } - - static public class LoggingForm - { - public boolean isLogging() - { - return logging; - } - - public void setLogging(boolean logging) - { - this.logging = logging; - } - - public boolean logging = false; - } - - @RequiresLogin - @AllowedDuringUpgrade - @AllowedBeforeInitialUserIsSet - public static class GetSessionLogEventsAction extends ReadOnlyApiAction - { - @Override - public void checkPermissions() - { - super.checkPermissions(); - if (!getUser().isPlatformDeveloper()) - throw new UnauthorizedException(); - } - - @Override - public ApiResponse execute(Object o, BindException errors) - { - Integer eventId = null; - try - { - String s = getViewContext().getRequest().getParameter("eventId"); - if (null != s) - eventId = Integer.parseInt(s); - } - catch (NumberFormatException ignored) {} - ApiSimpleResponse res = new ApiSimpleResponse(); - res.put("success", true); - res.put("events", SessionAppender.getLoggingEvents(getViewContext().getRequest(), eventId)); - return res; - } - } - - @RequiresLogin - @AllowedBeforeInitialUserIsSet - @AllowedDuringUpgrade - @IgnoresAllocationTracking /* ignore so that we don't get an update in the UI for each time it requests the newest data */ - public static class GetTrackedAllocationsAction extends ReadOnlyApiAction - { - @Override - public void checkPermissions() - { - super.checkPermissions(); - if (!getUser().isPlatformDeveloper()) - throw new UnauthorizedException(); - } - - @Override - public ApiResponse execute(Object o, BindException errors) - { - long requestId = 0; - try - { - String s = getViewContext().getRequest().getParameter("requestId"); - if (null != s) - requestId = Long.parseLong(s); - } - catch (NumberFormatException ignored) {} - List requests = MemTracker.getInstance().getNewRequests(requestId); - List> jsonRequests = new ArrayList<>(requests.size()); - for (RequestInfo requestInfo : requests) - { - Map m = new HashMap<>(); - m.put("requestId", requestInfo.getId()); - m.put("url", requestInfo.getUrl()); - m.put("date", requestInfo.getDate()); - - - List> sortedObjects = sortByCounts(requestInfo); - - List> jsonObjects = new ArrayList<>(sortedObjects.size()); - for (Map.Entry entry : sortedObjects) - { - Map jsonObject = new HashMap<>(); - jsonObject.put("name", entry.getKey()); - jsonObject.put("count", entry.getValue()); - jsonObjects.add(jsonObject); - } - m.put("objects", jsonObjects); - jsonRequests.add(m); - } - return new ApiSimpleResponse("requests", jsonRequests); - } - - private List> sortByCounts(RequestInfo requestInfo) - { - List> objects = new ArrayList<>(requestInfo.getObjects().entrySet()); - objects.sort(Map.Entry.comparingByValue(Comparator.reverseOrder())); - return objects; - } - } - - @RequiresLogin - @AllowedDuringUpgrade - @AllowedBeforeInitialUserIsSet - public static class TrackedAllocationsViewerAction extends SimpleViewAction - { - @Override - public void checkPermissions() - { - super.checkPermissions(); - if (!getUser().isPlatformDeveloper()) - throw new UnauthorizedException(); - } - - @Override - public ModelAndView getView(Object o, BindException errors) - { - getPageConfig().setTemplate(Template.Print); - return new JspView<>("/org/labkey/core/admin/memTrackerViewer.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresLogin - @AllowedDuringUpgrade - @AllowedBeforeInitialUserIsSet - public static class SessionLoggingAction extends FormViewAction - { - @Override - public void checkPermissions() - { - super.checkPermissions(); - if (!getContainer().hasPermission(getUser(), PlatformDeveloperPermission.class)) - throw new UnauthorizedException(); - } - - @Override - public boolean handlePost(LoggingForm form, BindException errors) - { - boolean on = SessionAppender.isLogging(getViewContext().getRequest()); - if (form.logging != on) - { - if (!form.logging) - LogManager.getLogger(AdminController.class).info("turn session logging OFF"); - SessionAppender.setLoggingForSession(getViewContext().getRequest(), form.logging); - if (form.logging) - LogManager.getLogger(AdminController.class).info("turn session logging ON"); - } - return true; - } - - @Override - public void validateCommand(LoggingForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(LoggingForm o, boolean reshow, BindException errors) - { - SessionAppender.setLoggingForSession(getViewContext().getRequest(), true); - getPageConfig().setTemplate(Template.Print); - return new LoggingView(); - } - - @Override - public ActionURL getSuccessURL(LoggingForm o) - { - return new ActionURL(SessionLoggingAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Admin Console", new ActionURL(ShowAdminAction.class, getContainer()).getLocalURIString()); - root.addChild("View Event Log"); - } - } - - static class LoggingView extends JspView - { - LoggingView() - { - super("/org/labkey/core/admin/logging.jsp", null); - } - } - - public static class LogForm - { - private String _message; - private String _level; - - public String getMessage() - { - return _message; - } - - public void setMessage(String message) - { - _message = message; - } - - public String getLevel() - { - return _level; - } - - public void setLevel(String level) - { - _level = level; - } - } - - - // Simple action that writes "message" parameter to the labkey log. Used by the test harness to indicate when - // each test begins and ends. Message parameter is output as sent, except that \n is translated to newline. - @RequiresLogin - public static class LogAction extends MutatingApiAction - { - @Override - public ApiResponse execute(LogForm logForm, BindException errors) - { - // Could use %A0 for newline in the middle of the message, however, parameter values get trimmed so translate - // \n to newlines to allow them at the beginning or end of the message as well. - StringBuilder message = new StringBuilder(); - message.append(StringUtils.replace(logForm.getMessage(), "\\n", "\n")); - - Level level = Level.toLevel(logForm.getLevel(), Level.INFO); - CLIENT_LOG.log(level, message); - return new ApiSimpleResponse("success", true); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public class ValidateDomainsAction extends FormHandlerAction - { - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - // Find a valid pipeline root - we don't really care which one, we just need somewhere to write the log file - for (Container project : Arrays.asList(ContainerManager.getSharedContainer(), ContainerManager.getHomeContainer())) - { - PipeRoot root = PipelineService.get().findPipelineRoot(project); - if (root != null && root.isValid()) - { - ViewBackgroundInfo info = getViewBackgroundInfo(); - PipelineJob job = new ValidateDomainsPipelineJob(info, root); - PipelineService.get().queueJob(job); - return true; - } - } - return false; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return urlProvider(PipelineUrls.class).urlBegin(ContainerManager.getRoot()); - } - } - - public static class ModulesForm - { - private double[] _ignore = new double[0]; // Module versions to ignore (filter out of the results) - private boolean _managedOnly = false; - private boolean _unmanagedOnly = false; - - public double[] getIgnore() - { - return _ignore; - } - - public void setIgnore(double[] ignore) - { - _ignore = ignore; - } - - private Set getIgnoreSet() - { - return new LinkedHashSet<>(Arrays.asList(ArrayUtils.toObject(_ignore))); - } - - public boolean isManagedOnly() - { - return _managedOnly; - } - - @SuppressWarnings("unused") - public void setManagedOnly(boolean managedOnly) - { - _managedOnly = managedOnly; - } - - public boolean isUnmanagedOnly() - { - return _unmanagedOnly; - } - - @SuppressWarnings("unused") - public void setUnmanagedOnly(boolean unmanagedOnly) - { - _unmanagedOnly = unmanagedOnly; - } - } - - public enum ManageFilter - { - ManagedOnly - { - @Override - public boolean accept(Module module) - { - return null != module && module.shouldManageVersion(); - } - }, - UnmanagedOnly - { - @Override - public boolean accept(Module module) - { - return null != module && !module.shouldManageVersion(); - } - }, - All - { - @Override - public boolean accept(Module module) - { - return true; - } - }; - - public abstract boolean accept(Module module); - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public class ModulesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ModulesForm form, BindException errors) - { - ModuleLoader ml = ModuleLoader.getInstance(); - boolean hasAdminOpsPerm = getContainer().hasPermission(getUser(), AdminOperationsPermission.class); - - Collection unknownModules = ml.getUnknownModuleContexts().values(); - Collection knownModules = ml.getAllModuleContexts(); - knownModules.removeAll(unknownModules); - - Set ignoreSet = form.getIgnoreSet(); - HtmlString managedLink = HtmlString.EMPTY_STRING; - HtmlString unmanagedLink = HtmlString.EMPTY_STRING; - - // Option to filter out all modules whose version shouldn't be managed, or whose version matches the previous release - // version or 0.00. This can be helpful during the end-of-release consolidation process. Show the link only in dev mode. - if (AppProps.getInstance().isDevMode()) - { - if (ignoreSet.isEmpty() && !form.isManagedOnly()) - { - String lowestSchemaVersion = ModuleContext.formatVersion(Constants.getLowestSchemaVersion()); - ActionURL url = new ActionURL(ModulesAction.class, ContainerManager.getRoot()); - url.addParameter("ignore", "0.00," + lowestSchemaVersion); - url.addParameter("managedOnly", true); - managedLink = LinkBuilder.labkeyLink("Click here to ignore null, " + lowestSchemaVersion + " and unmanaged modules", url).getHtmlString(); - } - else - { - List ignore = ignoreSet - .stream() - .map(ModuleContext::formatVersion) - .collect(Collectors.toCollection(LinkedList::new)); - - String ignoreString = ignore.isEmpty() ? null : ignore.toString(); - String unmanaged = form.isManagedOnly() ? "unmanaged" : null; - - managedLink = HtmlString.of("(Currently ignoring " + Joiner.on(" and ").skipNulls().join(new String[]{ignoreString, unmanaged}) + ") "); - } - - if (!form.isUnmanagedOnly()) - { - ActionURL url = new ActionURL(ModulesAction.class, ContainerManager.getRoot()); - url.addParameter("unmanagedOnly", true); - unmanagedLink = LinkBuilder.labkeyLink("Click here to show unmanaged modules only", url).getHtmlString(); - } - else - { - unmanagedLink = HtmlString.of("(Currently showing unmanaged modules only)"); - } - } - - ManageFilter filter = form.isManagedOnly() ? ManageFilter.ManagedOnly : (form.isUnmanagedOnly() ? ManageFilter.UnmanagedOnly : ManageFilter.All); - - HtmlStringBuilder deleteInstructions = HtmlStringBuilder.of(); - if (hasAdminOpsPerm) - { - deleteInstructions.unsafeAppend("

    ").append( - "To delete a module that does not have a delete link, first delete its .module file and exploded module directory from your Labkey deployment directory, and restart the server. " + - "Module files are typically deployed in /modules and /externalModules.") - .unsafeAppend("

    ").append( - LinkBuilder.labkeyLink("Create new empty module", getCreateURL())); - } - - HtmlStringBuilder docLink = HtmlStringBuilder.of(); - docLink.unsafeAppend("

    ").append("Additional modules available, click ").append(new HelpTopic("defaultModules").getSimpleLinkHtml("here")).append(" to learn more."); - - HtmlStringBuilder knownDescription = HtmlStringBuilder.of() - .append("Each of these modules is installed and has a valid module file. ").append(managedLink).append(unmanagedLink).append(deleteInstructions).append(docLink); - HttpView known = new ModulesView(knownModules, "Known", knownDescription.getHtmlString(), null, ignoreSet, filter); - - HtmlStringBuilder unknownDescription = HtmlStringBuilder.of() - .append(1 == unknownModules.size() ? "This module" : "Each of these modules").append(" has been installed on this server " + - "in the past but the corresponding module file is currently missing or invalid. Possible explanations: the " + - "module is no longer part of the deployed distribution, the module has been renamed, the server location where the module " + - "is stored is not accessible, or the module file is corrupted.") - .unsafeAppend("

    ").append("The delete links below will remove all record of a module from the database tables."); - HtmlString noModulesDescription = HtmlString.of("A module is considered \"unknown\" if it was installed on this server " + - "in the past but the corresponding module file is currently missing or invalid. This server has no unknown modules."); - HttpView unknown = new ModulesView(unknownModules, "Unknown", unknownDescription.getHtmlString(), noModulesDescription, Collections.emptySet(), filter); - - return new VBox(known, unknown); - } - - private class ModulesView extends WebPartView - { - private final Collection _contexts; - private final HtmlString _descriptionHtml; - private final HtmlString _noModulesDescriptionHtml; - private final Set _ignoreVersions; - private final ManageFilter _manageFilter; - - private ModulesView(Collection contexts, String type, HtmlString descriptionHtml, HtmlString noModulesDescriptionHtml, Set ignoreVersions, ManageFilter manageFilter) - { - super(FrameType.PORTAL); - List sorted = new ArrayList<>(contexts); - sorted.sort(Comparator.comparing(ModuleContext::getName, String.CASE_INSENSITIVE_ORDER)); - - _contexts = sorted; - _descriptionHtml = descriptionHtml; - _noModulesDescriptionHtml = noModulesDescriptionHtml; - _ignoreVersions = ignoreVersions; - _manageFilter = manageFilter; - setTitle(type + " Modules"); - } - - @Override - protected void renderView(Object model, HtmlWriter out) - { - boolean isDevMode = AppProps.getInstance().isDevMode(); - boolean hasAdminOpsPerm = getUser().hasRootPermission(AdminOperationsPermission.class); - boolean hasUploadModulePerm = getUser().hasRootPermission(UploadFileBasedModulePermission.class); - final AtomicInteger rowCount = new AtomicInteger(); - ExplodedModuleService moduleService = !hasUploadModulePerm ? null : ServiceRegistry.get().getService(ExplodedModuleService.class); - final File externalModulesDir = moduleService==null ? null : moduleService.getExternalModulesDirectory(); - final Path relativeRoot = ModuleLoader.getInstance().getCoreModule().getExplodedPath().getParentFile().getParentFile().toPath(); - - if (_contexts.isEmpty()) - { - out.write(_noModulesDescriptionHtml); - } - else - { - DIV( - DIV(_descriptionHtml), - TABLE(cl("labkey-data-region-legacy","labkey-show-borders","labkey-data-region-header-lock"), - TR( - TD(cl("labkey-column-header"),"Name"), - TD(cl("labkey-column-header"),"Release Version"), - TD(cl("labkey-column-header"),"Schema Version"), - TD(cl("labkey-column-header"),"Class"), - TD(cl("labkey-column-header"),"Location"), - TD(cl("labkey-column-header"),"Schemas"), - !AppProps.getInstance().isDevMode() ? null : TD(cl("labkey-column-header"),""), // edit actions - null == externalModulesDir ? null : TD(cl("labkey-column-header"),""), // upload actions - !hasAdminOpsPerm ? null : TD(cl("labkey-column-header"),"") // delete actions - ), - _contexts.stream() - .filter(moduleContext -> !_ignoreVersions.contains(moduleContext.getInstalledVersion())) - .map(moduleContext -> new Pair<>(moduleContext,ModuleLoader.getInstance().getModule(moduleContext.getName()))) - .filter(pair -> _manageFilter.accept(pair.getValue())) - .map(pair -> - { - ModuleContext moduleContext = pair.getKey(); - Module module = pair.getValue(); - List schemas = moduleContext.getSchemaList(); - Double schemaVersion = moduleContext.getSchemaVersion(); - boolean replaceableModule = false; - if (null != module && module.getClass() == SimpleModule.class && schemas.isEmpty()) - { - File zip = module.getZippedPath(); - if (null != zip && zip.getParentFile().equals(externalModulesDir)) - replaceableModule = true; - } - boolean deleteableModule = replaceableModule || null == module; - String className = StringUtils.trimToEmpty(moduleContext.getClassName()); - String fullPathToModule = ""; - String shortPathToModule = ""; - if (null != module) - { - Path p = module.getExplodedPath().toPath(); - if (null != module.getZippedPath()) - p = module.getZippedPath().toPath(); - if (isDevMode && ModuleEditorService.get().canEditSourceModule(module)) - if (!module.getExplodedPath().getPath().equals(module.getSourcePath())) - p = Paths.get(module.getSourcePath()); - fullPathToModule = p.toString(); - shortPathToModule = fullPathToModule; - Path rel = relativeRoot.relativize(p); - if (!rel.startsWith("..")) - shortPathToModule = rel.toString(); - } - ActionURL moduleEditorUrl = getModuleEditorURL(moduleContext.getName()); - - return TR(cl(rowCount.getAndIncrement()%2==0 ? "labkey-alternate-row" : "labkey-row").at(style,"vertical-align:top;"), - TD(moduleContext.getName()), - TD(at(style,"white-space:nowrap;"), null != module ? module.getReleaseVersion() : NBSP), - TD(null != schemaVersion ? ModuleContext.formatVersion(schemaVersion) : NBSP), - TD(SPAN(at(title,className), className.substring(className.lastIndexOf(".")+1))), - TD(SPAN(at(title,fullPathToModule),shortPathToModule)), - TD(schemas.stream().map(s -> createHtmlFragment(s, BR()))), - !AppProps.getInstance().isDevMode() ? null : TD((null == moduleEditorUrl) ? NBSP : LinkBuilder.labkeyLink("Edit module", moduleEditorUrl)), - null == externalModulesDir ? null : TD(!replaceableModule ? NBSP : LinkBuilder.labkeyLink("Upload Module", getUpdateURL(moduleContext.getName()))), - !hasAdminOpsPerm ? null : TD(!deleteableModule ? NBSP : LinkBuilder.labkeyLink("Delete Module" + (schemas.isEmpty() ? "" : (" and Schema" + (schemas.size() > 1 ? "s" : ""))), getDeleteURL(moduleContext.getName()))) - ); - }) - ) - ).appendTo(out); - } - } - } - - private ActionURL getDeleteURL(String name) - { - ActionURL url = ModuleEditorService.get().getDeleteModuleURL(name); - if (null != url) - return url; - url = new ActionURL(DeleteModuleAction.class, ContainerManager.getRoot()); - url.addParameter("name", name); - return url; - } - - private ActionURL getUpdateURL(String name) - { - ActionURL url = ModuleEditorService.get().getUpdateModuleURL(name); - if (null != url) - return url; - url = new ActionURL(UpdateModuleAction.class, ContainerManager.getRoot()); - url.addParameter("name", name); - return url; - } - - private ActionURL getModuleEditorURL(String name) - { - return ModuleEditorService.get().getModuleEditorURL(name); - } - - private ActionURL getCreateURL() - { - ActionURL url = ModuleEditorService.get().getCreateModuleURL(); - if (null != url) - return url; - url = new ActionURL(CreateModuleAction.class, ContainerManager.getRoot()); - return url; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("defaultModules"); - addAdminNavTrail(root, "Modules", getClass()); - } - } - - public static class SchemaVersionTestCase extends Assert - { - @Test - public void verifyMinimumSchemaVersion() - { - List modulesTooLow = ModuleLoader.getInstance().getModules().stream() - .filter(ManageFilter.ManagedOnly::accept) - .filter(m -> null != m.getSchemaVersion()) - .filter(m -> m.getSchemaVersion() > 0.00 && m.getSchemaVersion() < Constants.getLowestSchemaVersion()) - .toList(); - - if (!modulesTooLow.isEmpty()) - fail("The following module" + (1 == modulesTooLow.size() ? " needs its schema version" : "s need their schema versions") + " increased to " + ModuleContext.formatVersion(Constants.getLowestSchemaVersion()) + ": " + modulesTooLow); - } - - @Test - public void modulesWithSchemaVersionButNoScripts() - { - // Flag all modules that have a schema version but don't have scripts. Their schema version should be null. - List moduleNames = ModuleLoader.getInstance().getModules().stream() - .filter(m -> m.getSchemaVersion() != null) - .filter(m -> m instanceof DefaultModule dm && !dm.hasScripts()) - .map(m -> m.getName() + ": " + m.getSchemaVersion()) - .toList(); - - if (!moduleNames.isEmpty()) - fail("The following module" + (1 == moduleNames.size() ? "" : "s") + " should have a null schema version: " + moduleNames); - } - } - - public static class ModuleForm - { - private String _name; - - public String getName() - { - return _name; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setName(String name) - { - _name = name; - } - - @NotNull - private ModuleContext getModuleContext() - { - ModuleLoader ml = ModuleLoader.getInstance(); - ModuleContext ctx = ml.getModuleContextFromDatabase(getName()); - - if (null == ctx) - throw new NotFoundException("Module not found"); - - return ctx; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public class DeleteModuleAction extends ConfirmAction - { - @Override - public void validateCommand(ModuleForm form, Errors errors) - { - } - - @Override - public ModelAndView getConfirmView(ModuleForm form, BindException errors) - { - if (getPageConfig().getTitle() == null) - setTitle("Delete Module"); - - ModuleContext ctx = form.getModuleContext(); - Module module = ModuleLoader.getInstance().getModule(ctx.getName()); - boolean hasSchemas = !ctx.getSchemaList().isEmpty(); - boolean hasFiles = false; - if (null != module) - hasFiles = null!=module.getExplodedPath() && module.getExplodedPath().isDirectory() || null!=module.getZippedPath() && module.getZippedPath().isFile(); - - HtmlStringBuilder description = HtmlStringBuilder.of("\"" + ctx.getName() + "\" module"); - HtmlStringBuilder skippedSchemas = HtmlStringBuilder.of(); - if (hasSchemas) - { - SchemaActions schemaActions = ModuleLoader.getInstance().getSchemaActions(module, ctx); - List deleteList = schemaActions.deleteList(); - List skipList = schemaActions.skipList(); - - // List all the schemas that will be deleted - if (!deleteList.isEmpty()) - { - description.append(" and delete all data in "); - description.append(deleteList.size() > 1 ? "these schemas: " + StringUtils.join(deleteList, ", ") : "the \"" + deleteList.get(0) + "\" schema"); - } - - // For unknown modules, also list the schemas that won't be deleted - if (!skipList.isEmpty()) - { - skippedSchemas.append(HtmlString.BR); - skipList.forEach(sam -> skippedSchemas.append(HtmlString.BR) - .append("Note: Schema \"") - .append(sam.schema()) - .append("\" will not be deleted because it's in use by module \"") - .append(sam.module()) - .append("\"")); - } - } - - return new HtmlView(DIV( - !hasFiles ? null : DIV(cl("labkey-warning-messages"), - "This module still has files on disk. Consider, first stopping the server, deleting these files, and restarting the server before continuing.", - null==module.getExplodedPath()?null:UL(LI(module.getExplodedPath().getPath())), - null==module.getZippedPath()?null:UL(LI(module.getZippedPath().getPath())) - ), - BR(), - "Are you sure you want to remove the ", description, "? ", - "This operation cannot be undone!", - skippedSchemas, - BR(), - !hasFiles ? null : "Deleting modules on a running server could leave it in an unpredictable state; be sure to restart your server." - )); - } - - @Override - public boolean handlePost(ModuleForm form, BindException errors) - { - ModuleLoader.getInstance().removeModule(form.getModuleContext()); - - return true; - } - - @Override - public @NotNull URLHelper getSuccessURL(ModuleForm form) - { - return new ActionURL(ModulesAction.class, ContainerManager.getRoot()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class UpdateModuleAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ModuleForm moduleForm, BindException errors) throws Exception - { - return new HtmlView(HtmlString.of("This is a premium feature, please refer to our documentation on www.labkey.org")); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class CreateModuleAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ModuleForm moduleForm, BindException errors) throws Exception - { - return new HtmlView(HtmlString.of("This is a premium feature, please refer to our documentation on www.labkey.org")); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - public static class OptionalFeatureForm - { - private String feature; - private boolean enabled; - - public String getFeature() - { - return feature; - } - - public void setFeature(String feature) - { - this.feature = feature; - } - - public boolean isEnabled() - { - return enabled; - } - - public void setEnabled(boolean enabled) - { - this.enabled = enabled; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - @ActionNames("OptionalFeature, ExperimentalFeature") - public static class OptionalFeatureAction extends BaseApiAction - { - @Override - protected ModelAndView handleGet() throws Exception - { - return handlePost(); // 'execute' ensures that only POSTs are mutating - } - - @Override - public ApiResponse execute(OptionalFeatureForm form, BindException errors) - { - String feature = StringUtils.trimToNull(form.getFeature()); - if (feature == null) - throw new ApiUsageException("feature is required"); - - OptionalFeatureService svc = OptionalFeatureService.get(); - if (svc == null) - throw new IllegalStateException(); - - Map ret = new HashMap<>(); - ret.put("feature", feature); - - if (isPost()) - { - ret.put("previouslyEnabled", svc.isFeatureEnabled(feature)); - svc.setFeatureEnabled(feature, form.isEnabled(), getUser()); - } - - ret.put("enabled", svc.isFeatureEnabled(feature)); - return new ApiSimpleResponse(ret); - } - } - - public static class OptionalFeaturesForm - { - private String _type; - private boolean _showHidden; - - public String getType() - { - return _type; - } - - @SuppressWarnings("unused") - public void setType(String type) - { - _type = type; - } - - public @NotNull FeatureType getTypeEnum() - { - return EnumUtils.getEnum(FeatureType.class, getType(), FeatureType.Experimental); - } - - public boolean isShowHidden() - { - return _showHidden; - } - - @SuppressWarnings("unused") - public void setShowHidden(boolean showHidden) - { - _showHidden = showHidden; - } - } - - @RequiresPermission(TroubleshooterPermission.class) - public class OptionalFeaturesAction extends SimpleViewAction - { - private FeatureType _type; - - @Override - public ModelAndView getView(OptionalFeaturesForm form, BindException errors) - { - _type = form.getTypeEnum(); - JspView view = new JspView<>("/org/labkey/core/admin/optionalFeatures.jsp", form); - view.setFrame(WebPartView.FrameType.NONE); - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("experimental"); - addAdminNavTrail(root, _type.name() + " Features", getClass()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class ProductFeatureAction extends BaseApiAction - { - @Override - protected ModelAndView handleGet() throws Exception - { - return handlePost(); // 'execute' ensures that only POSTs are mutating - } - - @Override - public ApiResponse execute(ProductConfigForm form, BindException errors) - { - String productKey = StringUtils.trimToNull(form.getProductKey()); - - Map ret = new HashMap<>(); - - if (isPost()) - { - ProductConfiguration.setProductKey(productKey); - } - - ret.put("productKey", new ProductConfiguration().getCurrentProductKey()); - return new ApiSimpleResponse(ret); - } - } - - public static class ProductConfigForm - { - private String productKey; - - public String getProductKey() - { - return productKey; - } - - public void setProductKey(String productKey) - { - this.productKey = productKey; - } - - } - - @AdminConsoleAction - @RequiresPermission(AdminOperationsPermission.class) - public class ProductConfigurationAction extends SimpleViewAction - { - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Product Configuration", getClass()); - } - - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - JspView view = new JspView<>("/org/labkey/core/admin/productConfiguration.jsp"); - view.setFrame(WebPartView.FrameType.NONE); - return view; - } - } - - - public static class FolderTypesBean - { - private final Collection _allFolderTypes; - private final Collection _enabledFolderTypes; - private final FolderType _defaultFolderType; - - public FolderTypesBean(Collection allFolderTypes, Collection enabledFolderTypes, FolderType defaultFolderType) - { - _allFolderTypes = allFolderTypes; - _enabledFolderTypes = enabledFolderTypes; - _defaultFolderType = defaultFolderType; - } - - public Collection getAllFolderTypes() - { - return _allFolderTypes; - } - - public Collection getEnabledFolderTypes() - { - return _enabledFolderTypes; - } - - public FolderType getDefaultFolderType() - { - return _defaultFolderType; - } - } - - @AdminConsoleAction - @RequiresPermission(AdminPermission.class) - public class FolderTypesAction extends FormViewAction - { - @Override - public void validateCommand(Object form, Errors errors) - { - } - - @Override - public ModelAndView getView(Object form, boolean reshow, BindException errors) - { - FolderTypesBean bean; - if (reshow) - { - bean = getOptionsFromRequest(); - } - else - { - FolderTypeManager manager = FolderTypeManager.get(); - var defaultFolderType = manager.getDefaultFolderType(); - // If a default folder type has not yet been configuration use "Collaboration" folder type as the default - defaultFolderType = defaultFolderType != null ? defaultFolderType : manager.getFolderType(CollaborationFolderType.TYPE_NAME); - boolean userHasEnableRestrictedModulesPermission = getContainer().hasEnableRestrictedModules(getUser()); - bean = new FolderTypesBean(manager.getAllFolderTypes(), manager.getEnabledFolderTypes(userHasEnableRestrictedModulesPermission), defaultFolderType); - } - - return new JspView<>("/org/labkey/core/admin/enabledFolderTypes.jsp", bean, errors); - } - - @Override - public boolean handlePost(Object form, BindException errors) - { - FolderTypesBean bean = getOptionsFromRequest(); - var defaultFolderType = bean.getDefaultFolderType(); - if (defaultFolderType == null) - { - errors.reject(ERROR_MSG, "Please select a default folder type."); - return false; - } - var enabledFolderTypes = bean.getEnabledFolderTypes(); - if (!enabledFolderTypes.contains(defaultFolderType)) - { - errors.reject(ERROR_MSG, "Folder type selected as the default, '" + defaultFolderType.getName() + "', must be enabled."); - return false; - } - - FolderTypeManager.get().setEnabledFolderTypes(enabledFolderTypes, defaultFolderType); - return true; - } - - private FolderTypesBean getOptionsFromRequest() - { - var allFolderTypes = FolderTypeManager.get().getAllFolderTypes(); - List enabledFolderTypes = new ArrayList<>(); - FolderType defaultFolderType = null; - String defaultFolderTypeParam = getViewContext().getRequest().getParameter(FolderTypeManager.FOLDER_TYPE_DEFAULT); - - for (FolderType folderType : FolderTypeManager.get().getAllFolderTypes()) - { - boolean enabled = Boolean.TRUE.toString().equalsIgnoreCase(getViewContext().getRequest().getParameter(folderType.getName())); - if (enabled) - { - enabledFolderTypes.add(folderType); - } - if (folderType.getName().equals(defaultFolderTypeParam)) - { - defaultFolderType = folderType; - } - } - return new FolderTypesBean(allFolderTypes, enabledFolderTypes, defaultFolderType); - } - - @Override - public URLHelper getSuccessURL(Object form) - { - return getShowAdminURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Folder Types", getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class CustomizeMenuAction extends MutatingApiAction - { - @Override - public ApiResponse execute(CustomizeMenuForm form, BindException errors) - { - if (null != form.getUrl()) - { - String errorMessage = StringExpressionFactory.validateURL(form.getUrl()); - if (null != errorMessage) - { - errors.reject(ERROR_MSG, errorMessage); - return new ApiSimpleResponse("success", false); - } - } - - setCustomizeMenuForm(form, getContainer(), getUser()); - return new ApiSimpleResponse("success", true); - } - } - - protected static final String CUSTOMMENU_SCHEMA = "customMenuSchemaName"; - protected static final String CUSTOMMENU_QUERY = "customMenuQueryName"; - protected static final String CUSTOMMENU_VIEW = "customMenuViewName"; - protected static final String CUSTOMMENU_COLUMN = "customMenuColumnName"; - protected static final String CUSTOMMENU_FOLDER = "customMenuFolderName"; - protected static final String CUSTOMMENU_TITLE = "customMenuTitle"; - protected static final String CUSTOMMENU_URL = "customMenuUrl"; - protected static final String CUSTOMMENU_ROOTFOLDER = "customMenuRootFolder"; - protected static final String CUSTOMMENU_FOLDERTYPES = "customMenuFolderTypes"; - protected static final String CUSTOMMENU_CHOICELISTQUERY = "customMenuChoiceListQuery"; - protected static final String CUSTOMMENU_INCLUDEALLDESCENDANTS = "customIncludeAllDescendants"; - protected static final String CUSTOMMENU_CURRENTPROJECTONLY = "customCurrentProjectOnly"; - - public static CustomizeMenuForm getCustomizeMenuForm(Portal.WebPart webPart) - { - CustomizeMenuForm form = new CustomizeMenuForm(); - Map menuProps = webPart.getPropertyMap(); - - String schemaName = menuProps.get(CUSTOMMENU_SCHEMA); - String queryName = menuProps.get(CUSTOMMENU_QUERY); - String columnName = menuProps.get(CUSTOMMENU_COLUMN); - String viewName = menuProps.get(CUSTOMMENU_VIEW); - String folderName = menuProps.get(CUSTOMMENU_FOLDER); - String title = menuProps.get(CUSTOMMENU_TITLE); if (null == title) title = "My Menu"; - String urlBottom = menuProps.get(CUSTOMMENU_URL); - String rootFolder = menuProps.get(CUSTOMMENU_ROOTFOLDER); - String folderTypes = menuProps.get(CUSTOMMENU_FOLDERTYPES); - String choiceListQueryString = menuProps.get(CUSTOMMENU_CHOICELISTQUERY); - boolean choiceListQuery = null == choiceListQueryString || choiceListQueryString.equalsIgnoreCase("true"); - String includeAllDescendantsString = menuProps.get(CUSTOMMENU_INCLUDEALLDESCENDANTS); - boolean includeAllDescendants = null == includeAllDescendantsString || includeAllDescendantsString.equalsIgnoreCase("true"); - String currentProjectOnlyString = menuProps.get(CUSTOMMENU_CURRENTPROJECTONLY); - boolean currentProjectOnly = null != currentProjectOnlyString && currentProjectOnlyString.equalsIgnoreCase("true"); - - form.setSchemaName(schemaName); - form.setQueryName(queryName); - form.setColumnName(columnName); - form.setViewName(viewName); - form.setFolderName(folderName); - form.setTitle(title); - form.setUrl(urlBottom); - form.setRootFolder(rootFolder); - form.setFolderTypes(folderTypes); - form.setChoiceListQuery(choiceListQuery); - form.setIncludeAllDescendants(includeAllDescendants); - form.setCurrentProjectOnly(currentProjectOnly); - - form.setWebPartIndex(webPart.getIndex()); - form.setPageId(webPart.getPageId()); - return form; - } - - private static void setCustomizeMenuForm(CustomizeMenuForm form, Container container, User user) - { - Portal.WebPart webPart = Portal.getPart(container, form.getPageId(), form.getWebPartIndex()); - if (null == webPart) - throw new NotFoundException(); - Map menuProps = webPart.getPropertyMap(); - - menuProps.put(CUSTOMMENU_SCHEMA, form.getSchemaName()); - menuProps.put(CUSTOMMENU_QUERY, form.getQueryName()); - menuProps.put(CUSTOMMENU_COLUMN, form.getColumnName()); - menuProps.put(CUSTOMMENU_VIEW, form.getViewName()); - menuProps.put(CUSTOMMENU_FOLDER, form.getFolderName()); - menuProps.put(CUSTOMMENU_TITLE, form.getTitle()); - menuProps.put(CUSTOMMENU_URL, form.getUrl()); - - // If root folder not specified, set as current container - menuProps.put(CUSTOMMENU_ROOTFOLDER, StringUtils.trimToNull(form.getRootFolder()) != null ? form.getRootFolder() : container.getPath()); - menuProps.put(CUSTOMMENU_FOLDERTYPES, form.getFolderTypes()); - menuProps.put(CUSTOMMENU_CHOICELISTQUERY, form.isChoiceListQuery() ? "true" : "false"); - menuProps.put(CUSTOMMENU_INCLUDEALLDESCENDANTS, form.isIncludeAllDescendants() ? "true" : "false"); - menuProps.put(CUSTOMMENU_CURRENTPROJECTONLY, form.isCurrentProjectOnly() ? "true" : "false"); - - Portal.updatePart(user, webPart); - } - - @RequiresPermission(AdminPermission.class) - public static class AddTabAction extends MutatingApiAction - { - public void validateCommand(TabActionForm form, Errors errors) - { - Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); - if(tabContainer.getFolderType() == FolderType.NONE) - { - errors.reject(ERROR_MSG, "Cannot add tabs to custom folder types."); - } - else - { - String name = form.getTabName(); - if (StringUtils.isEmpty(name)) - { - errors.reject(ERROR_MSG, "A tab name must be specified."); - return; - } - - // Note: The name, which shows up on the url, is trimmed to 50 characters. The caption, which is derived - // from the name, and is editable, is allowed to be 64 characters, so we only error if passed something - // longer than 64 characters. - if (name.length() > 64) - { - errors.reject(ERROR_MSG, "Tab name cannot be longer than 64 characters."); - return; - } - - if (name.length() > 50) - name = name.substring(0, 50).trim(); - - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); - CaseInsensitiveHashMap folderTabMap = new CaseInsensitiveHashMap<>(); - - for (FolderTab tab : tabContainer.getFolderType().getDefaultTabs()) - { - folderTabMap.put(tab.getName(), tab); - } - - if (pages.containsKey(name)) - { - errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); - return; - } - - for (Portal.PortalPage page : pages.values()) - { - if (page.getCaption() != null && page.getCaption().equals(name)) - { - errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); - return; - } - else if (folderTabMap.containsKey(page.getPageId())) - { - if (folderTabMap.get(page.getPageId()).getCaption(getViewContext()).equalsIgnoreCase(name)) - { - errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); - return; - } - } - } - } - } - - @Override - public ApiResponse execute(TabActionForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - - validateCommand(form, errors); - - if(errors.hasErrors()) - { - return response; - } - - Container container = getContainer().getContainerFor(ContainerType.DataType.tabParent); - String name = form.getTabName(); - String caption = form.getTabName(); - - // The name, which shows up on the url, is trimmed to 50 characters. The caption, which is derived from the - // name, and is editable, is allowed to be 64 characters. - if (name.length() > 50) - name = name.substring(0, 50).trim(); - - Portal.saveParts(container, name); - Portal.addProperty(container, name, Portal.PROP_CUSTOMTAB); - - if (!name.equals(caption)) - { - // If we had to truncate the name then we want to set the caption to the un-truncated version of the name. - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(container, true)); - Portal.PortalPage page = pages.get(name); - // Get a mutable copy - page = page.copy(); - page.setCaption(caption); - Portal.updatePortalPage(container, page); - } - - ActionURL tabURL = new ActionURL(ProjectController.BeginAction.class, container); - tabURL.addParameter("pageId", name); - response.put("url", tabURL); - response.put("success", true); - return response; - } - } - - @RequiresPermission(AdminPermission.class) - public static class ShowTabAction extends MutatingApiAction - { - public void validateCommand(TabActionForm form, Errors errors) - { - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(getContainer().getContainerFor(ContainerType.DataType.tabParent), true)); - - if (form.getTabPageId() == null) - { - errors.reject(ERROR_MSG, "PageId cannot be blank."); - } - - if (!pages.containsKey(form.getTabPageId())) - { - errors.reject(ERROR_MSG, "Page cannot be found. Check with your system administrator."); - } - } - - @Override - public ApiResponse execute(TabActionForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); - - validateCommand(form, errors); - if (errors.hasErrors()) - return response; - - Portal.showPage(tabContainer, form.getTabPageId()); - ActionURL tabURL = new ActionURL(ProjectController.BeginAction.class, tabContainer); - tabURL.addParameter("pageId", form.getTabPageId()); - response.put("url", tabURL); - response.put("success", true); - return response; - } - } - - - public static class TabActionForm extends ReturnUrlForm - { - // This class is used for tab related actions (add, rename, show, etc.) - String _tabName; - String _tabPageId; - - public String getTabName() - { - return _tabName; - } - - public void setTabName(String name) - { - _tabName = name; - } - - public String getTabPageId() - { - return _tabPageId; - } - - public void setTabPageId(String tabPageId) - { - _tabPageId = tabPageId; - } - } - - @RequiresPermission(AdminPermission.class) - public class MoveTabAction extends MutatingApiAction - { - @Override - public ApiResponse execute(MoveTabForm form, BindException errors) - { - final Map properties = new HashMap<>(); - Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); - Portal.PortalPage tab = pages.get(form.getPageId()); - - if (null != tab) - { - int oldIndex = tab.getIndex(); - Portal.PortalPage pageToSwap = handleMovePortalPage(tabContainer, getUser(), tab, form.getDirection()); - - if (null != pageToSwap) - { - properties.put("oldIndex", oldIndex); - properties.put("newIndex", tab.getIndex()); - properties.put("pageId", tab.getPageId()); - properties.put("pageIdToSwap", pageToSwap.getPageId()); - } - else - { - properties.put("error", "Unable to move tab."); - } - } - else - { - properties.put("error", "Requested tab does not exist."); - } - - return new ApiSimpleResponse(properties); - } - } - - public static class MoveTabForm implements HasViewContext - { - private int _direction; - private String _pageId; - private ViewContext _viewContext; - - public int getDirection() - { - // 0 moves left, 1 moves right. - return _direction; - } - - public void setDirection(int direction) - { - _direction = direction; - } - - public String getPageId() - { - return _pageId; - } - - public void setPageId(String pageId) - { - _pageId = pageId; - } - - @Override - public ViewContext getViewContext() - { - return _viewContext; - } - - @Override - public void setViewContext(ViewContext viewContext) - { - _viewContext = viewContext; - } - } - - private Portal.PortalPage handleMovePortalPage(Container c, User user, Portal.PortalPage page, int direction) - { - Map pageMap = new CaseInsensitiveHashMap<>(); - for (Portal.PortalPage pp : Portal.getTabPages(c, true)) - pageMap.put(pp.getPageId(), pp); - - for (FolderTab folderTab : c.getFolderType().getDefaultTabs()) - { - if (pageMap.containsKey(folderTab.getName())) - { - // Issue 46233 : folder tabs can conditionally hide/show themselves at render time, these need to - // be excluded when adjusting the relative indexes. - if (!folderTab.isVisible(c, user)) - pageMap.remove(folderTab.getName()); - } - } - List pagesList = new ArrayList<>(pageMap.values()); - pagesList.sort(Comparator.comparingInt(Portal.PortalPage::getIndex)); - - int visibleIndex; - for (visibleIndex = 0; visibleIndex < pagesList.size(); visibleIndex++) - { - if (pagesList.get(visibleIndex).getIndex() == page.getIndex()) - { - break; - } - } - - if (visibleIndex == pagesList.size()) - { - return null; - } - - if (direction == Portal.MOVE_DOWN) - { - if (visibleIndex == pagesList.size() - 1) - { - return page; - } - - Portal.PortalPage nextPage = pagesList.get(visibleIndex + 1); - - if (null == nextPage) - return null; - Portal.swapPageIndexes(c, page, nextPage); - return nextPage; - } - else - { - if (visibleIndex < 1) - { - return page; - } - - Portal.PortalPage prevPage = pagesList.get(visibleIndex - 1); - - if (null == prevPage) - return null; - Portal.swapPageIndexes(c, page, prevPage); - return prevPage; - } - } - - @RequiresPermission(AdminPermission.class) - public static class RenameTabAction extends MutatingApiAction - { - public void validateCommand(TabActionForm form, Errors errors) - { - Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); - - if (tabContainer.getFolderType() == FolderType.NONE) - { - errors.reject(ERROR_MSG, "Cannot change tab names in custom folder types."); - } - else - { - String name = form.getTabName(); - if (StringUtils.isEmpty(name)) - { - errors.reject(ERROR_MSG, "A tab name must be specified."); - return; - } - - if (name.length() > 64) - { - errors.reject(ERROR_MSG, "Tab name cannot be longer than 64 characters."); - return; - } - - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); - Portal.PortalPage pageToChange = pages.get(form.getTabPageId()); - if (null == pageToChange) - { - errors.reject(ERROR_MSG, "Page cannot be found. Check with your system administrator."); - return; - } - - for (Portal.PortalPage page : pages.values()) - { - if (!page.equals(pageToChange)) - { - if (null != page.getCaption() && page.getCaption().equalsIgnoreCase(name)) - { - errors.reject(ERROR_MSG, "A tab with the same name already exists in this folder."); - return; - } - if (page.getPageId().equalsIgnoreCase(name)) - { - if (null != page.getCaption() || Portal.DEFAULT_PORTAL_PAGE_ID.equalsIgnoreCase(name)) - errors.reject(ERROR_MSG, "You cannot change a tab's name to another tab's original name even if the original name is not visible."); - else - errors.reject(ERROR_MSG, "A tab with the same name already exists in this folder."); - return; - } - } - } - - List folderTabs = tabContainer.getFolderType().getDefaultTabs(); - for (FolderTab folderTab : folderTabs) - { - String folderTabCaption = folderTab.getCaption(getViewContext()); - if (!folderTab.getName().equalsIgnoreCase(pageToChange.getPageId()) && null != folderTabCaption && folderTabCaption.equalsIgnoreCase(name)) - { - errors.reject(ERROR_MSG, "You cannot change a tab's name to another tab's original name even if the original name is not visible."); - return; - } - } - } - } - - @Override - public ApiResponse execute(TabActionForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - validateCommand(form, errors); - - if (errors.hasErrors()) - { - return response; - } - - Container container = getContainer().getContainerFor(ContainerType.DataType.tabParent); - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(container, true)); - Portal.PortalPage page = pages.get(form.getTabPageId()); - page = page.copy(); - page.setCaption(form.getTabName()); - // Update the page the caption is saved. - Portal.updatePortalPage(container, page); - - response.put("success", true); - return response; - } - } - - @RequiresPermission(ReadPermission.class) - public static class ClearDeletedTabFoldersAction extends MutatingApiAction - { - @Override - public ApiResponse execute(DeletedFoldersForm form, BindException errors) - { - if (isBlank(form.getContainerPath())) - throw new NotFoundException(); - Container container = ContainerManager.getForPath(form.getContainerPath()); - for (String tabName : form.getResurrectFolders()) - { - ContainerManager.clearContainerTabDeleted(container, tabName, form.getNewFolderType()); - } - return new ApiSimpleResponse("success", true); - } - } - - @SuppressWarnings("unused") - public static class DeletedFoldersForm - { - private String _containerPath; - private String _newFolderType; - private List _resurrectFolders; - - public List getResurrectFolders() - { - return _resurrectFolders; - } - - public void setResurrectFolders(List resurrectFolders) - { - _resurrectFolders = resurrectFolders; - } - - public String getContainerPath() - { - return _containerPath; - } - - public void setContainerPath(String containerPath) - { - _containerPath = containerPath; - } - - public String getNewFolderType() - { - return _newFolderType; - } - - public void setNewFolderType(String newFolderType) - { - _newFolderType = newFolderType; - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetFolderTabsAction extends ReadOnlyApiAction - { - @Override - public Object execute(Object form, BindException errors) throws Exception - { - var data = getContainer() - .getFolderType() - .getAppBar(getViewContext(), getPageConfig()) - .getButtons() - .stream() - .map(this::getProperties) - .toList(); - - return success(data); - } - - private Map getProperties(NavTree navTree) - { - Map props = new HashMap<>(); - props.put("id", navTree.getId()); - props.put("text", navTree.getText()); - props.put("href", navTree.getHref()); - props.put("disabled", navTree.isDisabled()); - return props; - } - } - - @SuppressWarnings("unused") - public static class ShortURLForm - { - private String _shortURL; - private String _fullURL; - private boolean _delete; - - private List _savedShortURLs; - - public void setShortURL(String shortURL) - { - _shortURL = shortURL; - } - - public void setFullURL(String fullURL) - { - _fullURL = fullURL; - } - - public void setDelete(boolean delete) - { - _delete = delete; - } - - public String getShortURL() - { - return _shortURL; - } - - public String getFullURL() - { - return _fullURL; - } - - public boolean isDelete() - { - return _delete; - } - } - - public abstract static class AbstractShortURLAdminAction extends FormViewAction - { - @Override - public void validateCommand(ShortURLForm target, Errors errors) {} - - @Override - public boolean handlePost(ShortURLForm form, BindException errors) throws Exception - { - String shortURL = StringUtils.trimToEmpty(form.getShortURL()); - if (StringUtils.isEmpty(shortURL)) - { - errors.addError(new LabKeyError("Short URL must not be blank")); - } - if (shortURL.endsWith(".url")) - shortURL = shortURL.substring(0,shortURL.length()-".url".length()); - if (shortURL.contains("#") || shortURL.contains("/") || shortURL.contains(".")) - { - errors.addError(new LabKeyError("Short URLs may not contain '#' or '/' or '.'")); - } - URLHelper fullURL = null; - if (!form.isDelete()) - { - String trimmedFullURL = StringUtils.trimToNull(form.getFullURL()); - if (trimmedFullURL == null) - { - errors.addError(new LabKeyError("Target URL must not be blank")); - } - else - { - try - { - fullURL = new URLHelper(trimmedFullURL); - } - catch (URISyntaxException e) - { - errors.addError(new LabKeyError("Invalid Target URL. " + e.getMessage())); - } - } - } - if (errors.getErrorCount() > 0) - { - return false; - } - - ShortURLService service = ShortURLService.get(); - if (form.isDelete()) - { - ShortURLRecord shortURLRecord = service.resolveShortURL(shortURL); - if (shortURLRecord == null) - { - throw new NotFoundException("No such short URL: " + shortURL); - } - try - { - service.deleteShortURL(shortURLRecord, getUser()); - } - catch (ValidationException e) - { - errors.addError(new LabKeyError("Error deleting short URL:")); - for(ValidationError error: e.getErrors()) - { - errors.addError(new LabKeyError(error.getMessage())); - } - } - - if (errors.getErrorCount() > 0) - { - return false; - } - } - else - { - ShortURLRecord shortURLRecord = service.saveShortURL(shortURL, fullURL, getUser()); - MutableSecurityPolicy policy = new MutableSecurityPolicy(SecurityPolicyManager.getPolicy(shortURLRecord)); - // Add a role assignment to let another group manage the URL. This grants permission to the journal - // to change where the URL redirects you to after they copy the data - SecurityPolicyManager.savePolicy(policy, getUser()); - } - return true; - } - } - - @AdminConsoleAction - public class ShortURLAdminAction extends AbstractShortURLAdminAction - { - @Override - public ModelAndView getView(ShortURLForm form, boolean reshow, BindException errors) - { - JspView newView = new JspView<>("/org/labkey/core/admin/createNewShortURL.jsp", form, errors); - boolean isAppAdmin = getUser().hasRootPermission(ApplicationAdminPermission.class); - newView.setTitle(isAppAdmin ? "Create New Short URL" : "Short URLs"); - newView.setFrame(WebPartView.FrameType.PORTAL); - - QuerySettings qSettings = new QuerySettings(getViewContext(), "ShortURL", CoreQuerySchema.SHORT_URL_TABLE_NAME); - qSettings.setBaseSort(new Sort("-Created")); - QueryView existingView = new QueryView(new CoreQuerySchema(getUser(), getContainer()), qSettings, null); - if (!isAppAdmin) - { - existingView.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); - } - existingView.setTitle("Existing Short URLs"); - existingView.setFrame(WebPartView.FrameType.PORTAL); - - return new VBox(newView, existingView); - } - - @Override - public URLHelper getSuccessURL(ShortURLForm form) - { - return new ActionURL(ShortURLAdminAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("shortURL"); - addAdminNavTrail(root, "Short URL Admin", getClass()); - } - } - - @RequiresPermission(ApplicationAdminPermission.class) - public class UpdateShortURLAction extends AbstractShortURLAdminAction - { - @Override - public ModelAndView getView(ShortURLForm form, boolean reshow, BindException errors) - { - var shortUrlRecord = ShortURLService.get().resolveShortURL(form.getShortURL()); - if (shortUrlRecord == null) - { - errors.addError(new LabKeyError("Short URL does not exist: " + form.getShortURL())); - return new SimpleErrorView(errors); - } - form.setFullURL(shortUrlRecord.getFullURL()); - - JspView view = new JspView<>("/org/labkey/core/admin/updateShortURL.jsp", form, errors); - view.setTitle("Update Short URL"); - view.setFrame(WebPartView.FrameType.PORTAL); - return view; - } - - @Override - public URLHelper getSuccessURL(ShortURLForm form) - { - return new ActionURL(ShortURLAdminAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("shortURL"); - addAdminNavTrail(root, "Update Short URL", getClass()); - } - } - - // API for reporting client-side exceptions. - // UNDONE: Throttle by IP to avoid DOS from buggy clients. - @Marshal(Marshaller.Jackson) - @SuppressWarnings("UnusedDeclaration") - @RequiresLogin // Issue 52520: Prevent bots from submitting reports - @IgnoresForbiddenProjectCheck // Skip the "forbidden project" check since it disallows root - public static class LogClientExceptionAction extends MutatingApiAction - { - @Override - public Object execute(ExceptionForm form, BindException errors) - { - String errorCode = ExceptionUtil.logClientExceptionToMothership( - form.getStackTrace(), - form.getExceptionMessage(), - form.getBrowser(), - null, - form.getRequestURL(), - form.getReferrerURL(), - form.getUsername() - ); - - Map results = new HashMap<>(); - results.put("errorCode", errorCode); - results.put("loggedToMothership", errorCode != null); - - return success(results); - } - } - - @SuppressWarnings("unused") - public static class ExceptionForm - { - private String _exceptionMessage; - private String _stackTrace; - private String _requestURL; - private String _browser; - private String _username; - private String _referrerURL; - private String _file; - private String _line; - private String _platform; - - public String getExceptionMessage() - { - return _exceptionMessage; - } - - public void setExceptionMessage(String exceptionMessage) - { - _exceptionMessage = exceptionMessage; - } - - public String getUsername() - { - return _username; - } - - public void setUsername(String username) - { - _username = username; - } - - public String getStackTrace() - { - return _stackTrace; - } - - public void setStackTrace(String stackTrace) - { - _stackTrace = stackTrace; - } - - public String getRequestURL() - { - return _requestURL; - } - - public void setRequestURL(String requestURL) - { - _requestURL = requestURL; - } - - public String getBrowser() - { - return _browser; - } - - public void setBrowser(String browser) - { - _browser = browser; - } - - public String getReferrerURL() - { - return _referrerURL; - } - - public void setReferrerURL(String referrerURL) - { - _referrerURL = referrerURL; - } - - public String getFile() - { - return _file; - } - - public void setFile(String file) - { - _file = file; - } - - public String getLine() - { - return _line; - } - - public void setLine(String line) - { - _line = line; - } - - public String getPlatform() - { - return _platform; - } - - public void setPlatform(String platform) - { - _platform = platform; - } - } - - - /** generate URLS to seed web-site scanner */ - @SuppressWarnings("UnusedDeclaration") - @RequiresSiteAdmin - public static class SpiderAction extends SimpleViewAction - { - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Spider Initialization"); - } - - @Override - public ModelAndView getView(Object o, BindException errors) - { - List urls = new ArrayList<>(1000); - - if (getContainer().equals(ContainerManager.getRoot())) - { - for (Container c : ContainerManager.getAllChildren(ContainerManager.getRoot())) - { - urls.add(c.getStartURL(getUser()).toString()); - urls.add(new ActionURL(SpiderAction.class, c).toString()); - } - - Container home = ContainerManager.getHomeContainer(); - for (ActionDescriptor d : SpringActionController.getRegisteredActionDescriptors()) - { - ActionURL url = new ActionURL(d.getControllerName(), d.getPrimaryName(), home); - urls.add(url.toString()); - } - } - else - { - DefaultSchema def = DefaultSchema.get(getUser(), getContainer()); - def.getSchemaNames().forEach(name -> - { - QuerySchema q = def.getSchema(name); - if (null == q) - return; - var tableNames = q.getTableNames(); - if (null == tableNames) - return; - tableNames.forEach(table -> - { - try - { - var t = q.getTable(table); - if (null != t) - { - ActionURL grid = t.getGridURL(getContainer()); - if (null != grid) - urls.add(grid.toString()); - else - urls.add(new ActionURL("query", "executeQuery.view", getContainer()) - .addParameter("schemaName", q.getSchemaName()) - .addParameter("query.queryName", t.getName()) - .toString()); - } - } - catch (Exception x) - { - // pass - } - }); - }); - - ModuleLoader.getInstance().getModules().forEach(m -> - { - ActionURL url = m.getTabURL(getContainer(), getUser()); - if (null != url) - urls.add(url.toString()); - }); - } - - return new HtmlView(DIV(urls.stream().map(url -> createHtmlFragment(A(at(href,url),url),BR())))); - } - } - - @SuppressWarnings("UnusedDeclaration") - @RequiresPermission(TroubleshooterPermission.class) - public static class TestMothershipReportAction extends ReadOnlyApiAction - { - @Override - public Object execute(MothershipReportSelectionForm form, BindException errors) throws Exception - { - MothershipReport report; - MothershipReport.Target target = form.isTestMode() ? MothershipReport.Target.test : MothershipReport.Target.local; - if (MothershipReport.Type.CheckForUpdates.toString().equals(form.getType())) - { - report = UsageReportingLevel.generateReport(UsageReportingLevel.valueOf(form.getLevel()), target); - } - else - { - report = ExceptionUtil.createReportFromThrowable(getViewContext().getRequest(), - new SQLException("Intentional exception for testing purposes", "400"), - (String)getViewContext().getRequest().getAttribute(ViewServlet.ORIGINAL_URL_STRING), - target, - ExceptionReportingLevel.valueOf(form.getLevel()), null, null, null); - } - - final Map params; - if (report == null) - { - params = new LinkedHashMap<>(); - } - else - { - params = report.getJsonFriendlyParams(); - if (form.isSubmit()) - { - report.setForwardedFor(form.getForwardedFor()); - report.run(); - if (null != report.getUpgradeMessage()) - params.put("upgradeMessage", report.getUpgradeMessage()); - } - } - if (form.isDownload()) - { - ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, "metrics.json"); - } - return new ApiSimpleResponse(params); - } - } - - - static class MothershipReportSelectionForm - { - private String _type = MothershipReport.Type.CheckForUpdates.toString(); - private String _level = UsageReportingLevel.ON.toString(); - private boolean _submit = false; - private boolean _download = false; - private String _forwardedFor = null; - // indicates action is being invoked for dev/test - private boolean _testMode = false; - - public String getType() - { - return _type; - } - - public void setType(String type) - { - _type = type; - } - - public String getLevel() - { - return _level; - } - - public void setLevel(String level) - { - _level = StringUtils.upperCase(level); - } - - public boolean isSubmit() - { - return _submit; - } - - public void setSubmit(boolean submit) - { - _submit = submit; - } - - public String getForwardedFor() - { - return _forwardedFor; - } - - public void setForwardedFor(String forwardedFor) - { - _forwardedFor = forwardedFor; - } - - public boolean isTestMode() - { - return _testMode; - } - - public void setTestMode(boolean testMode) - { - _testMode = testMode; - } - - public boolean isDownload() - { - return _download; - } - - public void setDownload(boolean download) - { - _download = download; - } - } - - - @RequiresPermission(TroubleshooterPermission.class) - public class SuspiciousAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - Collection list = BlockListFilter.reportSuspicious(); - HtmlStringBuilder html = HtmlStringBuilder.of(); - if (list.isEmpty()) - { - html.append("No suspicious activity.\n"); - } - else - { - html.unsafeAppend("") - .unsafeAppend("\n"); - for (BlockListFilter.Suspicious s : list) - { - html.unsafeAppend("\n"); - } - html.unsafeAppend("
    host (user)user-agentcount
    ") - .append(s.host); - if (!isBlank(s.user)) - html.append(HtmlString.NBSP).append("(" + s.user + ")"); - html.unsafeAppend("") - .append(s.userAgent) - .unsafeAppend("") - .append(s.count) - .unsafeAppend("
    "); - } - return new HtmlView(html); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Suspicious activity", SuspiciousAction.class); - } - } - - /** This is a very crude API right now, mostly using default serialization of pre-existing objects - * NOTE: callers should expect that the return shape of this method may and will change in non-backward-compatible ways - */ - @Marshal(Marshaller.Jackson) - @RequiresNoPermission - @AllowedBeforeInitialUserIsSet - public static class ConfigurationSummaryAction extends ReadOnlyApiAction - { - @Override - public Object execute(Object o, BindException errors) - { - if (!getContainer().isRoot()) - throw new NotFoundException("Must be invoked in the root"); - - // requires site-admin, unless there are no users - if (!UserManager.hasNoRealUsers() && !getContainer().hasPermission(getUser(), AdminOperationsPermission.class)) - throw new UnauthorizedException(); - - return getConfigurationJson(); - } - - @Override - protected ObjectMapper createResponseObjectMapper() - { - ObjectMapper result = JsonUtil.createDefaultMapper(); - result.addMixIn(ExternalScriptEngineDefinitionImpl.class, IgnorePasswordMixIn.class); - return result; - } - - /* returns a jackson serializable object that reports superset of information returned in admin console */ - private JSONObject getConfigurationJson() - { - JSONObject res = new JSONObject(); - - res.put("server", AdminBean.getPropertyMap()); - - final Map> sets = new TreeMap<>(); - new SqlSelector(CoreSchema.getInstance().getScope(), - new SQLFragment("SELECT category, name, value FROM prop.propertysets PS inner join prop.properties P on PS.\"set\" = P.\"set\"\n" + - "WHERE objectid = ? AND category IN ('SiteConfig') AND encryption='None' AND LOWER(name) NOT LIKE '%password%'", ContainerManager.getRoot())).forEachMap(m -> - { - String category = (String)m.get("category"); - String name = (String)m.get("name"); - Object value = m.get("value"); - if (!sets.containsKey(category)) - sets.put(category, new TreeMap<>()); - sets.get(category).put(name,value); - } - ); - res.put("siteSettings", sets); - - HealthCheck.Result result = HealthCheckRegistry.get().checkHealth(Arrays.asList("all")); - res.put("health", result); - - LabKeyScriptEngineManager mgr = LabKeyScriptEngineManager.get(); - res.put("scriptEngines", mgr.getEngineDefinitions()); - - return res; - } - } - - @JsonIgnoreProperties(value = { "password", "changePassword", "configuration" }) - private static class IgnorePasswordMixIn - { - } - - @AdminConsoleAction() - public class AllowListAction extends FormViewAction - { - private AllowListType _type; - - @Override - public void validateCommand(AllowListForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(AllowListForm form, boolean reshow, BindException errors) - { - _type = form.getTypeEnum(); - - form.setExistingValuesList(form.getTypeEnum().getValues()); - - JspView newView = new JspView<>("/org/labkey/core/admin/addNewListValue.jsp", form, errors); - newView.setTitle("Register New " + form.getTypeEnum().getTitle()); - newView.setFrame(WebPartView.FrameType.PORTAL); - JspView existingView = new JspView<>("/org/labkey/core/admin/existingListValues.jsp", form, errors); - existingView.setTitle("Existing " + form.getTypeEnum().getTitle() + "s"); - existingView.setFrame(WebPartView.FrameType.PORTAL); - - return new VBox(newView, existingView); - } - - @Override - public boolean handlePost(AllowListForm form, BindException errors) throws Exception - { - AllowListType allowListType = form.getTypeEnum(); - //handle delete of existing value - if (form.isDelete()) - { - String urlToDelete = form.getExistingValue(); - List values = new ArrayList<>(allowListType.getValues()); - for (String value : values) - { - if (null != urlToDelete && urlToDelete.trim().equalsIgnoreCase(value.trim())) - { - values.remove(value); - allowListType.setValues(values, getUser()); - break; - } - } - } - //handle updates - clicking on Save button under Existing will save the updated urls - else if (form.isSaveAll()) - { - Set validatedValues = form.validateValues(errors); - if (errors.hasErrors()) - return false; - - allowListType.setValues(validatedValues.stream().toList(), getUser()); - } - //save new external value - else if (form.isSaveNew()) - { - Set valueSet = form.validateNewValue(errors); - if (errors.hasErrors()) - return false; - - allowListType.setValues(valueSet, getUser()); - } - - return true; - } - - @Override - public URLHelper getSuccessURL(AllowListForm form) - { - return form.getTypeEnum().getSuccessURL(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic(_type.getHelpTopic()); - addAdminNavTrail(root, String.format("%1$s Admin", _type.getTitle()), getClass()); - } - } - - public static class AllowListForm - { - private String _newValue; - private String _existingValue; - private boolean _delete; - private String _existingValues; - private boolean _saveAll; - private boolean _saveNew; - private String _type; - - private List _existingValuesList; - - public String getNewValue() - { - return _newValue; - } - - @SuppressWarnings("unused") - public void setNewValue(String newValue) - { - _newValue = newValue; - } - - public String getExistingValue() - { - return _existingValue; - } - - @SuppressWarnings("unused") - public void setExistingValue(String existingValue) - { - _existingValue = existingValue; - } - - public boolean isDelete() - { - return _delete; - } - - @SuppressWarnings("unused") - public void setDelete(boolean delete) - { - _delete = delete; - } - - public String getExistingValues() - { - return _existingValues; - } - - @SuppressWarnings("unused") - public void setExistingValues(String existingValues) - { - _existingValues = existingValues; - } - - public boolean isSaveAll() - { - return _saveAll; - } - - @SuppressWarnings("unused") - public void setSaveAll(boolean saveAll) - { - _saveAll = saveAll; - } - - public boolean isSaveNew() - { - return _saveNew; - } - - @SuppressWarnings("unused") - public void setSaveNew(boolean saveNew) - { - _saveNew = saveNew; - } - - public List getExistingValuesList() - { - //for updated urls that comes in as String values from the jsp/html form - if (null != getExistingValues()) - { - // The JavaScript delimits with "\n". Not sure where these "\r"s are coming from, but we need to strip them. - return new ArrayList<>(Arrays.asList(getExistingValues().replace("\r", "").split("\n"))); - } - return _existingValuesList; - } - - public void setExistingValuesList(List valuesList) - { - _existingValuesList = valuesList; - } - - public String getType() - { - return _type; - } - - @SuppressWarnings("unused") - public void setType(String type) - { - _type = type; - } - - @NotNull - public AllowListType getTypeEnum() - { - return EnumUtils.getEnum(AllowListType.class, getType(), AllowListType.Redirect); - } - - @JsonIgnore - public Set validateNewValue(BindException errors) - { - String value = StringUtils.trimToEmpty(getNewValue()); - getTypeEnum().validateValueFormat(value, errors); - if (errors.hasErrors()) - return null; - - Set valueSet = new CaseInsensitiveHashSet(getTypeEnum().getValues()); - checkDuplicatesByAddition(value, valueSet, errors); - return valueSet; - } - - @JsonIgnore - public Set validateValues(BindException errors) - { - List values = getExistingValuesList(); //get values from the form, this includes updated values - Set valueSet = new CaseInsensitiveHashSet(); - - if (null != values && !values.isEmpty()) - { - for (String value : values) - { - getTypeEnum().validateValueFormat(value, errors); - if (errors.hasErrors()) - continue; - - checkDuplicatesByAddition(value, valueSet, errors); - } - } - - return valueSet; - } - - /** - * Adds value to value set unless it is a duplicate, in which case it adds an error - * @param value to check - * @param valueSet of existing values - * @param errors collections of errors observed - */ - @JsonIgnore - private void checkDuplicatesByAddition(String value, Set valueSet, BindException errors) - { - String trimValue = StringUtils.trimToEmpty(value); - if (!valueSet.add(trimValue)) - errors.addError(new LabKeyError(String.format("'%1$s' already exists. Duplicate values not allowed.", trimValue))); - } - } - - @AdminConsoleAction - public static class DeleteAllValuesAction extends FormHandlerAction - { - @Override - public void validateCommand(AllowListForm form, Errors errors) - { - } - - @Override - public boolean handlePost(AllowListForm form, BindException errors) throws Exception - { - form.getTypeEnum().setValues(Collections.emptyList(), getUser()); - return true; - } - - @Override - public URLHelper getSuccessURL(AllowListForm form) - { - return form.getTypeEnum().getSuccessURL(getContainer()); - } - } - - public static class ExternalSourcesForm - { - private boolean _delete; - private boolean _saveNew; - private boolean _saveAll; - - private String _newDirective; - private String _newHost; - private String _existingValue; - private String _existingValues; - - public boolean isDelete() - { - return _delete; - } - - @SuppressWarnings("unused") - public void setDelete(boolean delete) - { - _delete = delete; - } - - public boolean isSaveNew() - { - return _saveNew; - } - - @SuppressWarnings("unused") - public void setSaveNew(boolean saveNew) - { - _saveNew = saveNew; - } - - public boolean isSaveAll() - { - return _saveAll; - } - - @SuppressWarnings("unused") - public void setSaveAll(boolean saveAll) - { - _saveAll = saveAll; - } - - public String getNewDirective() - { - return _newDirective; - } - - @SuppressWarnings("unused") - public void setNewDirective(String newDirective) - { - _newDirective = newDirective; - } - - public String getNewHost() - { - return _newHost; - } - - @SuppressWarnings("unused") - public void setNewHost(String newHost) - { - _newHost = newHost; - } - - public String getExistingValue() - { - return _existingValue; - } - - @SuppressWarnings("unused") - public void setExistingValue(String existingValue) - { - _existingValue = existingValue; - } - - public List getExistingValues() - { - return Arrays.stream(StringUtils.trimToEmpty(_existingValues).split("\n")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .toList(); - } - - @SuppressWarnings("unused") - public void setExistingValues(String existingValues) - { - _existingValues = existingValues; - } - - private AllowedHost getExistingAllowedHost(BindException errors) - { - return getAllowedHost(getExistingValue(), errors); - } - - private AllowedHost getAllowedHost(String value, BindException errors) - { - String[] parts = value.split("\\|", 2); // Stop after the first bar to produce two parts - if (parts.length != 2) - { - errors.addError(new LabKeyError("Can't parse allowed host.")); - return null; - } - return validateHost(parts[0], parts[1], errors); - } - - private List getExistingAllowedHosts(BindException errors) - { - List existing = getExistingValues().stream() - .map(value-> getAllowedHost(value, errors)) - .toList(); - - if (errors.hasErrors()) - return null; - - return checkDuplicates(existing, errors); - } - - private List validateNewAllowedHost(BindException errors) throws JsonProcessingException - { - AllowedHost newAllowedHost = validateHost(getNewDirective(), getNewHost(), errors); - - if (errors.hasErrors()) - return null; - - List hosts = getSavedAllowedHosts(); - hosts.add(newAllowedHost); - - return checkDuplicates(hosts, errors); - } - - // Lenient for now: no unknown directives, no blank hosts or hosts with semicolons - public static AllowedHost validateHost(String directiveString, String host, BindException errors) - { - AllowedHost ret = null; - - if (StringUtils.isEmpty(directiveString)) - { - errors.addError(new LabKeyError("Directive must not be blank")); - } - else if (StringUtils.isEmpty(host)) - { - errors.addError(new LabKeyError("Host must not be blank")); - } - else if (host.contains(";")) - { - errors.addError(new LabKeyError("Semicolons are not allowed in host names")); - } - else - { - Directive directive = EnumUtils.getEnum(Directive.class, directiveString); - - if (null == directive) - { - errors.addError(new LabKeyError("Unknown directive: " + directiveString)); - } - else - { - ret = new AllowedHost(directive, host.trim()); - } - } - - return ret; - } - - /** - * Check for duplicates in hosts: within each Directive, hosts are checked using case-insensitive comparisons - - * @param hosts a list of AllowedHost objects to check for duplicates - * @param errors errors to populate - * @return hosts if there are no duplicates, otherwise {@code null} - */ - public static @Nullable List checkDuplicates(List hosts, BindException errors) - { - // Not a simple Set check since we want host check to be case-insensitive - MultiValuedMap map = new CaseInsensitiveHashSetValuedMap<>(); - - hosts.forEach(allowedHost -> { - String host = allowedHost.host().trim(); - if (!map.put(allowedHost.directive(), host)) - errors.addError(new LabKeyError(String.format("'%1$s' already exists. Duplicate values are not allowed.", allowedHost))); - }); - - return errors.hasErrors() ? null : hosts; - } - - // Returns a mutable list - public List getSavedAllowedHosts() throws JsonProcessingException - { - return AllowedExternalResourceHosts.readAllowedHosts(); - } - } - - @AdminConsoleAction() - public class ExternalSourcesAction extends FormViewAction - { - @Override - public void validateCommand(ExternalSourcesForm form, Errors errors) - { - } - - @Override - public ModelAndView getView(ExternalSourcesForm form, boolean reshow, BindException errors) - { - boolean isTroubleshooter = !getContainer().hasPermission(getUser(), ApplicationAdminPermission.class); - - JspView newView = new JspView<>("/org/labkey/core/admin/addNewExternalSource.jsp", form, errors); - newView.setTitle(isTroubleshooter ? "Overview" : "Register New External Resource Host"); - newView.setFrame(WebPartView.FrameType.PORTAL); - JspView existingView = new JspView<>("/org/labkey/core/admin/existingExternalSources.jsp", form, errors); - existingView.setTitle("Existing External Resource Hosts"); - existingView.setFrame(WebPartView.FrameType.PORTAL); - - return new VBox(newView, existingView); - } - - private static final Object HOST_LOCK = new Object(); - - @Override - public boolean handlePost(ExternalSourcesForm form, BindException errors) throws Exception - { - List allowedHosts = null; - - // Multiple requests could access this in parallel, so synchronize access, Issue 53457 - synchronized (HOST_LOCK) - { - //handle delete of an existing value - if (form.isDelete()) - { - AllowedHost subToDelete = form.getExistingAllowedHost(errors); - if (errors.hasErrors()) - return false; - allowedHosts = form.getSavedAllowedHosts(); - var iter = allowedHosts.listIterator(); - while (iter.hasNext()) - { - AllowedHost sub = iter.next(); - if (sub.equals(subToDelete)) - { - iter.remove(); - break; - } - } - } - //handle updates - clicking on Save button under Existing will save the updated hosts - else if (form.isSaveAll()) - { - allowedHosts = form.getExistingAllowedHosts(errors); - if (errors.hasErrors()) - return false; - } - //save new external value - else if (form.isSaveNew()) - { - allowedHosts = form.validateNewAllowedHost(errors); - } - - if (errors.hasErrors()) - return false; - - AllowedExternalResourceHosts.saveAllowedHosts(allowedHosts, getUser()); - } - - return true; - } - - @Override - public URLHelper getSuccessURL(ExternalSourcesForm form) - { - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("externalHosts"); - addAdminNavTrail(root, "Allowed External Resource Hosts", getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ProjectSettingsAction extends ProjectSettingsViewPostAction - { - @Override - protected LookAndFeelView getTabView(ProjectSettingsForm form, boolean reshow, BindException errors) - { - return new LookAndFeelView(errors); - } - - @Override - public void validateCommand(ProjectSettingsForm form, Errors errors) - { - } - - @Override - public boolean handlePost(ProjectSettingsForm form, BindException errors) throws Exception - { - return saveProjectSettings(getContainer(), getUser(), form, errors); - } - } - - private static boolean saveProjectSettings(Container c, User user, ProjectSettingsForm form, BindException errors) - { - WriteableLookAndFeelProperties props = LookAndFeelProperties.getWriteableInstance(c); - boolean hasAdminOpsPerm = c.hasPermission(user, AdminOperationsPermission.class); - - // Site-only properties - - if (c.isRoot()) - { - DateParsingMode dateParsingMode = DateParsingMode.fromString(form.getDateParsingMode()); - props.setDateParsingMode(dateParsingMode); - - if (hasAdminOpsPerm) - { - String customWelcome = form.getCustomWelcome(); - String welcomeUrl = StringUtils.trimToNull(customWelcome); - if ("/".equals(welcomeUrl) || AppProps.getInstance().getContextPath().equalsIgnoreCase(welcomeUrl)) - { - errors.reject(SpringActionController.ERROR_MSG, "Invalid welcome URL. The url cannot equal '/' or the contextPath (" + AppProps.getInstance().getContextPath() + ")"); - } - else - { - props.setCustomWelcome(welcomeUrl); - } - } - } - - // Site & project properties - - boolean shouldInherit = form.getShouldInherit(); - if (shouldInherit != SecurityManager.shouldNewSubfoldersInheritPermissions(c)) - { - SecurityManager.setNewSubfoldersInheritPermissions(c, user, shouldInherit); - } - - setProperty(form.isSystemDescriptionInherited(), props::clearSystemDescription, () -> props.setSystemDescription(form.getSystemDescription())); - setProperty(form.isSystemShortNameInherited(), props::clearSystemShortName, () -> props.setSystemShortName(form.getSystemShortName())); - setProperty(form.isThemeNameInherited(), props::clearThemeName, () -> props.setThemeName(form.getThemeName())); - setProperty(form.isFolderDisplayModeInherited(), props::clearFolderDisplayMode, () -> props.setFolderDisplayMode(FolderDisplayMode.fromString(form.getFolderDisplayMode()))); - setProperty(form.isApplicationMenuDisplayModeInherited(), props::clearApplicationMenuDisplayMode, () -> props.setApplicationMenuDisplayMode(FolderDisplayMode.fromString(form.getApplicationMenuDisplayMode()))); - setProperty(form.isHelpMenuEnabledInherited(), props::clearHelpMenuEnabled, () -> props.setHelpMenuEnabled(form.isHelpMenuEnabled())); - setProperty(form.isDiscussionEnabledInherited(), props::clearDiscussionEnabled, () -> props.setDiscussionEnabled(form.isDiscussionEnabled())); - - // a few properties on this page should be restricted to operational permissions (i.e. site admin) - if (hasAdminOpsPerm) - { - setProperty(form.isSystemEmailAddressInherited(), props::clearSystemEmailAddress, () -> { - String systemEmailAddress = form.getSystemEmailAddress(); - try - { - // this will throw an InvalidEmailException for invalid email addresses - ValidEmail email = new ValidEmail(systemEmailAddress); - props.setSystemEmailAddress(email); - } - catch (ValidEmail.InvalidEmailException e) - { - errors.reject(SpringActionController.ERROR_MSG, "Invalid System Email Address: [" - + e.getBadEmail() + "]. Please enter a valid email address."); - } - }); - - setProperty(form.isCustomLoginInherited(), props::clearCustomLogin, () -> { - String customLogin = form.getCustomLogin(); - if (props.isValidUrl(customLogin)) - { - props.setCustomLogin(customLogin); - } - else - { - errors.reject(SpringActionController.ERROR_MSG, "Invalid login URL. Should be in the form -."); - } - }); - } - - setProperty(form.isCompanyNameInherited(), props::clearCompanyName, () -> props.setCompanyName(form.getCompanyName())); - setProperty(form.isLogoHrefInherited(), props::clearLogoHref, () -> props.setLogoHref(form.getLogoHref())); - setProperty(form.isReportAProblemPathInherited(), props::clearReportAProblemPath, () -> props.setReportAProblemPath(form.getReportAProblemPath())); - setProperty(form.isSupportEmailInherited(), props::clearSupportEmail, () -> { - String supportEmail = form.getSupportEmail(); - - if (!isBlank(supportEmail)) - { - try - { - // this will throw an InvalidEmailException for invalid email addresses - ValidEmail email = new ValidEmail(supportEmail); - props.setSupportEmail(email.toString()); - } - catch (ValidEmail.InvalidEmailException e) - { - errors.reject(SpringActionController.ERROR_MSG, "Invalid Support Email Address: [" - + e.getBadEmail() + "]. Please enter a valid email address."); - } - } - else - { - // This stores a blank value, not null (which would mean inherit) - props.setSupportEmail(null); - } - }); - - boolean noErrors = !saveFolderSettings(c, user, props, form, errors); - - if (noErrors) - { - // Bump the look & feel revision so browsers retrieve the new theme stylesheet - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - } - - return noErrors; - } - - private static void setProperty(boolean inherited, Runnable clear, Runnable set) - { - if (inherited) - clear.run(); - else - set.run(); - } - - // Same as ProjectSettingsAction, but provides special admin console permissions handling - @AdminConsoleAction(ApplicationAdminPermission.class) - public static class LookAndFeelSettingsAction extends ProjectSettingsAction - { - @Override - protected TYPE getType() - { - return TYPE.LookAndFeelSettings; - } - } - - @RequiresPermission(AdminPermission.class) - public static class UpdateContainerSettingsAction extends MutatingApiAction - { - @Override - public Object execute(FolderSettingsForm form, BindException errors) - { - boolean saved = saveFolderSettings(getContainer(), getUser(), LookAndFeelProperties.getWriteableFolderInstance(getContainer()), form, errors); - - ApiSimpleResponse response = new ApiSimpleResponse(); - response.put("success", saved && !errors.hasErrors()); - return response; - } - } - - @RequiresPermission(AdminPermission.class) - public static class ResourcesAction extends ProjectSettingsViewPostAction - { - @Override - protected JspView getTabView(Object o, boolean reshow, BindException errors) - { - LookAndFeelBean bean = new LookAndFeelBean(); - return new JspView<>("/org/labkey/core/admin/lookAndFeelResources.jsp", bean, errors); - } - - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) - { - Container c = getContainer(); - Map fileMap = getFileMap(); - - for (ResourceType type : ResourceType.values()) - { - MultipartFile file = fileMap.get(type.name()); - - if (file != null && !file.isEmpty()) - { - try - { - type.save(file, c, getUser()); - } - catch (Exception e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - return false; - } - } - } - - // Note that audit logging happens via the attachment code, so we don't log separately here - - // Bump the look & feel revision so browsers retrieve the new logo, custom stylesheet, etc. - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - - return true; - } - } - - // Same as ResourcesAction, but provides special admin console permissions handling - @AdminConsoleAction - public static class AdminConsoleResourcesAction extends ResourcesAction - { - @Override - protected TYPE getType() - { - return TYPE.LookAndFeelSettings; - } - } - - @RequiresPermission(AdminPermission.class) - public static class MenuBarAction extends ProjectSettingsViewAction - { - @Override - protected HttpView getTabView() - { - if (getContainer().isRoot()) - return HtmlView.err("Menu bar must be configured for each project separately."); - - WebPartView v = new JspView<>("/org/labkey/core/admin/editMenuBar.jsp", null); - v.setView("menubar", new VBox()); - Portal.populatePortalView(getViewContext(), Portal.DEFAULT_PORTAL_PAGE_ID, v, false, true, true, false); - - return v; - } - } - - @RequiresPermission(AdminPermission.class) - public static class FilesAction extends ProjectSettingsViewPostAction - { - @Override - protected HttpView getTabView(FilesForm form, boolean reshow, BindException errors) - { - Container c = getContainer(); - - if (c.isRoot()) - return HtmlView.err("Files must be configured for each project separately."); - - if (!reshow || form.isPipelineRootForm()) - { - try - { - AdminController.setFormAndConfirmMessage(getViewContext(), form); - } - catch (IllegalArgumentException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - } - VBox box = new VBox(); - JspView view = new JspView<>("/org/labkey/core/admin/view/filesProjectSettings.jsp", form, errors); - String title = "Configure File Root"; - if (CloudStoreService.get() != null) - title += " And Enable Cloud Stores"; - view.setTitle(title); - box.addView(view); - - // only site admins (i.e. AdminOperationsPermission) can configure the pipeline root - if (c.hasPermission(getViewContext().getUser(), AdminOperationsPermission.class)) - { - SetupForm setupForm = SetupForm.init(c); - setupForm.setShowAdditionalOptionsLink(true); - setupForm.setErrors(errors); - PipeRoot pipeRoot = SetupForm.getPipelineRoot(c); - - if (pipeRoot != null) - { - for (String errorMessage : pipeRoot.validate()) - errors.addError(new LabKeyError(errorMessage)); - } - JspView pipelineView = (JspView) PipelineService.get().getSetupView(setupForm); - pipelineView.setTitle("Configure Data Processing Pipeline"); - box.addView(pipelineView); - } - - return box; - } - - @Override - public void validateCommand(FilesForm form, Errors errors) - { - if (!form.isPipelineRootForm() && !form.isDisableFileSharing() && !form.hasSiteDefaultRoot() && !form.isCloudFileRoot()) - { - String root = StringUtils.trimToNull(form.getFolderRootPath()); - if (root != null) - { - File f = new File(root); - if (!f.exists() || !f.isDirectory()) - { - errors.reject(SpringActionController.ERROR_MSG, "File root '" + root + "' does not appear to be a valid directory accessible to the server at " + getViewContext().getRequest().getServerName() + "."); - } - } - else - errors.reject(SpringActionController.ERROR_MSG, "A Project specified file root cannot be blank, to disable file sharing for this project, select the disable option."); - } - else if (form.isCloudFileRoot()) - { - AdminController.validateCloudFileRoot(form, getContainer(), errors); - } - } - - @Override - public boolean handlePost(FilesForm form, BindException errors) throws Exception - { - FileContentService service = FileContentService.get(); - if (service != null) - { - if (form.isPipelineRootForm()) - return PipelineService.get().savePipelineSetup(getViewContext(), form, errors); - else - { - AdminController.setFileRootFromForm(getViewContext(), form, errors); - } - } - - // Cloud settings - AdminController.setEnabledCloudStores(getViewContext(), form, errors); - - return !errors.hasErrors(); - } - - @Override - public URLHelper getSuccessURL(FilesForm form) - { - ActionURL url = new AdminController.AdminUrlsImpl().getProjectSettingsFileURL(getContainer()); - if (form.isPipelineRootForm()) - { - url.addParameter("piperootSet", true); - } - else - { - if (form.isFileRootChanged()) - url.addParameter("rootSet", form.getMigrateFilesOption()); - if (form.isEnabledCloudStoresChanged()) - url.addParameter("cloudChanged", true); - } - return url; - } - } - - public static class LookAndFeelView extends JspView - { - LookAndFeelView(BindException errors) - { - super("/org/labkey/core/admin/lookAndFeelProperties.jsp", new LookAndFeelBean(), errors); - } - } - - - public static class LookAndFeelBean - { - public final HtmlString helpLink = new HelpTopic("customizeLook").getSimpleLinkHtml("more info..."); - public final HtmlString welcomeLink = new HelpTopic("customizeLook").getSimpleLinkHtml("more info..."); - public final HtmlString customColumnRestrictionHelpLink = new HelpTopic("chartTrouble").getSimpleLinkHtml("more info..."); - } - - @RequiresPermission(AdminPermission.class) - public static class AdjustSystemTimestampsAction extends FormViewAction - { - @Override - public void addNavTrail(NavTree root) - { - } - - @Override - public void validateCommand(AdjustTimestampsForm form, Errors errors) - { - if (form.getHourDelta() == null || form.getHourDelta() == 0) - errors.reject(ERROR_MSG, "You must specify a non-zero value for 'Hour Delta'"); - } - - @Override - public ModelAndView getView(AdjustTimestampsForm form, boolean reshow, BindException errors) throws Exception - { - return new JspView<>("/org/labkey/core/admin/adjustTimestamps.jsp", form, errors); - } - - private void updateFields(TableInfo tInfo, Collection fieldNames, int delta) - { - SQLFragment sql = new SQLFragment(); - DbSchema schema = tInfo.getSchema(); - String comma = ""; - List updating = new ArrayList<>(); - - for (String fieldName: fieldNames) - { - ColumnInfo col = tInfo.getColumn(FieldKey.fromParts(fieldName)); - if (col != null && col.getJdbcType() == JdbcType.TIMESTAMP) - { - updating.add(fieldName); - if (sql.isEmpty()) - sql.append("UPDATE ").append(tInfo, "").append(" SET "); - sql.append(comma) - .append(String.format(" %s = {fn timestampadd(SQL_TSI_HOUR, %d, %s)}", col.getSelectIdentifier(), delta, col.getSelectIdentifier())); - comma = ", "; - } - } - - if (!sql.isEmpty()) - { - logger.info(String.format("Updating %s in table %s.%s", updating, schema.getName(), tInfo.getName())); - logger.debug(sql.toDebugString()); - int numRows = new SqlExecutor(schema).execute(sql); - logger.info(String.format("Updated %d rows for table %s.%s", numRows, schema.getName(), tInfo.getName())); - } - } - - @Override - public boolean handlePost(AdjustTimestampsForm form, BindException errors) throws Exception - { - List toUpdate = Arrays.asList("Created", "Modified", "lastIndexed", "diCreated", "diModified"); - logger.info("Adjusting all " + toUpdate + " timestamp fields in all tables by " + form.getHourDelta() + " hours."); - DbScope scope = DbScope.getLabKeyScope(); - try (DbScope.Transaction t = scope.ensureTransaction()) - { - ModuleLoader.getInstance().getModules().forEach(module -> { - logger.info("==> Beginning update of timestamps for module: " + module.getName()); - module.getSchemaNames().stream().sorted().forEach(schemaName -> { - DbSchema schema = DbSchema.get(schemaName, DbSchemaType.Module); - scope.invalidateSchema(schema); // Issue 44452: assure we have a fresh set of tables to work from - schema.getTableNames().forEach(tableName -> { - TableInfo tInfo = schema.getTable(tableName); - if (tInfo.getTableType() == DatabaseTableType.TABLE) - { - updateFields(tInfo, toUpdate, form.getHourDelta()); - } - }); - }); - logger.info("<== DONE updating timestamps for module: " + module.getName()); - }); - t.commit(); - } - logger.info("DONE Adjusting all " + toUpdate + " timestamp fields in all tables by " + form.getHourDelta() + " hours."); - return true; - } - - @Override - public URLHelper getSuccessURL(AdjustTimestampsForm adjustTimestampsForm) - { - return PageFlowUtil.urlProvider(AdminUrls.class).getAdminConsoleURL(); - } - } - - public static class AdjustTimestampsForm - { - private Integer hourDelta; - - public Integer getHourDelta() - { - return hourDelta; - } - - public void setHourDelta(Integer hourDelta) - { - this.hourDelta = hourDelta; - } - } - - @RequiresPermission(TroubleshooterPermission.class) - public class ViewUsageStatistics extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("ViewUsageStatistics")); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Usage Statistics", this.getClass()); - } - } - - private static final URI LABKEY_ORG_REPORT_ACTION; - - static - { - LABKEY_ORG_REPORT_ACTION = URI.create("https://www.labkey.org/admin-contentSecurityPolicyReport.api"); - } - - @RequiresNoPermission - @CSRF(CSRF.Method.NONE) - public static class ContentSecurityPolicyReportAction extends ReadOnlyApiAction - { - private static final Logger _log = LogHelper.getLogger(ContentSecurityPolicyReportAction.class, "CSP warnings"); - - // recent reports, to help avoid log spam - private static final Map reports = Collections.synchronizedMap(new LRUMap<>(20)); - - @Override - public Object execute(SimpleApiJsonForm form, BindException errors) throws Exception - { - var ret = new JSONObject().put("success", true); - - // fail fast - if (!_log.isWarnEnabled()) - return ret; - - var request = getViewContext().getRequest(); - assert null != request; - - var userAgent = request.getHeader("User-Agent"); - if (PageFlowUtil.isRobotUserAgent(userAgent) && !_log.isDebugEnabled()) - return ret; - - // NOTE User may be "guest", and will always be guest if being relayed to labkey.org - var jsonObj = form.getJsonObject(); - if (null != jsonObj) - { - JSONObject cspReport = jsonObj.optJSONObject("csp-report"); - if (cspReport != null) - { - String blockedUri = cspReport.optString("blocked-uri", null); - - // Issue 52933 - suppress base-uri problems from a crawler or bot on labkey.org - if (blockedUri != null && - blockedUri.startsWith("https://labkey.org%2C") && - blockedUri.endsWith("undefined") && - !_log.isDebugEnabled()) - { - return ret; - } - - String urlString = cspReport.optString("document-uri", null); - if (urlString != null) - { - String path = new URLHelper(urlString).deleteParameters().getURIString(); - if (null == reports.put(path, Boolean.TRUE) || _log.isDebugEnabled()) - { - // Don't modify forwarded reports; they already have user, ip, user-agent, etc. from the forwarding server. - boolean forwarded = jsonObj.optBoolean("forwarded", false); - if (!forwarded) - { - User user = getUser(); - String email = null; - // If the user is not logged in, we may still be able to snag the email address from our cookie - if (user.isGuest()) - email = LoginController.getEmailFromCookie(getViewContext().getRequest()); - if (null == email) - email = user.getEmail(); - jsonObj.put("user", email); - String ipAddress = request.getHeader("X-FORWARDED-FOR"); - if (ipAddress == null) - ipAddress = request.getRemoteAddr(); - jsonObj.put("ip", ipAddress); - if (isNotBlank(userAgent)) - jsonObj.put("user-agent", userAgent); - String labkeyVersion = request.getParameter("labkeyVersion"); - if (null != labkeyVersion) - jsonObj.put("labkeyVersion", labkeyVersion); - String cspVersion = request.getParameter("cspVersion"); - if (null != cspVersion) - jsonObj.put("cspVersion", cspVersion); - } - - var jsonStr = jsonObj.toString(2); - _log.warn("ContentSecurityPolicy warning on page: {}\n{}", urlString, jsonStr); - - if (!forwarded && OptionalFeatureService.get().isFeatureEnabled(ContentSecurityPolicyFilter.FEATURE_FLAG_FORWARD_CSP_REPORTS)) - { - jsonObj.put("forwarded", true); - - // Create an HttpClient - HttpClient client = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(10)) - .build(); - - // Create the POST request - HttpRequest remoteRequest = HttpRequest.newBuilder() - .uri(LABKEY_ORG_REPORT_ACTION) - .header("Content-Type", request.getContentType()) // Use whatever the browser set - .POST(HttpRequest.BodyPublishers.ofString(jsonObj.toString(2))) - .build(); - - // Send the request and get the response - HttpResponse response = client.send(remoteRequest, HttpResponse.BodyHandlers.ofString()); - - if (response.statusCode() != 200) - { - _log.error("ContentSecurityPolicy report forwarding to https://www.labkey.org failed: {}\n{}", response.statusCode(), response.body()); - } - else - { - JSONObject jsonResponse = new JSONObject(response.body()); - boolean success = jsonResponse.optBoolean("success", false); - if (!success) - { - _log.error("ContentSecurityPolicy report forwarding to https://www.labkey.org failed: {}", jsonResponse); - } - } - } - } - } - } - } - return ret; - } - } - - - public static class TestCase extends AbstractActionPermissionTest - { - @Override - @Test - public void testActionPermissions() - { - User user = TestContext.get().getUser(); - assertTrue(user.hasSiteAdminPermission()); - - AdminController controller = new AdminController(); - - // @RequiresPermission(ReadPermission.class) - assertForReadPermission(user, false, - new GetModulesAction(), - new GetFolderTabsAction(), - new ClearDeletedTabFoldersAction() - ); - - // @RequiresPermission(DeletePermission.class) - assertForUpdateOrDeletePermission(user, - new DeleteFolderAction() - ); - - // @RequiresPermission(AdminPermission.class) - assertForAdminPermission(user, - controller.new CustomizeEmailAction(), - controller.new FolderAliasesAction(), - controller.new MoveFolderAction(), - controller.new MoveTabAction(), - controller.new RenameFolderAction(), - controller.new ReorderFoldersAction(), - controller.new ReorderFoldersApiAction(), - controller.new SiteValidationAction(), - new AddTabAction(), - new ConfirmProjectMoveAction(), - new CreateFolderAction(), - new CustomizeMenuAction(), - new DeleteCustomEmailAction(), - new FilesAction(), - new MenuBarAction(), - new ProjectSettingsAction(), - new RenameContainerAction(), - new RenameTabAction(), - new ResetPropertiesAction(), - new ResetQueryStatisticsAction(), - new ResetResourceAction(), - new ResourcesAction(), - new RevertFolderAction(), - new SetFolderPermissionsAction(), - new SetInitialFolderSettingsAction(), - new ShowTabAction() - ); - - //TODO @RequiresPermission(AdminReadPermission.class) - //controller.new TestMothershipReportAction() - - // @RequiresPermission(AdminOperationsPermission.class) - assertForAdminOperationsPermission(ContainerManager.getRoot(), user, - controller.new DbCheckerAction(), - controller.new DeleteModuleAction(), - controller.new DoCheckAction(), - controller.new EmailTestAction(), - controller.new ShowNetworkDriveTestAction(), - controller.new ValidateDomainsAction(), - new OptionalFeatureAction(), - new GetSchemaXmlDocAction(), - new RecreateViewsAction() - ); - - // @AdminConsoleAction - assertForAdminPermission(ContainerManager.getRoot(), user, - controller.new ActionsAction(), - controller.new CachesAction(), - controller.new ConfigureSystemMaintenanceAction(), - controller.new CustomizeSiteAction(), - controller.new DumpHeapAction(), - controller.new ExecutionPlanAction(), - controller.new FolderTypesAction(), - controller.new MemTrackerAction(), - controller.new ModulesAction(), - controller.new QueriesAction(), - controller.new QueryStackTracesAction(), - controller.new ResetErrorMarkAction(), - controller.new ShortURLAdminAction(), - controller.new ShowAllErrorsAction(), - controller.new ShowErrorsSinceMarkAction(), - controller.new ShowPrimaryLogAction(), - controller.new ShowCspReportLogAction(), - controller.new ShowThreadsAction(), - new ExportActionsAction(), - new ExportQueriesAction(), - new MemoryChartAction(), - new ShowAdminAction() - ); - - // @RequiresSiteAdmin - assertForRequiresSiteAdmin(user, - controller.new EnvironmentVariablesAction(), - controller.new SystemMaintenanceAction(), - controller.new SystemPropertiesAction(), - new GetPendingRequestCountAction(), - new InstallCompleteAction(), - new NewInstallSiteSettingsAction() - ); - - assertForTroubleshooterPermission(ContainerManager.getRoot(), user, - controller.new OptionalFeaturesAction(), - controller.new ShowModuleErrorsAction(), - new ModuleStatusAction() - ); - } - } - - public static class SerializationTest extends PipelineJob.TestSerialization - { - static class TestJob extends PipelineJob - { - ImpersonationContext _impersonationContext; - ImpersonationContext _impersonationContext1; - ImpersonationContext _impersonationContext2; - - @Override - public URLHelper getStatusHref() - { - return null; - } - - @Override - public String getDescription() - { - return "Test Job"; - } - } - - @Test - public void testSerialization() - { - TestJob job = new TestJob(); - TestContext ctx = TestContext.get(); - ViewContext viewContext = new ViewContext(); - viewContext.setContainer(ContainerManager.getSharedContainer()); - viewContext.setUser(ctx.getUser()); - RoleImpersonationContextFactory factory = new RoleImpersonationContextFactory( - viewContext.getContainer(), viewContext.getUser(), - Collections.singleton(RoleManager.getRole(SharedViewEditorRole.class)), Collections.emptySet(), null); - job._impersonationContext = factory.getImpersonationContext(); - - try - { - UserImpersonationContextFactory factory1 = new UserImpersonationContextFactory(viewContext.getContainer(), viewContext.getUser(), - UserManager.getGuestUser(), null); - job._impersonationContext1 = factory1.getImpersonationContext(); - } - catch (Exception e) - { - LOG.error("Invalid user email for impersonating."); - } - - GroupImpersonationContextFactory factory2 = new GroupImpersonationContextFactory(viewContext.getContainer(), viewContext.getUser(), - GroupManager.getGroup(ContainerManager.getRoot(), "Users", GroupEnumType.SITE), null); - job._impersonationContext2 = factory2.getImpersonationContext(); - testSerialize(job, LOG); - } - } - - public static class WorkbookDeleteTestCase extends Assert - { - private static final String FOLDER_NAME = "WorkbookDeleteTestCaseFolder"; - private static final String TEST_EMAIL = "testDelete@myDomain.com"; - - @Test - public void testWorkbookDelete() throws Exception - { - doCleanup(); - - Container project = ContainerManager.createContainer(ContainerManager.getRoot(), FOLDER_NAME, TestContext.get().getUser()); - Container workbook = ContainerManager.createContainer(project, null, "Title1", null, WorkbookContainerType.NAME, TestContext.get().getUser()); - - ValidEmail email = new ValidEmail(TEST_EMAIL); - SecurityManager.NewUserStatus newUserStatus = SecurityManager.addUser(email, null); - User nonAdminUser = newUserStatus.getUser(); - MutableSecurityPolicy policy = new MutableSecurityPolicy(project.getPolicy()); - policy.addRoleAssignment(nonAdminUser, ReaderRole.class); - SecurityPolicyManager.savePolicy(policy, TestContext.get().getUser()); - - // User lacks any permission, throw unauthorized for parent and workbook: - HttpServletRequest request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, project), nonAdminUser, Map.of("Content-Type", "application/json"), null); - MockHttpServletResponse response = ViewServlet.mockDispatch(request, null); - Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); - - request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, workbook), nonAdminUser, Map.of("Content-Type", "application/json"), null); - response = ViewServlet.mockDispatch(request, null); - Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); - - // Grant permission, should be able to delete the workbook but not parent: - policy = new MutableSecurityPolicy(project.getPolicy()); - policy.addRoleAssignment(nonAdminUser, EditorRole.class); - SecurityPolicyManager.savePolicy(policy, TestContext.get().getUser()); - - request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, project), nonAdminUser, Map.of("Content-Type", "application/json"), null); - response = ViewServlet.mockDispatch(request, null); - Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); - - // Hitting delete action results in a redirect: - request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, workbook), nonAdminUser, Map.of("Content-Type", "application/json"), null); - response = ViewServlet.mockDispatch(request, null); - Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FOUND, response.getStatus()); - - doCleanup(); - } - - protected static void doCleanup() throws Exception - { - Container project = ContainerManager.getForPath(FOLDER_NAME); - if (project != null) - { - ContainerManager.deleteAll(project, TestContext.get().getUser()); - } - - if (UserManager.userExists(new ValidEmail(TEST_EMAIL))) - { - User u = UserManager.getUser(new ValidEmail(TEST_EMAIL)); - UserManager.deleteUser(u.getUserId()); - } - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.core.admin; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; +import com.google.common.util.concurrent.UncheckedExecutionException; +import jakarta.mail.MessagingException; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.map.LRUMap; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.xmlbeans.XmlOptions; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jfree.chart.ChartFactory; +import org.jfree.chart.ChartUtilities; +import org.jfree.chart.JFreeChart; +import org.jfree.chart.plot.PlotOrientation; +import org.jfree.data.category.DefaultCategoryDataset; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.Constants; +import org.labkey.api.action.ApiResponse; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.action.BaseApiAction; +import org.labkey.api.action.BaseViewAction; +import org.labkey.api.action.ConfirmAction; +import org.labkey.api.action.ExportAction; +import org.labkey.api.action.FormHandlerAction; +import org.labkey.api.action.FormViewAction; +import org.labkey.api.action.HasViewContext; +import org.labkey.api.action.IgnoresAllocationTracking; +import org.labkey.api.action.LabKeyError; +import org.labkey.api.action.Marshal; +import org.labkey.api.action.Marshaller; +import org.labkey.api.action.MutatingApiAction; +import org.labkey.api.action.QueryViewAction; +import org.labkey.api.action.ReadOnlyApiAction; +import org.labkey.api.action.ReturnUrlForm; +import org.labkey.api.action.SimpleApiJsonForm; +import org.labkey.api.action.SimpleErrorView; +import org.labkey.api.action.SimpleRedirectAction; +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AbstractFolderContext.ExportType; +import org.labkey.api.admin.AdminBean; +import org.labkey.api.admin.AdminUrls; +import org.labkey.api.admin.FolderExportContext; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.admin.FolderWriter; +import org.labkey.api.admin.FolderWriterImpl; +import org.labkey.api.admin.HealthCheck; +import org.labkey.api.admin.HealthCheckRegistry; +import org.labkey.api.admin.ImportOptions; +import org.labkey.api.admin.StaticLoggerGetter; +import org.labkey.api.admin.TableXmlUtils; +import org.labkey.api.admin.sitevalidation.SiteValidationResult; +import org.labkey.api.admin.sitevalidation.SiteValidationResultList; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.provider.ContainerAuditProvider; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.cache.CacheStats; +import org.labkey.api.cache.TrackingCache; +import org.labkey.api.cloud.CloudStoreService; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.CaseInsensitiveHashSetValuedMap; +import org.labkey.api.collections.LabKeyCollectors; +import org.labkey.api.compliance.ComplianceFolderSettings; +import org.labkey.api.compliance.ComplianceService; +import org.labkey.api.compliance.PhiColumnBehavior; +import org.labkey.api.data.ButtonBar; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.ConnectionWrapper; +import org.labkey.api.data.Container; +import org.labkey.api.data.Container.ContainerException; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ContainerType; +import org.labkey.api.data.ConvertHelper; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DataColumn; +import org.labkey.api.data.DataRegion; +import org.labkey.api.data.DataRegionSelection; +import org.labkey.api.data.DatabaseTableType; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DisplayColumn; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MenuButton; +import org.labkey.api.data.MvUtil; +import org.labkey.api.data.NormalContainerType; +import org.labkey.api.data.PHI; +import org.labkey.api.data.RenderContext; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.Sort; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TransactionFilter; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.data.dialect.SqlDialect.ExecutionPlanType; +import org.labkey.api.data.queryprofiler.QueryProfiler; +import org.labkey.api.data.queryprofiler.QueryProfiler.QueryStatTsvWriter; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Lookup; +import org.labkey.api.files.FileContentService; +import org.labkey.api.message.settings.AbstractConfigTypeProvider.EmailConfigFormImpl; +import org.labkey.api.message.settings.MessageConfigService; +import org.labkey.api.message.settings.MessageConfigService.ConfigTypeProvider; +import org.labkey.api.message.settings.MessageConfigService.NotificationOption; +import org.labkey.api.message.settings.MessageConfigService.UserPreference; +import org.labkey.api.miniprofiler.RequestInfo; +import org.labkey.api.module.AllowedBeforeInitialUserIsSet; +import org.labkey.api.module.AllowedDuringUpgrade; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.FolderType; +import org.labkey.api.module.FolderTypeManager; +import org.labkey.api.module.IgnoresForbiddenProjectCheck; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleHtmlView; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.ModuleLoader.SchemaActions; +import org.labkey.api.module.ModuleLoader.SchemaAndModule; +import org.labkey.api.module.SimpleModule; +import org.labkey.api.moduleeditor.api.ModuleEditorService; +import org.labkey.api.pipeline.DirectoryNotDeletedException; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineStatusFile; +import org.labkey.api.pipeline.PipelineStatusUrls; +import org.labkey.api.pipeline.PipelineUrls; +import org.labkey.api.pipeline.PipelineValidationException; +import org.labkey.api.pipeline.view.SetupForm; +import org.labkey.api.products.ProductRegistry; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QuerySchema; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.RuntimeValidationException; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reports.ExternalScriptEngineDefinition; +import org.labkey.api.reports.LabKeyScriptEngineManager; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.ActionNames; +import org.labkey.api.security.AdminConsoleAction; +import org.labkey.api.security.CSRF; +import org.labkey.api.security.Directive; +import org.labkey.api.security.Group; +import org.labkey.api.security.GroupManager; +import org.labkey.api.security.IgnoresTermsOfUse; +import org.labkey.api.security.LoginUrls; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.RequiresLogin; +import org.labkey.api.security.RequiresNoPermission; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.RequiresSiteAdmin; +import org.labkey.api.security.RoleAssignment; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicy; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.SecurityUrls; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.UserPrincipal; +import org.labkey.api.security.ValidEmail; +import org.labkey.api.security.impersonation.GroupImpersonationContextFactory; +import org.labkey.api.security.impersonation.ImpersonationContext; +import org.labkey.api.security.impersonation.RoleImpersonationContextFactory; +import org.labkey.api.security.impersonation.UserImpersonationContextFactory; +import org.labkey.api.security.permissions.AbstractActionPermissionTest; +import org.labkey.api.security.permissions.AdminOperationsPermission; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.ApplicationAdminPermission; +import org.labkey.api.security.permissions.CreateProjectPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.PlatformDeveloperPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.SiteAdminPermission; +import org.labkey.api.security.permissions.TroubleshooterPermission; +import org.labkey.api.security.permissions.UploadFileBasedModulePermission; +import org.labkey.api.security.roles.EditorRole; +import org.labkey.api.security.roles.FolderAdminRole; +import org.labkey.api.security.roles.ProjectAdminRole; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.security.roles.Role; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.security.roles.SharedViewEditorRole; +import org.labkey.api.services.ServiceRegistry; +import org.labkey.api.settings.AdminConsole; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.ConceptURIProperties; +import org.labkey.api.settings.DateParsingMode; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.settings.LookAndFeelPropertiesManager.ResourceType; +import org.labkey.api.settings.NetworkDriveProps; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.settings.OptionalFeatureService.FeatureType; +import org.labkey.api.settings.ProductConfiguration; +import org.labkey.api.settings.WriteableAppProps; +import org.labkey.api.settings.WriteableFolderLookAndFeelProperties; +import org.labkey.api.settings.WriteableLookAndFeelProperties; +import org.labkey.api.util.ButtonBuilder; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.DOM; +import org.labkey.api.util.DOM.Renderable; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.DebugInfoDumper; +import org.labkey.api.util.ExceptionReportingLevel; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.FolderDisplayMode; +import org.labkey.api.util.GUID; +import org.labkey.api.util.HelpTopic; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.HtmlStringBuilder; +import org.labkey.api.util.HttpsUtil; +import org.labkey.api.util.JsonUtil; +import org.labkey.api.util.LinkBuilder; +import org.labkey.api.util.MailHelper; +import org.labkey.api.util.MemTracker; +import org.labkey.api.util.MemTracker.HeldReference; +import org.labkey.api.util.MothershipReport; +import org.labkey.api.util.NetworkDrive; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.ResponseHelper; +import org.labkey.api.util.SafeToRenderEnum; +import org.labkey.api.util.SessionAppender; +import org.labkey.api.util.StringExpressionFactory; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.util.SystemMaintenance.SystemMaintenanceProperties; +import org.labkey.api.util.SystemMaintenanceJob; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.Tuple3; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.UniqueID; +import org.labkey.api.util.UsageReportingLevel; +import org.labkey.api.util.emailTemplate.EmailTemplate; +import org.labkey.api.util.emailTemplate.EmailTemplateService; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.DataView; +import org.labkey.api.view.FolderManagement.FolderManagementViewAction; +import org.labkey.api.view.FolderManagement.FolderManagementViewPostAction; +import org.labkey.api.view.FolderManagement.ProjectSettingsViewAction; +import org.labkey.api.view.FolderManagement.ProjectSettingsViewPostAction; +import org.labkey.api.view.FolderManagement.TYPE; +import org.labkey.api.view.FolderTab; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.Portal; +import org.labkey.api.view.RedirectException; +import org.labkey.api.view.ShortURLRecord; +import org.labkey.api.view.ShortURLService; +import org.labkey.api.view.TabStripView; +import org.labkey.api.view.URLException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.VBox; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.ViewServlet; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.EmptyView; +import org.labkey.api.view.template.PageConfig; +import org.labkey.api.view.template.PageConfig.Template; +import org.labkey.api.wiki.WikiRendererType; +import org.labkey.api.wiki.WikiRenderingService; +import org.labkey.api.writer.FileSystemFile; +import org.labkey.api.writer.HtmlWriter; +import org.labkey.api.writer.ZipFile; +import org.labkey.api.writer.ZipUtil; +import org.labkey.bootstrap.ExplodedModuleService; +import org.labkey.core.admin.miniprofiler.MiniProfilerController; +import org.labkey.core.admin.sitevalidation.SiteValidationJob; +import org.labkey.core.admin.sql.SqlScriptController; +import org.labkey.core.login.LoginController; +import org.labkey.core.portal.CollaborationFolderType; +import org.labkey.core.portal.ProjectController; +import org.labkey.core.query.CoreQuerySchema; +import org.labkey.core.query.PostgresUserSchema; +import org.labkey.core.reports.ExternalScriptEngineDefinitionImpl; +import org.labkey.core.security.AllowedExternalResourceHosts; +import org.labkey.core.security.AllowedExternalResourceHosts.AllowedHost; +import org.labkey.core.security.BlockListFilter; +import org.labkey.core.security.SecurityController; +import org.labkey.data.xml.TablesDocument; +import org.labkey.filters.ContentSecurityPolicyFilter; +import org.labkey.security.xml.GroupEnumType; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.ModelAndView; +import org.springframework.web.servlet.mvc.Controller; + +import java.awt.*; +import java.beans.Introspector; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ClassLoadingMXBean; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryType; +import java.lang.management.MemoryUsage; +import java.lang.management.OperatingSystemMXBean; +import java.lang.management.RuntimeMXBean; +import java.lang.management.ThreadMXBean; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.SQLException; +import java.text.DecimalFormat; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.labkey.api.data.MultiValuedRenderContext.VALUE_DELIMITER_REGEX; +import static org.labkey.api.settings.AdminConsole.SettingsLinkType.Configuration; +import static org.labkey.api.settings.AdminConsole.SettingsLinkType.Diagnostics; +import static org.labkey.api.util.DOM.A; +import static org.labkey.api.util.DOM.Attribute.href; +import static org.labkey.api.util.DOM.Attribute.method; +import static org.labkey.api.util.DOM.Attribute.name; +import static org.labkey.api.util.DOM.Attribute.style; +import static org.labkey.api.util.DOM.Attribute.title; +import static org.labkey.api.util.DOM.Attribute.type; +import static org.labkey.api.util.DOM.Attribute.value; +import static org.labkey.api.util.DOM.BR; +import static org.labkey.api.util.DOM.DIV; +import static org.labkey.api.util.DOM.LI; +import static org.labkey.api.util.DOM.SPAN; +import static org.labkey.api.util.DOM.STYLE; +import static org.labkey.api.util.DOM.TABLE; +import static org.labkey.api.util.DOM.TD; +import static org.labkey.api.util.DOM.TR; +import static org.labkey.api.util.DOM.UL; +import static org.labkey.api.util.DOM.at; +import static org.labkey.api.util.DOM.cl; +import static org.labkey.api.util.DOM.createHtmlFragment; +import static org.labkey.api.util.HtmlString.NBSP; +import static org.labkey.api.util.logging.LogHelper.getLabKeyLogDir; +import static org.labkey.api.view.FolderManagement.EVERY_CONTAINER; +import static org.labkey.api.view.FolderManagement.FOLDERS_AND_PROJECTS; +import static org.labkey.api.view.FolderManagement.FOLDERS_ONLY; +import static org.labkey.api.view.FolderManagement.NOT_ROOT; +import static org.labkey.api.view.FolderManagement.PROJECTS_ONLY; +import static org.labkey.api.view.FolderManagement.ROOT; +import static org.labkey.api.view.FolderManagement.addTab; + +public class AdminController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver( + AdminController.class, + FileListAction.class, + FilesSiteSettingsAction.class, + UpdateFilePathsAction.class + ); + + private static final Logger LOG = LogHelper.getLogger(AdminController.class, "Admin-related UI and APIs"); + private static final Logger CLIENT_LOG = LogHelper.getLogger(LogAction.class, "Client/browser logging submitted to server"); + private static final String HEAP_MEMORY_KEY = "Total Heap Memory"; + + private static long _errorMark = 0; + private static long _primaryLogMark = 0; + + public static void registerAdminConsoleLinks() + { + Container root = ContainerManager.getRoot(); + + // Configuration + AdminConsole.addLink(Configuration, "authentication", urlProvider(LoginUrls.class).getConfigureURL()); + AdminConsole.addLink(Configuration, "email customization", new ActionURL(CustomizeEmailAction.class, root), AdminPermission.class); + AdminConsole.addLink(Configuration, "deprecated features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Deprecated.name()), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "experimental features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Experimental.name()), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "optional features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Optional.name()), TroubleshooterPermission.class); + if (!ProductRegistry.getProducts().isEmpty()) + AdminConsole.addLink(Configuration, "product configuration", new ActionURL(ProductConfigurationAction.class, root), AdminOperationsPermission.class); + // TODO move to FileContentModule + if (ModuleLoader.getInstance().hasModule("FileContent")) + AdminConsole.addLink(Configuration, "files", new ActionURL(FilesSiteSettingsAction.class, root), AdminOperationsPermission.class); + AdminConsole.addLink(Configuration, "folder types", new ActionURL(FolderTypesAction.class, root), AdminPermission.class); + AdminConsole.addLink(Configuration, "look and feel settings", new ActionURL(LookAndFeelSettingsAction.class, root)); + AdminConsole.addLink(Configuration, "missing value indicators", new AdminUrlsImpl().getMissingValuesURL(root), AdminPermission.class); + AdminConsole.addLink(Configuration, "project display order", new ActionURL(ReorderFoldersAction.class, root), AdminPermission.class); + AdminConsole.addLink(Configuration, "short urls", new ActionURL(ShortURLAdminAction.class, root), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "site settings", new AdminUrlsImpl().getCustomizeSiteURL()); + AdminConsole.addLink(Configuration, "system maintenance", new ActionURL(ConfigureSystemMaintenanceAction.class, root)); + AdminConsole.addLink(Configuration, "allowed external redirect hosts", new ActionURL(AllowListAction.class, root).addParameter("type", AllowListType.Redirect.name()), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "allowed external resource hosts", new ActionURL(ExternalSourcesAction.class, root), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "allowed file extensions", new ActionURL(AllowListAction.class, root).addParameter("type", AllowListType.FileExtension.name()), TroubleshooterPermission.class); + + // Diagnostics + AdminConsole.addLink(Diagnostics, "actions", new ActionURL(ActionsAction.class, root)); + AdminConsole.addLink(Diagnostics, "attachments", new ActionURL(AttachmentsAction.class, root)); + AdminConsole.addLink(Diagnostics, "caches", new ActionURL(CachesAction.class, root)); + AdminConsole.addLink(Diagnostics, "check database", new ActionURL(DbCheckerAction.class, root), AdminOperationsPermission.class); + AdminConsole.addLink(Diagnostics, "credits", new ActionURL(CreditsAction.class, root)); + AdminConsole.addLink(Diagnostics, "dump heap", new ActionURL(DumpHeapAction.class, root)); + AdminConsole.addLink(Diagnostics, "environment variables", new ActionURL(EnvironmentVariablesAction.class, root), SiteAdminPermission.class); + AdminConsole.addLink(Diagnostics, "memory usage", new ActionURL(MemTrackerAction.class, root)); + + if (CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) + { + AdminConsole.addLink(Diagnostics, "postgres activity", new ActionURL(PostgresStatActivityAction.class, root)); + AdminConsole.addLink(Diagnostics, "postgres locks", new ActionURL(PostgresLocksAction.class, root)); + } + + AdminConsole.addLink(Diagnostics, "profiler", new ActionURL(MiniProfilerController.ManageAction.class, root)); + AdminConsole.addLink(Diagnostics, "queries", getQueriesURL(null)); + AdminConsole.addLink(Diagnostics, "reset site errors", new ActionURL(ResetErrorMarkAction.class, root), AdminPermission.class); + AdminConsole.addLink(Diagnostics, "running threads", new ActionURL(ShowThreadsAction.class, root)); + AdminConsole.addLink(Diagnostics, "site validation", new ActionURL(ConfigureSiteValidationAction.class, root), AdminPermission.class); + AdminConsole.addLink(Diagnostics, "sql scripts", new ActionURL(SqlScriptController.ScriptsAction.class, root), AdminOperationsPermission.class); + AdminConsole.addLink(Diagnostics, "suspicious activity", new ActionURL(SuspiciousAction.class, root)); + AdminConsole.addLink(Diagnostics, "system properties", new ActionURL(SystemPropertiesAction.class, root), SiteAdminPermission.class); + AdminConsole.addLink(Diagnostics, "test email configuration", new ActionURL(EmailTestAction.class, root), AdminOperationsPermission.class); + AdminConsole.addLink(Diagnostics, "view all site errors", new ActionURL(ShowAllErrorsAction.class, root)); + AdminConsole.addLink(Diagnostics, "view all site errors since reset", new ActionURL(ShowErrorsSinceMarkAction.class, root)); + AdminConsole.addLink(Diagnostics, "view csp report log file", new ActionURL(ShowCspReportLogAction.class, root)); + AdminConsole.addLink(Diagnostics, "view primary site log file", new ActionURL(ShowPrimaryLogAction.class, root)); + } + + public static void registerManagementTabs() + { + addTab(TYPE.FolderManagement, "Folder Tree", "folderTree", EVERY_CONTAINER, ManageFoldersAction.class); + addTab(TYPE.FolderManagement, "Folder Type", "folderType", NOT_ROOT, FolderTypeAction.class); + addTab(TYPE.FolderManagement, "Missing Values", "mvIndicators", EVERY_CONTAINER, MissingValuesAction.class); + addTab(TYPE.FolderManagement, "Module Properties", "props", c -> { + if (!c.isRoot()) + { + // Show module properties tab only if a module w/ properties to set is present for current folder + for (Module m : c.getActiveModules()) + if (!m.getModuleProperties().isEmpty()) + return true; + } + + return false; + }, ModulePropertiesAction.class); + addTab(TYPE.FolderManagement, "Concepts", "concepts", c -> { + // Show Concepts tab only if the experiment module is enabled in this container + return c.getActiveModules().contains(ModuleLoader.getInstance().getModule(ExperimentService.MODULE_NAME)); + }, AdminController.ConceptsAction.class); + // Show Notifications tab only if we have registered notification providers + addTab(TYPE.FolderManagement, "Notifications", "notifications", c -> NOT_ROOT.test(c) && !MessageConfigService.get().getConfigTypes().isEmpty(), NotificationsAction.class); + addTab(TYPE.FolderManagement, "Export", "export", NOT_ROOT, ExportFolderAction.class); + addTab(TYPE.FolderManagement, "Import", "import", NOT_ROOT, ImportFolderAction.class); + addTab(TYPE.FolderManagement, "Files", "files", FOLDERS_AND_PROJECTS, FileRootsAction.class); + addTab(TYPE.FolderManagement, "Formats", "settings", FOLDERS_ONLY, FolderSettingsAction.class); + addTab(TYPE.FolderManagement, "Information", "info", NOT_ROOT, FolderInformationAction.class); + addTab(TYPE.FolderManagement, "R Config", "rConfig", NOT_ROOT, RConfigurationAction.class); + + addTab(TYPE.ProjectSettings, "Properties", "properties", PROJECTS_ONLY, ProjectSettingsAction.class); + addTab(TYPE.ProjectSettings, "Resources", "resources", PROJECTS_ONLY, ResourcesAction.class); + addTab(TYPE.ProjectSettings, "Menu Bar", "menubar", PROJECTS_ONLY, MenuBarAction.class); + addTab(TYPE.ProjectSettings, "Files", "files", PROJECTS_ONLY, FilesAction.class); + + addTab(TYPE.LookAndFeelSettings, "Properties", "properties", ROOT, LookAndFeelSettingsAction.class); + addTab(TYPE.LookAndFeelSettings, "Resources", "resources", ROOT, AdminConsoleResourcesAction.class); + } + + public AdminController() + { + setActionResolver(_actionResolver); + } + + @RequiresNoPermission + public static class BeginAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(Object o) + { + return getShowAdminURL(); + } + } + + private void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action) + { + addAdminNavTrail(root, childTitle, action, getContainer()); + } + + private static void addAdminNavTrail(NavTree root, @NotNull Container container) + { + if (container.isRoot()) + root.addChild("Admin Console", getShowAdminURL().setFragment("links")); + } + + private static void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action, @NotNull Container container) + { + addAdminNavTrail(root, container); + root.addChild(childTitle, new ActionURL(action, container)); + } + + public static ActionURL getShowAdminURL() + { + return new ActionURL(ShowAdminAction.class, ContainerManager.getRoot()); + } + + @Override + protected void beforeAction(Controller action) throws ServletException + { + super.beforeAction(action); + if (action instanceof BaseViewAction viewaction) + viewaction.getPageConfig().setRobotsNone(); + } + + @AdminConsoleAction + public static class ShowAdminAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/admin.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + URLHelper returnUrl = getViewContext().getActionURL().getReturnUrl(); + if (null != returnUrl) + root.addChild("Return to Project", returnUrl); + root.addChild("Admin Console"); + setHelpTopic("siteManagement"); + } + } + + @RequiresPermission(TroubleshooterPermission.class) + public class ShowModuleErrorsAction extends SimpleViewAction + { + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Module Errors", this.getClass()); + } + + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/moduleErrors.jsp"); + } + } + + public static class AdminUrlsImpl implements AdminUrls + { + @Override + public ActionURL getModuleErrorsURL() + { + return new ActionURL(ShowModuleErrorsAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getAdminConsoleURL() + { + return getShowAdminURL(); + } + + @Override + public ActionURL getModuleStatusURL(URLHelper returnUrl) + { + return AdminController.getModuleStatusURL(returnUrl); + } + + @Override + public ActionURL getCustomizeSiteURL() + { + return new ActionURL(CustomizeSiteAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getCustomizeSiteURL(boolean upgradeInProgress) + { + ActionURL url = getCustomizeSiteURL(); + + if (upgradeInProgress) + url.addParameter("upgradeInProgress", "1"); + + return url; + } + + @Override + public ActionURL getProjectSettingsURL(Container c) + { + return new ActionURL(ProjectSettingsAction.class, LookAndFeelProperties.getSettingsContainer(c)); + } + + ActionURL getLookAndFeelResourcesURL(Container c) + { + return c.isRoot() ? new ActionURL(AdminConsoleResourcesAction.class, c) : new ActionURL(ResourcesAction.class, LookAndFeelProperties.getSettingsContainer(c)); + } + + @Override + public ActionURL getProjectSettingsMenuURL(Container c) + { + return new ActionURL(MenuBarAction.class, LookAndFeelProperties.getSettingsContainer(c)); + } + + @Override + public ActionURL getProjectSettingsFileURL(Container c) + { + return new ActionURL(FilesAction.class, LookAndFeelProperties.getSettingsContainer(c)); + } + + @Override + public ActionURL getCustomizeEmailURL(@NotNull Container c, @Nullable Class selectedTemplate, @Nullable URLHelper returnUrl) + { + return getCustomizeEmailURL(c, selectedTemplate == null ? null : selectedTemplate.getName(), returnUrl); + } + + public ActionURL getCustomizeEmailURL(@NotNull Container c, @Nullable String selectedTemplate, @Nullable URLHelper returnUrl) + { + ActionURL url = new ActionURL(CustomizeEmailAction.class, c); + if (selectedTemplate != null) + { + url.addParameter("templateClass", selectedTemplate); + } + if (returnUrl != null) + { + url.addReturnUrl(returnUrl); + } + return url; + } + + public ActionURL getResetLookAndFeelPropertiesURL(Container c) + { + return new ActionURL(ResetPropertiesAction.class, c); + } + + @Override + public ActionURL getMaintenanceURL(URLHelper returnUrl) + { + ActionURL url = new ActionURL(MaintenanceAction.class, ContainerManager.getRoot()); + if (returnUrl != null) + url.addReturnUrl(returnUrl); + return url; + } + + @Override + public ActionURL getModulesDetailsURL() + { + return new ActionURL(ModulesAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getDeleteModuleURL(String moduleName) + { + return new ActionURL(DeleteModuleAction.class, ContainerManager.getRoot()).addParameter("name", moduleName); + } + + @Override + public ActionURL getManageFoldersURL(Container c) + { + return new ActionURL(ManageFoldersAction.class, c); + } + + @Override + public ActionURL getFolderTypeURL(Container c) + { + return new ActionURL(FolderTypeAction.class, c); + } + + @Override + public ActionURL getExportFolderURL(Container c) + { + return new ActionURL(ExportFolderAction.class, c); + } + + @Override + public ActionURL getImportFolderURL(Container c) + { + return new ActionURL(ImportFolderAction.class, c); + } + + @Override + public ActionURL getCreateProjectURL(@Nullable ActionURL returnUrl) + { + return getCreateFolderURL(ContainerManager.getRoot(), returnUrl); + } + + @Override + public ActionURL getCreateFolderURL(Container c, @Nullable ActionURL returnUrl) + { + ActionURL result = new ActionURL(CreateFolderAction.class, c); + if (returnUrl != null) + { + result.addReturnUrl(returnUrl); + } + return result; + } + + public ActionURL getSetFolderPermissionsURL(Container c) + { + return new ActionURL(SetFolderPermissionsAction.class, c); + } + + @Override + public void addAdminNavTrail(NavTree root, @NotNull Container container) + { + AdminController.addAdminNavTrail(root, container); + } + + @Override + public void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action, @NotNull Container container) + { + AdminController.addAdminNavTrail(root, childTitle, action, container); + } + + @Override + public void addModulesNavTrail(NavTree root, String childTitle, @NotNull Container container) + { + if (container.isRoot()) + addAdminNavTrail(root, "Modules", ModulesAction.class, container); + + root.addChild(childTitle); + } + + @Override + public ActionURL getFileRootsURL(Container c) + { + return new ActionURL(FileRootsAction.class, c); + } + + @Override + public ActionURL getLookAndFeelSettingsURL(Container c) + { + if (c.isRoot()) + return getSiteLookAndFeelSettingsURL(); + else if (c.isProject()) + return getProjectSettingsURL(c); + else + return getFolderSettingsURL(c); + } + + @Override + public ActionURL getSiteLookAndFeelSettingsURL() + { + return new ActionURL(LookAndFeelSettingsAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getFolderSettingsURL(Container c) + { + return new ActionURL(FolderSettingsAction.class, c); + } + + @Override + public ActionURL getNotificationsURL(Container c) + { + return new ActionURL(NotificationsAction.class, c); + } + + @Override + public ActionURL getModulePropertiesURL(Container c) + { + return new ActionURL(ModulePropertiesAction.class, c); + } + + @Override + public ActionURL getMissingValuesURL(Container c) + { + return new ActionURL(MissingValuesAction.class, c); + } + + public ActionURL getInitialFolderSettingsURL(Container c) + { + return new ActionURL(SetInitialFolderSettingsAction.class, c); + } + + @Override + public ActionURL getMemTrackerURL() + { + return new ActionURL(MemTrackerAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getFilesSiteSettingsURL() + { + return new ActionURL(FilesSiteSettingsAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getSessionLoggingURL() + { + return new ActionURL(SessionLoggingAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getTrackedAllocationsViewerURL() + { + return new ActionURL(TrackedAllocationsViewerAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getSystemMaintenanceURL() + { + return new ActionURL(ConfigureSystemMaintenanceAction.class, ContainerManager.getRoot()); + } + + public static ActionURL getDeprecatedFeaturesURL() + { + return new ActionURL(OptionalFeaturesAction.class, ContainerManager.getRoot()).addParameter("type", FeatureType.Deprecated.name()); + } + } + + public static class MaintenanceBean + { + public HtmlString content; + public ActionURL loginURL; + } + + /** + * During upgrade, startup, or maintenance mode, the user will be redirected to + * MaintenanceAction and only admin users will be allowed to log into the server. + * The maintenance.jsp page checks startup is complete or adminOnly mode is turned off + * and will redirect to the returnUrl or the loginURL. + * See Issue 18758 for more information. + */ + @RequiresNoPermission + @AllowedDuringUpgrade + @IgnoresAllocationTracking + public static class MaintenanceAction extends SimpleViewAction + { + private String _title = "Maintenance in progress"; + + @Override + public ModelAndView getView(ReturnUrlForm form, BindException errors) + { + if (!getUser().hasSiteAdminPermission()) + { + getViewContext().getResponse().setStatus(HttpServletResponse.SC_UNAUTHORIZED); + } + getPageConfig().setTemplate(Template.Dialog); + + boolean upgradeInProgress = ModuleLoader.getInstance().isUpgradeInProgress(); + boolean startupInProgress = ModuleLoader.getInstance().isStartupInProgress(); + boolean maintenanceMode = AppProps.getInstance().isUserRequestedAdminOnlyMode(); + + HtmlString content = HtmlString.of("This site is currently undergoing maintenance, only site admins may login at this time."); + if (upgradeInProgress) + { + _title = "Upgrade in progress"; + content = HtmlString.of("Upgrade in progress: only site admins may login at this time. Your browser will be redirected when startup is complete."); + } + else if (startupInProgress) + { + _title = "Startup in progress"; + content = HtmlString.of("Startup in progress: only site admins may login at this time. Your browser will be redirected when startup is complete."); + } + else if (maintenanceMode) + { + WikiRenderingService wikiService = WikiRenderingService.get(); + content = wikiService.getFormattedHtml(WikiRendererType.RADEOX, ModuleLoader.getInstance().getAdminOnlyMessage(), "Admin only message"); + } + + if (content == null) + content = HtmlString.of(_title); + + ActionURL loginURL = null; + if (getUser().isGuest()) + { + URLHelper returnUrl = form.getReturnUrlHelper(); + if (returnUrl != null) + loginURL = urlProvider(LoginUrls.class).getLoginURL(ContainerManager.getRoot(), returnUrl); + else + loginURL = urlProvider(LoginUrls.class).getLoginURL(); + } + + MaintenanceBean bean = new MaintenanceBean(); + bean.content = content; + bean.loginURL = loginURL; + + JspView view = new JspView<>("/org/labkey/core/admin/maintenance.jsp", bean, errors); + view.setTitle(_title); + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild(_title); + } + } + + /** + * Similar to SqlScriptController.GetModuleStatusAction except that Guest is allowed to check that the startup is complete. + */ + @RequiresNoPermission + @AllowedDuringUpgrade + @IgnoresAllocationTracking + public static class StartupStatusAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(Object o, BindException errors) + { + JSONObject result = new JSONObject(); + result.put("startupComplete", ModuleLoader.getInstance().isStartupComplete()); + result.put("adminOnly", AppProps.getInstance().isUserRequestedAdminOnlyMode()); + + return new ApiSimpleResponse(result); + } + } + + @RequiresSiteAdmin + @IgnoresTermsOfUse + public static class GetPendingRequestCountAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(Object o, BindException errors) + { + JSONObject result = new JSONObject(); + result.put("pendingRequestCount", TransactionFilter.getPendingRequestCount() - 1 /* Exclude this request */); + + return new ApiSimpleResponse(result); + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetModulesAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(GetModulesForm form, BindException errors) + { + Container c = ContainerManager.getForPath(getContainer().getPath()); + + ApiSimpleResponse response = new ApiSimpleResponse(); + + List> qinfos = new ArrayList<>(); + + FolderType folderType = c.getFolderType(); + List allModules = new ArrayList<>(ModuleLoader.getInstance().getModules()); + allModules.sort(Comparator.comparing(module -> module.getTabName(getViewContext()), String.CASE_INSENSITIVE_ORDER)); + + //note: this has been altered to use Container.getRequiredModules() instead of FolderType + //this is b/c a parent container must consider child workbooks when determining the set of requiredModules + Set requiredModules = c.getRequiredModules(); //folderType.getActiveModules() != null ? folderType.getActiveModules() : new HashSet(); + Set activeModules = c.getActiveModules(getUser()); + + for (Module m : allModules) + { + Map qinfo = new HashMap<>(); + + qinfo.put("name", m.getName()); + qinfo.put("required", requiredModules.contains(m)); + qinfo.put("active", activeModules.contains(m) || requiredModules.contains(m)); + qinfo.put("enabled", (m.getTabDisplayMode() == Module.TabDisplayMode.DISPLAY_USER_PREFERENCE || + m.getTabDisplayMode() == Module.TabDisplayMode.DISPLAY_USER_PREFERENCE_DEFAULT) && !requiredModules.contains(m)); + qinfo.put("tabName", m.getTabName(getViewContext())); + qinfo.put("requireSitePermission", m.getRequireSitePermission()); + qinfos.add(qinfo); + } + + response.put("modules", qinfos); + response.put("folderType", folderType.getName()); + + return response; + } + } + + public static class GetModulesForm + { + } + + @RequiresNoPermission + @AllowedDuringUpgrade + // This action is invoked by HttpsUtil.checkSslRedirectConfiguration(), often while upgrade is in progress + public static class GuidAction extends ExportAction + { + @Override + public void export(Object o, HttpServletResponse response, BindException errors) throws Exception + { + response.getWriter().write(GUID.makeGUID()); + } + } + + /** + * Preform health checks corresponding to the given categories. + */ + @Marshal(Marshaller.Jackson) + @RequiresNoPermission + @AllowedDuringUpgrade + @AllowedBeforeInitialUserIsSet + public static class HealthCheckAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(HealthCheckForm form, BindException errors) throws Exception + { + if (!ModuleLoader.getInstance().isStartupComplete()) + return new ApiSimpleResponse("healthy", false); + + Collection categories = form.getCategories() == null ? Collections.singleton(HealthCheckRegistry.DEFAULT_CATEGORY) : Arrays.asList(form.getCategories().split(",")); + HealthCheck.Result checkResult = HealthCheckRegistry.get().checkHealth(categories); + + checkResult.getDetails().put("healthy", checkResult.isHealthy()); + + if (getUser().hasRootAdminPermission()) + { + return new ApiSimpleResponse(checkResult.getDetails()); + } + else + { + if (!checkResult.isHealthy()) + { + try (var writer = createResponseWriter()) + { + writer.writeResponse(HttpServletResponse.SC_SERVICE_UNAVAILABLE, "Server isn't ready yet"); + } + return null; + } + + return new ApiSimpleResponse("healthy", checkResult.isHealthy()); + } + } + } + + public static class HealthCheckForm + { + private String _categories; // if null, all categories will be checked. + + public String getCategories() + { + return _categories; + } + + @SuppressWarnings("unused") + public void setCategories(String categories) + { + _categories = categories; + } + } + + // No security checks... anyone (even guests) can view the credits page + @RequiresNoPermission + public class CreditsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + VBox views = new VBox(); + List modules = new ArrayList<>(ModuleLoader.getInstance().getModules()); + modules.sort(Comparator.comparing(Module::getName, String.CASE_INSENSITIVE_ORDER)); + + addCreditsViews(views, modules, "jars.txt", "JAR"); + addCreditsViews(views, modules, "scripts.txt", "Script, Icon and Font"); + addCreditsViews(views, modules, "source.txt", "Java Source Code"); + addCreditsViews(views, modules, "executables.txt", "Executable"); + + return views; + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Credits", this.getClass()); + } + } + + private void addCreditsViews(VBox views, List modules, String creditsFile, String fileType) throws IOException + { + for (Module module : modules) + { + String wikiSource = getCreditsFile(module, creditsFile); + + if (null != wikiSource) + { + String title = fileType + " Files Distributed with the " + module.getName() + " Module"; + CreditsView credits = new CreditsView(wikiSource, title); + views.addView(credits); + } + } + } + + private static class CreditsView extends WebPartView + { + private Renderable _html; + + CreditsView(@Nullable String wikiSource, String title) + { + super(title); + + wikiSource = StringUtils.trimToEmpty(wikiSource); + + if (StringUtils.isNotEmpty(wikiSource)) + { + WikiRenderingService wikiService = WikiRenderingService.get(); + HtmlString html = wikiService.getFormattedHtml(WikiRendererType.RADEOX, wikiSource, "Credits page"); + _html = DOM.createHtmlFragment(STYLE(at(type, "text/css"), "tr.table-odd td { background-color: #EEEEEE; }"), html); + } + } + + @Override + public void renderView(Object model, HtmlWriter out) + { + out.write(_html); + } + } + + private static String getCreditsFile(Module module, String filename) throws IOException + { + // credits files are in /resources/credits + InputStream is = module.getResourceStream("credits/" + filename); + + return null == is ? null : PageFlowUtil.getStreamContentsAsString(is); + } + + private void validateNetworkDrive(NetworkDriveForm form, Errors errors) + { + if (isBlank(form.getNetworkDriveUser()) || isBlank(form.getNetworkDrivePath()) || + isBlank(form.getNetworkDrivePassword()) || isBlank(form.getNetworkDriveLetter())) + { + errors.reject(ERROR_MSG, "All fields are required"); + } + else if (form.getNetworkDriveLetter().trim().length() > 1) + { + errors.reject(ERROR_MSG, "Network drive letter must be a single character"); + } + else + { + char letter = form.getNetworkDriveLetter().trim().toLowerCase().charAt(0); + + if (letter < 'a' || letter > 'z') + { + errors.reject(ERROR_MSG, "Network drive letter must be a letter"); + } + } + } + + public static class ResourceForm + { + private String _resource; + + public String getResource() + { + return _resource; + } + + public void setResource(String resource) + { + _resource = resource; + } + + public ResourceType getResourceType() + { + return ResourceType.valueOf(_resource); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ResetResourceAction extends FormHandlerAction + { + @Override + public void validateCommand(ResourceForm target, Errors errors) + { + } + + @Override + public boolean handlePost(ResourceForm form, BindException errors) throws Exception + { + form.getResourceType().delete(getContainer(), getUser()); + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + return true; + } + + @Override + public URLHelper getSuccessURL(ResourceForm form) + { + return new AdminUrlsImpl().getLookAndFeelResourcesURL(getContainer()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ResetPropertiesAction extends FormHandlerAction + { + private URLHelper _returnUrl; + + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + Container c = getContainer(); + boolean folder = !(c.isRoot() || c.isProject()); + boolean hasAdminOpsPerm = c.hasPermission(getUser(), AdminOperationsPermission.class); + + WriteableFolderLookAndFeelProperties props = folder ? LookAndFeelProperties.getWriteableFolderInstance(c) : LookAndFeelProperties.getWriteableInstance(c); + props.clear(hasAdminOpsPerm); + props.save(); + // TODO: Audit log? + + AdminUrls urls = new AdminUrlsImpl(); + + // Folder-level settings are just display formats and measure/dimension flags -- no need to increment L&F revision + if (!folder) + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + + _returnUrl = urls.getLookAndFeelSettingsURL(c); + + return true; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return _returnUrl; + } + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public class CustomizeSiteAction extends FormViewAction + { + @Override + public ModelAndView getView(SiteSettingsForm form, boolean reshow, BindException errors) + { + if (form.isUpgradeInProgress()) + getPageConfig().setTemplate(Template.Dialog); + + SiteSettingsBean bean = new SiteSettingsBean(form.isUpgradeInProgress()); + setHelpTopic("configAdmin"); + return new JspView<>("/org/labkey/core/admin/customizeSite.jsp", bean, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Customize Site", this.getClass()); + } + + @Override + public void validateCommand(SiteSettingsForm form, Errors errors) + { + if (form.isShowRibbonMessage() && StringUtils.isEmpty(form.getRibbonMessage())) + { + errors.reject(ERROR_MSG, "Cannot enable the ribbon message without providing a message to show"); + } + if (form.getMaxBLOBSize() < 0) + { + errors.reject(ERROR_MSG, "Maximum BLOB size cannot be negative"); + } + int hardCap = Math.max(WriteableAppProps.SOFT_MAX_BLOB_SIZE, AppProps.getInstance().getMaxBLOBSize()); + if (form.getMaxBLOBSize() > hardCap) + { + errors.reject(ERROR_MSG, "Maximum BLOB size cannot be set higher than " + hardCap + " bytes"); + } + if (form.getSslPort() < 1 || form.getSslPort() > 65535) + { + errors.reject(ERROR_MSG, "HTTPS port must be between 1 and 65,535"); + } + if (form.getReadOnlyHttpRequestTimeout() < 0) + { + errors.reject(ERROR_MSG, "HTTP timeout must be non-negative"); + } + if (form.getMemoryUsageDumpInterval() < 0) + { + errors.reject(ERROR_MSG, "Memory logging frequency must be non-negative"); + } + } + + @Override + public boolean handlePost(SiteSettingsForm form, BindException errors) throws Exception + { + HttpServletRequest request = getViewContext().getRequest(); + + // We only need to check that SSL is running if the user isn't already using SSL + if (form.isSslRequired() && !(request.isSecure() && (form.getSslPort() == request.getServerPort()))) + { + URL testURL = new URL("https", request.getServerName(), form.getSslPort(), AppProps.getInstance().getContextPath()); + Pair sslResponse = HttpsUtil.testHttpsUrl(testURL, "Ensure that the web server is configured for SSL and the port is correct. If SSL is enabled, try saving these settings while connected via SSL."); + + if (sslResponse != null) + { + errors.reject(ERROR_MSG, sslResponse.first); + return false; + } + } + + if (form.getReadOnlyHttpRequestTimeout() < 0) + { + errors.reject(ERROR_MSG, "Read only HTTP request timeout must be non-negative"); + } + + WriteableAppProps props = AppProps.getWriteableInstance(); + + props.setPipelineToolsDir(form.getPipelineToolsDirectory()); + props.setNavAccessOpen(form.isNavAccessOpen()); + props.setSSLRequired(form.isSslRequired()); + boolean sslSettingChanged = AppProps.getInstance().isSSLRequired() != form.isSslRequired(); + props.setSSLPort(form.getSslPort()); + props.setMemoryUsageDumpInterval(form.getMemoryUsageDumpInterval()); + props.setReadOnlyHttpRequestTimeout(form.getReadOnlyHttpRequestTimeout()); + props.setMaxBLOBSize(form.getMaxBLOBSize()); + props.setExt3Required(form.isExt3Required()); + props.setExt3APIRequired(form.isExt3APIRequired()); + props.setSelfReportExceptions(form.isSelfReportExceptions()); + + props.setAdminOnlyMessage(form.getAdminOnlyMessage()); + props.setShowRibbonMessage(form.isShowRibbonMessage()); + props.setRibbonMessage(form.getRibbonMessage()); + props.setUserRequestedAdminOnlyMode(form.isAdminOnlyMode()); + + props.setAllowApiKeys(form.isAllowApiKeys()); + props.setApiKeyExpirationSeconds(form.getApiKeyExpirationSeconds()); + props.setAllowSessionKeys(form.isAllowSessionKeys()); + + try + { + ExceptionReportingLevel level = ExceptionReportingLevel.valueOf(form.getExceptionReportingLevel()); + props.setExceptionReportingLevel(level); + } + catch (IllegalArgumentException ignored) + { + } + + try + { + if (form.getUsageReportingLevel() != null) + { + UsageReportingLevel level = UsageReportingLevel.valueOf(form.getUsageReportingLevel()); + props.setUsageReportingLevel(level); + } + } + catch (IllegalArgumentException ignored) + { + } + + props.setAdministratorContactEmail(form.getAdministratorContactEmail() == null ? null : form.getAdministratorContactEmail().trim()); + + if (null != form.getBaseServerURL()) + { + if (form.isSslRequired() && !form.getBaseServerURL().startsWith("https")) + { + errors.reject(ERROR_MSG, "Invalid Base Server URL. SSL connection is required. Consider https://."); + return false; + } + + try + { + props.setBaseServerUrl(form.getBaseServerURL()); + } + catch (URISyntaxException e) + { + errors.reject(ERROR_MSG, "Invalid Base Server URL, \"" + e.getMessage() + "\"." + + "Please enter a valid base URL containing the protocol, hostname, and port if required. " + + "The webapp context path should not be included. " + + "For example: \"https://www.example.com\" or \"http://www.labkey.org:8080\" and not \"http://www.example.com/labkey/\""); + return false; + } + } + + String frameOption = StringUtils.trimToEmpty(form.getXFrameOption()); + if (!frameOption.equals("DENY") && !frameOption.equals("SAMEORIGIN") && !frameOption.equals("ALLOW")) + { + errors.reject(ERROR_MSG, "XFrameOption must equal DENY, or SAMEORIGIN, or ALLOW"); + return false; + } + props.setXFrameOption(frameOption); + props.setIncludeServerHttpHeader(form.isIncludeServerHttpHeader()); + + props.save(getViewContext().getUser()); + UsageReportingLevel.reportNow(); + if (sslSettingChanged) + ContentSecurityPolicyFilter.regenerateSubstitutionMap(); + + return true; + } + + @Override + public ActionURL getSuccessURL(SiteSettingsForm form) + { + if (form.isUpgradeInProgress()) + { + return AppProps.getInstance().getHomePageActionURL(); + } + else + { + return new AdminUrlsImpl().getAdminConsoleURL(); + } + } + } + + public static class NetworkDriveForm + { + private String _networkDriveLetter; + private String _networkDrivePath; + private String _networkDriveUser; + private String _networkDrivePassword; + + public String getNetworkDriveLetter() + { + return _networkDriveLetter; + } + + public void setNetworkDriveLetter(String networkDriveLetter) + { + _networkDriveLetter = networkDriveLetter; + } + + public String getNetworkDrivePassword() + { + return _networkDrivePassword; + } + + public void setNetworkDrivePassword(String networkDrivePassword) + { + _networkDrivePassword = networkDrivePassword; + } + + public String getNetworkDrivePath() + { + return _networkDrivePath; + } + + public void setNetworkDrivePath(String networkDrivePath) + { + _networkDrivePath = networkDrivePath; + } + + public String getNetworkDriveUser() + { + return _networkDriveUser; + } + + public void setNetworkDriveUser(String networkDriveUser) + { + _networkDriveUser = networkDriveUser; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + @AdminConsoleAction + public class MapNetworkDriveAction extends FormViewAction + { + @Override + public void validateCommand(NetworkDriveForm form, Errors errors) + { + validateNetworkDrive(form, errors); + } + + @Override + public ModelAndView getView(NetworkDriveForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/mapNetworkDrive.jsp", null, errors); + } + + @Override + public boolean handlePost(NetworkDriveForm form, BindException errors) throws Exception + { + NetworkDriveProps.setNetworkDriveLetter(form.getNetworkDriveLetter().trim()); + NetworkDriveProps.setNetworkDrivePath(form.getNetworkDrivePath().trim()); + NetworkDriveProps.setNetworkDriveUser(form.getNetworkDriveUser().trim()); + NetworkDriveProps.setNetworkDrivePassword(form.getNetworkDrivePassword().trim()); + + return true; + } + + @Override + public URLHelper getSuccessURL(NetworkDriveForm siteSettingsForm) + { + return new ActionURL(FilesSiteSettingsAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("setRoots#map"); + addAdminNavTrail(root, "Map Network Drive", this.getClass()); + } + } + + public static class SiteSettingsBean + { + public final boolean _upgradeInProgress; + public final boolean _showSelfReportExceptions; + + private SiteSettingsBean(boolean upgradeInProgress) + { + _upgradeInProgress = upgradeInProgress; + _showSelfReportExceptions = MothershipReport.isShowSelfReportExceptions(); + } + + public HtmlString getSiteSettingsHelpLink(String fragment) + { + return new HelpTopic("configAdmin", fragment).getSimpleLinkHtml("more info..."); + } + } + + public static class SetRibbonMessageForm + { + private Boolean _show = null; + private String _message = null; + + public Boolean isShow() + { + return _show; + } + + public void setShow(Boolean show) + { + _show = show; + } + + public String getMessage() + { + return _message; + } + + public void setMessage(String message) + { + _message = message; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class SetRibbonMessageAction extends MutatingApiAction + { + @Override + public Object execute(SetRibbonMessageForm form, BindException errors) throws Exception + { + if (form.isShow() != null || form.getMessage() != null) + { + WriteableAppProps props = AppProps.getWriteableInstance(); + + if (form.isShow() != null) + props.setShowRibbonMessage(form.isShow()); + + if (form.getMessage() != null) + props.setRibbonMessage(form.getMessage()); + + props.save(getViewContext().getUser()); + } + + return null; + } + } + + @RequiresPermission(AdminPermission.class) + public class ConfigureSiteValidationAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + return new JspView<>("/org/labkey/core/admin/sitevalidation/configureSiteValidation.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("siteValidation"); + addAdminNavTrail(root, "Configure " + (getContainer().isRoot() ? "Site" : "Folder") + " Validation", getClass()); + } + } + + public static class SiteValidationForm + { + private List _providers; + private boolean _includeSubfolders = false; + private transient Consumer _logger = s -> { + }; // No-op by default + + public List getProviders() + { + return _providers; + } + + public void setProviders(List providers) + { + _providers = providers; + } + + public boolean isIncludeSubfolders() + { + return _includeSubfolders; + } + + public void setIncludeSubfolders(boolean includeSubfolders) + { + _includeSubfolders = includeSubfolders; + } + + public Consumer getLogger() + { + return _logger; + } + + public void setLogger(Consumer logger) + { + _logger = logger; + } + } + + @RequiresPermission(AdminPermission.class) + public class SiteValidationAction extends SimpleViewAction + { + @Override + public ModelAndView getView(SiteValidationForm form, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/sitevalidation/siteValidation.jsp", form); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("siteValidation"); + addAdminNavTrail(root, (getContainer().isRoot() ? "Site" : "Folder") + " Validation", getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class SiteValidationBackgroundAction extends FormHandlerAction + { + private ActionURL _redirectUrl; + + @Override + public void validateCommand(SiteValidationForm form, Errors errors) + { + } + + @Override + public boolean handlePost(SiteValidationForm form, BindException errors) throws PipelineValidationException + { + ViewBackgroundInfo vbi = new ViewBackgroundInfo(getContainer(), getUser(), null); + PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); + SiteValidationJob job = new SiteValidationJob(vbi, root, form); + PipelineService.get().queueJob(job); + String jobGuid = job.getJobGUID(); + + if (null == jobGuid) + throw new NotFoundException("Unable to determine pipeline job GUID"); + + Long jobId = PipelineService.get().getJobId(getUser(), getContainer(), jobGuid); + + if (null == jobId) + throw new NotFoundException("Unable to determine pipeline job ID"); + + PipelineStatusUrls urls = urlProvider(PipelineStatusUrls.class); + _redirectUrl = urls.urlDetails(getContainer(), jobId); + + return true; + } + + @Override + public URLHelper getSuccessURL(SiteValidationForm form) + { + return _redirectUrl; + } + } + + public static class ViewValidationResultsForm + { + private int _rowId; + + public int getRowId() + { + return _rowId; + } + + public void setRowId(int rowId) + { + _rowId = rowId; + } + } + + @RequiresPermission(AdminPermission.class) + public class ViewValidationResultsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ViewValidationResultsForm form, BindException errors) throws Exception + { + PipelineStatusFile statusFile = PipelineService.get().getStatusFile(form.getRowId()); + if (null == statusFile) + throw new NotFoundException("Status file not found"); + if (!getContainer().equals(statusFile.lookupContainer())) + throw new UnauthorizedException("Wrong container"); + + String logFilePath = statusFile.getFilePath(); + String htmlFilePath = FileUtil.getBaseName(logFilePath) + ".html"; + File htmlFile = new File(htmlFilePath); + + if (!htmlFile.exists()) + throw new NotFoundException("Results file not found"); + return new HtmlView(HtmlString.unsafe(PageFlowUtil.getFileContentsAsString(htmlFile))); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("siteValidation"); + addAdminNavTrail(root, "View " + (getContainer().isRoot() ? "Site" : "Folder") + " Validation Results", getClass()); + } + } + + public interface FileManagementForm + { + String getFolderRootPath(); + + void setFolderRootPath(String folderRootPath); + + String getFileRootOption(); + + void setFileRootOption(String fileRootOption); + + String getConfirmMessage(); + + void setConfirmMessage(String confirmMessage); + + boolean isDisableFileSharing(); + + boolean hasSiteDefaultRoot(); + + String[] getEnabledCloudStore(); + + @SuppressWarnings("unused") + void setEnabledCloudStore(String[] enabledCloudStore); + + boolean isCloudFileRoot(); + + @Nullable + String getCloudRootName(); + + void setCloudRootName(String cloudRootName); + + void setFileRootChanged(boolean changed); + + void setEnabledCloudStoresChanged(boolean changed); + + String getMigrateFilesOption(); + + void setMigrateFilesOption(String migrateFilesOption); + + default boolean isFolderSetup() + { + return false; + } + } + + public enum MigrateFilesOption implements SafeToRenderEnum + { + leave + { + @Override + public String description() + { + return "Source files not copied or moved"; + } + }, + copy + { + @Override + public String description() + { + return "Copy source files to destination"; + } + }, + move + { + @Override + public String description() + { + return "Move source files to destination"; + } + }; + + public abstract String description(); + } + + public static class ProjectSettingsForm extends FolderSettingsForm + { + // Site-only properties + private String _dateParsingMode; + private String _customWelcome; + + // Site & project properties + private boolean _shouldInherit; // new subfolders should inherit parent permissions + private String _systemDescription; + private boolean _systemDescriptionInherited; + private String _systemShortName; + private boolean _systemShortNameInherited; + private String _themeName; + private boolean _themeNameInherited; + private String _folderDisplayMode; + private boolean _folderDisplayModeInherited; + private String _applicationMenuDisplayMode; + private boolean _applicationMenuDisplayModeInherited; + private boolean _helpMenuEnabled; + private boolean _helpMenuEnabledInherited; + private boolean _discussionEnabled; + private boolean _discussionEnabledInherited; + private String _logoHref; + private boolean _logoHrefInherited; + private String _companyName; + private boolean _companyNameInherited; + private String _systemEmailAddress; + private boolean _systemEmailAddressInherited; + private String _reportAProblemPath; + private boolean _reportAProblemPathInherited; + private String _supportEmail; + private boolean _supportEmailInherited; + private String _customLogin; + private boolean _customLoginInherited; + + // Site-only properties + + public String getDateParsingMode() + { + return _dateParsingMode; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setDateParsingMode(String dateParsingMode) + { + _dateParsingMode = dateParsingMode; + } + + public String getCustomWelcome() + { + return _customWelcome; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCustomWelcome(String customWelcome) + { + _customWelcome = customWelcome; + } + + // Site & project properties + + public boolean getShouldInherit() + { + return _shouldInherit; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setShouldInherit(boolean b) + { + _shouldInherit = b; + } + + public String getSystemDescription() + { + return _systemDescription; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemDescription(String systemDescription) + { + _systemDescription = systemDescription; + } + + public boolean isSystemDescriptionInherited() + { + return _systemDescriptionInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemDescriptionInherited(boolean systemDescriptionInherited) + { + _systemDescriptionInherited = systemDescriptionInherited; + } + + public String getSystemShortName() + { + return _systemShortName; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemShortName(String systemShortName) + { + _systemShortName = systemShortName; + } + + public boolean isSystemShortNameInherited() + { + return _systemShortNameInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemShortNameInherited(boolean systemShortNameInherited) + { + _systemShortNameInherited = systemShortNameInherited; + } + + public String getThemeName() + { + return _themeName; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setThemeName(String themeName) + { + _themeName = themeName; + } + + public boolean isThemeNameInherited() + { + return _themeNameInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setThemeNameInherited(boolean themeNameInherited) + { + _themeNameInherited = themeNameInherited; + } + + public String getFolderDisplayMode() + { + return _folderDisplayMode; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setFolderDisplayMode(String folderDisplayMode) + { + _folderDisplayMode = folderDisplayMode; + } + + public boolean isFolderDisplayModeInherited() + { + return _folderDisplayModeInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setFolderDisplayModeInherited(boolean folderDisplayModeInherited) + { + _folderDisplayModeInherited = folderDisplayModeInherited; + } + + public String getApplicationMenuDisplayMode() + { + return _applicationMenuDisplayMode; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setApplicationMenuDisplayMode(String displayMode) + { + _applicationMenuDisplayMode = displayMode; + } + + public boolean isApplicationMenuDisplayModeInherited() + { + return _applicationMenuDisplayModeInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setApplicationMenuDisplayModeInherited(boolean applicationMenuDisplayModeInherited) + { + _applicationMenuDisplayModeInherited = applicationMenuDisplayModeInherited; + } + + public boolean isHelpMenuEnabled() + { + return _helpMenuEnabled; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setHelpMenuEnabled(boolean helpMenuEnabled) + { + _helpMenuEnabled = helpMenuEnabled; + } + + public boolean isHelpMenuEnabledInherited() + { + return _helpMenuEnabledInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setHelpMenuEnabledInherited(boolean helpMenuEnabledInherited) + { + _helpMenuEnabledInherited = helpMenuEnabledInherited; + } + + public boolean isDiscussionEnabled() + { + return _discussionEnabled; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setDiscussionEnabled(boolean discussionEnabled) + { + _discussionEnabled = discussionEnabled; + } + + public boolean isDiscussionEnabledInherited() + { + return _discussionEnabledInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setDiscussionEnabledInherited(boolean discussionEnabledInherited) + { + _discussionEnabledInherited = discussionEnabledInherited; + } + + public String getLogoHref() + { + return _logoHref; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setLogoHref(String logoHref) + { + _logoHref = logoHref; + } + + public boolean isLogoHrefInherited() + { + return _logoHrefInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setLogoHrefInherited(boolean logoHrefInherited) + { + _logoHrefInherited = logoHrefInherited; + } + + public String getReportAProblemPath() + { + return _reportAProblemPath; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setReportAProblemPath(String reportAProblemPath) + { + _reportAProblemPath = reportAProblemPath; + } + + public boolean isReportAProblemPathInherited() + { + return _reportAProblemPathInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setReportAProblemPathInherited(boolean reportAProblemPathInherited) + { + _reportAProblemPathInherited = reportAProblemPathInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSupportEmail(String supportEmail) + { + _supportEmail = supportEmail; + } + + public String getSupportEmail() + { + return _supportEmail; + } + + public boolean isSupportEmailInherited() + { + return _supportEmailInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSupportEmailInherited(boolean supportEmailInherited) + { + _supportEmailInherited = supportEmailInherited; + } + + public String getSystemEmailAddress() + { + return _systemEmailAddress; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemEmailAddress(String systemEmailAddress) + { + _systemEmailAddress = systemEmailAddress; + } + + public boolean isSystemEmailAddressInherited() + { + return _systemEmailAddressInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemEmailAddressInherited(boolean systemEmailAddressInherited) + { + _systemEmailAddressInherited = systemEmailAddressInherited; + } + + public String getCompanyName() + { + return _companyName; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCompanyName(String companyName) + { + _companyName = companyName; + } + + public boolean isCompanyNameInherited() + { + return _companyNameInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCompanyNameInherited(boolean companyNameInherited) + { + _companyNameInherited = companyNameInherited; + } + + public String getCustomLogin() + { + return _customLogin; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCustomLogin(String customLogin) + { + _customLogin = customLogin; + } + + public boolean isCustomLoginInherited() + { + return _customLoginInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCustomLoginInherited(boolean customLoginInherited) + { + _customLoginInherited = customLoginInherited; + } + } + + public enum FileRootProp implements SafeToRenderEnum + { + disable, + siteDefault, + folderOverride, + cloudRoot + } + + public static class FilesForm extends SetupForm implements FileManagementForm + { + private boolean _fileRootChanged; + private boolean _enabledCloudStoresChanged; + private String _cloudRootName; + private String _migrateFilesOption; + private String[] _enabledCloudStore; + private String _fileRootOption; + private String _folderRootPath; + + public boolean isFileRootChanged() + { + return _fileRootChanged; + } + + @Override + public void setFileRootChanged(boolean changed) + { + _fileRootChanged = changed; + } + + public boolean isEnabledCloudStoresChanged() + { + return _enabledCloudStoresChanged; + } + + @Override + public void setEnabledCloudStoresChanged(boolean enabledCloudStoresChanged) + { + _enabledCloudStoresChanged = enabledCloudStoresChanged; + } + + @Override + public boolean isDisableFileSharing() + { + return FileRootProp.disable.name().equals(getFileRootOption()); + } + + @Override + public boolean hasSiteDefaultRoot() + { + return FileRootProp.siteDefault.name().equals(getFileRootOption()); + } + + @Override + public String[] getEnabledCloudStore() + { + return _enabledCloudStore; + } + + @Override + public void setEnabledCloudStore(String[] enabledCloudStore) + { + _enabledCloudStore = enabledCloudStore; + } + + @Override + public boolean isCloudFileRoot() + { + return FileRootProp.cloudRoot.name().equals(getFileRootOption()); + } + + @Override + @Nullable + public String getCloudRootName() + { + return _cloudRootName; + } + + @Override + public void setCloudRootName(String cloudRootName) + { + _cloudRootName = cloudRootName; + } + + @Override + public String getMigrateFilesOption() + { + return _migrateFilesOption; + } + + @Override + public void setMigrateFilesOption(String migrateFilesOption) + { + _migrateFilesOption = migrateFilesOption; + } + + @Override + public String getFolderRootPath() + { + return _folderRootPath; + } + + @Override + public void setFolderRootPath(String folderRootPath) + { + _folderRootPath = folderRootPath; + } + + @Override + public String getFileRootOption() + { + return _fileRootOption; + } + + @Override + public void setFileRootOption(String fileRootOption) + { + _fileRootOption = fileRootOption; + } + } + + @SuppressWarnings("unused") + public static class SiteSettingsForm + { + private boolean _upgradeInProgress = false; + + private String _pipelineToolsDirectory; + private boolean _sslRequired; + private boolean _adminOnlyMode; + private boolean _showRibbonMessage; + private boolean _ext3Required; + private boolean _ext3APIRequired; + private boolean _selfReportExceptions; + private String _adminOnlyMessage; + private String _ribbonMessage; + private int _sslPort; + private int _memoryUsageDumpInterval; + private int _readOnlyHttpRequestTimeout; + private int _maxBLOBSize; + private String _exceptionReportingLevel; + private String _usageReportingLevel; + private String _administratorContactEmail; + + private String _baseServerURL; + private String _callbackPassword; + private boolean _allowApiKeys; + private int _apiKeyExpirationSeconds; + private boolean _allowSessionKeys; + private boolean _navAccessOpen; + + private String _XFrameOption; + private boolean _includeServerHttpHeader; + + public String getPipelineToolsDirectory() + { + return _pipelineToolsDirectory; + } + + public void setPipelineToolsDirectory(String pipelineToolsDirectory) + { + _pipelineToolsDirectory = pipelineToolsDirectory; + } + + public boolean isNavAccessOpen() + { + return _navAccessOpen; + } + + public void setNavAccessOpen(boolean navAccessOpen) + { + _navAccessOpen = navAccessOpen; + } + + public boolean isSslRequired() + { + return _sslRequired; + } + + public void setSslRequired(boolean sslRequired) + { + _sslRequired = sslRequired; + } + + public boolean isExt3Required() + { + return _ext3Required; + } + + public void setExt3Required(boolean ext3Required) + { + _ext3Required = ext3Required; + } + + public boolean isExt3APIRequired() + { + return _ext3APIRequired; + } + + public void setExt3APIRequired(boolean ext3APIRequired) + { + _ext3APIRequired = ext3APIRequired; + } + + public int getSslPort() + { + return _sslPort; + } + + public void setSslPort(int sslPort) + { + _sslPort = sslPort; + } + + public boolean isAdminOnlyMode() + { + return _adminOnlyMode; + } + + public void setAdminOnlyMode(boolean adminOnlyMode) + { + _adminOnlyMode = adminOnlyMode; + } + + public String getAdminOnlyMessage() + { + return _adminOnlyMessage; + } + + public void setAdminOnlyMessage(String adminOnlyMessage) + { + _adminOnlyMessage = adminOnlyMessage; + } + + public boolean isSelfReportExceptions() + { + return _selfReportExceptions; + } + + public void setSelfReportExceptions(boolean selfReportExceptions) + { + _selfReportExceptions = selfReportExceptions; + } + + public String getExceptionReportingLevel() + { + return _exceptionReportingLevel; + } + + public void setExceptionReportingLevel(String exceptionReportingLevel) + { + _exceptionReportingLevel = exceptionReportingLevel; + } + + public String getUsageReportingLevel() + { + return _usageReportingLevel; + } + + public void setUsageReportingLevel(String usageReportingLevel) + { + _usageReportingLevel = usageReportingLevel; + } + + public String getAdministratorContactEmail() + { + return _administratorContactEmail; + } + + public void setAdministratorContactEmail(String administratorContactEmail) + { + _administratorContactEmail = administratorContactEmail; + } + + public boolean isUpgradeInProgress() + { + return _upgradeInProgress; + } + + public void setUpgradeInProgress(boolean upgradeInProgress) + { + _upgradeInProgress = upgradeInProgress; + } + + public int getMemoryUsageDumpInterval() + { + return _memoryUsageDumpInterval; + } + + public void setMemoryUsageDumpInterval(int memoryUsageDumpInterval) + { + _memoryUsageDumpInterval = memoryUsageDumpInterval; + } + + public int getReadOnlyHttpRequestTimeout() + { + return _readOnlyHttpRequestTimeout; + } + + public void setReadOnlyHttpRequestTimeout(int timeout) + { + _readOnlyHttpRequestTimeout = timeout; + } + + public int getMaxBLOBSize() + { + return _maxBLOBSize; + } + + public void setMaxBLOBSize(int maxBLOBSize) + { + _maxBLOBSize = maxBLOBSize; + } + + public String getBaseServerURL() + { + return _baseServerURL; + } + + public void setBaseServerURL(String baseServerURL) + { + _baseServerURL = baseServerURL; + } + + public String getCallbackPassword() + { + return _callbackPassword; + } + + public void setCallbackPassword(String callbackPassword) + { + _callbackPassword = callbackPassword; + } + + public boolean isShowRibbonMessage() + { + return _showRibbonMessage; + } + + public void setShowRibbonMessage(boolean showRibbonMessage) + { + _showRibbonMessage = showRibbonMessage; + } + + public String getRibbonMessage() + { + return _ribbonMessage; + } + + public void setRibbonMessage(String ribbonMessage) + { + _ribbonMessage = ribbonMessage; + } + + public boolean isAllowApiKeys() + { + return _allowApiKeys; + } + + public void setAllowApiKeys(boolean allowApiKeys) + { + _allowApiKeys = allowApiKeys; + } + + public int getApiKeyExpirationSeconds() + { + return _apiKeyExpirationSeconds; + } + + public void setApiKeyExpirationSeconds(int apiKeyExpirationSeconds) + { + _apiKeyExpirationSeconds = apiKeyExpirationSeconds; + } + + public boolean isAllowSessionKeys() + { + return _allowSessionKeys; + } + + public void setAllowSessionKeys(boolean allowSessionKeys) + { + _allowSessionKeys = allowSessionKeys; + } + + public String getXFrameOption() + { + return _XFrameOption; + } + + public void setXFrameOption(String XFrameOption) + { + _XFrameOption = XFrameOption; + } + + public boolean isIncludeServerHttpHeader() + { + return _includeServerHttpHeader; + } + + public void setIncludeServerHttpHeader(boolean includeServerHttpHeader) + { + _includeServerHttpHeader = includeServerHttpHeader; + } + } + + + @AdminConsoleAction + public class ShowThreadsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + // Log to labkey.log as well as showing through the browser + DebugInfoDumper.dumpThreads(3); + return new JspView<>("/org/labkey/core/admin/threads.jsp", new ThreadsBean()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("dumpDebugging#threads"); + addAdminNavTrail(root, "Current Threads", this.getClass()); + } + } + + private abstract class AbstractPostgresAction extends QueryViewAction + { + private final String _queryName; + + protected AbstractPostgresAction(String queryName) + { + super(QueryExportForm.class); + _queryName = queryName; + } + + @Override + protected QueryView createQueryView(QueryExportForm form, BindException errors, boolean forExport, @Nullable String dataRegion) throws Exception + { + if (!CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) + { + throw new NotFoundException("Only available with Postgres as the primary database"); + } + + QuerySettings qSettings = new QuerySettings(getViewContext(), "query", _queryName); + QueryView result = new QueryView(new PostgresUserSchema(getUser(), getContainer()), qSettings, errors) + { + @Override + public DataView createDataView() + { + // Troubleshooters don't have normal read access to the root container so grant them special access + // for these queries + DataView view = super.createDataView(); + view.getRenderContext().getViewContext().addContextualRole(ReaderRole.class); + return view; + } + }; + result.setTitle(_queryName); + result.setFrame(WebPartView.FrameType.PORTAL); + return result; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("postgresActivity"); + addAdminNavTrail(root, "Postgres " + _queryName, this.getClass()); + } + + } + + @AdminConsoleAction + public class PostgresStatActivityAction extends AbstractPostgresAction + { + public PostgresStatActivityAction() + { + super(PostgresUserSchema.POSTGRES_STAT_ACTIVITY_TABLE_NAME); + } + } + + @AdminConsoleAction + public class PostgresLocksAction extends AbstractPostgresAction + { + public PostgresLocksAction() + { + super(PostgresUserSchema.POSTGRES_LOCKS_TABLE_NAME); + } + } + + @AdminConsoleAction + public class DumpHeapAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + File destination = DebugInfoDumper.dumpHeap(); + return new HtmlView(HtmlString.of("Heap dumped to " + destination.getAbsolutePath())); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("dumpHeap"); + addAdminNavTrail(root, "Heap dump", getClass()); + } + } + + + public static class ThreadsBean + { + public Map> spids; + public List threads; + public Map stackTraces; + + ThreadsBean() + { + stackTraces = Thread.getAllStackTraces(); + threads = new ArrayList<>(stackTraces.keySet()); + threads.sort(Comparator.comparing(Thread::getName, String.CASE_INSENSITIVE_ORDER)); + + spids = new HashMap<>(); + + for (Thread t : threads) + { + spids.put(t, ConnectionWrapper.getSPIDsForThread(t)); + } + } + } + + + @RequiresPermission(AdminOperationsPermission.class) + public class ShowNetworkDriveTestAction extends SimpleViewAction + { + @Override + public void validate(NetworkDriveForm form, BindException errors) + { + validateNetworkDrive(form, errors); + } + + @Override + public ModelAndView getView(NetworkDriveForm form, BindException errors) + { + NetworkDrive testDrive = new NetworkDrive(); + testDrive.setPassword(form.getNetworkDrivePassword()); + testDrive.setPath(form.getNetworkDrivePath()); + testDrive.setUser(form.getNetworkDriveUser()); + TestNetworkDriveBean bean = new TestNetworkDriveBean(); + + if (!errors.hasErrors()) + { + char driveLetter = form.getNetworkDriveLetter().trim().charAt(0); + try + { + String mountError = testDrive.mount(driveLetter); + if (mountError != null) + { + errors.reject(ERROR_MSG, mountError); + } + else + { + File f = new File(driveLetter + ":\\"); + if (!f.exists()) + { + errors.reject(ERROR_MSG, "Could not access network drive"); + } + else + { + String[] fileNames = f.list(); + if (fileNames == null) + fileNames = new String[0]; + Arrays.sort(fileNames); + bean.setFiles(fileNames); + } + } + } + catch (IOException | InterruptedException e) + { + errors.reject(ERROR_MSG, "Error mounting drive: " + e); + } + try + { + testDrive.unmount(driveLetter); + } + catch (IOException | InterruptedException e) + { + errors.reject(ERROR_MSG, "Error mounting drive: " + e); + } + } + + getPageConfig().setTemplate(Template.Dialog); + return new JspView<>("/org/labkey/core/admin/testNetworkDrive.jsp", bean, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Test Mapping Network Drive"); + } + } + + + @AdminConsoleAction(ApplicationAdminPermission.class) + public class ResetErrorMarkAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) + { + return HtmlView.of("Are you sure you want to reset the site errors?"); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + File errorLogFile = getErrorLogFile(); + _errorMark = errorLogFile.length(); + + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + } + + @Override + public @NotNull URLHelper getSuccessURL(Object o) + { + return getShowAdminURL(); + } + } + + abstract public static class ShowLogAction extends ExportAction + { + @Override + public final void export(Object o, HttpServletResponse response, BindException errors) throws IOException + { + getPageConfig().setNoIndex(); + export(response); + } + + protected abstract void export(HttpServletResponse response) throws IOException; + } + + @AdminConsoleAction + public class ShowErrorsSinceMarkAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, _errorMark, getErrorLogFile()); + } + } + + @AdminConsoleAction + public class ShowAllErrorsAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, 0, getErrorLogFile()); + } + } + + @AdminConsoleAction(ApplicationAdminPermission.class) + public class ResetPrimaryLogMarkAction extends MutatingApiAction + { + @Override + public Object execute(Object o, BindException errors) throws Exception + { + File logFile = getPrimaryLogFile(); + _primaryLogMark = logFile.length(); + return null; + } + } + + @AdminConsoleAction + public class ShowPrimaryLogSinceMarkAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, _primaryLogMark, getPrimaryLogFile()); + } + } + + @AdminConsoleAction + public class ShowPrimaryLogAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, 0, getPrimaryLogFile()); + } + } + + @AdminConsoleAction + public class ShowCspReportLogAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, 0, getCspReportLogFile()); + } + } + + private File getErrorLogFile() + { + return new File(getLabKeyLogDir(), "labkey-errors.log"); + } + + private File getPrimaryLogFile() + { + return new File(getLabKeyLogDir(), "labkey.log"); + } + + private File getCspReportLogFile() + { + return new File(getLabKeyLogDir(), "csp-report.log"); + } + + private static ActionURL getActionsURL() + { + return new ActionURL(ActionsAction.class, ContainerManager.getRoot()); + } + + + @AdminConsoleAction + public class ActionsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new ActionsTabStrip(); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("actionsDiagnostics"); + addAdminNavTrail(root, "Actions", this.getClass()); + } + } + + private static class ActionsTabStrip extends TabStripView + { + @Override + public List getTabList() + { + List tabs = new ArrayList<>(3); + + tabs.add(new TabInfo("Summary", "summary", getActionsURL())); + tabs.add(new TabInfo("Details", "details", getActionsURL())); + tabs.add(new TabInfo("Exceptions", "exceptions", getActionsURL())); + + return tabs; + } + + @Override + public HttpView getTabView(String tabId) + { + if ("exceptions".equals(tabId)) + return new ActionsExceptionsView(); + return new ActionsView(!"details".equals(tabId)); + } + } + + @AdminConsoleAction + public static class ExportActionsAction extends ExportAction + { + @Override + public void export(Object form, HttpServletResponse response, BindException errors) throws Exception + { + try (ActionsTsvWriter writer = new ActionsTsvWriter()) + { + writer.write(response); + } + } + } + + private static ActionURL getQueriesURL(@Nullable String statName) + { + ActionURL url = new ActionURL(QueriesAction.class, ContainerManager.getRoot()); + + if (null != statName) + url.addParameter("stat", statName); + + return url; + } + + + @AdminConsoleAction + public class QueriesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(QueriesForm form, BindException errors) + { + String buttonHTML = ""; + if (getUser().hasRootAdminPermission()) + buttonHTML += PageFlowUtil.button("Reset All Statistics").href(getResetQueryStatisticsURL()).usePost() + " "; + buttonHTML += PageFlowUtil.button("Export").href(getExportQueriesURL()) + "

    "; + + return QueryProfiler.getInstance().getReportView(form.getStat(), buttonHTML, AdminController::getQueriesURL, + AdminController::getQueryStackTracesURL); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("queryPerf"); + addAdminNavTrail(root, "Queries", this.getClass()); + } + } + + public static class QueriesForm + { + private String _stat = "Count"; + + public String getStat() + { + return _stat; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setStat(String stat) + { + _stat = stat; + } + } + + + private static ActionURL getQueryStackTracesURL(String sqlHash) + { + ActionURL url = new ActionURL(QueryStackTracesAction.class, ContainerManager.getRoot()); + url.addParameter("sqlHash", sqlHash); + return url; + } + + + @AdminConsoleAction + public class QueryStackTracesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + return QueryProfiler.getInstance().getStackTraceView(form.getSqlHash(), AdminController::getExecutionPlanURL); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Queries", QueriesAction.class); + root.addChild("Query Stack Traces"); + } + } + + + private static ActionURL getExecutionPlanURL(String sqlHash) + { + ActionURL url = new ActionURL(ExecutionPlanAction.class, ContainerManager.getRoot()); + url.addParameter("sqlHash", sqlHash); + return url; + } + + + @AdminConsoleAction + public class ExecutionPlanAction extends SimpleViewAction + { + private String _sqlHash; + private ExecutionPlanType _type; + + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + _sqlHash = form.getSqlHash(); + _type = EnumUtils.getEnum(ExecutionPlanType.class, form.getType()); + if (null == _type) + throw new NotFoundException("Unknown execution plan type"); + + return QueryProfiler.getInstance().getExecutionPlanView(form.getSqlHash(), _type, form.isLog()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Queries", QueriesAction.class); + root.addChild("Query Stack Traces", getQueryStackTracesURL(_sqlHash)); + root.addChild(_type.getDescription()); + } + } + + + public static class QueryForm + { + private String _sqlHash; + private String _type = "Estimated"; // All dialects support Estimated + private boolean _log = false; + + public String getSqlHash() + { + return _sqlHash; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSqlHash(String sqlHash) + { + _sqlHash = sqlHash; + } + + public String getType() + { + return _type; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setType(String type) + { + _type = type; + } + + public boolean isLog() + { + return _log; + } + + public void setLog(boolean log) + { + _log = log; + } + } + + + private ActionURL getExportQueriesURL() + { + return new ActionURL(ExportQueriesAction.class, ContainerManager.getRoot()); + } + + + @AdminConsoleAction + public static class ExportQueriesAction extends ExportAction + { + @Override + public void export(Object o, HttpServletResponse response, BindException errors) throws Exception + { + try (QueryStatTsvWriter writer = new QueryStatTsvWriter()) + { + writer.setFilenamePrefix("SQL_Queries"); + writer.write(response); + } + } + } + + private static ActionURL getResetQueryStatisticsURL() + { + return new ActionURL(ResetQueryStatisticsAction.class, ContainerManager.getRoot()); + } + + + @RequiresPermission(AdminPermission.class) + public static class ResetQueryStatisticsAction extends FormHandlerAction + { + @Override + public void validateCommand(QueriesForm target, Errors errors) + { + } + + @Override + public boolean handlePost(QueriesForm form, BindException errors) throws Exception + { + QueryProfiler.getInstance().resetAllStatistics(); + return true; + } + + @Override + public URLHelper getSuccessURL(QueriesForm form) + { + return getQueriesURL(form.getStat()); + } + } + + + @AdminConsoleAction + public class CachesAction extends SimpleViewAction + { + private final DecimalFormat commaf0 = new DecimalFormat("#,##0"); + private final DecimalFormat percent = new DecimalFormat("0%"); + + @Override + public ModelAndView getView(MemForm form, BindException errors) + { + if (form.isClearCaches()) + { + LOG.info("Clearing Introspector caches"); + Introspector.flushCaches(); + LOG.info("Purging all caches"); + CacheManager.clearAllKnownCaches(); + ActionURL redirect = getViewContext().cloneActionURL().deleteParameter("clearCaches"); + throw new RedirectException(redirect); + } + + List> caches = CacheManager.getKnownCaches(); + + if (form.getDebugName() != null) + { + for (TrackingCache cache : caches) + { + if (form.getDebugName().equals(cache.getDebugName())) + { + LOG.info("Purging cache: " + cache.getDebugName()); + cache.clear(); + } + } + ActionURL redirect = getViewContext().cloneActionURL().deleteParameter("debugName"); + throw new RedirectException(redirect); + } + + List cacheStats = new ArrayList<>(); + List transactionStats = new ArrayList<>(); + + for (TrackingCache cache : caches) + { + cacheStats.add(CacheManager.getCacheStats(cache)); + transactionStats.add(CacheManager.getTransactionCacheStats(cache)); + } + + HtmlStringBuilder html = HtmlStringBuilder.of(); + + html.append(LinkBuilder.labkeyLink("Clear Caches and Refresh", getCachesURL(true, false))); + html.append(LinkBuilder.labkeyLink("Refresh", getCachesURL(false, false))); + + html.unsafeAppend("

    \n"); + appendStats(html, "Caches", cacheStats, false); + + html.unsafeAppend("

    \n"); + appendStats(html, "Transaction Caches", transactionStats, true); + + return new HtmlView(html); + } + + private void appendStats(HtmlStringBuilder html, String title, List allStats, boolean skipUnusedCaches) + { + List stats = skipUnusedCaches ? + allStats.stream() + .filter(stat->stat.getMaxSize() > 0) + .collect(Collectors.toCollection((Supplier>) ArrayList::new)) : + allStats; + + Collections.sort(stats); + + html.unsafeAppend("

    "); + html.append(title); + html.append(" (").append(stats.size()).unsafeAppend(")

    \n"); + + html.unsafeAppend("\n"); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + + long size = 0; + long gets = 0; + long misses = 0; + long puts = 0; + long expirations = 0; + long evictions = 0; + long removes = 0; + long clears = 0; + int rowCount = 0; + + for (CacheStats stat : stats) + { + size += stat.getSize(); + gets += stat.getGets(); + misses += stat.getMisses(); + puts += stat.getPuts(); + expirations += stat.getExpirations(); + evictions += stat.getEvictions(); + removes += stat.getRemoves(); + clears += stat.getClears(); + + html.unsafeAppend(""); + + appendDescription(html, stat.getDescription(), stat.getCreationStackTrace()); + + Long limit = stat.getLimit(); + long maxSize = stat.getMaxSize(); + + appendLongs(html, limit, maxSize, stat.getSize(), stat.getGets(), stat.getMisses(), stat.getPuts(), stat.getExpirations(), stat.getEvictions(), stat.getRemoves(), stat.getClears()); + appendDoubles(html, stat.getMissRatio()); + + html.unsafeAppend("\n"); + + if (null != limit && maxSize >= limit) + html.unsafeAppend(""); + + html.unsafeAppend("\n"); + rowCount++; + } + + double ratio = 0 != gets ? misses / (double)gets : 0; + html.unsafeAppend(""); + + appendLongs(html, null, null, size, gets, misses, puts, expirations, evictions, removes, clears); + appendDoubles(html, ratio); + + html.unsafeAppend("\n"); + html.unsafeAppend("
    Debug NameLimitMax SizeCurrent SizeGetsMissesPutsExpirationsEvictionsRemovesClearsMiss PercentageClear
    ").append(LinkBuilder.labkeyLink("Clear", getCacheURL(stat.getDescription()))).unsafeAppend("This cache has been limited
    Total
    \n"); + } + + private static final List PREFIXES_TO_SKIP = List.of( + "java.base/java.lang.Thread.getStackTrace", + "org.labkey.api.cache.CacheManager", + "org.labkey.api.cache.Throttle", + "org.labkey.api.data.DatabaseCache", + "org.labkey.api.module.ModuleResourceCache" + ); + + private void appendDescription(HtmlStringBuilder html, String description, @Nullable StackTraceElement[] creationStackTrace) + { + StringBuilder sb = new StringBuilder(); + + if (creationStackTrace != null) + { + boolean trimming = true; + for (StackTraceElement element : creationStackTrace) + { + // Skip the first few uninteresting stack trace elements to highlight the caller we care about + if (trimming) + { + if (PREFIXES_TO_SKIP.stream().anyMatch(prefix->element.toString().startsWith(prefix))) + continue; + + trimming = false; + } + sb.append(element); + sb.append("\n"); + } + } + + if (!sb.isEmpty()) + { + String message = PageFlowUtil.jsString(sb); + String id = "id" + UniqueID.getServerSessionScopedUID(); + html.append(DOM.createHtmlFragment(TD(A(at(href, "#").id(id), description)))); + HttpView.currentPageConfig().addHandler(id, "click", "alert(" + message + ");return false;"); + } + } + + private void appendLongs(HtmlStringBuilder html, Long... stats) + { + for (Long stat : stats) + { + if (null == stat) + html.unsafeAppend(" "); + else + html.unsafeAppend("").append(commaf0.format(stat)).unsafeAppend(""); + } + } + + private void appendDoubles(HtmlStringBuilder html, double... stats) + { + for (double stat : stats) + html.unsafeAppend("").append(percent.format(stat)).unsafeAppend(""); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("cachesDiagnostics"); + addAdminNavTrail(root, "Cache Statistics", this.getClass()); + } + } + + @RequiresSiteAdmin + public class EnvironmentVariablesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/properties.jsp", System.getenv()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Environment Variables", this.getClass()); + } + } + + @RequiresSiteAdmin + public class SystemPropertiesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView>("/org/labkey/core/admin/properties.jsp", new HashMap(System.getProperties())); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "System Properties", this.getClass()); + } + } + + + public static class ConfigureSystemMaintenanceForm + { + private String _maintenanceTime; + private Set _enable = Collections.emptySet(); + private boolean _enableSystemMaintenance = true; + + public String getMaintenanceTime() + { + return _maintenanceTime; + } + + @SuppressWarnings("unused") + public void setMaintenanceTime(String maintenanceTime) + { + _maintenanceTime = maintenanceTime; + } + + public Set getEnable() + { + return _enable; + } + + @SuppressWarnings("unused") + public void setEnable(Set enable) + { + _enable = enable; + } + + public boolean isEnableSystemMaintenance() + { + return _enableSystemMaintenance; + } + + @SuppressWarnings("unused") + public void setEnableSystemMaintenance(boolean enableSystemMaintenance) + { + _enableSystemMaintenance = enableSystemMaintenance; + } + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public class ConfigureSystemMaintenanceAction extends FormViewAction + { + @Override + public void validateCommand(ConfigureSystemMaintenanceForm form, Errors errors) + { + Date date = SystemMaintenance.parseSystemMaintenanceTime(form.getMaintenanceTime()); + + if (null == date) + errors.reject(ERROR_MSG, "Invalid format for system maintenance time"); + } + + @Override + public ModelAndView getView(ConfigureSystemMaintenanceForm form, boolean reshow, BindException errors) + { + SystemMaintenanceProperties prop = SystemMaintenance.getProperties(); + return new JspView<>("/org/labkey/core/admin/systemMaintenance.jsp", prop, errors); + } + + @Override + public boolean handlePost(ConfigureSystemMaintenanceForm form, BindException errors) + { + SystemMaintenance.setTimeDisabled(!form.isEnableSystemMaintenance()); + SystemMaintenance.setProperties(form.getEnable(), form.getMaintenanceTime()); + + return true; + } + + @Override + public URLHelper getSuccessURL(ConfigureSystemMaintenanceForm form) + { + return new AdminUrlsImpl().getAdminConsoleURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Configure System Maintenance", this.getClass()); + } + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public static class ResetSystemMaintenanceAction extends FormHandlerAction + { + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + SystemMaintenance.clearProperties(); + return true; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return new AdminUrlsImpl().getAdminConsoleURL(); + } + } + + public static class SystemMaintenanceForm + { + private String _taskName; + private boolean _test = false; + + public String getTaskName() + { + return _taskName; + } + + @SuppressWarnings("unused") + public void setTaskName(String taskName) + { + _taskName = taskName; + } + + public boolean isTest() + { + return _test; + } + + public void setTest(boolean test) + { + _test = test; + } + } + + @RequiresSiteAdmin + public class SystemMaintenanceAction extends FormHandlerAction + { + private Long _jobId = null; + private URLHelper _url = null; + + @Override + public void validateCommand(SystemMaintenanceForm form, Errors errors) + { + } + + @Override + public ModelAndView getSuccessView(SystemMaintenanceForm form) throws IOException + { + // Send the pipeline job details absolute URL back to the test + sendPlainText(_url.getURIString()); + + // Suppress templates, divs, etc. + getPageConfig().setTemplate(Template.None); + return new EmptyView(); + } + + @Override + public boolean handlePost(SystemMaintenanceForm form, BindException errors) + { + String jobGuid = new SystemMaintenanceJob(form.getTaskName(), getUser()).call(); + + if (null != jobGuid) + _jobId = PipelineService.get().getJobId(getUser(), getContainer(), jobGuid); + + PipelineStatusUrls urls = urlProvider(PipelineStatusUrls.class); + _url = null != _jobId ? urls.urlDetails(getContainer(), _jobId) : urls.urlBegin(getContainer()); + + return true; + } + + @Override + public URLHelper getSuccessURL(SystemMaintenanceForm form) + { + // In the standard case, redirect to the pipeline details URL + // If the test is invoking system maintenance then return the URL instead + return form.isTest() ? null : _url; + } + } + + @AdminConsoleAction + public class AttachmentsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return AttachmentService.get().getAdminView(getViewContext().getActionURL()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Attachments", getClass()); + } + } + + @AdminConsoleAction + public class FindAttachmentParentsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return AttachmentService.get().getFindAttachmentParentsView(); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Find Attachment Parents", getClass()); + } + } + + public static ActionURL getMemTrackerURL(boolean clearCaches, boolean gc) + { + ActionURL url = new ActionURL(MemTrackerAction.class, ContainerManager.getRoot()); + + if (clearCaches) + url.addParameter(MemForm.Params.clearCaches, "1"); + + if (gc) + url.addParameter(MemForm.Params.gc, "1"); + + return url; + } + + public static ActionURL getCachesURL(boolean clearCaches, boolean gc) + { + ActionURL url = new ActionURL(CachesAction.class, ContainerManager.getRoot()); + + if (clearCaches) + url.addParameter(MemForm.Params.clearCaches, "1"); + + if (gc) + url.addParameter(MemForm.Params.gc, "1"); + + return url; + } + + public static ActionURL getCacheURL(String debugName) + { + ActionURL url = new ActionURL(CachesAction.class, ContainerManager.getRoot()); + + url.addParameter(MemForm.Params.debugName, debugName); + + return url; + } + + private static volatile String lastCacheMemUsed = null; + + @AdminConsoleAction + public class MemTrackerAction extends SimpleViewAction + { + @Override + public ModelAndView getView(MemForm form, BindException errors) + { + Set objectsToIgnore = MemTracker.getInstance().beforeReport(); + + boolean gc = form.isGc(); + boolean cc = form.isClearCaches(); + + if (getUser().hasRootAdminPermission() && (gc || cc)) + { + // If both are requested then try to determine and record cache memory usage + if (gc && cc) + { + // gc once to get an accurate free memory read + long before = gc(); + clearCaches(); + // gc again now that we cleared caches + long cacheMemoryUsed = before - gc(); + + // Difference could be < 0 if JVM or other threads have performed gc, in which case we can't guesstimate cache memory usage + String cacheMemUsed = cacheMemoryUsed > 0 ? FileUtils.byteCountToDisplaySize(cacheMemoryUsed) : "Unknown"; + LOG.info("Estimate of cache memory used: " + cacheMemUsed); + lastCacheMemUsed = cacheMemUsed; + } + else if (cc) + { + clearCaches(); + } + else + { + gc(); + } + + LOG.info("Cache clearing and garbage collecting complete"); + } + + return new JspView<>("/org/labkey/core/admin/memTracker.jsp", new MemBean(getViewContext().getRequest(), objectsToIgnore)); + } + + /** @return estimated current memory usage, post-garbage collection */ + private long gc() + { + LOG.info("Garbage collecting"); + System.gc(); + // This is more reliable than relying on just free memory size, as the VM can grow/shrink the heap at will + return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + } + + private void clearCaches() + { + LOG.info("Clearing Introspector caches"); + Introspector.flushCaches(); + LOG.info("Purging all caches"); + CacheManager.clearAllKnownCaches(); + LOG.info("Purging SearchService queues"); + SearchService.get().purgeQueues(); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("memTracker"); + addAdminNavTrail(root, "Memory usage -- " + DateUtil.formatDateTime(getContainer()), this.getClass()); + } + } + + public static class MemForm + { + private enum Params {clearCaches, debugName, gc} + + private boolean _clearCaches = false; + private boolean _gc = false; + private String _debugName; + + public boolean isClearCaches() + { + return _clearCaches; + } + + @SuppressWarnings("unused") + public void setClearCaches(boolean clearCaches) + { + _clearCaches = clearCaches; + } + + public boolean isGc() + { + return _gc; + } + + @SuppressWarnings("unused") + public void setGc(boolean gc) + { + _gc = gc; + } + + public String getDebugName() + { + return _debugName; + } + + @SuppressWarnings("unused") + public void setDebugName(String debugName) + { + _debugName = debugName; + } + } + + public static class MemBean + { + public final List> memoryUsages = new ArrayList<>(); + public final List> systemProperties = new ArrayList<>(); + public final List references; + public final List graphNames = new ArrayList<>(); + public final List activeThreads = new LinkedList<>(); + + public boolean assertsEnabled = false; + + private MemBean(HttpServletRequest request, Set objectsToIgnore) + { + MemTracker memTracker = MemTracker.getInstance(); + List all = memTracker.getReferences(); + long threadId = Thread.currentThread().getId(); + + // Attempt to detect other threads running labkey code -- mem tracker page will warn if any are found + for (Thread thread : new ThreadsBean().threads) + { + if (thread.getId() == threadId) + continue; + + Thread.State state = thread.getState(); + + if (state == Thread.State.RUNNABLE || state == Thread.State.BLOCKED) + { + boolean labkeyThread = false; + + if (memTracker.shouldDisplay(thread)) + { + for (StackTraceElement element : thread.getStackTrace()) + { + String className = element.getClassName(); + + if (className.startsWith("org.labkey") || className.startsWith("org.fhcrc")) + { + labkeyThread = true; + break; + } + } + } + + if (labkeyThread) + { + String threadInfo = thread.getName(); + TransactionFilter.RequestTracker uri = TransactionFilter.getRequestSummary(thread); + if (null != uri) + threadInfo += "; processing URL " + uri; + activeThreads.add(threadInfo); + } + } + } + + // ignore recently allocated + long start = ViewServlet.getRequestStartTime(request) - 2000; + references = new ArrayList<>(all.size()); + + for (HeldReference r : all) + { + if (r.getThreadId() == threadId && r.getAllocationTime() >= start) + continue; + + if (objectsToIgnore.contains(r.getReference())) + continue; + + references.add(r); + } + + // memory: + graphNames.add("Heap"); + graphNames.add("Non Heap"); + + MemoryMXBean membean = ManagementFactory.getMemoryMXBean(); + if (membean != null) + { + memoryUsages.add(Tuple3.of(true, HEAP_MEMORY_KEY, getUsage(membean.getHeapMemoryUsage()))); + } + + List pools = ManagementFactory.getMemoryPoolMXBeans(); + for (MemoryPoolMXBean pool : pools) + { + if (pool.getType() == MemoryType.HEAP) + { + memoryUsages.add(Tuple3.of(false, pool.getName() + " " + pool.getType(), getUsage(pool))); + graphNames.add(pool.getName()); + } + } + + if (membean != null) + { + memoryUsages.add(Tuple3.of(true, "Total Non-heap Memory", getUsage(membean.getNonHeapMemoryUsage()))); + } + + for (MemoryPoolMXBean pool : pools) + { + if (pool.getType() == MemoryType.NON_HEAP) + { + memoryUsages.add(Tuple3.of(false, pool.getName() + " " + pool.getType(), getUsage(pool))); + graphNames.add(pool.getName()); + } + } + + for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) + { + memoryUsages.add(Tuple3.of(true, "Buffer pool " + pool.getName(), new MemoryUsageSummary(pool))); + graphNames.add(pool.getName()); + } + + DecimalFormat commaf0 = new DecimalFormat("#,##0"); + + + // class loader: + ClassLoadingMXBean classbean = ManagementFactory.getClassLoadingMXBean(); + if (classbean != null) + { + systemProperties.add(new Pair<>("Loaded Class Count", commaf0.format(classbean.getLoadedClassCount()))); + systemProperties.add(new Pair<>("Unloaded Class Count", commaf0.format(classbean.getUnloadedClassCount()))); + systemProperties.add(new Pair<>("Total Loaded Class Count", commaf0.format(classbean.getTotalLoadedClassCount()))); + } + + // runtime: + RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean(); + if (runtimeBean != null) + { + systemProperties.add(new Pair<>("VM Start Time", DateUtil.formatIsoDateShortTime(new Date(runtimeBean.getStartTime())))); + long upTime = runtimeBean.getUptime(); // round to sec + upTime = upTime - (upTime % 1000); + systemProperties.add(new Pair<>("VM Uptime", DateUtil.formatDuration(upTime))); + systemProperties.add(new Pair<>("VM Version", runtimeBean.getVmVersion())); + systemProperties.add(new Pair<>("VM Classpath", runtimeBean.getClassPath())); + } + + // threads: + ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + if (threadBean != null) + { + systemProperties.add(new Pair<>("Thread Count", threadBean.getThreadCount())); + systemProperties.add(new Pair<>("Peak Thread Count", threadBean.getPeakThreadCount())); + long[] deadlockedThreads = threadBean.findMonitorDeadlockedThreads(); + systemProperties.add(new Pair<>("Deadlocked Thread Count", deadlockedThreads != null ? deadlockedThreads.length : 0)); + } + + // threads: + List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); + for (GarbageCollectorMXBean gcBean : gcBeans) + { + systemProperties.add(new Pair<>(gcBean.getName() + " GC count", gcBean.getCollectionCount())); + systemProperties.add(new Pair<>(gcBean.getName() + " GC time", DateUtil.formatDuration(gcBean.getCollectionTime()))); + } + + String cacheMem = lastCacheMemUsed; + + if (null != cacheMem) + systemProperties.add(new Pair<>("Most Recent Estimated Cache Memory Usage", cacheMem)); + + OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); + if (osBean != null) + { + systemProperties.add(new Pair<>("CPU count", osBean.getAvailableProcessors())); + + DecimalFormat f3 = new DecimalFormat("0.000"); + + if (osBean instanceof com.sun.management.OperatingSystemMXBean sunOsBean) + { + systemProperties.add(new Pair<>("Total OS memory", FileUtils.byteCountToDisplaySize(sunOsBean.getTotalMemorySize()))); + systemProperties.add(new Pair<>("Free OS memory", FileUtils.byteCountToDisplaySize(sunOsBean.getFreeMemorySize()))); + systemProperties.add(new Pair<>("OS CPU load", f3.format(sunOsBean.getCpuLoad()))); + systemProperties.add(new Pair<>("JVM CPU load", f3.format(sunOsBean.getProcessCpuLoad()))); + } + } + + //noinspection ConstantConditions + assert assertsEnabled = true; + } + } + + private static MemoryUsageSummary getUsage(MemoryPoolMXBean pool) + { + try + { + return getUsage(pool.getUsage()); + } + catch (IllegalArgumentException x) + { + // sometimes we get usage>committed exception with older versions of JRockit + return null; + } + } + + public static class MemoryUsageSummary + { + + public final long _init; + public final long _used; + public final long _committed; + public final long _max; + + public MemoryUsageSummary(MemoryUsage usage) + { + _init = usage.getInit(); + _used = usage.getUsed(); + _committed = usage.getCommitted(); + _max = usage.getMax(); + } + + public MemoryUsageSummary(BufferPoolMXBean pool) + { + _init = -1; + _used = pool.getMemoryUsed(); + _committed = _used; + _max = pool.getTotalCapacity(); + } + } + + private static MemoryUsageSummary getUsage(MemoryUsage usage) + { + if (null == usage) + return null; + + try + { + return new MemoryUsageSummary(usage); + } + catch (IllegalArgumentException x) + { + // sometime we get usage>committed exception with older verions of JRockit + return null; + } + } + + public static class ChartForm + { + private String _type; + + public String getType() + { + return _type; + } + + public void setType(String type) + { + _type = type; + } + } + + private static class MemoryCategory implements Comparable + { + private final String _type; + private final double _mb; + + public MemoryCategory(String type, double mb) + { + _type = type; + _mb = mb; + } + + @Override + public int compareTo(@NotNull MemoryCategory o) + { + return Double.compare(getMb(), o.getMb()); + } + + public String getType() + { + return _type; + } + + public double getMb() + { + return _mb; + } + } + + @AdminConsoleAction + public static class MemoryChartAction extends ExportAction + { + @Override + public void export(ChartForm form, HttpServletResponse response, BindException errors) throws Exception + { + MemoryUsage usage = null; + boolean showLegend = false; + String title = form.getType(); + if ("Heap".equals(form.getType())) + { + usage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + showLegend = true; + } + else if ("Non Heap".equals(form.getType())) + usage = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage(); + else + { + List pools = ManagementFactory.getMemoryPoolMXBeans(); + for (Iterator it = pools.iterator(); it.hasNext() && usage == null;) + { + MemoryPoolMXBean pool = it.next(); + if (form.getType().equals(pool.getName())) + usage = pool.getUsage(); + } + } + + Pair divisor = null; + + List types = new ArrayList<>(4); + + if (usage == null) + { + boolean found = false; + for (Iterator it = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class).iterator(); it.hasNext() && !found;) + { + BufferPoolMXBean pool = it.next(); + if (form.getType().equals(pool.getName())) + { + long total = pool.getTotalCapacity(); + long used = pool.getMemoryUsed(); + + divisor = getDivisor(total); + + title = "Buffer pool " + title; + + if (total > 0 || used > 0) + { + types.add(new MemoryCategory("Used", used / divisor.first)); + types.add(new MemoryCategory("Max", total / divisor.first)); + } + found = true; + } + } + if (!found) + { + throw new NotFoundException(); + } + } + else + { + if (usage.getInit() > 0 || usage.getUsed() > 0 || usage.getCommitted() > 0 || usage.getMax() > 0) + { + divisor = getDivisor(Math.max(usage.getInit(), Math.max(usage.getUsed(), Math.max(usage.getCommitted(), usage.getMax())))); + + types.add(new MemoryCategory("Init", (double) usage.getInit() / divisor.first)); + types.add(new MemoryCategory("Used", (double) usage.getUsed() / divisor.first)); + types.add(new MemoryCategory("Committed", (double) usage.getCommitted() / divisor.first)); + types.add(new MemoryCategory("Max", (double) usage.getMax() / divisor.first)); + } + } + + if (divisor != null) + { + title += " (" + divisor.second + ")"; + } + + DefaultCategoryDataset dataset = new DefaultCategoryDataset(); + + Collections.sort(types); + + for (int i = 0; i < types.size(); i++) + { + double mbPastPrevious = i > 0 ? types.get(i).getMb() - types.get(i - 1).getMb() : types.get(i).getMb(); + dataset.addValue(mbPastPrevious, types.get(i).getType(), ""); + } + + JFreeChart chart = ChartFactory.createStackedBarChart(title, null, null, dataset, PlotOrientation.HORIZONTAL, showLegend, false, false); + chart.getTitle().setFont(new Font("SansSerif", Font.BOLD, 14)); + response.setContentType("image/png"); + + ChartUtilities.writeChartAsPNG(response.getOutputStream(), chart, showLegend ? 800 : 398, showLegend ? 100 : 70); + } + + private Pair getDivisor(long l) + { + if (l > 4096L * 1024L * 1024L) + { + return Pair.of(1024L * 1024L * 1024L, "GB"); + } + if (l > 4096L * 1024L) + { + return Pair.of(1024L * 1024L, "MB"); + } + if (l > 4096L) + { + return Pair.of(1024L, "KB"); + } + + return Pair.of(1L, "bytes"); + + } + } + + public static class MemoryStressForm + { + private int _threads = 3; + private int _arraySize = 20_000; + private int _arrayCount = 10_000; + private float _percentChurn = 0.50f; + private int _delay = 20; + private int _iterations = 500; + + public int getThreads() + { + return _threads; + } + + public void setThreads(int threads) + { + _threads = threads; + } + + public int getArraySize() + { + return _arraySize; + } + + public void setArraySize(int arraySize) + { + _arraySize = arraySize; + } + + public int getArrayCount() + { + return _arrayCount; + } + + public void setArrayCount(int arrayCount) + { + _arrayCount = arrayCount; + } + + public float getPercentChurn() + { + return _percentChurn; + } + + public void setPercentChurn(float percentChurn) + { + _percentChurn = percentChurn; + } + + public int getDelay() + { + return _delay; + } + + public void setDelay(int delay) + { + _delay = delay; + } + + public int getIterations() + { + return _iterations; + } + + public void setIterations(int iterations) + { + _iterations = iterations; + } + } + + @RequiresSiteAdmin + public class MemoryStressTestAction extends FormViewAction + { + @Override + public void validateCommand(MemoryStressForm target, Errors errors) + { + + } + + @Override + public ModelAndView getView(MemoryStressForm memoryStressForm, boolean reshow, BindException errors) throws Exception + { + return new HtmlView( + DOM.LK.FORM(at(method, "POST"), + DOM.LK.ERRORS(errors.getBindingResult()), + DOM.BR(), DOM.BR(), + "This utility action will do a lot of memory allocation to test the memory configuration of the host.", + DOM.BR(), DOM.BR(), + "It spins up threads, all of which allocate a specified number byte arrays of specified length.", + DOM.BR(), + "The threads sleep for the delay period, and then replace the specified percent of arrays with new ones.", + DOM.BR(), + "They continue for the specified number of allocations.", + DOM.BR(), + "The memory actively held is approximately (threads * array count * array length).", + DOM.BR(), + "The memory turnover is based on the churn percentage, array length, delay, and iterations.", + DOM.BR(), DOM.BR(), + DOM.TABLE( + DOM.TR(DOM.TD("Thread count"), DOM.TD(DOM.INPUT(at(name, "threads", value, memoryStressForm._threads)))), + DOM.TR(DOM.TD("Byte array count"), DOM.TD(DOM.INPUT(at(name, "arrayCount", value, memoryStressForm._arrayCount)))), + DOM.TR(DOM.TD("Byte array size"), DOM.TD(DOM.INPUT(at(name, "arraySize", value, memoryStressForm._arraySize)))), + DOM.TR(DOM.TD("Iterations"), DOM.TD(DOM.INPUT(at(name, "iterations", value, memoryStressForm._iterations)))), + DOM.TR(DOM.TD("Delay between iterations (ms)"), DOM.TD(DOM.INPUT(at(name, "delay", value, memoryStressForm._delay)))), + DOM.TR(DOM.TD("Percent churn per iteration (0.0 - 1.0)"), DOM.TD(DOM.INPUT(at(name, "percentChurn", value, memoryStressForm._percentChurn)))) + ), + new ButtonBuilder("Perform stress test").submit(true).build()) + ); + } + + @Override + public boolean handlePost(MemoryStressForm memoryStressForm, BindException errors) throws Exception + { + List threads = new ArrayList<>(); + for (int i = 0; i < memoryStressForm._threads; i++) + { + Thread t = new Thread(() -> + { + Random r = new Random(); + byte[][] arrays = new byte[memoryStressForm._arrayCount][]; + // Initialize the arrays + for (int a = 0; a < arrays.length; a++) + { + arrays[a] = new byte[memoryStressForm._arraySize]; + } + + for (int iter = 0; iter < memoryStressForm._iterations; iter++) + { + try + { + Thread.sleep(memoryStressForm._delay); + } + catch (InterruptedException ignored) {} + + // Swap the contents based on our desired percent churn + for (int a = 0; a < arrays.length; a++) + { + if (r.nextFloat() <= memoryStressForm._percentChurn) + { + arrays[a] = new byte[memoryStressForm._arraySize]; + } + } + } + }); + t.setUncaughtExceptionHandler((t2, e) -> { + LOG.error("Stress test exception", e); + errors.reject(null, "Stress test exception: " + e); + }); + t.start(); + threads.add(t); + } + + for (Thread thread : threads) + { + thread.join(); + } + + return !errors.hasErrors(); + } + + @Override + public URLHelper getSuccessURL(MemoryStressForm memoryStressForm) + { + return new ActionURL(MemTrackerAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Memory Usage", MemTrackerAction.class); + root.addChild("Memory Stress Test"); + } + } + + public static ActionURL getModuleStatusURL(URLHelper returnUrl) + { + ActionURL url = new ActionURL(ModuleStatusAction.class, ContainerManager.getRoot()); + if (returnUrl != null) + url.addReturnUrl(returnUrl); + return url; + } + + public static class ModuleStatusBean + { + public String verb; + public String verbing; + public ActionURL nextURL; + } + + @RequiresPermission(TroubleshooterPermission.class) + @AllowedDuringUpgrade + @IgnoresAllocationTracking + public static class ModuleStatusAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ReturnUrlForm form, BindException errors) + { + ModuleLoader loader = ModuleLoader.getInstance(); + VBox vbox = new VBox(); + ModuleStatusBean bean = new ModuleStatusBean(); + + if (loader.isNewInstall()) + bean.nextURL = new ActionURL(NewInstallSiteSettingsAction.class, ContainerManager.getRoot()); + else if (form.getReturnUrl() != null) + { + try + { + bean.nextURL = form.getReturnActionURL(); + } + catch (URLException x) + { + // might not be an ActionURL e.g. /labkey/_webdav/home + } + } + if (null == bean.nextURL) + bean.nextURL = new ActionURL(InstallCompleteAction.class, ContainerManager.getRoot()); + + if (loader.isNewInstall()) + bean.verb = "Install"; + else if (loader.isUpgradeRequired() || loader.isUpgradeInProgress()) + bean.verb = "Upgrade"; + else + bean.verb = "Start"; + + if (loader.isNewInstall()) + bean.verbing = "Installing"; + else if (loader.isUpgradeRequired() || loader.isUpgradeInProgress()) + bean.verbing = "Upgrading"; + else + bean.verbing = "Starting"; + + JspView statusView = new JspView<>("/org/labkey/core/admin/moduleStatus.jsp", bean, errors); + vbox.addView(statusView); + + getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); + + getPageConfig().setTemplate(Template.Wizard); + getPageConfig().setTitle(bean.verb + " Modules"); + setHelpTopic(ModuleLoader.getInstance().isNewInstall() ? "config" : "upgrade"); + + return vbox; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + public static class NewInstallSiteSettingsForm extends FileSettingsForm + { + private String _notificationEmail; + private String _siteName; + + public String getNotificationEmail() + { + return _notificationEmail; + } + + public void setNotificationEmail(String notificationEmail) + { + _notificationEmail = notificationEmail; + } + + public String getSiteName() + { + return _siteName; + } + + public void setSiteName(String siteName) + { + _siteName = siteName; + } + } + + @RequiresSiteAdmin + public static class NewInstallSiteSettingsAction extends AbstractFileSiteSettingsAction + { + public NewInstallSiteSettingsAction() + { + super(NewInstallSiteSettingsForm.class); + } + + @Override + public void validateCommand(NewInstallSiteSettingsForm form, Errors errors) + { + super.validateCommand(form, errors); + + if (isBlank(form.getNotificationEmail())) + { + errors.reject(SpringActionController.ERROR_MSG, "Notification email address may not be blank."); + } + try + { + ValidEmail email = new ValidEmail(form.getNotificationEmail()); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + } + + @Override + public boolean handlePost(NewInstallSiteSettingsForm form, BindException errors) throws Exception + { + boolean success = super.handlePost(form, errors); + if (success) + { + WriteableLookAndFeelProperties lafProps = LookAndFeelProperties.getWriteableInstance(ContainerManager.getRoot()); + try + { + lafProps.setSystemEmailAddress(new ValidEmail(form.getNotificationEmail())); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + return false; + } + lafProps.setSystemShortName(form.getSiteName()); + lafProps.save(); + + // Send an immediate report now that they've set up their account and defaults, and then every 24 hours after. + UsageReportingLevel.reportNow(); + + return true; + } + return false; + } + + @Override + public ModelAndView getView(NewInstallSiteSettingsForm form, boolean reshow, BindException errors) + { + if (!reshow) + { + File root = _svc.getSiteDefaultRoot(); + + if (root.exists()) + form.setRootPath(FileUtil.getAbsoluteCaseSensitiveFile(root).getAbsolutePath()); + + LookAndFeelProperties props = LookAndFeelProperties.getInstance(ContainerManager.getRoot()); + form.setSiteName(props.getShortName()); + form.setNotificationEmail(props.getSystemEmailAddress()); + } + + JspView view = new JspView<>("/org/labkey/core/admin/newInstallSiteSettings.jsp", form, errors); + + getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); + getPageConfig().setTitle("Set Defaults"); + getPageConfig().setTemplate(Template.Wizard); + + return view; + } + + @Override + public URLHelper getSuccessURL(NewInstallSiteSettingsForm form) + { + return new ActionURL(InstallCompleteAction.class, ContainerManager.getRoot()); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresSiteAdmin + public static class InstallCompleteAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + JspView view = new JspView<>("/org/labkey/core/admin/installComplete.jsp"); + + getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); + getPageConfig().setTitle("Complete"); + getPageConfig().setTemplate(Template.Wizard); + + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + public static List getInstallUpgradeWizardSteps() + { + List navTrail = new ArrayList<>(); + if (ModuleLoader.getInstance().isNewInstall()) + { + navTrail.add(new NavTree("Account Setup")); + navTrail.add(new NavTree("Install Modules")); + navTrail.add(new NavTree("Set Defaults")); + } + else if (ModuleLoader.getInstance().isUpgradeRequired() || ModuleLoader.getInstance().isUpgradeInProgress()) + { + navTrail.add(new NavTree("Upgrade Modules")); + } + else + { + navTrail.add(new NavTree("Start Modules")); + } + navTrail.add(new NavTree("Complete")); + return navTrail; + } + + @RequiresPermission(AdminOperationsPermission.class) + public class DbCheckerAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/checkDatabase.jsp", new DataCheckForm()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Database Check Tools", this.getClass()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public class DoCheckAction extends SimpleViewAction + { + @Override + public ModelAndView getView(DataCheckForm form, BindException errors) + { + try (var ignore=SpringActionController.ignoreSqlUpdates()) + { + ActionURL currentUrl = getViewContext().cloneActionURL(); + String fixRequested = currentUrl.getParameter("_fix"); + HtmlStringBuilder contentBuilder = HtmlStringBuilder.of(HtmlString.unsafe("
    ")); + + if (null != fixRequested) + { + HtmlString sqlCheck = HtmlString.EMPTY_STRING; + if (fixRequested.equalsIgnoreCase("container")) + sqlCheck = DbSchema.checkAllContainerCols(getUser(), true); + else if (fixRequested.equalsIgnoreCase("descriptor")) + sqlCheck = OntologyManager.doProjectColumnCheck(true); + contentBuilder.append(sqlCheck); + } + else + { + LOG.info("Starting database check"); // Debugging test timeout + LOG.info("Checking container column references"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Checking Container Column References..."); + HtmlString strTemp = DbSchema.checkAllContainerCols(getUser(), false); + if (!strTemp.isEmpty()) + { + contentBuilder.append(strTemp); + currentUrl = getViewContext().cloneActionURL(); + currentUrl.addParameter("_fix", "container"); + contentBuilder.unsafeAppend("

        ") + .append(" click ") + .append(LinkBuilder.simpleLink("here", currentUrl)) + .append(" to attempt recovery."); + } + + LOG.info("Checking PropertyDescriptor and DomainDescriptor consistency"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Checking PropertyDescriptor and DomainDescriptor consistency..."); + strTemp = OntologyManager.doProjectColumnCheck(false); + if (!strTemp.isEmpty()) + { + contentBuilder.append(strTemp); + currentUrl = getViewContext().cloneActionURL(); + currentUrl.addParameter("_fix", "descriptor"); + contentBuilder.unsafeAppend("

        ") + .append(" click ") + .append(LinkBuilder.simpleLink("here", currentUrl)) + .append(" to attempt recovery."); + } + + LOG.info("Checking Schema consistency with tableXML"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Checking Schema consistency with tableXML.") + .unsafeAppend("

    "); + Set schemas = DbSchema.getAllSchemasToTest(); + + for (DbSchema schema : schemas) + { + SiteValidationResultList schemaResult = TableXmlUtils.compareXmlToMetaData(schema, form.getFull(), false, true); + List results = schemaResult.getResults(null); + if (results.isEmpty()) + { + contentBuilder.unsafeAppend("") + .append(schema.getDisplayName()) + .append(": OK") + .unsafeAppend("
    "); + } + else + { + contentBuilder.unsafeAppend("") + .append(schema.getDisplayName()) + .unsafeAppend(""); + for (var r : results) + { + HtmlString item = r.getMessage().isEmpty() ? NBSP : r.getMessage(); + contentBuilder.unsafeAppend("
  • ") + .append(item) + .unsafeAppend("
    • \n"); + } + contentBuilder.unsafeAppend(""); + } + } + + LOG.info("Checking consistency of provisioned storage"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Checking Consistency of Provisioned Storage...\n"); + StorageProvisioner.ProvisioningReport pr = StorageProvisioner.get().getProvisioningReport(); + contentBuilder.append(String.format("%d domains use Storage Provisioner", pr.getProvisionedDomains().size())); + for (StorageProvisioner.ProvisioningReport.DomainReport dr : pr.getProvisionedDomains()) + { + for (String error : dr.getErrors()) + { + contentBuilder.unsafeAppend("
    ") + .append(error) + .unsafeAppend("
    "); + } + } + for (String error : pr.getGlobalErrors()) + { + contentBuilder.unsafeAppend("
    ") + .append(error) + .unsafeAppend("
    "); + } + + LOG.info("Database check complete"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Database Consistency checker complete"); + } + + contentBuilder.unsafeAppend("
    "); + + return new HtmlView(contentBuilder); + } + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Database Tools", this.getClass()); + } + } + + public static class DataCheckForm + { + private String _dbSchema = ""; + private boolean _full = false; + + public List modules = ModuleLoader.getInstance().getModules(); + public DataCheckForm(){} + + public List getModules() { return modules; } + public String getDbSchema() { return _dbSchema; } + @SuppressWarnings("unused") + public void setDbSchema(String dbSchema){ _dbSchema = dbSchema; } + public boolean getFull() { return _full; } + public void setFull(boolean full) { _full = full; } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class GetSchemaXmlDocAction extends ExportAction + { + @Override + public void export(DataCheckForm form, HttpServletResponse response, BindException errors) throws Exception + { + String fullyQualifiedSchemaName = form.getDbSchema(); + if (null == fullyQualifiedSchemaName || fullyQualifiedSchemaName.isEmpty()) + { + throw new NotFoundException("Must specify dbSchema parameter"); + } + + boolean bFull = form.getFull(); + + Pair scopeAndSchemaName = DbSchema.getDbScopeAndSchemaName(fullyQualifiedSchemaName); + TablesDocument tdoc = TableXmlUtils.createXmlDocumentFromDatabaseMetaData(scopeAndSchemaName.first, scopeAndSchemaName.second, bFull); + StringWriter sw = new StringWriter(); + + XmlOptions xOpt = new XmlOptions(); + xOpt.setSavePrettyPrint(); + xOpt.setUseDefaultNamespace(); + + tdoc.save(sw, xOpt); + + sw.flush(); + PageFlowUtil.streamFileBytes(response, fullyQualifiedSchemaName + ".xml", sw.toString().getBytes(StringUtilsLabKey.DEFAULT_CHARSET), true); + } + } + + @RequiresPermission(AdminPermission.class) + public static class FolderInformationAction extends FolderManagementViewAction + { + @Override + protected HtmlView getTabView() + { + Container c = getContainer(); + User currentUser = getUser(); + + User createdBy = UserManager.getUser(c.getCreatedBy()); + Map propValueMap = new LinkedHashMap<>(); + propValueMap.put("Path", c.getPath()); + propValueMap.put("Name", c.getName()); + propValueMap.put("Displayed Title", c.getTitle()); + propValueMap.put("EntityId", c.getId()); + propValueMap.put("RowId", c.getRowId()); + propValueMap.put("Created", DateUtil.formatDateTime(c, c.getCreated())); + propValueMap.put("Created By", (createdBy != null ? createdBy.getDisplayName(currentUser) : "<" + c.getCreatedBy() + ">")); + propValueMap.put("Folder Type", c.getFolderType().getName()); + propValueMap.put("Description", c.getDescription()); + + return new HtmlView(PageFlowUtil.getDataRegionHtmlForPropertyObjects(propValueMap)); + } + } + + public static class MissingValuesForm + { + private boolean _inheritMvIndicators; + private String[] _mvIndicators; + private String[] _mvLabels; + + public boolean isInheritMvIndicators() + { + return _inheritMvIndicators; + } + + public void setInheritMvIndicators(boolean inheritMvIndicators) + { + _inheritMvIndicators = inheritMvIndicators; + } + + public String[] getMvIndicators() + { + return _mvIndicators; + } + + public void setMvIndicators(String[] mvIndicators) + { + _mvIndicators = mvIndicators; + } + + public String[] getMvLabels() + { + return _mvLabels; + } + + public void setMvLabels(String[] mvLabels) + { + _mvLabels = mvLabels; + } + } + + @RequiresPermission(AdminPermission.class) + public static class MissingValuesAction extends FolderManagementViewPostAction + { + @Override + protected JspView getTabView(MissingValuesForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/mvIndicators.jsp", form, errors); + } + + @Override + public void validateCommand(MissingValuesForm form, Errors errors) + { + } + + @Override + public boolean handlePost(MissingValuesForm form, BindException errors) + { + if (form.isInheritMvIndicators()) + { + MvUtil.inheritMvIndicators(getContainer()); + return true; + } + else + { + // Javascript should have enforced any constraints + MvUtil.assignMvIndicators(getContainer(), form.getMvIndicators(), form.getMvLabels()); + return true; + } + } + } + + @SuppressWarnings("unused") + public static class RConfigForm + { + private Integer _reportEngine; + private Integer _pipelineEngine; + private boolean _overrideDefault; + + public Integer getReportEngine() + { + return _reportEngine; + } + + public void setReportEngine(Integer reportEngine) + { + _reportEngine = reportEngine; + } + + public Integer getPipelineEngine() + { + return _pipelineEngine; + } + + public void setPipelineEngine(Integer pipelineEngine) + { + _pipelineEngine = pipelineEngine; + } + + public boolean getOverrideDefault() + { + return _overrideDefault; + } + + public void setOverrideDefault(String overrideDefault) + { + _overrideDefault = "override".equals(overrideDefault); + } + } + + @RequiresPermission(AdminPermission.class) + public static class RConfigurationAction extends FolderManagementViewPostAction + { + @Override + protected HttpView getTabView(RConfigForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/rConfiguration.jsp", form, errors); + } + + @Override + public void validateCommand(RConfigForm form, Errors errors) + { + if (form.getOverrideDefault()) + { + if (form.getReportEngine() == null) + errors.reject(ERROR_MSG, "Please select a valid report engine configuration"); + if (form.getPipelineEngine() == null) + errors.reject(ERROR_MSG, "Please select a valid pipeline engine configuration"); + } + } + + @Override + public URLHelper getSuccessURL(RConfigForm rConfigForm) + { + return getContainer().getStartURL(getUser()); + } + + @Override + public boolean handlePost(RConfigForm rConfigForm, BindException errors) throws Exception + { + LabKeyScriptEngineManager mgr = LabKeyScriptEngineManager.get(); + if (null != mgr) + { + try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) + { + if (rConfigForm.getOverrideDefault()) + { + ExternalScriptEngineDefinition reportEngine = mgr.getEngineDefinition(rConfigForm.getReportEngine(), ExternalScriptEngineDefinition.Type.R); + ExternalScriptEngineDefinition pipelineEngine = mgr.getEngineDefinition(rConfigForm.getPipelineEngine(), ExternalScriptEngineDefinition.Type.R); + + if (reportEngine != null) + mgr.setEngineScope(getContainer(), reportEngine, LabKeyScriptEngineManager.EngineContext.report); + if (pipelineEngine != null) + mgr.setEngineScope(getContainer(), pipelineEngine, LabKeyScriptEngineManager.EngineContext.pipeline); + } + else + { + // need to clear the current scope (if any) + ExternalScriptEngineDefinition reportEngine = mgr.getScopedEngine(getContainer(), "r", LabKeyScriptEngineManager.EngineContext.report, false); + ExternalScriptEngineDefinition pipelineEngine = mgr.getScopedEngine(getContainer(), "r", LabKeyScriptEngineManager.EngineContext.pipeline, false); + + if (reportEngine != null) + mgr.removeEngineScope(getContainer(), reportEngine, LabKeyScriptEngineManager.EngineContext.report); + if (pipelineEngine != null) + mgr.removeEngineScope(getContainer(), pipelineEngine, LabKeyScriptEngineManager.EngineContext.pipeline); + } + transaction.commit(); + } + return true; + } + return false; + } + } + + @SuppressWarnings("unused") + public static class ExportFolderForm + { + private String[] _types; + private int _location; + private String _format = "new"; // As of 14.3, this is the only supported format. But leave in place for the future. + private String _exportType; + private boolean _includeSubfolders; + private PHI _exportPhiLevel; // Input: max level when viewing form + private boolean _shiftDates; + private boolean _alternateIds; + private boolean _maskClinic; + + public String[] getTypes() + { + return _types; + } + + public void setTypes(String[] types) + { + _types = types; + } + + public int getLocation() + { + return _location; + } + + public void setLocation(int location) + { + _location = location; + } + + public String getFormat() + { + return _format; + } + + public void setFormat(String format) + { + _format = format; + } + + public ExportType getExportType() + { + if ("study".equals(_exportType)) + return ExportType.STUDY; + else + return ExportType.ALL; + } + + public void setExportType(String exportType) + { + _exportType = exportType; + } + + public boolean isIncludeSubfolders() + { + return _includeSubfolders; + } + + public void setIncludeSubfolders(boolean includeSubfolders) + { + _includeSubfolders = includeSubfolders; + } + + public PHI getExportPhiLevel() + { + return null != _exportPhiLevel ? _exportPhiLevel : PHI.NotPHI; + } + + public void setExportPhiLevel(PHI exportPhiLevel) + { + _exportPhiLevel = exportPhiLevel; + } + + public boolean isShiftDates() + { + return _shiftDates; + } + + public void setShiftDates(boolean shiftDates) + { + _shiftDates = shiftDates; + } + + public boolean isAlternateIds() + { + return _alternateIds; + } + + public void setAlternateIds(boolean alternateIds) + { + _alternateIds = alternateIds; + } + + public boolean isMaskClinic() + { + return _maskClinic; + } + + public void setMaskClinic(boolean maskClinic) + { + _maskClinic = maskClinic; + } + } + + public enum ExportOption + { + PipelineRootAsFiles("file root as multiple files") + { + @Override + public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception + { + PipeRoot root = PipelineService.get().findPipelineRoot(container); + if (root == null || !root.isValid()) + { + throw new NotFoundException("No valid pipeline root found"); + } + else if (root.isCloudRoot()) + { + errors.reject(ERROR_MSG, "Cannot export as individual files when root is in the cloud"); + } + else + { + File exportDir = root.resolvePath(PipelineService.EXPORT_DIR); + try + { + writer.write(container, ctx, new FileSystemFile(exportDir)); + } + catch (ContainerException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + return urlProvider(PipelineUrls.class).urlBrowse(container); + } + return null; + } + }, + + PipelineRootAsZip("file root as a single zip file") + { + @Override + public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception + { + PipeRoot root = PipelineService.get().findPipelineRoot(container); + if (root == null || !root.isValid()) + { + throw new NotFoundException("No valid pipeline root found"); + } + Path exportDir = root.resolveToNioPath(PipelineService.EXPORT_DIR); + FileUtil.createDirectories(exportDir); + exportFolderToFile(exportDir, container, writer, ctx, errors); + return urlProvider(PipelineUrls.class).urlBrowse(container); + } + }, + DownloadAsZip("browser download as a zip file") + { + @Override + public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception + { + try + { + // Export to a temporary file first so exceptions are displayed by the standard error page, Issue #44152 + // Same pattern as ExportListArchiveAction + Path tempDir = FileUtil.getTempDirectory().toPath(); + Path tempZipFile = exportFolderToFile(tempDir, container, writer, ctx, errors); + + // No exceptions, so stream the resulting zip file to the browser and delete it + try (OutputStream os = ZipFile.getOutputStream(response, tempZipFile.getFileName().toString())) + { + Files.copy(tempZipFile, os); + } + finally + { + Files.delete(tempZipFile); + } + } + catch (ContainerException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + return null; + } + }; + + private final String _description; + + ExportOption(String description) + { + _description = description; + } + + public String getDescription() + { + return _description; + } + + public abstract ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception; + + Path exportFolderToFile(Path exportDir, Container container, FolderWriterImpl writer, FolderExportContext ctx, BindException errors) throws Exception + { + String filename = FileUtil.makeFileNameWithTimestamp(container.getName(), "folder.zip"); + + try (ZipFile zip = new ZipFile(exportDir, filename)) + { + writer.write(container, ctx, zip); + } + catch (Container.ContainerException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + + return exportDir.resolve(filename); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ExportFolderAction extends FolderManagementViewPostAction + { + private ActionURL _successURL = null; + + @Override + public ModelAndView getView(ExportFolderForm exportFolderForm, boolean reshow, BindException errors) throws Exception + { + // In export-to-browser do nothing (leave the export page in place). We just exported to the response, so + // rendering a view would throw. + return reshow && !errors.hasErrors() ? null : super.getView(exportFolderForm, reshow, errors); + } + + @Override + protected HttpView getTabView(ExportFolderForm form, boolean reshow, BindException errors) + { + form.setExportType(PageFlowUtil.filter(getViewContext().getActionURL().getParameter("exportType"))); + + ComplianceFolderSettings settings = ComplianceService.get().getFolderSettings(getContainer(), User.getAdminServiceUser()); + PhiColumnBehavior columnBehavior = null==settings ? PhiColumnBehavior.show : settings.getPhiColumnBehavior(); + PHI maxAllowedPhiForExport = PhiColumnBehavior.show == columnBehavior ? PHI.Restricted : ComplianceService.get().getMaxAllowedPhi(getContainer(), getUser()); + form.setExportPhiLevel(maxAllowedPhiForExport); + + return new JspView<>("/org/labkey/core/admin/exportFolder.jsp", form, errors); + } + + @Override + public void validateCommand(ExportFolderForm form, Errors errors) + { + } + + @Override + public boolean handlePost(ExportFolderForm form, BindException errors) throws Exception + { + Container container = getContainer(); + if (container.isRoot()) + { + throw new NotFoundException(); + } + + ExportOption exportOption = null; + if (form.getLocation() >= 0 && form.getLocation() < ExportOption.values().length) + { + exportOption = ExportOption.values()[form.getLocation()]; + } + if (exportOption == null) + { + throw new NotFoundException("Invalid export location: " + form.getLocation()); + } + ContainerManager.checkContainerValidity(container); + + FolderWriterImpl writer = new FolderWriterImpl(); + FolderExportContext ctx = new FolderExportContext(getUser(), container, PageFlowUtil.set(form.getTypes()), + form.getFormat(), form.isIncludeSubfolders(), form.getExportPhiLevel(), form.isShiftDates(), + form.isAlternateIds(), form.isMaskClinic(), new StaticLoggerGetter(FolderWriterImpl.LOG)); + + AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, container, "Folder export initiated to " + exportOption.getDescription() + " " + (form.isIncludeSubfolders() ? "including" : "excluding") + " subfolders."); + AuditLogService.get().addEvent(getUser(), event); + + _successURL = exportOption.initiateExport(container, errors, writer, ctx, getViewContext().getResponse()); + + return !errors.hasErrors(); + } + + @Override + public URLHelper getSuccessURL(ExportFolderForm exportFolderForm) + { + return _successURL; + } + } + + public static class ImportFolderForm + { + private boolean _createSharedDatasets; + private boolean _validateQueries; + private boolean _failForUndefinedVisits; + private boolean _advancedImportOptions; + private String _sourceTemplateFolder; + private String _sourceTemplateFolderId; + private String _origin; + + public boolean isCreateSharedDatasets() + { + return _createSharedDatasets; + } + + public void setCreateSharedDatasets(boolean createSharedDatasets) + { + _createSharedDatasets = createSharedDatasets; + } + + public boolean isValidateQueries() + { + return _validateQueries; + } + + public boolean isFailForUndefinedVisits() + { + return _failForUndefinedVisits; + } + + public void setFailForUndefinedVisits(boolean failForUndefinedVisits) + { + _failForUndefinedVisits = failForUndefinedVisits; + } + + public void setValidateQueries(boolean validateQueries) + { + _validateQueries = validateQueries; + } + + public boolean isAdvancedImportOptions() + { + return _advancedImportOptions; + } + + public void setAdvancedImportOptions(boolean advancedImportOptions) + { + _advancedImportOptions = advancedImportOptions; + } + + public String getSourceTemplateFolder() + { + return _sourceTemplateFolder; + } + + @SuppressWarnings("unused") + public void setSourceTemplateFolder(String sourceTemplateFolder) + { + _sourceTemplateFolder = sourceTemplateFolder; + } + + public String getSourceTemplateFolderId() + { + return _sourceTemplateFolderId; + } + + @SuppressWarnings("unused") + public void setSourceTemplateFolderId(String sourceTemplateFolderId) + { + _sourceTemplateFolderId = sourceTemplateFolderId; + } + + public String getOrigin() + { + return _origin; + } + + public void setOrigin(String origin) + { + _origin = origin; + } + + public Container getSourceTemplateFolderContainer() + { + if (null == getSourceTemplateFolderId()) + return null; + return ContainerManager.getForId(getSourceTemplateFolderId().replace(',', ' ').trim()); + } + } + + @RequiresPermission(AdminPermission.class) + public class ImportFolderAction extends FolderManagementViewPostAction + { + private ActionURL _successURL; + + @Override + protected HttpView getTabView(ImportFolderForm form, boolean reshow, BindException errors) + { + // default the createSharedDatasets and validateQueries to true if this is not a form error reshow + if (!errors.hasErrors()) + { + form.setCreateSharedDatasets(true); + form.setValidateQueries(true); + } + + return new JspView<>("/org/labkey/core/admin/importFolder.jsp", form, errors); + } + + @Override + public void validateCommand(ImportFolderForm form, Errors errors) + { + // don't allow import into the root container + if (getContainer().isRoot()) + { + throw new NotFoundException(); + } + } + + @Override + public boolean handlePost(ImportFolderForm form, BindException errors) throws Exception + { + ViewContext context = getViewContext(); + ActionURL url = context.getActionURL(); + User user = getUser(); + Container container = getContainer(); + PipeRoot pipelineRoot; + Path pipelineUnzipDir; // Should be local & writable + PipelineUrls pipelineUrlProvider; + + if (form.getOrigin() == null) + { + form.setOrigin("Folder"); + } + + // make sure we have a pipeline url provider to use for the success URL redirect + pipelineUrlProvider = urlProvider(PipelineUrls.class); + if (pipelineUrlProvider == null) + { + errors.reject("folderImport", "Pipeline url provider does not exist."); + return false; + } + + // make sure that the pipeline root is valid for this container + pipelineRoot = PipelineService.get().findPipelineRoot(container); + if (!PipelineService.get().hasValidPipelineRoot(container) || pipelineRoot == null) + { + errors.reject("folderImport", "Pipeline root not set or does not exist on disk."); + return false; + } + + // make sure we are able to delete any existing unzip dir in the pipeline root + try + { + pipelineUnzipDir = pipelineRoot.deleteImportDirectory(null); + } + catch (DirectoryNotDeletedException e) + { + errors.reject("studyImport", "Import failed: Could not delete the directory \"" + PipelineService.UNZIP_DIR + "\""); + return false; + } + + FolderImportConfig fiConfig; + if (!StringUtils.isEmpty(form.getSourceTemplateFolder())) + { + fiConfig = getFolderImportConfigFromTemplateFolder(form, pipelineUnzipDir, errors); + } + else + { + fiConfig = getFolderFromZipArchive(pipelineUnzipDir, errors); + if (fiConfig == null || errors.hasErrors()) + { + return false; + } + } + + // get the folder.xml file from the unzipped import archive + Path archiveXml = pipelineUnzipDir.resolve("folder.xml"); + if (!Files.exists(archiveXml)) + { + errors.reject("folderImport", "This archive doesn't contain a folder.xml file."); + return false; + } + + ImportOptions options = new ImportOptions(getContainer().getId(), user.getUserId()); + options.setSkipQueryValidation(!form.isValidateQueries()); + options.setCreateSharedDatasets(form.isCreateSharedDatasets()); + options.setFailForUndefinedVisits(form.isFailForUndefinedVisits()); + options.setAdvancedImportOptions(form.isAdvancedImportOptions()); + options.setActivity(ComplianceService.get().getCurrentActivity(getViewContext())); + + // if the option is selected to show the advanced import options, redirect to there + if (form.isAdvancedImportOptions()) + { + // archiveFile is the zip of the source template folder located in the current container's unzip dir + _successURL = pipelineUrlProvider.urlStartFolderImport(getContainer(), fiConfig.archiveFile, options, fiConfig.fromTemplateSourceFolder); + return true; + } + + // finally, create the study or folder import pipeline job + _successURL = pipelineUrlProvider.urlBegin(container); + PipelineService.get().runFolderImportJob(container, user, url, archiveXml, fiConfig.originalFileName, pipelineRoot, options); + + return !errors.hasErrors(); + } + + private @Nullable FolderImportConfig getFolderFromZipArchive(Path pipelineUnzipDir, BindException errors) + { + // user chose to import from a zip file + Map map = getFileMap(); + + // make sure we have a single file selected for import + if (map.size() != 1) + { + errors.reject("folderImport", "You must select a valid zip archive (folder or study)."); + return null; + } + + // make sure the file is not empty and that it has a .zip extension + MultipartFile zipFile = map.values().iterator().next(); + if (0 == zipFile.getSize() || isBlank(zipFile.getOriginalFilename()) || !zipFile.getOriginalFilename().toLowerCase().endsWith(".zip")) + { + errors.reject("folderImport", "You must select a valid zip archive (folder or study)."); + return null; + } + + // copy and unzip the uploaded import archive zip file to the pipeline unzip dir + try + { + Path pipelineUnzipFile = pipelineUnzipDir.resolve(zipFile.getOriginalFilename()); + FileUtil.createDirectories(pipelineUnzipFile.getParent()); // Non-pipeline import sometimes fails here on Windows (shrug) + FileUtil.createFile(pipelineUnzipFile); + try (OutputStream os = Files.newOutputStream(pipelineUnzipFile)) + { + FileUtil.copyData(zipFile.getInputStream(), os); + } + ZipUtil.unzipToDirectory(pipelineUnzipFile, pipelineUnzipDir); + + return new FolderImportConfig( + false, + zipFile.getOriginalFilename(), + pipelineUnzipFile, + pipelineUnzipFile + ); + } + catch (FileNotFoundException e) + { + LOG.debug("Failed to import '" + zipFile.getOriginalFilename() + "'.", e); + errors.reject("folderImport", "File not found."); + return null; + } + catch (IOException e) + { + LOG.debug("Failed to import '" + zipFile.getOriginalFilename() + "'.", e); + errors.reject("folderImport", "Unable to unzip folder archive."); + return null; + } + } + + private FolderImportConfig getFolderImportConfigFromTemplateFolder(final ImportFolderForm form, final Path pipelineUnzipDir, final BindException errors) throws Exception + { + // user choose to import from a template source folder + Container sourceContainer = form.getSourceTemplateFolderContainer(); + + // In order to support the Advanced import options to import into multiple target folders we need to zip + // the source template folder so that the zip file can be passed to the pipeline processes. + FolderExportContext ctx = new FolderExportContext(getUser(), sourceContainer, + getRegisteredFolderWritersForImplicitExport(sourceContainer), "new", false, + PHI.NotPHI, false, false, false, new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); + FolderWriterImpl writer = new FolderWriterImpl(); + String zipFileName = FileUtil.makeFileNameWithTimestamp(sourceContainer.getName(), "folder.zip"); + try (ZipFile zip = new ZipFile(pipelineUnzipDir, zipFileName)) + { + writer.write(sourceContainer, ctx, zip); + } + catch (Container.ContainerException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + Path implicitZipFile = pipelineUnzipDir.resolve(zipFileName); + + // To support the simple import option unzip the zip file to the pipeline unzip dir of the current container + ZipUtil.unzipToDirectory(implicitZipFile, pipelineUnzipDir); + + return new FolderImportConfig( + StringUtils.isNotEmpty(form.getSourceTemplateFolderId()), + implicitZipFile.getFileName().toString(), + implicitZipFile, + null + ); + } + + private static class FolderImportConfig { + Path pipelineUnzipFile; + String originalFileName; + Path archiveFile; + boolean fromTemplateSourceFolder; + + public FolderImportConfig(boolean fromTemplateSourceFolder, String originalFileName, Path archiveFile, @Nullable Path pipelineUnzipFile) + { + this.originalFileName = originalFileName; + this.archiveFile = archiveFile; + this.fromTemplateSourceFolder = fromTemplateSourceFolder; + this.pipelineUnzipFile = pipelineUnzipFile; + } + } + + @Override + public URLHelper getSuccessURL(ImportFolderForm importFolderForm) + { + return _successURL; + } + } + + private Set getRegisteredFolderWritersForImplicitExport(Container sourceContainer) + { + // this method is very similar to CoreController.GetRegisteredFolderWritersAction.execute() method, but instead of + // of building up a map of Writer object names to display in the UI, we are instead adding them to the list of Writers + // to apply during the implicit export. + Set registeredFolderWriters = new HashSet<>(); + FolderSerializationRegistry registry = FolderSerializationRegistry.get(); + if (null == registry) + { + throw new RuntimeException(); + } + Collection registeredWriters = registry.getRegisteredFolderWriters(); + for (FolderWriter writer : registeredWriters) + { + String dataType = writer.getDataType(); + boolean excludeForDataspace = sourceContainer.isDataspace() && "Study".equals(dataType); + boolean excludeForTemplate = !writer.includeWithTemplate(); + + if (dataType != null && writer.show(sourceContainer) && !excludeForDataspace && !excludeForTemplate) + { + registeredFolderWriters.add(dataType); + + // for each Writer also determine if there are related children Writers, if so include them also + Collection> childWriters = writer.getChildren(true, true); + if (!childWriters.isEmpty()) + { + for (org.labkey.api.writer.Writer child : childWriters) + { + dataType = child.getDataType(); + if (dataType != null) + registeredFolderWriters.add(dataType); + } + } + } + } + return registeredFolderWriters; + } + + public static class FolderSettingsForm + { + private String _defaultDateFormat; + private boolean _defaultDateFormatInherited; + private String _defaultDateTimeFormat; + private boolean _defaultDateTimeFormatInherited; + private String _defaultTimeFormat; + private boolean _defaultTimeFormatInherited; + private String _defaultNumberFormat; + private boolean _defaultNumberFormatInherited; + private boolean _restrictedColumnsEnabled; + private boolean _restrictedColumnsEnabledInherited; + + public String getDefaultDateFormat() + { + return _defaultDateFormat; + } + + @SuppressWarnings("unused") + public void setDefaultDateFormat(String defaultDateFormat) + { + _defaultDateFormat = defaultDateFormat; + } + + public boolean isDefaultDateFormatInherited() + { + return _defaultDateFormatInherited; + } + + @SuppressWarnings("unused") + public void setDefaultDateFormatInherited(boolean defaultDateFormatInherited) + { + _defaultDateFormatInherited = defaultDateFormatInherited; + } + + public String getDefaultDateTimeFormat() + { + return _defaultDateTimeFormat; + } + + @SuppressWarnings("unused") + public void setDefaultDateTimeFormat(String defaultDateTimeFormat) + { + _defaultDateTimeFormat = defaultDateTimeFormat; + } + + public boolean isDefaultDateTimeFormatInherited() + { + return _defaultDateTimeFormatInherited; + } + + @SuppressWarnings("unused") + public void setDefaultDateTimeFormatInherited(boolean defaultDateTimeFormatInherited) + { + _defaultDateTimeFormatInherited = defaultDateTimeFormatInherited; + } + + public String getDefaultTimeFormat() + { + return _defaultTimeFormat; + } + + @SuppressWarnings("UnusedDeclaration") + public void setDefaultTimeFormat(String defaultTimeFormat) + { + _defaultTimeFormat = defaultTimeFormat; + } + + public boolean isDefaultTimeFormatInherited() + { + return _defaultTimeFormatInherited; + } + + @SuppressWarnings("unused") + public void setDefaultTimeFormatInherited(boolean defaultTimeFormatInherited) + { + _defaultTimeFormatInherited = defaultTimeFormatInherited; + } + + public String getDefaultNumberFormat() + { + return _defaultNumberFormat; + } + + @SuppressWarnings("unused") + public void setDefaultNumberFormat(String defaultNumberFormat) + { + _defaultNumberFormat = defaultNumberFormat; + } + + public boolean isDefaultNumberFormatInherited() + { + return _defaultNumberFormatInherited; + } + + @SuppressWarnings("unused") + public void setDefaultNumberFormatInherited(boolean defaultNumberFormatInherited) + { + _defaultNumberFormatInherited = defaultNumberFormatInherited; + } + + public boolean areRestrictedColumnsEnabled() + { + return _restrictedColumnsEnabled; + } + + @SuppressWarnings("unused") + public void setRestrictedColumnsEnabled(boolean restrictedColumnsEnabled) + { + _restrictedColumnsEnabled = restrictedColumnsEnabled; + } + + public boolean isRestrictedColumnsEnabledInherited() + { + return _restrictedColumnsEnabledInherited; + } + + @SuppressWarnings("unused") + public void setRestrictedColumnsEnabledInherited(boolean restrictedColumnsEnabledInherited) + { + _restrictedColumnsEnabledInherited = restrictedColumnsEnabledInherited; + } + } + + @RequiresPermission(AdminPermission.class) + public static class FolderSettingsAction extends FolderManagementViewPostAction + { + @Override + protected LookAndFeelView getTabView(FolderSettingsForm form, boolean reshow, BindException errors) + { + return new LookAndFeelView(errors); + } + + @Override + public void validateCommand(FolderSettingsForm form, Errors errors) + { + } + + @Override + public boolean handlePost(FolderSettingsForm form, BindException errors) + { + return saveFolderSettings(getContainer(), getUser(), LookAndFeelProperties.getWriteableFolderInstance(getContainer()), form, errors); + } + } + + // Validate and populate the folder settings; save & log all changes + private static boolean saveFolderSettings(Container c, User user, WriteableFolderLookAndFeelProperties props, FolderSettingsForm form, BindException errors) + { + validateAndSaveFormat(form.getDefaultDateFormat(), form.isDefaultDateFormatInherited(), props::clearDefaultDateFormat, props::setDefaultDateFormat, errors, "date display format"); + validateAndSaveFormat(form.getDefaultDateTimeFormat(), form.isDefaultDateTimeFormatInherited(), props::clearDefaultDateTimeFormat, props::setDefaultDateTimeFormat, errors, "date-time display format"); + validateAndSaveFormat(form.getDefaultTimeFormat(), form.isDefaultTimeFormatInherited(), props::clearDefaultTimeFormat, props::setDefaultTimeFormat, errors, "time display format"); + validateAndSaveFormat(form.getDefaultNumberFormat(), form.isDefaultNumberFormatInherited(), props::clearDefaultNumberFormat, props::setDefaultNumberFormat, errors, "number display format"); + + setProperty(form.isRestrictedColumnsEnabledInherited(), props::clearRestrictedColumnsEnabled, () -> props.setRestrictedColumnsEnabled(form.areRestrictedColumnsEnabled())); + + if (!errors.hasErrors()) + { + props.save(); + + //write an audit log event + props.writeAuditLogEvent(c, user); + } + + return !errors.hasErrors(); + } + + private interface FormatSaver + { + void save(String format) throws IllegalArgumentException; + } + + private static void validateAndSaveFormat(String format, boolean inherited, Runnable clearer, FormatSaver saver, BindException errors, String what) + { + String defaultFormat = StringUtils.trimToNull(format); + if (inherited) + { + clearer.run(); + } + else + { + try + { + saver.save(defaultFormat); + } + catch (IllegalArgumentException e) + { + errors.reject(ERROR_MSG, "Invalid " + what + ": " + e.getMessage()); + } + } + } + + @RequiresPermission(AdminPermission.class) + public static class ModulePropertiesAction extends FolderManagementViewAction + { + @Override + protected JspView getTabView() + { + return new JspView<>("/org/labkey/core/project/modulePropertiesAdmin.jsp"); + } + } + + @SuppressWarnings("unused") + public static class FolderTypeForm + { + private String[] _activeModules = new String[ModuleLoader.getInstance().getModules().size()]; + private String _defaultModule; + private String _folderType; + private boolean _wizard; + + public String[] getActiveModules() + { + return _activeModules; + } + + public void setActiveModules(String[] activeModules) + { + _activeModules = activeModules; + } + + public String getDefaultModule() + { + return _defaultModule; + } + + public void setDefaultModule(String defaultModule) + { + _defaultModule = defaultModule; + } + + public String getFolderType() + { + return _folderType; + } + + public void setFolderType(String folderType) + { + _folderType = folderType; + } + + public boolean isWizard() + { + return _wizard; + } + + public void setWizard(boolean wizard) + { + _wizard = wizard; + } + } + + @RequiresPermission(AdminPermission.class) + @IgnoresTermsOfUse // At the moment, compliance configuration is very sensitive to active modules, so allow those adjustments + public static class FolderTypeAction extends FolderManagementViewPostAction + { + private ActionURL _successURL = null; + + @Override + protected JspView getTabView(FolderTypeForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/folderType.jsp", form, errors); + } + + @Override + public void validateCommand(FolderTypeForm form, Errors errors) + { + boolean fEmpty = true; + for (String module : form._activeModules) + { + if (module != null) + { + fEmpty = false; + break; + } + } + if (fEmpty && "None".equals(form.getFolderType())) + { + errors.reject(SpringActionController.ERROR_MSG, "Error: Please select at least one module to display."); + } + } + + @Override + public boolean handlePost(FolderTypeForm form, BindException errors) + { + Container container = getContainer(); + if (container.isRoot()) + { + throw new NotFoundException(); + } + + String[] modules = form.getActiveModules(); + + if (modules.length == 0) + { + errors.reject(null, "At least one module must be selected"); + return false; + } + + Set activeModules = new HashSet<>(); + for (String moduleName : modules) + { + Module module = ModuleLoader.getInstance().getModule(moduleName); + if (module != null) + activeModules.add(module); + } + + if (null == StringUtils.trimToNull(form.getFolderType()) || FolderType.NONE.getName().equals(form.getFolderType())) + { + container.setFolderType(FolderType.NONE, getUser(), errors, activeModules); + Module defaultModule = ModuleLoader.getInstance().getModule(form.getDefaultModule()); + container.setDefaultModule(defaultModule); + } + else + { + FolderType folderType = FolderTypeManager.get().getFolderType(form.getFolderType()); + if (container.isContainerTab() && folderType.hasContainerTabs()) + errors.reject(null, "You cannot set a tab folder to a folder type that also has tab folders"); + else + container.setFolderType(folderType, getUser(), errors, activeModules); + } + if (errors.hasErrors()) + return false; + + if (form.isWizard()) + { + _successURL = urlProvider(SecurityUrls.class).getContainerURL(container); + _successURL.addParameter("wizard", Boolean.TRUE.toString()); + } + else + _successURL = container.getFolderType().getStartURL(container, getUser()); + + return true; + } + + @Override + public URLHelper getSuccessURL(FolderTypeForm folderTypeForm) + { + return _successURL; + } + } + + @SuppressWarnings("unused") + public static class FileRootsForm extends SetupForm implements FileManagementForm + { + private String _folderRootPath; + private String _fileRootOption; + private String _cloudRootName; + private boolean _isFolderSetup; + private boolean _fileRootChanged; + private boolean _enabledCloudStoresChanged; + private String _migrateFilesOption; + + // cloud settings + private String[] _enabledCloudStore; + //file management + @Override + public String getFolderRootPath() + { + return _folderRootPath; + } + + @Override + public void setFolderRootPath(String folderRootPath) + { + _folderRootPath = folderRootPath; + } + + @Override + public String getFileRootOption() + { + return _fileRootOption; + } + + @Override + public void setFileRootOption(String fileRootOption) + { + _fileRootOption = fileRootOption; + } + + @Override + public String[] getEnabledCloudStore() + { + return _enabledCloudStore; + } + + @Override + public void setEnabledCloudStore(String[] enabledCloudStore) + { + _enabledCloudStore = enabledCloudStore; + } + + @Override + public boolean isDisableFileSharing() + { + return FileRootProp.disable.name().equals(getFileRootOption()); + } + + @Override + public boolean hasSiteDefaultRoot() + { + return FileRootProp.siteDefault.name().equals(getFileRootOption()); + } + + @Override + public boolean isCloudFileRoot() + { + return FileRootProp.cloudRoot.name().equals(getFileRootOption()); + } + + @Override + @Nullable + public String getCloudRootName() + { + return _cloudRootName; + } + + @Override + public void setCloudRootName(String cloudRootName) + { + _cloudRootName = cloudRootName; + } + + @Override + public boolean isFolderSetup() + { + return _isFolderSetup; + } + + public void setFolderSetup(boolean folderSetup) + { + _isFolderSetup = folderSetup; + } + + public boolean isFileRootChanged() + { + return _fileRootChanged; + } + + @Override + public void setFileRootChanged(boolean changed) + { + _fileRootChanged = changed; + } + + public boolean isEnabledCloudStoresChanged() + { + return _enabledCloudStoresChanged; + } + + @Override + public void setEnabledCloudStoresChanged(boolean enabledCloudStoresChanged) + { + _enabledCloudStoresChanged = enabledCloudStoresChanged; + } + + @Override + public String getMigrateFilesOption() + { + return _migrateFilesOption; + } + + @Override + public void setMigrateFilesOption(String migrateFilesOption) + { + _migrateFilesOption = migrateFilesOption; + } + } + + @RequiresPermission(AdminPermission.class) + public class FileRootsStandAloneAction extends FormViewAction + { + @Override + public ModelAndView getView(FileRootsForm form, boolean reShow, BindException errors) + { + JspView view = getFileRootsView(form, errors, getReshow()); + view.setFrame(WebPartView.FrameType.NONE); + + getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(getContainer(), getContainer().getParent())); + getPageConfig().setTemplate(PageConfig.Template.Wizard); + getPageConfig().setTitle("Change File Root"); + return view; + } + + @Override + public void validateCommand(FileRootsForm form, Errors errors) + { + validateCloudFileRoot(form, getContainer(), errors); + } + + @Override + public boolean handlePost(FileRootsForm form, BindException errors) throws Exception + { + return handleFileRootsPost(form, errors); + } + + @Override + public ActionURL getSuccessURL(FileRootsForm form) + { + ActionURL url = new ActionURL(FileRootsStandAloneAction.class, getContainer()) + .addParameter("folderSetup", true) + .addReturnUrl(getViewContext().getActionURL().getReturnUrl()); + + if (form.isFileRootChanged()) + url.addParameter("rootSet", form.getMigrateFilesOption()); + if (form.isEnabledCloudStoresChanged()) + url.addParameter("cloudChanged", true); + return url; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + /** + * This standalone file root management action can be used on folder types that do not support + * the normal 'Manage Folder' UI. Not currently linked in the UI, but available for direct URL + * navigation when a workbook needs it. + */ + @RequiresPermission(AdminPermission.class) + public class ManageFileRootAction extends FormViewAction + { + @Override + public ModelAndView getView(FileRootsForm form, boolean reShow, BindException errors) + { + JspView view = getFileRootsView(form, errors, getReshow()); + getPageConfig().setTitle("Manage File Root"); + return view; + } + + @Override + public void validateCommand(FileRootsForm form, Errors errors) + { + validateCloudFileRoot(form, getContainer(), errors); + } + + @Override + public boolean handlePost(FileRootsForm form, BindException errors) throws Exception + { + return handleFileRootsPost(form, errors); + } + + @Override + public ActionURL getSuccessURL(FileRootsForm form) + { + ActionURL url = getContainer().getStartURL(getUser()); + + if (getViewContext().getActionURL().getReturnUrl() != null) + { + url.addReturnUrl(getViewContext().getActionURL().getReturnUrl()); + } + + return url; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresPermission(AdminPermission.class) + public class FileRootsAction extends FolderManagementViewPostAction + { + @Override + protected HttpView getTabView(FileRootsForm form, boolean reshow, BindException errors) + { + return getFileRootsView(form, errors, getReshow()); + } + + @Override + public void validateCommand(FileRootsForm form, Errors errors) + { + validateCloudFileRoot(form, getContainer(), errors); + } + + @Override + public boolean handlePost(FileRootsForm form, BindException errors) throws Exception + { + return handleFileRootsPost(form, errors); + } + + @Override + public ActionURL getSuccessURL(FileRootsForm form) + { + ActionURL url = new AdminController.AdminUrlsImpl().getFileRootsURL(getContainer()); + + if (form.isFileRootChanged()) + url.addParameter("rootSet", form.getMigrateFilesOption()); + if (form.isEnabledCloudStoresChanged()) + url.addParameter("cloudChanged", true); + return url; + } + } + + private JspView getFileRootsView(FileRootsForm form, BindException errors, boolean reshow) + { + JspView view = new JspView<>("/org/labkey/core/admin/view/filesProjectSettings.jsp", form, errors); + String title = "Configure File Root"; + if (CloudStoreService.get() != null) + title += " And Enable Cloud Stores"; + view.setTitle(title); + view.setFrame(WebPartView.FrameType.DIV); + try + { + if (!reshow) + setFormAndConfirmMessage(getViewContext(), form); + } + catch (IllegalArgumentException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + + return view; + } + + private boolean handleFileRootsPost(FileRootsForm form, BindException errors) throws Exception + { + if (form.isPipelineRootForm()) + { + return PipelineService.get().savePipelineSetup(getViewContext(), form, errors); + } + else + { + setFileRootFromForm(getViewContext(), form, errors); + setEnabledCloudStores(getViewContext(), form, errors); + return !errors.hasErrors(); + } + } + + public static void validateCloudFileRoot(FileManagementForm form, Container container, Errors errors) + { + FileContentService service = FileContentService.get(); + if (null != service) + { + boolean isOrDefaultsToCloudRoot = form.isCloudFileRoot(); + String cloudRootName = form.getCloudRootName(); + if (!isOrDefaultsToCloudRoot && form.hasSiteDefaultRoot()) + { + Path defaultRootPath = service.getDefaultRootPath(container, false); + cloudRootName = service.getDefaultRootInfo(container).getCloudName(); + isOrDefaultsToCloudRoot = (null != defaultRootPath && FileUtil.hasCloudScheme(defaultRootPath)); + } + + if (isOrDefaultsToCloudRoot && null != cloudRootName) + { + if (null != form.getEnabledCloudStore()) + { + for (String storeName : form.getEnabledCloudStore()) + { + if (Strings.CI.equals(cloudRootName, storeName)) + return; + } + } + // Didn't find cloud root in enabled list + errors.reject(ERROR_MSG, "Cannot disable cloud store used as File Root."); + } + } + } + + public static void setFileRootFromForm(ViewContext ctx, FileManagementForm form, BindException errors) + { + boolean changed = false; + boolean shouldCopyMove = false; + FileContentService service = FileContentService.get(); + if (null != service) + { + // If we need to copy/move files based on the FileRoot change, we need to check children that use the default and move them, too. + // And we need to capture the source roots for each of those, because changing this parent file root changes the child source roots. + MigrateFilesOption migrateFilesOption = null != form.getMigrateFilesOption() ? + MigrateFilesOption.valueOf(form.getMigrateFilesOption()) : + MigrateFilesOption.leave; + List> sourceInfos = + ((MigrateFilesOption.leave.equals(migrateFilesOption) && !form.isFolderSetup()) || form.isDisableFileSharing()) ? + Collections.emptyList() : + getCopySourceInfo(service, ctx.getContainer()); + + if (form.isDisableFileSharing()) + { + if (!service.isFileRootDisabled(ctx.getContainer())) + { + service.disableFileRoot(ctx.getContainer()); + changed = true; + } + } + else if (form.hasSiteDefaultRoot()) + { + if (service.isFileRootDisabled(ctx.getContainer()) || !service.isUseDefaultRoot(ctx.getContainer())) + { + service.setIsUseDefaultRoot(ctx.getContainer(), true); + changed = true; + shouldCopyMove = true; + } + } + else if (form.isCloudFileRoot()) + { + throwIfUnauthorizedFileRootChange(ctx, service, form); + String cloudRootName = form.getCloudRootName(); + if (null != cloudRootName && + (!service.isCloudRoot(ctx.getContainer()) || + !cloudRootName.equalsIgnoreCase(service.getCloudRootName(ctx.getContainer())))) + { + service.setIsUseDefaultRoot(ctx.getContainer(), false); + service.setCloudRoot(ctx.getContainer(), cloudRootName); + try + { + PipelineService.get().setPipelineRoot(ctx.getUser(), ctx.getContainer(), PipelineService.PRIMARY_ROOT, false); + if (form.isFolderSetup() && !sourceInfos.isEmpty()) + { + // File root was set to cloud storage, remove folder created + Path fromPath = FileUtil.stringToPath(sourceInfos.get(0).first, sourceInfos.get(0).second); // sourceInfos paths should be encoded + if (FileContentService.FILES_LINK.equals(FileUtil.getFileName(fromPath))) + { + try + { + Files.deleteIfExists(fromPath.getParent()); + } + catch (IOException e) + { + LOG.warn("Could not delete directory '" + FileUtil.pathToString(fromPath.getParent()) + "'"); + } + } + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + changed = true; + shouldCopyMove = true; + } + } + else + { + throwIfUnauthorizedFileRootChange(ctx, service, form); + String root = StringUtils.trimToNull(form.getFolderRootPath()); + if (root != null) + { + URI uri = FileUtil.createUri(root, false); // root is unencoded + Path path = FileUtil.getPath(ctx.getContainer(), uri); + if (null == path || !Files.exists(path)) + { + errors.reject(ERROR_MSG, "File root '" + root + "' does not appear to be a valid directory accessible to the server at " + ctx.getRequest().getServerName() + "."); + } + else + { + Path currentFileRootPath = service.getFileRootPath(ctx.getContainer()); + if (null == currentFileRootPath || !root.equalsIgnoreCase(currentFileRootPath.toAbsolutePath().toString())) + { + service.setIsUseDefaultRoot(ctx.getContainer(), false); + service.setFileRootPath(ctx.getContainer(), root); + changed = true; + shouldCopyMove = true; + } + } + } + else + { + service.setFileRootPath(ctx.getContainer(), null); + changed = true; + } + } + + if (!errors.hasErrors()) + { + if (changed && shouldCopyMove && !MigrateFilesOption.leave.equals(migrateFilesOption)) + { + // Make sure we have pipeRoot before starting jobs, even though each subfolder needs to get its own + PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(ctx.getContainer()); + if (null != pipeRoot) + { + try + { + initiateCopyFilesPipelineJobs(ctx, sourceInfos, pipeRoot, migrateFilesOption); + } + catch (PipelineValidationException e) + { + throw new RuntimeValidationException(e); + } + } + else + { + LOG.warn("Change File Root: Can't copy or move files with no pipeline root"); + } + } + + form.setFileRootChanged(changed); + if (changed && null != ctx.getUser()) + { + setFormAndConfirmMessage(ctx.getContainer(), form, true, false, migrateFilesOption.name()); + String comment = (ctx.getContainer().isProject() ? "Project " : "Folder ") + ctx.getContainer().getPath() + ": " + form.getConfirmMessage(); + AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, ctx.getContainer(), comment); + AuditLogService.get().addEvent(ctx.getUser(), event); + } + } + } + } + + private static List> getCopySourceInfo(FileContentService service, Container container) + { + + List> sourceInfo = new ArrayList<>(); + addCopySourceInfo(service, container, sourceInfo, true); + return sourceInfo; + } + + private static void addCopySourceInfo(FileContentService service, Container container, List> sourceInfo, boolean isRoot) + { + if (isRoot || service.isUseDefaultRoot(container)) + { + Path sourceFileRootDir = service.getFileRootPath(container, FileContentService.ContentType.files); + if (null != sourceFileRootDir) + { + String pathStr = FileUtil.pathToString(sourceFileRootDir); + if (null != pathStr) + sourceInfo.add(new Pair<>(container, pathStr)); + else + throw new RuntimeValidationException("Unexpected error converting path to string"); + } + } + for (Container childContainer : container.getChildren()) + addCopySourceInfo(service, childContainer, sourceInfo, false); + } + + private static void initiateCopyFilesPipelineJobs(ViewContext ctx, @NotNull List> sourceInfos, PipeRoot pipeRoot, + MigrateFilesOption migrateFilesOption) throws PipelineValidationException + { + CopyFileRootPipelineJob job = new CopyFileRootPipelineJob(ctx.getContainer(), ctx.getUser(), sourceInfos, pipeRoot, migrateFilesOption); + PipelineService.get().queueJob(job); + } + + private static void throwIfUnauthorizedFileRootChange(ViewContext ctx, FileContentService service, FileManagementForm form) + { + // test permissions. only site admins are able to turn on a custom file root for a folder + // this is only relevant if the folder is either being switched to a custom file root, + // or if the file root is changed. + if (!service.isUseDefaultRoot(ctx.getContainer())) + { + Path fileRootPath = service.getFileRootPath(ctx.getContainer()); + if (null != fileRootPath) + { + String absolutePath = FileUtil.getAbsolutePath(ctx.getContainer(), fileRootPath); + if (Strings.CI.equals(absolutePath, form.getFolderRootPath())) + { + if (!ctx.getUser().hasRootPermission(AdminOperationsPermission.class)) + throw new UnauthorizedException("Only site admins can change file roots"); + } + } + } + } + + public static void setEnabledCloudStores(ViewContext ctx, FileManagementForm form, BindException errors) + { + String[] enabledCloudStores = form.getEnabledCloudStore(); + CloudStoreService cloud = CloudStoreService.get(); + if (cloud != null) + { + Set enabled = Collections.emptySet(); + if (enabledCloudStores != null) + enabled = new HashSet<>(Arrays.asList(enabledCloudStores)); + + try + { + // Check if anything changed + boolean changed = false; + Collection storeNames = cloud.getEnabledCloudStores(ctx.getContainer()); + if (enabled.size() != storeNames.size()) + changed = true; + else + if (!enabled.containsAll(storeNames)) + changed = true; + if (changed) + cloud.setEnabledCloudStores(ctx.getContainer(), enabled); + form.setEnabledCloudStoresChanged(changed); + } + catch (UncheckedExecutionException e) + { + LOG.debug("Failed to configure cloud store(s).", e); + // UncheckedExecutionException with cause org.jclouds.blobstore.ContainerNotFoundException + // is what BlobStore hands us if bucket (S3 container) does not exist + if (null != e.getCause()) + errors.reject(ERROR_MSG, e.getCause().getMessage()); + else + throw e; + } + catch (RuntimeException e) + { + LOG.debug("Failed to configure cloud store(s).", e); + errors.reject(ERROR_MSG, e.getMessage()); + } + } + } + + + public static void setFormAndConfirmMessage(ViewContext ctx, FileManagementForm form) throws IllegalArgumentException + { + String rootSetParam = ctx.getActionURL().getParameter("rootSet"); + boolean fileRootChanged = null != rootSetParam && !"false".equalsIgnoreCase(rootSetParam); + String cloudChangedParam = ctx.getActionURL().getParameter("cloudChanged"); + boolean enabledCloudChanged = "true".equalsIgnoreCase(cloudChangedParam); + setFormAndConfirmMessage(ctx.getContainer(), form, fileRootChanged, enabledCloudChanged, rootSetParam); + } + + public static void setFormAndConfirmMessage(Container container, FileManagementForm form, boolean fileRootChanged, boolean enabledCloudChanged, + String migrateFilesOption) throws IllegalArgumentException + { + FileContentService service = FileContentService.get(); + String confirmMessage = null; + + String migrateFilesMessage = ""; + if (fileRootChanged && !form.isFolderSetup()) + { + if (MigrateFilesOption.leave.name().equals(migrateFilesOption)) + migrateFilesMessage = ". Existing files not copied or moved."; + else if (MigrateFilesOption.copy.name().equals(migrateFilesOption)) + { + migrateFilesMessage = ". Existing files copied."; + form.setMigrateFilesOption(migrateFilesOption); + } + else if (MigrateFilesOption.move.name().equals(migrateFilesOption)) + { + migrateFilesMessage = ". Existing files moved."; + form.setMigrateFilesOption(migrateFilesOption); + } + } + + if (service != null) + { + if (service.isFileRootDisabled(container)) + { + form.setFileRootOption(FileRootProp.disable.name()); + if (fileRootChanged) + confirmMessage = "File sharing has been disabled for this " + container.getContainerNoun(); + } + else if (service.isUseDefaultRoot(container)) + { + form.setFileRootOption(FileRootProp.siteDefault.name()); + Path root = service.getFileRootPath(container); + if (root != null && Files.exists(root) && fileRootChanged) + confirmMessage = "The file root is set to a default of: " + FileUtil.getAbsolutePath(container, root) + migrateFilesMessage; + } + else if (!service.isCloudRoot(container)) + { + Path root = service.getFileRootPath(container); + + form.setFileRootOption(FileRootProp.folderOverride.name()); + if (root != null) + { + String absolutePath = FileUtil.getAbsolutePath(container, root); + form.setFolderRootPath(absolutePath); + if (Files.exists(root)) + { + if (fileRootChanged) + confirmMessage = "The file root is set to: " + absolutePath + migrateFilesMessage; + } + } + } + else + { + form.setFileRootOption(FileRootProp.cloudRoot.name()); + form.setCloudRootName(service.getCloudRootName(container)); + Path root = service.getFileRootPath(container); + if (root != null && fileRootChanged) + { + confirmMessage = "The file root is set to: " + FileUtil.getCloudRootPathString(form.getCloudRootName()) + migrateFilesMessage; + } + } + } + + if (fileRootChanged && confirmMessage != null) + form.setConfirmMessage(confirmMessage); + else if (enabledCloudChanged) + form.setConfirmMessage("The enabled cloud stores changed."); + } + + @RequiresPermission(AdminPermission.class) + public static class ManageFoldersAction extends FolderManagementViewAction + { + @Override + protected HttpView getTabView() + { + return new JspView<>("/org/labkey/core/admin/manageFolders.jsp"); + } + } + + public static class NotificationsForm + { + private String _provider; + + public String getProvider() + { + return _provider; + } + + public void setProvider(String provider) + { + _provider = provider; + } + } + + private static final String DATA_REGION_NAME = "Users"; + + @RequiresPermission(AdminPermission.class) + public static class NotificationsAction extends FolderManagementViewPostAction + { + @Override + protected HttpView getTabView(NotificationsForm form, boolean reshow, BindException errors) + { + final String key = DataRegionSelection.getSelectionKey("core", CoreQuerySchema.USERS_MSG_SETTINGS_TABLE_NAME, null, DATA_REGION_NAME); + DataRegionSelection.clearAll(getViewContext(), key); + + QuerySettings settings = new QuerySettings(getViewContext(), DATA_REGION_NAME, CoreQuerySchema.USERS_MSG_SETTINGS_TABLE_NAME); + settings.setAllowChooseView(true); + settings.getBaseSort().insertSortColumn(FieldKey.fromParts("DisplayName")); + + UserSchema schema = QueryService.get().getUserSchema(getViewContext().getUser(), getViewContext().getContainer(), SchemaKey.fromParts(CoreQuerySchema.NAME)); + QueryView queryView = new QueryView(schema, settings, errors) + { + @Override + public List getDisplayColumns() + { + List columns = new ArrayList<>(); + SecurityPolicy policy = getContainer().getPolicy(); + Set assignmentSet = new HashSet<>(); + + for (RoleAssignment assignment : policy.getAssignments()) + { + Group g = SecurityManager.getGroup(assignment.getUserId()); + if (g != null) + assignmentSet.add(g.getName()); + } + + for (DisplayColumn col : super.getDisplayColumns()) + { + if (col.getName().equalsIgnoreCase("Groups")) + columns.add(new FolderGroupColumn(assignmentSet, col.getColumnInfo())); + else + columns.add(col); + } + return columns; + } + + @Override + protected void populateButtonBar(DataView dataView, ButtonBar bar) + { + try + { + // add the provider configuration menu items to the admin panel button + MenuButton adminButton = new MenuButton("Update user settings"); + adminButton.setRequiresSelection(true); + for (ConfigTypeProvider provider : MessageConfigService.get().getConfigTypes()) + adminButton.addMenuItem("For " + provider.getName().toLowerCase(), "userSettings_"+provider.getName()+"(LABKEY.DataRegions.Users.getSelectionCount())" ); + + bar.add(adminButton); + super.populateButtonBar(dataView, bar); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }; + queryView.setShadeAlternatingRows(true); + queryView.setShowBorders(true); + queryView.setShowDetailsColumn(false); + queryView.setShowRecordSelectors(true); + queryView.setFrame(WebPartView.FrameType.NONE); + queryView.disableContainerFilterSelection(); + queryView.setButtonBarPosition(DataRegion.ButtonBarPosition.TOP); + + VBox defaultsView = new VBox( + HtmlView.unsafe( + "
    Default settings
    " + + "You can change this folder's default settings for email notifications here.") + ); + + PanelConfig config = new PanelConfig(getViewContext().getActionURL().clone(), key); + for (ConfigTypeProvider provider : MessageConfigService.get().getConfigTypes()) + { + defaultsView.addView(new JspView<>("/org/labkey/core/admin/view/notifySettings.jsp", provider.createConfigForm(getViewContext(), config))); + } + + return new VBox( + new JspView<>("/org/labkey/core/admin/view/folderSettingsHeader.jsp", null, errors), + defaultsView, + new VBox( + HtmlView.unsafe( + "
    User settings
    " + + "The list below contains all users with read access to this folder who are able to receive notifications. Each user's current
    " + + "notification setting is visible in the appropriately named column.

    " + + "To bulk edit individual settings: select one or more users, click the 'Update user settings' menu, and select the notification type."), + queryView + ) + ); + } + + @Override + public void validateCommand(NotificationsForm form, Errors errors) + { + ConfigTypeProvider provider = MessageConfigService.get().getConfigType(form.getProvider()); + + if (provider != null) + provider.validateCommand(getViewContext(), errors); + } + + @Override + public boolean handlePost(NotificationsForm form, BindException errors) throws Exception + { + ConfigTypeProvider provider = MessageConfigService.get().getConfigType(form.getProvider()); + + if (provider != null) + { + return provider.handlePost(getViewContext(), errors); + } + errors.reject(SpringActionController.ERROR_MSG, "Unable to find the selected config provider"); + return false; + } + } + + public static class NotifyOptionsForm + { + private String _type; + + public String getType() + { + return _type; + } + + public void setType(String type) + { + _type = type; + } + + public ConfigTypeProvider getProvider() + { + return MessageConfigService.get().getConfigType(getType()); + } + } + + /** + * Action to populate an Ext store with email notification options for admin settings + */ + @RequiresPermission(AdminPermission.class) + public static class GetEmailOptionsAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(NotifyOptionsForm form, BindException errors) + { + ApiSimpleResponse resp = new ApiSimpleResponse(); + + ConfigTypeProvider provider = form.getProvider(); + if (provider != null) + { + List options = new ArrayList<>(); + + // if the list of options is not for the folder default, add an option to use the folder default + if (getViewContext().get("isDefault") == null) + options.add(PageFlowUtil.map("id", -1, "label", "Folder default")); + + for (NotificationOption option : provider.getOptions()) + { + options.add(PageFlowUtil.map("id", option.getEmailOptionId(), "label", option.getEmailOption())); + } + resp.put("success", true); + if (!options.isEmpty()) + resp.put("options", options); + } + else + resp.put("success", false); + + return resp; + } + } + + @RequiresPermission(AdminPermission.class) + public static class SetBulkEmailOptionsAction extends MutatingApiAction + { + @Override + public ApiResponse execute(EmailConfigFormImpl form, BindException errors) + { + ApiSimpleResponse resp = new ApiSimpleResponse(); + ConfigTypeProvider provider = form.getProvider(); + String srcIdentifier = getContainer().getId(); + + Set selections = DataRegionSelection.getSelected(getViewContext(), form.getDataRegionSelectionKey(), true); + + if (!selections.isEmpty() && provider != null) + { + int newOption = form.getIndividualEmailOption(); + + for (String user : selections) + { + User projectUser = UserManager.getUser(Integer.parseInt(user)); + UserPreference pref = provider.getPreference(getContainer(), projectUser, srcIdentifier); + + int currentEmailOption = pref != null ? pref.getEmailOptionId() : -1; + + //has this projectUser's option changed? if so, update + //creating new record in EmailPrefs table if there isn't one, or deleting if set back to folder default + if (currentEmailOption != newOption) + { + provider.savePreference(getUser(), getContainer(), projectUser, newOption, srcIdentifier); + } + } + resp.put("success", true); + } + else + { + resp.put("success", false); + resp.put("message", "There were no users selected"); + } + return resp; + } + } + + /** Renders only the groups that are assigned roles in this container */ + private static class FolderGroupColumn extends DataColumn + { + private final Set _assignmentSet; + + public FolderGroupColumn(Set assignmentSet, ColumnInfo col) + { + super(col); + _assignmentSet = assignmentSet; + } + + @Override + public void renderGridCellContents(RenderContext ctx, HtmlWriter out) + { + String value = (String)ctx.get(getBoundColumn().getDisplayField().getFieldKey()); + + if (value != null) + { + out.write(Arrays.stream(value.split(VALUE_DELIMITER_REGEX)) + .filter(_assignmentSet::contains) + .map(HtmlString::of) + .collect(LabKeyCollectors.joining(HtmlString.unsafe(",
    ")))); + } + } + } + + private static class PanelConfig implements MessageConfigService.PanelInfo + { + private final ActionURL _returnUrl; + private final String _dataRegionSelectionKey; + + public PanelConfig(ActionURL returnUrl, String selectionKey) + { + _returnUrl = returnUrl; + _dataRegionSelectionKey = selectionKey; + } + + @Override + public ActionURL getReturnUrl() + { + return _returnUrl; + } + + @Override + public String getDataRegionSelectionKey() + { + return _dataRegionSelectionKey; + } + } + + public static class ConceptsForm + { + private String _conceptURI; + private String _containerId; + private String _schemaName; + private String _queryName; + + public String getConceptURI() + { + return _conceptURI; + } + + public void setConceptURI(String conceptURI) + { + _conceptURI = conceptURI; + } + + public String getContainerId() + { + return _containerId; + } + + public void setContainerId(String containerId) + { + _containerId = containerId; + } + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public String getQueryName() + { + return _queryName; + } + + public void setQueryName(String queryName) + { + _queryName = queryName; + } + } + + @RequiresPermission(AdminPermission.class) + public static class ConceptsAction extends FolderManagementViewPostAction + { + @Override + protected HttpView getTabView(ConceptsForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/manageConcepts.jsp", form, errors); + } + + @Override + public void validateCommand(ConceptsForm form, Errors errors) + { + // validate that the required input fields are provided + String missingRequired = "", sep = ""; + if (form.getConceptURI() == null) + { + missingRequired += "conceptURI"; + sep = ", "; + } + if (form.getSchemaName() == null) + { + missingRequired += sep + "schemaName"; + sep = ", "; + } + if (form.getQueryName() == null) + missingRequired += sep + "queryName"; + if (!missingRequired.isEmpty()) + errors.reject(SpringActionController.ERROR_MSG, "Missing required field(s): " + missingRequired + "."); + + // validate that, if provided, the containerId matches an existing container + Container postContainer = null; + if (form.getContainerId() != null) + { + postContainer = ContainerManager.getForId(form.getContainerId()); + if (postContainer == null) + errors.reject(SpringActionController.ERROR_MSG, "Container does not exist for containerId provided."); + } + + // validate that the schema and query names provided exist + if (form.getSchemaName() != null && form.getQueryName() != null) + { + Container c = postContainer != null ? postContainer : getContainer(); + UserSchema schema = QueryService.get().getUserSchema(getUser(), c, form.getSchemaName()); + if (schema == null) + errors.reject(SpringActionController.ERROR_MSG, "UserSchema '" + form.getSchemaName() + "' not found."); + else if (schema.getTable(form.getQueryName()) == null) + errors.reject(SpringActionController.ERROR_MSG, "Table '" + form.getSchemaName() + "." + form.getQueryName() + "' not found."); + } + } + + @Override + public boolean handlePost(ConceptsForm form, BindException errors) + { + Lookup lookup = new Lookup(ContainerManager.getForId(form.getContainerId()), form.getSchemaName(), form.getQueryName()); + ConceptURIProperties.setLookup(getContainer(), form.getConceptURI(), lookup); + + return true; + } + } + + @RequiresPermission(AdminPermission.class) + public class FolderAliasesAction extends FormViewAction + { + @Override + public void validateCommand(FolderAliasesForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(FolderAliasesForm form, boolean reshow, BindException errors) + { + return new JspView("/org/labkey/core/admin/folderAliases.jsp"); + } + + @Override + public boolean handlePost(FolderAliasesForm form, BindException errors) + { + List aliases = new ArrayList<>(); + if (form.getAliases() != null) + { + StringTokenizer st = new StringTokenizer(form.getAliases(), "\n\r", false); + while (st.hasMoreTokens()) + { + String alias = st.nextToken().trim(); + if (!alias.startsWith("/")) + { + alias = "/" + alias; + } + while (alias.endsWith("/")) + { + alias = alias.substring(0, alias.lastIndexOf('/')); + } + aliases.add(alias); + } + } + ContainerManager.saveAliasesForContainer(getContainer(), aliases, getUser()); + + return true; + } + + @Override + public ActionURL getSuccessURL(FolderAliasesForm form) + { + return getManageFoldersURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Folder Aliases: " + getContainer().getPath(), this.getClass()); + } + } + + public static class FolderAliasesForm + { + private String _aliases; + + public String getAliases() + { + return _aliases; + } + + @SuppressWarnings("unused") + public void setAliases(String aliases) + { + _aliases = aliases; + } + } + + @RequiresPermission(AdminPermission.class) + public class CustomizeEmailAction extends FormViewAction + { + @Override + public void validateCommand(CustomEmailForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(CustomEmailForm form, boolean reshow, BindException errors) + { + JspView result = new JspView<>("/org/labkey/core/admin/customizeEmail.jsp", form, errors); + result.setTitle("Email Template"); + return result; + } + + @Override + public boolean handlePost(CustomEmailForm form, BindException errors) + { + if (form.getTemplateClass() != null) + { + EmailTemplate template = EmailTemplateService.get().createTemplate(form.getTemplateClass()); + + template.setSubject(form.getEmailSubject()); + template.setSenderName(form.getEmailSender()); + template.setReplyToEmail(form.getEmailReplyTo()); + template.setBody(form.getEmailMessage()); + + String[] errorStrings = new String[1]; + if (template.isValid(errorStrings)) // TODO: Pass in errors collection directly? Should also build a list of all validation errors and display them all. + EmailTemplateService.get().saveEmailTemplate(template, getContainer()); + else + errors.reject(ERROR_MSG, errorStrings[0]); + } + + return !errors.hasErrors(); + } + + @Override + public ActionURL getSuccessURL(CustomEmailForm form) + { + ActionURL result = new ActionURL(CustomizeEmailAction.class, getContainer()); + result.replaceParameter("templateClass", form.getTemplateClass()); + if (form.getReturnActionURL() != null) + { + result.replaceParameter(ActionURL.Param.returnUrl, form.getReturnUrl()); + } + return result; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("customEmail"); + addAdminNavTrail(root, "Customize " + (getContainer().isRoot() ? "Site-Wide" : StringUtils.capitalize(getContainer().getContainerNoun()) + "-Level") + " Email", this.getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class DeleteCustomEmailAction extends FormHandlerAction + { + @Override + public void validateCommand(CustomEmailForm target, Errors errors) + { + } + + @Override + public boolean handlePost(CustomEmailForm form, BindException errors) throws Exception + { + if (form.getTemplateClass() != null) + { + EmailTemplate template = EmailTemplateService.get().createTemplate(form.getTemplateClass()); + template.setSubject(form.getEmailSubject()); + template.setBody(form.getEmailMessage()); + + EmailTemplateService.get().deleteEmailTemplate(template, getContainer()); + } + return true; + } + + @Override + public URLHelper getSuccessURL(CustomEmailForm form) + { + return new AdminUrlsImpl().getCustomizeEmailURL(getContainer(), form.getTemplateClass(), form.getReturnUrlHelper()); + } + } + + @SuppressWarnings("unused") + public static class CustomEmailForm extends ReturnUrlForm + { + private String _templateClass; + private String _emailSubject; + private String _emailSender; + private String _emailReplyTo; + private String _emailMessage; + private String _templateDescription; + + public void setTemplateClass(String name){_templateClass = name;} + public String getTemplateClass(){return _templateClass;} + public void setEmailSubject(String subject){_emailSubject = subject;} + public String getEmailSubject(){return _emailSubject;} + public void setEmailSender(String sender){_emailSender = sender;} + public String getEmailSender(){return _emailSender;} + public void setEmailMessage(String body){_emailMessage = body;} + public String getEmailMessage(){return _emailMessage;} + public String getEmailReplyTo(){return _emailReplyTo;} + public void setEmailReplyTo(String emailReplyTo){_emailReplyTo = emailReplyTo;} + + public String getTemplateDescription() + { + return _templateDescription; + } + + public void setTemplateDescription(String templateDescription) + { + _templateDescription = templateDescription; + } + } + + private ActionURL getManageFoldersURL() + { + return new AdminUrlsImpl().getManageFoldersURL(getContainer()); + } + + public static class ManageFoldersForm extends ReturnUrlForm + { + private String name; + private String title; + private boolean titleSameAsName; + private String folder; + private String target; + private String folderType; + private String defaultModule; + private String[] activeModules; + private boolean hasLoaded = false; + private boolean showAll; + private boolean confirmed = false; + private boolean addAlias = false; + private String templateSourceId; + private String[] templateWriterTypes; + private boolean templateIncludeSubfolders = false; + private String[] targets; + private PHI _exportPhiLevel = PHI.NotPHI; + + public boolean getHasLoaded() + { + return hasLoaded; + } + + public void setHasLoaded(boolean hasLoaded) + { + this.hasLoaded = hasLoaded; + } + + public String[] getActiveModules() + { + return activeModules; + } + + public void setActiveModules(String[] activeModules) + { + this.activeModules = activeModules; + } + + public String getDefaultModule() + { + return defaultModule; + } + + public void setDefaultModule(String defaultModule) + { + this.defaultModule = defaultModule; + } + + public boolean isShowAll() + { + return showAll; + } + + public void setShowAll(boolean showAll) + { + this.showAll = showAll; + } + + public String getFolder() + { + return folder; + } + + public void setFolder(String folder) + { + this.folder = folder; + } + + public String getName() + { + return name; + } + + public String getTitle() + { + return title; + } + + public void setTitle(String title) + { + this.title = title; + } + + public boolean isTitleSameAsName() + { + return titleSameAsName; + } + + public void setTitleSameAsName(boolean updateTitle) + { + this.titleSameAsName = updateTitle; + } + public void setName(String name) + { + this.name = name; + } + + public boolean isConfirmed() + { + return confirmed; + } + + public void setConfirmed(boolean confirmed) + { + this.confirmed = confirmed; + } + + public String getFolderType() + { + return folderType; + } + + public void setFolderType(String folderType) + { + this.folderType = folderType; + } + + public boolean isAddAlias() + { + return addAlias; + } + + public void setAddAlias(boolean addAlias) + { + this.addAlias = addAlias; + } + + public String getTarget() + { + return target; + } + + public void setTarget(String target) + { + this.target = target; + } + + public void setTemplateSourceId(String templateSourceId) + { + this.templateSourceId = templateSourceId; + } + + public String getTemplateSourceId() + { + return templateSourceId; + } + + public Container getTemplateSourceContainer() + { + if (null == getTemplateSourceId()) + return null; + return ContainerManager.getForId(getTemplateSourceId()); + } + + public String[] getTemplateWriterTypes() + { + return templateWriterTypes; + } + + public void setTemplateWriterTypes(String[] templateWriterTypes) + { + this.templateWriterTypes = templateWriterTypes; + } + + public boolean getTemplateIncludeSubfolders() + { + return templateIncludeSubfolders; + } + + public void setTemplateIncludeSubfolders(boolean templateIncludeSubfolders) + { + this.templateIncludeSubfolders = templateIncludeSubfolders; + } + + public String[] getTargets() + { + return targets; + } + + public void setTargets(String[] targets) + { + this.targets = targets; + } + + public PHI getExportPhiLevel() + { + return _exportPhiLevel; + } + + public void setExportPhiLevel(PHI exportPhiLevel) + { + _exportPhiLevel = exportPhiLevel; + } + + /** + * Note: this is designed to allow code to specify a set of children to delete in bulk. The main use-case is workbooks, + * but it will work for non-workbook children as well. + */ + public List getTargetContainers(final Container currentContainer) throws IllegalArgumentException + { + if (getTargets() != null) + { + final List targets = new ArrayList<>(); + final List directChildren = ContainerManager.getChildren(currentContainer); + + Arrays.stream(getTargets()).forEach(x -> { + Container c = ContainerManager.getForId(x); + if (c == null) + { + try + { + Integer rowId = ConvertHelper.convert(x, Integer.class); + if (rowId > 0) + c = ContainerManager.getForRowId(rowId); + } + catch (ConversionException e) + { + //ignore + } + } + + if (c != null) + { + if (!c.equals(currentContainer)) + { + if (!directChildren.contains(c)) + { + throw new IllegalArgumentException("Folder " + c.getPath() + " is not a direct child of the current folder: " + currentContainer.getPath()); + } + + if (c.getContainerType().canHaveChildren()) + { + throw new IllegalArgumentException("Multi-folder delete is not supported for containers of type: " + c.getContainerType().getName()); + } + } + + targets.add(c); + } + else + { + throw new IllegalArgumentException("Unable to find folder with ID or RowId of: " + x); + } + }); + + return targets; + } + else + { + return Collections.singletonList(currentContainer); + } + } + } + + public static class RenameContainerForm + { + private String name; + private String title; + private boolean addAlias = true; + + public String getName() + { + return name; + } + + public void setName(String name) + { + this.name = name; + } + + public String getTitle() + { + return title; + } + + public void setTitle(String title) + { + this.title = title; + } + + public boolean isAddAlias() + { + return addAlias; + } + + public void setAddAlias(boolean addAlias) + { + this.addAlias = addAlias; + } + } + + // Note that validation checks occur in ContainerManager.rename() + @RequiresPermission(AdminPermission.class) + public static class RenameContainerAction extends MutatingApiAction + { + @Override + public ApiResponse execute(RenameContainerForm form, BindException errors) + { + Container container = getContainer(); + String name = StringUtils.trimToNull(form.getName()); + String title = StringUtils.trimToNull(form.getTitle()); + + String nameValue = name; + String titleValue = title; + if (name == null && title == null) + { + errors.reject(ERROR_MSG, "Please specify a name or a title."); + return new ApiSimpleResponse("success", false); + } + else if (name != null && title == null) + { + titleValue = name; + } + else if (name == null) + { + nameValue = container.getName(); + } + + boolean addAlias = form.isAddAlias(); + + try + { + Container c = ContainerManager.rename(container, getUser(), nameValue, titleValue, addAlias); + return new ApiSimpleResponse(c.toJSON(getUser())); + } + catch (Exception e) + { + errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : "Failed to rename folder. An error has occurred."); + return new ApiSimpleResponse("success", false); + } + } + } + + @RequiresPermission(AdminPermission.class) + public class RenameFolderAction extends FormViewAction + { + private ActionURL _returnUrl; + + @Override + public void validateCommand(ManageFoldersForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/renameFolder.jsp", form, errors); + } + + @Override + public boolean handlePost(ManageFoldersForm form, BindException errors) + { + try + { + String title = form.isTitleSameAsName() ? null : StringUtils.trimToNull(form.getTitle()); + Container c = ContainerManager.rename(getContainer(), getUser(), form.getName(), title, form.isAddAlias()); + _returnUrl = new AdminUrlsImpl().getManageFoldersURL(c); + return true; + } + catch (Exception e) + { + errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : "Failed to rename folder. An error has occurred."); + } + + return false; + } + + @Override + public ActionURL getSuccessURL(ManageFoldersForm form) + { + return _returnUrl; + } + + @Override + public void addNavTrail(NavTree root) + { + getPageConfig().setFocusId("name"); + String containerType = getContainer().isProject() ? "Project" : "Folder"; + addAdminNavTrail(root, "Change " + containerType + " Name Settings", this.getClass()); + } + } + + public static class MoveFolderTreeView extends JspView + { + private MoveFolderTreeView(ManageFoldersForm form, BindException errors) + { + super("/org/labkey/core/admin/moveFolder.jsp", form, errors); + } + } + + @RequiresPermission(AdminPermission.class) + @ActionNames("ShowMoveFolderTree,MoveFolder") + public class MoveFolderAction extends FormViewAction + { + boolean showConfirmPage = false; + boolean moveFailed = false; + + @Override + public void validateCommand(ManageFoldersForm form, Errors errors) + { + Container c = getContainer(); + + if (c.isRoot()) + throw new NotFoundException("Can't move the root folder."); // Don't show move tree from root + + if (c.equals(ContainerManager.getSharedContainer()) || c.equals(ContainerManager.getHomeContainer())) + errors.reject(ERROR_MSG, "Moving /Shared or /home is not possible."); + + Container newParent = isBlank(form.getTarget()) ? null : ContainerManager.getForPath(form.getTarget()); + if (null == newParent) + { + errors.reject(ERROR_MSG, "Target '" + form.getTarget() + "' folder does not exist."); + } + else if (!newParent.hasPermission(getUser(), AdminPermission.class)) + { + throw new UnauthorizedException(); + } + else if (newParent.hasChild(c.getName())) + { + errors.reject(ERROR_MSG, "Error: The selected folder already has a folder with that name. Please select a different location (or Cancel)."); + } + } + + @Override + public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) throws Exception + { + if (showConfirmPage) + return new JspView<>("/org/labkey/core/admin/confirmProjectMove.jsp", form); + if (moveFailed) + return new SimpleErrorView(errors); + else + return new MoveFolderTreeView(form, errors); + } + + @Override + public boolean handlePost(ManageFoldersForm form, BindException errors) throws Exception + { + Container c = getContainer(); + Container newParent = ContainerManager.getForPath(form.getTarget()); + Container oldProject = c.getProject(); + Container newProject = newParent.isRoot() ? c : newParent.getProject(); + + if (!oldProject.getId().equals(newProject.getId()) && !form.isConfirmed()) + { + showConfirmPage = true; + return false; // reshow + } + + try + { + ContainerManager.move(c, newParent, getUser()); + } + catch (ValidationException e) + { + moveFailed = true; + getPageConfig().setTemplate(Template.Dialog); + for (ValidationError validationError : e.getErrors()) + { + errors.addError(new LabKeyError(validationError.getMessage())); + } + if (!errors.hasErrors()) + errors.addError(new LabKeyError("Move failed")); + return false; + } + + if (form.isAddAlias()) + { + List newAliases = new ArrayList<>(ContainerManager.getAliasesForContainer(c)); + newAliases.add(c.getPath()); + ContainerManager.saveAliasesForContainer(c, newAliases, getUser()); + } + return true; + } + + @Override + public URLHelper getSuccessURL(ManageFoldersForm manageFoldersForm) + { + Container c = getContainer(); + c = ContainerManager.getForId(c.getId()); // Reload container to populate new location + return new AdminUrlsImpl().getManageFoldersURL(c); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Folder Management", getManageFoldersURL()); + root.addChild("Move Folder"); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ConfirmProjectMoveAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ManageFoldersForm form, BindException errors) + { + getPageConfig().setTemplate(Template.Dialog); + return new JspView<>("/org/labkey/core/admin/confirmProjectMove.jsp", form); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Confirm Project Move"); + } + } + + private static abstract class AbstractCreateFolderAction extends FormViewAction + { + private ActionURL _successURL; + + @Override + public void validateCommand(FORM target, Errors errors) + { + } + + @Override + public ModelAndView getView(FORM form, boolean reshow, BindException errors) + { + VBox vbox = new VBox(); + + if (!reshow) + { + FolderType folderType = FolderTypeManager.get().getDefaultFolderType(); + if (null != folderType) + { + // If a default folder type has been configured by a site admin set that as the default folder type choice + form.setFolderType(folderType.getName()); + } + form.setExportPhiLevel(ComplianceService.get().getMaxAllowedPhi(getContainer(), getUser())); + } + JspView statusView = new JspView<>("/org/labkey/core/admin/createFolder.jsp", form, errors); + vbox.addView(statusView); + + Container c = getViewContext().getContainerNoTab(); // Cannot create subfolder of tab folder + + setHelpTopic("createProject"); + getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(null, c)); + getPageConfig().setTemplate(Template.Wizard); + + if (c.isRoot()) + getPageConfig().setTitle("Create Project"); + else + { + String title = "Create Folder"; + + title += " in /"; + if (c == ContainerManager.getHomeContainer()) + title += "Home"; + else + title += c.getName(); + + getPageConfig().setTitle(title); + } + + return vbox; + } + + @Override + public boolean handlePost(FORM form, BindException errors) throws Exception + { + Container parent = getViewContext().getContainerNoTab(); + String folderName = StringUtils.trimToNull(form.getName()); + String folderTitle = (form.isTitleSameAsName() || folderName.equals(form.getTitle())) ? null : form.getTitle(); + StringBuilder error = new StringBuilder(); + Consumer afterCreateHandler = getAfterCreateHandler(form); + + Container container; + + if (Container.isLegalName(folderName, parent.isRoot(), error)) + { + if (parent.hasChild(folderName)) + { + if (parent.isRoot()) + { + error.append("The server already has a project with this name."); + } + else + { + error.append("The ").append(parent.isProject() ? "project " : "folder ").append(parent.getPath()).append(" already has a folder with this name."); + } + } + else + { + String folderType = form.getFolderType(); + + if (null == folderType) + { + errors.reject(null, "Folder type must be specified"); + return false; + } + + if ("Template".equals(folderType)) // Create folder from selected template + { + Container sourceContainer = form.getTemplateSourceContainer(); + if (null == sourceContainer) + { + errors.reject(null, "Source template folder not selected"); + return false; + } + else if (!sourceContainer.hasPermission(getUser(), AdminPermission.class)) + { + errors.reject(null, "User does not have administrator permissions to the source container"); + return false; + } + else if (!sourceContainer.hasEnableRestrictedModules(getUser()) && sourceContainer.hasRestrictedActiveModule(sourceContainer.getActiveModules())) + { + errors.reject(null, "The source folder has a restricted module for which you do not have permission."); + return false; + } + + FolderExportContext exportCtx = new FolderExportContext(getUser(), sourceContainer, PageFlowUtil.set(form.getTemplateWriterTypes()), "new", + form.getTemplateIncludeSubfolders(), form.getExportPhiLevel(), false, false, false, + new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); + + container = ContainerManager.createContainerFromTemplate(parent, folderName, folderTitle, sourceContainer, getUser(), exportCtx, afterCreateHandler); + } + else + { + FolderType type = FolderTypeManager.get().getFolderType(folderType); + + if (type == null) + { + errors.reject(null, "Folder type not recognized"); + return false; + } + + String[] modules = form.getActiveModules(); + + if (null == StringUtils.trimToNull(folderType) || FolderType.NONE.getName().equals(folderType)) + { + if (null == modules || modules.length == 0) + { + errors.reject(null, "At least one module must be selected"); + return false; + } + } + + // Work done in this lambda will not fire container events. Only fireCreateContainer() will be called. + Consumer configureContainer = (newContainer) -> + { + afterCreateHandler.accept(newContainer); + newContainer.setFolderType(type, getUser(), errors); + + if (null == StringUtils.trimToNull(folderType) || FolderType.NONE.getName().equals(folderType)) + { + Set activeModules = new HashSet<>(); + for (String moduleName : modules) + { + Module module = ModuleLoader.getInstance().getModule(moduleName); + if (module != null) + activeModules.add(module); + } + + newContainer.setFolderType(FolderType.NONE, getUser(), errors, activeModules); + Module defaultModule = ModuleLoader.getInstance().getModule(form.getDefaultModule()); + newContainer.setDefaultModule(defaultModule); + } + }; + container = ContainerManager.createContainer(parent, folderName, folderTitle, null, NormalContainerType.NAME, getUser(), null, configureContainer); + } + + _successURL = new AdminUrlsImpl().getSetFolderPermissionsURL(container); + _successURL.addParameter("wizard", Boolean.TRUE.toString()); + + return true; + } + } + + errors.reject(ERROR_MSG, "Error: " + error + " Please enter a different name."); + return false; + } + + /** + * Return a Consumer that provides post-creation handling on the new Container + */ + abstract public Consumer getAfterCreateHandler(FORM form); + + @Override + protected String getCommandClassMethodName() + { + return "getAfterCreateHandler"; + } + + @Override + public ActionURL getSuccessURL(FORM form) + { + return _successURL; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresPermission(AdminPermission.class) + public static class CreateFolderAction extends AbstractCreateFolderAction + { + @Override + public Consumer getAfterCreateHandler(ManageFoldersForm form) + { + // No special handling + return container -> {}; + } + } + + public static class CreateProjectForm extends ManageFoldersForm + { + private boolean _assignProjectAdmin = false; + + public boolean isAssignProjectAdmin() + { + return _assignProjectAdmin; + } + + @SuppressWarnings("unused") + public void setAssignProjectAdmin(boolean assignProjectAdmin) + { + _assignProjectAdmin = assignProjectAdmin; + } + } + + @RequiresPermission(CreateProjectPermission.class) + public static class CreateProjectAction extends AbstractCreateFolderAction + { + @Override + public void validateCommand(CreateProjectForm target, Errors errors) + { + super.validateCommand(target, errors); + if (!getContainer().isRoot()) + errors.reject(ERROR_MSG, "Must be invoked from the root"); + } + + @Override + public Consumer getAfterCreateHandler(CreateProjectForm form) + { + if (form.isAssignProjectAdmin()) + { + return c -> { + MutableSecurityPolicy policy = new MutableSecurityPolicy(c.getPolicy()); + policy.addRoleAssignment(getUser(), ProjectAdminRole.class); + User savePolicyUser = getUser(); + if (c.isProject() && !c.hasPermission(savePolicyUser, AdminPermission.class) && ContainerManager.getRoot().hasPermission(savePolicyUser, CreateProjectPermission.class)) + { + // Special case for project creators who don't necessarily yet have permission to save the policy of + // the project they just created + savePolicyUser = User.getAdminServiceUser(); + } + + SecurityPolicyManager.savePolicy(policy, savePolicyUser); + }; + } + else + { + return c -> {}; + } + } + } + + @RequiresPermission(AdminPermission.class) + public static class SetFolderPermissionsAction extends FormViewAction + { + private ActionURL _successURL; + + @Override + public void validateCommand(SetFolderPermissionsForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(SetFolderPermissionsForm form, boolean reshow, BindException errors) + { + VBox vbox = new VBox(); + + JspView statusView = new JspView<>("/org/labkey/core/admin/setFolderPermissions.jsp", form, errors); + vbox.addView(statusView); + + Container c = getContainer(); + getPageConfig().setTitle("Users / Permissions"); + getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(c, c.getParent())); + getPageConfig().setTemplate(Template.Wizard); + setHelpTopic("createProject"); + + return vbox; + } + + @Override + public boolean handlePost(SetFolderPermissionsForm form, BindException errors) + { + Container c = getContainer(); + String permissionType = form.getPermissionType(); + + if(c.isProject()){ + _successURL = new AdminUrlsImpl().getInitialFolderSettingsURL(c); + } + else + { + List extraSteps = getContainer().getFolderType().getExtraSetupSteps(getContainer()); + if (extraSteps.isEmpty()) + { + if (form.isAdvanced()) + { + _successURL = new SecurityController.SecurityUrlsImpl().getPermissionsURL(getContainer()); + } + else + { + _successURL = getContainer().getStartURL(getUser()); + } + } + else + { + _successURL = new ActionURL(extraSteps.get(0).getHref()); + } + } + + if(permissionType == null){ + errors.reject(ERROR_MSG, "You must select one of the options for permissions."); + return false; + } + + switch (permissionType) + { + case "CurrentUser" -> { + MutableSecurityPolicy policy = new MutableSecurityPolicy(c); + Role role = RoleManager.getRole(c.isProject() ? ProjectAdminRole.class : FolderAdminRole.class); + policy.addRoleAssignment(getUser(), role); + SecurityPolicyManager.savePolicy(policy, getUser()); + } + case "Inherit" -> SecurityManager.setInheritPermissions(c); + case "CopyExistingProject" -> { + String targetProject = form.getTargetProject(); + if (targetProject == null) + { + errors.reject(ERROR_MSG, "In order to copy permissions from an existing project, you must pick a project."); + return false; + } + Container source = ContainerManager.getForId(targetProject); + if (source == null) + { + source = ContainerManager.getForPath(targetProject); + } + if (source == null) + { + throw new NotFoundException("An unknown project was specified to copy permissions from: " + targetProject); + } + Map groupMap = GroupManager.copyGroupsToContainer(source, c, getUser()); + + //copy role assignments + SecurityPolicy op = SecurityPolicyManager.getPolicy(source); + MutableSecurityPolicy np = new MutableSecurityPolicy(c); + for (RoleAssignment assignment : op.getAssignments()) + { + int userId = assignment.getUserId(); + UserPrincipal p = SecurityManager.getPrincipal(userId); + Role r = assignment.getRole(); + + if (p instanceof Group g) + { + if (!g.isProjectGroup()) + { + np.addRoleAssignment(p, r, false); + } + else + { + np.addRoleAssignment(groupMap.get(p), r, false); + } + } + else + { + np.addRoleAssignment(p, r, false); + } + } + SecurityPolicyManager.savePolicy(np, getUser()); + } + default -> throw new UnsupportedOperationException("An Unknown permission type was supplied: " + permissionType); + } + _successURL.addParameter("wizard", Boolean.TRUE.toString()); + + return true; + } + + @Override + public ActionURL getSuccessURL(SetFolderPermissionsForm form) + { + return _successURL; + } + + @Override + public void addNavTrail(NavTree root) + { + getPageConfig().setFocusId("name"); + } + } + + public static class SetFolderPermissionsForm + { + private String targetProject; + private String permissionType; + private boolean advanced; + + public String getPermissionType() + { + return permissionType; + } + + @SuppressWarnings("unused") + public void setPermissionType(String permissionType) + { + this.permissionType = permissionType; + } + + public String getTargetProject() + { + return targetProject; + } + + @SuppressWarnings("unused") + public void setTargetProject(String targetProject) + { + this.targetProject = targetProject; + } + + public boolean isAdvanced() + { + return advanced; + } + + @SuppressWarnings("unused") + public void setAdvanced(boolean advanced) + { + this.advanced = advanced; + } + } + + @RequiresPermission(AdminPermission.class) + public static class SetInitialFolderSettingsAction extends FormViewAction + { + private ActionURL _successURL; + + @Override + public void validateCommand(FilesForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(FilesForm form, boolean reshow, BindException errors) + { + VBox vbox = new VBox(); + Container c = getContainer(); + + JspView statusView = new JspView<>("/org/labkey/core/admin/setInitialFolderSettings.jsp", form, errors); + vbox.addView(statusView); + + getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(c, c.getParent())); + getPageConfig().setTemplate(Template.Wizard); + + String noun = c.isProject() ? "Project": "Folder"; + getPageConfig().setTitle(noun + " Settings"); + + return vbox; + } + + @Override + public boolean handlePost(FilesForm form, BindException errors) + { + Container c = getContainer(); + String folderRootPath = StringUtils.trimToNull(form.getFolderRootPath()); + String fileRootOption = form.getFileRootOption() != null ? form.getFileRootOption() : "default"; + + if(folderRootPath == null && !fileRootOption.equals("default")) + { + errors.reject(ERROR_MSG, "Error: Must supply a default file location."); + return false; + } + + FileContentService service = FileContentService.get(); + if(fileRootOption.equals("default")) + { + service.setIsUseDefaultRoot(c, true); + } + // Requires AdminOperationsPermission to set file root + else if (c.hasPermission(getUser(), AdminOperationsPermission.class)) + { + if (!service.isValidProjectRoot(folderRootPath)) + { + errors.reject(ERROR_MSG, "File root '" + folderRootPath + "' does not appear to be a valid directory accessible to the server at " + getViewContext().getRequest().getServerName() + "."); + return false; + } + + service.setIsUseDefaultRoot(c.getProject(), false); + service.setFileRootPath(c.getProject(), folderRootPath); + } + + List extraSteps = getContainer().getFolderType().getExtraSetupSteps(getContainer()); + if (extraSteps.isEmpty()) + { + _successURL = getContainer().getStartURL(getUser()); + } + else + { + _successURL = new ActionURL(extraSteps.get(0).getHref()); + } + + return true; + } + + @Override + public ActionURL getSuccessURL(FilesForm form) + { + return _successURL; + } + + @Override + public void addNavTrail(NavTree root) + { + getPageConfig().setFocusId("name"); + setHelpTopic("createProject"); + } + } + + @RequiresPermission(DeletePermission.class) + public static class DeleteWorkbooksAction extends SimpleRedirectAction + { + public void validateCommand(ReturnUrlForm target, Errors errors) + { + Set ids = DataRegionSelection.getSelected(getViewContext(), true); + if (ids.isEmpty()) + { + errors.reject(ERROR_MSG, "No IDs provided"); + } + } + + @Override + public @Nullable URLHelper getRedirectURL(ReturnUrlForm form) throws Exception + { + Set ids = DataRegionSelection.getSelected(getViewContext(), true); + + ActionURL ret = new ActionURL(DeleteFolderAction.class, getContainer()); + ids.forEach(id -> ret.addParameter("targets", id)); + + ret.replaceParameter(ActionURL.Param.returnUrl, form.getReturnUrl()); + + return ret; + } + } + + //NOTE: some types of containers can be deleted by non-admin users, provided they have DeletePermission on the parent + @RequiresPermission(DeletePermission.class) + public static class DeleteFolderAction extends FormViewAction + { + private final List _deleted = new ArrayList<>(); + + @Override + public void validateCommand(ManageFoldersForm form, Errors errors) + { + try + { + List targets = form.getTargetContainers(getContainer()); + for (Container target : targets) + { + if (!ContainerManager.isDeletable(target)) + errors.reject(ERROR_MSG, "The path " + target.getPath() + " is not deletable."); + + if (target.isProject() && !getUser().hasRootAdminPermission()) + { + throw new UnauthorizedException(); + } + + Class permClass = target.getPermissionNeededToDelete(); + if (!target.hasPermission(getUser(), permClass)) + { + Permission perm = RoleManager.getPermission(permClass); + throw new UnauthorizedException("Cannot delete folder: " + target.getName() + ". " + perm.getName() + " permission required"); + } + + if (target.hasChildren() && !ContainerManager.hasTreePermission(target, getUser(), AdminPermission.class)) + { + throw new UnauthorizedException("Deleting the " + target.getContainerNoun() + " " + target.getName() + " requires admin permissions on that folder and all children. You do not have admin permission on all subfolders."); + } + + if (target.equals(ContainerManager.getSharedContainer()) || target.equals(ContainerManager.getHomeContainer())) + errors.reject(ERROR_MSG, "Deleting /Shared or /home is not possible."); + } + } + catch (IllegalArgumentException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + } + + @Override + public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) + { + getPageConfig().setTemplate(Template.Dialog); + return new JspView<>("/org/labkey/core/admin/deleteFolder.jsp", form); + } + + @Override + public boolean handlePost(ManageFoldersForm form, BindException errors) + { + List targets = form.getTargetContainers(getContainer()); + + // Must be site/app admin to delete a project + for (Container c : targets) + { + ContainerManager.deleteAll(c, getUser()); + } + + _deleted.addAll(targets); + + return true; + } + + @Override + public ActionURL getSuccessURL(ManageFoldersForm form) + { + // Note: because in some scenarios we might be deleting children of the current contaner, in those cases we remain in this folder: + // If we just deleted a project then redirect to the home page, otherwise back to managing the project folders + if (_deleted.size() == 1 && _deleted.get(0).equals(getContainer())) + { + Container c = getContainer(); + if (c.isProject()) + return AppProps.getInstance().getHomePageActionURL(); + else + return new AdminUrlsImpl().getManageFoldersURL(c.getParent()); + } + else + { + if (form.getReturnUrl() != null) + { + return form.getReturnActionURL(); + } + else + { + return getContainer().getStartURL(getUser()); + } + } + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Confirm " + getContainer().getContainerNoun() + " deletion"); + } + } + + @RequiresPermission(AdminPermission.class) + public class ReorderFoldersAction extends FormViewAction + { + @Override + public void validateCommand(FolderReorderForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(FolderReorderForm folderReorderForm, boolean reshow, BindException errors) + { + return new JspView("/org/labkey/core/admin/reorderFolders.jsp"); + } + + @Override + public boolean handlePost(FolderReorderForm form, BindException errors) + { + return ReorderFolders(form, errors); + } + + @Override + public ActionURL getSuccessURL(FolderReorderForm folderReorderForm) + { + if (getContainer().isRoot()) + return getShowAdminURL(); + else + return getManageFoldersURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + String title = "Reorder " + (getContainer().isRoot() || getContainer().getParent().isRoot() ? "Projects" : "Folders"); + addAdminNavTrail(root, title, this.getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public class ReorderFoldersApiAction extends MutatingApiAction + { + @Override + public ApiResponse execute(FolderReorderForm form, BindException errors) + { + return new ApiSimpleResponse("success", ReorderFolders(form, errors)); + } + } + + private boolean ReorderFolders(FolderReorderForm form, BindException errors) + { + Container parent = getContainer().isRoot() ? getContainer() : getContainer().getParent(); + if (form.isResetToAlphabetical()) + ContainerManager.setChildOrderToAlphabetical(parent); + else if (form.getOrder() != null) + { + List children = parent.getChildren(); + String[] order = form.getOrder().split(";"); + Map nameToContainer = new HashMap<>(); + for (Container child : children) + nameToContainer.put(child.getName(), child); + List sorted = new ArrayList<>(children.size()); + for (String childName : order) + { + Container child = nameToContainer.get(childName); + sorted.add(child); + } + + try + { + ContainerManager.setChildOrder(parent, sorted); + } + catch (ContainerException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + return false; + } + } + + return true; + } + + public static class FolderReorderForm + { + private String _order; + private boolean _resetToAlphabetical; + + public String getOrder() + { + return _order; + } + + @SuppressWarnings("unused") + public void setOrder(String order) + { + _order = order; + } + + public boolean isResetToAlphabetical() + { + return _resetToAlphabetical; + } + + @SuppressWarnings("unused") + public void setResetToAlphabetical(boolean resetToAlphabetical) + { + _resetToAlphabetical = resetToAlphabetical; + } + } + + @RequiresPermission(AdminPermission.class) + public static class RevertFolderAction extends MutatingApiAction + { + @Override + public ApiResponse execute(RevertFolderForm form, BindException errors) + { + if (isBlank(form.getContainerPath())) + throw new NotFoundException(); + + boolean success = false; + Container revertContainer = ContainerManager.getForPath(form.getContainerPath()); + if (null != revertContainer) + { + if (revertContainer.isContainerTab()) + { + FolderTab tab = revertContainer.getParent().getFolderType().findTab(revertContainer.getName()); + if (null != tab) + { + FolderType origFolderType = tab.getFolderType(); + if (null != origFolderType) + { + revertContainer.setFolderType(origFolderType, getUser(), errors); + if (!errors.hasErrors()) + success = true; + } + } + } + else if (revertContainer.getFolderType().hasContainerTabs()) + { + try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) + { + List children = revertContainer.getChildren(); + for (Container container : children) + { + if (container.isContainerTab()) + { + FolderTab tab = revertContainer.getFolderType().findTab(container.getName()); + if (null != tab) + { + FolderType origFolderType = tab.getFolderType(); + if (null != origFolderType) + { + container.setFolderType(origFolderType, getUser(), errors); + } + } + } + } + if (!errors.hasErrors()) + { + transaction.commit(); + success = true; + } + } + } + } + return new ApiSimpleResponse("success", success); + } + } + + public static class RevertFolderForm + { + private String _containerPath; + + public String getContainerPath() + { + return _containerPath; + } + + public void setContainerPath(String containerPath) + { + _containerPath = containerPath; + } + } + + public static class EmailTestForm + { + private String _to; + private String _body; + private ConfigurationException _exception; + + public String getTo() + { + return _to; + } + + public void setTo(String to) + { + _to = to; + } + + public String getBody() + { + return _body; + } + + public void setBody(String body) + { + _body = body; + } + + public ConfigurationException getException() + { + return _exception; + } + + public void setException(ConfigurationException exception) + { + _exception = exception; + } + + public String getFrom(Container c) + { + LookAndFeelProperties props = LookAndFeelProperties.getInstance(c); + return props.getSystemEmailAddress(); + } + } + + @AdminConsoleAction + @RequiresPermission(AdminOperationsPermission.class) + public class EmailTestAction extends FormViewAction + { + @Override + public void validateCommand(EmailTestForm form, Errors errors) + { + if(null == form.getTo() || form.getTo().isEmpty()) + { + errors.reject(ERROR_MSG, "To field cannot be blank."); + form.setException(new ConfigurationException("To field cannot be blank")); + return; + } + + try + { + ValidEmail email = new ValidEmail(form.getTo()); + } + catch(ValidEmail.InvalidEmailException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + form.setException(new ConfigurationException(e.getMessage())); + } + } + + @Override + public ModelAndView getView(EmailTestForm form, boolean reshow, BindException errors) + { + JspView testView = new JspView<>("/org/labkey/core/admin/emailTest.jsp", form); + testView.setTitle("Send a Test Email"); + + if(null != MailHelper.getSession() && null != MailHelper.getSession().getProperties()) + { + JspView emailPropsView = new JspView<>("/org/labkey/core/admin/emailProps.jsp"); + emailPropsView.setTitle("Current Email Settings"); + + return new VBox(emailPropsView, testView); + } + else + return testView; + } + + @Override + public boolean handlePost(EmailTestForm form, BindException errors) throws Exception + { + if (errors.hasErrors()) + { + return false; + } + + LookAndFeelProperties props = LookAndFeelProperties.getInstance(getContainer()); + try + { + MailHelper.ViewMessage msg = MailHelper.createMessage(props.getSystemEmailAddress(), new ValidEmail(form.getTo()).toString()); + msg.setSubject("Test email message sent from " + props.getShortName()); + msg.setText(PageFlowUtil.filter(form.getBody())); + + try + { + MailHelper.send(msg, getUser(), getContainer()); + } + catch (ConfigurationException e) + { + form.setException(e); + return false; + } + catch (Exception e) + { + form.setException(new ConfigurationException(e.getMessage())); + return false; + } + } + catch (MessagingException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + return false; + } + return true; + } + + @Override + public URLHelper getSuccessURL(EmailTestForm emailTestForm) + { + return new ActionURL(EmailTestAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Test Email Configuration", getClass()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class RecreateViewsAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) + { + getPageConfig().setShowHeader(false); + getPageConfig().setTitle("Recreate Views?"); + return new HtmlView(HtmlString.of("Are you sure you want to drop and recreate all module views?")); + } + + @Override + public boolean handlePost(Object o, BindException errors) + { + ModuleLoader.getInstance().recreateViews(); + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + } + + @Override + public @NotNull ActionURL getSuccessURL(Object o) + { + return AppProps.getInstance().getHomePageActionURL(); + } + } + + static public class LoggingForm + { + public boolean isLogging() + { + return logging; + } + + public void setLogging(boolean logging) + { + this.logging = logging; + } + + public boolean logging = false; + } + + @RequiresLogin + @AllowedDuringUpgrade + @AllowedBeforeInitialUserIsSet + public static class GetSessionLogEventsAction extends ReadOnlyApiAction + { + @Override + public void checkPermissions() + { + super.checkPermissions(); + if (!getUser().isPlatformDeveloper()) + throw new UnauthorizedException(); + } + + @Override + public ApiResponse execute(Object o, BindException errors) + { + Integer eventId = null; + try + { + String s = getViewContext().getRequest().getParameter("eventId"); + if (null != s) + eventId = Integer.parseInt(s); + } + catch (NumberFormatException ignored) {} + ApiSimpleResponse res = new ApiSimpleResponse(); + res.put("success", true); + res.put("events", SessionAppender.getLoggingEvents(getViewContext().getRequest(), eventId)); + return res; + } + } + + @RequiresLogin + @AllowedBeforeInitialUserIsSet + @AllowedDuringUpgrade + @IgnoresAllocationTracking /* ignore so that we don't get an update in the UI for each time it requests the newest data */ + public static class GetTrackedAllocationsAction extends ReadOnlyApiAction + { + @Override + public void checkPermissions() + { + super.checkPermissions(); + if (!getUser().isPlatformDeveloper()) + throw new UnauthorizedException(); + } + + @Override + public ApiResponse execute(Object o, BindException errors) + { + long requestId = 0; + try + { + String s = getViewContext().getRequest().getParameter("requestId"); + if (null != s) + requestId = Long.parseLong(s); + } + catch (NumberFormatException ignored) {} + List requests = MemTracker.getInstance().getNewRequests(requestId); + List> jsonRequests = new ArrayList<>(requests.size()); + for (RequestInfo requestInfo : requests) + { + Map m = new HashMap<>(); + m.put("requestId", requestInfo.getId()); + m.put("url", requestInfo.getUrl()); + m.put("date", requestInfo.getDate()); + + + List> sortedObjects = sortByCounts(requestInfo); + + List> jsonObjects = new ArrayList<>(sortedObjects.size()); + for (Map.Entry entry : sortedObjects) + { + Map jsonObject = new HashMap<>(); + jsonObject.put("name", entry.getKey()); + jsonObject.put("count", entry.getValue()); + jsonObjects.add(jsonObject); + } + m.put("objects", jsonObjects); + jsonRequests.add(m); + } + return new ApiSimpleResponse("requests", jsonRequests); + } + + private List> sortByCounts(RequestInfo requestInfo) + { + List> objects = new ArrayList<>(requestInfo.getObjects().entrySet()); + objects.sort(Map.Entry.comparingByValue(Comparator.reverseOrder())); + return objects; + } + } + + @RequiresLogin + @AllowedDuringUpgrade + @AllowedBeforeInitialUserIsSet + public static class TrackedAllocationsViewerAction extends SimpleViewAction + { + @Override + public void checkPermissions() + { + super.checkPermissions(); + if (!getUser().isPlatformDeveloper()) + throw new UnauthorizedException(); + } + + @Override + public ModelAndView getView(Object o, BindException errors) + { + getPageConfig().setTemplate(Template.Print); + return new JspView<>("/org/labkey/core/admin/memTrackerViewer.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresLogin + @AllowedDuringUpgrade + @AllowedBeforeInitialUserIsSet + public static class SessionLoggingAction extends FormViewAction + { + @Override + public void checkPermissions() + { + super.checkPermissions(); + if (!getContainer().hasPermission(getUser(), PlatformDeveloperPermission.class)) + throw new UnauthorizedException(); + } + + @Override + public boolean handlePost(LoggingForm form, BindException errors) + { + boolean on = SessionAppender.isLogging(getViewContext().getRequest()); + if (form.logging != on) + { + if (!form.logging) + LogManager.getLogger(AdminController.class).info("turn session logging OFF"); + SessionAppender.setLoggingForSession(getViewContext().getRequest(), form.logging); + if (form.logging) + LogManager.getLogger(AdminController.class).info("turn session logging ON"); + } + return true; + } + + @Override + public void validateCommand(LoggingForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(LoggingForm o, boolean reshow, BindException errors) + { + SessionAppender.setLoggingForSession(getViewContext().getRequest(), true); + getPageConfig().setTemplate(Template.Print); + return new LoggingView(); + } + + @Override + public ActionURL getSuccessURL(LoggingForm o) + { + return new ActionURL(SessionLoggingAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Admin Console", new ActionURL(ShowAdminAction.class, getContainer()).getLocalURIString()); + root.addChild("View Event Log"); + } + } + + static class LoggingView extends JspView + { + LoggingView() + { + super("/org/labkey/core/admin/logging.jsp", null); + } + } + + public static class LogForm + { + private String _message; + private String _level; + + public String getMessage() + { + return _message; + } + + public void setMessage(String message) + { + _message = message; + } + + public String getLevel() + { + return _level; + } + + public void setLevel(String level) + { + _level = level; + } + } + + + // Simple action that writes "message" parameter to the labkey log. Used by the test harness to indicate when + // each test begins and ends. Message parameter is output as sent, except that \n is translated to newline. + @RequiresLogin + public static class LogAction extends MutatingApiAction + { + @Override + public ApiResponse execute(LogForm logForm, BindException errors) + { + // Could use %A0 for newline in the middle of the message, however, parameter values get trimmed so translate + // \n to newlines to allow them at the beginning or end of the message as well. + StringBuilder message = new StringBuilder(); + message.append(StringUtils.replace(logForm.getMessage(), "\\n", "\n")); + + Level level = Level.toLevel(logForm.getLevel(), Level.INFO); + CLIENT_LOG.log(level, message); + return new ApiSimpleResponse("success", true); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public class ValidateDomainsAction extends FormHandlerAction + { + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + // Find a valid pipeline root - we don't really care which one, we just need somewhere to write the log file + for (Container project : Arrays.asList(ContainerManager.getSharedContainer(), ContainerManager.getHomeContainer())) + { + PipeRoot root = PipelineService.get().findPipelineRoot(project); + if (root != null && root.isValid()) + { + ViewBackgroundInfo info = getViewBackgroundInfo(); + PipelineJob job = new ValidateDomainsPipelineJob(info, root); + PipelineService.get().queueJob(job); + return true; + } + } + return false; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return urlProvider(PipelineUrls.class).urlBegin(ContainerManager.getRoot()); + } + } + + public static class ModulesForm + { + private double[] _ignore = new double[0]; // Module versions to ignore (filter out of the results) + private boolean _managedOnly = false; + private boolean _unmanagedOnly = false; + + public double[] getIgnore() + { + return _ignore; + } + + public void setIgnore(double[] ignore) + { + _ignore = ignore; + } + + private Set getIgnoreSet() + { + return new LinkedHashSet<>(Arrays.asList(ArrayUtils.toObject(_ignore))); + } + + public boolean isManagedOnly() + { + return _managedOnly; + } + + @SuppressWarnings("unused") + public void setManagedOnly(boolean managedOnly) + { + _managedOnly = managedOnly; + } + + public boolean isUnmanagedOnly() + { + return _unmanagedOnly; + } + + @SuppressWarnings("unused") + public void setUnmanagedOnly(boolean unmanagedOnly) + { + _unmanagedOnly = unmanagedOnly; + } + } + + public enum ManageFilter + { + ManagedOnly + { + @Override + public boolean accept(Module module) + { + return null != module && module.shouldManageVersion(); + } + }, + UnmanagedOnly + { + @Override + public boolean accept(Module module) + { + return null != module && !module.shouldManageVersion(); + } + }, + All + { + @Override + public boolean accept(Module module) + { + return true; + } + }; + + public abstract boolean accept(Module module); + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public class ModulesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ModulesForm form, BindException errors) + { + ModuleLoader ml = ModuleLoader.getInstance(); + boolean hasAdminOpsPerm = getContainer().hasPermission(getUser(), AdminOperationsPermission.class); + + Collection unknownModules = ml.getUnknownModuleContexts().values(); + Collection knownModules = ml.getAllModuleContexts(); + knownModules.removeAll(unknownModules); + + Set ignoreSet = form.getIgnoreSet(); + HtmlString managedLink = HtmlString.EMPTY_STRING; + HtmlString unmanagedLink = HtmlString.EMPTY_STRING; + + // Option to filter out all modules whose version shouldn't be managed, or whose version matches the previous release + // version or 0.00. This can be helpful during the end-of-release consolidation process. Show the link only in dev mode. + if (AppProps.getInstance().isDevMode()) + { + if (ignoreSet.isEmpty() && !form.isManagedOnly()) + { + String lowestSchemaVersion = ModuleContext.formatVersion(Constants.getLowestSchemaVersion()); + ActionURL url = new ActionURL(ModulesAction.class, ContainerManager.getRoot()); + url.addParameter("ignore", "0.00," + lowestSchemaVersion); + url.addParameter("managedOnly", true); + managedLink = LinkBuilder.labkeyLink("Click here to ignore null, " + lowestSchemaVersion + " and unmanaged modules", url).getHtmlString(); + } + else + { + List ignore = ignoreSet + .stream() + .map(ModuleContext::formatVersion) + .collect(Collectors.toCollection(LinkedList::new)); + + String ignoreString = ignore.isEmpty() ? null : ignore.toString(); + String unmanaged = form.isManagedOnly() ? "unmanaged" : null; + + managedLink = HtmlString.of("(Currently ignoring " + Joiner.on(" and ").skipNulls().join(new String[]{ignoreString, unmanaged}) + ") "); + } + + if (!form.isUnmanagedOnly()) + { + ActionURL url = new ActionURL(ModulesAction.class, ContainerManager.getRoot()); + url.addParameter("unmanagedOnly", true); + unmanagedLink = LinkBuilder.labkeyLink("Click here to show unmanaged modules only", url).getHtmlString(); + } + else + { + unmanagedLink = HtmlString.of("(Currently showing unmanaged modules only)"); + } + } + + ManageFilter filter = form.isManagedOnly() ? ManageFilter.ManagedOnly : (form.isUnmanagedOnly() ? ManageFilter.UnmanagedOnly : ManageFilter.All); + + HtmlStringBuilder deleteInstructions = HtmlStringBuilder.of(); + if (hasAdminOpsPerm) + { + deleteInstructions.unsafeAppend("

    ").append( + "To delete a module that does not have a delete link, first delete its .module file and exploded module directory from your Labkey deployment directory, and restart the server. " + + "Module files are typically deployed in /modules and /externalModules.") + .unsafeAppend("

    ").append( + LinkBuilder.labkeyLink("Create new empty module", getCreateURL())); + } + + HtmlStringBuilder docLink = HtmlStringBuilder.of(); + docLink.unsafeAppend("

    ").append("Additional modules available, click ").append(new HelpTopic("defaultModules").getSimpleLinkHtml("here")).append(" to learn more."); + + HtmlStringBuilder knownDescription = HtmlStringBuilder.of() + .append("Each of these modules is installed and has a valid module file. ").append(managedLink).append(unmanagedLink).append(deleteInstructions).append(docLink); + HttpView known = new ModulesView(knownModules, "Known", knownDescription.getHtmlString(), null, ignoreSet, filter); + + HtmlStringBuilder unknownDescription = HtmlStringBuilder.of() + .append(1 == unknownModules.size() ? "This module" : "Each of these modules").append(" has been installed on this server " + + "in the past but the corresponding module file is currently missing or invalid. Possible explanations: the " + + "module is no longer part of the deployed distribution, the module has been renamed, the server location where the module " + + "is stored is not accessible, or the module file is corrupted.") + .unsafeAppend("

    ").append("The delete links below will remove all record of a module from the database tables."); + HtmlString noModulesDescription = HtmlString.of("A module is considered \"unknown\" if it was installed on this server " + + "in the past but the corresponding module file is currently missing or invalid. This server has no unknown modules."); + HttpView unknown = new ModulesView(unknownModules, "Unknown", unknownDescription.getHtmlString(), noModulesDescription, Collections.emptySet(), filter); + + return new VBox(known, unknown); + } + + private class ModulesView extends WebPartView + { + private final Collection _contexts; + private final HtmlString _descriptionHtml; + private final HtmlString _noModulesDescriptionHtml; + private final Set _ignoreVersions; + private final ManageFilter _manageFilter; + + private ModulesView(Collection contexts, String type, HtmlString descriptionHtml, HtmlString noModulesDescriptionHtml, Set ignoreVersions, ManageFilter manageFilter) + { + super(FrameType.PORTAL); + List sorted = new ArrayList<>(contexts); + sorted.sort(Comparator.comparing(ModuleContext::getName, String.CASE_INSENSITIVE_ORDER)); + + _contexts = sorted; + _descriptionHtml = descriptionHtml; + _noModulesDescriptionHtml = noModulesDescriptionHtml; + _ignoreVersions = ignoreVersions; + _manageFilter = manageFilter; + setTitle(type + " Modules"); + } + + @Override + protected void renderView(Object model, HtmlWriter out) + { + boolean isDevMode = AppProps.getInstance().isDevMode(); + boolean hasAdminOpsPerm = getUser().hasRootPermission(AdminOperationsPermission.class); + boolean hasUploadModulePerm = getUser().hasRootPermission(UploadFileBasedModulePermission.class); + final AtomicInteger rowCount = new AtomicInteger(); + ExplodedModuleService moduleService = !hasUploadModulePerm ? null : ServiceRegistry.get().getService(ExplodedModuleService.class); + final File externalModulesDir = moduleService==null ? null : moduleService.getExternalModulesDirectory(); + final Path relativeRoot = ModuleLoader.getInstance().getCoreModule().getExplodedPath().getParentFile().getParentFile().toPath(); + + if (_contexts.isEmpty()) + { + out.write(_noModulesDescriptionHtml); + } + else + { + DIV( + DIV(_descriptionHtml), + TABLE(cl("labkey-data-region-legacy","labkey-show-borders","labkey-data-region-header-lock"), + TR( + TD(cl("labkey-column-header"),"Name"), + TD(cl("labkey-column-header"),"Release Version"), + TD(cl("labkey-column-header"),"Schema Version"), + TD(cl("labkey-column-header"),"Class"), + TD(cl("labkey-column-header"),"Location"), + TD(cl("labkey-column-header"),"Schemas"), + !AppProps.getInstance().isDevMode() ? null : TD(cl("labkey-column-header"),""), // edit actions + null == externalModulesDir ? null : TD(cl("labkey-column-header"),""), // upload actions + !hasAdminOpsPerm ? null : TD(cl("labkey-column-header"),"") // delete actions + ), + _contexts.stream() + .filter(moduleContext -> !_ignoreVersions.contains(moduleContext.getInstalledVersion())) + .map(moduleContext -> new Pair<>(moduleContext,ModuleLoader.getInstance().getModule(moduleContext.getName()))) + .filter(pair -> _manageFilter.accept(pair.getValue())) + .map(pair -> + { + ModuleContext moduleContext = pair.getKey(); + Module module = pair.getValue(); + List schemas = moduleContext.getSchemaList(); + Double schemaVersion = moduleContext.getSchemaVersion(); + boolean replaceableModule = false; + if (null != module && module.getClass() == SimpleModule.class && schemas.isEmpty()) + { + File zip = module.getZippedPath(); + if (null != zip && zip.getParentFile().equals(externalModulesDir)) + replaceableModule = true; + } + boolean deleteableModule = replaceableModule || null == module; + String className = StringUtils.trimToEmpty(moduleContext.getClassName()); + String fullPathToModule = ""; + String shortPathToModule = ""; + if (null != module) + { + Path p = module.getExplodedPath().toPath(); + if (null != module.getZippedPath()) + p = module.getZippedPath().toPath(); + if (isDevMode && ModuleEditorService.get().canEditSourceModule(module)) + if (!module.getExplodedPath().getPath().equals(module.getSourcePath())) + p = Paths.get(module.getSourcePath()); + fullPathToModule = p.toString(); + shortPathToModule = fullPathToModule; + Path rel = relativeRoot.relativize(p); + if (!rel.startsWith("..")) + shortPathToModule = rel.toString(); + } + ActionURL moduleEditorUrl = getModuleEditorURL(moduleContext.getName()); + + return TR(cl(rowCount.getAndIncrement()%2==0 ? "labkey-alternate-row" : "labkey-row").at(style,"vertical-align:top;"), + TD(moduleContext.getName()), + TD(at(style,"white-space:nowrap;"), null != module ? module.getReleaseVersion() : NBSP), + TD(null != schemaVersion ? ModuleContext.formatVersion(schemaVersion) : NBSP), + TD(SPAN(at(title,className), className.substring(className.lastIndexOf(".")+1))), + TD(SPAN(at(title,fullPathToModule),shortPathToModule)), + TD(schemas.stream().map(s -> createHtmlFragment(s, BR()))), + !AppProps.getInstance().isDevMode() ? null : TD((null == moduleEditorUrl) ? NBSP : LinkBuilder.labkeyLink("Edit module", moduleEditorUrl)), + null == externalModulesDir ? null : TD(!replaceableModule ? NBSP : LinkBuilder.labkeyLink("Upload Module", getUpdateURL(moduleContext.getName()))), + !hasAdminOpsPerm ? null : TD(!deleteableModule ? NBSP : LinkBuilder.labkeyLink("Delete Module" + (schemas.isEmpty() ? "" : (" and Schema" + (schemas.size() > 1 ? "s" : ""))), getDeleteURL(moduleContext.getName()))) + ); + }) + ) + ).appendTo(out); + } + } + } + + private ActionURL getDeleteURL(String name) + { + ActionURL url = ModuleEditorService.get().getDeleteModuleURL(name); + if (null != url) + return url; + url = new ActionURL(DeleteModuleAction.class, ContainerManager.getRoot()); + url.addParameter("name", name); + return url; + } + + private ActionURL getUpdateURL(String name) + { + ActionURL url = ModuleEditorService.get().getUpdateModuleURL(name); + if (null != url) + return url; + url = new ActionURL(UpdateModuleAction.class, ContainerManager.getRoot()); + url.addParameter("name", name); + return url; + } + + private ActionURL getModuleEditorURL(String name) + { + return ModuleEditorService.get().getModuleEditorURL(name); + } + + private ActionURL getCreateURL() + { + ActionURL url = ModuleEditorService.get().getCreateModuleURL(); + if (null != url) + return url; + url = new ActionURL(CreateModuleAction.class, ContainerManager.getRoot()); + return url; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("defaultModules"); + addAdminNavTrail(root, "Modules", getClass()); + } + } + + public static class SchemaVersionTestCase extends Assert + { + @Test + public void verifyMinimumSchemaVersion() + { + List modulesTooLow = ModuleLoader.getInstance().getModules().stream() + .filter(ManageFilter.ManagedOnly::accept) + .filter(m -> null != m.getSchemaVersion()) + .filter(m -> m.getSchemaVersion() > 0.00 && m.getSchemaVersion() < Constants.getLowestSchemaVersion()) + .toList(); + + if (!modulesTooLow.isEmpty()) + fail("The following module" + (1 == modulesTooLow.size() ? " needs its schema version" : "s need their schema versions") + " increased to " + ModuleContext.formatVersion(Constants.getLowestSchemaVersion()) + ": " + modulesTooLow); + } + + @Test + public void modulesWithSchemaVersionButNoScripts() + { + // Flag all modules that have a schema version but don't have scripts. Their schema version should be null. + List moduleNames = ModuleLoader.getInstance().getModules().stream() + .filter(m -> m.getSchemaVersion() != null) + .filter(m -> m instanceof DefaultModule dm && !dm.hasScripts()) + .map(m -> m.getName() + ": " + m.getSchemaVersion()) + .toList(); + + if (!moduleNames.isEmpty()) + fail("The following module" + (1 == moduleNames.size() ? "" : "s") + " should have a null schema version: " + moduleNames); + } + } + + public static class ModuleForm + { + private String _name; + + public String getName() + { + return _name; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setName(String name) + { + _name = name; + } + + @NotNull + private ModuleContext getModuleContext() + { + ModuleLoader ml = ModuleLoader.getInstance(); + ModuleContext ctx = ml.getModuleContextFromDatabase(getName()); + + if (null == ctx) + throw new NotFoundException("Module not found"); + + return ctx; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public class DeleteModuleAction extends ConfirmAction + { + @Override + public void validateCommand(ModuleForm form, Errors errors) + { + } + + @Override + public ModelAndView getConfirmView(ModuleForm form, BindException errors) + { + if (getPageConfig().getTitle() == null) + setTitle("Delete Module"); + + ModuleContext ctx = form.getModuleContext(); + Module module = ModuleLoader.getInstance().getModule(ctx.getName()); + boolean hasSchemas = !ctx.getSchemaList().isEmpty(); + boolean hasFiles = false; + if (null != module) + hasFiles = null!=module.getExplodedPath() && module.getExplodedPath().isDirectory() || null!=module.getZippedPath() && module.getZippedPath().isFile(); + + HtmlStringBuilder description = HtmlStringBuilder.of("\"" + ctx.getName() + "\" module"); + HtmlStringBuilder skippedSchemas = HtmlStringBuilder.of(); + if (hasSchemas) + { + SchemaActions schemaActions = ModuleLoader.getInstance().getSchemaActions(module, ctx); + List deleteList = schemaActions.deleteList(); + List skipList = schemaActions.skipList(); + + // List all the schemas that will be deleted + if (!deleteList.isEmpty()) + { + description.append(" and delete all data in "); + description.append(deleteList.size() > 1 ? "these schemas: " + StringUtils.join(deleteList, ", ") : "the \"" + deleteList.get(0) + "\" schema"); + } + + // For unknown modules, also list the schemas that won't be deleted + if (!skipList.isEmpty()) + { + skippedSchemas.append(HtmlString.BR); + skipList.forEach(sam -> skippedSchemas.append(HtmlString.BR) + .append("Note: Schema \"") + .append(sam.schema()) + .append("\" will not be deleted because it's in use by module \"") + .append(sam.module()) + .append("\"")); + } + } + + return new HtmlView(DIV( + !hasFiles ? null : DIV(cl("labkey-warning-messages"), + "This module still has files on disk. Consider, first stopping the server, deleting these files, and restarting the server before continuing.", + null==module.getExplodedPath()?null:UL(LI(module.getExplodedPath().getPath())), + null==module.getZippedPath()?null:UL(LI(module.getZippedPath().getPath())) + ), + BR(), + "Are you sure you want to remove the ", description, "? ", + "This operation cannot be undone!", + skippedSchemas, + BR(), + !hasFiles ? null : "Deleting modules on a running server could leave it in an unpredictable state; be sure to restart your server." + )); + } + + @Override + public boolean handlePost(ModuleForm form, BindException errors) + { + ModuleLoader.getInstance().removeModule(form.getModuleContext()); + + return true; + } + + @Override + public @NotNull URLHelper getSuccessURL(ModuleForm form) + { + return new ActionURL(ModulesAction.class, ContainerManager.getRoot()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class UpdateModuleAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ModuleForm moduleForm, BindException errors) throws Exception + { + return new HtmlView(HtmlString.of("This is a premium feature, please refer to our documentation on www.labkey.org")); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class CreateModuleAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ModuleForm moduleForm, BindException errors) throws Exception + { + return new HtmlView(HtmlString.of("This is a premium feature, please refer to our documentation on www.labkey.org")); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + public static class OptionalFeatureForm + { + private String feature; + private boolean enabled; + + public String getFeature() + { + return feature; + } + + public void setFeature(String feature) + { + this.feature = feature; + } + + public boolean isEnabled() + { + return enabled; + } + + public void setEnabled(boolean enabled) + { + this.enabled = enabled; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + @ActionNames("OptionalFeature, ExperimentalFeature") + public static class OptionalFeatureAction extends BaseApiAction + { + @Override + protected ModelAndView handleGet() throws Exception + { + return handlePost(); // 'execute' ensures that only POSTs are mutating + } + + @Override + public ApiResponse execute(OptionalFeatureForm form, BindException errors) + { + String feature = StringUtils.trimToNull(form.getFeature()); + if (feature == null) + throw new ApiUsageException("feature is required"); + + OptionalFeatureService svc = OptionalFeatureService.get(); + if (svc == null) + throw new IllegalStateException(); + + Map ret = new HashMap<>(); + ret.put("feature", feature); + + if (isPost()) + { + ret.put("previouslyEnabled", svc.isFeatureEnabled(feature)); + svc.setFeatureEnabled(feature, form.isEnabled(), getUser()); + } + + ret.put("enabled", svc.isFeatureEnabled(feature)); + return new ApiSimpleResponse(ret); + } + } + + public static class OptionalFeaturesForm + { + private String _type; + private boolean _showHidden; + + public String getType() + { + return _type; + } + + @SuppressWarnings("unused") + public void setType(String type) + { + _type = type; + } + + public @NotNull FeatureType getTypeEnum() + { + return EnumUtils.getEnum(FeatureType.class, getType(), FeatureType.Experimental); + } + + public boolean isShowHidden() + { + return _showHidden; + } + + @SuppressWarnings("unused") + public void setShowHidden(boolean showHidden) + { + _showHidden = showHidden; + } + } + + @RequiresPermission(TroubleshooterPermission.class) + public class OptionalFeaturesAction extends SimpleViewAction + { + private FeatureType _type; + + @Override + public ModelAndView getView(OptionalFeaturesForm form, BindException errors) + { + _type = form.getTypeEnum(); + JspView view = new JspView<>("/org/labkey/core/admin/optionalFeatures.jsp", form); + view.setFrame(WebPartView.FrameType.NONE); + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("experimental"); + addAdminNavTrail(root, _type.name() + " Features", getClass()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class ProductFeatureAction extends BaseApiAction + { + @Override + protected ModelAndView handleGet() throws Exception + { + return handlePost(); // 'execute' ensures that only POSTs are mutating + } + + @Override + public ApiResponse execute(ProductConfigForm form, BindException errors) + { + String productKey = StringUtils.trimToNull(form.getProductKey()); + + Map ret = new HashMap<>(); + + if (isPost()) + { + ProductConfiguration.setProductKey(productKey); + } + + ret.put("productKey", new ProductConfiguration().getCurrentProductKey()); + return new ApiSimpleResponse(ret); + } + } + + public static class ProductConfigForm + { + private String productKey; + + public String getProductKey() + { + return productKey; + } + + public void setProductKey(String productKey) + { + this.productKey = productKey; + } + + } + + @AdminConsoleAction + @RequiresPermission(AdminOperationsPermission.class) + public class ProductConfigurationAction extends SimpleViewAction + { + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Product Configuration", getClass()); + } + + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + JspView view = new JspView<>("/org/labkey/core/admin/productConfiguration.jsp"); + view.setFrame(WebPartView.FrameType.NONE); + return view; + } + } + + + public static class FolderTypesBean + { + private final Collection _allFolderTypes; + private final Collection _enabledFolderTypes; + private final FolderType _defaultFolderType; + + public FolderTypesBean(Collection allFolderTypes, Collection enabledFolderTypes, FolderType defaultFolderType) + { + _allFolderTypes = allFolderTypes; + _enabledFolderTypes = enabledFolderTypes; + _defaultFolderType = defaultFolderType; + } + + public Collection getAllFolderTypes() + { + return _allFolderTypes; + } + + public Collection getEnabledFolderTypes() + { + return _enabledFolderTypes; + } + + public FolderType getDefaultFolderType() + { + return _defaultFolderType; + } + } + + @AdminConsoleAction + @RequiresPermission(AdminPermission.class) + public class FolderTypesAction extends FormViewAction + { + @Override + public void validateCommand(Object form, Errors errors) + { + } + + @Override + public ModelAndView getView(Object form, boolean reshow, BindException errors) + { + FolderTypesBean bean; + if (reshow) + { + bean = getOptionsFromRequest(); + } + else + { + FolderTypeManager manager = FolderTypeManager.get(); + var defaultFolderType = manager.getDefaultFolderType(); + // If a default folder type has not yet been configuration use "Collaboration" folder type as the default + defaultFolderType = defaultFolderType != null ? defaultFolderType : manager.getFolderType(CollaborationFolderType.TYPE_NAME); + boolean userHasEnableRestrictedModulesPermission = getContainer().hasEnableRestrictedModules(getUser()); + bean = new FolderTypesBean(manager.getAllFolderTypes(), manager.getEnabledFolderTypes(userHasEnableRestrictedModulesPermission), defaultFolderType); + } + + return new JspView<>("/org/labkey/core/admin/enabledFolderTypes.jsp", bean, errors); + } + + @Override + public boolean handlePost(Object form, BindException errors) + { + FolderTypesBean bean = getOptionsFromRequest(); + var defaultFolderType = bean.getDefaultFolderType(); + if (defaultFolderType == null) + { + errors.reject(ERROR_MSG, "Please select a default folder type."); + return false; + } + var enabledFolderTypes = bean.getEnabledFolderTypes(); + if (!enabledFolderTypes.contains(defaultFolderType)) + { + errors.reject(ERROR_MSG, "Folder type selected as the default, '" + defaultFolderType.getName() + "', must be enabled."); + return false; + } + + FolderTypeManager.get().setEnabledFolderTypes(enabledFolderTypes, defaultFolderType); + return true; + } + + private FolderTypesBean getOptionsFromRequest() + { + var allFolderTypes = FolderTypeManager.get().getAllFolderTypes(); + List enabledFolderTypes = new ArrayList<>(); + FolderType defaultFolderType = null; + String defaultFolderTypeParam = getViewContext().getRequest().getParameter(FolderTypeManager.FOLDER_TYPE_DEFAULT); + + for (FolderType folderType : FolderTypeManager.get().getAllFolderTypes()) + { + boolean enabled = Boolean.TRUE.toString().equalsIgnoreCase(getViewContext().getRequest().getParameter(folderType.getName())); + if (enabled) + { + enabledFolderTypes.add(folderType); + } + if (folderType.getName().equals(defaultFolderTypeParam)) + { + defaultFolderType = folderType; + } + } + return new FolderTypesBean(allFolderTypes, enabledFolderTypes, defaultFolderType); + } + + @Override + public URLHelper getSuccessURL(Object form) + { + return getShowAdminURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Folder Types", getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class CustomizeMenuAction extends MutatingApiAction + { + @Override + public ApiResponse execute(CustomizeMenuForm form, BindException errors) + { + if (null != form.getUrl()) + { + String errorMessage = StringExpressionFactory.validateURL(form.getUrl()); + if (null != errorMessage) + { + errors.reject(ERROR_MSG, errorMessage); + return new ApiSimpleResponse("success", false); + } + } + + setCustomizeMenuForm(form, getContainer(), getUser()); + return new ApiSimpleResponse("success", true); + } + } + + protected static final String CUSTOMMENU_SCHEMA = "customMenuSchemaName"; + protected static final String CUSTOMMENU_QUERY = "customMenuQueryName"; + protected static final String CUSTOMMENU_VIEW = "customMenuViewName"; + protected static final String CUSTOMMENU_COLUMN = "customMenuColumnName"; + protected static final String CUSTOMMENU_FOLDER = "customMenuFolderName"; + protected static final String CUSTOMMENU_TITLE = "customMenuTitle"; + protected static final String CUSTOMMENU_URL = "customMenuUrl"; + protected static final String CUSTOMMENU_ROOTFOLDER = "customMenuRootFolder"; + protected static final String CUSTOMMENU_FOLDERTYPES = "customMenuFolderTypes"; + protected static final String CUSTOMMENU_CHOICELISTQUERY = "customMenuChoiceListQuery"; + protected static final String CUSTOMMENU_INCLUDEALLDESCENDANTS = "customIncludeAllDescendants"; + protected static final String CUSTOMMENU_CURRENTPROJECTONLY = "customCurrentProjectOnly"; + + public static CustomizeMenuForm getCustomizeMenuForm(Portal.WebPart webPart) + { + CustomizeMenuForm form = new CustomizeMenuForm(); + Map menuProps = webPart.getPropertyMap(); + + String schemaName = menuProps.get(CUSTOMMENU_SCHEMA); + String queryName = menuProps.get(CUSTOMMENU_QUERY); + String columnName = menuProps.get(CUSTOMMENU_COLUMN); + String viewName = menuProps.get(CUSTOMMENU_VIEW); + String folderName = menuProps.get(CUSTOMMENU_FOLDER); + String title = menuProps.get(CUSTOMMENU_TITLE); if (null == title) title = "My Menu"; + String urlBottom = menuProps.get(CUSTOMMENU_URL); + String rootFolder = menuProps.get(CUSTOMMENU_ROOTFOLDER); + String folderTypes = menuProps.get(CUSTOMMENU_FOLDERTYPES); + String choiceListQueryString = menuProps.get(CUSTOMMENU_CHOICELISTQUERY); + boolean choiceListQuery = null == choiceListQueryString || choiceListQueryString.equalsIgnoreCase("true"); + String includeAllDescendantsString = menuProps.get(CUSTOMMENU_INCLUDEALLDESCENDANTS); + boolean includeAllDescendants = null == includeAllDescendantsString || includeAllDescendantsString.equalsIgnoreCase("true"); + String currentProjectOnlyString = menuProps.get(CUSTOMMENU_CURRENTPROJECTONLY); + boolean currentProjectOnly = null != currentProjectOnlyString && currentProjectOnlyString.equalsIgnoreCase("true"); + + form.setSchemaName(schemaName); + form.setQueryName(queryName); + form.setColumnName(columnName); + form.setViewName(viewName); + form.setFolderName(folderName); + form.setTitle(title); + form.setUrl(urlBottom); + form.setRootFolder(rootFolder); + form.setFolderTypes(folderTypes); + form.setChoiceListQuery(choiceListQuery); + form.setIncludeAllDescendants(includeAllDescendants); + form.setCurrentProjectOnly(currentProjectOnly); + + form.setWebPartIndex(webPart.getIndex()); + form.setPageId(webPart.getPageId()); + return form; + } + + private static void setCustomizeMenuForm(CustomizeMenuForm form, Container container, User user) + { + Portal.WebPart webPart = Portal.getPart(container, form.getPageId(), form.getWebPartIndex()); + if (null == webPart) + throw new NotFoundException(); + Map menuProps = webPart.getPropertyMap(); + + menuProps.put(CUSTOMMENU_SCHEMA, form.getSchemaName()); + menuProps.put(CUSTOMMENU_QUERY, form.getQueryName()); + menuProps.put(CUSTOMMENU_COLUMN, form.getColumnName()); + menuProps.put(CUSTOMMENU_VIEW, form.getViewName()); + menuProps.put(CUSTOMMENU_FOLDER, form.getFolderName()); + menuProps.put(CUSTOMMENU_TITLE, form.getTitle()); + menuProps.put(CUSTOMMENU_URL, form.getUrl()); + + // If root folder not specified, set as current container + menuProps.put(CUSTOMMENU_ROOTFOLDER, StringUtils.trimToNull(form.getRootFolder()) != null ? form.getRootFolder() : container.getPath()); + menuProps.put(CUSTOMMENU_FOLDERTYPES, form.getFolderTypes()); + menuProps.put(CUSTOMMENU_CHOICELISTQUERY, form.isChoiceListQuery() ? "true" : "false"); + menuProps.put(CUSTOMMENU_INCLUDEALLDESCENDANTS, form.isIncludeAllDescendants() ? "true" : "false"); + menuProps.put(CUSTOMMENU_CURRENTPROJECTONLY, form.isCurrentProjectOnly() ? "true" : "false"); + + Portal.updatePart(user, webPart); + } + + @RequiresPermission(AdminPermission.class) + public static class AddTabAction extends MutatingApiAction + { + public void validateCommand(TabActionForm form, Errors errors) + { + Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); + if(tabContainer.getFolderType() == FolderType.NONE) + { + errors.reject(ERROR_MSG, "Cannot add tabs to custom folder types."); + } + else + { + String name = form.getTabName(); + if (StringUtils.isEmpty(name)) + { + errors.reject(ERROR_MSG, "A tab name must be specified."); + return; + } + + // Note: The name, which shows up on the url, is trimmed to 50 characters. The caption, which is derived + // from the name, and is editable, is allowed to be 64 characters, so we only error if passed something + // longer than 64 characters. + if (name.length() > 64) + { + errors.reject(ERROR_MSG, "Tab name cannot be longer than 64 characters."); + return; + } + + if (name.length() > 50) + name = name.substring(0, 50).trim(); + + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); + CaseInsensitiveHashMap folderTabMap = new CaseInsensitiveHashMap<>(); + + for (FolderTab tab : tabContainer.getFolderType().getDefaultTabs()) + { + folderTabMap.put(tab.getName(), tab); + } + + if (pages.containsKey(name)) + { + errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); + return; + } + + for (Portal.PortalPage page : pages.values()) + { + if (page.getCaption() != null && page.getCaption().equals(name)) + { + errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); + return; + } + else if (folderTabMap.containsKey(page.getPageId())) + { + if (folderTabMap.get(page.getPageId()).getCaption(getViewContext()).equalsIgnoreCase(name)) + { + errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); + return; + } + } + } + } + } + + @Override + public ApiResponse execute(TabActionForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + + validateCommand(form, errors); + + if(errors.hasErrors()) + { + return response; + } + + Container container = getContainer().getContainerFor(ContainerType.DataType.tabParent); + String name = form.getTabName(); + String caption = form.getTabName(); + + // The name, which shows up on the url, is trimmed to 50 characters. The caption, which is derived from the + // name, and is editable, is allowed to be 64 characters. + if (name.length() > 50) + name = name.substring(0, 50).trim(); + + Portal.saveParts(container, name); + Portal.addProperty(container, name, Portal.PROP_CUSTOMTAB); + + if (!name.equals(caption)) + { + // If we had to truncate the name then we want to set the caption to the un-truncated version of the name. + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(container, true)); + Portal.PortalPage page = pages.get(name); + // Get a mutable copy + page = page.copy(); + page.setCaption(caption); + Portal.updatePortalPage(container, page); + } + + ActionURL tabURL = new ActionURL(ProjectController.BeginAction.class, container); + tabURL.addParameter("pageId", name); + response.put("url", tabURL); + response.put("success", true); + return response; + } + } + + @RequiresPermission(AdminPermission.class) + public static class ShowTabAction extends MutatingApiAction + { + public void validateCommand(TabActionForm form, Errors errors) + { + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(getContainer().getContainerFor(ContainerType.DataType.tabParent), true)); + + if (form.getTabPageId() == null) + { + errors.reject(ERROR_MSG, "PageId cannot be blank."); + } + + if (!pages.containsKey(form.getTabPageId())) + { + errors.reject(ERROR_MSG, "Page cannot be found. Check with your system administrator."); + } + } + + @Override + public ApiResponse execute(TabActionForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); + + validateCommand(form, errors); + if (errors.hasErrors()) + return response; + + Portal.showPage(tabContainer, form.getTabPageId()); + ActionURL tabURL = new ActionURL(ProjectController.BeginAction.class, tabContainer); + tabURL.addParameter("pageId", form.getTabPageId()); + response.put("url", tabURL); + response.put("success", true); + return response; + } + } + + + public static class TabActionForm extends ReturnUrlForm + { + // This class is used for tab related actions (add, rename, show, etc.) + String _tabName; + String _tabPageId; + + public String getTabName() + { + return _tabName; + } + + public void setTabName(String name) + { + _tabName = name; + } + + public String getTabPageId() + { + return _tabPageId; + } + + public void setTabPageId(String tabPageId) + { + _tabPageId = tabPageId; + } + } + + @RequiresPermission(AdminPermission.class) + public class MoveTabAction extends MutatingApiAction + { + @Override + public ApiResponse execute(MoveTabForm form, BindException errors) + { + final Map properties = new HashMap<>(); + Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); + Portal.PortalPage tab = pages.get(form.getPageId()); + + if (null != tab) + { + int oldIndex = tab.getIndex(); + Portal.PortalPage pageToSwap = handleMovePortalPage(tabContainer, getUser(), tab, form.getDirection()); + + if (null != pageToSwap) + { + properties.put("oldIndex", oldIndex); + properties.put("newIndex", tab.getIndex()); + properties.put("pageId", tab.getPageId()); + properties.put("pageIdToSwap", pageToSwap.getPageId()); + } + else + { + properties.put("error", "Unable to move tab."); + } + } + else + { + properties.put("error", "Requested tab does not exist."); + } + + return new ApiSimpleResponse(properties); + } + } + + public static class MoveTabForm implements HasViewContext + { + private int _direction; + private String _pageId; + private ViewContext _viewContext; + + public int getDirection() + { + // 0 moves left, 1 moves right. + return _direction; + } + + public void setDirection(int direction) + { + _direction = direction; + } + + public String getPageId() + { + return _pageId; + } + + public void setPageId(String pageId) + { + _pageId = pageId; + } + + @Override + public ViewContext getViewContext() + { + return _viewContext; + } + + @Override + public void setViewContext(ViewContext viewContext) + { + _viewContext = viewContext; + } + } + + private Portal.PortalPage handleMovePortalPage(Container c, User user, Portal.PortalPage page, int direction) + { + Map pageMap = new CaseInsensitiveHashMap<>(); + for (Portal.PortalPage pp : Portal.getTabPages(c, true)) + pageMap.put(pp.getPageId(), pp); + + for (FolderTab folderTab : c.getFolderType().getDefaultTabs()) + { + if (pageMap.containsKey(folderTab.getName())) + { + // Issue 46233 : folder tabs can conditionally hide/show themselves at render time, these need to + // be excluded when adjusting the relative indexes. + if (!folderTab.isVisible(c, user)) + pageMap.remove(folderTab.getName()); + } + } + List pagesList = new ArrayList<>(pageMap.values()); + pagesList.sort(Comparator.comparingInt(Portal.PortalPage::getIndex)); + + int visibleIndex; + for (visibleIndex = 0; visibleIndex < pagesList.size(); visibleIndex++) + { + if (pagesList.get(visibleIndex).getIndex() == page.getIndex()) + { + break; + } + } + + if (visibleIndex == pagesList.size()) + { + return null; + } + + if (direction == Portal.MOVE_DOWN) + { + if (visibleIndex == pagesList.size() - 1) + { + return page; + } + + Portal.PortalPage nextPage = pagesList.get(visibleIndex + 1); + + if (null == nextPage) + return null; + Portal.swapPageIndexes(c, page, nextPage); + return nextPage; + } + else + { + if (visibleIndex < 1) + { + return page; + } + + Portal.PortalPage prevPage = pagesList.get(visibleIndex - 1); + + if (null == prevPage) + return null; + Portal.swapPageIndexes(c, page, prevPage); + return prevPage; + } + } + + @RequiresPermission(AdminPermission.class) + public static class RenameTabAction extends MutatingApiAction + { + public void validateCommand(TabActionForm form, Errors errors) + { + Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); + + if (tabContainer.getFolderType() == FolderType.NONE) + { + errors.reject(ERROR_MSG, "Cannot change tab names in custom folder types."); + } + else + { + String name = form.getTabName(); + if (StringUtils.isEmpty(name)) + { + errors.reject(ERROR_MSG, "A tab name must be specified."); + return; + } + + if (name.length() > 64) + { + errors.reject(ERROR_MSG, "Tab name cannot be longer than 64 characters."); + return; + } + + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); + Portal.PortalPage pageToChange = pages.get(form.getTabPageId()); + if (null == pageToChange) + { + errors.reject(ERROR_MSG, "Page cannot be found. Check with your system administrator."); + return; + } + + for (Portal.PortalPage page : pages.values()) + { + if (!page.equals(pageToChange)) + { + if (null != page.getCaption() && page.getCaption().equalsIgnoreCase(name)) + { + errors.reject(ERROR_MSG, "A tab with the same name already exists in this folder."); + return; + } + if (page.getPageId().equalsIgnoreCase(name)) + { + if (null != page.getCaption() || Portal.DEFAULT_PORTAL_PAGE_ID.equalsIgnoreCase(name)) + errors.reject(ERROR_MSG, "You cannot change a tab's name to another tab's original name even if the original name is not visible."); + else + errors.reject(ERROR_MSG, "A tab with the same name already exists in this folder."); + return; + } + } + } + + List folderTabs = tabContainer.getFolderType().getDefaultTabs(); + for (FolderTab folderTab : folderTabs) + { + String folderTabCaption = folderTab.getCaption(getViewContext()); + if (!folderTab.getName().equalsIgnoreCase(pageToChange.getPageId()) && null != folderTabCaption && folderTabCaption.equalsIgnoreCase(name)) + { + errors.reject(ERROR_MSG, "You cannot change a tab's name to another tab's original name even if the original name is not visible."); + return; + } + } + } + } + + @Override + public ApiResponse execute(TabActionForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + validateCommand(form, errors); + + if (errors.hasErrors()) + { + return response; + } + + Container container = getContainer().getContainerFor(ContainerType.DataType.tabParent); + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(container, true)); + Portal.PortalPage page = pages.get(form.getTabPageId()); + page = page.copy(); + page.setCaption(form.getTabName()); + // Update the page the caption is saved. + Portal.updatePortalPage(container, page); + + response.put("success", true); + return response; + } + } + + @RequiresPermission(ReadPermission.class) + public static class ClearDeletedTabFoldersAction extends MutatingApiAction + { + @Override + public ApiResponse execute(DeletedFoldersForm form, BindException errors) + { + if (isBlank(form.getContainerPath())) + throw new NotFoundException(); + Container container = ContainerManager.getForPath(form.getContainerPath()); + for (String tabName : form.getResurrectFolders()) + { + ContainerManager.clearContainerTabDeleted(container, tabName, form.getNewFolderType()); + } + return new ApiSimpleResponse("success", true); + } + } + + @SuppressWarnings("unused") + public static class DeletedFoldersForm + { + private String _containerPath; + private String _newFolderType; + private List _resurrectFolders; + + public List getResurrectFolders() + { + return _resurrectFolders; + } + + public void setResurrectFolders(List resurrectFolders) + { + _resurrectFolders = resurrectFolders; + } + + public String getContainerPath() + { + return _containerPath; + } + + public void setContainerPath(String containerPath) + { + _containerPath = containerPath; + } + + public String getNewFolderType() + { + return _newFolderType; + } + + public void setNewFolderType(String newFolderType) + { + _newFolderType = newFolderType; + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetFolderTabsAction extends ReadOnlyApiAction + { + @Override + public Object execute(Object form, BindException errors) throws Exception + { + var data = getContainer() + .getFolderType() + .getAppBar(getViewContext(), getPageConfig()) + .getButtons() + .stream() + .map(this::getProperties) + .toList(); + + return success(data); + } + + private Map getProperties(NavTree navTree) + { + Map props = new HashMap<>(); + props.put("id", navTree.getId()); + props.put("text", navTree.getText()); + props.put("href", navTree.getHref()); + props.put("disabled", navTree.isDisabled()); + return props; + } + } + + @SuppressWarnings("unused") + public static class ShortURLForm + { + private String _shortURL; + private String _fullURL; + private boolean _delete; + + private List _savedShortURLs; + + public void setShortURL(String shortURL) + { + _shortURL = shortURL; + } + + public void setFullURL(String fullURL) + { + _fullURL = fullURL; + } + + public void setDelete(boolean delete) + { + _delete = delete; + } + + public String getShortURL() + { + return _shortURL; + } + + public String getFullURL() + { + return _fullURL; + } + + public boolean isDelete() + { + return _delete; + } + } + + public abstract static class AbstractShortURLAdminAction extends FormViewAction + { + @Override + public void validateCommand(ShortURLForm target, Errors errors) {} + + @Override + public boolean handlePost(ShortURLForm form, BindException errors) throws Exception + { + String shortURL = StringUtils.trimToEmpty(form.getShortURL()); + if (StringUtils.isEmpty(shortURL)) + { + errors.addError(new LabKeyError("Short URL must not be blank")); + } + if (shortURL.endsWith(".url")) + shortURL = shortURL.substring(0,shortURL.length()-".url".length()); + if (shortURL.contains("#") || shortURL.contains("/") || shortURL.contains(".")) + { + errors.addError(new LabKeyError("Short URLs may not contain '#' or '/' or '.'")); + } + URLHelper fullURL = null; + if (!form.isDelete()) + { + String trimmedFullURL = StringUtils.trimToNull(form.getFullURL()); + if (trimmedFullURL == null) + { + errors.addError(new LabKeyError("Target URL must not be blank")); + } + else + { + try + { + fullURL = new URLHelper(trimmedFullURL); + } + catch (URISyntaxException e) + { + errors.addError(new LabKeyError("Invalid Target URL. " + e.getMessage())); + } + } + } + if (errors.getErrorCount() > 0) + { + return false; + } + + ShortURLService service = ShortURLService.get(); + if (form.isDelete()) + { + ShortURLRecord shortURLRecord = service.resolveShortURL(shortURL); + if (shortURLRecord == null) + { + throw new NotFoundException("No such short URL: " + shortURL); + } + try + { + service.deleteShortURL(shortURLRecord, getUser()); + } + catch (ValidationException e) + { + errors.addError(new LabKeyError("Error deleting short URL:")); + for(ValidationError error: e.getErrors()) + { + errors.addError(new LabKeyError(error.getMessage())); + } + } + + if (errors.getErrorCount() > 0) + { + return false; + } + } + else + { + ShortURLRecord shortURLRecord = service.saveShortURL(shortURL, fullURL, getUser()); + MutableSecurityPolicy policy = new MutableSecurityPolicy(SecurityPolicyManager.getPolicy(shortURLRecord)); + // Add a role assignment to let another group manage the URL. This grants permission to the journal + // to change where the URL redirects you to after they copy the data + SecurityPolicyManager.savePolicy(policy, getUser()); + } + return true; + } + } + + @AdminConsoleAction + public class ShortURLAdminAction extends AbstractShortURLAdminAction + { + @Override + public ModelAndView getView(ShortURLForm form, boolean reshow, BindException errors) + { + JspView newView = new JspView<>("/org/labkey/core/admin/createNewShortURL.jsp", form, errors); + boolean isAppAdmin = getUser().hasRootPermission(ApplicationAdminPermission.class); + newView.setTitle(isAppAdmin ? "Create New Short URL" : "Short URLs"); + newView.setFrame(WebPartView.FrameType.PORTAL); + + QuerySettings qSettings = new QuerySettings(getViewContext(), "ShortURL", CoreQuerySchema.SHORT_URL_TABLE_NAME); + qSettings.setBaseSort(new Sort("-Created")); + QueryView existingView = new QueryView(new CoreQuerySchema(getUser(), getContainer()), qSettings, null); + if (!isAppAdmin) + { + existingView.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); + } + existingView.setTitle("Existing Short URLs"); + existingView.setFrame(WebPartView.FrameType.PORTAL); + + return new VBox(newView, existingView); + } + + @Override + public URLHelper getSuccessURL(ShortURLForm form) + { + return new ActionURL(ShortURLAdminAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("shortURL"); + addAdminNavTrail(root, "Short URL Admin", getClass()); + } + } + + @RequiresPermission(ApplicationAdminPermission.class) + public class UpdateShortURLAction extends AbstractShortURLAdminAction + { + @Override + public ModelAndView getView(ShortURLForm form, boolean reshow, BindException errors) + { + var shortUrlRecord = ShortURLService.get().resolveShortURL(form.getShortURL()); + if (shortUrlRecord == null) + { + errors.addError(new LabKeyError("Short URL does not exist: " + form.getShortURL())); + return new SimpleErrorView(errors); + } + form.setFullURL(shortUrlRecord.getFullURL()); + + JspView view = new JspView<>("/org/labkey/core/admin/updateShortURL.jsp", form, errors); + view.setTitle("Update Short URL"); + view.setFrame(WebPartView.FrameType.PORTAL); + return view; + } + + @Override + public URLHelper getSuccessURL(ShortURLForm form) + { + return new ActionURL(ShortURLAdminAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("shortURL"); + addAdminNavTrail(root, "Update Short URL", getClass()); + } + } + + // API for reporting client-side exceptions. + // UNDONE: Throttle by IP to avoid DOS from buggy clients. + @Marshal(Marshaller.Jackson) + @SuppressWarnings("UnusedDeclaration") + @RequiresLogin // Issue 52520: Prevent bots from submitting reports + @IgnoresForbiddenProjectCheck // Skip the "forbidden project" check since it disallows root + public static class LogClientExceptionAction extends MutatingApiAction + { + @Override + public Object execute(ExceptionForm form, BindException errors) + { + String errorCode = ExceptionUtil.logClientExceptionToMothership( + form.getStackTrace(), + form.getExceptionMessage(), + form.getBrowser(), + null, + form.getRequestURL(), + form.getReferrerURL(), + form.getUsername() + ); + + Map results = new HashMap<>(); + results.put("errorCode", errorCode); + results.put("loggedToMothership", errorCode != null); + + return success(results); + } + } + + @SuppressWarnings("unused") + public static class ExceptionForm + { + private String _exceptionMessage; + private String _stackTrace; + private String _requestURL; + private String _browser; + private String _username; + private String _referrerURL; + private String _file; + private String _line; + private String _platform; + + public String getExceptionMessage() + { + return _exceptionMessage; + } + + public void setExceptionMessage(String exceptionMessage) + { + _exceptionMessage = exceptionMessage; + } + + public String getUsername() + { + return _username; + } + + public void setUsername(String username) + { + _username = username; + } + + public String getStackTrace() + { + return _stackTrace; + } + + public void setStackTrace(String stackTrace) + { + _stackTrace = stackTrace; + } + + public String getRequestURL() + { + return _requestURL; + } + + public void setRequestURL(String requestURL) + { + _requestURL = requestURL; + } + + public String getBrowser() + { + return _browser; + } + + public void setBrowser(String browser) + { + _browser = browser; + } + + public String getReferrerURL() + { + return _referrerURL; + } + + public void setReferrerURL(String referrerURL) + { + _referrerURL = referrerURL; + } + + public String getFile() + { + return _file; + } + + public void setFile(String file) + { + _file = file; + } + + public String getLine() + { + return _line; + } + + public void setLine(String line) + { + _line = line; + } + + public String getPlatform() + { + return _platform; + } + + public void setPlatform(String platform) + { + _platform = platform; + } + } + + + /** generate URLS to seed web-site scanner */ + @SuppressWarnings("UnusedDeclaration") + @RequiresSiteAdmin + public static class SpiderAction extends SimpleViewAction + { + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Spider Initialization"); + } + + @Override + public ModelAndView getView(Object o, BindException errors) + { + List urls = new ArrayList<>(1000); + + if (getContainer().equals(ContainerManager.getRoot())) + { + for (Container c : ContainerManager.getAllChildren(ContainerManager.getRoot())) + { + urls.add(c.getStartURL(getUser()).toString()); + urls.add(new ActionURL(SpiderAction.class, c).toString()); + } + + Container home = ContainerManager.getHomeContainer(); + for (ActionDescriptor d : SpringActionController.getRegisteredActionDescriptors()) + { + ActionURL url = new ActionURL(d.getControllerName(), d.getPrimaryName(), home); + urls.add(url.toString()); + } + } + else + { + DefaultSchema def = DefaultSchema.get(getUser(), getContainer()); + def.getSchemaNames().forEach(name -> + { + QuerySchema q = def.getSchema(name); + if (null == q) + return; + var tableNames = q.getTableNames(); + if (null == tableNames) + return; + tableNames.forEach(table -> + { + try + { + var t = q.getTable(table); + if (null != t) + { + ActionURL grid = t.getGridURL(getContainer()); + if (null != grid) + urls.add(grid.toString()); + else + urls.add(new ActionURL("query", "executeQuery.view", getContainer()) + .addParameter("schemaName", q.getSchemaName()) + .addParameter("query.queryName", t.getName()) + .toString()); + } + } + catch (Exception x) + { + // pass + } + }); + }); + + ModuleLoader.getInstance().getModules().forEach(m -> + { + ActionURL url = m.getTabURL(getContainer(), getUser()); + if (null != url) + urls.add(url.toString()); + }); + } + + return new HtmlView(DIV(urls.stream().map(url -> createHtmlFragment(A(at(href,url),url),BR())))); + } + } + + @SuppressWarnings("UnusedDeclaration") + @RequiresPermission(TroubleshooterPermission.class) + public static class TestMothershipReportAction extends ReadOnlyApiAction + { + @Override + public Object execute(MothershipReportSelectionForm form, BindException errors) throws Exception + { + MothershipReport report; + MothershipReport.Target target = form.isTestMode() ? MothershipReport.Target.test : MothershipReport.Target.local; + if (MothershipReport.Type.CheckForUpdates.toString().equals(form.getType())) + { + report = UsageReportingLevel.generateReport(UsageReportingLevel.valueOf(form.getLevel()), target); + } + else + { + report = ExceptionUtil.createReportFromThrowable(getViewContext().getRequest(), + new SQLException("Intentional exception for testing purposes", "400"), + (String)getViewContext().getRequest().getAttribute(ViewServlet.ORIGINAL_URL_STRING), + target, + ExceptionReportingLevel.valueOf(form.getLevel()), null, null, null); + } + + final Map params; + if (report == null) + { + params = new LinkedHashMap<>(); + } + else + { + params = report.getJsonFriendlyParams(); + if (form.isSubmit()) + { + report.setForwardedFor(form.getForwardedFor()); + report.run(); + if (null != report.getUpgradeMessage()) + params.put("upgradeMessage", report.getUpgradeMessage()); + } + } + if (form.isDownload()) + { + ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, "metrics.json"); + } + return new ApiSimpleResponse(params); + } + } + + + static class MothershipReportSelectionForm + { + private String _type = MothershipReport.Type.CheckForUpdates.toString(); + private String _level = UsageReportingLevel.ON.toString(); + private boolean _submit = false; + private boolean _download = false; + private String _forwardedFor = null; + // indicates action is being invoked for dev/test + private boolean _testMode = false; + + public String getType() + { + return _type; + } + + public void setType(String type) + { + _type = type; + } + + public String getLevel() + { + return _level; + } + + public void setLevel(String level) + { + _level = StringUtils.upperCase(level); + } + + public boolean isSubmit() + { + return _submit; + } + + public void setSubmit(boolean submit) + { + _submit = submit; + } + + public String getForwardedFor() + { + return _forwardedFor; + } + + public void setForwardedFor(String forwardedFor) + { + _forwardedFor = forwardedFor; + } + + public boolean isTestMode() + { + return _testMode; + } + + public void setTestMode(boolean testMode) + { + _testMode = testMode; + } + + public boolean isDownload() + { + return _download; + } + + public void setDownload(boolean download) + { + _download = download; + } + } + + + @RequiresPermission(TroubleshooterPermission.class) + public class SuspiciousAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + Collection list = BlockListFilter.reportSuspicious(); + HtmlStringBuilder html = HtmlStringBuilder.of(); + if (list.isEmpty()) + { + html.append("No suspicious activity.\n"); + } + else + { + html.unsafeAppend("") + .unsafeAppend("\n"); + for (BlockListFilter.Suspicious s : list) + { + html.unsafeAppend("\n"); + } + html.unsafeAppend("
    host (user)user-agentcount
    ") + .append(s.host); + if (!isBlank(s.user)) + html.append(HtmlString.NBSP).append("(" + s.user + ")"); + html.unsafeAppend("") + .append(s.userAgent) + .unsafeAppend("") + .append(s.count) + .unsafeAppend("
    "); + } + return new HtmlView(html); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Suspicious activity", SuspiciousAction.class); + } + } + + /** This is a very crude API right now, mostly using default serialization of pre-existing objects + * NOTE: callers should expect that the return shape of this method may and will change in non-backward-compatible ways + */ + @Marshal(Marshaller.Jackson) + @RequiresNoPermission + @AllowedBeforeInitialUserIsSet + public static class ConfigurationSummaryAction extends ReadOnlyApiAction + { + @Override + public Object execute(Object o, BindException errors) + { + if (!getContainer().isRoot()) + throw new NotFoundException("Must be invoked in the root"); + + // requires site-admin, unless there are no users + if (!UserManager.hasNoRealUsers() && !getContainer().hasPermission(getUser(), AdminOperationsPermission.class)) + throw new UnauthorizedException(); + + return getConfigurationJson(); + } + + @Override + protected ObjectMapper createResponseObjectMapper() + { + ObjectMapper result = JsonUtil.createDefaultMapper(); + result.addMixIn(ExternalScriptEngineDefinitionImpl.class, IgnorePasswordMixIn.class); + return result; + } + + /* returns a jackson serializable object that reports superset of information returned in admin console */ + private JSONObject getConfigurationJson() + { + JSONObject res = new JSONObject(); + + res.put("server", AdminBean.getPropertyMap()); + + final Map> sets = new TreeMap<>(); + new SqlSelector(CoreSchema.getInstance().getScope(), + new SQLFragment("SELECT category, name, value FROM prop.propertysets PS inner join prop.properties P on PS.\"set\" = P.\"set\"\n" + + "WHERE objectid = ? AND category IN ('SiteConfig') AND encryption='None' AND LOWER(name) NOT LIKE '%password%'", ContainerManager.getRoot())).forEachMap(m -> + { + String category = (String)m.get("category"); + String name = (String)m.get("name"); + Object value = m.get("value"); + if (!sets.containsKey(category)) + sets.put(category, new TreeMap<>()); + sets.get(category).put(name,value); + } + ); + res.put("siteSettings", sets); + + HealthCheck.Result result = HealthCheckRegistry.get().checkHealth(Arrays.asList("all")); + res.put("health", result); + + LabKeyScriptEngineManager mgr = LabKeyScriptEngineManager.get(); + res.put("scriptEngines", mgr.getEngineDefinitions()); + + return res; + } + } + + @JsonIgnoreProperties(value = { "password", "changePassword", "configuration" }) + private static class IgnorePasswordMixIn + { + } + + @AdminConsoleAction() + public class AllowListAction extends FormViewAction + { + private AllowListType _type; + + @Override + public void validateCommand(AllowListForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(AllowListForm form, boolean reshow, BindException errors) + { + _type = form.getTypeEnum(); + + form.setExistingValuesList(form.getTypeEnum().getValues()); + + JspView newView = new JspView<>("/org/labkey/core/admin/addNewListValue.jsp", form, errors); + newView.setTitle("Register New " + form.getTypeEnum().getTitle()); + newView.setFrame(WebPartView.FrameType.PORTAL); + JspView existingView = new JspView<>("/org/labkey/core/admin/existingListValues.jsp", form, errors); + existingView.setTitle("Existing " + form.getTypeEnum().getTitle() + "s"); + existingView.setFrame(WebPartView.FrameType.PORTAL); + + return new VBox(newView, existingView); + } + + @Override + public boolean handlePost(AllowListForm form, BindException errors) throws Exception + { + AllowListType allowListType = form.getTypeEnum(); + //handle delete of existing value + if (form.isDelete()) + { + String urlToDelete = form.getExistingValue(); + List values = new ArrayList<>(allowListType.getValues()); + for (String value : values) + { + if (null != urlToDelete && urlToDelete.trim().equalsIgnoreCase(value.trim())) + { + values.remove(value); + allowListType.setValues(values, getUser()); + break; + } + } + } + //handle updates - clicking on Save button under Existing will save the updated urls + else if (form.isSaveAll()) + { + Set validatedValues = form.validateValues(errors); + if (errors.hasErrors()) + return false; + + allowListType.setValues(validatedValues.stream().toList(), getUser()); + } + //save new external value + else if (form.isSaveNew()) + { + Set valueSet = form.validateNewValue(errors); + if (errors.hasErrors()) + return false; + + allowListType.setValues(valueSet, getUser()); + } + + return true; + } + + @Override + public URLHelper getSuccessURL(AllowListForm form) + { + return form.getTypeEnum().getSuccessURL(getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic(_type.getHelpTopic()); + addAdminNavTrail(root, String.format("%1$s Admin", _type.getTitle()), getClass()); + } + } + + public static class AllowListForm + { + private String _newValue; + private String _existingValue; + private boolean _delete; + private String _existingValues; + private boolean _saveAll; + private boolean _saveNew; + private String _type; + + private List _existingValuesList; + + public String getNewValue() + { + return _newValue; + } + + @SuppressWarnings("unused") + public void setNewValue(String newValue) + { + _newValue = newValue; + } + + public String getExistingValue() + { + return _existingValue; + } + + @SuppressWarnings("unused") + public void setExistingValue(String existingValue) + { + _existingValue = existingValue; + } + + public boolean isDelete() + { + return _delete; + } + + @SuppressWarnings("unused") + public void setDelete(boolean delete) + { + _delete = delete; + } + + public String getExistingValues() + { + return _existingValues; + } + + @SuppressWarnings("unused") + public void setExistingValues(String existingValues) + { + _existingValues = existingValues; + } + + public boolean isSaveAll() + { + return _saveAll; + } + + @SuppressWarnings("unused") + public void setSaveAll(boolean saveAll) + { + _saveAll = saveAll; + } + + public boolean isSaveNew() + { + return _saveNew; + } + + @SuppressWarnings("unused") + public void setSaveNew(boolean saveNew) + { + _saveNew = saveNew; + } + + public List getExistingValuesList() + { + //for updated urls that comes in as String values from the jsp/html form + if (null != getExistingValues()) + { + // The JavaScript delimits with "\n". Not sure where these "\r"s are coming from, but we need to strip them. + return new ArrayList<>(Arrays.asList(getExistingValues().replace("\r", "").split("\n"))); + } + return _existingValuesList; + } + + public void setExistingValuesList(List valuesList) + { + _existingValuesList = valuesList; + } + + public String getType() + { + return _type; + } + + @SuppressWarnings("unused") + public void setType(String type) + { + _type = type; + } + + @NotNull + public AllowListType getTypeEnum() + { + return EnumUtils.getEnum(AllowListType.class, getType(), AllowListType.Redirect); + } + + @JsonIgnore + public Set validateNewValue(BindException errors) + { + String value = StringUtils.trimToEmpty(getNewValue()); + getTypeEnum().validateValueFormat(value, errors); + if (errors.hasErrors()) + return null; + + Set valueSet = new CaseInsensitiveHashSet(getTypeEnum().getValues()); + checkDuplicatesByAddition(value, valueSet, errors); + return valueSet; + } + + @JsonIgnore + public Set validateValues(BindException errors) + { + List values = getExistingValuesList(); //get values from the form, this includes updated values + Set valueSet = new CaseInsensitiveHashSet(); + + if (null != values && !values.isEmpty()) + { + for (String value : values) + { + getTypeEnum().validateValueFormat(value, errors); + if (errors.hasErrors()) + continue; + + checkDuplicatesByAddition(value, valueSet, errors); + } + } + + return valueSet; + } + + /** + * Adds value to value set unless it is a duplicate, in which case it adds an error + * @param value to check + * @param valueSet of existing values + * @param errors collections of errors observed + */ + @JsonIgnore + private void checkDuplicatesByAddition(String value, Set valueSet, BindException errors) + { + String trimValue = StringUtils.trimToEmpty(value); + if (!valueSet.add(trimValue)) + errors.addError(new LabKeyError(String.format("'%1$s' already exists. Duplicate values not allowed.", trimValue))); + } + } + + @AdminConsoleAction + public static class DeleteAllValuesAction extends FormHandlerAction + { + @Override + public void validateCommand(AllowListForm form, Errors errors) + { + } + + @Override + public boolean handlePost(AllowListForm form, BindException errors) throws Exception + { + form.getTypeEnum().setValues(Collections.emptyList(), getUser()); + return true; + } + + @Override + public URLHelper getSuccessURL(AllowListForm form) + { + return form.getTypeEnum().getSuccessURL(getContainer()); + } + } + + public static class ExternalSourcesForm + { + private boolean _delete; + private boolean _saveNew; + private boolean _saveAll; + + private String _newDirective; + private String _newHost; + private String _existingValue; + private String _existingValues; + + public boolean isDelete() + { + return _delete; + } + + @SuppressWarnings("unused") + public void setDelete(boolean delete) + { + _delete = delete; + } + + public boolean isSaveNew() + { + return _saveNew; + } + + @SuppressWarnings("unused") + public void setSaveNew(boolean saveNew) + { + _saveNew = saveNew; + } + + public boolean isSaveAll() + { + return _saveAll; + } + + @SuppressWarnings("unused") + public void setSaveAll(boolean saveAll) + { + _saveAll = saveAll; + } + + public String getNewDirective() + { + return _newDirective; + } + + @SuppressWarnings("unused") + public void setNewDirective(String newDirective) + { + _newDirective = newDirective; + } + + public String getNewHost() + { + return _newHost; + } + + @SuppressWarnings("unused") + public void setNewHost(String newHost) + { + _newHost = newHost; + } + + public String getExistingValue() + { + return _existingValue; + } + + @SuppressWarnings("unused") + public void setExistingValue(String existingValue) + { + _existingValue = existingValue; + } + + public List getExistingValues() + { + return Arrays.stream(StringUtils.trimToEmpty(_existingValues).split("\n")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .toList(); + } + + @SuppressWarnings("unused") + public void setExistingValues(String existingValues) + { + _existingValues = existingValues; + } + + private AllowedHost getExistingAllowedHost(BindException errors) + { + return getAllowedHost(getExistingValue(), errors); + } + + private AllowedHost getAllowedHost(String value, BindException errors) + { + String[] parts = value.split("\\|", 2); // Stop after the first bar to produce two parts + if (parts.length != 2) + { + errors.addError(new LabKeyError("Can't parse allowed host.")); + return null; + } + return validateHost(parts[0], parts[1], errors); + } + + private List getExistingAllowedHosts(BindException errors) + { + List existing = getExistingValues().stream() + .map(value-> getAllowedHost(value, errors)) + .toList(); + + if (errors.hasErrors()) + return null; + + return checkDuplicates(existing, errors); + } + + private List validateNewAllowedHost(BindException errors) throws JsonProcessingException + { + AllowedHost newAllowedHost = validateHost(getNewDirective(), getNewHost(), errors); + + if (errors.hasErrors()) + return null; + + List hosts = getSavedAllowedHosts(); + hosts.add(newAllowedHost); + + return checkDuplicates(hosts, errors); + } + + // Lenient for now: no unknown directives, no blank hosts or hosts with semicolons + public static AllowedHost validateHost(String directiveString, String host, BindException errors) + { + AllowedHost ret = null; + + if (StringUtils.isEmpty(directiveString)) + { + errors.addError(new LabKeyError("Directive must not be blank")); + } + else if (StringUtils.isEmpty(host)) + { + errors.addError(new LabKeyError("Host must not be blank")); + } + else if (host.contains(";")) + { + errors.addError(new LabKeyError("Semicolons are not allowed in host names")); + } + else + { + Directive directive = EnumUtils.getEnum(Directive.class, directiveString); + + if (null == directive) + { + errors.addError(new LabKeyError("Unknown directive: " + directiveString)); + } + else + { + ret = new AllowedHost(directive, host.trim()); + } + } + + return ret; + } + + /** + * Check for duplicates in hosts: within each Directive, hosts are checked using case-insensitive comparisons + + * @param hosts a list of AllowedHost objects to check for duplicates + * @param errors errors to populate + * @return hosts if there are no duplicates, otherwise {@code null} + */ + public static @Nullable List checkDuplicates(List hosts, BindException errors) + { + // Not a simple Set check since we want host check to be case-insensitive + MultiValuedMap map = new CaseInsensitiveHashSetValuedMap<>(); + + hosts.forEach(allowedHost -> { + String host = allowedHost.host().trim(); + if (!map.put(allowedHost.directive(), host)) + errors.addError(new LabKeyError(String.format("'%1$s' already exists. Duplicate values are not allowed.", allowedHost))); + }); + + return errors.hasErrors() ? null : hosts; + } + + // Returns a mutable list + public List getSavedAllowedHosts() throws JsonProcessingException + { + return AllowedExternalResourceHosts.readAllowedHosts(); + } + } + + @AdminConsoleAction() + public class ExternalSourcesAction extends FormViewAction + { + @Override + public void validateCommand(ExternalSourcesForm form, Errors errors) + { + } + + @Override + public ModelAndView getView(ExternalSourcesForm form, boolean reshow, BindException errors) + { + boolean isTroubleshooter = !getContainer().hasPermission(getUser(), ApplicationAdminPermission.class); + + JspView newView = new JspView<>("/org/labkey/core/admin/addNewExternalSource.jsp", form, errors); + newView.setTitle(isTroubleshooter ? "Overview" : "Register New External Resource Host"); + newView.setFrame(WebPartView.FrameType.PORTAL); + JspView existingView = new JspView<>("/org/labkey/core/admin/existingExternalSources.jsp", form, errors); + existingView.setTitle("Existing External Resource Hosts"); + existingView.setFrame(WebPartView.FrameType.PORTAL); + + return new VBox(newView, existingView); + } + + private static final Object HOST_LOCK = new Object(); + + @Override + public boolean handlePost(ExternalSourcesForm form, BindException errors) throws Exception + { + List allowedHosts = null; + + // Multiple requests could access this in parallel, so synchronize access, Issue 53457 + synchronized (HOST_LOCK) + { + //handle delete of an existing value + if (form.isDelete()) + { + AllowedHost subToDelete = form.getExistingAllowedHost(errors); + if (errors.hasErrors()) + return false; + allowedHosts = form.getSavedAllowedHosts(); + var iter = allowedHosts.listIterator(); + while (iter.hasNext()) + { + AllowedHost sub = iter.next(); + if (sub.equals(subToDelete)) + { + iter.remove(); + break; + } + } + } + //handle updates - clicking on Save button under Existing will save the updated hosts + else if (form.isSaveAll()) + { + allowedHosts = form.getExistingAllowedHosts(errors); + if (errors.hasErrors()) + return false; + } + //save new external value + else if (form.isSaveNew()) + { + allowedHosts = form.validateNewAllowedHost(errors); + } + + if (errors.hasErrors()) + return false; + + AllowedExternalResourceHosts.saveAllowedHosts(allowedHosts, getUser()); + } + + return true; + } + + @Override + public URLHelper getSuccessURL(ExternalSourcesForm form) + { + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("externalHosts"); + addAdminNavTrail(root, "Allowed External Resource Hosts", getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ProjectSettingsAction extends ProjectSettingsViewPostAction + { + @Override + protected LookAndFeelView getTabView(ProjectSettingsForm form, boolean reshow, BindException errors) + { + return new LookAndFeelView(errors); + } + + @Override + public void validateCommand(ProjectSettingsForm form, Errors errors) + { + } + + @Override + public boolean handlePost(ProjectSettingsForm form, BindException errors) throws Exception + { + return saveProjectSettings(getContainer(), getUser(), form, errors); + } + } + + private static boolean saveProjectSettings(Container c, User user, ProjectSettingsForm form, BindException errors) + { + WriteableLookAndFeelProperties props = LookAndFeelProperties.getWriteableInstance(c); + boolean hasAdminOpsPerm = c.hasPermission(user, AdminOperationsPermission.class); + + // Site-only properties + + if (c.isRoot()) + { + DateParsingMode dateParsingMode = DateParsingMode.fromString(form.getDateParsingMode()); + props.setDateParsingMode(dateParsingMode); + + if (hasAdminOpsPerm) + { + String customWelcome = form.getCustomWelcome(); + String welcomeUrl = StringUtils.trimToNull(customWelcome); + if ("/".equals(welcomeUrl) || AppProps.getInstance().getContextPath().equalsIgnoreCase(welcomeUrl)) + { + errors.reject(SpringActionController.ERROR_MSG, "Invalid welcome URL. The url cannot equal '/' or the contextPath (" + AppProps.getInstance().getContextPath() + ")"); + } + else + { + props.setCustomWelcome(welcomeUrl); + } + } + } + + // Site & project properties + + boolean shouldInherit = form.getShouldInherit(); + if (shouldInherit != SecurityManager.shouldNewSubfoldersInheritPermissions(c)) + { + SecurityManager.setNewSubfoldersInheritPermissions(c, user, shouldInherit); + } + + setProperty(form.isSystemDescriptionInherited(), props::clearSystemDescription, () -> props.setSystemDescription(form.getSystemDescription())); + setProperty(form.isSystemShortNameInherited(), props::clearSystemShortName, () -> props.setSystemShortName(form.getSystemShortName())); + setProperty(form.isThemeNameInherited(), props::clearThemeName, () -> props.setThemeName(form.getThemeName())); + setProperty(form.isFolderDisplayModeInherited(), props::clearFolderDisplayMode, () -> props.setFolderDisplayMode(FolderDisplayMode.fromString(form.getFolderDisplayMode()))); + setProperty(form.isApplicationMenuDisplayModeInherited(), props::clearApplicationMenuDisplayMode, () -> props.setApplicationMenuDisplayMode(FolderDisplayMode.fromString(form.getApplicationMenuDisplayMode()))); + setProperty(form.isHelpMenuEnabledInherited(), props::clearHelpMenuEnabled, () -> props.setHelpMenuEnabled(form.isHelpMenuEnabled())); + setProperty(form.isDiscussionEnabledInherited(), props::clearDiscussionEnabled, () -> props.setDiscussionEnabled(form.isDiscussionEnabled())); + + // a few properties on this page should be restricted to operational permissions (i.e. site admin) + if (hasAdminOpsPerm) + { + setProperty(form.isSystemEmailAddressInherited(), props::clearSystemEmailAddress, () -> { + String systemEmailAddress = form.getSystemEmailAddress(); + try + { + // this will throw an InvalidEmailException for invalid email addresses + ValidEmail email = new ValidEmail(systemEmailAddress); + props.setSystemEmailAddress(email); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(SpringActionController.ERROR_MSG, "Invalid System Email Address: [" + + e.getBadEmail() + "]. Please enter a valid email address."); + } + }); + + setProperty(form.isCustomLoginInherited(), props::clearCustomLogin, () -> { + String customLogin = form.getCustomLogin(); + if (props.isValidUrl(customLogin)) + { + props.setCustomLogin(customLogin); + } + else + { + errors.reject(SpringActionController.ERROR_MSG, "Invalid login URL. Should be in the form -."); + } + }); + } + + setProperty(form.isCompanyNameInherited(), props::clearCompanyName, () -> props.setCompanyName(form.getCompanyName())); + setProperty(form.isLogoHrefInherited(), props::clearLogoHref, () -> props.setLogoHref(form.getLogoHref())); + setProperty(form.isReportAProblemPathInherited(), props::clearReportAProblemPath, () -> props.setReportAProblemPath(form.getReportAProblemPath())); + setProperty(form.isSupportEmailInherited(), props::clearSupportEmail, () -> { + String supportEmail = form.getSupportEmail(); + + if (!isBlank(supportEmail)) + { + try + { + // this will throw an InvalidEmailException for invalid email addresses + ValidEmail email = new ValidEmail(supportEmail); + props.setSupportEmail(email.toString()); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(SpringActionController.ERROR_MSG, "Invalid Support Email Address: [" + + e.getBadEmail() + "]. Please enter a valid email address."); + } + } + else + { + // This stores a blank value, not null (which would mean inherit) + props.setSupportEmail(null); + } + }); + + boolean noErrors = !saveFolderSettings(c, user, props, form, errors); + + if (noErrors) + { + // Bump the look & feel revision so browsers retrieve the new theme stylesheet + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + } + + return noErrors; + } + + private static void setProperty(boolean inherited, Runnable clear, Runnable set) + { + if (inherited) + clear.run(); + else + set.run(); + } + + // Same as ProjectSettingsAction, but provides special admin console permissions handling + @AdminConsoleAction(ApplicationAdminPermission.class) + public static class LookAndFeelSettingsAction extends ProjectSettingsAction + { + @Override + protected TYPE getType() + { + return TYPE.LookAndFeelSettings; + } + } + + @RequiresPermission(AdminPermission.class) + public static class UpdateContainerSettingsAction extends MutatingApiAction + { + @Override + public Object execute(FolderSettingsForm form, BindException errors) + { + boolean saved = saveFolderSettings(getContainer(), getUser(), LookAndFeelProperties.getWriteableFolderInstance(getContainer()), form, errors); + + ApiSimpleResponse response = new ApiSimpleResponse(); + response.put("success", saved && !errors.hasErrors()); + return response; + } + } + + @RequiresPermission(AdminPermission.class) + public static class ResourcesAction extends ProjectSettingsViewPostAction + { + @Override + protected JspView getTabView(Object o, boolean reshow, BindException errors) + { + LookAndFeelBean bean = new LookAndFeelBean(); + return new JspView<>("/org/labkey/core/admin/lookAndFeelResources.jsp", bean, errors); + } + + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) + { + Container c = getContainer(); + Map fileMap = getFileMap(); + + for (ResourceType type : ResourceType.values()) + { + MultipartFile file = fileMap.get(type.name()); + + if (file != null && !file.isEmpty()) + { + try + { + type.save(file, c, getUser()); + } + catch (Exception e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + return false; + } + } + } + + // Note that audit logging happens via the attachment code, so we don't log separately here + + // Bump the look & feel revision so browsers retrieve the new logo, custom stylesheet, etc. + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + + return true; + } + } + + // Same as ResourcesAction, but provides special admin console permissions handling + @AdminConsoleAction + public static class AdminConsoleResourcesAction extends ResourcesAction + { + @Override + protected TYPE getType() + { + return TYPE.LookAndFeelSettings; + } + } + + @RequiresPermission(AdminPermission.class) + public static class MenuBarAction extends ProjectSettingsViewAction + { + @Override + protected HttpView getTabView() + { + if (getContainer().isRoot()) + return HtmlView.err("Menu bar must be configured for each project separately."); + + WebPartView v = new JspView<>("/org/labkey/core/admin/editMenuBar.jsp", null); + v.setView("menubar", new VBox()); + Portal.populatePortalView(getViewContext(), Portal.DEFAULT_PORTAL_PAGE_ID, v, false, true, true, false); + + return v; + } + } + + @RequiresPermission(AdminPermission.class) + public static class FilesAction extends ProjectSettingsViewPostAction + { + @Override + protected HttpView getTabView(FilesForm form, boolean reshow, BindException errors) + { + Container c = getContainer(); + + if (c.isRoot()) + return HtmlView.err("Files must be configured for each project separately."); + + if (!reshow || form.isPipelineRootForm()) + { + try + { + AdminController.setFormAndConfirmMessage(getViewContext(), form); + } + catch (IllegalArgumentException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + } + VBox box = new VBox(); + JspView view = new JspView<>("/org/labkey/core/admin/view/filesProjectSettings.jsp", form, errors); + String title = "Configure File Root"; + if (CloudStoreService.get() != null) + title += " And Enable Cloud Stores"; + view.setTitle(title); + box.addView(view); + + // only site admins (i.e. AdminOperationsPermission) can configure the pipeline root + if (c.hasPermission(getViewContext().getUser(), AdminOperationsPermission.class)) + { + SetupForm setupForm = SetupForm.init(c); + setupForm.setShowAdditionalOptionsLink(true); + setupForm.setErrors(errors); + PipeRoot pipeRoot = SetupForm.getPipelineRoot(c); + + if (pipeRoot != null) + { + for (String errorMessage : pipeRoot.validate()) + errors.addError(new LabKeyError(errorMessage)); + } + JspView pipelineView = (JspView) PipelineService.get().getSetupView(setupForm); + pipelineView.setTitle("Configure Data Processing Pipeline"); + box.addView(pipelineView); + } + + return box; + } + + @Override + public void validateCommand(FilesForm form, Errors errors) + { + if (!form.isPipelineRootForm() && !form.isDisableFileSharing() && !form.hasSiteDefaultRoot() && !form.isCloudFileRoot()) + { + String root = StringUtils.trimToNull(form.getFolderRootPath()); + if (root != null) + { + File f = new File(root); + if (!f.exists() || !f.isDirectory()) + { + errors.reject(SpringActionController.ERROR_MSG, "File root '" + root + "' does not appear to be a valid directory accessible to the server at " + getViewContext().getRequest().getServerName() + "."); + } + } + else + errors.reject(SpringActionController.ERROR_MSG, "A Project specified file root cannot be blank, to disable file sharing for this project, select the disable option."); + } + else if (form.isCloudFileRoot()) + { + AdminController.validateCloudFileRoot(form, getContainer(), errors); + } + } + + @Override + public boolean handlePost(FilesForm form, BindException errors) throws Exception + { + FileContentService service = FileContentService.get(); + if (service != null) + { + if (form.isPipelineRootForm()) + return PipelineService.get().savePipelineSetup(getViewContext(), form, errors); + else + { + AdminController.setFileRootFromForm(getViewContext(), form, errors); + } + } + + // Cloud settings + AdminController.setEnabledCloudStores(getViewContext(), form, errors); + + return !errors.hasErrors(); + } + + @Override + public URLHelper getSuccessURL(FilesForm form) + { + ActionURL url = new AdminController.AdminUrlsImpl().getProjectSettingsFileURL(getContainer()); + if (form.isPipelineRootForm()) + { + url.addParameter("piperootSet", true); + } + else + { + if (form.isFileRootChanged()) + url.addParameter("rootSet", form.getMigrateFilesOption()); + if (form.isEnabledCloudStoresChanged()) + url.addParameter("cloudChanged", true); + } + return url; + } + } + + public static class LookAndFeelView extends JspView + { + LookAndFeelView(BindException errors) + { + super("/org/labkey/core/admin/lookAndFeelProperties.jsp", new LookAndFeelBean(), errors); + } + } + + + public static class LookAndFeelBean + { + public final HtmlString helpLink = new HelpTopic("customizeLook").getSimpleLinkHtml("more info..."); + public final HtmlString welcomeLink = new HelpTopic("customizeLook").getSimpleLinkHtml("more info..."); + public final HtmlString customColumnRestrictionHelpLink = new HelpTopic("chartTrouble").getSimpleLinkHtml("more info..."); + } + + @RequiresPermission(AdminPermission.class) + public static class AdjustSystemTimestampsAction extends FormViewAction + { + @Override + public void addNavTrail(NavTree root) + { + } + + @Override + public void validateCommand(AdjustTimestampsForm form, Errors errors) + { + if (form.getHourDelta() == null || form.getHourDelta() == 0) + errors.reject(ERROR_MSG, "You must specify a non-zero value for 'Hour Delta'"); + } + + @Override + public ModelAndView getView(AdjustTimestampsForm form, boolean reshow, BindException errors) throws Exception + { + return new JspView<>("/org/labkey/core/admin/adjustTimestamps.jsp", form, errors); + } + + private void updateFields(TableInfo tInfo, Collection fieldNames, int delta) + { + SQLFragment sql = new SQLFragment(); + DbSchema schema = tInfo.getSchema(); + String comma = ""; + List updating = new ArrayList<>(); + + for (String fieldName: fieldNames) + { + ColumnInfo col = tInfo.getColumn(FieldKey.fromParts(fieldName)); + if (col != null && col.getJdbcType() == JdbcType.TIMESTAMP) + { + updating.add(fieldName); + if (sql.isEmpty()) + sql.append("UPDATE ").append(tInfo, "").append(" SET "); + sql.append(comma) + .append(String.format(" %s = {fn timestampadd(SQL_TSI_HOUR, %d, %s)}", col.getSelectIdentifier(), delta, col.getSelectIdentifier())); + comma = ", "; + } + } + + if (!sql.isEmpty()) + { + logger.info(String.format("Updating %s in table %s.%s", updating, schema.getName(), tInfo.getName())); + logger.debug(sql.toDebugString()); + int numRows = new SqlExecutor(schema).execute(sql); + logger.info(String.format("Updated %d rows for table %s.%s", numRows, schema.getName(), tInfo.getName())); + } + } + + @Override + public boolean handlePost(AdjustTimestampsForm form, BindException errors) throws Exception + { + List toUpdate = Arrays.asList("Created", "Modified", "lastIndexed", "diCreated", "diModified"); + logger.info("Adjusting all " + toUpdate + " timestamp fields in all tables by " + form.getHourDelta() + " hours."); + DbScope scope = DbScope.getLabKeyScope(); + try (DbScope.Transaction t = scope.ensureTransaction()) + { + ModuleLoader.getInstance().getModules().forEach(module -> { + logger.info("==> Beginning update of timestamps for module: " + module.getName()); + module.getSchemaNames().stream().sorted().forEach(schemaName -> { + DbSchema schema = DbSchema.get(schemaName, DbSchemaType.Module); + scope.invalidateSchema(schema); // Issue 44452: assure we have a fresh set of tables to work from + schema.getTableNames().forEach(tableName -> { + TableInfo tInfo = schema.getTable(tableName); + if (tInfo.getTableType() == DatabaseTableType.TABLE) + { + updateFields(tInfo, toUpdate, form.getHourDelta()); + } + }); + }); + logger.info("<== DONE updating timestamps for module: " + module.getName()); + }); + t.commit(); + } + logger.info("DONE Adjusting all " + toUpdate + " timestamp fields in all tables by " + form.getHourDelta() + " hours."); + return true; + } + + @Override + public URLHelper getSuccessURL(AdjustTimestampsForm adjustTimestampsForm) + { + return PageFlowUtil.urlProvider(AdminUrls.class).getAdminConsoleURL(); + } + } + + public static class AdjustTimestampsForm + { + private Integer hourDelta; + + public Integer getHourDelta() + { + return hourDelta; + } + + public void setHourDelta(Integer hourDelta) + { + this.hourDelta = hourDelta; + } + } + + @RequiresPermission(TroubleshooterPermission.class) + public class ViewUsageStatistics extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("ViewUsageStatistics")); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Usage Statistics", this.getClass()); + } + } + + private static final URI LABKEY_ORG_REPORT_ACTION; + + static + { + LABKEY_ORG_REPORT_ACTION = URI.create("https://www.labkey.org/admin-contentSecurityPolicyReport.api"); + } + + @RequiresNoPermission + @CSRF(CSRF.Method.NONE) + public static class ContentSecurityPolicyReportAction extends ReadOnlyApiAction + { + private static final Logger _log = LogHelper.getLogger(ContentSecurityPolicyReportAction.class, "CSP warnings"); + + // recent reports, to help avoid log spam + private static final Map reports = Collections.synchronizedMap(new LRUMap<>(20)); + + @Override + public Object execute(SimpleApiJsonForm form, BindException errors) throws Exception + { + var ret = new JSONObject().put("success", true); + + // fail fast + if (!_log.isWarnEnabled()) + return ret; + + var request = getViewContext().getRequest(); + assert null != request; + + var userAgent = request.getHeader("User-Agent"); + if (PageFlowUtil.isRobotUserAgent(userAgent) && !_log.isDebugEnabled()) + return ret; + + // NOTE User may be "guest", and will always be guest if being relayed to labkey.org + var jsonObj = form.getJsonObject(); + if (null != jsonObj) + { + JSONObject cspReport = jsonObj.optJSONObject("csp-report"); + if (cspReport != null) + { + String blockedUri = cspReport.optString("blocked-uri", null); + + // Issue 52933 - suppress base-uri problems from a crawler or bot on labkey.org + if (blockedUri != null && + blockedUri.startsWith("https://labkey.org%2C") && + blockedUri.endsWith("undefined") && + !_log.isDebugEnabled()) + { + return ret; + } + + String urlString = cspReport.optString("document-uri", null); + if (urlString != null) + { + String path = new URLHelper(urlString).deleteParameters().getURIString(); + if (null == reports.put(path, Boolean.TRUE) || _log.isDebugEnabled()) + { + // Don't modify forwarded reports; they already have user, ip, user-agent, etc. from the forwarding server. + boolean forwarded = jsonObj.optBoolean("forwarded", false); + if (!forwarded) + { + User user = getUser(); + String email = null; + // If the user is not logged in, we may still be able to snag the email address from our cookie + if (user.isGuest()) + email = LoginController.getEmailFromCookie(getViewContext().getRequest()); + if (null == email) + email = user.getEmail(); + jsonObj.put("user", email); + String ipAddress = request.getHeader("X-FORWARDED-FOR"); + if (ipAddress == null) + ipAddress = request.getRemoteAddr(); + jsonObj.put("ip", ipAddress); + if (isNotBlank(userAgent)) + jsonObj.put("user-agent", userAgent); + String labkeyVersion = request.getParameter("labkeyVersion"); + if (null != labkeyVersion) + jsonObj.put("labkeyVersion", labkeyVersion); + String cspVersion = request.getParameter("cspVersion"); + if (null != cspVersion) + jsonObj.put("cspVersion", cspVersion); + } + + var jsonStr = jsonObj.toString(2); + _log.warn("ContentSecurityPolicy warning on page: {}\n{}", urlString, jsonStr); + + if (!forwarded && OptionalFeatureService.get().isFeatureEnabled(ContentSecurityPolicyFilter.FEATURE_FLAG_FORWARD_CSP_REPORTS)) + { + jsonObj.put("forwarded", true); + + // Create an HttpClient + HttpClient client = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .build(); + + // Create the POST request + HttpRequest remoteRequest = HttpRequest.newBuilder() + .uri(LABKEY_ORG_REPORT_ACTION) + .header("Content-Type", request.getContentType()) // Use whatever the browser set + .POST(HttpRequest.BodyPublishers.ofString(jsonObj.toString(2))) + .build(); + + // Send the request and get the response + HttpResponse response = client.send(remoteRequest, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) + { + _log.error("ContentSecurityPolicy report forwarding to https://www.labkey.org failed: {}\n{}", response.statusCode(), response.body()); + } + else + { + JSONObject jsonResponse = new JSONObject(response.body()); + boolean success = jsonResponse.optBoolean("success", false); + if (!success) + { + _log.error("ContentSecurityPolicy report forwarding to https://www.labkey.org failed: {}", jsonResponse); + } + } + } + } + } + } + } + return ret; + } + } + + + public static class TestCase extends AbstractActionPermissionTest + { + @Override + @Test + public void testActionPermissions() + { + User user = TestContext.get().getUser(); + assertTrue(user.hasSiteAdminPermission()); + + AdminController controller = new AdminController(); + + // @RequiresPermission(ReadPermission.class) + assertForReadPermission(user, false, + new GetModulesAction(), + new GetFolderTabsAction(), + new ClearDeletedTabFoldersAction() + ); + + // @RequiresPermission(DeletePermission.class) + assertForUpdateOrDeletePermission(user, + new DeleteFolderAction() + ); + + // @RequiresPermission(AdminPermission.class) + assertForAdminPermission(user, + controller.new CustomizeEmailAction(), + controller.new FolderAliasesAction(), + controller.new MoveFolderAction(), + controller.new MoveTabAction(), + controller.new RenameFolderAction(), + controller.new ReorderFoldersAction(), + controller.new ReorderFoldersApiAction(), + controller.new SiteValidationAction(), + new AddTabAction(), + new ConfirmProjectMoveAction(), + new CreateFolderAction(), + new CustomizeMenuAction(), + new DeleteCustomEmailAction(), + new FilesAction(), + new MenuBarAction(), + new ProjectSettingsAction(), + new RenameContainerAction(), + new RenameTabAction(), + new ResetPropertiesAction(), + new ResetQueryStatisticsAction(), + new ResetResourceAction(), + new ResourcesAction(), + new RevertFolderAction(), + new SetFolderPermissionsAction(), + new SetInitialFolderSettingsAction(), + new ShowTabAction() + ); + + //TODO @RequiresPermission(AdminReadPermission.class) + //controller.new TestMothershipReportAction() + + // @RequiresPermission(AdminOperationsPermission.class) + assertForAdminOperationsPermission(ContainerManager.getRoot(), user, + controller.new DbCheckerAction(), + controller.new DeleteModuleAction(), + controller.new DoCheckAction(), + controller.new EmailTestAction(), + controller.new ShowNetworkDriveTestAction(), + controller.new ValidateDomainsAction(), + new OptionalFeatureAction(), + new GetSchemaXmlDocAction(), + new RecreateViewsAction() + ); + + // @AdminConsoleAction + assertForAdminPermission(ContainerManager.getRoot(), user, + controller.new ActionsAction(), + controller.new CachesAction(), + controller.new ConfigureSystemMaintenanceAction(), + controller.new CustomizeSiteAction(), + controller.new DumpHeapAction(), + controller.new ExecutionPlanAction(), + controller.new FolderTypesAction(), + controller.new MemTrackerAction(), + controller.new ModulesAction(), + controller.new QueriesAction(), + controller.new QueryStackTracesAction(), + controller.new ResetErrorMarkAction(), + controller.new ShortURLAdminAction(), + controller.new ShowAllErrorsAction(), + controller.new ShowErrorsSinceMarkAction(), + controller.new ShowPrimaryLogAction(), + controller.new ShowCspReportLogAction(), + controller.new ShowThreadsAction(), + new ExportActionsAction(), + new ExportQueriesAction(), + new MemoryChartAction(), + new ShowAdminAction() + ); + + // @RequiresSiteAdmin + assertForRequiresSiteAdmin(user, + controller.new EnvironmentVariablesAction(), + controller.new SystemMaintenanceAction(), + controller.new SystemPropertiesAction(), + new GetPendingRequestCountAction(), + new InstallCompleteAction(), + new NewInstallSiteSettingsAction() + ); + + assertForTroubleshooterPermission(ContainerManager.getRoot(), user, + controller.new OptionalFeaturesAction(), + controller.new ShowModuleErrorsAction(), + new ModuleStatusAction() + ); + } + } + + public static class SerializationTest extends PipelineJob.TestSerialization + { + static class TestJob extends PipelineJob + { + ImpersonationContext _impersonationContext; + ImpersonationContext _impersonationContext1; + ImpersonationContext _impersonationContext2; + + @Override + public URLHelper getStatusHref() + { + return null; + } + + @Override + public String getDescription() + { + return "Test Job"; + } + } + + @Test + public void testSerialization() + { + TestJob job = new TestJob(); + TestContext ctx = TestContext.get(); + ViewContext viewContext = new ViewContext(); + viewContext.setContainer(ContainerManager.getSharedContainer()); + viewContext.setUser(ctx.getUser()); + RoleImpersonationContextFactory factory = new RoleImpersonationContextFactory( + viewContext.getContainer(), viewContext.getUser(), + Collections.singleton(RoleManager.getRole(SharedViewEditorRole.class)), Collections.emptySet(), null); + job._impersonationContext = factory.getImpersonationContext(); + + try + { + UserImpersonationContextFactory factory1 = new UserImpersonationContextFactory(viewContext.getContainer(), viewContext.getUser(), + UserManager.getGuestUser(), null); + job._impersonationContext1 = factory1.getImpersonationContext(); + } + catch (Exception e) + { + LOG.error("Invalid user email for impersonating."); + } + + GroupImpersonationContextFactory factory2 = new GroupImpersonationContextFactory(viewContext.getContainer(), viewContext.getUser(), + GroupManager.getGroup(ContainerManager.getRoot(), "Users", GroupEnumType.SITE), null); + job._impersonationContext2 = factory2.getImpersonationContext(); + testSerialize(job, LOG); + } + } + + public static class WorkbookDeleteTestCase extends Assert + { + private static final String FOLDER_NAME = "WorkbookDeleteTestCaseFolder"; + private static final String TEST_EMAIL = "testDelete@myDomain.com"; + + @Test + public void testWorkbookDelete() throws Exception + { + doCleanup(); + + Container project = ContainerManager.createContainer(ContainerManager.getRoot(), FOLDER_NAME, TestContext.get().getUser()); + Container workbook = ContainerManager.createContainer(project, null, "Title1", null, WorkbookContainerType.NAME, TestContext.get().getUser()); + + ValidEmail email = new ValidEmail(TEST_EMAIL); + SecurityManager.NewUserStatus newUserStatus = SecurityManager.addUser(email, null); + User nonAdminUser = newUserStatus.getUser(); + MutableSecurityPolicy policy = new MutableSecurityPolicy(project.getPolicy()); + policy.addRoleAssignment(nonAdminUser, ReaderRole.class); + SecurityPolicyManager.savePolicy(policy, TestContext.get().getUser()); + + // User lacks any permission, throw unauthorized for parent and workbook: + HttpServletRequest request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, project), nonAdminUser, Map.of("Content-Type", "application/json"), null); + MockHttpServletResponse response = ViewServlet.mockDispatch(request, null); + Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); + + request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, workbook), nonAdminUser, Map.of("Content-Type", "application/json"), null); + response = ViewServlet.mockDispatch(request, null); + Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); + + // Grant permission, should be able to delete the workbook but not parent: + policy = new MutableSecurityPolicy(project.getPolicy()); + policy.addRoleAssignment(nonAdminUser, EditorRole.class); + SecurityPolicyManager.savePolicy(policy, TestContext.get().getUser()); + + request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, project), nonAdminUser, Map.of("Content-Type", "application/json"), null); + response = ViewServlet.mockDispatch(request, null); + Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); + + // Hitting delete action results in a redirect: + request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, workbook), nonAdminUser, Map.of("Content-Type", "application/json"), null); + response = ViewServlet.mockDispatch(request, null); + Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FOUND, response.getStatus()); + + doCleanup(); + } + + protected static void doCleanup() throws Exception + { + Container project = ContainerManager.getForPath(FOLDER_NAME); + if (project != null) + { + ContainerManager.deleteAll(project, TestContext.get().getUser()); + } + + if (UserManager.userExists(new ValidEmail(TEST_EMAIL))) + { + User u = UserManager.getUser(new ValidEmail(TEST_EMAIL)); + UserManager.deleteUser(u.getUserId()); + } + } + } +} diff --git a/core/src/org/labkey/core/attachment/AttachmentServiceImpl.java b/core/src/org/labkey/core/attachment/AttachmentServiceImpl.java index 184ae14ccf1..4f9925a20ac 100644 --- a/core/src/org/labkey/core/attachment/AttachmentServiceImpl.java +++ b/core/src/org/labkey/core/attachment/AttachmentServiceImpl.java @@ -1,1879 +1,1876 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.core.attachment; - -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.attachments.Attachment; -import org.labkey.api.attachments.AttachmentDirectory; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.attachments.AttachmentParent; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.attachments.AttachmentType; -import org.labkey.api.attachments.DocumentWriter; -import org.labkey.api.attachments.FileAttachmentFile; -import org.labkey.api.attachments.SpringAttachmentFile; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.Sets; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.ColumnRenderProperties; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DatabaseTableType; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.Parameter; -import org.labkey.api.data.ResultSetView; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SchemaTableInfo; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.exp.Lsid; -import org.labkey.api.files.FileContentService; -import org.labkey.api.files.MissingRootDirectoryException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.AuthenticationLogoAttachmentParent; -import org.labkey.api.security.SecurableResource; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.settings.AppProps; -import org.labkey.api.test.TestWhen; -import org.labkey.api.util.ContainerUtil; -import org.labkey.api.util.FileStream; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.HtmlStringBuilder; -import org.labkey.api.util.MimeMap; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.Path; -import org.labkey.api.util.ResponseHelper; -import org.labkey.api.util.ResultSetUtil; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.URLHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.VBox; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartView; -import org.labkey.api.webdav.AbstractDocumentResource; -import org.labkey.api.webdav.AbstractWebdavResourceCollection; -import org.labkey.api.webdav.DavException; -import org.labkey.api.webdav.WebdavResolver; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.core.admin.AdminController; -import org.labkey.core.query.AttachmentAuditProvider; -import org.springframework.http.ContentDisposition; -import org.springframework.mock.web.MockMultipartFile; -import org.springframework.validation.BindException; -import org.springframework.web.multipart.MultipartFile; - -import java.beans.PropertyChangeEvent; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; - -public class AttachmentServiceImpl implements AttachmentService, ContainerManager.ContainerListener -{ - private static final String UPLOAD_LOG = ".upload.log"; - private static final Map ATTACHMENT_TYPE_MAP = new HashMap<>(); - private static final Set ATTACHMENT_COLUMNS = Set.of("Parent", "Container", "DocumentName", "DocumentSize", "DocumentType", "Created", "CreatedBy", "LastIndexed"); - - public AttachmentServiceImpl() - { - ContainerManager.addContainerListener(this); - } - - @Override - public void download(HttpServletResponse response, AttachmentParent parent, String filename, @Nullable String alias, boolean inlineIfPossible) throws ServletException, IOException - { - if (null == filename || filename.isEmpty()) - { - throw new NotFoundException(); - } - - boolean canInline = MimeMap.DEFAULT.canInlineFor(filename); - - // Default to rendering inline when possible, but let caller force download as an attachment - boolean asAttachment = !canInline || !inlineIfPossible; - - response.reset(); - writeDocument(new ResponseWriter(response), parent, filename, alias, asAttachment); - - User user = null; - try - { - ViewContext context = HttpView.currentContext(); - if (context != null) - user = context.getUser(); - } - catch (RuntimeException ignored) - { - } - - // Change in behavior added in 11.1: no longer audit download events for the guest user - if (null != user && !user.isGuest() && asAttachment) - { - addAuditEvent(user, parent, filename, "The attachment " + filename + " was downloaded"); - } - } - - - @Override - public void download(HttpServletResponse response, AttachmentParent parent, String filename, boolean inlineIfPossible) throws ServletException, IOException - { - download(response, parent, filename, null, inlineIfPossible); - } - - - @Override - public void addAuditEvent(User user, AttachmentParent parent, String filename, String comment) - { - if (user == null) - throw new IllegalArgumentException("Cannot create attachment audit events for the null user."); - - if (parent != null) - { - Container c = ContainerManager.getForId(parent.getContainerId()); - AttachmentAuditProvider.AttachmentAuditEvent attachmentEvent = new AttachmentAuditProvider.AttachmentAuditEvent(c == null ? ContainerManager.getRoot() : c, comment); - - attachmentEvent.setAttachmentParentEntityId(parent.getEntityId()); - attachmentEvent.setAttachment(filename); - - AuditLogService.get().addEvent(user, attachmentEvent); - - if (parent instanceof AttachmentDirectory adParent) - { - FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(c, comment); - try - { - event.setDirectory(adParent.getFileSystemDirectory().getPath()); - } - catch (MissingRootDirectoryException ex) - { - // UNDONE: AttachmentDirectory.getFileSystemPath()... - event.setDirectory("path not found"); - } - event.setFile(filename); - AuditLogService.get().addEvent(user, event); - } - } - } - - @Override - public HttpView getHistoryView(ViewContext context, AttachmentParent parent, BindException errors) - { - UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); - if (schema != null) - { - checkSecurityPolicy(context.getUser(), parent); - QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(AttachmentAuditProvider.COLUMN_NAME_CONTAINER), parent.getContainerId()); - filter.addCondition(FieldKey.fromParts(AttachmentAuditProvider.COLUMN_NAME_ATTACHMENT_PARENT_ENTITY_ID), parent.getEntityId()); - - settings.setBaseFilter(filter); - settings.setQueryName(AttachmentService.ATTACHMENT_AUDIT_EVENT); - - QueryView view = schema.createView(context, settings, errors); - view.setTitle("Attachments History:"); - - return view; - } - return null; - } - - private void validateAttachmentSize(AttachmentFile file) throws IOException - { - int maxSize = AppProps.getInstance().getMaxBLOBSize(); - if (file.getSize() > maxSize) - { - throw new AttachmentService.FileTooLargeException(file, maxSize); - } - } - - @Override - public void validateAttachmentSizes(AttachmentParent parent, List files) throws IOException - { - File fileLocation = parent instanceof AttachmentDirectory ? ((AttachmentDirectory) parent).getFileSystemDirectory() : null; - - // Only validate if we're putting the file in the database - if (null == fileLocation) - { - for (AttachmentFile file : files) - { - validateAttachmentSize(file); - } - } - } - - - @Override - public synchronized void addAttachments(AttachmentParent parent, List files, @NotNull User user) throws IOException - { - if (null == user) - throw new IllegalArgumentException("Cannot add attachments for the null user"); - - if (null == files || files.isEmpty()) - return; - - List duplicates = findDuplicates(files); - if (duplicates.size() > 0) - { - throw new AttachmentService.DuplicateFilenameException(duplicates); - } - - Set filesToSkip = new TreeSet<>(); - File fileLocation = parent instanceof AttachmentDirectory ? ((AttachmentDirectory) parent).getFileSystemDirectory() : null; - - for (AttachmentFile file : files) - { - if (parent != null && exists(parent, file.getFilename())) - { - filesToSkip.add(file.getFilename()); - continue; - } - - HashMap hm = new HashMap<>(); - if (null == fileLocation) - { - validateAttachmentSize(file); - hm.put("Document", file); - } - else - ((AttachmentDirectory)parent).addAttachment(user, file); - - hm.put("DocumentName", file.getFilename()); - hm.put("DocumentSize", file.getSize()); - hm.put("DocumentType", file.getContentType()); - hm.put("Parent", parent.getEntityId()); - hm.put("Container", parent.getContainerId()); - Table.insert(user, coreTables().getTableInfoDocuments(), hm); - - addAuditEvent(user, parent, file.getFilename(), "The attachment " + file.getFilename() + " was added"); - } - - AttachmentCache.removeAttachments(parent); - - if (!filesToSkip.isEmpty()) - throw new AttachmentService.DuplicateFilenameException(filesToSkip); - } - - @Override - public HttpView getErrorView(List files, BindException errors, URLHelper returnUrl) - { - boolean hasErrors = null != errors && errors.hasErrors(); - HtmlString errorHtml = getErrorHtml(files); // TODO: Get rid of getErrorHtml() -- use errors collection - - if (null == errorHtml && !hasErrors) - return null; - - try - { - return new ErrorView(errorHtml, errors, returnUrl); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - - - private @Nullable HtmlString getErrorHtml(List files) - { - HtmlStringBuilder builder = HtmlStringBuilder.of(); - - for (AttachmentFile file : files) - { - String error = file.getError(); - - if (null != error) - builder.append(error).unsafeAppend("

    "); - } - - HtmlString html = builder.getHtmlString(); - - return HtmlString.isEmpty(html) ? null : html; - } - - - public static class ErrorView extends JspView - { - public HtmlString errorHtml; - public URLHelper returnUrl; - - private ErrorView(HtmlString errorHtml, BindException errors, URLHelper returnUrl) - { - super("/org/labkey/core/attachment/showErrors.jsp", new Object(), errors); - this.errorHtml = errorHtml; - this.returnUrl = returnUrl; - } - } - - - @Override - public void deleteAttachments(AttachmentParent parent) - { - deleteAttachments(Collections.singleton(parent)); - } - - @Override - public void deleteAttachments(Collection parents) - { - for (AttachmentParent parent : parents) - { - List atts = getAttachments(parent); - - // No attachments, or perhaps container doesn't match entityid - if (atts.isEmpty()) - continue; - - checkSecurityPolicy(parent); // Only check policy if there are attachments (a client may delete attachment and policy, but attempt to delete again) - deleteIndexedAttachments(parent, atts); - - new SqlExecutor(coreTables().getSchema()).execute(sqlCascadeDelete(parent)); - if (parent instanceof AttachmentDirectory) - ((AttachmentDirectory)parent).deleteAttachment(HttpView.currentContext().getUser(), null); - AttachmentCache.removeAttachments(parent); - } - } - - @Override - public void deleteIndexedAttachments(List parentIds) - { - TableSelector ts = new TableSelector(CoreSchema.getInstance().getTableInfoDocuments(), - PageFlowUtil.set("Parent", "DocumentName"), - new SimpleFilter(FieldKey.fromParts("Parent"), parentIds, CompareType.IN), null); - - try (ResultSet rs = ts.getResultSet()) - { - while (rs.next()) - { - deleteIndexedAttachment(rs.getString("Parent"), rs.getString("DocumentName")); - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - } - - @Override - public void clearLastIndexed(List parentIds) - { - SimpleFilter filter = new SimpleFilter(new SimpleFilter.InClause(FieldKey.fromParts("Parent"), parentIds)) - .addClause(new SimpleFilter.SQLClause("LastIndexed IS NOT NULL", null)); - SQLFragment sql = new SQLFragment("UPDATE core.Documents SET LastIndexed = NULL ") - .append(filter.getSQLFragment(CoreSchema.getInstance().getSqlDialect())); - new SqlExecutor(CoreSchema.getInstance().getSchema()).execute(sql); - } - - private void deleteIndexedAttachment(AttachmentParent parent, String name) - { - deleteIndexedAttachment(parent.getEntityId(), name); - } - - private void deleteIndexedAttachment(String parent, String name) - { - SearchService ss = SearchService.get(); - if (ss != null) - ss.deleteResource(makeDocId(parent, name)); - new SqlExecutor(CoreSchema.getInstance().getSchema()).execute(new SQLFragment( - "UPDATE core.Documents SET LastIndexed = NULL WHERE LastIndexed IS NOT NULL AND Parent = ? AND DocumentName = ?", parent, name) - ); - } - - private void deleteIndexedAttachments(AttachmentParent parent, List atts) - { - for (Attachment att : atts) - deleteIndexedAttachment(parent, att.getName()); - } - - @Override - public void deleteIndexedAttachments(AttachmentParent parent) - { - List atts = getAttachments(parent); - deleteIndexedAttachments(parent, atts); - } - - private void _deleteAttachment(AttachmentParent parent, String name, @Nullable User auditUser) - { - checkSecurityPolicy(auditUser, parent); // Only check policy if there are attachments (a client may delete attachment and policy, but attempt to delete again) - deleteIndexedAttachment(parent, name); - - new SqlExecutor(coreTables().getSchema()).execute(sqlDelete(parent, name)); - if (parent instanceof AttachmentDirectory) - ((AttachmentDirectory)parent).deleteAttachment(auditUser, name); - - if (null != auditUser) - addAuditEvent(auditUser, parent, name, "The attachment " + name + " was deleted"); - } - - - @Override - public void deleteAttachment(AttachmentParent parent, String name, @Nullable User auditUser) - { - Attachment att = getAttachmentHelper(parent, name); - - if (null != att) - { - _deleteAttachment(parent, name, auditUser); - AttachmentCache.removeAttachments(parent); - } - } - - @Override - public void deleteAttachments(AttachmentParent parent, Collection names, @Nullable User auditUser) - { - Map attachmentMap = getAttachments(parent, names); - - for (Attachment attachment : attachmentMap.values()) - { - _deleteAttachment(parent, attachment.getName(), null); - } - - AttachmentCache.removeAttachments(parent); - } - - - @Override - public void renameAttachment(AttachmentParent parent, String oldName, String newName, User auditUser) throws IOException - { - File dir = null; - File dest = null; - File src = null; - - checkSecurityPolicy(auditUser, parent); - if (parent instanceof AttachmentDirectory) - { - dir = ((AttachmentDirectory)parent).getFileSystemDirectory(); - src = new File(dir,oldName); - dest = new File(dir,newName); - if (!src.exists()) - throw new FileNotFoundException(oldName); - if (dest.exists()) - throw new AttachmentService.DuplicateFilenameException(newName); - - // make sure newName attachment doesn't exist. if it does exist, it's already orphaned - deleteAttachment(parent, newName, null); - } - - if (exists(parent, newName)) - throw new AttachmentService.DuplicateFilenameException(newName); - - new SqlExecutor(coreTables().getSchema()).execute(sqlRename(parent, oldName, newName)); - - // rename the file in the filesystem only if an Attachment directory and the db rename succeeded - if (null != dir) - src.renameTo(dest); - - AttachmentCache.removeAttachments(parent); - - addAuditEvent(auditUser, parent, newName, "The attachment " + oldName + " was renamed " + newName); - } - - - // Copies an attachment -- same container, same parent, but new name. - @Override - public void copyAttachment(AttachmentParent parent, Attachment a, String newName, User auditUser) throws IOException - { - checkSecurityPolicy(auditUser, parent); - a.setName(newName); - DatabaseAttachmentFile file = new DatabaseAttachmentFile(a); - addAttachments(parent, Collections.singletonList(file), auditUser); - } - - @Override - public void moveAttachments(Container newContainer, List parents, User auditUser) throws IOException - { - SearchService ss = SearchService.get(); - for (AttachmentParent parent : parents) - { - checkSecurityPolicy(auditUser, parent); - int rowsChanged = new SqlExecutor(coreTables().getSchema()).execute(sqlMove(parent, newContainer)); - if (rowsChanged > 0) - { - List atts = getAttachments(parent); - String filename; - for (Attachment att : atts) - { - filename = att.getName(); - if (parent instanceof AttachmentDirectory parentDir) - { - File currentDir = parentDir.getFileSystemDirectoryPath().toFile(); - File newDir = parentDir.getFileSystemDirectoryPath(newContainer, true).toFile(); - File src = new File(currentDir, filename); - File dest = new File(newDir, filename); - if (!src.exists()) - throw new FileNotFoundException(src.getAbsolutePath()); - if (dest.exists()) - throw new AttachmentService.DuplicateFilenameException(dest.getAbsolutePath()); - } - deleteIndexedAttachment(parent, filename); - addAuditEvent(auditUser, parent, filename, "The attachment " + filename + " was moved"); - } - AttachmentCache.removeAttachments(parent); - } - } - } - - /** may return fewer AttachmentFile than Attachment, if there have been deletions */ - @Override - public @NotNull List getAttachmentFiles(AttachmentParent parent, Collection attachments) throws IOException - { - checkSecurityPolicy(parent); - List files = new ArrayList<>(attachments.size()); - - for (Attachment attachment : attachments) - { - if (parent instanceof AttachmentDirectory) - { - File f = new File(((AttachmentDirectory)parent).getFileSystemDirectory(), attachment.getName()); - files.add(new FileAttachmentFile(f)); - } - else - { - try - { - files.add(new DatabaseAttachmentFile(attachment)); - } - catch (FileNotFoundException x) - { - // - } - } - } - return files; - } - - - private boolean exists(AttachmentParent parent, String filename) - { - return null != getAttachmentHelper(parent, filename); - } - - private List findDuplicates(List files) - { - Set fileNames = new HashSet<>(); - List duplicates = new ArrayList<>(); - for (AttachmentFile file : files) - { - if (!fileNames.add(file.getFilename())) - { - duplicates.add(file.getFilename()); - } - } - return duplicates; - } - - @Override - public @NotNull List getAttachments(AttachmentParent parent) - { - checkSecurityPolicy(parent); - Map mapFromDatabase = AttachmentCache.getAttachments(parent); - List attachmentsFromDatabase = Collections.unmodifiableList(new ArrayList<>(mapFromDatabase.values())); - - File parentDir = null; - - try - { - parentDir = parent instanceof AttachmentDirectory ? ((AttachmentDirectory) parent).getFileSystemDirectory() : null; - } - catch (MissingRootDirectoryException ex) - { - /* no problem */ - } - - if (null == parentDir || !parentDir.exists()) - return attachmentsFromDatabase; - - for (Attachment att : attachmentsFromDatabase) - att.setFile(new File(parentDir, att.getName())); - - //OK, make sure that the list really reflects what is in the file system. - List attList = new ArrayList<>(); - - File[] fileList = parentDir.listFiles(file -> !file.isDirectory() && !(file.getName().charAt(0) == '.') && !file.isHidden()); - - Set attachmentNames = new CaseInsensitiveHashSet(); - - for (Attachment attachment : attachmentsFromDatabase) - { - attachmentNames.add(attachment.getName()); - attList.add(attachment); - } - - if (null != fileList) - { - for (File file : fileList) - { - if (!attachmentNames.contains(file.getName())) - attList.add(attachmentFromFile(parent, file)); - } - } - - return Collections.unmodifiableList(attList); - } - - - /** Does not work for file system parents */ - @Override - public List> listAttachmentsForIndexing(Collection parents, Date modifiedSince) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Parent"), parents, CompareType.IN); - var since = new SearchService.LastIndexedClause(coreTables().getTableInfoDocuments(), modifiedSince, null); - - if (!since.isEmpty()) - filter.addClause(since); - - final ArrayList> ret = new ArrayList<>(); - - new TableSelector(coreTables().getTableInfoDocuments(), - PageFlowUtil.set("Parent", "DocumentName", "LastIndexed"), - filter, - new Sort("+Created")).forEach(rs -> { - String parent = rs.getString(1); - String name = rs.getString(2); - Date last = rs.getTimestamp(3); - if (last != null && last.getTime() == SearchService.failDate.getTime()) - return; - ret.add(new Pair<>(parent, name)); - }); - - return ret; - } - - /** Collection resource with all attachments for this parent */ - @Override - public WebdavResource getAttachmentResource(Path path, AttachmentParent parent) - { - // NOTE parent does not supply ACL, but should? - // acl = parent.getAcl() - checkSecurityPolicy(parent); - Container c = ContainerManager.getForId(parent.getContainerId()); - if (null == c) - return null; - - return new AttachmentCollection(path, parent, c); - } - - @Override - public WebdavResource getDocumentResource(Path path, ActionURL downloadURL, String displayTitle, AttachmentParent parent, String name, SearchService.SearchCategory cat) - { - checkSecurityPolicy(parent); - return new AttachmentResource(path, downloadURL, displayTitle, parent, name, cat); - } - - private Attachment attachmentFromFile(AttachmentParent parent, File file) - { - Attachment attachment = new Attachment(); - attachment.setParent(parent.getEntityId()); - attachment.setContainer(parent.getContainerId()); - attachment.setDocumentName(file.getName()); - attachment.setCreated(new Date(file.lastModified())); - attachment.setFile(file); - - return attachment; - } - - @Override - public void registerAttachmentType(AttachmentType type) - { - ATTACHMENT_TYPE_MAP.put(type.getUniqueName(), type); - } - - @Override - public HttpView getAdminView(ActionURL currentUrl) - { - String requestedType = currentUrl.getParameter("type"); - AttachmentType attachmentType = null != requestedType ? ATTACHMENT_TYPE_MAP.get(requestedType) : null; - - if (null == attachmentType) - { - boolean findAttachmentParents = "1".equals(currentUrl.getParameter("find")); - - // The first query lists all the attachment types and the attachment counts for each. A separate select from - // core.Documents for each type is needed to associate the Type values with the associated rows. - List selectStatements = new LinkedList<>(); - - for (AttachmentType type : ATTACHMENT_TYPE_MAP.values()) - { - SQLFragment selectStatement = new SQLFragment(); - - // Adding unique column RowId ensures we get the proper count - selectStatement.append("SELECT RowId, CAST(").appendValue(type.getUniqueName()).append(" AS VARCHAR(500)) AS Type FROM ") - .append(CoreSchema.getInstance().getTableInfoDocuments(), "d") - .append(" WHERE "); - addAndVerifyWhereSql(type, selectStatement); - selectStatement.append("\n"); - - selectStatements.add(selectStatement); - } - - SQLFragment allSql = new SQLFragment("SELECT Type, COUNT(*) AS Count FROM (\n"); - allSql.append(SQLFragment.join(selectStatements, "UNION\n")); - allSql.append(") u\nGROUP BY Type\nORDER BY Type"); - ActionURL linkUrl = currentUrl.clone().deleteParameters().addParameter("type", null); - - // The second query shows all attachments that we can't associate with a type. We just need to assemble a big - // WHERE NOT clause that ORs the conditions from every registered type. - SQLFragment whereSql = new SQLFragment(); - String sep = ""; - - for (AttachmentType type : ATTACHMENT_TYPE_MAP.values()) - { - whereSql.append(sep); - sep = " OR"; - whereSql.append("\n("); - addAndVerifyWhereSql(type, whereSql); - whereSql.append(")"); - } - - SQLFragment unknownSql = new SQLFragment("SELECT d.Container, c.Name, d.Parent, d.DocumentName"); - - if (findAttachmentParents) - unknownSql.append(", e.TableName"); - - unknownSql.append(" FROM core.Documents d\n"); - unknownSql.append("INNER JOIN core.Containers c ON c.EntityId = d.Container\n"); - - Set schemasToIgnore = Sets.newCaseInsensitiveHashSet(currentUrl.getParameterValues("ignore")); - - if (findAttachmentParents) - { - unknownSql.append("LEFT OUTER JOIN (\n"); - addSelectAllEntityIdsSql(unknownSql, schemasToIgnore); - unknownSql.append(") e ON e.EntityId = d.Parent\n"); - } - - unknownSql.append("WHERE NOT ("); - unknownSql.append(whereSql); - unknownSql.append(")\n"); - unknownSql.append("ORDER BY Container, Parent, DocumentName"); - - WebPartView unknownView = getResultSetView(unknownSql, "Unknown Attachments", null); - NavTree navMenu = new NavTree(); - - if (!findAttachmentParents) - { - navMenu.addChild(new NavTree("Search for Attachment Parents (Be Patient)", - new ActionURL(AdminController.AttachmentsAction.class, ContainerManager.getRoot()).addParameter("find", 1).addParameter("ignore", "Audit")) - ); - } - else - { - navMenu.addChild(new NavTree("Remove TableName Column", - new ActionURL(AdminController.AttachmentsAction.class, ContainerManager.getRoot())) - ); - - if (schemasToIgnore.isEmpty()) - { - navMenu.addChild(new NavTree("Ignore Audit Schema", - new ActionURL(AdminController.AttachmentsAction.class, ContainerManager.getRoot()).addParameter("find", 1).addParameter("ignore", "Audit")) - ); - } - else - { - navMenu.addChild(new NavTree("Include All Schemas", - new ActionURL(AdminController.AttachmentsAction.class, ContainerManager.getRoot()).addParameter("find", 1)) - ); - } - } - unknownView.setNavMenu(navMenu); - - return new VBox(getResultSetView(allSql, "Attachment Types and Counts", linkUrl), unknownView); - } - else - { - // This query lists all the documents associated with a single type. - SQLFragment oneTypeSql = new SQLFragment("SELECT d.Container, c.Name, d.Parent, d.DocumentName FROM core.Documents d\n" + - "INNER JOIN core.Containers c ON c.EntityId = d.Container\n" + - "WHERE "); - addAndVerifyWhereSql(attachmentType, oneTypeSql); - oneTypeSql.append("\nORDER BY Container, Parent, DocumentName"); - - return getResultSetView(oneTypeSql, attachmentType.getUniqueName() + " Attachments", null); - } - } - - private void addAndVerifyWhereSql(AttachmentType attachmentType, SQLFragment sql) - { - int initialLength = sql.length(); - attachmentType.addWhereSql(sql, "d.Parent", "d.DocumentName"); - if (initialLength == sql.length()) - throw new UnsupportedOperationException("AttachmentType: '" + attachmentType.getUniqueName() + "' did not update attachment WHERE clause."); - } - - @Override - // Joins each row of core.Documents to the table(s) (if any) that contain an entityid matching the document's parent - public HttpView getFindAttachmentParentsView() - { - SQLFragment sql = new SQLFragment("SELECT RowId, CreatedBy, Created, ModifiedBy, Modified, Container, DocumentName, TableName FROM core.Documents LEFT OUTER JOIN (\n"); - addSelectAllEntityIdsSql(sql, Sets.newCaseInsensitiveHashSet("Audit")); - sql.append(") c ON EntityId = Parent\nORDER BY TableName, DocumentName, Container"); - - return getResultSetView(sql, "Probable Attachment Parents", null); - } - - // Creates a two-column query of ID and table name that selects from every possible attachment parent column in the labkey database: - // - Enumerate all tables in all schemas in the labkey scope - // - Enumerate columns and identify potential attachment parents (currently, EntityId columns and ObjectIds extracted from LSIDs) - // - Create a UNION query that selects the candidate ids along with a constant column that lists the table name - private void addSelectAllEntityIdsSql(SQLFragment sql, Set userRequestedSchemasToIgnore) - { - List selectStatements = new LinkedList<>(); - Set schemasToIgnore = Sets.newCaseInsensitiveHashSet(userRequestedSchemasToIgnore); - - // Temp schema causes problems because materialized tables disappear but stay in the cached list. This is probably a bug with - // MaterializedQueryHelper... it should clear the temp DbSchema when it deletes a temp table. TODO: fix MQH & remove this workaround - schemasToIgnore.add("temp"); - - DbScope.getLabKeyScope().getSchemaNames().stream() - .filter(schemaName->!schemasToIgnore.contains(schemaName)) // Exclude unwanted schema names - .map(schemaName->DbSchema.get(schemaName, DbSchemaType.Bare)) - .forEach(schema-> schema.getTableNames().stream() - .map(schema::getTable) - .filter(table->table.getTableType() == DatabaseTableType.TABLE) // We just want the underlying tables (no views or virtual tables) - .map(SchemaTableInfo::getColumns) - .flatMap(Collection::stream) - .filter(ColumnRenderProperties::isStringType) - .forEach(c->addSelectStatement(selectStatements, c)) - ); - - sql.append(StringUtils.join(selectStatements, " UNION\n")); - } - - private void addSelectStatement(List selectStatements, ColumnInfo column) - { - String expression; - String where = null; - - if (StringUtils.containsIgnoreCase(column.getName(), "EntityId")) - { - // TODO convert all this to use SQLFragment - expression = column.getSelectIdentifier().getSql().getRawSQL(); - } - else if (StringUtils.endsWithIgnoreCase(column.getName(), "LSID")) - { - Pair pair = Lsid.getSqlExpressionToExtractObjectId(column.getSelectIdentifier().getSql().getRawSQL(), column.getSqlDialect()); - expression = pair.first; - where = pair.second; - } - else - { - return; - } - - TableInfo table = column.getParentTable(); - selectStatements.add(" SELECT " + expression + " AS EntityId, " + table.getSqlDialect().quoteStringLiteral(table.getSelectName()) + " AS TableName FROM " + table.getSelectName() + (null != where ? " WHERE " + where : "") + "\n"); - } - - private WebPartView getResultSetView(SQLFragment sql, String title, @Nullable ActionURL linkUrl) - { - SqlSelector selector = new SqlSelector(DbScope.getLabKeyScope(), sql); - ResultSet rs = selector.getResultSet(); - - return null != linkUrl ? new ResultSetView(rs, title, "Type", linkUrl) : new ResultSetView(rs, title); - } - - @Override - public @Nullable Attachment getAttachment(AttachmentParent parent, String name) - { - checkSecurityPolicy(parent); - return getAttachmentHelper(parent, name); - } - - @Override - public Map getAttachments(AttachmentParent parent, Collection names) - { - checkSecurityPolicy(parent); - - if (names == null || names.isEmpty()) - return Collections.emptyMap(); - - Map attachments = new HashMap<>(); - - if (parent instanceof AttachmentDirectory) - { - for (Attachment attachment : getAttachments(parent)) - if (names.contains(attachment.getName())) - attachments.put(attachment.getName(), attachment); - } - else - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Parent"), parent.getEntityId()); - filter.addCondition(FieldKey.fromParts("DocumentName"), names, CompareType.IN); - // Note: we are intentionally skipping the AttachmentCache here. If we hit the cache here while getting - // attachments from the global attachment parent we'll load every single attachment into the cache first, - // which could be very expensive on servers that have a ton of attachments. - List attachmentsList = new TableSelector(CoreSchema.getInstance().getTableInfoDocuments(), - ATTACHMENT_COLUMNS, - filter, - new Sort("+RowId")).getArrayList(Attachment.class); - - for (Attachment attachment : attachmentsList) - attachments.put(attachment.getName(), attachment); - } - - return attachments; - } - - private @Nullable Attachment getAttachmentHelper(AttachmentParent parent, String name) - { - if (parent instanceof AttachmentDirectory) - { - for (Attachment attachment : getAttachments(parent)) - if (name.equals(attachment.getName())) - return attachment; - - return null; - } - else - { - return AttachmentCache.getAttachments(parent).get(name); - } - } - - - @Override - public void containerCreated(Container c, User user) - { - } - - - @Override - public void propertyChange(PropertyChangeEvent propertyChangeEvent) - { - } - - @Override - public void containerDeleted(Container c, User user) - { - // TODO: do we need to get each document and remove its security policy? - ContainerUtil.purgeTable(coreTables().getTableInfoDocuments(), c, null); - AttachmentCache.removeAttachments(c); - } - - @Override - public void containerMoved(Container c, Container oldParent, User user) - { - } - - @NotNull - @Override - public Collection canMove(Container c, Container newParent, User user) - { - return Collections.emptyList(); - } - - private void writeDocument(DocumentWriter writer, AttachmentParent parent, String name, @Nullable String alias, boolean asAttachment) throws ServletException, IOException - { - checkSecurityPolicy(parent); - Connection conn = null; - PreparedStatement stmt = null; - ResultSet rs = null; - DbSchema schema = coreTables().getSchema(); - - if (alias == null) - alias = name; - - try (Parameter.ParameterList jdbcParameters = new Parameter.ParameterList()) - { - // we don't want a RowSet, so execute directly (not Table.executeQuery()) - conn = schema.getScope().getConnection(); - if (null == parent.getEntityId()) - stmt = Table.prepareStatement(conn, sqlRootDocument(), Collections.singletonList(name), jdbcParameters); - else - stmt = Table.prepareStatement(conn, sqlDocument(), Arrays.asList(parent.getContainerId(), parent.getEntityId(), name), jdbcParameters); - - rs = stmt.executeQuery(); - - OutputStream out; - InputStream s; - - if (parent instanceof AttachmentDirectory) - { - File parentDir = ((AttachmentDirectory) parent).getFileSystemDirectory(); - if (!parentDir.exists()) - throw new NotFoundException("No parent directory for downloaded file " + alias + ". Please contact an administrator."); - File file = new File(parentDir, name); - if (!file.exists()) - throw new NotFoundException("Could not find file " + alias); - - if (asAttachment) - writer.setContentDisposition(ContentDisposition.attachment().filename(alias, StandardCharsets.UTF_8).build()); - s = new FileInputStream(file); - } - else - { - if (!rs.next()) - { - throw new NotFoundException(); - } - - writer.setContentType(rs.getString("DocumentType")); - if (asAttachment) - writer.setContentDisposition(ContentDisposition.builder("attachment").filename(alias, StandardCharsets.UTF_8).build()); - else - writer.setContentDisposition(ContentDisposition.builder("inline").filename(alias, StandardCharsets.UTF_8).build()); - - int size = rs.getInt("DocumentSize"); - if (size > 0) - writer.setContentLength(size); - - s = rs.getBinaryStream("Document"); - if (null == s) - return; - } - - out = writer.getOutputStream(); - - try - { - IOUtils.copy(s, out); - } - finally - { - s.close(); - } - } - catch (SQLException x) - { - throw new ServletException(x); - } - finally - { - ResultSetUtil.close(rs); - ResultSetUtil.close(stmt); - if (conn != null) - { - schema.getScope().releaseConnection(conn); - } - } - } - - // CONSIDER: Return success/failure notification so caller can take action (render a default document) in all the failure scenarios. - @Override - public void writeDocument(DocumentWriter writer, AttachmentParent parent, String name, boolean asAttachment) throws ServletException, IOException - { - writeDocument(writer, parent, name, null, asAttachment); - } - - - @Override - @NotNull - public InputStream getInputStream(AttachmentParent parent, String name) throws FileNotFoundException - { - checkSecurityPolicy(parent); - Connection conn = null; - PreparedStatement stmt = null; - ResultSet rs = null; - final DbSchema schema = coreTables().getSchema(); - - try (Parameter.ParameterList jdbcParameters = new Parameter.ParameterList()) - { - // we don't want a RowSet, so execute directly (not Table.executeQuery()) - conn = schema.getScope().getConnection(); - if (null == parent.getEntityId()) - stmt = Table.prepareStatement(conn, sqlRootDocument(), Collections.singletonList(name), jdbcParameters); - else - stmt = Table.prepareStatement(conn, sqlDocument(), Arrays.asList(parent.getContainerId(), parent.getEntityId(), name), jdbcParameters); - - rs = stmt.executeQuery(); - - if (parent instanceof AttachmentDirectory) - { - File parentDir = ((AttachmentDirectory) parent).getFileSystemDirectory(); - if (!parentDir.exists()) - throw new FileNotFoundException("No parent directory for downloaded file " + name + ". Please contact an administrator."); - File file = new File(parentDir, name); - stmt.close(); - stmt = null; - rs.close(); - rs = null; - return new FileInputStream(file); - } - else - { - if (!rs.next()) - throw new FileNotFoundException(name); - final int size = rs.getInt("DocumentSize"); - InputStream is = rs.getBinaryStream("Document"); - - final Connection fconn = conn; - final PreparedStatement fstmt = stmt; - final ResultSet frs = rs; - InputStream ret = new FilterInputStream(is) - { - @Override - public void close() throws IOException - { - ResultSetUtil.close(frs); - ResultSetUtil.close(fstmt); - schema.getScope().releaseConnection(fconn); - super.close(); - } - - // slight hack here to get the size cheaply - @Override - public int available() - { - return size; - } - }; - stmt = null; - rs = null; - conn = null; - return ret; - } - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - finally - { - ResultSetUtil.close(rs); - ResultSetUtil.close(stmt); - if (null != conn) schema.getScope().releaseConnection(conn); - } - } - - - private CoreSchema coreTables() - { - return CoreSchema.getInstance(); - } - - private String sqlDocument() - { - return "SELECT DocumentType, DocumentSize, Document FROM " + coreTables().getTableInfoDocuments() + " WHERE Container = ? AND Parent = ? AND DocumentName = ?"; - } - - private String sqlRootDocument() - { - return "SELECT DocumentType, DocumentSize, Document FROM " + coreTables().getTableInfoDocuments() + " WHERE Parent IS NULL AND DocumentName = ?"; - } - - private SQLFragment sqlCascadeDelete(AttachmentParent parent) - { - return new SQLFragment("DELETE FROM " + coreTables().getTableInfoDocuments() + " WHERE Container = ? AND Parent = ?", parent.getContainerId(), parent.getEntityId()); - } - - private SQLFragment sqlDelete(AttachmentParent parent, String name) - { - return new SQLFragment("DELETE FROM " + coreTables().getTableInfoDocuments() + " WHERE Container = ? AND Parent = ? AND DocumentName = ?", parent.getContainerId(), parent.getEntityId(), name); - } - - private SQLFragment sqlRename(AttachmentParent parent, String oldName, String newName) - { - return new SQLFragment("UPDATE " + coreTables().getTableInfoDocuments() + " SET DocumentName = ? WHERE Container = ? AND Parent = ? AND DocumentName = ?", - newName, parent.getContainerId(), parent.getEntityId(), oldName); - } - - private SQLFragment sqlMove(AttachmentParent parent, Container newContainer) - { - // TODO: consider an inClause - return new SQLFragment("UPDATE " + coreTables().getTableInfoDocuments() + " SET Container = ? WHERE Container = ? AND Parent=?", - newContainer.getEntityId(), parent.getContainerId(), parent.getEntityId()); - } - - private static class ResponseWriter implements DocumentWriter - { - private final HttpServletResponse _response; - - public ResponseWriter(HttpServletResponse response) - { - _response = response; - } - - @Override - public void setContentType(String contentType) - { - _response.setContentType(contentType); - } - - @Override - public void setContentDisposition(ContentDisposition value) - { - ResponseHelper.setContentDisposition(_response, value); - } - - @Override - public void setContentLength(int size) - { - _response.setContentLength(size); - } - - @Override - public OutputStream getOutputStream() throws IOException - { - return _response.getOutputStream(); - } - } - - /** two cases - * regular file attachments: assume that this collection resource will be wrapped rather then exposed directly in the tree - * filesets: expect that this will be directly exposed in the tree - */ - private class AttachmentCollection extends AbstractWebdavResourceCollection - { - private final AttachmentParent _parent; - - AttachmentCollection(Path path, AttachmentParent parent, SecurableResource resource) - { - super(path); - _parent = parent; - setSecurableResource(resource); - } - - - @Override - public boolean exists() - { - FileContentService svc = FileContentService.get(); - if (_parent instanceof AttachmentDirectory) - { - if (null == ((AttachmentDirectory)_parent).getName()) - { - try - { - return null != svc.getMappedAttachmentDirectory(ContainerManager.getForId(_parent.getContainerId()), false); - } - catch (MissingRootDirectoryException x) - { - return false; - } - } - else - { - return null != svc.getRegisteredDirectory(ContainerManager.getForId(_parent.getContainerId()), ((AttachmentDirectory)_parent).getName()); - } - } - return true; - } - - - @Override - public WebdavResource find(Path.Part name) - { - Attachment a = getAttachment(_parent, name.toString()); - - if (null != a) - return new AttachmentResource(this, _parent, a); - else - return new AttachmentResource(this, _parent, name.toString()); - } - - - @Override - public Collection listNames() - { - List attachments = getAttachments(_parent); - ArrayList names = new ArrayList<>(attachments.size()); - - for (Attachment a : attachments) - { - if (null != a.getFile() && !a.getFile().exists()) - continue; - names.add(a.getName()); - } - - return names; - } - - - @Override - public Collection list() - { - List attachments = getAttachments(_parent); - ArrayList resources = new ArrayList<>(attachments.size()); - - for (Attachment a : attachments) - { - if (null != a.getFile() && !a.getFile().exists()) - continue; - resources.add(new AttachmentResource(this, _parent, a)); - } - - return resources; - } - } - - private static String makeDocId(AttachmentParent parent, String name) - { - return makeDocId(parent.getEntityId(), name); - } - - private static String makeDocId(String parentId, String name) - { - return "attachment:/" + parentId + "/" + PageFlowUtil.encode(name); - } - - private class AttachmentResource extends AbstractDocumentResource - { - WebdavResource _folder; - final AttachmentParent _parent; - final String _name; - long _created = Long.MIN_VALUE; - User _createdBy = null; - ActionURL _downloadUrl = null; - Attachment _cached = null; - final String _docid; - - AttachmentResource(Path path, ActionURL downloadURL, String displayTitle, AttachmentParent parent, String name, SearchService.SearchCategory cat) - { - super(path); - - Container c = ContainerManager.getForId(parent.getContainerId()); - _containerId = new GUID(parent.getContainerId()); - if (null != c) - setSecurableResource(c); - _downloadUrl = downloadURL; - _parent = parent; - _name = name; - _docid = makeDocId(parent,name); - initSearchProperties(name, displayTitle, cat); - } - - @Override - public boolean allowInline() - { - if (isFile() && "text/html".equals(getContentType())) - return false; - return super.allowInline(); - } - - private void initSearchProperties(String name, @Nullable String displayTitle, @Nullable SearchService.SearchCategory cat) - { - setSearchProperty(SearchService.PROPERTY.keywordsMed, FileUtil.getSearchKeywords(name)); - setSearchProperty(SearchService.PROPERTY.title, null != displayTitle ? displayTitle : name); - - if (null == cat) - setSearchCategory(SearchService.fileCategory); - else - setSearchProperty(SearchService.PROPERTY.categories, SearchService.fileCategory + StringUtils.capitalize(cat.toString())); - } - - AttachmentResource(@NotNull WebdavResource folder, @NotNull AttachmentParent parent, @NotNull Attachment attachment) - { - this(folder, parent, attachment.getName()); - - _created = attachment.getCreated().getTime(); - _createdBy = UserManager.getUser(attachment.getCreatedBy()); - _cached = attachment; - } - - - AttachmentResource(@NotNull WebdavResource folder, @NotNull AttachmentParent parent, @NotNull String name) - { - super(folder.getPath(), name); - _containerId = new GUID(parent.getContainerId()); - _folder = folder; - Container c = ContainerManager.getForId(parent.getContainerId()); - if (c != null) - setSecurableResource(c); - _name = name; - _parent = parent; - _docid = makeDocId(parent,name); - - initSearchProperties(name, null, null); - } - - - @Override - public String getDocumentId() - { - return makeDocId(_parent, _name); - } - - Attachment getAttachment() - { - if (null != _cached) - return _cached; - return AttachmentService.get().getAttachment(_parent, _name); - } - - - @Override - public String getContentType() - { - return super.getContentType(); - } - - @Override - public String getExecuteHref(ViewContext context) - { - if (null != _downloadUrl) - return _downloadUrl.getLocalURIString(); - return super.getExecuteHref(context); - } - - @Override - public boolean exists() - { - Attachment r = getAttachment(); - if (r == null) - return false; - if (null != r.getFile()) - return r.getFile().exists(); - return true; - } - - @Override - public boolean isCollection() - { - return false; - } - - @Override - public boolean canRename(User user, boolean forRename) - { - return false; - } - - @Override - public boolean delete(User user) - { - if (user != null && !canDelete(user, true, null)) - return false; - AttachmentService.get().deleteAttachment(_parent, _name, user); - return true; - } - - @Override - public FileStream getFileStream(User user) throws IOException - { - Attachment r = getAttachment(); - if (null == r) - return null; - List files = getAttachmentFiles(_parent, Collections.singletonList(r)); - if (files.isEmpty()) - throw new FileNotFoundException(r.getName()); - return files.get(0); - } - - @Override - public InputStream getInputStream(User user) throws IOException - { - return AttachmentService.get().getInputStream(_parent, _name); - } - - @Override - public void moveFrom(User user, WebdavResource r) throws IOException, DavException - { - if (r instanceof AttachmentResource from) - { - if (from._parent == _parent) - { - renameAttachment(_parent, from.getName(), getName(), user); - return; - } - } - super.moveFrom(user, r); - } - - @Override - public long copyFrom(User user, final FileStream in) throws IOException - { - try - { - AttachmentFile file = new AttachmentFile() - { - @Override - public long getSize() throws IOException - { - return in.getSize(); - } - - @Override - public String getError() - { - return null; - } - - @Override - public String getFilename() - { - return getName(); - } - - @Override - public String getContentType() - { - return PageFlowUtil.getContentTypeFor(getFilename()); - } - - @Override - public InputStream openInputStream() throws IOException - { - return in.openInputStream(); - } - - @Override - public void closeInputStream() throws IOException - { - in.closeInputStream(); - } - }; - - if (AttachmentServiceImpl.this.exists(_parent,_name)) - deleteAttachment(_parent, _name, user); - addAttachments(_parent, Collections.singletonList(file), user); - } - catch (AttachmentService.DuplicateFilenameException x) - { - throw new IOException(x); - } - finally - { - in.closeInputStream(); - } - // UNDONE return real length if anyone cares - return 0; - } - - @Override - public WebdavResource parent() - { - return _folder; - } - - @Override - public long getCreated() - { - return _created; - } - - @Override - public User getCreatedBy() - { - return _createdBy; - } - - @Override - public long getLastModified() - { - return getCreated(); - } - - @Override - public User getModifiedBy() - { - return _createdBy; - } - - @Override - public long getContentLength() - { - Attachment a = getAttachment(); - if (null == a) - return 0; - - if (_parent instanceof AttachmentDirectory) - { - try - { - File dir = ((AttachmentDirectory)_parent).getFileSystemDirectory(); - File file = new File(dir,a.getName()); - return file.exists() ? file.length() : 0; - } - catch (MissingRootDirectoryException x) - { - return 0; - } - } - else - { - // UNDONE - // return a.getSize(); - try (InputStream is = getInputStream(null)) - { - if (null != is) - { - long size = 0; - if (is instanceof FileInputStream) - size = ((FileInputStream) is).getChannel().size(); - else if (is instanceof FilterInputStream) - size = is.available(); - - return size; - } - } - catch (IOException ignored) - { - } - return 0; - } - } - - @Override - public Set> getPermissions(User user) - { - return super.getPermissions(user); - } - - @Override - public File getFile() - { - if (_parent instanceof AttachmentDirectory) - { - try - { - File dir = ((AttachmentDirectory)_parent).getFileSystemDirectory(); - return new File(dir,getName()); - } - catch (MissingRootDirectoryException x) - { - return null; - } - } - return null; - } - - @Override - public String getAbsolutePath(User user) - { - Container container = null != getContainerId() ? ContainerManager.getForId(getContainerId()) : null; - if (null != container && SecurityManager.canSeeFilePaths(container, user)) - { - File file = getFile(); - return null != file ? file.getAbsolutePath() : null; - } - return null; - } - - - @Override - @NotNull - public List getHistory() - { - //noinspection unchecked - return Collections.EMPTY_LIST; - } - - @Override - public void setLastIndexed(long ms, long modified) - { - new SqlExecutor(CoreSchema.getInstance().getSchema()).execute(new SQLFragment( - "UPDATE core.Documents SET LastIndexed=? WHERE Parent=? and DocumentName=?", - new Date(ms), _parent.getEntityId(), _name)); - } - } - - private void checkSecurityPolicy(AttachmentParent attachmentParent) throws UnauthorizedException - { - // No-op: AttachmentParent no longer provides getSecurityPolicy() - } - - private void checkSecurityPolicy(User user, AttachmentParent attachmentParent) throws UnauthorizedException - { - // No-op: AttachmentParent no longer provides getSecurityPolicy() - } - - // - //JUnit TestCase - // - - @TestWhen(TestWhen.When.BVT) - public static class TestCase extends Assert - { - private static final String _testDirName = "/_jUnitAttachment"; - - @Test - public void testDirectories() throws IOException - { - User user = TestContext.get().getUser(); - assertNotNull("Should have access to a user", user); - - // clean up if anything was left over from last time - if (null != ContainerManager.getForPath(_testDirName)) - ContainerManager.deleteAll(ContainerManager.getForPath(_testDirName), user); - - Container proj = ContainerManager.ensureContainer(_testDirName, TestContext.get().getUser()); - Container folder = ContainerManager.ensureContainer(_testDirName + "/Test", TestContext.get().getUser()); - - FileContentService fileService = FileContentService.get(); - AttachmentService svc = AttachmentService.get(); - - File curRoot = fileService.getFileRoot(proj); - assertTrue(curRoot.isDirectory()); - - AttachmentDirectory attachParent = fileService.getMappedAttachmentDirectory(folder, true); - File attachDir = attachParent.getFileSystemDirectory(); - assertTrue(attachDir.exists()); - - MultipartFile f = new MockMultipartFile("file.txt", "file.txt", "text/plain", "Hello World".getBytes()); - Map fileMap = new HashMap<>(); - fileMap.put("file.txt", f); - List files = SpringAttachmentFile.createList(fileMap); - - svc.addAttachments(attachParent, files, user); - List att = svc.getAttachments(attachParent); - assertEquals(1, att.size()); - assertTrue(att.get(0).getFile().exists()); - - // test rename - String oldName = f.getName(); - String newName = "newname.txt"; - svc.renameAttachment(attachParent, oldName, newName, user); - assertNull(svc.getAttachment(attachParent, oldName)); - assertNotNull(svc.getAttachment(attachParent, newName)); - // put things back as we found them... - svc.renameAttachment(attachParent, newName, oldName, user); - - - assertTrue(new File(attachDir, UPLOAD_LOG).exists()); - - File otherDir = new File(attachDir, "subdir"); - otherDir.mkdir(); - AttachmentDirectory namedParent = fileService.registerDirectory(folder, "test", otherDir.getCanonicalPath(), false); - - AttachmentDirectory namedParentTest = fileService.getRegisteredDirectory(folder, "test"); - assertNotNull(namedParentTest); - assertSameFile(namedParentTest.getFileSystemDirectory(), namedParent.getFileSystemDirectory()); - - svc.addAttachments(namedParent, files, user); - att = svc.getAttachments(namedParent); - assertEquals(1, att.size()); - assertTrue(att.get(0).getFile().exists()); - assertSameFile(new File(otherDir, "file.txt"), att.get(0).getFile()); - assertTrue(new File(otherDir, UPLOAD_LOG).exists()); - - fileService.unregisterDirectory(folder, "test"); - namedParentTest = fileService.getRegisteredDirectory(folder, "test"); - assertNull(namedParentTest); - - File relativeDir = new File(attachDir, "subdir2"); - relativeDir.mkdirs(); - AttachmentDirectory relativeParent = fileService.registerDirectory(folder, "relative", FileUtil.getAbsoluteCaseSensitiveFile(relativeDir).getAbsolutePath(), false); - - AttachmentDirectory relativeParentTest = fileService.getRegisteredDirectory(folder, "relative"); - assertNotNull(relativeParentTest); - assertSameFile(relativeParentTest.getFileSystemDirectory(), relativeParent.getFileSystemDirectory()); - - svc.addAttachments(relativeParent, files, user); - att = svc.getAttachments(relativeParent); - assertEquals(1, att.size()); - - File expectedFile1 = att.get(0).getFile(); - File expectedFile2 = new File(relativeDir, UPLOAD_LOG); - - assertTrue(expectedFile1.exists()); - assertEquals(new File(relativeDir, "file.txt"), expectedFile1); - assertTrue(expectedFile2.exists()); - - - // Slight detour... test FileAttachmentFile using these just-created files before we delete them - testFileAttachmentFiles(expectedFile1, expectedFile2, user); - - // clean up - ContainerManager.deleteAll(proj, user); - } - - private void assertSameFile(File a, File b) - { - if (a.equals(b)) - return; - try - { - a = a.getCanonicalFile(); - b = b.getCanonicalFile(); - assertEquals(a,b); - } - catch (IOException x) - { - fail(x.getMessage()); - } - } - - private void testFileAttachmentFiles(File file1, File file2, User user) throws IOException - { - AttachmentFile aFile1 = new FileAttachmentFile(file1); - AttachmentFile aFile2 = new FileAttachmentFile(file2); - - AttachmentService service = AttachmentService.get(); - AttachmentParent root = AuthenticationLogoAttachmentParent.get(); - service.deleteAttachment(root, file1.getName(), user); - service.deleteAttachment(root, file2.getName(), user); - - List attachments = service.getAttachments(root); - int originalCount = attachments.size(); - - service.addAttachments(root, Arrays.asList(aFile1, aFile2), user); - attachments = service.getAttachments(root); - assertEquals((originalCount + 2), attachments.size()); - - service.deleteAttachment(root, file1.getName(), user); - attachments = service.getAttachments(root); - assertEquals((originalCount + 1), attachments.size()); - - service.deleteAttachment(root, file2.getName(), user); - attachments = service.getAttachments(root); - assertEquals(originalCount, attachments.size()); - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.core.attachment; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.attachments.Attachment; +import org.labkey.api.attachments.AttachmentDirectory; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.attachments.AttachmentParent; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.attachments.AttachmentType; +import org.labkey.api.attachments.DocumentWriter; +import org.labkey.api.attachments.FileAttachmentFile; +import org.labkey.api.attachments.SpringAttachmentFile; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.Sets; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.ColumnRenderProperties; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DatabaseTableType; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.Parameter; +import org.labkey.api.data.ResultSetView; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SchemaTableInfo; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.exp.Lsid; +import org.labkey.api.files.FileContentService; +import org.labkey.api.files.MissingRootDirectoryException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.AuthenticationLogoAttachmentParent; +import org.labkey.api.security.SecurableResource; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.settings.AppProps; +import org.labkey.api.test.TestWhen; +import org.labkey.api.util.ContainerUtil; +import org.labkey.api.util.FileStream; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.HtmlStringBuilder; +import org.labkey.api.util.MimeMap; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.Path; +import org.labkey.api.util.ResponseHelper; +import org.labkey.api.util.ResultSetUtil; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.URLHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.VBox; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartView; +import org.labkey.api.webdav.AbstractDocumentResource; +import org.labkey.api.webdav.AbstractWebdavResourceCollection; +import org.labkey.api.webdav.DavException; +import org.labkey.api.webdav.WebdavResolver; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.core.admin.AdminController; +import org.labkey.core.query.AttachmentAuditProvider; +import org.springframework.http.ContentDisposition; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.validation.BindException; +import org.springframework.web.multipart.MultipartFile; + +import java.beans.PropertyChangeEvent; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +public class AttachmentServiceImpl implements AttachmentService, ContainerManager.ContainerListener +{ + private static final String UPLOAD_LOG = ".upload.log"; + private static final Map ATTACHMENT_TYPE_MAP = new HashMap<>(); + private static final Set ATTACHMENT_COLUMNS = Set.of("Parent", "Container", "DocumentName", "DocumentSize", "DocumentType", "Created", "CreatedBy", "LastIndexed"); + + public AttachmentServiceImpl() + { + ContainerManager.addContainerListener(this); + } + + @Override + public void download(HttpServletResponse response, AttachmentParent parent, String filename, @Nullable String alias, boolean inlineIfPossible) throws ServletException, IOException + { + if (null == filename || filename.isEmpty()) + { + throw new NotFoundException(); + } + + boolean canInline = MimeMap.DEFAULT.canInlineFor(filename); + + // Default to rendering inline when possible, but let caller force download as an attachment + boolean asAttachment = !canInline || !inlineIfPossible; + + response.reset(); + writeDocument(new ResponseWriter(response), parent, filename, alias, asAttachment); + + User user = null; + try + { + ViewContext context = HttpView.currentContext(); + if (context != null) + user = context.getUser(); + } + catch (RuntimeException ignored) + { + } + + // Change in behavior added in 11.1: no longer audit download events for the guest user + if (null != user && !user.isGuest() && asAttachment) + { + addAuditEvent(user, parent, filename, "The attachment " + filename + " was downloaded"); + } + } + + + @Override + public void download(HttpServletResponse response, AttachmentParent parent, String filename, boolean inlineIfPossible) throws ServletException, IOException + { + download(response, parent, filename, null, inlineIfPossible); + } + + + @Override + public void addAuditEvent(User user, AttachmentParent parent, String filename, String comment) + { + if (user == null) + throw new IllegalArgumentException("Cannot create attachment audit events for the null user."); + + if (parent != null) + { + Container c = ContainerManager.getForId(parent.getContainerId()); + AttachmentAuditProvider.AttachmentAuditEvent attachmentEvent = new AttachmentAuditProvider.AttachmentAuditEvent(c == null ? ContainerManager.getRoot() : c, comment); + + attachmentEvent.setAttachmentParentEntityId(parent.getEntityId()); + attachmentEvent.setAttachment(filename); + + AuditLogService.get().addEvent(user, attachmentEvent); + + if (parent instanceof AttachmentDirectory adParent) + { + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(c, comment); + try + { + event.setDirectory(adParent.getFileSystemDirectory().getPath()); + } + catch (MissingRootDirectoryException ex) + { + // UNDONE: AttachmentDirectory.getFileSystemPath()... + event.setDirectory("path not found"); + } + event.setFile(filename); + AuditLogService.get().addEvent(user, event); + } + } + } + + @Override + public HttpView getHistoryView(ViewContext context, AttachmentParent parent, BindException errors) + { + UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); + if (schema != null) + { + checkSecurityPolicy(context.getUser(), parent); + QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(AttachmentAuditProvider.COLUMN_NAME_CONTAINER), parent.getContainerId()); + filter.addCondition(FieldKey.fromParts(AttachmentAuditProvider.COLUMN_NAME_ATTACHMENT_PARENT_ENTITY_ID), parent.getEntityId()); + + settings.setBaseFilter(filter); + settings.setQueryName(AttachmentService.ATTACHMENT_AUDIT_EVENT); + + QueryView view = schema.createView(context, settings, errors); + view.setTitle("Attachments History:"); + + return view; + } + return null; + } + + private void validateAttachmentSize(AttachmentFile file) throws IOException + { + int maxSize = AppProps.getInstance().getMaxBLOBSize(); + if (file.getSize() > maxSize) + { + throw new AttachmentService.FileTooLargeException(file, maxSize); + } + } + + @Override + public void validateAttachmentSizes(AttachmentParent parent, List files) throws IOException + { + File fileLocation = parent instanceof AttachmentDirectory ? ((AttachmentDirectory) parent).getFileSystemDirectory() : null; + + // Only validate if we're putting the file in the database + if (null == fileLocation) + { + for (AttachmentFile file : files) + { + validateAttachmentSize(file); + } + } + } + + + @Override + public synchronized void addAttachments(AttachmentParent parent, List files, @NotNull User user) throws IOException + { + if (null == user) + throw new IllegalArgumentException("Cannot add attachments for the null user"); + + if (null == files || files.isEmpty()) + return; + + List duplicates = findDuplicates(files); + if (duplicates.size() > 0) + { + throw new AttachmentService.DuplicateFilenameException(duplicates); + } + + Set filesToSkip = new TreeSet<>(); + File fileLocation = parent instanceof AttachmentDirectory ? ((AttachmentDirectory) parent).getFileSystemDirectory() : null; + + for (AttachmentFile file : files) + { + if (parent != null && exists(parent, file.getFilename())) + { + filesToSkip.add(file.getFilename()); + continue; + } + + HashMap hm = new HashMap<>(); + if (null == fileLocation) + { + validateAttachmentSize(file); + hm.put("Document", file); + } + else + ((AttachmentDirectory)parent).addAttachment(user, file); + + hm.put("DocumentName", file.getFilename()); + hm.put("DocumentSize", file.getSize()); + hm.put("DocumentType", file.getContentType()); + hm.put("Parent", parent.getEntityId()); + hm.put("Container", parent.getContainerId()); + Table.insert(user, coreTables().getTableInfoDocuments(), hm); + + addAuditEvent(user, parent, file.getFilename(), "The attachment " + file.getFilename() + " was added"); + } + + AttachmentCache.removeAttachments(parent); + + if (!filesToSkip.isEmpty()) + throw new AttachmentService.DuplicateFilenameException(filesToSkip); + } + + @Override + public HttpView getErrorView(List files, BindException errors, URLHelper returnUrl) + { + boolean hasErrors = null != errors && errors.hasErrors(); + HtmlString errorHtml = getErrorHtml(files); // TODO: Get rid of getErrorHtml() -- use errors collection + + if (null == errorHtml && !hasErrors) + return null; + + try + { + return new ErrorView(errorHtml, errors, returnUrl); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + + private @Nullable HtmlString getErrorHtml(List files) + { + HtmlStringBuilder builder = HtmlStringBuilder.of(); + + for (AttachmentFile file : files) + { + String error = file.getError(); + + if (null != error) + builder.append(error).unsafeAppend("

    "); + } + + HtmlString html = builder.getHtmlString(); + + return HtmlString.isEmpty(html) ? null : html; + } + + + public static class ErrorView extends JspView + { + public HtmlString errorHtml; + public URLHelper returnUrl; + + private ErrorView(HtmlString errorHtml, BindException errors, URLHelper returnUrl) + { + super("/org/labkey/core/attachment/showErrors.jsp", new Object(), errors); + this.errorHtml = errorHtml; + this.returnUrl = returnUrl; + } + } + + + @Override + public void deleteAttachments(AttachmentParent parent) + { + deleteAttachments(Collections.singleton(parent)); + } + + @Override + public void deleteAttachments(Collection parents) + { + for (AttachmentParent parent : parents) + { + List atts = getAttachments(parent); + + // No attachments, or perhaps container doesn't match entityid + if (atts.isEmpty()) + continue; + + checkSecurityPolicy(parent); // Only check policy if there are attachments (a client may delete attachment and policy, but attempt to delete again) + deleteIndexedAttachments(parent, atts); + + new SqlExecutor(coreTables().getSchema()).execute(sqlCascadeDelete(parent)); + if (parent instanceof AttachmentDirectory) + ((AttachmentDirectory)parent).deleteAttachment(HttpView.currentContext().getUser(), null); + AttachmentCache.removeAttachments(parent); + } + } + + @Override + public void deleteIndexedAttachments(List parentIds) + { + TableSelector ts = new TableSelector(CoreSchema.getInstance().getTableInfoDocuments(), + PageFlowUtil.set("Parent", "DocumentName"), + new SimpleFilter(FieldKey.fromParts("Parent"), parentIds, CompareType.IN), null); + + try (ResultSet rs = ts.getResultSet()) + { + while (rs.next()) + { + deleteIndexedAttachment(rs.getString("Parent"), rs.getString("DocumentName")); + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + } + + @Override + public void clearLastIndexed(List parentIds) + { + SimpleFilter filter = new SimpleFilter(new SimpleFilter.InClause(FieldKey.fromParts("Parent"), parentIds)) + .addClause(new SimpleFilter.SQLClause("LastIndexed IS NOT NULL", null)); + SQLFragment sql = new SQLFragment("UPDATE core.Documents SET LastIndexed = NULL ") + .append(filter.getSQLFragment(CoreSchema.getInstance().getSqlDialect())); + new SqlExecutor(CoreSchema.getInstance().getSchema()).execute(sql); + } + + private void deleteIndexedAttachment(AttachmentParent parent, String name) + { + deleteIndexedAttachment(parent.getEntityId(), name); + } + + private void deleteIndexedAttachment(String parent, String name) + { + SearchService.get().deleteResource(makeDocId(parent, name)); + new SqlExecutor(CoreSchema.getInstance().getSchema()).execute(new SQLFragment( + "UPDATE core.Documents SET LastIndexed = NULL WHERE LastIndexed IS NOT NULL AND Parent = ? AND DocumentName = ?", parent, name) + ); + } + + private void deleteIndexedAttachments(AttachmentParent parent, List atts) + { + for (Attachment att : atts) + deleteIndexedAttachment(parent, att.getName()); + } + + @Override + public void deleteIndexedAttachments(AttachmentParent parent) + { + List atts = getAttachments(parent); + deleteIndexedAttachments(parent, atts); + } + + private void _deleteAttachment(AttachmentParent parent, String name, @Nullable User auditUser) + { + checkSecurityPolicy(auditUser, parent); // Only check policy if there are attachments (a client may delete attachment and policy, but attempt to delete again) + deleteIndexedAttachment(parent, name); + + new SqlExecutor(coreTables().getSchema()).execute(sqlDelete(parent, name)); + if (parent instanceof AttachmentDirectory) + ((AttachmentDirectory)parent).deleteAttachment(auditUser, name); + + if (null != auditUser) + addAuditEvent(auditUser, parent, name, "The attachment " + name + " was deleted"); + } + + + @Override + public void deleteAttachment(AttachmentParent parent, String name, @Nullable User auditUser) + { + Attachment att = getAttachmentHelper(parent, name); + + if (null != att) + { + _deleteAttachment(parent, name, auditUser); + AttachmentCache.removeAttachments(parent); + } + } + + @Override + public void deleteAttachments(AttachmentParent parent, Collection names, @Nullable User auditUser) + { + Map attachmentMap = getAttachments(parent, names); + + for (Attachment attachment : attachmentMap.values()) + { + _deleteAttachment(parent, attachment.getName(), null); + } + + AttachmentCache.removeAttachments(parent); + } + + + @Override + public void renameAttachment(AttachmentParent parent, String oldName, String newName, User auditUser) throws IOException + { + File dir = null; + File dest = null; + File src = null; + + checkSecurityPolicy(auditUser, parent); + if (parent instanceof AttachmentDirectory) + { + dir = ((AttachmentDirectory)parent).getFileSystemDirectory(); + src = new File(dir,oldName); + dest = new File(dir,newName); + if (!src.exists()) + throw new FileNotFoundException(oldName); + if (dest.exists()) + throw new AttachmentService.DuplicateFilenameException(newName); + + // make sure newName attachment doesn't exist. if it does exist, it's already orphaned + deleteAttachment(parent, newName, null); + } + + if (exists(parent, newName)) + throw new AttachmentService.DuplicateFilenameException(newName); + + new SqlExecutor(coreTables().getSchema()).execute(sqlRename(parent, oldName, newName)); + + // rename the file in the filesystem only if an Attachment directory and the db rename succeeded + if (null != dir) + src.renameTo(dest); + + AttachmentCache.removeAttachments(parent); + + addAuditEvent(auditUser, parent, newName, "The attachment " + oldName + " was renamed " + newName); + } + + + // Copies an attachment -- same container, same parent, but new name. + @Override + public void copyAttachment(AttachmentParent parent, Attachment a, String newName, User auditUser) throws IOException + { + checkSecurityPolicy(auditUser, parent); + a.setName(newName); + DatabaseAttachmentFile file = new DatabaseAttachmentFile(a); + addAttachments(parent, Collections.singletonList(file), auditUser); + } + + @Override + public void moveAttachments(Container newContainer, List parents, User auditUser) throws IOException + { + for (AttachmentParent parent : parents) + { + checkSecurityPolicy(auditUser, parent); + int rowsChanged = new SqlExecutor(coreTables().getSchema()).execute(sqlMove(parent, newContainer)); + if (rowsChanged > 0) + { + List atts = getAttachments(parent); + String filename; + for (Attachment att : atts) + { + filename = att.getName(); + if (parent instanceof AttachmentDirectory parentDir) + { + File currentDir = parentDir.getFileSystemDirectoryPath().toFile(); + File newDir = parentDir.getFileSystemDirectoryPath(newContainer, true).toFile(); + File src = new File(currentDir, filename); + File dest = new File(newDir, filename); + if (!src.exists()) + throw new FileNotFoundException(src.getAbsolutePath()); + if (dest.exists()) + throw new AttachmentService.DuplicateFilenameException(dest.getAbsolutePath()); + } + deleteIndexedAttachment(parent, filename); + addAuditEvent(auditUser, parent, filename, "The attachment " + filename + " was moved"); + } + AttachmentCache.removeAttachments(parent); + } + } + } + + /** may return fewer AttachmentFile than Attachment, if there have been deletions */ + @Override + public @NotNull List getAttachmentFiles(AttachmentParent parent, Collection attachments) throws IOException + { + checkSecurityPolicy(parent); + List files = new ArrayList<>(attachments.size()); + + for (Attachment attachment : attachments) + { + if (parent instanceof AttachmentDirectory) + { + File f = new File(((AttachmentDirectory)parent).getFileSystemDirectory(), attachment.getName()); + files.add(new FileAttachmentFile(f)); + } + else + { + try + { + files.add(new DatabaseAttachmentFile(attachment)); + } + catch (FileNotFoundException x) + { + // + } + } + } + return files; + } + + + private boolean exists(AttachmentParent parent, String filename) + { + return null != getAttachmentHelper(parent, filename); + } + + private List findDuplicates(List files) + { + Set fileNames = new HashSet<>(); + List duplicates = new ArrayList<>(); + for (AttachmentFile file : files) + { + if (!fileNames.add(file.getFilename())) + { + duplicates.add(file.getFilename()); + } + } + return duplicates; + } + + @Override + public @NotNull List getAttachments(AttachmentParent parent) + { + checkSecurityPolicy(parent); + Map mapFromDatabase = AttachmentCache.getAttachments(parent); + List attachmentsFromDatabase = Collections.unmodifiableList(new ArrayList<>(mapFromDatabase.values())); + + File parentDir = null; + + try + { + parentDir = parent instanceof AttachmentDirectory ? ((AttachmentDirectory) parent).getFileSystemDirectory() : null; + } + catch (MissingRootDirectoryException ex) + { + /* no problem */ + } + + if (null == parentDir || !parentDir.exists()) + return attachmentsFromDatabase; + + for (Attachment att : attachmentsFromDatabase) + att.setFile(new File(parentDir, att.getName())); + + //OK, make sure that the list really reflects what is in the file system. + List attList = new ArrayList<>(); + + File[] fileList = parentDir.listFiles(file -> !file.isDirectory() && !(file.getName().charAt(0) == '.') && !file.isHidden()); + + Set attachmentNames = new CaseInsensitiveHashSet(); + + for (Attachment attachment : attachmentsFromDatabase) + { + attachmentNames.add(attachment.getName()); + attList.add(attachment); + } + + if (null != fileList) + { + for (File file : fileList) + { + if (!attachmentNames.contains(file.getName())) + attList.add(attachmentFromFile(parent, file)); + } + } + + return Collections.unmodifiableList(attList); + } + + + /** Does not work for file system parents */ + @Override + public List> listAttachmentsForIndexing(Collection parents, Date modifiedSince) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Parent"), parents, CompareType.IN); + var since = new SearchService.LastIndexedClause(coreTables().getTableInfoDocuments(), modifiedSince, null); + + if (!since.isEmpty()) + filter.addClause(since); + + final ArrayList> ret = new ArrayList<>(); + + new TableSelector(coreTables().getTableInfoDocuments(), + PageFlowUtil.set("Parent", "DocumentName", "LastIndexed"), + filter, + new Sort("+Created")).forEach(rs -> { + String parent = rs.getString(1); + String name = rs.getString(2); + Date last = rs.getTimestamp(3); + if (last != null && last.getTime() == SearchService.failDate.getTime()) + return; + ret.add(new Pair<>(parent, name)); + }); + + return ret; + } + + /** Collection resource with all attachments for this parent */ + @Override + public WebdavResource getAttachmentResource(Path path, AttachmentParent parent) + { + // NOTE parent does not supply ACL, but should? + // acl = parent.getAcl() + checkSecurityPolicy(parent); + Container c = ContainerManager.getForId(parent.getContainerId()); + if (null == c) + return null; + + return new AttachmentCollection(path, parent, c); + } + + @Override + public WebdavResource getDocumentResource(Path path, ActionURL downloadURL, String displayTitle, AttachmentParent parent, String name, SearchService.SearchCategory cat) + { + checkSecurityPolicy(parent); + return new AttachmentResource(path, downloadURL, displayTitle, parent, name, cat); + } + + private Attachment attachmentFromFile(AttachmentParent parent, File file) + { + Attachment attachment = new Attachment(); + attachment.setParent(parent.getEntityId()); + attachment.setContainer(parent.getContainerId()); + attachment.setDocumentName(file.getName()); + attachment.setCreated(new Date(file.lastModified())); + attachment.setFile(file); + + return attachment; + } + + @Override + public void registerAttachmentType(AttachmentType type) + { + ATTACHMENT_TYPE_MAP.put(type.getUniqueName(), type); + } + + @Override + public HttpView getAdminView(ActionURL currentUrl) + { + String requestedType = currentUrl.getParameter("type"); + AttachmentType attachmentType = null != requestedType ? ATTACHMENT_TYPE_MAP.get(requestedType) : null; + + if (null == attachmentType) + { + boolean findAttachmentParents = "1".equals(currentUrl.getParameter("find")); + + // The first query lists all the attachment types and the attachment counts for each. A separate select from + // core.Documents for each type is needed to associate the Type values with the associated rows. + List selectStatements = new LinkedList<>(); + + for (AttachmentType type : ATTACHMENT_TYPE_MAP.values()) + { + SQLFragment selectStatement = new SQLFragment(); + + // Adding unique column RowId ensures we get the proper count + selectStatement.append("SELECT RowId, CAST(").appendValue(type.getUniqueName()).append(" AS VARCHAR(500)) AS Type FROM ") + .append(CoreSchema.getInstance().getTableInfoDocuments(), "d") + .append(" WHERE "); + addAndVerifyWhereSql(type, selectStatement); + selectStatement.append("\n"); + + selectStatements.add(selectStatement); + } + + SQLFragment allSql = new SQLFragment("SELECT Type, COUNT(*) AS Count FROM (\n"); + allSql.append(SQLFragment.join(selectStatements, "UNION\n")); + allSql.append(") u\nGROUP BY Type\nORDER BY Type"); + ActionURL linkUrl = currentUrl.clone().deleteParameters().addParameter("type", null); + + // The second query shows all attachments that we can't associate with a type. We just need to assemble a big + // WHERE NOT clause that ORs the conditions from every registered type. + SQLFragment whereSql = new SQLFragment(); + String sep = ""; + + for (AttachmentType type : ATTACHMENT_TYPE_MAP.values()) + { + whereSql.append(sep); + sep = " OR"; + whereSql.append("\n("); + addAndVerifyWhereSql(type, whereSql); + whereSql.append(")"); + } + + SQLFragment unknownSql = new SQLFragment("SELECT d.Container, c.Name, d.Parent, d.DocumentName"); + + if (findAttachmentParents) + unknownSql.append(", e.TableName"); + + unknownSql.append(" FROM core.Documents d\n"); + unknownSql.append("INNER JOIN core.Containers c ON c.EntityId = d.Container\n"); + + Set schemasToIgnore = Sets.newCaseInsensitiveHashSet(currentUrl.getParameterValues("ignore")); + + if (findAttachmentParents) + { + unknownSql.append("LEFT OUTER JOIN (\n"); + addSelectAllEntityIdsSql(unknownSql, schemasToIgnore); + unknownSql.append(") e ON e.EntityId = d.Parent\n"); + } + + unknownSql.append("WHERE NOT ("); + unknownSql.append(whereSql); + unknownSql.append(")\n"); + unknownSql.append("ORDER BY Container, Parent, DocumentName"); + + WebPartView unknownView = getResultSetView(unknownSql, "Unknown Attachments", null); + NavTree navMenu = new NavTree(); + + if (!findAttachmentParents) + { + navMenu.addChild(new NavTree("Search for Attachment Parents (Be Patient)", + new ActionURL(AdminController.AttachmentsAction.class, ContainerManager.getRoot()).addParameter("find", 1).addParameter("ignore", "Audit")) + ); + } + else + { + navMenu.addChild(new NavTree("Remove TableName Column", + new ActionURL(AdminController.AttachmentsAction.class, ContainerManager.getRoot())) + ); + + if (schemasToIgnore.isEmpty()) + { + navMenu.addChild(new NavTree("Ignore Audit Schema", + new ActionURL(AdminController.AttachmentsAction.class, ContainerManager.getRoot()).addParameter("find", 1).addParameter("ignore", "Audit")) + ); + } + else + { + navMenu.addChild(new NavTree("Include All Schemas", + new ActionURL(AdminController.AttachmentsAction.class, ContainerManager.getRoot()).addParameter("find", 1)) + ); + } + } + unknownView.setNavMenu(navMenu); + + return new VBox(getResultSetView(allSql, "Attachment Types and Counts", linkUrl), unknownView); + } + else + { + // This query lists all the documents associated with a single type. + SQLFragment oneTypeSql = new SQLFragment("SELECT d.Container, c.Name, d.Parent, d.DocumentName FROM core.Documents d\n" + + "INNER JOIN core.Containers c ON c.EntityId = d.Container\n" + + "WHERE "); + addAndVerifyWhereSql(attachmentType, oneTypeSql); + oneTypeSql.append("\nORDER BY Container, Parent, DocumentName"); + + return getResultSetView(oneTypeSql, attachmentType.getUniqueName() + " Attachments", null); + } + } + + private void addAndVerifyWhereSql(AttachmentType attachmentType, SQLFragment sql) + { + int initialLength = sql.length(); + attachmentType.addWhereSql(sql, "d.Parent", "d.DocumentName"); + if (initialLength == sql.length()) + throw new UnsupportedOperationException("AttachmentType: '" + attachmentType.getUniqueName() + "' did not update attachment WHERE clause."); + } + + @Override + // Joins each row of core.Documents to the table(s) (if any) that contain an entityid matching the document's parent + public HttpView getFindAttachmentParentsView() + { + SQLFragment sql = new SQLFragment("SELECT RowId, CreatedBy, Created, ModifiedBy, Modified, Container, DocumentName, TableName FROM core.Documents LEFT OUTER JOIN (\n"); + addSelectAllEntityIdsSql(sql, Sets.newCaseInsensitiveHashSet("Audit")); + sql.append(") c ON EntityId = Parent\nORDER BY TableName, DocumentName, Container"); + + return getResultSetView(sql, "Probable Attachment Parents", null); + } + + // Creates a two-column query of ID and table name that selects from every possible attachment parent column in the labkey database: + // - Enumerate all tables in all schemas in the labkey scope + // - Enumerate columns and identify potential attachment parents (currently, EntityId columns and ObjectIds extracted from LSIDs) + // - Create a UNION query that selects the candidate ids along with a constant column that lists the table name + private void addSelectAllEntityIdsSql(SQLFragment sql, Set userRequestedSchemasToIgnore) + { + List selectStatements = new LinkedList<>(); + Set schemasToIgnore = Sets.newCaseInsensitiveHashSet(userRequestedSchemasToIgnore); + + // Temp schema causes problems because materialized tables disappear but stay in the cached list. This is probably a bug with + // MaterializedQueryHelper... it should clear the temp DbSchema when it deletes a temp table. TODO: fix MQH & remove this workaround + schemasToIgnore.add("temp"); + + DbScope.getLabKeyScope().getSchemaNames().stream() + .filter(schemaName->!schemasToIgnore.contains(schemaName)) // Exclude unwanted schema names + .map(schemaName->DbSchema.get(schemaName, DbSchemaType.Bare)) + .forEach(schema-> schema.getTableNames().stream() + .map(schema::getTable) + .filter(table->table.getTableType() == DatabaseTableType.TABLE) // We just want the underlying tables (no views or virtual tables) + .map(SchemaTableInfo::getColumns) + .flatMap(Collection::stream) + .filter(ColumnRenderProperties::isStringType) + .forEach(c->addSelectStatement(selectStatements, c)) + ); + + sql.append(StringUtils.join(selectStatements, " UNION\n")); + } + + private void addSelectStatement(List selectStatements, ColumnInfo column) + { + String expression; + String where = null; + + if (StringUtils.containsIgnoreCase(column.getName(), "EntityId")) + { + // TODO convert all this to use SQLFragment + expression = column.getSelectIdentifier().getSql().getRawSQL(); + } + else if (StringUtils.endsWithIgnoreCase(column.getName(), "LSID")) + { + Pair pair = Lsid.getSqlExpressionToExtractObjectId(column.getSelectIdentifier().getSql().getRawSQL(), column.getSqlDialect()); + expression = pair.first; + where = pair.second; + } + else + { + return; + } + + TableInfo table = column.getParentTable(); + selectStatements.add(" SELECT " + expression + " AS EntityId, " + table.getSqlDialect().quoteStringLiteral(table.getSelectName()) + " AS TableName FROM " + table.getSelectName() + (null != where ? " WHERE " + where : "") + "\n"); + } + + private WebPartView getResultSetView(SQLFragment sql, String title, @Nullable ActionURL linkUrl) + { + SqlSelector selector = new SqlSelector(DbScope.getLabKeyScope(), sql); + ResultSet rs = selector.getResultSet(); + + return null != linkUrl ? new ResultSetView(rs, title, "Type", linkUrl) : new ResultSetView(rs, title); + } + + @Override + public @Nullable Attachment getAttachment(AttachmentParent parent, String name) + { + checkSecurityPolicy(parent); + return getAttachmentHelper(parent, name); + } + + @Override + public Map getAttachments(AttachmentParent parent, Collection names) + { + checkSecurityPolicy(parent); + + if (names == null || names.isEmpty()) + return Collections.emptyMap(); + + Map attachments = new HashMap<>(); + + if (parent instanceof AttachmentDirectory) + { + for (Attachment attachment : getAttachments(parent)) + if (names.contains(attachment.getName())) + attachments.put(attachment.getName(), attachment); + } + else + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Parent"), parent.getEntityId()); + filter.addCondition(FieldKey.fromParts("DocumentName"), names, CompareType.IN); + // Note: we are intentionally skipping the AttachmentCache here. If we hit the cache here while getting + // attachments from the global attachment parent we'll load every single attachment into the cache first, + // which could be very expensive on servers that have a ton of attachments. + List attachmentsList = new TableSelector(CoreSchema.getInstance().getTableInfoDocuments(), + ATTACHMENT_COLUMNS, + filter, + new Sort("+RowId")).getArrayList(Attachment.class); + + for (Attachment attachment : attachmentsList) + attachments.put(attachment.getName(), attachment); + } + + return attachments; + } + + private @Nullable Attachment getAttachmentHelper(AttachmentParent parent, String name) + { + if (parent instanceof AttachmentDirectory) + { + for (Attachment attachment : getAttachments(parent)) + if (name.equals(attachment.getName())) + return attachment; + + return null; + } + else + { + return AttachmentCache.getAttachments(parent).get(name); + } + } + + + @Override + public void containerCreated(Container c, User user) + { + } + + + @Override + public void propertyChange(PropertyChangeEvent propertyChangeEvent) + { + } + + @Override + public void containerDeleted(Container c, User user) + { + // TODO: do we need to get each document and remove its security policy? + ContainerUtil.purgeTable(coreTables().getTableInfoDocuments(), c, null); + AttachmentCache.removeAttachments(c); + } + + @Override + public void containerMoved(Container c, Container oldParent, User user) + { + } + + @NotNull + @Override + public Collection canMove(Container c, Container newParent, User user) + { + return Collections.emptyList(); + } + + private void writeDocument(DocumentWriter writer, AttachmentParent parent, String name, @Nullable String alias, boolean asAttachment) throws ServletException, IOException + { + checkSecurityPolicy(parent); + Connection conn = null; + PreparedStatement stmt = null; + ResultSet rs = null; + DbSchema schema = coreTables().getSchema(); + + if (alias == null) + alias = name; + + try (Parameter.ParameterList jdbcParameters = new Parameter.ParameterList()) + { + // we don't want a RowSet, so execute directly (not Table.executeQuery()) + conn = schema.getScope().getConnection(); + if (null == parent.getEntityId()) + stmt = Table.prepareStatement(conn, sqlRootDocument(), Collections.singletonList(name), jdbcParameters); + else + stmt = Table.prepareStatement(conn, sqlDocument(), Arrays.asList(parent.getContainerId(), parent.getEntityId(), name), jdbcParameters); + + rs = stmt.executeQuery(); + + OutputStream out; + InputStream s; + + if (parent instanceof AttachmentDirectory) + { + File parentDir = ((AttachmentDirectory) parent).getFileSystemDirectory(); + if (!parentDir.exists()) + throw new NotFoundException("No parent directory for downloaded file " + alias + ". Please contact an administrator."); + File file = new File(parentDir, name); + if (!file.exists()) + throw new NotFoundException("Could not find file " + alias); + + if (asAttachment) + writer.setContentDisposition(ContentDisposition.attachment().filename(alias, StandardCharsets.UTF_8).build()); + s = new FileInputStream(file); + } + else + { + if (!rs.next()) + { + throw new NotFoundException(); + } + + writer.setContentType(rs.getString("DocumentType")); + if (asAttachment) + writer.setContentDisposition(ContentDisposition.builder("attachment").filename(alias, StandardCharsets.UTF_8).build()); + else + writer.setContentDisposition(ContentDisposition.builder("inline").filename(alias, StandardCharsets.UTF_8).build()); + + int size = rs.getInt("DocumentSize"); + if (size > 0) + writer.setContentLength(size); + + s = rs.getBinaryStream("Document"); + if (null == s) + return; + } + + out = writer.getOutputStream(); + + try + { + IOUtils.copy(s, out); + } + finally + { + s.close(); + } + } + catch (SQLException x) + { + throw new ServletException(x); + } + finally + { + ResultSetUtil.close(rs); + ResultSetUtil.close(stmt); + if (conn != null) + { + schema.getScope().releaseConnection(conn); + } + } + } + + // CONSIDER: Return success/failure notification so caller can take action (render a default document) in all the failure scenarios. + @Override + public void writeDocument(DocumentWriter writer, AttachmentParent parent, String name, boolean asAttachment) throws ServletException, IOException + { + writeDocument(writer, parent, name, null, asAttachment); + } + + + @Override + @NotNull + public InputStream getInputStream(AttachmentParent parent, String name) throws FileNotFoundException + { + checkSecurityPolicy(parent); + Connection conn = null; + PreparedStatement stmt = null; + ResultSet rs = null; + final DbSchema schema = coreTables().getSchema(); + + try (Parameter.ParameterList jdbcParameters = new Parameter.ParameterList()) + { + // we don't want a RowSet, so execute directly (not Table.executeQuery()) + conn = schema.getScope().getConnection(); + if (null == parent.getEntityId()) + stmt = Table.prepareStatement(conn, sqlRootDocument(), Collections.singletonList(name), jdbcParameters); + else + stmt = Table.prepareStatement(conn, sqlDocument(), Arrays.asList(parent.getContainerId(), parent.getEntityId(), name), jdbcParameters); + + rs = stmt.executeQuery(); + + if (parent instanceof AttachmentDirectory) + { + File parentDir = ((AttachmentDirectory) parent).getFileSystemDirectory(); + if (!parentDir.exists()) + throw new FileNotFoundException("No parent directory for downloaded file " + name + ". Please contact an administrator."); + File file = new File(parentDir, name); + stmt.close(); + stmt = null; + rs.close(); + rs = null; + return new FileInputStream(file); + } + else + { + if (!rs.next()) + throw new FileNotFoundException(name); + final int size = rs.getInt("DocumentSize"); + InputStream is = rs.getBinaryStream("Document"); + + final Connection fconn = conn; + final PreparedStatement fstmt = stmt; + final ResultSet frs = rs; + InputStream ret = new FilterInputStream(is) + { + @Override + public void close() throws IOException + { + ResultSetUtil.close(frs); + ResultSetUtil.close(fstmt); + schema.getScope().releaseConnection(fconn); + super.close(); + } + + // slight hack here to get the size cheaply + @Override + public int available() + { + return size; + } + }; + stmt = null; + rs = null; + conn = null; + return ret; + } + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + finally + { + ResultSetUtil.close(rs); + ResultSetUtil.close(stmt); + if (null != conn) schema.getScope().releaseConnection(conn); + } + } + + + private CoreSchema coreTables() + { + return CoreSchema.getInstance(); + } + + private String sqlDocument() + { + return "SELECT DocumentType, DocumentSize, Document FROM " + coreTables().getTableInfoDocuments() + " WHERE Container = ? AND Parent = ? AND DocumentName = ?"; + } + + private String sqlRootDocument() + { + return "SELECT DocumentType, DocumentSize, Document FROM " + coreTables().getTableInfoDocuments() + " WHERE Parent IS NULL AND DocumentName = ?"; + } + + private SQLFragment sqlCascadeDelete(AttachmentParent parent) + { + return new SQLFragment("DELETE FROM " + coreTables().getTableInfoDocuments() + " WHERE Container = ? AND Parent = ?", parent.getContainerId(), parent.getEntityId()); + } + + private SQLFragment sqlDelete(AttachmentParent parent, String name) + { + return new SQLFragment("DELETE FROM " + coreTables().getTableInfoDocuments() + " WHERE Container = ? AND Parent = ? AND DocumentName = ?", parent.getContainerId(), parent.getEntityId(), name); + } + + private SQLFragment sqlRename(AttachmentParent parent, String oldName, String newName) + { + return new SQLFragment("UPDATE " + coreTables().getTableInfoDocuments() + " SET DocumentName = ? WHERE Container = ? AND Parent = ? AND DocumentName = ?", + newName, parent.getContainerId(), parent.getEntityId(), oldName); + } + + private SQLFragment sqlMove(AttachmentParent parent, Container newContainer) + { + // TODO: consider an inClause + return new SQLFragment("UPDATE " + coreTables().getTableInfoDocuments() + " SET Container = ? WHERE Container = ? AND Parent=?", + newContainer.getEntityId(), parent.getContainerId(), parent.getEntityId()); + } + + private static class ResponseWriter implements DocumentWriter + { + private final HttpServletResponse _response; + + public ResponseWriter(HttpServletResponse response) + { + _response = response; + } + + @Override + public void setContentType(String contentType) + { + _response.setContentType(contentType); + } + + @Override + public void setContentDisposition(ContentDisposition value) + { + ResponseHelper.setContentDisposition(_response, value); + } + + @Override + public void setContentLength(int size) + { + _response.setContentLength(size); + } + + @Override + public OutputStream getOutputStream() throws IOException + { + return _response.getOutputStream(); + } + } + + /** two cases + * regular file attachments: assume that this collection resource will be wrapped rather then exposed directly in the tree + * filesets: expect that this will be directly exposed in the tree + */ + private class AttachmentCollection extends AbstractWebdavResourceCollection + { + private final AttachmentParent _parent; + + AttachmentCollection(Path path, AttachmentParent parent, SecurableResource resource) + { + super(path); + _parent = parent; + setSecurableResource(resource); + } + + + @Override + public boolean exists() + { + FileContentService svc = FileContentService.get(); + if (_parent instanceof AttachmentDirectory) + { + if (null == ((AttachmentDirectory)_parent).getName()) + { + try + { + return null != svc.getMappedAttachmentDirectory(ContainerManager.getForId(_parent.getContainerId()), false); + } + catch (MissingRootDirectoryException x) + { + return false; + } + } + else + { + return null != svc.getRegisteredDirectory(ContainerManager.getForId(_parent.getContainerId()), ((AttachmentDirectory)_parent).getName()); + } + } + return true; + } + + + @Override + public WebdavResource find(Path.Part name) + { + Attachment a = getAttachment(_parent, name.toString()); + + if (null != a) + return new AttachmentResource(this, _parent, a); + else + return new AttachmentResource(this, _parent, name.toString()); + } + + + @Override + public Collection listNames() + { + List attachments = getAttachments(_parent); + ArrayList names = new ArrayList<>(attachments.size()); + + for (Attachment a : attachments) + { + if (null != a.getFile() && !a.getFile().exists()) + continue; + names.add(a.getName()); + } + + return names; + } + + + @Override + public Collection list() + { + List attachments = getAttachments(_parent); + ArrayList resources = new ArrayList<>(attachments.size()); + + for (Attachment a : attachments) + { + if (null != a.getFile() && !a.getFile().exists()) + continue; + resources.add(new AttachmentResource(this, _parent, a)); + } + + return resources; + } + } + + private static String makeDocId(AttachmentParent parent, String name) + { + return makeDocId(parent.getEntityId(), name); + } + + private static String makeDocId(String parentId, String name) + { + return "attachment:/" + parentId + "/" + PageFlowUtil.encode(name); + } + + private class AttachmentResource extends AbstractDocumentResource + { + WebdavResource _folder; + final AttachmentParent _parent; + final String _name; + long _created = Long.MIN_VALUE; + User _createdBy = null; + ActionURL _downloadUrl = null; + Attachment _cached = null; + final String _docid; + + AttachmentResource(Path path, ActionURL downloadURL, String displayTitle, AttachmentParent parent, String name, SearchService.SearchCategory cat) + { + super(path); + + Container c = ContainerManager.getForId(parent.getContainerId()); + _containerId = new GUID(parent.getContainerId()); + if (null != c) + setSecurableResource(c); + _downloadUrl = downloadURL; + _parent = parent; + _name = name; + _docid = makeDocId(parent,name); + initSearchProperties(name, displayTitle, cat); + } + + @Override + public boolean allowInline() + { + if (isFile() && "text/html".equals(getContentType())) + return false; + return super.allowInline(); + } + + private void initSearchProperties(String name, @Nullable String displayTitle, @Nullable SearchService.SearchCategory cat) + { + setSearchProperty(SearchService.PROPERTY.keywordsMed, FileUtil.getSearchKeywords(name)); + setSearchProperty(SearchService.PROPERTY.title, null != displayTitle ? displayTitle : name); + + if (null == cat) + setSearchCategory(SearchService.fileCategory); + else + setSearchProperty(SearchService.PROPERTY.categories, SearchService.fileCategory + StringUtils.capitalize(cat.toString())); + } + + AttachmentResource(@NotNull WebdavResource folder, @NotNull AttachmentParent parent, @NotNull Attachment attachment) + { + this(folder, parent, attachment.getName()); + + _created = attachment.getCreated().getTime(); + _createdBy = UserManager.getUser(attachment.getCreatedBy()); + _cached = attachment; + } + + + AttachmentResource(@NotNull WebdavResource folder, @NotNull AttachmentParent parent, @NotNull String name) + { + super(folder.getPath(), name); + _containerId = new GUID(parent.getContainerId()); + _folder = folder; + Container c = ContainerManager.getForId(parent.getContainerId()); + if (c != null) + setSecurableResource(c); + _name = name; + _parent = parent; + _docid = makeDocId(parent,name); + + initSearchProperties(name, null, null); + } + + + @Override + public String getDocumentId() + { + return makeDocId(_parent, _name); + } + + Attachment getAttachment() + { + if (null != _cached) + return _cached; + return AttachmentService.get().getAttachment(_parent, _name); + } + + + @Override + public String getContentType() + { + return super.getContentType(); + } + + @Override + public String getExecuteHref(ViewContext context) + { + if (null != _downloadUrl) + return _downloadUrl.getLocalURIString(); + return super.getExecuteHref(context); + } + + @Override + public boolean exists() + { + Attachment r = getAttachment(); + if (r == null) + return false; + if (null != r.getFile()) + return r.getFile().exists(); + return true; + } + + @Override + public boolean isCollection() + { + return false; + } + + @Override + public boolean canRename(User user, boolean forRename) + { + return false; + } + + @Override + public boolean delete(User user) + { + if (user != null && !canDelete(user, true, null)) + return false; + AttachmentService.get().deleteAttachment(_parent, _name, user); + return true; + } + + @Override + public FileStream getFileStream(User user) throws IOException + { + Attachment r = getAttachment(); + if (null == r) + return null; + List files = getAttachmentFiles(_parent, Collections.singletonList(r)); + if (files.isEmpty()) + throw new FileNotFoundException(r.getName()); + return files.get(0); + } + + @Override + public InputStream getInputStream(User user) throws IOException + { + return AttachmentService.get().getInputStream(_parent, _name); + } + + @Override + public void moveFrom(User user, WebdavResource r) throws IOException, DavException + { + if (r instanceof AttachmentResource from) + { + if (from._parent == _parent) + { + renameAttachment(_parent, from.getName(), getName(), user); + return; + } + } + super.moveFrom(user, r); + } + + @Override + public long copyFrom(User user, final FileStream in) throws IOException + { + try + { + AttachmentFile file = new AttachmentFile() + { + @Override + public long getSize() throws IOException + { + return in.getSize(); + } + + @Override + public String getError() + { + return null; + } + + @Override + public String getFilename() + { + return getName(); + } + + @Override + public String getContentType() + { + return PageFlowUtil.getContentTypeFor(getFilename()); + } + + @Override + public InputStream openInputStream() throws IOException + { + return in.openInputStream(); + } + + @Override + public void closeInputStream() throws IOException + { + in.closeInputStream(); + } + }; + + if (AttachmentServiceImpl.this.exists(_parent,_name)) + deleteAttachment(_parent, _name, user); + addAttachments(_parent, Collections.singletonList(file), user); + } + catch (AttachmentService.DuplicateFilenameException x) + { + throw new IOException(x); + } + finally + { + in.closeInputStream(); + } + // UNDONE return real length if anyone cares + return 0; + } + + @Override + public WebdavResource parent() + { + return _folder; + } + + @Override + public long getCreated() + { + return _created; + } + + @Override + public User getCreatedBy() + { + return _createdBy; + } + + @Override + public long getLastModified() + { + return getCreated(); + } + + @Override + public User getModifiedBy() + { + return _createdBy; + } + + @Override + public long getContentLength() + { + Attachment a = getAttachment(); + if (null == a) + return 0; + + if (_parent instanceof AttachmentDirectory) + { + try + { + File dir = ((AttachmentDirectory)_parent).getFileSystemDirectory(); + File file = new File(dir,a.getName()); + return file.exists() ? file.length() : 0; + } + catch (MissingRootDirectoryException x) + { + return 0; + } + } + else + { + // UNDONE + // return a.getSize(); + try (InputStream is = getInputStream(null)) + { + if (null != is) + { + long size = 0; + if (is instanceof FileInputStream) + size = ((FileInputStream) is).getChannel().size(); + else if (is instanceof FilterInputStream) + size = is.available(); + + return size; + } + } + catch (IOException ignored) + { + } + return 0; + } + } + + @Override + public Set> getPermissions(User user) + { + return super.getPermissions(user); + } + + @Override + public File getFile() + { + if (_parent instanceof AttachmentDirectory) + { + try + { + File dir = ((AttachmentDirectory)_parent).getFileSystemDirectory(); + return new File(dir,getName()); + } + catch (MissingRootDirectoryException x) + { + return null; + } + } + return null; + } + + @Override + public String getAbsolutePath(User user) + { + Container container = null != getContainerId() ? ContainerManager.getForId(getContainerId()) : null; + if (null != container && SecurityManager.canSeeFilePaths(container, user)) + { + File file = getFile(); + return null != file ? file.getAbsolutePath() : null; + } + return null; + } + + + @Override + @NotNull + public List getHistory() + { + //noinspection unchecked + return Collections.EMPTY_LIST; + } + + @Override + public void setLastIndexed(long ms, long modified) + { + new SqlExecutor(CoreSchema.getInstance().getSchema()).execute(new SQLFragment( + "UPDATE core.Documents SET LastIndexed=? WHERE Parent=? and DocumentName=?", + new Date(ms), _parent.getEntityId(), _name)); + } + } + + private void checkSecurityPolicy(AttachmentParent attachmentParent) throws UnauthorizedException + { + // No-op: AttachmentParent no longer provides getSecurityPolicy() + } + + private void checkSecurityPolicy(User user, AttachmentParent attachmentParent) throws UnauthorizedException + { + // No-op: AttachmentParent no longer provides getSecurityPolicy() + } + + // + //JUnit TestCase + // + + @TestWhen(TestWhen.When.BVT) + public static class TestCase extends Assert + { + private static final String _testDirName = "/_jUnitAttachment"; + + @Test + public void testDirectories() throws IOException + { + User user = TestContext.get().getUser(); + assertNotNull("Should have access to a user", user); + + // clean up if anything was left over from last time + if (null != ContainerManager.getForPath(_testDirName)) + ContainerManager.deleteAll(ContainerManager.getForPath(_testDirName), user); + + Container proj = ContainerManager.ensureContainer(_testDirName, TestContext.get().getUser()); + Container folder = ContainerManager.ensureContainer(_testDirName + "/Test", TestContext.get().getUser()); + + FileContentService fileService = FileContentService.get(); + AttachmentService svc = AttachmentService.get(); + + File curRoot = fileService.getFileRoot(proj); + assertTrue(curRoot.isDirectory()); + + AttachmentDirectory attachParent = fileService.getMappedAttachmentDirectory(folder, true); + File attachDir = attachParent.getFileSystemDirectory(); + assertTrue(attachDir.exists()); + + MultipartFile f = new MockMultipartFile("file.txt", "file.txt", "text/plain", "Hello World".getBytes()); + Map fileMap = new HashMap<>(); + fileMap.put("file.txt", f); + List files = SpringAttachmentFile.createList(fileMap); + + svc.addAttachments(attachParent, files, user); + List att = svc.getAttachments(attachParent); + assertEquals(1, att.size()); + assertTrue(att.get(0).getFile().exists()); + + // test rename + String oldName = f.getName(); + String newName = "newname.txt"; + svc.renameAttachment(attachParent, oldName, newName, user); + assertNull(svc.getAttachment(attachParent, oldName)); + assertNotNull(svc.getAttachment(attachParent, newName)); + // put things back as we found them... + svc.renameAttachment(attachParent, newName, oldName, user); + + + assertTrue(new File(attachDir, UPLOAD_LOG).exists()); + + File otherDir = new File(attachDir, "subdir"); + otherDir.mkdir(); + AttachmentDirectory namedParent = fileService.registerDirectory(folder, "test", otherDir.getCanonicalPath(), false); + + AttachmentDirectory namedParentTest = fileService.getRegisteredDirectory(folder, "test"); + assertNotNull(namedParentTest); + assertSameFile(namedParentTest.getFileSystemDirectory(), namedParent.getFileSystemDirectory()); + + svc.addAttachments(namedParent, files, user); + att = svc.getAttachments(namedParent); + assertEquals(1, att.size()); + assertTrue(att.get(0).getFile().exists()); + assertSameFile(new File(otherDir, "file.txt"), att.get(0).getFile()); + assertTrue(new File(otherDir, UPLOAD_LOG).exists()); + + fileService.unregisterDirectory(folder, "test"); + namedParentTest = fileService.getRegisteredDirectory(folder, "test"); + assertNull(namedParentTest); + + File relativeDir = new File(attachDir, "subdir2"); + relativeDir.mkdirs(); + AttachmentDirectory relativeParent = fileService.registerDirectory(folder, "relative", FileUtil.getAbsoluteCaseSensitiveFile(relativeDir).getAbsolutePath(), false); + + AttachmentDirectory relativeParentTest = fileService.getRegisteredDirectory(folder, "relative"); + assertNotNull(relativeParentTest); + assertSameFile(relativeParentTest.getFileSystemDirectory(), relativeParent.getFileSystemDirectory()); + + svc.addAttachments(relativeParent, files, user); + att = svc.getAttachments(relativeParent); + assertEquals(1, att.size()); + + File expectedFile1 = att.get(0).getFile(); + File expectedFile2 = new File(relativeDir, UPLOAD_LOG); + + assertTrue(expectedFile1.exists()); + assertEquals(new File(relativeDir, "file.txt"), expectedFile1); + assertTrue(expectedFile2.exists()); + + + // Slight detour... test FileAttachmentFile using these just-created files before we delete them + testFileAttachmentFiles(expectedFile1, expectedFile2, user); + + // clean up + ContainerManager.deleteAll(proj, user); + } + + private void assertSameFile(File a, File b) + { + if (a.equals(b)) + return; + try + { + a = a.getCanonicalFile(); + b = b.getCanonicalFile(); + assertEquals(a,b); + } + catch (IOException x) + { + fail(x.getMessage()); + } + } + + private void testFileAttachmentFiles(File file1, File file2, User user) throws IOException + { + AttachmentFile aFile1 = new FileAttachmentFile(file1); + AttachmentFile aFile2 = new FileAttachmentFile(file2); + + AttachmentService service = AttachmentService.get(); + AttachmentParent root = AuthenticationLogoAttachmentParent.get(); + service.deleteAttachment(root, file1.getName(), user); + service.deleteAttachment(root, file2.getName(), user); + + List attachments = service.getAttachments(root); + int originalCount = attachments.size(); + + service.addAttachments(root, Arrays.asList(aFile1, aFile2), user); + attachments = service.getAttachments(root); + assertEquals((originalCount + 2), attachments.size()); + + service.deleteAttachment(root, file1.getName(), user); + attachments = service.getAttachments(root); + assertEquals((originalCount + 1), attachments.size()); + + service.deleteAttachment(root, file2.getName(), user); + attachments = service.getAttachments(root); + assertEquals(originalCount, attachments.size()); + } + } +} diff --git a/core/src/org/labkey/core/webdav/DavController.java b/core/src/org/labkey/core/webdav/DavController.java index 8877ecc56af..23c381af920 100644 --- a/core/src/org/labkey/core/webdav/DavController.java +++ b/core/src/org/labkey/core/webdav/DavController.java @@ -801,9 +801,7 @@ else if ("GET".equals(method) && HttpUtil.isBrowser(getRequest())) setLastError(dex); if (dex.getStatus().equals(WebdavStatus.SC_NOT_FOUND)) { - SearchService ss = SearchService.get(); - if (null != ss) - ss.notFound((URLHelper)getRequest().getAttribute(ViewServlet.ORIGINAL_URL_URLHELPER)); + SearchService.get().notFound((URLHelper)getRequest().getAttribute(ViewServlet.ORIGINAL_URL_URLHELPER)); } getResponse().sendError(dex.getStatus(), dex.getMessage()); @@ -6503,9 +6501,7 @@ private void addToIndex(WebdavResource r) private void removeFromIndex(WebdavResource r) { _log.debug("removeFromIndex: " + r.getPath()); - SearchService ss = SearchService.get(); - if (null != ss) - ss.deleteResource(r.getDocumentId()); + SearchService.get().deleteResource(r.getDocumentId()); } diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index e571878f0b2..ab7d561b789 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -2039,18 +2039,15 @@ public boolean next() throws BatchValidationException else { final SearchService ss = SearchService.get(); - if (null != ss) - { - final ArrayList lsids = new ArrayList<>(_lsids); - final ArrayList rowIds = new LongArrayList(_rowIds); - Collections.sort(rowIds); - final Runnable indexTask = _indexFunction.apply(new SearchIndexDataKeys(rowIds, lsids)); + final ArrayList lsids = new ArrayList<>(_lsids); + final ArrayList rowIds = new LongArrayList(_rowIds); + Collections.sort(rowIds); + final Runnable indexTask = _indexFunction.apply(new SearchIndexDataKeys(rowIds, lsids)); - if (null != DbScope.getLabKeyScope()) - DbScope.getLabKeyScope().addCommitTask(indexTask, DbScope.CommitTaskOption.POSTCOMMIT); - else - indexTask.run(); - } + if (null != DbScope.getLabKeyScope()) + DbScope.getLabKeyScope().addCommitTask(indexTask, DbScope.CommitTaskOption.POSTCOMMIT); + else + indexTask.run(); } return hasNext; } diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 7456235c24b..4806f4fa949 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -1,1077 +1,1069 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment; - -import org.apache.commons.lang3.math.NumberUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayService; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DatabaseMigrationService; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationHandler; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpgradeCode; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.ExperimentRunType; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.DefaultExperimentDataHandler; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpDataClass; -import org.labkey.api.exp.api.ExpLineageService; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolAttachmentType; -import org.labkey.api.exp.api.ExpRunAttachmentType; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.FilterProtocolInputCriteria; -import org.labkey.api.exp.api.SampleTypeDomainKind; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.exp.property.DomainPropertyAuditProvider; -import org.labkey.api.exp.property.ExperimentProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.SystemProperty; -import org.labkey.api.exp.query.ExpDataClassTable; -import org.labkey.api.exp.query.ExpSampleTypeTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.exp.xar.LSIDRelativizer; -import org.labkey.api.exp.xar.LsidUtils; -import org.labkey.api.files.FileContentService; -import org.labkey.api.files.TableUpdaterFileListener; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.module.SpringModule; -import org.labkey.api.module.Summary; -import org.labkey.api.ontology.OntologyService; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.query.FilteredTable; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.JspTestCase; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.view.AlwaysAvailableWebPartFactory; -import org.labkey.api.view.BaseWebPartFactory; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.Portal; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.view.WebPartView; -import org.labkey.api.vocabulary.security.DesignVocabularyPermission; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.api.webdav.WebdavService; -import org.labkey.experiment.api.DataClassDomainKind; -import org.labkey.experiment.api.ExpDataClassImpl; -import org.labkey.experiment.api.ExpDataClassTableImpl; -import org.labkey.experiment.api.ExpDataClassType; -import org.labkey.experiment.api.ExpDataImpl; -import org.labkey.experiment.api.ExpDataTableImpl; -import org.labkey.experiment.api.ExpMaterialImpl; -import org.labkey.experiment.api.ExpProtocolImpl; -import org.labkey.experiment.api.ExpSampleTypeImpl; -import org.labkey.experiment.api.ExpSampleTypeTableImpl; -import org.labkey.experiment.api.ExperimentServiceImpl; -import org.labkey.experiment.api.ExperimentStressTest; -import org.labkey.experiment.api.GraphAlgorithms; -import org.labkey.experiment.api.LineageTest; -import org.labkey.experiment.api.LogDataType; -import org.labkey.experiment.api.Protocol; -import org.labkey.experiment.api.SampleTypeServiceImpl; -import org.labkey.experiment.api.UniqueValueCounterTestCase; -import org.labkey.experiment.api.VocabularyDomainKind; -import org.labkey.experiment.api.data.ChildOfCompareType; -import org.labkey.experiment.api.data.ChildOfMethod; -import org.labkey.experiment.api.data.LineageCompareType; -import org.labkey.experiment.api.data.ParentOfCompareType; -import org.labkey.experiment.api.data.ParentOfMethod; -import org.labkey.experiment.api.property.DomainImpl; -import org.labkey.experiment.api.property.DomainPropertyImpl; -import org.labkey.experiment.api.property.LengthValidator; -import org.labkey.experiment.api.property.LookupValidator; -import org.labkey.experiment.api.property.PropertyServiceImpl; -import org.labkey.experiment.api.property.RangeValidator; -import org.labkey.experiment.api.property.RegExValidator; -import org.labkey.experiment.api.property.StorageNameGenerator; -import org.labkey.experiment.api.property.StorageProvisionerImpl; -import org.labkey.experiment.api.property.TextChoiceValidator; -import org.labkey.experiment.controllers.exp.ExperimentController; -import org.labkey.experiment.controllers.property.PropertyController; -import org.labkey.experiment.defaults.DefaultValueServiceImpl; -import org.labkey.experiment.lineage.ExpLineageServiceImpl; -import org.labkey.experiment.lineage.LineagePerfTest; -import org.labkey.experiment.pipeline.ExperimentPipelineProvider; -import org.labkey.experiment.samples.DataClassFolderImporter; -import org.labkey.experiment.samples.DataClassFolderWriter; -import org.labkey.experiment.samples.SampleStatusFolderImporter; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; -import org.labkey.experiment.samples.SampleTypeFolderImporter; -import org.labkey.experiment.samples.SampleTypeFolderWriter; -import org.labkey.experiment.security.DataClassDesignerRole; -import org.labkey.experiment.security.SampleTypeDesignerRole; -import org.labkey.experiment.types.TypesController; -import org.labkey.experiment.xar.FolderXarImporterFactory; -import org.labkey.experiment.xar.FolderXarWriterFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; -import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; -import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; - -public class ExperimentModule extends SpringModule -{ - private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; - private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; - - public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; - - @Override - public String getName() - { - return MODULE_NAME; - } - - @Override - public Double getSchemaVersion() - { - return 25.009; - } - - @Nullable - @Override - public UpgradeCode getUpgradeCode() - { - return new ExperimentUpgradeCode(); - } - - @Override - protected void init() - { - addController("experiment", ExperimentController.class); - addController("experiment-types", TypesController.class); - addController("property", PropertyController.class); - ExperimentService.setInstance(new ExperimentServiceImpl()); - SampleTypeService.setInstance(new SampleTypeServiceImpl()); - DefaultValueService.setInstance(new DefaultValueServiceImpl()); - StorageProvisioner.setInstance(StorageProvisionerImpl.get()); - ExpLineageService.setInstance(new ExpLineageServiceImpl()); - - PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); - PropertyService.setInstance(propertyServiceImpl); - UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); - - UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); - - ExperimentProperty.register(); - SamplesSchema.register(this); - ExpSchema.register(this); - - PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); - PropertyService.get().registerDomainKind(new DataClassDomainKind()); - PropertyService.get().registerDomainKind(new VocabularyDomainKind()); - - QueryService.get().addCompareType(new ChildOfCompareType()); - QueryService.get().addCompareType(new ParentOfCompareType()); - QueryService.get().addCompareType(new LineageCompareType()); - QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); - QueryService.get().addQueryListener(new PropertyQueryChangeListener()); - - PropertyService.get().registerValidatorKind(new RegExValidator()); - PropertyService.get().registerValidatorKind(new RangeValidator()); - PropertyService.get().registerValidatorKind(new LookupValidator()); - PropertyService.get().registerValidatorKind(new LengthValidator()); - PropertyService.get().registerValidatorKind(new TextChoiceValidator()); - - ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); - ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); - ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); - ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); - ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); - - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", - "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false); - if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) - { - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", - "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); - } - else - { - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", - "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); - } - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", - "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); - - RoleManager.registerPermission(new DesignVocabularyPermission(), true); - RoleManager.registerRole(new SampleTypeDesignerRole()); - RoleManager.registerRole(new DataClassDesignerRole()); - - AttachmentService.get().registerAttachmentType(ExpRunAttachmentType.get()); - AttachmentService.get().registerAttachmentType(ExpProtocolAttachmentType.get()); - - WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); - ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); - - addModuleProperty(new LineageMaximumDepthModuleProperty(this)); - } - - @Override - public boolean hasScripts() - { - return true; - } - - @Override - @NotNull - protected Collection createWebPartFactories() - { - List result = new ArrayList<>(); - - BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); - } - }; - runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); - result.add(runGroupsFactory); - - BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunTypeWebPart(); - } - }; - result.add(runTypesFactory); - - result.add(new ExperimentRunWebPartFactory()); - BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); - result.add(sampleTypeFactory); - result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); - view.setTitle("Samples"); - return view; - } - }); - - result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); - } - }); - - BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - narrowProtocolFactory.addLegacyNames("Narrow Protocols"); - result.add(narrowProtocolFactory); - - return result; - } - - private void addDataResourceResolver(String categoryName) - { - SearchService ss = SearchService.get(); - - ss.addResourceResolver(categoryName, new SearchService.ResourceResolver() - { - @Override - public WebdavResource resolve(@NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return data.createIndexDocument(null); - } - - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); - if (idDataMap == null) - return null; - - Map> searchJsonMap = new HashMap<>(); - for (String resourceIdentifier : idDataMap.keySet()) - searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); - return searchJsonMap; - } - }); - } - - private void addDataClassResourceResolver(String categoryName) - { - SearchService ss = SearchService.get(); - - ss.addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); - if (dataClass == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); - - return properties; - } - }); - } - - private void addSampleTypeResourceResolver(String categoryName) - { - SearchService ss = SearchService.get(); - - ss.addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); - if (sampleType == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "sampleSet"); - - return properties; - } - }); - } - - private void addSampleResourceResolver(String categoryName) - { - SearchService ss = SearchService.get(); - - ss.addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); - if (material == null) - return null; - - return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Set rowIds = new HashSet<>(); - Map rowIdIdentifierMap = new LongHashMap<>(); - for (String resourceIdentifier : resourceIdentifiers) - { - long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId != 0) - { - rowIds.add(rowId); - rowIdIdentifierMap.put(rowId, resourceIdentifier); - } - } - - Map> searchJsonMap = new HashMap<>(); - for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) - { - searchJsonMap.put( - rowIdIdentifierMap.get(material.getRowId()), - ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() - ); - } - - return searchJsonMap; - } - }); - } - - @Override - protected void startupAfterSpringConfig(ModuleContext moduleContext) - { - SearchService ss = SearchService.get(); -// ss.addSearchCategory(OntologyManager.conceptCategory); - ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); - ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); - ss.addSearchCategory(ExpMaterialImpl.searchCategory); - ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); - ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataImpl.expDataCategory); - ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); - ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); - addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); - addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); - addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); - addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); - ss.addDocumentProvider(ExperimentServiceImpl.get()); - - PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); - ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); - ExperimentService.get().registerDataType(new LogDataType()); - - AuditLogService.get().registerAuditType(new DomainAuditProvider()); - AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); - AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); - - FileContentService fileContentService = FileContentService.get(); - if (null != fileContentService) - { - fileContentService.addFileListener(new ExpDataFileListener()); - fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); - fileContentService.addFileListener(new FileLinkFileListener()); - } - ContainerManager.addContainerListener( - new ContainerManager.AbstractContainerListener() - { - @Override - public void containerDeleted(Container c, User user) - { - try - { - ExperimentService.get().deleteAllExpObjInContainer(c, user); - } - catch (ExperimentException ee) - { - throw new RuntimeException(ee); - } - } - }, - // This is in the Last group because when a container is deleted, - // the Experiment listener needs to be called after the Study listener, - // because Study needs the metadata held by Experiment to delete properly. - // but it should be before the CoreContainerListener - ContainerManager.ContainerListener.Order.Last); - - if (ModuleLoader.getInstance().shouldInsertData()) - SystemProperty.registerProperties(); - - FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); - if (null != folderRegistry) - { - folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); - folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); - folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); - folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); - } - - AttachmentService.get().registerAttachmentType(ExpDataClassType.get()); - - WebdavService.get().addProvider(new ScriptsResourceProvider()); - - SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); - - UsageMetricsService svc = UsageMetricsService.get(); - if (null != svc) - { - svc.registerUsageMetrics(getName(), () -> { - Map results = new HashMap<>(); - - DbSchema schema = ExperimentService.get().getSchema(); - if (AssayService.get() != null) - { - Map assayMetrics = new HashMap<>(); - SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); - SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); - for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) - { - Map protocolMetrics = new HashMap<>(); - - // Run count across all assay designs of this type - SQLFragment runSQL = new SQLFragment(baseRunSQL); - runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); - protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); - - // Number of assay designs of this type - SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); - protocolSQL.add(assayProvider.getProtocolPattern()); - protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); - List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); - protocolMetrics.put("protocolCount", protocols.size()); - - List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); - - protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); - - // Primary implementation class - protocolMetrics.put("implementingClass", assayProvider.getClass()); - - assayMetrics.put(assayProvider.getName(), protocolMetrics); - } - assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); - - assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); - SQLFragment runsWithPlateSQL = new SQLFragment(""" - SELECT COUNT(*) FROM exp.experimentrun r - INNER JOIN exp.object o ON o.objectUri = r.lsid - INNER JOIN exp.objectproperty op ON op.objectId = o.objectId - WHERE op.propertyid IN ( - SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? - )"""); - assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); - assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); - - assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - Map sampleLookupCountMetrics = new HashMap<>(); - SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); - - SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); - sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); - sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( - """ - SELECT COUNT(*) FROM ( - SELECT PD.domainid, COUNT(*) AS PropCount - FROM exp.propertydescriptor D - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) - AND propertyuri LIKE ? - GROUP BY PD.domainid - ) X WHERE X.PropCount > 1""" - ); - resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); - - assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); - - - // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) - results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assay", assayMetrics); - } - - results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); - results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); - - if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries - { - Collection> numSampleCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); - results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( - map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") - ).count()); - } - UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); - FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); - - SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + - " FROM (\n" + - " -- updates that are marked as lineage updates\n" + - " (SELECT DISTINCT transactionId\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + - " AND comment = 'Sample was updated.'\n" + - " ) a1\n" + - " JOIN\n" + - " -- but have associated entries that are not lineage updates\n" + - " (SELECT DISTINCT transactionid\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + - " ON a1.transactionid = a2.transactionid\n" + - " )"); - - results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); - - results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); - results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); - results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); - results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); - results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); - results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); - - results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - String duplicateCaseInsensitiveSampleNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.material - WHERE materialsourceid IS NOT NULL - GROUP BY LOWER(name), materialsourceid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - String duplicateCaseInsensitiveDataNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.data - WHERE classid IS NOT NULL - GROUP BY LOWER(name), classid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); - results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); - - results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); - results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); - results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + - "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); - if (schema.getSqlDialect().isPostgreSQL()) - { - Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); - results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> - (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); - } - - results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); - results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); - results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); - - results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); - results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - - results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); - - results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); - - results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); - - results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); - results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); - - results.putAll(ExperimentService.get().getDomainMetrics()); - - return results; - }); - } - - // Work around foreign key cycle between ExperimentRun <-> ProtocolApplication by temporarily dropping FK_Run_WorfklowTask - DatabaseMigrationService.get().registerHandler(OntologyManager.getExpSchema(), new DefaultMigrationHandler() - { - @Override - public void beforeSchema(DbSchema targetSchema) - { - // Yes, the FK name is misspelled - new SqlExecutor(targetSchema).execute("ALTER TABLE exp.ExperimentRun DROP CONSTRAINT FK_Run_WorfklowTask"); - } - - @Override - public void afterSchema(DbSchema targetSchema) - { - new SqlExecutor(targetSchema).execute("ALTER TABLE exp.ExperimentRun ADD CONSTRAINT FK_Run_WorfklowTask FOREIGN KEY (WorkflowTask) REFERENCES exp.ProtocolApplication (RowId) MATCH SIMPLE ON DELETE SET NULL"); - } - }); - } - - @Override - @NotNull - public Collection getSummary(Container c) - { - Collection list = new LinkedList<>(); - int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); - if (runGroupCount > 0) - list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); - - User user = HttpView.currentContext().getUser(); - - Set runTypes = ExperimentService.get().getExperimentRunTypes(c); - for (ExperimentRunType runType : runTypes) - { - if (runType == ExperimentRunType.ALL_RUNS_TYPE) - continue; - - long runCount = runType.getRunCount(user, c); - if (runCount > 0) - list.add(runCount + " runs of type " + runType.getDescription()); - } - - int dataClassCount = ExperimentService.get().getDataClasses(c, null, false).size(); - if (dataClassCount > 0) - list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); - - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, null, false).size(); - if (sampleTypeCount > 0) - list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); - - return list; - } - - @Override - public @NotNull ArrayList getDetailedSummary(Container c, User user) - { - ArrayList summaries = new ArrayList<>(); - - // Assay types - long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); - if (assayTypeCount > 0) - summaries.add(new Summary(assayTypeCount, "Assay Type")); - - // Run count - int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); - if (runGroupCount > 0) - summaries.add(new Summary(runGroupCount, "Assay run")); - - // Number of Data Classes - List dataClasses = ExperimentService.get().getDataClasses(c, user, false); - int dataClassCount = dataClasses.size(); - if (dataClassCount > 0) - summaries.add(new Summary(dataClassCount, "Data Class")); - - ExpSchema expSchema = new ExpSchema(user, c); - - // Individual Data Class row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 47919: The "DataCount" column is filtered to only count data in the target container - if (table instanceof ExpDataClassTableImpl tableImpl) - tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpDataClassTable.Column.Name.name()); - columns.add(ExpDataClassTable.Column.DataCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - summaries.add(new Summary(count, entry.getKey())); - } - } - - // Sample Types - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, null, false).size(); - if (sampleTypeCount > 0) - summaries.add(new Summary(sampleTypeCount, "Sample Type")); - - // Individual Sample Type row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 51557: The "SampleCount" column is filtered to only count data in the target container - if (table instanceof ExpSampleTypeTableImpl tableImpl) - tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpSampleTypeTable.Column.Name.name()); - columns.add(ExpSampleTypeTable.Column.SampleCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - { - String name = entry.getKey(); - Summary s = name.equals("MixtureBatches") - ? new Summary(count, "Batch") - : new Summary(count, name); - summaries.add(s); - } - } - } - - return summaries; - } - - @Override - @NotNull - public Set getIntegrationTests() - { - return Set.of( - DomainImpl.TestCase.class, - DomainPropertyImpl.TestCase.class, - ExpDataTableImpl.TestCase.class, - ExperimentServiceImpl.AuditDomainUriTest.class, - ExperimentServiceImpl.LineageQueryTestCase.class, - ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, - ExperimentServiceImpl.TestCase.class, - ExperimentStressTest.class, - LineagePerfTest.class, - LineageTest.class, - OntologyManager.TestCase.class, - PropertyServiceImpl.TestCase.class, - StorageNameGenerator.TestCase.class, - StorageProvisionerImpl.TestCase.class, - UniqueValueCounterTestCase.class - ); - } - - @Override - public @NotNull Collection>> getIntegrationTestFactories() - { - List>> list = new ArrayList<>(super.getIntegrationTestFactories()); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); - return list; - } - - @NotNull - @Override - public Set getUnitTests() - { - return Set.of( - GraphAlgorithms.TestCase.class, - LSIDRelativizer.TestCase.class, - Lsid.TestCase.class, - LsidUtils.TestCase.class, - PropertyController.TestCase.class, - Quantity.TestCase.class, - Unit.TestCase.class - ); - } - - @Override - @NotNull - public Collection getSchemaNames() - { - return List.of( - ExpSchema.SCHEMA_NAME, - DataClassDomainKind.PROVISIONED_SCHEMA_NAME, - SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME - ); - } - - @NotNull - @Override - public Collection getProvisionedSchemaNames() - { - return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment; + +import org.apache.commons.lang3.math.NumberUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayService; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DatabaseMigrationService; +import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationHandler; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpgradeCode; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.ExperimentRunType; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.DefaultExperimentDataHandler; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpLineageService; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolAttachmentType; +import org.labkey.api.exp.api.ExpRunAttachmentType; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.FilterProtocolInputCriteria; +import org.labkey.api.exp.api.SampleTypeDomainKind; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.exp.property.DomainPropertyAuditProvider; +import org.labkey.api.exp.property.ExperimentProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.SystemProperty; +import org.labkey.api.exp.query.ExpDataClassTable; +import org.labkey.api.exp.query.ExpSampleTypeTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.exp.xar.LSIDRelativizer; +import org.labkey.api.exp.xar.LsidUtils; +import org.labkey.api.files.FileContentService; +import org.labkey.api.files.TableUpdaterFileListener; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.SpringModule; +import org.labkey.api.module.Summary; +import org.labkey.api.ontology.OntologyService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.JspTestCase; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.view.AlwaysAvailableWebPartFactory; +import org.labkey.api.view.BaseWebPartFactory; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.Portal; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; +import org.labkey.api.vocabulary.security.DesignVocabularyPermission; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; +import org.labkey.experiment.api.DataClassDomainKind; +import org.labkey.experiment.api.ExpDataClassImpl; +import org.labkey.experiment.api.ExpDataClassTableImpl; +import org.labkey.experiment.api.ExpDataClassType; +import org.labkey.experiment.api.ExpDataImpl; +import org.labkey.experiment.api.ExpDataTableImpl; +import org.labkey.experiment.api.ExpMaterialImpl; +import org.labkey.experiment.api.ExpProtocolImpl; +import org.labkey.experiment.api.ExpSampleTypeImpl; +import org.labkey.experiment.api.ExpSampleTypeTableImpl; +import org.labkey.experiment.api.ExperimentServiceImpl; +import org.labkey.experiment.api.ExperimentStressTest; +import org.labkey.experiment.api.GraphAlgorithms; +import org.labkey.experiment.api.LineageTest; +import org.labkey.experiment.api.LogDataType; +import org.labkey.experiment.api.Protocol; +import org.labkey.experiment.api.SampleTypeServiceImpl; +import org.labkey.experiment.api.UniqueValueCounterTestCase; +import org.labkey.experiment.api.VocabularyDomainKind; +import org.labkey.experiment.api.data.ChildOfCompareType; +import org.labkey.experiment.api.data.ChildOfMethod; +import org.labkey.experiment.api.data.LineageCompareType; +import org.labkey.experiment.api.data.ParentOfCompareType; +import org.labkey.experiment.api.data.ParentOfMethod; +import org.labkey.experiment.api.property.DomainImpl; +import org.labkey.experiment.api.property.DomainPropertyImpl; +import org.labkey.experiment.api.property.LengthValidator; +import org.labkey.experiment.api.property.LookupValidator; +import org.labkey.experiment.api.property.PropertyServiceImpl; +import org.labkey.experiment.api.property.RangeValidator; +import org.labkey.experiment.api.property.RegExValidator; +import org.labkey.experiment.api.property.StorageNameGenerator; +import org.labkey.experiment.api.property.StorageProvisionerImpl; +import org.labkey.experiment.api.property.TextChoiceValidator; +import org.labkey.experiment.controllers.exp.ExperimentController; +import org.labkey.experiment.controllers.property.PropertyController; +import org.labkey.experiment.defaults.DefaultValueServiceImpl; +import org.labkey.experiment.lineage.ExpLineageServiceImpl; +import org.labkey.experiment.lineage.LineagePerfTest; +import org.labkey.experiment.pipeline.ExperimentPipelineProvider; +import org.labkey.experiment.samples.DataClassFolderImporter; +import org.labkey.experiment.samples.DataClassFolderWriter; +import org.labkey.experiment.samples.SampleStatusFolderImporter; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; +import org.labkey.experiment.samples.SampleTypeFolderImporter; +import org.labkey.experiment.samples.SampleTypeFolderWriter; +import org.labkey.experiment.security.DataClassDesignerRole; +import org.labkey.experiment.security.SampleTypeDesignerRole; +import org.labkey.experiment.types.TypesController; +import org.labkey.experiment.xar.FolderXarImporterFactory; +import org.labkey.experiment.xar.FolderXarWriterFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; +import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; +import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; + +public class ExperimentModule extends SpringModule +{ + private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; + private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; + + public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; + + @Override + public String getName() + { + return MODULE_NAME; + } + + @Override + public Double getSchemaVersion() + { + return 25.009; + } + + @Nullable + @Override + public UpgradeCode getUpgradeCode() + { + return new ExperimentUpgradeCode(); + } + + @Override + protected void init() + { + addController("experiment", ExperimentController.class); + addController("experiment-types", TypesController.class); + addController("property", PropertyController.class); + ExperimentService.setInstance(new ExperimentServiceImpl()); + SampleTypeService.setInstance(new SampleTypeServiceImpl()); + DefaultValueService.setInstance(new DefaultValueServiceImpl()); + StorageProvisioner.setInstance(StorageProvisionerImpl.get()); + ExpLineageService.setInstance(new ExpLineageServiceImpl()); + + PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); + PropertyService.setInstance(propertyServiceImpl); + UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); + + UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); + + ExperimentProperty.register(); + SamplesSchema.register(this); + ExpSchema.register(this); + + PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); + PropertyService.get().registerDomainKind(new DataClassDomainKind()); + PropertyService.get().registerDomainKind(new VocabularyDomainKind()); + + QueryService.get().addCompareType(new ChildOfCompareType()); + QueryService.get().addCompareType(new ParentOfCompareType()); + QueryService.get().addCompareType(new LineageCompareType()); + QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); + QueryService.get().addQueryListener(new PropertyQueryChangeListener()); + + PropertyService.get().registerValidatorKind(new RegExValidator()); + PropertyService.get().registerValidatorKind(new RangeValidator()); + PropertyService.get().registerValidatorKind(new LookupValidator()); + PropertyService.get().registerValidatorKind(new LengthValidator()); + PropertyService.get().registerValidatorKind(new TextChoiceValidator()); + + ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); + ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); + ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); + ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); + ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); + + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", + "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false); + if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) + { + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", + "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); + } + else + { + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", + "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); + } + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", + "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); + + RoleManager.registerPermission(new DesignVocabularyPermission(), true); + RoleManager.registerRole(new SampleTypeDesignerRole()); + RoleManager.registerRole(new DataClassDesignerRole()); + + AttachmentService.get().registerAttachmentType(ExpRunAttachmentType.get()); + AttachmentService.get().registerAttachmentType(ExpProtocolAttachmentType.get()); + + WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); + ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); + + addModuleProperty(new LineageMaximumDepthModuleProperty(this)); + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + List result = new ArrayList<>(); + + BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); + } + }; + runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); + result.add(runGroupsFactory); + + BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunTypeWebPart(); + } + }; + result.add(runTypesFactory); + + result.add(new ExperimentRunWebPartFactory()); + BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); + result.add(sampleTypeFactory); + result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); + view.setTitle("Samples"); + return view; + } + }); + + result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); + } + }); + + BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + narrowProtocolFactory.addLegacyNames("Narrow Protocols"); + result.add(narrowProtocolFactory); + + return result; + } + + private void addDataResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() + { + @Override + public WebdavResource resolve(@NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return data.createIndexDocument(null); + } + + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); + if (idDataMap == null) + return null; + + Map> searchJsonMap = new HashMap<>(); + for (String resourceIdentifier : idDataMap.keySet()) + searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); + return searchJsonMap; + } + }); + } + + private void addDataClassResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); + if (dataClass == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); + + return properties; + } + }); + } + + private void addSampleTypeResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); + if (sampleType == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "sampleSet"); + + return properties; + } + }); + } + + private void addSampleResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); + if (material == null) + return null; + + return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Set rowIds = new HashSet<>(); + Map rowIdIdentifierMap = new LongHashMap<>(); + for (String resourceIdentifier : resourceIdentifiers) + { + long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId != 0) + { + rowIds.add(rowId); + rowIdIdentifierMap.put(rowId, resourceIdentifier); + } + } + + Map> searchJsonMap = new HashMap<>(); + for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) + { + searchJsonMap.put( + rowIdIdentifierMap.get(material.getRowId()), + ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() + ); + } + + return searchJsonMap; + } + }); + } + + @Override + protected void startupAfterSpringConfig(ModuleContext moduleContext) + { + SearchService ss = SearchService.get(); +// ss.addSearchCategory(OntologyManager.conceptCategory); + ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); + ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); + ss.addSearchCategory(ExpMaterialImpl.searchCategory); + ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); + ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataImpl.expDataCategory); + ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); + ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); + addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); + addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); + addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); + addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); + ss.addDocumentProvider(ExperimentServiceImpl.get()); + + PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); + ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); + ExperimentService.get().registerDataType(new LogDataType()); + + AuditLogService.get().registerAuditType(new DomainAuditProvider()); + AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); + AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); + + FileContentService fileContentService = FileContentService.get(); + if (null != fileContentService) + { + fileContentService.addFileListener(new ExpDataFileListener()); + fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); + fileContentService.addFileListener(new FileLinkFileListener()); + } + ContainerManager.addContainerListener( + new ContainerManager.AbstractContainerListener() + { + @Override + public void containerDeleted(Container c, User user) + { + try + { + ExperimentService.get().deleteAllExpObjInContainer(c, user); + } + catch (ExperimentException ee) + { + throw new RuntimeException(ee); + } + } + }, + // This is in the Last group because when a container is deleted, + // the Experiment listener needs to be called after the Study listener, + // because Study needs the metadata held by Experiment to delete properly. + // but it should be before the CoreContainerListener + ContainerManager.ContainerListener.Order.Last); + + if (ModuleLoader.getInstance().shouldInsertData()) + SystemProperty.registerProperties(); + + FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); + if (null != folderRegistry) + { + folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); + folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); + folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); + folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); + } + + AttachmentService.get().registerAttachmentType(ExpDataClassType.get()); + + WebdavService.get().addProvider(new ScriptsResourceProvider()); + + SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); + + UsageMetricsService svc = UsageMetricsService.get(); + if (null != svc) + { + svc.registerUsageMetrics(getName(), () -> { + Map results = new HashMap<>(); + + DbSchema schema = ExperimentService.get().getSchema(); + if (AssayService.get() != null) + { + Map assayMetrics = new HashMap<>(); + SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); + SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); + for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) + { + Map protocolMetrics = new HashMap<>(); + + // Run count across all assay designs of this type + SQLFragment runSQL = new SQLFragment(baseRunSQL); + runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); + protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); + + // Number of assay designs of this type + SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); + protocolSQL.add(assayProvider.getProtocolPattern()); + protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); + List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); + protocolMetrics.put("protocolCount", protocols.size()); + + List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); + + protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); + + // Primary implementation class + protocolMetrics.put("implementingClass", assayProvider.getClass()); + + assayMetrics.put(assayProvider.getName(), protocolMetrics); + } + assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); + + assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); + SQLFragment runsWithPlateSQL = new SQLFragment(""" + SELECT COUNT(*) FROM exp.experimentrun r + INNER JOIN exp.object o ON o.objectUri = r.lsid + INNER JOIN exp.objectproperty op ON op.objectId = o.objectId + WHERE op.propertyid IN ( + SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? + )"""); + assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); + assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); + + assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + Map sampleLookupCountMetrics = new HashMap<>(); + SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); + + SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); + sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); + sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( + """ + SELECT COUNT(*) FROM ( + SELECT PD.domainid, COUNT(*) AS PropCount + FROM exp.propertydescriptor D + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) + AND propertyuri LIKE ? + GROUP BY PD.domainid + ) X WHERE X.PropCount > 1""" + ); + resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); + + assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); + + + // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) + results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assay", assayMetrics); + } + + results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); + results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); + + if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries + { + Collection> numSampleCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); + results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( + map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") + ).count()); + } + UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); + FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); + + SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + + " FROM (\n" + + " -- updates that are marked as lineage updates\n" + + " (SELECT DISTINCT transactionId\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + + " AND comment = 'Sample was updated.'\n" + + " ) a1\n" + + " JOIN\n" + + " -- but have associated entries that are not lineage updates\n" + + " (SELECT DISTINCT transactionid\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + + " ON a1.transactionid = a2.transactionid\n" + + " )"); + + results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); + + results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); + results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); + results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); + results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); + results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); + results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); + + results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + String duplicateCaseInsensitiveSampleNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.material + WHERE materialsourceid IS NOT NULL + GROUP BY LOWER(name), materialsourceid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + String duplicateCaseInsensitiveDataNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.data + WHERE classid IS NOT NULL + GROUP BY LOWER(name), classid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); + results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); + + results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); + results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); + results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + + "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); + if (schema.getSqlDialect().isPostgreSQL()) + { + Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); + results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> + (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); + } + + results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); + results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); + results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); + + results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); + results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + + results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); + + results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); + + results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); + + results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); + results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); + + results.putAll(ExperimentService.get().getDomainMetrics()); + + return results; + }); + } + + // Work around foreign key cycle between ExperimentRun <-> ProtocolApplication by temporarily dropping FK_Run_WorfklowTask + DatabaseMigrationService.get().registerHandler(OntologyManager.getExpSchema(), new DefaultMigrationHandler() + { + @Override + public void beforeSchema(DbSchema targetSchema) + { + // Yes, the FK name is misspelled + new SqlExecutor(targetSchema).execute("ALTER TABLE exp.ExperimentRun DROP CONSTRAINT FK_Run_WorfklowTask"); + } + + @Override + public void afterSchema(DbSchema targetSchema) + { + new SqlExecutor(targetSchema).execute("ALTER TABLE exp.ExperimentRun ADD CONSTRAINT FK_Run_WorfklowTask FOREIGN KEY (WorkflowTask) REFERENCES exp.ProtocolApplication (RowId) MATCH SIMPLE ON DELETE SET NULL"); + } + }); + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + Collection list = new LinkedList<>(); + int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); + if (runGroupCount > 0) + list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); + + User user = HttpView.currentContext().getUser(); + + Set runTypes = ExperimentService.get().getExperimentRunTypes(c); + for (ExperimentRunType runType : runTypes) + { + if (runType == ExperimentRunType.ALL_RUNS_TYPE) + continue; + + long runCount = runType.getRunCount(user, c); + if (runCount > 0) + list.add(runCount + " runs of type " + runType.getDescription()); + } + + int dataClassCount = ExperimentService.get().getDataClasses(c, null, false).size(); + if (dataClassCount > 0) + list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); + + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, null, false).size(); + if (sampleTypeCount > 0) + list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); + + return list; + } + + @Override + public @NotNull ArrayList getDetailedSummary(Container c, User user) + { + ArrayList summaries = new ArrayList<>(); + + // Assay types + long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); + if (assayTypeCount > 0) + summaries.add(new Summary(assayTypeCount, "Assay Type")); + + // Run count + int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); + if (runGroupCount > 0) + summaries.add(new Summary(runGroupCount, "Assay run")); + + // Number of Data Classes + List dataClasses = ExperimentService.get().getDataClasses(c, user, false); + int dataClassCount = dataClasses.size(); + if (dataClassCount > 0) + summaries.add(new Summary(dataClassCount, "Data Class")); + + ExpSchema expSchema = new ExpSchema(user, c); + + // Individual Data Class row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 47919: The "DataCount" column is filtered to only count data in the target container + if (table instanceof ExpDataClassTableImpl tableImpl) + tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpDataClassTable.Column.Name.name()); + columns.add(ExpDataClassTable.Column.DataCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + summaries.add(new Summary(count, entry.getKey())); + } + } + + // Sample Types + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, null, false).size(); + if (sampleTypeCount > 0) + summaries.add(new Summary(sampleTypeCount, "Sample Type")); + + // Individual Sample Type row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 51557: The "SampleCount" column is filtered to only count data in the target container + if (table instanceof ExpSampleTypeTableImpl tableImpl) + tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpSampleTypeTable.Column.Name.name()); + columns.add(ExpSampleTypeTable.Column.SampleCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + { + String name = entry.getKey(); + Summary s = name.equals("MixtureBatches") + ? new Summary(count, "Batch") + : new Summary(count, name); + summaries.add(s); + } + } + } + + return summaries; + } + + @Override + @NotNull + public Set getIntegrationTests() + { + return Set.of( + DomainImpl.TestCase.class, + DomainPropertyImpl.TestCase.class, + ExpDataTableImpl.TestCase.class, + ExperimentServiceImpl.AuditDomainUriTest.class, + ExperimentServiceImpl.LineageQueryTestCase.class, + ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, + ExperimentServiceImpl.TestCase.class, + ExperimentStressTest.class, + LineagePerfTest.class, + LineageTest.class, + OntologyManager.TestCase.class, + PropertyServiceImpl.TestCase.class, + StorageNameGenerator.TestCase.class, + StorageProvisionerImpl.TestCase.class, + UniqueValueCounterTestCase.class + ); + } + + @Override + public @NotNull Collection>> getIntegrationTestFactories() + { + List>> list = new ArrayList<>(super.getIntegrationTestFactories()); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); + return list; + } + + @NotNull + @Override + public Set getUnitTests() + { + return Set.of( + GraphAlgorithms.TestCase.class, + LSIDRelativizer.TestCase.class, + Lsid.TestCase.class, + LsidUtils.TestCase.class, + PropertyController.TestCase.class, + Quantity.TestCase.class, + Unit.TestCase.class + ); + } + + @Override + @NotNull + public Collection getSchemaNames() + { + return List.of( + ExpSchema.SCHEMA_NAME, + DataClassDomainKind.PROVISIONED_SCHEMA_NAME, + SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME + ); + } + + @NotNull + @Override + public Collection getProvisionedSchemaNames() + { + return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); + } +} diff --git a/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java b/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java index 5537fab3d14..086d0ee91e5 100644 --- a/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java +++ b/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java @@ -1,8355 +1,8353 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.controllers.exp; - -import au.com.bytecode.opencsv.CSVWriter; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import jakarta.servlet.http.HttpSession; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.poi.openxml4j.exceptions.InvalidFormatException; -import org.apache.poi.ss.usermodel.Workbook; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.labkey.api.action.ApiJsonWriter; -import org.labkey.api.action.ApiResponse; -import org.labkey.api.action.ApiSimpleResponse; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.action.ExportAction; -import org.labkey.api.action.FormHandlerAction; -import org.labkey.api.action.FormViewAction; -import org.labkey.api.action.HasViewContext; -import org.labkey.api.action.Marshal; -import org.labkey.api.action.Marshaller; -import org.labkey.api.action.MutatingApiAction; -import org.labkey.api.action.QueryViewAction; -import org.labkey.api.action.ReadOnlyApiAction; -import org.labkey.api.action.ReturnUrlForm; -import org.labkey.api.action.SimpleApiJsonForm; -import org.labkey.api.action.SimpleErrorView; -import org.labkey.api.action.SimpleResponse; -import org.labkey.api.action.SimpleViewAction; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.assay.AssayFileWriter; -import org.labkey.api.assay.AssayProtocolSchema; -import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayService; -import org.labkey.api.assay.actions.UploadWizardAction; -import org.labkey.api.assay.security.DesignAssayPermission; -import org.labkey.api.attachments.AttachmentParent; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.attachments.BaseDownloadAction; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.data.ActionButton; -import org.labkey.api.data.BaseColumnInfo; -import org.labkey.api.data.ButtonBar; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DataRegion; -import org.labkey.api.data.DataRegionSelection; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DisplayColumn; -import org.labkey.api.data.ExcelWriter; -import org.labkey.api.data.MenuButton; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.ShowRows; -import org.labkey.api.data.SimpleDisplayColumn; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TSVWriter; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.exp.AbstractParameter; -import org.labkey.api.exp.DeleteForm; -import org.labkey.api.exp.DuplicateMaterialException; -import org.labkey.api.exp.ExperimentDataHandler; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.ExperimentRunForm; -import org.labkey.api.exp.ExperimentRunListView; -import org.labkey.api.exp.ExperimentRunType; -import org.labkey.api.exp.Identifiable; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.LsidManager; -import org.labkey.api.exp.LsidType; -import org.labkey.api.exp.ObjectProperty; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.ProtocolApplicationParameter; -import org.labkey.api.exp.XarContext; -import org.labkey.api.exp.api.DataClassDomainKindProperties; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpDataClass; -import org.labkey.api.exp.api.ExpExperiment; -import org.labkey.api.exp.api.ExpLineageOptions; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpMaterialRunInput; -import org.labkey.api.exp.api.ExpObject; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolApplication; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpRunAttachmentParent; -import org.labkey.api.exp.api.ExpRunEditor; -import org.labkey.api.exp.api.ExpRunItem; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.ExperimentUrls; -import org.labkey.api.exp.api.NameExpressionOptionService; -import org.labkey.api.exp.api.ResolveLsidsForm; -import org.labkey.api.exp.api.SampleTypeDomainKind; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainTemplate; -import org.labkey.api.exp.property.DomainTemplateGroup; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.query.ExpDataProtocolInputTable; -import org.labkey.api.exp.query.ExpInputTable; -import org.labkey.api.exp.query.ExpMaterialProtocolInputTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.exp.xar.LSIDRelativizer; -import org.labkey.api.exp.xar.LsidUtils; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.inventory.InventoryService; -import org.labkey.api.module.ModuleHtmlView; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineStatusFile; -import org.labkey.api.pipeline.PipelineUrls; -import org.labkey.api.pipeline.PipelineValidationException; -import org.labkey.api.qc.SampleStatusService; -import org.labkey.api.query.AbstractQueryImportAction; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.query.DuplicateKeyException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryAction; -import org.labkey.api.query.QueryDefinition; -import org.labkey.api.query.QueryException; -import org.labkey.api.query.QueryForm; -import org.labkey.api.query.QueryParam; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryUpdateForm; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.UserSchemaAction; -import org.labkey.api.reader.ColumnDescriptor; -import org.labkey.api.reader.DataLoader; -import org.labkey.api.reader.DataLoaderFactory; -import org.labkey.api.reader.ExcelFactory; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.ActionNames; -import org.labkey.api.security.RequiresAnyOf; -import org.labkey.api.security.RequiresNoPermission; -import org.labkey.api.security.RequiresPermission; -import org.labkey.api.security.SecurableResource; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.DesignDataClassPermission; -import org.labkey.api.security.permissions.DesignSampleTypePermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.SampleWorkflowDeletePermission; -import org.labkey.api.security.permissions.SiteAdminPermission; -import org.labkey.api.security.permissions.TroubleshooterPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.ConceptURIProperties; -import org.labkey.api.sql.LabKeySql; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.StudyUrls; -import org.labkey.api.study.publish.StudyPublishService; -import org.labkey.api.usageMetrics.SimpleMetricsService; -import org.labkey.api.util.DOM; -import org.labkey.api.util.DOM.LK; -import org.labkey.api.util.ErrorRenderer; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.FileStream; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.HtmlStringBuilder; -import org.labkey.api.util.ImageUtil; -import org.labkey.api.util.JSoupUtil; -import org.labkey.api.util.LinkBuilder; -import org.labkey.api.util.NetworkDrive; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.ResponseHelper; -import org.labkey.api.util.SafeToRender; -import org.labkey.api.util.SessionHelper; -import org.labkey.api.util.StringExpression; -import org.labkey.api.util.URLHelper; -import org.labkey.api.util.UniqueID; -import org.labkey.api.util.CsrfInput; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.BadRequestException; -import org.labkey.api.view.DataView; -import org.labkey.api.view.DataViewSnapshotSelectionForm; -import org.labkey.api.view.DetailsView; -import org.labkey.api.view.HBox; -import org.labkey.api.view.HtmlView; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.InsertView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.RedirectException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.UpdateView; -import org.labkey.api.view.VBox; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.ViewServlet; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.ClientDependency; -import org.labkey.api.view.template.PageConfig; -import org.labkey.experiment.ChooseExperimentTypeBean; -import org.labkey.experiment.ConfirmDeleteView; -import org.labkey.experiment.CustomPropertiesView; -import org.labkey.experiment.DataClassWebPart; -import org.labkey.experiment.DerivedSamplePropertyHelper; -import org.labkey.experiment.DotGraph; -import org.labkey.experiment.ExpDataFileListener; -import org.labkey.experiment.ExperimentRunDisplayColumn; -import org.labkey.experiment.ExperimentRunGraph; -import org.labkey.experiment.LineageGraphDisplayColumn; -import org.labkey.experiment.MissingFilesCheckInfo; -import org.labkey.experiment.MoveRunsBean; -import org.labkey.experiment.ParentChildView; -import org.labkey.experiment.ProtocolApplicationDisplayColumn; -import org.labkey.experiment.ProtocolDisplayColumn; -import org.labkey.experiment.ProtocolWebPart; -import org.labkey.experiment.RunGroupWebPart; -import org.labkey.experiment.SampleTypeDisplayColumn; -import org.labkey.experiment.SampleTypeWebPart; -import org.labkey.experiment.StandardAndCustomPropertiesView; -import org.labkey.experiment.XarExportPipelineJob; -import org.labkey.experiment.XarExportType; -import org.labkey.experiment.XarExporter; -import org.labkey.experiment.api.ClosureQueryHelper; -import org.labkey.experiment.api.DataClass; -import org.labkey.experiment.api.DataClassDomainKind; -import org.labkey.experiment.api.ExpDataClassAttachmentParent; -import org.labkey.experiment.api.ExpDataClassImpl; -import org.labkey.experiment.api.ExpDataImpl; -import org.labkey.experiment.api.ExpExperimentImpl; -import org.labkey.experiment.api.ExpMaterialImpl; -import org.labkey.experiment.api.ExpProtocolApplicationImpl; -import org.labkey.experiment.api.ExpProtocolImpl; -import org.labkey.experiment.api.ExpRunImpl; -import org.labkey.experiment.api.ExpSampleTypeImpl; -import org.labkey.experiment.api.Experiment; -import org.labkey.experiment.api.ExperimentServiceImpl; -import org.labkey.experiment.api.GraphAlgorithms; -import org.labkey.experiment.api.ProtocolActionStepDetail; -import org.labkey.experiment.api.SampleTypeServiceImpl; -import org.labkey.experiment.api.SampleTypeUpdateServiceDI; -import org.labkey.experiment.controllers.property.PropertyController; -import org.labkey.experiment.lineage.ExpLineageServiceImpl; -import org.labkey.experiment.pipeline.ExperimentPipelineJob; -import org.labkey.experiment.types.TypesController; -import org.labkey.experiment.xar.XarExportSelection; -import org.labkey.vfs.FileLike; -import org.springframework.beans.PropertyValue; -import org.springframework.beans.PropertyValues; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.validation.ObjectError; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.multipart.MultipartHttpServletRequest; -import org.springframework.web.servlet.ModelAndView; - -import javax.imageio.ImageIO; -import java.awt.*; -import java.awt.image.BufferedImage; -import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.TreeSet; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.toList; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; -import static org.labkey.api.exp.query.ExpSchema.TableType.DataInputs; -import static org.labkey.api.query.QueryImportPipelineJob.QUERY_IMPORT_NOTIFICATION_PROVIDER_PARAM; -import static org.labkey.api.query.QueryImportPipelineJob.QUERY_IMPORT_PIPELINE_DESCRIPTION_PARAM; -import static org.labkey.api.query.QueryImportPipelineJob.QUERY_IMPORT_PIPELINE_PROVIDER_PARAM; -import static org.labkey.api.util.DOM.A; -import static org.labkey.api.util.DOM.Attribute.action; -import static org.labkey.api.util.DOM.Attribute.href; -import static org.labkey.api.util.DOM.Attribute.id; -import static org.labkey.api.util.DOM.Attribute.method; -import static org.labkey.api.util.DOM.Attribute.name; -import static org.labkey.api.util.DOM.Attribute.size; -import static org.labkey.api.util.DOM.Attribute.src; -import static org.labkey.api.util.DOM.Attribute.target; -import static org.labkey.api.util.DOM.Attribute.type; -import static org.labkey.api.util.DOM.Attribute.value; -import static org.labkey.api.util.DOM.Attribute.width; -import static org.labkey.api.util.DOM.DIV; -import static org.labkey.api.util.DOM.IMG; -import static org.labkey.api.util.DOM.INPUT; -import static org.labkey.api.util.DOM.LI; -import static org.labkey.api.util.DOM.TABLE; -import static org.labkey.api.util.DOM.TD; -import static org.labkey.api.util.DOM.TR; -import static org.labkey.api.util.DOM.UL; -import static org.labkey.api.util.DOM.at; -import static org.labkey.api.util.DOM.cl; -import static org.labkey.experiment.ExpDataIterators.setContainerFilterForImport; -import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.update; - -public class ExperimentController extends SpringActionController -{ - private static final Logger _log = LogManager.getLogger(ExperimentController.class); - private static final DefaultActionResolver _actionResolver = new DefaultActionResolver( - ExperimentController.class - ); - private static final String GUEST_DIRECTORY_NAME = "guest"; - - public ExperimentController() - { - setActionResolver(_actionResolver); - } - - public static void ensureCorrectContainer(Container requestContainer, ExpObject object, ViewContext viewContext) - { - Container objectContainer = object.getContainer(); - if (!requestContainer.equals(objectContainer)) - { - ActionURL url = viewContext.cloneActionURL(); - url.setContainer(objectContainer); - throw new RedirectException(url); - } - } - - // Complete no-op, but leave in place in case we decide to adjust the base nav trail - private void addRootNavTrail(NavTree root) - { - // Intentionally don't add an "Experiment" node to the list because it's too overloaded. All content on the - // default action can be added to a portal page if desired. - } - - @Override - public PageConfig defaultPageConfig() - { - // set default help topic for controller - PageConfig config = super.defaultPageConfig(); - config.setHelpTopic("experiment"); - return config; - } - - @ActionNames("begin,gridView") - @RequiresPermission(ReadPermission.class) - public class BeginAction extends SimpleViewAction - { - @Override - public VBox getView(Object o, BindException errors) - { - VBox result = new VBox(); - - VBox runListView = createRunListView(20); - result.addView(runListView); - - RunGroupWebPart runGroups = new RunGroupWebPart(getViewContext(), false); - runGroups.showHeader(); - result.addView(runGroups); - - result.addView(new ProtocolWebPart(false, getViewContext())); - result.addView(new SampleTypeWebPart(false, getViewContext())); - result.addView(new DataClassWebPart(false, getViewContext(), null)); - - return result; - } - - @Override - public void addNavTrail(NavTree root) - { - addRootNavTrail(root); - root.addChild("Experiment"); - } - } - - @RequiresPermission(ReadPermission.class) - public class ShowRunsAction extends SimpleViewAction - { - @Override - public VBox getView(Object o, BindException errors) - { - return createRunListView(100); - } - - @Override - public void addNavTrail(NavTree root) - { - addRootNavTrail(root); - root.addChild("Experiment Runs"); - } - } - - private VBox createRunListView(int defaultMaxRows) - { - Set types = ExperimentService.get().getExperimentRunTypes(getContainer()); - ChooseExperimentTypeBean bean = new ChooseExperimentTypeBean(types, ExperimentRunType.getSelectedFilter(types, getViewContext().getRequest().getParameter("experimentRunFilter")), getViewContext().getActionURL().clone(), Collections.emptyList()); - JspView chooserView = new JspView<>("/org/labkey/experiment/experimentRunQueryHeader.jsp", bean); - - ExperimentRunListView view = ExperimentService.get().createExperimentRunWebPart(getViewContext(), bean.getSelectedFilter()); - view.setFrame(WebPartView.FrameType.NONE); - - // When paginated and the user hasn't explicitly set a maxRows, use the default maxRows size. - QuerySettings settings = view.getSettings(); - if (!settings.isMaxRowsSet() && settings.getShowRows() == ShowRows.PAGINATED) - { - settings.setMaxRows(defaultMaxRows); - } - - VBox result = new VBox(chooserView, view); - result.setFrame(WebPartView.FrameType.PORTAL); - return result; - } - - @RequiresPermission(ReadPermission.class) - @ActionNames("showRunGroups, showExperiments") - public class ShowRunGroupsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - RunGroupWebPart webPart = new RunGroupWebPart(getViewContext(), false); - webPart.setFrame(WebPartView.FrameType.NONE); - return webPart; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("runGroups"); - addRootNavTrail(root); - root.addChild("Run Groups"); - } - } - - public record Field(String domainURI, String domainName, String name, Container container) {} - public record MiniExpObject(Object rowId, String name) {} - public record TimelineSummary(MiniExpObject miniExpObject, String mostRecentValue) {} - public record ProblemType(String tableName, String fieldName, String pkName) { - public Object toHtml(List summaries) - { - return DOM.DIV( - DOM.H4(tableName), - DOM.TABLE(at(cl("table-condensed", "labkey-data-region", "table-bordered")), - DOM.THEAD(DOM.TH(pkName), DOM.TH(fieldName)), - summaries.stream().map(summary -> - DOM.TR(DOM.TD(summary.miniExpObject.name), DOM.TD(summary.mostRecentValue))) - )); - } - } - - @RequiresPermission(SiteAdminPermission.class) - public static class ReportLostFieldValuesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - // Find all the fields that could have lost data due to issue 52666 - TableInfo t = new ExpSchema(getUser(), ContainerManager.getRoot()).getTable(ExpSchema.TableType.Fields.name(), ContainerFilter.getUnsafeEverythingFilter()); - List fields = new TableSelector(t, - new SimpleFilter(FieldKey.fromParts("StorageColumnNameMatch"), false). - addCondition(FieldKey.fromParts("DomainURI"), ":AssayDomain-Data.", CompareType.DOES_NOT_CONTAIN), - null). - getArrayList(Field.class); - - // Prep audit table for querying - UserSchema auditSchema = AuditLogService.get().createSchema(getUser(), ContainerManager.getRoot()); - - Map> sampleTypeSummaries = new HashMap<>(); - Map> dataClassSummaries = new HashMap<>(); - Map> listSummaries = new HashMap<>(); - - Map> problematicFields = new LinkedHashMap<>(); - - for (Field field : fields) - { - String domainURI = field.domainURI; - String fieldName = field.name; - Container container = field.container; - Domain domain = PropertyService.get().getDomain(container, domainURI); - if (domain != null && domain.getDomainKind() != null) - { - TableInfo table = domain.getDomainKind().getTableInfo(getUser(), container, domain, ContainerFilter.getUnsafeEverythingFilter()); - - if (table != null) - { - // Drill into sample types - if (domain.getDomainKind().getClass().equals(SampleTypeDomainKind.class)) - { - // rows that currently have no value for the field with potential for data loss - List rowsWithNull = new TableSelector(table, - new HashSet<>(List.of("RowId", "Name")), - new SimpleFilter(new CompareType.CompareClause(FieldKey.fromParts(fieldName), CompareType.ISBLANK, null)), - null). - getArrayList(MiniExpObject.class); - - List fixupsNeeded = checkData( - rowsWithNull, - fieldName, - obj -> new SimpleFilter(FieldKey.fromParts("SampleId"), obj.rowId), - auditSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE, ContainerFilter.getUnsafeEverythingFilter())); - if (!fixupsNeeded.isEmpty()) - { - sampleTypeSummaries.put(new ProblemType(table.getName(), fieldName, "SampleID"), fixupsNeeded); - } - } - // and data classes/sample sources - if (domain.getDomainKind().getClass().equals(DataClassDomainKind.class)) - { - // rows samples that current have no value for the field with potential for data loss - List rowsWithNull = new TableSelector(table, - new HashSet<>(List.of("RowId", "Name")), - new SimpleFilter(new CompareType.CompareClause(FieldKey.fromParts(fieldName), CompareType.ISBLANK, null)), - null). - getArrayList(MiniExpObject.class); - - List fixupsNeeded = checkData( - rowsWithNull, - fieldName, - obj -> new SimpleFilter(FieldKey.fromParts("RowPk"), Objects.toString(obj.rowId)). - addCondition(FieldKey.fromParts("SchemaName"), "exp.data"). - addCondition(FieldKey.fromParts("QueryName"), domain.getName()), - auditSchema.getTable("QueryUpdateAuditEvent", ContainerFilter.getUnsafeEverythingFilter())); - - if (!fixupsNeeded.isEmpty()) - { - dataClassSummaries.put(new ProblemType(table.getName(), fieldName, "SourceID"), fixupsNeeded); - } - } - // and lists - if ("lists".equals(table.getUserSchema().getName())) - { - // rows samples that current have no value for the field with potential for data loss - List rowsWithNull = new ArrayList<>(); - - ColumnInfo entityIdCol = table.getColumn("EntityId"); - ColumnInfo pkCol = table.getPkColumns().get(0); - - new TableSelector(table, - List.of(entityIdCol, pkCol), - new SimpleFilter(new CompareType.CompareClause(FieldKey.fromParts(fieldName), CompareType.ISBLANK, null)), - null). - forEachResults(r -> - { - Object entityId = entityIdCol.getValue(r); - Object pk = pkCol.getValue(r); - rowsWithNull.add(new MiniExpObject(entityId, pk.toString())); - }); - - - List fixupsNeeded = checkData( - rowsWithNull, - fieldName, - obj -> new SimpleFilter(FieldKey.fromParts("ListItemEntityId"), obj.rowId), - auditSchema.getTable("ListAuditEvent", ContainerFilter.getUnsafeEverythingFilter())); - - if (!fixupsNeeded.isEmpty()) - { - listSummaries.put(new ProblemType(table.getName(), fieldName, table.getPkColumnNames().get(0)), fixupsNeeded); - } - } - - long totalRows = new TableSelector(table).getRowCount(); - long emptyRows = new TableSelector(table, new SimpleFilter(new CompareType.CompareClause(FieldKey.fromParts(fieldName), CompareType.ISBLANK, null)), null).getRowCount(); - problematicFields.put(field, Pair.of(totalRows, emptyRows)); - } - else - { - problematicFields.put(field, Pair.of(null, null)); - } - } - } - - return new HtmlView("Fixups Needed", - DOM.createHtmlFragment( - DOM.H2("Potentially Problematic Fields"), - problematicFields.isEmpty() ? "No problematic fields detected!" : - DOM.TABLE(at(cl("table-condensed", "labkey-data-region", "table-bordered")), - DOM.THEAD(DOM.TH("Domain Name"), DOM.TH("Domain URI"), DOM.TH("Field Name"), DOM.TH("Container"), DOM.TH("Total Rows"), DOM.TH("Rows with Nulls")), - problematicFields.entrySet().stream().map(e -> { - Field f = e.getKey(); - Pair counts = e.getValue(); - return DOM.TR( - DOM.TD(f.domainName), - DOM.TD(f.domainURI), - DOM.TD(f.name), - DOM.TD(f.container.getPath()), - DOM.TD(counts.first), - DOM.TD(counts.second) - ); - } - )), - - DOM.H2("Sample Types"), - sampleTypeSummaries.isEmpty() ? "No problems detected!" : - sampleTypeSummaries.entrySet().stream().map(e -> - e.getKey().toHtml(e.getValue())), - - DOM.H2("Data Classes"), - dataClassSummaries.isEmpty() ? "No problems detected!" : - dataClassSummaries.entrySet().stream().map(e -> - e.getKey().toHtml(e.getValue())), - - DOM.H2("Lists"), - listSummaries.isEmpty() ? "No problems detected!" : - listSummaries.entrySet().stream().map(e -> - e.getKey().toHtml(e.getValue())) - )); - } - - @NotNull - private List checkData(List rowsWithNull, String fieldName, Function filterGenerator, TableInfo auditTable) - { - List fixupsNeeded = new ArrayList<>(); - - // For each sample without a value today, check the audit history - for (MiniExpObject row : rowsWithNull) - { - // Order by RowId to get them in the sequence they happened in - var events = new TableSelector(auditTable, filterGenerator.apply(row), new Sort("RowId")).getArrayList(DetailedAuditTypeEvent.class); - // Remember the most recently set value - String mostRecentValue = null; - for (DetailedAuditTypeEvent event : events) - { - Map newValues = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(event.getNewRecordMap())); - if (newValues.containsKey(fieldName)) - { - // Will be the empty string if the value was intentionally set to blank - mostRecentValue = newValues.get(fieldName); - } - } - // If the value had been set before, and its most recent insert/update wasn't setting it blank, - // it's most likely a lost value - if (mostRecentValue != null && !mostRecentValue.isEmpty()) - { - fixupsNeeded.add(new TimelineSummary(row, mostRecentValue)); - } - } - return fixupsNeeded; - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Accidentally Nulled Field Report"); - } - } - - @RequiresPermission(ReadPermission.class) - public static class CreateHiddenRunGroupAction extends MutatingApiAction - { - @Override - public ApiResponse execute(SimpleApiJsonForm form, BindException errors) throws Exception - { - JSONObject json = form.getJsonObject(); - String selectionKey = json.optString("selectionKey", null); - List runs = new ArrayList<>(); - - // Accept either an explicit list of run IDs - if (json.has("runIds")) - { - JSONArray runIds = json.getJSONArray("runIds"); - for (int i = 0; i < runIds.length(); i++) - { - ExpRunImpl run = ExperimentServiceImpl.get().getExpRun(runIds.getInt(i)); - if (run != null) - { - runs.add(run); - } - } - } - // Or a reference to a DataRegion selection key - else if (selectionKey != null) - { - Set ids = DataRegionSelection.getSelectedIntegers(getViewContext(), selectionKey, false); - for (Long id : ids) - { - ExpRunImpl run = ExperimentServiceImpl.get().getExpRun(id); - if (run != null) - { - runs.add(run); - } - } - } - if (runs.isEmpty()) - { - throw new NotFoundException(); - } - - ExpExperiment group = ExperimentService.get().createHiddenRunGroup(getContainer(), getUser(), runs.toArray(new ExpRun[0])); - if (selectionKey != null) - DataRegionSelection.clearAll(getViewContext(), selectionKey); - - ApiSimpleResponse response = new ApiSimpleResponse(); - response.putBean(group, "rowId", "LSID", "name", "hidden"); - return response; - } - } - - - @RequiresPermission(ReadPermission.class) - public class DetailsAction extends QueryViewAction - { - private ExpExperimentImpl _experiment; - - public DetailsAction() - { - super(ExpObjectForm.class); - } - - private Pair> createViews(ExpObjectForm form, BindException errors) - { - _experiment = ExperimentServiceImpl.get().getExpExperiment(form.getRowId()); - if (_experiment == null) - { - throw new NotFoundException("Could not find an experiment with RowId " + form.getRowId()); - } - - if (!_experiment.getContainer().equals(getContainer())) - { - throw new RedirectException(getViewContext().cloneActionURL().setContainer(_experiment.getContainer())); - } - - List protocols = _experiment.getAllProtocols(); - - Set types = new TreeSet<>(ExperimentService.get().getExperimentRunTypes(getContainer())); - ExperimentRunType selectedType = ExperimentRunType.getSelectedFilter(types, getViewContext().getRequest().getParameter("experimentRunFilter")); - - ChooseExperimentTypeBean bean = new ChooseExperimentTypeBean(types, selectedType, getViewContext().getActionURL().clone(), protocols); - JspView chooserView = new JspView<>("/org/labkey/experiment/experimentRunQueryHeader.jsp", bean, errors); - - ExperimentRunListView runListView = ExperimentRunListView.createView(getViewContext(), bean.getSelectedFilter(), true); - runListView.getRunTable().setExperiment(_experiment); - runListView.setShowRemoveFromExperimentButton(true); - runListView.setShowDeleteButton(true); - runListView.setShowAddToRunGroupButton(true); - runListView.setShowExportButtons(true); - runListView.setShowMoveRunsButton(true); - return new Pair<>(runListView, chooserView); - } - - @Override - protected ModelAndView getHtmlView(ExpObjectForm form, BindException errors) throws Exception - { - Pair> views = createViews(form, errors); - - CustomPropertiesView customPropertiesView = new CustomPropertiesView(_experiment.getLSID(), getContainer()); - - TableInfo runGroupsTable = new ExpSchema(getUser(), getContainer()).getTable(ExpSchema.TableType.RunGroups); - - DetailsView detailsView = new DetailsView(new DataRegion(), _experiment.getRowId()); - detailsView.getDataRegion().setTable(runGroupsTable); - detailsView.getDataRegion().addColumns(runGroupsTable, "RowId,Name,Created,Modified,Contact,ExperimentDescriptionURL,Hypothesis,Comments"); - detailsView.getDataRegion().getDisplayColumn(0).setVisible(false); - detailsView.getDataRegion().getDisplayColumn(2).setWidth("60%"); - - ButtonBar bb = new ButtonBar(); - bb.setStyle(ButtonBar.Style.separateButtons); - ActionButton b = new ActionButton(ExperimentUrlsImpl.get().getShowUpdateURL(_experiment), "Edit"); - b.setDisplayPermission(UpdatePermission.class); - bb.add(b); - detailsView.getDataRegion().setButtonBar(bb); - if (_experiment.getBatchProtocol() != null) - { - detailsView.setTitle("Batch Details"); - detailsView.getDataRegion().addColumns(runGroupsTable, "BatchProtocolId"); - } - else - { - detailsView.setTitle("Run Group Details"); - } - - VBox runsVBox = new VBox(views.second, createInitializedQueryView(form, errors, false, null)); - runsVBox.setTitle("Experiment Runs"); - runsVBox.setFrame(WebPartView.FrameType.PORTAL); - - return new VBox(new StandardAndCustomPropertiesView(detailsView, customPropertiesView), runsVBox); - } - - @Override - protected ExperimentRunListView createQueryView(ExpObjectForm form, BindException errors, boolean forExport, String dataRegion) - { - return createViews(form, errors).first; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("runGroups"); - addRootNavTrail(root); - root.addChild("Run Groups", ExperimentUrlsImpl.get().getShowExperimentsURL(getContainer())); - root.addChild(_experiment.getName()); - } - } - - @RequiresPermission(ReadPermission.class) - public class ListSampleTypesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - SampleTypeWebPart view = new SampleTypeWebPart(false, getViewContext()); - view.setFrame(WebPartView.FrameType.NONE); - view.setErrorMessage(getViewContext().getRequest().getParameter("errorMessage")); - - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("sampleSets"); - addRootNavTrail(root); - root.addChild("Sample Types"); - } - } - - @RequiresPermission(ReadPermission.class) - public class ShowSampleTypeAction extends SimpleViewAction - { - private ExpSampleTypeImpl _sampleType; - - @Override - public ModelAndView getView(ExpObjectForm form, BindException errors) - { - _sampleType = SampleTypeServiceImpl.get().getSampleType(getContainer(), getUser(), form.getRowId()); - if (_sampleType == null && form.getLsid() != null) - { - if (form.getLsid().equalsIgnoreCase("Material") || form.getLsid().equalsIgnoreCase("Sample")) - { - // Not a real sample type - just show all the materials instead - throw new RedirectException(new ActionURL(ShowAllMaterialsAction.class, getContainer())); - } - // Check if the URL specifies the LSID, and stick the bean back into the form - _sampleType = SampleTypeServiceImpl.get().getSampleType(form.getLsid()); - } - - if (_sampleType == null) - { - throw new NotFoundException("No matching sample type found"); - } - - List allScopedSampleTypes = (List) SampleTypeService.get().getSampleTypes(getContainer(), getUser(), true); - if (!allScopedSampleTypes.contains(_sampleType)) - { - ensureCorrectContainer(getContainer(), _sampleType, getViewContext()); - } - - SamplesSchema schema = new SamplesSchema(getUser(), getContainer()); - QuerySettings settings = schema.getSettings(getViewContext(), "Material", _sampleType.getName()); - QueryView queryView = new SampleTypeContentsView(_sampleType, schema, settings, errors); - - DetailsView detailsView = new DetailsView(getSampleTypeRegion(getViewContext()), _sampleType.getRowId()); - detailsView.getDataRegion().getDisplayColumn("Name").setURL((ActionURL)null); - detailsView.getDataRegion().getDisplayColumn("LSID").setVisible(false); - detailsView.getDataRegion().getDisplayColumn("MaterialLSIDPrefix").setVisible(false); - detailsView.getDataRegion().getDisplayColumn("LabelColor").setVisible(false); - detailsView.getDataRegion().getDisplayColumn("MetricUnit").setVisible(false); - detailsView.getDataRegion().getDisplayColumn("Category").setVisible(false); - - detailsView.setTitle("Sample Type Properties"); - detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).setStyle(ButtonBar.Style.separateButtons); - - Container autoLinkContainer = _sampleType.getAutoLinkTargetContainer(); - if (null != autoLinkContainer) - { - DisplayColumn autoLinkTargetColumn = detailsView.getDataRegion().getDisplayColumn("autoLinkTargetContainer"); - autoLinkTargetColumn.setVisible(false); - - SimpleDisplayColumn displayAutoLinkTargetColumn = new SimpleDisplayColumn(); - displayAutoLinkTargetColumn.setCaption("Auto Link Target Container:"); - String path = autoLinkContainer.getPath(); - displayAutoLinkTargetColumn.setDisplayHtml(path.equals("/") ? "" : path); - detailsView.getDataRegion().addDisplayColumn(displayAutoLinkTargetColumn); - } - - DisplayColumn autoLinkCategoryColumn = detailsView.getDataRegion().getDisplayColumn("autoLinkCategory"); - autoLinkCategoryColumn.setVisible(false); - SimpleDisplayColumn displayAutoLinkCategoryColumn = new SimpleDisplayColumn(); - displayAutoLinkCategoryColumn.setCaption("Auto Link Category:"); - displayAutoLinkCategoryColumn.setDisplayHtml(_sampleType.getAutoLinkCategory()); - detailsView.getDataRegion().addDisplayColumn(displayAutoLinkCategoryColumn); - - if (_sampleType.hasNameAsIdCol()) - { - SimpleDisplayColumn nameIdCol = new SimpleDisplayColumn(); - nameIdCol.setCaption("Has Name Id Column:"); - nameIdCol.setDisplayHtml("true"); - detailsView.getDataRegion().addDisplayColumn(nameIdCol); - } - - if (_sampleType.hasIdColumns()) - { - SimpleDisplayColumn idCols = new SimpleDisplayColumn(); - idCols.setCaption("Id Column(s):"); - String names = _sampleType.getIdCols().stream() - .filter(Objects::nonNull) - .map(DomainProperty::getName) - .collect(Collectors.joining(", ")); - if (!names.isEmpty()) - { - idCols.setDisplayHtml(PageFlowUtil.filter(names)); - detailsView.getDataRegion().addDisplayColumn(idCols); - } - } - - if (_sampleType.getParentCol() != null) - { - SimpleDisplayColumn parentCol = new SimpleDisplayColumn(PageFlowUtil.filter(_sampleType.getParentCol().getName())); - parentCol.setCaption("Parent Column:"); - detailsView.getDataRegion().addDisplayColumn(parentCol); - } - - try - { - SimpleDisplayColumn importAliasCol = new SimpleDisplayColumn(); - importAliasCol.setCaption("Parent Import Alias(es):"); - if (!_sampleType.getImportAliases().isEmpty()) - importAliasCol.setDisplayHtml(PageFlowUtil.filter(StringUtils.join(_sampleType.getImportAliases().keySet(), ", "))); - detailsView.getDataRegion().addDisplayColumn(importAliasCol); - } - catch (IOException e) - { - // unable to parse import alias map from JSON - } - - if (!getContainer().equals(_sampleType.getContainer())) - { - ActionURL definitionURL = urlProvider(ExperimentUrls.class).getShowSampleTypeURL(_sampleType); - SimpleDisplayColumn definedInCol = new SimpleDisplayColumn("" + - PageFlowUtil.filter(_sampleType.getContainer().getPath()) + - ""); - definedInCol.setCaption("Defined In:"); - detailsView.getDataRegion().addDisplayColumn(definedInCol); - } - - // Not all sample types can be edited - DomainKind domainKind = _sampleType.getDomain().getDomainKind(); - if (domainKind != null && domainKind.canEditDefinition(getUser(), _sampleType.getDomain())) - { - if (domainKind instanceof SampleTypeDomainKind) - { - ActionURL updateURL = new ActionURL(EditSampleTypeAction.class, _sampleType.getContainer()); - updateURL.addParameter("RowId", _sampleType.getRowId()); - updateURL.addReturnUrl(getViewContext().getActionURL()); - - if (!getContainer().equals(_sampleType.getContainer())) - { - String editLink = updateURL.toString(); - ActionButton updateButton = new ActionButton("Edit Type"); - updateButton.setActionType(ActionButton.Action.SCRIPT); - updateButton.setScript("if (window.confirm('This sample type is defined in the " + _sampleType.getContainer().getPath() + " folder. Would you still like to edit it?')) { window.location = '" + editLink + "' }"); - detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).add(updateButton); - } - else - { - ActionButton updateButton = new ActionButton(updateURL, "Edit Type", ActionButton.Action.LINK); - updateButton.setDisplayPermission(DesignSampleTypePermission.class); - updateButton.setPrimary(true); - detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).add(updateButton); - } - - ActionURL deleteURL = new ActionURL(DeleteSampleTypesAction.class, _sampleType.getContainer()); - deleteURL.addParameter("singleObjectRowId", _sampleType.getRowId()); - deleteURL.addReturnUrl(ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer())); - ActionButton deleteButton = new ActionButton(deleteURL, "Delete Type", ActionButton.Action.LINK); - deleteButton.setDisplayPermission(DesignSampleTypePermission.class); - detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).add(deleteButton); - } - else - { - ActionURL editURL = domainKind.urlEditDefinition(_sampleType.getDomain(), new ViewBackgroundInfo(_sampleType.getContainer(), getUser(), getViewContext().getActionURL())); - if (editURL != null) - { - editURL.addReturnUrl(getViewContext().getActionURL()); - ActionButton editTypeButton = new ActionButton(editURL, "Edit Fields"); - editTypeButton.setDisplayPermission(UpdatePermission.class); - detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).add(editTypeButton); - } - } - } - - if (_sampleType.canImportMoreSamples()) - { - TableInfo table = queryView.getTable(); - if (table != null) - { - ActionURL importURL = table.getImportDataURL(getContainer()); - if (importURL != null) - { - importURL = importURL.clone(); - importURL.addReturnUrl(getViewContext().getActionURL()); - ActionButton uploadButton = new ActionButton(importURL, "Import More Samples", ActionButton.Action.LINK); - uploadButton.setDisplayPermission(UpdatePermission.class); - detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).add(uploadButton); - } - } - } - - var publish = StudyPublishService.get(); - if (AuditLogService.get().isViewable() && publish != null) - { - ContainerFilter cf = ContainerFilter.Type.CurrentAndSubfolders.create(getContainer(), getUser()); - ActionURL linkToStudyHistoryURL = publish.getPublishHistory(getContainer(), Dataset.PublishSource.SampleType, _sampleType.getRowId(), cf); - ActionButton linkToStudyHistoryButton = new ActionButton(linkToStudyHistoryURL, "Link to Study History", ActionButton.Action.LINK); - linkToStudyHistoryButton.setDisplayPermission(InsertPermission.class); - detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).add(linkToStudyHistoryButton); - } - - return new VBox(detailsView, queryView); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("sampleSets"); - ActionURL url = ExperimentUrls.get().getShowSampleTypeListURL(getContainer()); - addRootNavTrail(root); - root.addChild("Sample Types", url); - root.addChild("Sample Type " + _sampleType.getName()); - } - } - - @RequiresPermission(ReadPermission.class) - public class ShowAllMaterialsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - ExpSchema schema = new ExpSchema(getUser(), getContainer()); - QuerySettings settings = schema.getSettings(getViewContext(), "Materials", ExpSchema.TableType.Materials.toString()); - QueryView view = new QueryView(schema, settings, errors) - { - @Override - protected void populateButtonBar(DataView view, ButtonBar bar) - { - super.populateButtonBar(view, bar); - bar.add(SampleTypeContentsView.getDeriveSamplesButton(getContainer(),null)); - } - }; - view.setShowDetailsColumn(false); - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("sampleSets"); - addRootNavTrail(root); - root.addChild("All Materials"); - } - } - - /** - * Only shows standard and custom properties, not parent and child samples. Used for indexing - */ - @RequiresPermission(ReadPermission.class) - public class ShowMaterialSimpleAction extends SimpleViewAction - { - protected ExpMaterialImpl _material; - - @Override - public VBox getView(ExpObjectForm form, BindException errors) throws Exception - { - Container c = getContainer(); - _material = ExperimentServiceImpl.get().getExpMaterial(form.getRowId()); - if (_material == null && form.getLsid() != null) - { - _material = ExperimentServiceImpl.get().getExpMaterial(form.getLsid()); - } - if (_material == null) - { - throw new NotFoundException("Could not find a material with RowId " + form.getRowId()); - } - - ensureCorrectContainer(getContainer(), _material, getViewContext()); - - ExpRunImpl run = _material.getRun(); - ExpProtocol sourceProtocol = _material.getSourceProtocol(); - ExpProtocolApplication sourceProtocolApplication = _material.getSourceApplication(); - - DataRegion dr = new DataRegion(); - dr.addColumns(ExperimentServiceImpl.get().getTinfoMaterial().getUserEditableColumns()); - dr.removeColumns("RowId", "RunId", "LastIndexed", "LSID", "SourceApplicationId", "CpasType"); - - //dr.addColumns(extraProps); - dr.addDisplayColumn(new ExperimentRunDisplayColumn(run, "Source Experiment Run")); - dr.addDisplayColumn(new ProtocolDisplayColumn(sourceProtocol, "Source Protocol")); - dr.addDisplayColumn(new ProtocolApplicationDisplayColumn(sourceProtocolApplication, "Source Protocol Application")); - dr.addDisplayColumn(new LineageGraphDisplayColumn(_material, run)); - dr.addDisplayColumn(new SampleTypeDisplayColumn(_material)); - - //TODO: Can't yet edit materials uploaded from a material source - dr.setButtonBar(new ButtonBar()); - DetailsView detailsView = new DetailsView(dr, _material.getRowId()); - detailsView.setTitle("Standard Properties"); - detailsView.setFrame(WebPartView.FrameType.PORTAL); - - CustomPropertiesView cpv = new CustomPropertiesView(_material, c, getUser()); - - return new VBox(new StandardAndCustomPropertiesView(detailsView, cpv)); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("sampleSets"); - addRootNavTrail(root); - root.addChild("Sample Types", ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer())); - ExpSampleType sampleType = _material.getSampleType(); - if (sampleType != null) - { - root.addChild(sampleType.getName(), ExperimentUrlsImpl.get().getShowSampleTypeURL(sampleType)); - } - root.addChild("Sample " + _material.getName()); - } - } - - @RequiresPermission(ReadPermission.class) - public class ShowMaterialAction extends ShowMaterialSimpleAction - { - @Override - public VBox getView(ExpObjectForm form, BindException errors) throws Exception - { - VBox vbox = super.getView(form, errors); - - List materialsToInvestigate = new ArrayList<>(); - final Set successorRuns = new HashSet<>(); - materialsToInvestigate.add(_material); - Set investigatedMaterials = new HashSet<>(); - do - { - // Query for all the next tier of materials at once - issue 45402 - List followupRuns = ExperimentService.get().getRunsUsingMaterials(materialsToInvestigate); - - // Mark this set as investigated and reset for the next cycle - investigatedMaterials.addAll(materialsToInvestigate); - materialsToInvestigate = new ArrayList<>(); - - for (ExpRun r : followupRuns) - { - // Only expand the material outputs of the run if it's our first time visiting it - if (successorRuns.add(r)) - { - materialsToInvestigate.addAll(r.getMaterialOutputs()); - } - } - - if (successorRuns.size() > 1000) - { - // Give up - there may be a cycle or other problematic data - break; - } - - // Cull the ones we've already looked up - materialsToInvestigate.removeAll(investigatedMaterials); - } - while (!materialsToInvestigate.isEmpty()); - - HtmlStringBuilder updateLinks = HtmlStringBuilder.of(); - ExpSampleType st = _material.getSampleType(); - if (st != null && st.getContainer() != null && st.getContainer().hasPermission(getUser(), UpdatePermission.class)) - { - // XXX: ridiculous amount of work to get a update url expression for the sample type's table. - UserSchema samplesSchema = QueryService.get().getUserSchema(getUser(), st.getContainer(), "Samples"); - QueryDefinition queryDef = samplesSchema.getQueryDefForTable(st.getName()); - StringExpression expr = queryDef.urlExpr(QueryAction.updateQueryRow, null); - if (expr != null) - { - // Since we're building a detailsURL outside the context of a "row" need to set the correct - // container context on the generated expr. - ((DetailsURL) expr).setContainerContext(st.getContainer()); - String url = expr.eval(Collections.singletonMap(new FieldKey(null, "RowId"), _material.getRowId())); - updateLinks.append(LinkBuilder.labkeyLink("edit", url)).append(" "); - } - } - - if (getContainer().hasPermission(getUser(), InsertPermission.class)) - { - ActionURL deriveURL = new ActionURL(DeriveSamplesChooseTargetAction.class, getContainer()); - deriveURL.addParameter("rowIds", _material.getRowId()); - if (st != null) - deriveURL.addParameter("targetSampleTypeId", st.getRowId()); - - updateLinks.append(LinkBuilder.labkeyLink("derive samples from this sample", deriveURL)).append(" "); - } - - vbox.addView(new HtmlView(updateLinks)); - - ExperimentRunListView runListView = ExperimentRunListView.createView(getViewContext(), ExperimentRunType.ALL_RUNS_TYPE, true); - runListView.setShowRecordSelectors(false); - runListView.getRunTable().setRuns(successorRuns); - runListView.getRunTable().setContainerFilter(new ContainerFilter.AllFolders(getUser())); - runListView.setAllowableContainerFilterTypes(ContainerFilter.Type.Current, ContainerFilter.Type.CurrentAndSubfolders, ContainerFilter.Type.AllFolders); - runListView.setTitle("Runs associated with this material or a derived material"); - - ParentChildView pv = new ParentChildView(_material, getViewContext()); - vbox.addView(pv); - vbox.addView(runListView); - - return vbox; - } - } - - - // - // DataClass - // - - @RequiresPermission(ReadPermission.class) - public class ListDataClassAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - DataClassWebPart view = new DataClassWebPart(false, getViewContext(), null); - view.setFrame(WebPartView.FrameType.NONE); - view.setErrorMessage(getViewContext().getRequest().getParameter("errorMessage")); - - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("dataClass"); - addRootNavTrail(root); - root.addChild("Data Classes"); - } - } - - public static class DataClassForm extends ExpObjectForm - { - private String _name; - - public String getName() - { - return _name; - } - - public void setName(String name) - { - _name = name; - } - - public ExpDataClassImpl getDataClass(@Nullable Container container) - { - ExpDataClassImpl dataClass = null; - - if (getName() != null) - { - dataClass = ExperimentServiceImpl.get().getDataClass(getContainer(), getUser(), getName()); - if (dataClass == null) - throw new NotFoundException("No data class found for name '" + getName() + "'."); - } - else if (getRowId() > 0) - { - dataClass = ExperimentServiceImpl.get().getDataClass(getContainer(), getUser(), getRowId()); - } - - if (dataClass == null) - throw new NotFoundException("No data class found."); - else if (container != null && !container.equals(dataClass.getContainer())) - throw new NotFoundException("Data class is not defined in the given container."); - - return dataClass; - } - } - - @RequiresPermission(ReadPermission.class) - public class ShowDataClassAction extends SimpleViewAction - { - private ExpDataClassImpl _dataClass; - - @Override - public ModelAndView getView(DataClassForm form, BindException errors) - { - _dataClass = form.getDataClass(null); - return new VBox(getDataClassPropertiesView(), getDataClassContentsView(errors)); - } - - private DetailsView getDataClassPropertiesView() - { - ExpSchema expSchema = new ExpSchema(getUser(), _dataClass.getContainer()); - - TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, null); - QueryUpdateForm tvf = new QueryUpdateForm(table, getViewContext(), null); - tvf.setPkVal(_dataClass.getRowId()); - DetailsView detailsView = new DetailsView(tvf); - detailsView.setTitle("Data Class Properties"); - - ButtonBar bb = new ButtonBar(); - bb.setStyle(ButtonBar.Style.separateButtons); - boolean inDefinitionContainer = getContainer().equals(_dataClass.getContainer()); - - DomainKind domainKind = _dataClass.getDomain().getDomainKind(); - if (domainKind != null && domainKind.canEditDefinition(getUser(), _dataClass.getDomain())) - { - ActionURL updateURL = new ActionURL(EditDataClassAction.class, _dataClass.getContainer()); - updateURL.addParameter("rowId", _dataClass.getRowId()); - updateURL.addReturnUrl(urlProvider(ExperimentUrls.class).getShowDataClassURL(_dataClass.getContainer(), _dataClass.getRowId())); - - if (inDefinitionContainer) - { - ActionButton updateButton = new ActionButton(updateURL, "Edit Data Class", ActionButton.Action.LINK); - updateButton.setDisplayPermission(DesignDataClassPermission.class); - updateButton.setPrimary(true); - bb.add(updateButton); - } - else if (_dataClass.getContainer().hasPermission(getUser(), DesignDataClassPermission.class)) - { - ActionButton updateButton = new ActionButton("Edit Data Class"); - updateButton.setActionType(ActionButton.Action.SCRIPT); - updateButton.setScript("if (window.confirm('This data class is defined in the " + _dataClass.getContainer().getPath() + " folder. Would you still like to edit it?')) { window.location = '" + updateURL + "' }"); - updateButton.setPrimary(true); - bb.add(updateButton); - } - - ActionURL deleteURL = new ActionURL(DeleteDataClassAction.class, _dataClass.getContainer()); - deleteURL.addParameter("singleObjectRowId", _dataClass.getRowId()); - deleteURL.addReturnUrl(ExperimentUrlsImpl.get().getDataClassListURL(getContainer())); - ActionButton deleteButton = new ActionButton(deleteURL, "Delete Data Class", ActionButton.Action.LINK); - - if (inDefinitionContainer) - { - deleteButton.setDisplayPermission(DesignDataClassPermission.class); - bb.add(deleteButton); - } - else if (_dataClass.getContainer().hasPermission(getUser(), DesignDataClassPermission.class)) - { - bb.add(deleteButton); - } - } - detailsView.getDataRegion().setButtonBar(bb); - - if (!inDefinitionContainer) - { - ActionURL definitionURL = urlProvider(ExperimentUrls.class).getShowDataClassURL(_dataClass.getContainer(), _dataClass.getRowId()); - LinkBuilder link = LinkBuilder.simpleLink(_dataClass.getContainer().getPath(), definitionURL); - SimpleDisplayColumn definedInCol = new SimpleDisplayColumn(link.toString()); - definedInCol.setCaption("Defined In:"); - detailsView.getDataRegion().addDisplayColumn(definedInCol); - } - - return detailsView; - } - - private QueryView getDataClassContentsView(BindException errors) - { - UserSchema dataClassSchema = QueryService.get().getUserSchema(getUser(), getContainer(), ExpSchema.SCHEMA_EXP_DATA); - QuerySettings settings = dataClassSchema.getSettings(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, _dataClass.getName()); - - return new QueryView(dataClassSchema, settings, errors) - { - @Override - public @NotNull LinkedHashSet getClientDependencies() - { - LinkedHashSet resources = super.getClientDependencies(); - resources.add(ClientDependency.fromPath("Ext4")); - resources.add(ClientDependency.fromPath("dataregion/confirmDelete.js")); - return resources; - } - - @Override - public ActionButton createDeleteButton() - { - ActionButton button = super.createDeleteButton(); - if (button != null) - { - String dependencyText = ExperimentService.get() - .getObjectReferencers() - .stream() - .map(referencer -> referencer.getObjectReferenceDescription(ExpData.class)) - .collect(Collectors.joining(" or ")); - - button.setScript("LABKEY.dataregion.confirmDelete(" + - PageFlowUtil.jsString(getDataRegionName()) + ", " + - PageFlowUtil.jsString(ExpSchema.SCHEMA_EXP_DATA.toString()) + ", " + - PageFlowUtil.jsString(getQueryDef().getName()) + ", " + - "'experiment', 'getDataOperationConfirmationData.api', " + - PageFlowUtil.jsString(getSelectionKey()) + ", " + - "'data object', 'data objects', '" + dependencyText + "', {dataOperation: 'Delete'})"); - button.setRequiresSelection(true); - } - return button; - } - }; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("dataClass"); - addRootNavTrail(root); - root.addChild("Data Classes", ExperimentUrlsImpl.get().getDataClassListURL(getContainer())); - root.addChild(_dataClass.getName()); - } - } - - @RequiresPermission(DesignDataClassPermission.class) - public class DeleteDataClassAction extends AbstractDeleteAction - { - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("dataClass"); - super.addNavTrail(root); - } - - @Override - protected void deleteObjects(DeleteForm deleteForm) - { - List dataClasses = getDataClasses(deleteForm); - if (!ensureCorrectContainer(dataClasses)) - { - throw new UnauthorizedException(); - } - for (ExpRun run : getRuns(dataClasses)) - { - if (!run.getContainer().hasPermission(getUser(), DeletePermission.class)) - { - throw new UnauthorizedException(); - } - } - for (ExpDataClass dataClass : dataClasses) - { - dataClass.delete(getUser(), deleteForm.getUserComment()); - } - } - - @Override - public ModelAndView getView(DeleteForm deleteForm, boolean reshow, BindException errors) - { - List dataClasses = getDataClasses(deleteForm); - - if (!ensureCorrectContainer(dataClasses)) - { - throw new RedirectException(ExperimentUrlsImpl.get().getDataClassListURL(getContainer(), "To delete a data class, you must be in its folder or project.")); - } - - return new ConfirmDeleteView("Data Class", ShowDataClassAction.class, dataClasses, deleteForm, getRuns(dataClasses)); - } - - private List getDataClasses(DeleteForm deleteForm) - { - List dataClasses = new ArrayList<>(); - for (long rowId : deleteForm.getIds(false)) - { - ExpDataClass dataClass = ExperimentServiceImpl.get().getDataClass(getContainer(), getUser(), rowId); - if (dataClass != null) - { - dataClasses.add(dataClass); - } - } - return dataClasses; - } - - private boolean ensureCorrectContainer(List dataClasses) - { - for (ExpDataClass dataClass : dataClasses) - { - Container sourceContainer = dataClass.getContainer(); - if (!sourceContainer.equals(getContainer())) - { - return false; - } - } - return true; - } - - private List getRuns(List dataClasses) - { - if (!dataClasses.isEmpty()) - { - List runArray = ExperimentService.get().getRunsUsingDataClasses(dataClasses); - return ExperimentService.get().runsDeletedWithInput(runArray); - } - else - { - return Collections.emptyList(); - } - } - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(ReadPermission.class) - public static class GetDataClassPropertiesAction extends ReadOnlyApiAction - { - @Override - public Object execute(DataClassForm form, BindException errors) throws Exception - { - ExpDataClass dataClass = form.getDataClass(getContainer()); - if (dataClass != null) - return new DataClassDomainKindProperties(dataClass); - else - throw new NotFoundException("Data class does not exist in this container for rowId " + form.getRowId() + "."); - } - } - - @RequiresPermission(DesignDataClassPermission.class) - public static class EditDataClassAction extends SimpleViewAction - { - private ExpDataClassImpl _dataClass; - - @Override - public ModelAndView getView(DataClassForm form, BindException errors) - { - boolean create = form.getLSID() == null && form.getRowId() == 0 && form.getName() == null; - if (!create) - _dataClass = form.getDataClass(getContainer()); - - return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("dataClassDesigner")); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("dataClass"); - - root.addChild("Data Classes", ExperimentUrlsImpl.get().getDataClassListURL(getContainer())); - if (_dataClass == null) - { - root.addChild("Create Data Class"); - } - else - { - root.addChild(_dataClass.getName(), ExperimentUrlsImpl.get().getShowDataClassURL(getContainer(), _dataClass.getRowId())); - root.addChild("Update Data Class"); - } - } - } - - @RequiresPermission(DesignDataClassPermission.class) - public static class CreateDataClassFromTemplateAction extends FormViewAction - { - private ActionURL _successUrl; - private Map _domainTemplates; - - @Override - public void validateCommand(CreateDataClassFromTemplateForm form, Errors errors) - { - String name = null; - _domainTemplates = DomainTemplateGroup.getAllTemplates(getContainer()); - - if (!_domainTemplates.containsKey(form.getDomainTemplate())) - { - errors.reject(ERROR_MSG, "Unknown template selected: " + form.getDomainTemplate()); - } - else - { - DomainTemplate template = _domainTemplates.get(form.getDomainTemplate()); - name = template.getTemplateName(); - - // Issue 40230: if template includes sample type option, verify that it exists - if (template.getOptions().containsKey("sampleSet")) - { - String sampleTypeName = template.getOptions().get("sampleSet").toString(); - ExpSampleType sampleType = SampleTypeServiceImpl.get().getSampleType(getContainer(), getUser(), sampleTypeName); - if (sampleType == null) - errors.reject(ERROR_MSG, "Unable to find a sample type in this container with name: " + sampleTypeName + "."); - } - } - - if (StringUtils.isBlank(name)) - errors.reject(ERROR_MSG, "DataClass template selection is required."); - else if (ExperimentService.get().getDataClass(getContainer(), getUser(), name) != null) - errors.reject(ERROR_MSG, "DataClass '" + name + "' already exists."); - - } - - @Override - public ModelAndView getView(CreateDataClassFromTemplateForm form, boolean reshow, BindException errors) - { - Set templates = DomainTemplateGroup.getTemplatesForDomainKind(getContainer(), DataClassDomainKind.NAME); - form.setAvailableDomainTemplateNames(templates); - - Set messages = new HashSet<>(); - Map groups = DomainTemplateGroup.getAllGroups(getContainer()); - for (DomainTemplateGroup g : groups.values()) - messages.addAll(g.getErrors()); - form.setXmlParseErrors(messages); - - return new JspView<>("/org/labkey/experiment/createDataClassFromTemplate.jsp", form, errors); - } - - @Override - public boolean handlePost(CreateDataClassFromTemplateForm form, BindException errors) throws Exception - { - DomainTemplate template = _domainTemplates.get(form.getDomainTemplate()); - Domain domain = DomainUtil.createDomain(template, getContainer(), getUser(), form.getName()); - - _successUrl = domain.getDomainKind().urlEditDefinition(domain, getViewContext()); - return true; - } - - @Override - public URLHelper getSuccessURL(CreateDataClassFromTemplateForm form) - { - return _successUrl; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("dataClass"); - root.addChild("Create Data Class from Template"); - } - } - - public static class CreateDataClassFromTemplateForm extends DataClass - { - private String _domainTemplate; - private Set _availableDomainTemplateNames; - private Set _xmlParseErrors; - private final ReturnUrlForm _returnUrlForm = new ReturnUrlForm(); - - public String getDomainTemplate() - { - return _domainTemplate; - } - - public void setDomainTemplate(String domainTemplate) - { - _domainTemplate = domainTemplate; - } - - public Set getAvailableDomainTemplateNames() - { - return _availableDomainTemplateNames; - } - - public void setAvailableDomainTemplateNames(Set availableDomainTemplateNames) - { - _availableDomainTemplateNames = availableDomainTemplateNames; - } - - public Set getXmlParseErrors() - { - return _xmlParseErrors; - } - - public void setXmlParseErrors(Set xmlParseErrors) - { - _xmlParseErrors = xmlParseErrors; - } - - @Nullable - public String getReturnUrl() - { - return _returnUrlForm.getReturnUrl(); - } - - public void setReturnUrl(String s) - { - _returnUrlForm.setReturnUrl(s); - } - } - - public static class ConceptURIForm - { - private String _conceptURI; - - public String getConceptURI() - { - return _conceptURI; - } - - public void setConceptURI(String conceptURI) - { - _conceptURI = conceptURI; - } - } - - @RequiresPermission(AdminPermission.class) - public static class RemoveConceptMappingAction extends MutatingApiAction - { - @Override - public void validateForm(ConceptURIForm form, Errors errors) - { - if (form.getConceptURI() == null || ConceptURIProperties.getLookup(getContainer(), form.getConceptURI()) == null) - errors.reject(ERROR_MSG, "Concept URI not found: " + form.getConceptURI()); - } - - @Override - public Object execute(ConceptURIForm form, BindException errors) - { - ConceptURIProperties.removeLookup(getContainer(), form.getConceptURI()); - return new ApiSimpleResponse("success", true); - } - } - - @RequiresPermission(ReadPermission.class) - public static class RunAttachmentDownloadAction extends BaseDownloadAction - { - @Nullable - @Override - public Pair getAttachment(AttachmentForm form) - { - if (form.getLsid() == null || form.getName() == null) - throw new NotFoundException("Error: missing required param 'lsid' or 'name'."); - - ExpRun run = ExperimentService.get().getExpRun(form.getLsid()); - if (run == null) - throw new NotFoundException("Run not found: " + form.getLsid()); - - if (!run.getContainer().equals(getContainer())) - { - if (run.getContainer().hasPermission(getUser(), ReadPermission.class)) - throw new RedirectException(getViewContext().cloneActionURL().setContainer(run.getContainer())); - else - throw new NotFoundException("Run not found"); - } - - AttachmentParent parent = new ExpRunAttachmentParent(run); - return new Pair<>(parent, form.getName()); - } - } - - @RequiresPermission(ReadPermission.class) - public static class DataClassAttachmentDownloadAction extends BaseDownloadAction - { - @Nullable - @Override - public Pair getAttachment(AttachmentForm form) - { - if (form.getLsid() == null || form.getName() == null) - throw new NotFoundException("Error: missing required param 'lsid' or 'name'."); - - Lsid lsid = new Lsid(form.getLsid()); - ExpData data = ExperimentServiceImpl.get().getExpData(lsid.toString()); - if (data == null) - throw new NotFoundException("Error: Data object not found for the given LSID: " + lsid); - AttachmentParent parent = new ExpDataClassAttachmentParent(data.getContainer(), lsid); - - return new Pair<>(parent, form.getName()); - } - } - - public static class AttachmentForm extends LsidForm implements BaseDownloadAction.InlineDownloader - { - private String _name; - private boolean _inline = true; - - public String getName() - { - return _name; - } - - public void setName(String name) - { - _name = name; - } - - @Override - public boolean isInline() - { - return _inline; - } - - public void setInline(boolean inline) - { - _inline = inline; - } - } - - // - // END DataClass actions - // - - public static ActionURL getRunGraphURL(Container c, long runId) - { - return new ActionURL(ShowRunGraphAction.class, c).addParameter("rowId", runId); - } - - - @RequiresPermission(ReadPermission.class) - public class ShowRunGraphAction extends AbstractShowRunAction - { - @Override - protected VBox createLowerView(ExpRunImpl experimentRun, BindException errors) - { - return new VBox( - createRunViewTabs(experimentRun, false, true, true), - new ExperimentRunGraphView(experimentRun, false)); - } - } - - - @RequiresPermission(ReadPermission.class) - public static class DownloadGraphAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ExperimentRunForm form, BindException errors) throws Exception - { - boolean detail = form.isDetail(); - String focus = form.getFocus(); - String focusType = form.getFocusType(); - - ExpRunImpl experimentRun = (ExpRunImpl) form.lookupRun(); - ensureCorrectContainer(getContainer(), experimentRun, getViewContext()); - - ExperimentRunGraph.RunGraphFiles files; - try - { - files = ExperimentRunGraph.generateRunGraph(getViewContext(), experimentRun, detail, focus, focusType); - } - catch (ExperimentException e) - { - PageFlowUtil.streamTextAsImage(getViewContext().getResponse(), "ERROR: " + e.getMessage(), 600, 150, Color.RED); - return null; - } - - try - { - PageFlowUtil.streamFile(getViewContext().getResponse(), new File(files.getImageFile().getAbsolutePath()), false); - } - catch (FileNotFoundException e) - { - getViewContext().getResponse().sendRedirect(getViewContext().getRequest().getContextPath() + "/experiment/ExperimentRunNotFound.gif"); - } - finally - { - files.release(); - } - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - throw new UnsupportedOperationException(); - } - } - - private abstract class AbstractShowRunAction extends SimpleViewAction - { - private ExpRunImpl _experimentRun; - - @Override - public ModelAndView getView(ExperimentRunForm form, BindException errors) - { - _experimentRun = (ExpRunImpl) form.lookupRun(); - ensureCorrectContainer(getContainer(), _experimentRun, getViewContext()); - - VBox vbox = new VBox(); - - JspView detailsView = new JspView<>("/org/labkey/experiment/ExperimentRunDetails.jsp", _experimentRun); - detailsView.setTitle("Standard Properties"); - - var attachmentParent = new ExpRunAttachmentParent(_experimentRun); - var attachments = AttachmentService.get().getAttachments(attachmentParent) - .stream() - .map(att -> Pair.of(att.getName(), new ActionURL(RunAttachmentDownloadAction.class, _experimentRun.getContainer()).addParameter("name", att.getName()).addParameter("lsid", _experimentRun.getLSID()))) - .collect(toList()); - CustomPropertiesView cpv = new CustomPropertiesView(_experimentRun.getLSID(), getContainer(), attachments); - - vbox.addView(new StandardAndCustomPropertiesView(detailsView, cpv)); - - HtmlStringBuilder updateLinks = HtmlStringBuilder.of(); - List runEditors = ExperimentService.get().getRunEditors(); - for (ExpRunEditor editor : runEditors) - { - if (editor.isProtocolEditor(form.lookupRun().getProtocol())) - { - updateLinks.append(LinkBuilder.labkeyLink("edit " + editor.getDisplayName() + " run", editor.getEditUrl(getContainer()).addParameter("rowId", form.getRowId()))); - } - } - - if (!updateLinks.isEmpty()) - { - HtmlView view = new HtmlView(updateLinks); - vbox.addView(view); - } - - VBox lowerView = createLowerView(_experimentRun, errors); - lowerView.setFrame(WebPartView.FrameType.PORTAL); - lowerView.setTitle("Run Details"); - NavTree tree = new NavTree(""); - File runRoot = _experimentRun.getFilePathRoot(); - if (NetworkDrive.exists(runRoot)) - { - if (!runRoot.isDirectory()) - { - runRoot = runRoot.getParentFile(); - } - PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(_experimentRun.getContainer()); - if (pipelineRoot != null) - { - if (pipelineRoot.isUnderRoot(runRoot)) - { - String path = pipelineRoot.relativePath(runRoot); - tree.addChild("View Files", urlProvider(PipelineUrls.class).urlBrowse(_experimentRun.getContainer(), null, path)); - } - } - } - - final String exportFilesFormId = "exportFilesForm"; - NavTree downloadFiles = new NavTree("Download all files"); - downloadFiles.setScript("document.getElementById('" + exportFilesFormId + "').submit();"); - tree.addChild(downloadFiles); - - // CONSIDER: Show modal dialog using ExperimentService.get().createRunExportView() - NavTree exportXarFiles = new NavTree("Export XAR"); - exportXarFiles.setScript("LABKEY.Experiment.exportRuns({runIds: [" + _experimentRun.getRowId() + "] });"); - tree.addChild(exportXarFiles); - - lowerView.setNavMenu(tree); - lowerView.setIsWebPart(false); - - vbox.addView(lowerView); - vbox.addView(new ExperimentRunGroupsView(getUser(), getContainer(), _experimentRun, getViewContext().getActionURL(), errors)); - - DOM.Renderable exportFilesForm = LK.FORM(at( - id, exportFilesFormId, - method, "POST", - action, new ActionURL(ExportRunFilesAction.class, _experimentRun.getContainer())), - INPUT(at(type, "hidden", - name, DataRegionSelection.DATA_REGION_SELECTION_KEY, - value, "ExportSingleRun")), - INPUT(at(type, "hidden", - name, DataRegion.SELECT_CHECKBOX_NAME, - value, _experimentRun.getRowId())), - INPUT(at(type, "hidden", - name, "zipFileName", - value, _experimentRun.getName() + ".zip"))); - - HtmlView hiddenFormView = new HtmlView(exportFilesForm); - vbox.addView(hiddenFormView); - - return vbox; - } - - protected abstract VBox createLowerView(ExpRunImpl experimentRun, BindException errors); - - @Override - public void addNavTrail(NavTree root) - { - root.addChild(_experimentRun.getName()); - } - } - - public static class ToggleRunExperimentMembershipForm - { - private int _runId; - private int _experimentId; - private boolean _included; - - public int getRunId() - { - return _runId; - } - - public void setRunId(int runId) - { - _runId = runId; - } - - public int getExperimentId() - { - return _experimentId; - } - - public void setExperimentId(int experimentId) - { - _experimentId = experimentId; - } - - public boolean isIncluded() - { - return _included; - } - - public void setIncluded(boolean included) - { - _included = included; - } - } - - @RequiresPermission(UpdatePermission.class) - public static class ToggleRunExperimentMembershipAction extends FormHandlerAction - { - @Override - public boolean handlePost(ToggleRunExperimentMembershipForm form, BindException errors) - { - ExpRun run = ExperimentService.get().getExpRun(form.getRunId()); - // Check if the user has permission to update this run - if (run == null || !run.getContainer().hasPermission(getUser(), UpdatePermission.class)) - { - throw new NotFoundException(); - } - - ExpExperiment exp = ExperimentService.get().getExpExperiment(form.getExperimentId()); - if (exp == null) - { - throw new NotFoundException(); - } - // Check if this - if (!ExperimentService.get().getExperiments(run.getContainer(), getUser(), true, false).contains(exp)) - { - throw new NotFoundException(); - } - // Users must have permission to view, but not necessarily update, the container the holds the run group - if (!exp.getContainer().hasPermission(getUser(), ReadPermission.class)) - { - throw new UnauthorizedException(); - } - - if (form.isIncluded()) - { - exp.addRuns(getUser(), run); - } - else - { - exp.removeRun(getUser(), run); - } - - return true; - } - - @Override - public URLHelper getSuccessURL(ToggleRunExperimentMembershipForm form) - { - return null; - } - - @Override - public void validateCommand(ToggleRunExperimentMembershipForm target, Errors errors) - { - } - } - - private HtmlView createRunViewTabs(ExpRun expRun, boolean showGraphSummary, boolean showGraphDetail, boolean showText) - { - return new HtmlView( - TABLE(cl("labkey-tab-strip"), - TR( - createTabSpacer(false), - createTab("Graph Summary View", ExperimentUrlsImpl.get().getRunGraphURL(expRun), !showGraphSummary), - createTabSpacer(false), - createTab("Graph Detail View", ExperimentUrlsImpl.get().getRunGraphDetailURL(expRun), !showGraphDetail), - createTabSpacer(false), - createTab("Text View", ExperimentUrlsImpl.get().getRunTextURL(expRun), !showText), - createTabSpacer(true)))); - } - - private DOM.Renderable createTab(String text, ActionURL url, boolean selected) - { - return TD(cl(selected,"labkey-tab-selected", "labkey-tab"), - A(at(href, url), text)); - } - - private DOM.Renderable createTabSpacer(boolean fullWidth) - { - return TD(cl("labkey-tab-space").at(fullWidth, width, "100%"), - IMG(at(src, AppProps.getInstance().getContextPath() + "/_.gif", width, "5"))); - } - - @RequiresPermission(ReadPermission.class) - public class ShowRunTextAction extends AbstractShowRunAction - { - @Override - protected VBox createLowerView(ExpRunImpl expRun, BindException errors) - { - JspView applicationsView = new JspView<>("/org/labkey/experiment/ProtocolApplications.jsp", expRun); - applicationsView.setFrame(WebPartView.FrameType.TITLE); - applicationsView.setTitle("Protocol Applications"); - - HtmlView toggleView = createRunViewTabs(expRun, true, true, false); - - QuerySettings runDataInputsSettings = new QuerySettings(getViewContext(), "RunDataInputs", DataInputs.name()); - UsageQueryView runDataInputsView = new UsageQueryView("Data Inputs", getViewContext(), expRun, ExpProtocol.ApplicationType.ExperimentRun, runDataInputsSettings, errors); - runDataInputsView.setButtonBarPosition(DataRegion.ButtonBarPosition.TOP); - - QuerySettings runDataOutputsSettings = new QuerySettings(getViewContext(), "RunDataOutputs", DataInputs.name()); - UsageQueryView runDataOutputsView = new UsageQueryView("Data Outputs", getViewContext(), expRun, ExpProtocol.ApplicationType.ExperimentRunOutput, runDataOutputsSettings, errors); - runDataOutputsView.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); - - QuerySettings runMaterialInputsSetting = new QuerySettings(getViewContext(), "RunMaterialInputs", ExpSchema.TableType.MaterialInputs.name()); - UsageQueryView runMaterialInputsView = new UsageQueryView("Material Inputs", getViewContext(), expRun, ExpProtocol.ApplicationType.ExperimentRun, runMaterialInputsSetting, errors); - runMaterialInputsView.setButtonBarPosition(DataRegion.ButtonBarPosition.TOP); - - QuerySettings runMaterialOutputsSettings = new QuerySettings(getViewContext(), "RunMaterialOutputs", ExpSchema.TableType.MaterialInputs.name()); - UsageQueryView runMaterialOutputsView = new UsageQueryView("Material Outputs", getViewContext(), expRun, ExpProtocol.ApplicationType.ExperimentRunOutput, runMaterialOutputsSettings, errors); - runMaterialOutputsView.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); - - HBox inputsView = new HBox(runDataInputsView, runMaterialInputsView); - HBox registeredInputsView = new HBox(); - - var expService = ExperimentService.get(); - expService.getRunInputsViewProviders().forEach(provider -> - { - var queryView = provider.createView(getViewContext(), expRun, errors); - if (queryView != null) - { - registeredInputsView.addView(queryView); - } - }); - HBox outputsView = new HBox(runDataOutputsView, runMaterialOutputsView); - HBox registeredOutputsView = new HBox(); - expService.getRunOutputsViewProviders().forEach(provider -> - { - var queryView = provider.createView(getViewContext(), expRun, errors); - if (queryView != null) - { - registeredOutputsView.addView(queryView); - } - }); - - var vBox = new VBox(); - vBox.addView(toggleView); - vBox.addView(inputsView); - if (!registeredInputsView.isEmpty()) - vBox.addView(registeredInputsView); - vBox.addView(outputsView); - if (!registeredOutputsView.isEmpty()) - vBox.addView(registeredOutputsView); - vBox.addView(applicationsView); - - return vBox; - } - } - - private static class UsageQueryView extends QueryView - { - private final ExpRun _run; - private final ExpProtocol.ApplicationType _type; - - public UsageQueryView(String title, ViewContext context, ExpRun run, ExpProtocol.ApplicationType type, - QuerySettings settings, BindException errors) - { - super(new ExpSchema(context.getUser(), context.getContainer()), settings, errors); - setTitle(title); - setFrame(FrameType.TITLE); - _run = run; - _type = type; - setShowBorders(true); - setShadeAlternatingRows(true); - setShowExportButtons(false); - setShowPagination(false); - disableContainerFilterSelection(); - } - - @Override - protected TableInfo createTable() - { - String tableName = getSettings().getQueryName(); - ExpInputTable tableInfo = (ExpInputTable) getSchema().getTable(tableName, new ContainerFilter.AllFolders(getUser()), true, true); - tableInfo.setRun(_run, _type); - tableInfo.setLocked(true); - return tableInfo; - } - } - - - public static ActionURL getShowRunGraphDetailURL(Container c, long rowId) - { - ActionURL url = new ActionURL(ShowRunGraphDetailAction.class, c); - url.addParameter("rowId", rowId); - return url; - } - - - @RequiresPermission(ReadPermission.class) - public class ShowRunGraphDetailAction extends AbstractShowRunAction - { - @Override - protected VBox createLowerView(ExpRunImpl run, BindException errors) - { - ExperimentRunGraphView gw = new ExperimentRunGraphView(run, true); - if (null != getViewContext().getActionURL().getParameter("focus")) - gw.setFocus(getViewContext().getActionURL().getParameter("focus")); - if (null != getViewContext().getActionURL().getParameter("focusType")) - gw.setFocusType(getViewContext().getActionURL().getParameter("focusType")); - return new VBox(createRunViewTabs(run, true, false, true), gw); - } - } - - private abstract class AbstractDataAction extends SimpleViewAction - { - protected ExpDataImpl _data; - - @Override - public final ModelAndView getView(DataForm form, BindException errors) throws Exception - { - _data = form.lookupData(); - if (_data == null) - { - throw new NotFoundException("Could not find a data with RowId " + form.getRowId()); - } - - ensureCorrectContainer(getContainer(), _data, getViewContext()); - return getDataView(form, errors); - } - - protected abstract ModelAndView getDataView(DataForm form, BindException errors) throws Exception; - - @Override - public void addNavTrail(NavTree root) - { - addRootNavTrail(root); - root.addChild("Data " + _data.getName()); - } - } - - @RequiresPermission(ReadPermission.class) - public class ShowDataAction extends AbstractDataAction - { - @Override - public ModelAndView getDataView(DataForm form, BindException errors) - { - ExpRun run = _data.getRun(); - ExpProtocol sourceProtocol = _data.getSourceProtocol(); - ExpProtocolApplication sourceProtocolApplication = _data.getSourceApplication(); - ExpDataClass dataClass = _data.getDataClass(getUser()); - - ExpSchema schema = new ExpSchema(getUser(), getContainer()); - TableInfo table; - long pk; - if (dataClass == null) - { - table = schema.getDatasTable(); - pk = _data.getRowId(); - } - else - { - table = schema.getSchema(ExpSchema.NestedSchemas.data).getTable(dataClass.getName()); - pk = new TableSelector(table, Collections.singleton("rowId"), new SimpleFilter(FieldKey.fromParts("lsid"), _data.getLSID()), null).getObject(Integer.class); - } - - DataRegion dr = new DataRegion(); - dr.setTable(table); - List cols = table.getColumns().stream().filter(ColumnInfo::isShownInDetailsView).collect(toList()); - dr.addColumns(cols); - dr.removeColumns("RowId", "Created", "CreatedBy", "Modified", "ModifiedBy", "DataFileUrl", "Run", "LSID", "CpasType", "SourceApplicationId", "Folder", "Generated"); - dr.addDisplayColumn(new ExperimentRunDisplayColumn(run, "Source Experiment Run")); - dr.addDisplayColumn(new ProtocolDisplayColumn(sourceProtocol, "Source Protocol")); - dr.addDisplayColumn(new ProtocolApplicationDisplayColumn(sourceProtocolApplication, "Source Protocol Application")); - dr.addDisplayColumn(new LineageGraphDisplayColumn(_data, run)); - DetailsView detailsView = new DetailsView(dr, pk); - detailsView.setTitle("Standard Properties"); - detailsView.setFrame(WebPartView.FrameType.PORTAL); - ButtonBar bb = new ButtonBar(); - bb.setStyle(ButtonBar.Style.separateButtons); - - ExperimentDataHandler handler = _data.findDataHandler(); - ActionURL viewDataURL = handler == null ? null : handler.getContentURL(_data); - if (viewDataURL != null) - { - bb.add(new ActionButton("View data", viewDataURL)); - } - - if (_data.isPathAccessible()) - { - bb.add(new ActionButton("View file", ExperimentUrlsImpl.get().getShowFileURL(_data, true))); - bb.add(new ActionButton("Download file", ExperimentUrlsImpl.get().getShowFileURL(_data, false))); - - if (getContainer().hasPermission(getUser(), InsertPermission.class)) - { - String relativePath = null; - PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); - if (root != null) - { - Path rootFile = root.getRootNioPath(); - Path dataFile = _data.getFilePath(); - if (dataFile != null) - { - Path pathRelative; - try - { - pathRelative = rootFile.relativize(dataFile); - if (null != pathRelative) - relativePath = pathRelative.toString(); - } - catch (IllegalArgumentException e) - { - // dataFile not relative to root - } - } - } - ActionURL browseURL = urlProvider(PipelineUrls.class).urlBrowse(getContainer(), getViewContext().getActionURL(), relativePath); - bb.add(new ActionButton("Browse in pipeline", browseURL)); - } - } - - // add links to any other exp.data that share the same dataFileUrl path - var altDataList = ExperimentService.get().getAllExpDataByURL(_data.getDataFileUrl(), getContainer()); - altDataList.removeIf(_data::equals); - if (!altDataList.isEmpty()) - { - MenuButton menu = new MenuButton("Alternate Data"); - for (ExpData altData : altDataList) - { - ExpRun altDataRun = altData.getRun(); - StringBuilder sb = new StringBuilder(altData.getName()); - if (altDataRun != null) - sb.append(" created by run '").append(altDataRun.getName()).append("' (").append(altDataRun.getProtocol().getName()).append(")"); - menu.addMenuItem(sb.toString(), altData.detailsURL()); - } - bb.add(menu); - } - - dr.setButtonBarPosition(DataRegion.ButtonBarPosition.TOP); - dr.setButtonBar(bb); - - CustomPropertiesView cpv = new CustomPropertiesView(_data.getLSID(), getContainer()); - HBox hbox = new StandardAndCustomPropertiesView(detailsView, cpv); - - VBox vbox = new VBox(hbox); - - ParentChildView pv = new ParentChildView(_data, getViewContext()); - vbox.addView(pv); - - ExperimentRunListView runListView = ExperimentRunListView.createView(getViewContext(), ExperimentRunType.ALL_RUNS_TYPE, true); - runListView.getRunTable().setInputData(_data); - runListView.getRunTable().setContainerFilter(new ContainerFilter.AllFolders(getUser())); - runListView.getRunTable().setLocked(true); - runListView.setTitle("Runs using this data as an input"); - vbox.addView(runListView); - - if (_data.isInlineImage() && _data.isFileOnDisk()) - { - ActionURL showFileURL = new ActionURL(ShowFileAction.class, getContainer()).addParameter("rowId", _data.getRowId()); - HtmlView imageView = new HtmlView(IMG(at(src, showFileURL))); - return new VBox(vbox, imageView); - } - return vbox; - } - } - - @RequiresPermission(AdminPermission.class) - public static class CheckDataFileAction extends MutatingApiAction - { - private ExpDataImpl _data; - - @Override - public void validateForm(DataFileForm form, Errors errors) - { - _data = form.lookupData(); - if (_data == null) - { - errors.reject(ERROR_MSG, "No ExpData found for id: " + form.getRowId()); - } - } - - @Override - public ApiResponse execute(DataFileForm form, BindException errors) - { - File dataFile = _data.getFile(); - Container dataContainer = _data.getContainer(); - boolean fileExists = _data.isFileOnDisk(); - boolean fileExistsAtCurrent = false; - File newDataFile = null; - - ApiSimpleResponse response = new ApiSimpleResponse(); - response.put("dataFileUrl", _data.getDataFileUrl()); - response.put("fileExists", fileExists); - response.put("containerPath", dataContainer.getPath()); - - if (!fileExists) - { - PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(dataContainer); - if (pipelineRoot != null && pipelineRoot.isValid() && dataFile != null) - { - newDataFile = pipelineRoot.resolvePath("/" + AssayFileWriter.DIR_NAME + "/" + dataFile.getName()); - fileExistsAtCurrent = NetworkDrive.exists(newDataFile); - response.put("fileExistsAtCurrent", fileExistsAtCurrent); - } - } - - // if the current dataFileUrl does not exist on disk and we have the file at the current - // pipeline root /assaydata dir, fix the dataFileUrl value - if (form.isAttemptFilePathFix()) - { - if (fileExistsAtCurrent) - { - ExpDataFileListener fileListener = new ExpDataFileListener(); - fileListener.fileMoved(dataFile, newDataFile, getUser(), dataContainer); - response.put("filePathFixed", true); - - // update the ExpData object so that we can get the new dataFileUrl - _data = form.lookupData(); - response.put("newDataFileUrl", _data.getDataFileUrl()); - } - else - { - response.put("filePathFixed", false); - } - } - - response.put("success", true); - return response; - } - } - - public static class DataFileForm extends DataForm - { - private boolean _attemptFilePathFix; - - public boolean isAttemptFilePathFix() - { - return _attemptFilePathFix; - } - - public void setAttemptFilePathFix(boolean attemptFilePathFix) - { - _attemptFilePathFix = attemptFilePathFix; - } - } - - @RequiresPermission(ReadPermission.class) - public class ShowFileAction extends AbstractDataAction - { - @Override - protected ModelAndView getDataView(DataForm form, BindException errors) throws IOException - { - if (!_data.isPathAccessible()) - { - throw new NotFoundException("Data file " + _data.getDataFileUrl() + " does not exist on disk"); - } - - PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); - if (root != null && !root.isUnderRoot(_data.getFilePath())) - { - // Issue 35649: ImmPort module "publish" creates exp.data object in this container for paths that originate in a different container - FileContentService fileSvc = FileContentService.get(); - if (fileSvc == null) - throw new UnauthorizedException("Data file is not under the pipeline root for this folder"); - - List containers = fileSvc.getContainersForFilePath(_data.getFilePath()); - if (containers.isEmpty() || containers.stream().noneMatch(c -> c.hasPermission(getUser(), ReadPermission.class))) - throw new UnauthorizedException("Data file is not under the pipeline root for this folder"); - } - - //Issues 25667 and 31152 - if (form.isInline()) - { - ExperimentDataHandler h = _data.findDataHandler(); - if (h != null) - { - URLHelper url = h.getShowFileURL(_data); - if (url != null) - { - throw new RedirectException(url, false); - } - } - } - - try - { - Path realContent = _data.getFilePath(); - if (null == realContent) - throw new IllegalStateException("Path not found."); - - boolean inline = _data.isInlineImage() || form.isInline() || "inlineImage".equalsIgnoreCase(form.getFormat()); - if (_data.isInlineImage() && form.getMaxDimension() != null) - { - try (InputStream inputStream = Files.newInputStream(realContent)) - { - BufferedImage image = ImageIO.read(inputStream); - // If image, create a thumbnail, otherwise fall through as a regular download attempt - if (image != null) - { - int imageMax = Math.max(image.getHeight(), image.getWidth()); - if (imageMax > form.getMaxDimension().intValue()) - { - double scale = (double) form.getMaxDimension().intValue() / (double) imageMax; - ByteArrayOutputStream bOut = new ByteArrayOutputStream(); - ImageUtil.resizeImage(image, bOut, scale, 1); - PageFlowUtil.streamFileBytes(getViewContext().getResponse(), FileUtil.getFileName(realContent) + ".png", bOut.toByteArray(), !inline); - return null; - } - } - } - } - - boolean extended = "jsonTSVExtended".equalsIgnoreCase(form.getFormat()); - boolean ignoreTypes = "jsonTSVIgnoreTypes".equalsIgnoreCase(form.getFormat()); - if ("jsonTSV".equalsIgnoreCase(form.getFormat()) || extended || ignoreTypes) - { - if (!FileUtil.hasCloudScheme(realContent)) // TODO: handle streaming from S3 to JSON - streamToJSON(realContent.toFile(), form.getFormat(), -1, null); - return null; - } - - try (InputStream inputStream = Files.newInputStream(realContent)) - { - PageFlowUtil.streamFile(getViewContext().getResponse(), Collections.emptyMap(), FileUtil.getFileName(realContent), inputStream, !inline); - } - } - catch (IOException e) - { - try - { - // Try to write the exception back to the caller if we haven't already flushed the buffer - ApiJsonWriter writer = new ApiJsonWriter(getViewContext().getResponse()); - writer.writeResponse(e); - } - catch (IllegalStateException ise) - { - // Most likely that a disconnected client caused the IOException writing back the response - } - } - - return null; - } - } - - - public static class ParseForm - { - String format = "jsonTSV"; - int maxRows = -1; - - public String getFormat() - { - return format; - } - - public void setFormat(String format) - { - this.format = format; - } - - public int getMaxRows() - { - return maxRows; - } - - public void setMaxRows(int maxRow) - { - this.maxRows = maxRow; - } - } - - @RequiresNoPermission - public class ParseFileAction extends MutatingApiAction - { - @Override - public Object execute(ParseForm form, BindException errors) throws Exception - { - if (!(getViewContext().getRequest() instanceof MultipartHttpServletRequest)) - throw new BadRequestException("Expected MultipartHttpServletRequest when posting files."); - - MultipartFile formFile = getFileMap().get("file"); - if (formFile == null) - { - return true; - } - - File tempFile = null; - try - { - tempFile = FileUtil.createTempFile("parse", formFile.getOriginalFilename()); - FileUtil.copyData(formFile.getInputStream(), tempFile); - streamToJSON(tempFile, form.getFormat(), form.getMaxRows(), formFile.getOriginalFilename()); - } - finally - { - if (null != tempFile) - tempFile.delete(); - } - return null; - } - } - - - // SampleTypeTest - private void streamToJSON(File realContent, String format, int maxRow, String originalFileName) throws IOException - { - String lowerCaseFileName = realContent.getName().toLowerCase(); - boolean extended = "jsonTSVExtended".equalsIgnoreCase(format); - boolean ignoreTypes = "jsonTSVIgnoreTypes".equalsIgnoreCase(format); - - JSONArray sheetsArray; - if (lowerCaseFileName.endsWith(".xls") || lowerCaseFileName.endsWith(".xlsx")) - { - try - { - sheetsArray = ExcelFactory.convertExcelToJSON(realContent, extended, maxRow); - } - catch (InvalidFormatException e) - { - throw new NotFoundException("Could not open " + realContent.getName(), e); - } - } - else - { - DataLoaderFactory dlf = DataLoader.get().findFactory(realContent, null); - if (null == dlf) - { - throw new ApiUsageException("Unable to parse file " + realContent + ", it is likely of an unsupported file type"); - } - - try (DataLoader tabLoader = dlf.createLoader(realContent, true)) - { - tabLoader.setScanAheadLineCount(5000); - ColumnDescriptor[] cols = tabLoader.getColumns(); - - if (ignoreTypes) - for (ColumnDescriptor col : cols) - col.clazz = String.class; - - JSONArray rowsArray = new JSONArray(); - JSONArray headerArray = new JSONArray(); - for (ColumnDescriptor col : cols) - { - if (extended) - { - JSONObject valueObject = new JSONObject(); - valueObject.put("value", col.name); - headerArray.put(valueObject); - } - else - { - headerArray.put(col.name); - } - } - rowsArray.put(headerArray); - for (Map rowMap : tabLoader) - { - // headers count as a row to be consistent - if (maxRow > -1 && maxRow <= rowsArray.length() + 1) - break; - - JSONArray rowArray = new JSONArray(); - for (ColumnDescriptor col : cols) - { - Object value = rowMap.get(col.name); - if (extended) - { - JSONObject valueObject = new JSONObject(); - valueObject.put("value", value); - rowArray.put(valueObject); - } - else - { - rowArray.put(value); - } - } - rowsArray.put(rowArray); - } - - JSONObject sheetJSON = new JSONObject(); - sheetJSON.put("name", "flat"); - sheetJSON.put("data", rowsArray); - sheetsArray = new JSONArray(); - sheetsArray.put(sheetJSON); - } - } - - try (ApiJsonWriter writer = new ApiJsonWriter(getViewContext().getResponse())) - { - JSONObject workbookJSON = new JSONObject(); - workbookJSON.put("fileName", realContent.getName()); - workbookJSON.put("sheets", sheetsArray); - if (originalFileName != null) - workbookJSON.put("originalFileName", originalFileName); - writer.writeResponse(new ApiSimpleResponse(workbookJSON)); - } - } - - - public static class ConvertArraysToExcelForm - { - private String _json; - - public String getJson() - { - return _json; - } - - public void setJson(String json) - { - _json = json; - } - } - - @RequiresPermission(ReadPermission.class) - public static class ConvertArraysToExcelAction extends ExportAction - { - @Override - public void validate(ConvertArraysToExcelForm form, BindException errors) - { - if (form.getJson() == null) - { - errors.reject(ERROR_MSG, "Unable to convert to Excel - no spreadsheet data given"); - } - } - - @Override - public void export(ConvertArraysToExcelForm form, HttpServletResponse response, BindException errors) throws Exception - { - try - { - JSONObject rootObject; - JSONArray sheetsArray; - if (form.getJson() == null || form.getJson().trim().isEmpty()) - { - // Create JSON so that we return an empty file - rootObject = new JSONObject(); - sheetsArray = new JSONArray(); - JSONObject sheetObject = new JSONObject(); - sheetsArray.put(sheetObject); - } - else - { - rootObject = new JSONObject(form.getJson()); - sheetsArray = rootObject.getJSONArray("sheets"); - } - String filename = rootObject.has("fileName") ? rootObject.getString("fileName") : "ExcelExport.xls"; - ExcelWriter.ExcelDocumentType docType = filename.toLowerCase().endsWith(".xlsx") ? ExcelWriter.ExcelDocumentType.xlsx : ExcelWriter.ExcelDocumentType.xls; - - try (Workbook workbook = ExcelFactory.createFromArray(sheetsArray, docType)) - { - response.setContentType(docType.getMimeType()); - ResponseHelper.setContentDisposition(response, ResponseHelper.ContentDispositionType.attachment, filename); - ResponseHelper.setPrivate(response); - workbook.write(response.getOutputStream()); - - JSONObject qInfo = rootObject.has("queryinfo") ? rootObject.getJSONObject("queryinfo") : null; - if (qInfo != null) - { - QueryService.get().addAuditEvent(getUser(), getContainer(), qInfo.getString("schema"), - qInfo.getString("query"), getViewContext().getActionURL(), - rootObject.getString("auditMessage") + filename, - null); - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "convertTable", "asExcel"); - } - } - } - catch (JSONException | ClassCastException e) - { - // We can get a ClassCastException if we expect an array and get a simple String, for example - ExceptionUtil.renderErrorView(getViewContext(), getPageConfig(), ErrorRenderer.ErrorType.notFound, HttpServletResponse.SC_BAD_REQUEST, "Failed to convert to Excel - invalid input", e, false, false); - } - } - } - - @RequiresPermission(ReadPermission.class) - public static class ConvertArraysToTableAction extends ExportAction - { - @Override - public void validate(ConvertArraysToExcelForm form, BindException errors) - { - if (form.getJson() == null) - { - errors.reject(ERROR_MSG, "Unable to convert to table - no data given"); - } - } - - @Override - public void export(ConvertArraysToExcelForm form, HttpServletResponse response, BindException errors) throws Exception - { - try - { - JSONObject rootObject; - JSONArray rowsArray; - if (form.getJson() == null || form.getJson().trim().isEmpty()) - { - // Create JSON so that we return an empty file - rootObject = new JSONObject(); - rowsArray = new JSONArray(); - } - else - { - rootObject = new JSONObject(form.getJson()); - rowsArray = rootObject.getJSONArray("rows"); - } - - TSVWriter.DELIM delimType = (!rootObject.isNull("delim") ? TSVWriter.DELIM.valueOf(rootObject.getString("delim")) : TSVWriter.DELIM.TAB); - TSVWriter.QUOTE quoteType = (!rootObject.isNull("quoteChar") ? TSVWriter.QUOTE.valueOf(rootObject.getString("quoteChar")) : TSVWriter.QUOTE.NONE); - String filenamePrefix = (!rootObject.isNull("fileNamePrefix") ? rootObject.getString("fileNamePrefix") : "Export"); - String filename = filenamePrefix + "." + delimType.extension; - String newlineChar = !rootObject.isNull("newlineChar") ? rootObject.getString("newlineChar") : "\n"; - - PageFlowUtil.prepareResponseForFile(response, Collections.emptyMap(), filename, true); - response.setContentType(delimType.contentType); - - //NOTE: we could also have used TSVWriter; however, this is in use elsewhere and we dont need a custom subclass - try (CSVWriter writer = new CSVWriter(response.getWriter(), delimType.delim, quoteType.quoteChar, newlineChar)) - { - for (int i = 0; i < rowsArray.length(); i++) - { - List objectList = rowsArray.getJSONArray(i).toList(); - Iterator it = objectList.iterator(); - List list = new ArrayList<>(); - - while (it.hasNext()) - { - Object o = it.next(); - if (o != null) - list.add(o.toString()); - else - list.add(""); - } - - writer.writeNext(list.toArray(new String[0])); - } - } - - JSONObject qInfo = rootObject.optJSONObject("queryinfo"); - if (qInfo != null) - { - QueryService.get().addAuditEvent(getUser(), getContainer(), qInfo.getString("schema"), qInfo.getString("query"), - getViewContext().getActionURL(), - rootObject.getString("auditMessage") + filename, - rowsArray.length()); - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "convertTable", "asDelimited"); - } - } - catch (JSONException e) - { - ExceptionUtil.renderErrorView(getViewContext(), getPageConfig(), ErrorRenderer.ErrorType.notFound, HttpServletResponse.SC_BAD_REQUEST, "Failed to convert to table - invalid input", e, false, false); - } - } - } - - - public static class ConvertHtmlToExcelForm - { - private String _baseUrl; - private String _htmlFragment; - private String _name = "workbook.xls"; - - - public String getName() - { - return _name; - } - - public void setName(String name) - { - _name = name; - } - - public String getBaseUrl() - { - return _baseUrl; - } - - public void setBaseUrl(String baseUrl) - { - _baseUrl = baseUrl; - } - - public String getHtmlFragment() - { - return _htmlFragment; - } - - public void setHtmlFragment(String htmlFragment) - { - _htmlFragment = htmlFragment; - } - } - - - @RequiresPermission(ReadPermission.class) - public static class ConvertHtmlToExcelAction extends FormViewAction - { - String _responseHtml = null; - - @Override - public void validateCommand(ConvertHtmlToExcelForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(ConvertHtmlToExcelForm form, boolean reshow, BindException errors) - { - String html = - "
    " + - "" + - new CsrfInput(getViewContext()) + - ""; - return HtmlView.unsafe(html); - } - - @Override - public boolean handlePost(ConvertHtmlToExcelForm form, BindException errors) - { - ActionURL url = getViewContext().getActionURL(); - String base = url.getBaseServerURI(); - if (!base.endsWith("/")) base += "/"; - - String baseTag = ""; - SafeToRender css = PageFlowUtil.getStylesheetIncludes(getContainer()); - String htmlFragment = StringUtils.trimToEmpty(form.getHtmlFragment()); - String html = "" + baseTag + css + "" + htmlFragment + ""; - - // UNDONE: strip script - List tidyErrors = new ArrayList<>(); - String tidy = JSoupUtil.tidyHTML(html, false, tidyErrors); - - if (!tidyErrors.isEmpty()) - { - for (String err : tidyErrors) - { - errors.reject(ERROR_MSG, err); - } - return false; - } - - _responseHtml = tidy; - return true; - } - - @Override - public ModelAndView getSuccessView(ConvertHtmlToExcelForm form) - { - ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, form.getName()); - getPageConfig().setTemplate(PageConfig.Template.None); - HtmlView v = HtmlView.unsafe(_responseHtml); - v.setContentType("application/vnd.ms-excel"); - v.setFrame(WebPartView.FrameType.NONE); - return v; - } - - @Override - public URLHelper getSuccessURL(ConvertHtmlToExcelForm convertHtmlToExcelForm) - { - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - - public static ActionURL getShowApplicationURL(Container c, long rowId) - { - ActionURL url = new ActionURL(ShowApplicationAction.class, c); - url.addParameter("rowId", rowId); - - return url; - } - - - @RequiresPermission(ReadPermission.class) - public class ShowApplicationAction extends SimpleViewAction - { - private ExpProtocolApplicationImpl _app; - private ExpRun _run; - - @Override - public ModelAndView getView(ExpObjectForm form, BindException errors) - { - _app = ExperimentServiceImpl.get().getExpProtocolApplication(form.getRowId()); - if (_app == null) - { - throw new NotFoundException("Could not find Protocol Application"); - } - _run = _app.getRun(); - if (_run == null) - { - throw new NotFoundException("No experiment run associated with Protocol Application"); - } - ensureCorrectContainer(getContainer(), _app, getViewContext()); - - ExpProtocol protocol = _app.getProtocol(); - - DataRegion dr = new DataRegion(); - dr.addColumns(ExperimentServiceImpl.get().getTinfoProtocolApplication().getUserEditableColumns()); - DetailsView detailsView = new DetailsView(dr, form.getRowId()); - dr.removeColumns("RunId", "ProtocolLSID", "RowId", "LSID"); - dr.addDisplayColumn(new ExperimentRunDisplayColumn(_run)); - dr.addDisplayColumn(new ProtocolDisplayColumn(protocol)); - dr.addDisplayColumn(new LineageGraphDisplayColumn(_app, _run)); - detailsView.setTitle("Protocol Application"); - - Container c = getContainer(); - ApplicationOutputGrid outMGrid = new ApplicationOutputGrid(c, _app.getRowId(), ExperimentServiceImpl.get().getTinfoMaterial()); - ApplicationOutputGrid outDGrid = new ApplicationOutputGrid(c, _app.getRowId(), ExperimentServiceImpl.get().getTinfoData()); - Map map = new HashMap<>(); - for (ProtocolApplicationParameter param : ExperimentService.get().getProtocolApplicationParameters(_app.getRowId())) - { - map.put(param.getOntologyEntryURI(), param); - } - - JspView> paramsView = new JspView<>("/org/labkey/experiment/Parameters.jsp", map); - paramsView.setTitle("Protocol Application Parameters"); - CustomPropertiesView cpv = new CustomPropertiesView(_app.getLSID(), c); - return new VBox(new StandardAndCustomPropertiesView(detailsView, cpv), paramsView, outMGrid, outDGrid); - } - - @Override - public void addNavTrail(NavTree root) - { - addRootNavTrail(root); - root.addChild("Experiment Run", ExperimentUrlsImpl.get().getRunGraphDetailURL(_run)); - root.addChild("Protocol Application " + _app.getName()); - } - } - - @RequiresPermission(ReadPermission.class) - public class ShowProtocolGridAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new ProtocolWebPart(false, getViewContext()); - } - - @Override - public void addNavTrail(NavTree root) - { - addRootNavTrail(root); - root.addChild("Protocols"); - } - } - - @RequiresPermission(ReadPermission.class) - public class ProtocolDetailsAction extends SimpleViewAction - { - private ExpProtocolImpl _protocol; - - @Override - public ModelAndView getView(ExpObjectForm form, BindException errors) - { - _protocol = ExperimentServiceImpl.get().getExpProtocol(form.getRowId()); - if (_protocol == null) - { - _protocol = ExperimentServiceImpl.get().getExpProtocol(form.getLSID()); - } - - if (_protocol == null) - { - throw new NotFoundException("Unable to find a matching protocol"); - } - ensureCorrectContainer(getContainer(), _protocol, getViewContext()); - - JspView detailsView = new JspView<>("/org/labkey/experiment/ProtocolDetails.jsp", _protocol); - detailsView.setTitle("Standard Properties"); - - CustomPropertiesView cpv = new CustomPropertiesView(_protocol.getLSID(), getContainer()); - ProtocolParametersView parametersView = new ProtocolParametersView(_protocol); - - VBox protocolDetails = new VBox(); - protocolDetails.setFrame(WebPartView.FrameType.PORTAL); - protocolDetails.setTitle("Protocol Details"); - protocolDetails.addView(new ProtocolInputOutputsView(_protocol, errors)); - - JspView stepsView = new JspView<>("/org/labkey/experiment/ProtocolSteps.jsp", _protocol); - stepsView.setTitle("Protocol Steps"); - stepsView.setFrame(WebPartView.FrameType.TITLE); - protocolDetails.addView(stepsView); - - ExpSchema schema = new ExpSchema(getUser(), getContainer()); - ExperimentRunListView runView = new ExperimentRunListView(schema, ExperimentRunListView.getRunListQuerySettings(schema, getViewContext(), ExpSchema.TableType.Runs.name(), true), ExperimentRunType.ALL_RUNS_TYPE) - { - @Override - public DataView createDataView() - { - DataView result = super.createDataView(); - result.getRenderContext().setBaseFilter(new SimpleFilter(FieldKey.fromParts("Protocol", "LSID"), _protocol.getLSID())); - return result; - } - }; - - runView.setTitle("Runs Using This Protocol"); - - return new VBox(new StandardAndCustomPropertiesView(detailsView, cpv), parametersView, protocolDetails, runView); - } - - @Override - public void addNavTrail(NavTree root) - { - addRootNavTrail(root); - root.addChild("Protocols", ExperimentUrlsImpl.get().getProtocolGridURL(getContainer())); - root.addChild("Protocol: " + _protocol.getName()); - } - } - - public class ProtocolInputOutputsView extends VBox - { - ProtocolInputOutputsView(ExpProtocol protocol, Errors errors) - { - HBox inputsView = new HBox(); - addView(inputsView); - - HBox outputsView = new HBox(); - addView(outputsView); - - UserSchema expSchema = QueryService.get().getUserSchema(getUser(), getContainer(), ExpSchema.SCHEMA_NAME); - - class ProtocolInputGrid extends QueryView - { - public ProtocolInputGrid(String title, QuerySettings settings, @Nullable Errors errors) - { - super(expSchema, settings, errors); - - setFrame(FrameType.TITLE); - setTitle(title); - setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); - setShowBorders(true); - setShadeAlternatingRows(true); - setShowExportButtons(false); - setShowPagination(false); - disableContainerFilterSelection(); - } - } - - // INPUTS - - QuerySettings materialInputsSettings = expSchema.getSettings("mpi", ExpSchema.TableType.MaterialProtocolInputs.toString()); - materialInputsSettings.getBaseFilter().addCondition(FieldKey.fromParts(ExpMaterialProtocolInputTable.Column.Protocol.toString()), protocol.getRowId()); - materialInputsSettings.setFieldKeys(Arrays.asList( - FieldKey.fromParts(ExpMaterialProtocolInputTable.Column.Name.toString()), - FieldKey.fromParts(ExpMaterialProtocolInputTable.Column.SampleSet.toString()) - )); - QueryView materialInputsView = new ProtocolInputGrid("Material Inputs", materialInputsSettings, errors); - inputsView.addView(materialInputsView); - - QuerySettings dataInputsSettings = expSchema.getSettings("dpi", ExpSchema.TableType.DataProtocolInputs.toString()); - dataInputsSettings.getBaseFilter().addCondition(FieldKey.fromParts(ExpDataProtocolInputTable.Column.Protocol.toString()), protocol.getRowId()); - dataInputsSettings.setFieldKeys(Arrays.asList( - FieldKey.fromParts(ExpDataProtocolInputTable.Column.Name.toString()), - FieldKey.fromParts(ExpDataProtocolInputTable.Column.DataClass.toString()) - )); - QueryView dataInputsView = new ProtocolInputGrid("Data Inputs", dataInputsSettings, errors); - inputsView.addView(dataInputsView); - - // OUTPUTS - - QuerySettings materialOutputsSettings = expSchema.getSettings("mpo", ExpSchema.TableType.MaterialProtocolInputs.toString()); - materialOutputsSettings.getBaseFilter().addCondition(FieldKey.fromParts(ExpMaterialProtocolInputTable.Column.Protocol.toString()), protocol.getRowId()); - materialOutputsSettings.setFieldKeys(Arrays.asList( - FieldKey.fromParts(ExpMaterialProtocolInputTable.Column.Name.toString()), - FieldKey.fromParts(ExpMaterialProtocolInputTable.Column.SampleSet.toString()) - )); - QueryView materialOutputsView = new ProtocolInputGrid("Material Outputs", materialOutputsSettings, errors); - outputsView.addView(materialOutputsView); - - QuerySettings dataOutputsSettings = expSchema.getSettings("dpo", ExpSchema.TableType.DataProtocolInputs.toString()); - dataOutputsSettings.getBaseFilter().addCondition(FieldKey.fromParts(ExpDataProtocolInputTable.Column.Protocol.toString()), protocol.getRowId()); - dataOutputsSettings.setFieldKeys(Arrays.asList( - FieldKey.fromParts(ExpDataProtocolInputTable.Column.Name.toString()), - FieldKey.fromParts(ExpDataProtocolInputTable.Column.DataClass.toString()) - )); - QueryView dataOutputsView = new ProtocolInputGrid("Data Outputs", dataOutputsSettings, errors); - outputsView.addView(dataOutputsView); - } - } - - - @RequiresPermission(ReadPermission.class) - public class ProtocolPredecessorsAction extends SimpleViewAction - { - private ExpProtocol _parentProtocol; - private ProtocolActionStepDetail _actionStep; - - @Override - public ModelAndView getView(Object o, BindException errors) - { - ActionURL url = getViewContext().getActionURL(); - - String parentProtocolLSID = url.getParameter("ParentLSID"); - int actionSequence; - try - { - actionSequence = Integer.parseInt(url.getParameter("Sequence")); - } - catch (NumberFormatException e) - { - throw new NotFoundException("Could not find SequenceId " + url.getParameter("Sequence")); - } - - _parentProtocol = ExperimentService.get().getExpProtocol(parentProtocolLSID); - if (_parentProtocol == null) - { - throw new NotFoundException("Unable to find a matching protocol"); - } - - ensureCorrectContainer(getContainer(), _parentProtocol, getViewContext()); - - _actionStep = ExperimentServiceImpl.get().getProtocolActionStepDetail(parentProtocolLSID, actionSequence); - - if (_actionStep == null) - { - throw new NotFoundException("Unable to find a matching protocol action step"); - } - - ExpProtocol childProtocol = ExperimentService.get().getExpProtocol(_actionStep.getChildProtocolLSID()); - - JspView detailsView = new JspView<>("/org/labkey/experiment/ProtocolDetails.jsp", childProtocol); - detailsView.setTitle("Standard Properties"); - - CustomPropertiesView cpv = new CustomPropertiesView(childProtocol.getLSID(), getContainer()); - - ProtocolParametersView parametersView = new ProtocolParametersView(childProtocol); - - VBox protocolDetails = new VBox(); - protocolDetails.setFrame(WebPartView.FrameType.PORTAL); - protocolDetails.setTitle("Protocol Details"); - protocolDetails.addView(new ProtocolInputOutputsView(childProtocol, errors)); - protocolDetails.addView(new ProtocolSuccessorPredecessorView(parentProtocolLSID, actionSequence, getContainer(), "PredecessorChildLSID", "PredecessorSequence", "ActionSequence", "Protocol Predecessors")); - protocolDetails.addView(new ProtocolSuccessorPredecessorView(parentProtocolLSID, actionSequence, getContainer(), "ChildProtocolLSID", "ActionSequence", "PredecessorSequence", "Protocol Successors")); - - return new VBox(new StandardAndCustomPropertiesView(detailsView, cpv), parametersView, protocolDetails); - } - - @Override - public void addNavTrail(NavTree root) - { - addRootNavTrail(root); - root.addChild("Protocols", ExperimentUrlsImpl.get().getProtocolGridURL(getContainer())); - root.addChild("Parent Protocol '" + _parentProtocol.getName() + "'", ExperimentUrlsImpl.get().getProtocolDetailsURL(_parentProtocol)); - root.addChild("Protocol Step: " + _actionStep.getName()); - } - } - - public static class DataForm - { - private boolean _inline; - private long _rowId; - private String _lsid; - private Integer _maxDimension; - private String _format; - - public boolean isInline() - { - return _inline; - } - - public void setInline(boolean inline) - { - _inline = inline; - } - - public long getRowId() - { - return _rowId; - } - - public void setRowId(long rowId) - { - _rowId = rowId; - } - - public String getLsid() - { - return _lsid; - } - - public void setLsid(String lsid) - { - _lsid = lsid; - } - - public ExpDataImpl lookupData() - { - ExpDataImpl result = ExperimentServiceImpl.get().getExpData(getRowId()); - if (result == null && getLsid() != null) - { - result = ExperimentServiceImpl.get().getExpData(getLsid()); - } - return result; - } - - public Integer getMaxDimension() - { - return _maxDimension; - } - - public void setMaxDimension(Integer maxDimension) - { - _maxDimension = maxDimension; - } - - public String getFormat() - { - return _format; - } - - public void setFormat(String format) - { - _format = format; - } - } - - public static class ExpObjectForm extends QueryViewAction.QueryExportForm - { - private long _rowId; - private String _lsid; - - public String getLsid() - { - return _lsid; - } - - public void setLsid(String lsid) - { - _lsid = lsid; - } - - public String getLSID() - { - return getLsid(); - } - - public void setLSID(String lsid) - { - setLsid(lsid); - } - - public long getRowId() - { - return _rowId; - } - - public void setRowId(long rowId) - { - _rowId = rowId; - } - } - - @RequiresAnyOf({DeletePermission.class, SampleWorkflowDeletePermission.class}) - public class DeleteSelectedExpRunsAction extends AbstractDeleteAction - { - @Override - public void addNavTrail(NavTree root) - { - // UNDONE: Need help topic on Runs - setHelpTopic("experiment"); - super.addNavTrail(root); - } - - @Override - public ModelAndView getView(DeleteForm deleteForm, boolean reshow, BindException errors) - { - List runs = new ArrayList<>(); - - Map idToRunMap = new LongHashMap<>(); - for (long runId : deleteForm.getIds(false)) - { - ExpRun run = ExperimentService.get().getExpRun(runId); - if (run != null) - { - if (!run.canDelete(getUser())) - throw new UnauthorizedException("You do not have permission to delete " + - (ExpProtocol.isSampleWorkflowProtocol(run.getProtocol().getLSID()) ? "jobs" : "runs") - + " in " + run.getContainer()); - - runs.add(run); - idToRunMap.put(run.getRowId(), run); - } - } - - Map referencedItems = new LongHashMap<>(); - List referenceDescriptions = new ArrayList<>(); - AssayService assayService = AssayService.get(); - if (!idToRunMap.isEmpty() && assayService != null ) - { - // using the first run as a representative, since all interactions here are (I believe) using the same protocol. - ExpProtocol protocol = runs.get(0).getProtocol(); - AssayProvider provider = assayService.getProvider(protocol); - if (provider != null) - { - SchemaKey key = AssayProtocolSchema.schemaName(provider, protocol); - ExperimentService.get().getObjectReferencers() - .forEach(referencer -> { - Collection referenced = referencer.getItemsWithReferences( - idToRunMap.keySet(), - key.toString(), - "Runs" - ); - referenced.forEach(id -> referencedItems.put(id, idToRunMap.get(id))); - referenceDescriptions.add(referencer.getObjectReferenceDescription(ExpRun.class)); - } - ); - } - - } - - List> permissionDatasetRows = new ArrayList<>(); - List> noPermissionDatasetRows = new ArrayList<>(); - if (StudyPublishService.get() != null) - { - for (Dataset dataset : StudyPublishService.get().getDatasetsForAssayRuns(runs, getUser())) - { - ActionURL url = urlProvider(StudyUrls.class).getDatasetURL(dataset.getContainer(), dataset.getDatasetId()); - TableInfo t = dataset.getTableInfo(getUser()); - if (null != t && t.hasPermission(getUser(),DeletePermission.class)) - { - permissionDatasetRows.add(new Pair<>(dataset, url)); - } - else - { - noPermissionDatasetRows.add(new Pair<>(dataset, url)); - } - } - } - - return new ConfirmDeleteView( - "run", - ShowRunGraphAction.class, - runs.stream().filter(run -> !referencedItems.containsKey(run.getRowId())).toList(), - deleteForm, - Collections.emptyList(), - "dataset(s) have one or more rows which", - permissionDatasetRows, - noPermissionDatasetRows, - referencedItems.values().stream().toList(), - referenceDescriptions.stream().filter(Objects::nonNull).collect(Collectors.joining(", or "))); - } - - @Override - protected void deleteObjects(DeleteForm deleteForm) - { - ExperimentServiceImpl.get().deleteExperimentRunsByRowIds(getContainer(), getUser(), deleteForm.getUserComment(), deleteForm.getIds(false)); - } - } - - public static class DeleteRunForm - { - private int _runId; - - public int getRunId() - { - return _runId; - } - - public void setRunId(int runId) - { - _runId = runId; - } - } - - /** - * Separate delete action from the client API - */ - @RequiresAnyOf({DeletePermission.class, SampleWorkflowDeletePermission.class}) - public static class DeleteRunAction extends MutatingApiAction - { - @Override - public ApiResponse execute(DeleteRunForm form, BindException errors) - { - ExpRun run = ExperimentService.get().getExpRun(form.getRunId()); - if (run == null) - { - throw new NotFoundException("Could not find run with ID " + form.getRunId()); - } - if (!run.canDelete(getUser())) - throw new UnauthorizedException("You do not have permission to delete " - + (ExpProtocol.isSampleWorkflowProtocol(run.getProtocol().getLSID()) ? "jobs" : "runs") + " in this container."); - - run.delete(getUser()); - return new ApiSimpleResponse("success", true); - } - } - - - @RequiresAnyOf({DeletePermission.class, SampleWorkflowDeletePermission.class}) - public static class DeleteRunsAction extends AbstractDeleteAPIAction - { - @Override - protected ApiSimpleResponse deleteObjects(CascadeDeleteForm form) - { - Set runIdsToDelete = new HashSet<>(form.getIds(true)); - Set runIdsCascadeDeleted = new HashSet<>(); - - if (form.isCascade()) - { - for (long runId : runIdsToDelete) - { - ExpRun run = ExperimentService.get().getExpRun(runId); - if (run != null) - addReplacesRuns(run, runIdsCascadeDeleted); - } - - if (!runIdsCascadeDeleted.isEmpty()) - runIdsToDelete.addAll(runIdsCascadeDeleted); - } - - ExperimentService.get().deleteExperimentRunsByRowIds(getContainer(), getUser(), form.getUserComment(), runIdsToDelete); - - ApiSimpleResponse response = new ApiSimpleResponse("success", true); - response.put("runIdsDeleted", runIdsToDelete); - if (!runIdsCascadeDeleted.isEmpty()) - response.put("runIdsCascadeDeleted", runIdsCascadeDeleted); - return response; - } - - private void addReplacesRuns(ExpRun run, Set runIds) - { - for (ExpRun replacedRun : run.getReplacesRuns()) - { - runIds.add(replacedRun.getRowId()); - addReplacesRuns(replacedRun, runIds); - } - } - } - - private abstract static class AbstractDeleteAPIAction extends MutatingApiAction - { - @Override - public void validateForm(CascadeDeleteForm form, Errors errors) - { - if (form.getSingleObjectRowId() == null && form.getDataRegionSelectionKey() == null && form.getRowIds() == null) - errors.reject(ERROR_REQUIRED, "Either singleObjectRowId, dataRegionSelectionKey, or rowIds is required"); - } - - @Override - public ApiResponse execute(CascadeDeleteForm form, BindException errors) throws Exception - { - ApiSimpleResponse response; - - try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) - { - tx.addCommitTask(form::clearSelected, POSTCOMMIT); - - response = deleteObjects(form); - tx.commit(); - } - - if (null != response.get("success")) - response.put("success", !errors.hasErrors()); - - return response; - } - - protected abstract ApiSimpleResponse deleteObjects(CascadeDeleteForm form) throws Exception; - } - - public static class CascadeDeleteForm extends DeleteForm - { - private boolean _cascade; - - public boolean isCascade() - { - return _cascade; - } - - public void setCascade(boolean cascade) - { - _cascade = cascade; - } - } - - private abstract static class AbstractDeleteAction extends FormViewAction - { - @Override - public void validateCommand(DeleteForm target, Errors errors) - { - } - - @Override - public boolean handlePost(DeleteForm deleteForm, BindException errors) throws Exception - { - if (!deleteForm.isForceDelete()) - { - return false; - } - else - { - try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) - { - tx.addCommitTask(deleteForm::clearSelected, POSTCOMMIT); - - deleteObjects(deleteForm); - tx.commit(); - } - catch (BatchValidationException v) - { - v.addToErrors(errors); - } - - return !errors.hasErrors(); - } - } - - @Override - public ActionURL getSuccessURL(DeleteForm form) - { - return form.getSuccessActionURL(ExperimentUrlsImpl.get().getOverviewURL(getContainer())); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Confirm Deletion"); - } - - protected abstract void deleteObjects(DeleteForm form) throws Exception; - } - - @RequiresPermission(DesignAssayPermission.class) - public static class DeleteProtocolByRowIdsAPIAction extends AbstractDeleteAPIAction - { - @Override - protected ApiSimpleResponse deleteObjects(CascadeDeleteForm form) - { - for (ExpProtocol protocol : getProtocolsForDeletion(form)) - { - if (!protocol.getContainer().hasPermission(getUser(), DesignAssayPermission.class)) - throw new UnauthorizedException("You do not have sufficient permissions to delete this assay design."); - - protocol.delete(getUser(), form.getUserComment()); - } - - return new ApiSimpleResponse(); - } - } - - public static List getProtocolsForDeletion(DeleteForm form) - { - List protocols = new ArrayList<>(); - for (long protocolId : form.getIds(false)) - { - ExpProtocol protocol = ExperimentService.get().getExpProtocol(protocolId); - if (protocol != null) - { - protocols.add(protocol); - } - } - return protocols; - } - - @RequiresPermission(DesignAssayPermission.class) - public class DeleteProtocolByRowIdsAction extends AbstractDeleteAction - { - @Override - public void addNavTrail(NavTree root) - { - // UNDONE: Need help topic on protocols - setHelpTopic("experiment"); - super.addNavTrail(root); - } - - @Override - public ModelAndView getView(DeleteForm form, boolean reshow, BindException errors) - { - List runs = ExperimentService.get().getExpRunsForProtocolIds(false, form.getIds(false)); - List protocols = getProtocolsForDeletion(form); - String noun = "Assay Design"; - List> deleteableDatasets = new ArrayList<>(); - List> noPermissionDatasets = new ArrayList<>(); - if (AssayService.get() != null && StudyService.get() != null) - { - for (ExpProtocol protocol : protocols) - { - if (!protocol.getContainer().hasPermission(getUser(), DesignAssayPermission.class)) - throw new UnauthorizedException("You do not have sufficient permissions to delete this assay design."); - - if (AssayService.get().getProvider(protocol) == null) - { - noun = "Protocol"; - } - for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(protocol.getRowId(), Dataset.PublishSource.Assay)) - { - Pair entry = new Pair<>(dataset, urlProvider(StudyUrls.class).getDatasetURL(dataset.getContainer(), dataset.getDatasetId())); - if (dataset.canDeleteDefinition(getUser())) - { - deleteableDatasets.add(entry); - } - else - { - noPermissionDatasets.add(entry); - } - } - } - } - - return new ConfirmDeleteView(noun, ProtocolDetailsAction.class, protocols, form, runs, "Dataset", deleteableDatasets, noPermissionDatasets, Collections.emptyList(), null); - } - - @Override - protected void deleteObjects(DeleteForm form) - { - for (ExpProtocol protocol : getProtocolsForDeletion(form)) - { - protocol.delete(getUser(), form.getUserComment()); - } - } - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(ReadPermission.class) - public static class GetDataOperationConfirmationDataAction extends ReadOnlyApiAction - { - @Override - public void validateForm(DataOperationConfirmationForm form, Errors errors) - { - if (form.getDataRegionSelectionKey() == null && form.getRowIds() == null) - errors.reject(ERROR_REQUIRED, "You must provide either a set of rowIds or a dataRegionSelectionKey"); - if (form.getDataOperation() == null) - errors.reject(ERROR_REQUIRED, "An operation type must be provided."); - } - - @Override - public Object execute(DataOperationConfirmationForm form, BindException errors) - { - Collection requestIds = form.getIds(false); - ExperimentServiceImpl service = ExperimentServiceImpl.get(); - List allData = service.getExpDatas(requestIds); - - Set notAllowedIds = new HashSet<>(); - if (form.getDataOperation() == ExpDataImpl.DataOperations.Delete) - service.getObjectReferencers().forEach(referencer -> - notAllowedIds.addAll(referencer.getItemsWithReferences(requestIds, "exp.data"))); - - Map>> response = ExperimentServiceImpl.partitionRequestedOperationObjects(getUser(), requestIds, notAllowedIds, allData); - - Collection containers = new HashSet<>(); - Collection notPermittedIds = new ArrayList<>(); - Class permClass = form.getDataOperation().getPermissionClass(); - for (ExpDataImpl expData : allData) - { - Container c = expData.getContainer(); - if (c.hasPermission(getUser(), ReadPermission.class)) - containers.add(c); - if (permClass != null && !c.hasPermission(getUser(), permClass)) - notPermittedIds.add(expData.getRowId()); - } - - NameExpressionOptionService svc = NameExpressionOptionService.get(); - response.put("containers", containers.stream().map(c -> Map.of( - "id", c.getEntityId(), - "path", (Object) c.getPath(), - "permitted", permClass == null || c.hasPermission(getUser(), permClass), - "canEditName", svc.getAllowUserSpecificNamesValue(c) - )).toList()); - - response.put("notPermitted", notPermittedIds.stream().map(id -> Map.of("RowId", (Object) id)).toList()); - - return success(response); - } - } - - - public static class DataOperationConfirmationForm extends DataViewSnapshotSelectionForm - { - private ExpDataImpl.DataOperations _dataOperation; - - public ExpDataImpl.DataOperations getDataOperation() - { - return _dataOperation; - } - - public void setDataOperation(ExpDataImpl.DataOperations dataOperation) - { - _dataOperation = dataOperation; - } - - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(ReadPermission.class) - public static class GetMaterialOperationConfirmationDataAction extends ReadOnlyApiAction - { - @Override - public void validateForm(MaterialOperationConfirmationForm form, Errors errors) - { - if (form.getDataRegionSelectionKey() == null && form.getRowIds() == null) - errors.reject(ERROR_REQUIRED, "You must provide either a set of rowIds or a dataRegionSelectionKey."); - if (form.getSampleOperation() == null) - errors.reject(ERROR_REQUIRED, "An operation type must be provided."); - } - - @Override - public Object execute(MaterialOperationConfirmationForm form, BindException errors) - { - Set requestIds = form.getIds(false); - ExperimentServiceImpl service = ExperimentServiceImpl.get(); - List allMaterials = service.getExpMaterials(requestIds); - - Set notAllowedIds = new HashSet<>(); - // We prevent deletion if a sample is used as a parent, has assay data, is used in a job, etc. - if (form.getSampleOperation() == SampleTypeService.SampleOperations.Delete) - service.getObjectReferencers().forEach(referencer -> - notAllowedIds.addAll(referencer.getItemsWithReferences(requestIds, "samples"))); - - if (SampleStatusService.get().supportsSampleStatus()) - notAllowedIds.addAll(service.findIdsNotPermittedForOperation(allMaterials, form.getSampleOperation())); - - Map>> response = ExperimentServiceImpl.partitionRequestedOperationObjects(getUser(), requestIds, notAllowedIds, allMaterials); - - Collection containers = new HashSet<>(); - Collection notPermittedIds = new ArrayList<>(); - Class permClass = form.getSampleOperation().getPermissionClass(); - for (ExpMaterial material : allMaterials) - { - Container c = material.getContainer(); - if (c.hasPermission(getUser(), ReadPermission.class)) - containers.add(c); - if (permClass != null && !c.hasPermission(getUser(), permClass)) - notPermittedIds.add(material.getRowId()); - } - - NameExpressionOptionService svc = NameExpressionOptionService.get(); - - response.put("containers", containers.stream().map(c -> Map.of( - "id", c.getEntityId(), - "path", (Object) c.getPath(), - "permitted", permClass == null || c.hasPermission(getUser(), permClass), - "canEditName", svc.getAllowUserSpecificNamesValue(c) - )).toList()); - - response.put("notPermitted", notPermittedIds.stream().map(id -> Map.of("RowId", (Object) id)).toList()); - - if (form.getSampleOperation() == SampleTypeService.SampleOperations.Delete) - // String 'associatedDatasets' must be synced to its handling in confirmDelete.js, confirmDelete() - response.put("associatedDatasets", ExperimentServiceImpl.includeLinkedToStudyText(allMaterials, requestIds, getUser(), getContainer())); - - return success(response); - } - } - - public static class MaterialOperationConfirmationForm extends DataViewSnapshotSelectionForm - { - private SampleTypeService.SampleOperations _sampleOperation; - - public SampleTypeService.SampleOperations getSampleOperation() - { - return _sampleOperation; - } - - public void setSampleOperation(SampleTypeService.SampleOperations sampleOperation) - { - _sampleOperation = sampleOperation; - } - } - - @RequiresPermission(DeletePermission.class) - public class DeleteSelectedDataAction extends AbstractDeleteAction - { - @Override - public void addNavTrail(NavTree root) - { - // UNDONE: Need help topic on Datas - setHelpTopic("experiment"); - super.addNavTrail(root); - } - - @Override - protected void deleteObjects(DeleteForm deleteForm) throws Exception - { - List datas = getDatas(deleteForm, false); - - for (ExpRun run : getRuns(datas)) - { - if (!run.getContainer().hasPermission(getUser(), DeletePermission.class)) - throw new UnauthorizedException(); - } - - // Issue 32076: Delete the exp.Data objects using QueryUpdateService so trigger scripts will be executed - Map, List> byDataClass = datas.stream().collect(Collectors.groupingBy(d -> Optional.ofNullable(d.getDataClass(null)))); - for (Optional opt : byDataClass.keySet()) - { - SchemaKey schemaKey; - String queryName; - ExpDataClass dc = opt.orElse(null); - List ds = byDataClass.get(opt); - if (dc == null) - { - // Reference to exp.Data table - schemaKey = ExpSchema.SCHEMA_EXP; - queryName = ExpSchema.TableType.Data.name(); - } - else - { - // Reference to exp.data. table - schemaKey = ExpSchema.SCHEMA_EXP_DATA; - queryName = dc.getName(); - } - - UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaKey); - if (schema == null) - throw new IllegalStateException("Failed to get schema '" + schemaKey + "'"); - - TableInfo table = schema.getTable(queryName); - if (table == null) - throw new IllegalStateException("Failed to get table '" + queryName + "' in schema '" + schemaKey + "'"); - - QueryUpdateService qus = table.getUpdateService(); - if (qus == null) - throw new IllegalStateException(); - - qus.deleteRows(getUser(), getContainer(), toKeys(ds), null, null); - } - } - - protected List> toKeys(List datas) - { - return datas.stream().map(d -> CaseInsensitiveHashMap.of("rowId", d.getRowId())).collect(toList()); - } - - @Override - public ModelAndView getView(DeleteForm deleteForm, boolean reshow, BindException errors) - { - if (errors.hasErrors()) - return new SimpleErrorView(errors, false); - - List datas = getDatas(deleteForm, false); - List runs = getRuns(datas); - - return new ConfirmDeleteView("Data", ShowDataAction.class, datas, deleteForm, runs); - } - - private List getRuns(List datas) - { - List runArray = ExperimentService.get().getRunsUsingDatas(datas); - return new ArrayList<>(ExperimentService.get().runsDeletedWithInput(runArray)); - } - - private List getDatas(DeleteForm deleteForm, boolean clear) - { - List datas = new ArrayList<>(); - for (long dataId : deleteForm.getIds(clear)) - { - ExpData data = ExperimentService.get().getExpData(dataId); - if (data != null) - { - datas.add(data); - } - } - return datas; - } - } - - @RequiresPermission(DeletePermission.class) - public class DeleteSelectedExperimentsAction extends AbstractDeleteAction - { - @Override - protected void deleteObjects(DeleteForm deleteForm) - { - for (ExpExperiment exp : lookupExperiments(deleteForm)) - { - exp.delete(getUser()); - } - } - - @Override - public ModelAndView getView(DeleteForm deleteForm, boolean reshow, BindException errors) - { - List experiments = lookupExperiments(deleteForm); - - List runs = new ArrayList<>(); - boolean allBatches = true; - for (ExpExperiment experiment : experiments) - { - // Deleting a batch also deletes all of its runs - if (experiment.getBatchProtocol() != null) - { - runs.addAll(experiment.getRuns()); - } - else - { - allBatches = false; - } - } - - return new ConfirmDeleteView(allBatches ? "batch" : "run group", DetailsAction.class, experiments, deleteForm, runs); - } - - private List lookupExperiments(DeleteForm deleteForm) - { - List experiments = new ArrayList<>(); - for (long experimentId : deleteForm.getIds(false)) - { - ExpExperiment experiment = ExperimentService.get().getExpExperiment(experimentId); - if (experiment != null) - { - experiments.add(experiment); - } - } - return experiments; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("runGroups"); - super.addNavTrail(root); - } - } - - @RequiresPermission(DesignSampleTypePermission.class) - public class DeleteSampleTypesAction extends AbstractDeleteAction - { - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("sampleSets"); - super.addNavTrail(root); - } - - @Override - protected void deleteObjects(DeleteForm deleteForm) - { - List sampleTypes = getSampleTypes(deleteForm); - if (sampleTypes.isEmpty()) - { - throw new NotFoundException("No sample types found for ids provided."); - } - if (!ensureCorrectContainer(sampleTypes)) - { - throw new UnauthorizedException(); - } - - for (ExpRun run : getRuns(sampleTypes)) - { - if (!run.getContainer().hasPermission(getUser(), DeletePermission.class)) - { - throw new UnauthorizedException(); - } - } - - for (ExpSampleType source : sampleTypes) - { - Domain domain = source.getDomain(); - if (!domain.getDomainKind().canDeleteDefinition(getUser(), domain)) - { - throw new UnauthorizedException(); - } - - source.delete(getUser(), deleteForm.getUserComment()); - } - } - - @Override - public ModelAndView getView(DeleteForm deleteForm, boolean reshow, BindException errors) - { - List sampleTypes = getSampleTypes(deleteForm); - if (!ensureCorrectContainer(sampleTypes)) - { - throw new RedirectException(ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer(), "To delete a sample type, you must be in its folder or project.")); - } - - List> deleteableDatasets = new ArrayList<>(); - List> noPermissionDatasets = new ArrayList<>(); - if (StudyService.get() != null && StudyPublishService.get() != null) - { - for (ExpSampleType sampleType: sampleTypes) - { - for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(sampleType.getRowId(), Dataset.PublishSource.SampleType)) - { - ActionURL datasetURL = StudyService.get().getDatasetURL(getContainer(), dataset.getDatasetId()); - Pair entry = new Pair<>(dataset, datasetURL); - if (dataset.canDeleteDefinition(getUser())) - { - deleteableDatasets.add(entry); - } - else - { - noPermissionDatasets.add(entry); - } - } - } - } - return new ConfirmDeleteView("Sample Type", ShowSampleTypeAction.class, sampleTypes, deleteForm, getRuns(sampleTypes), "Dataset", deleteableDatasets, noPermissionDatasets, Collections.emptyList(), null); - } - - private List getSampleTypes(DeleteForm deleteForm) - { - List sources = new ArrayList<>(); - for (long rowId : deleteForm.getIds(false)) - { - ExpSampleType sampleType = SampleTypeService.get().getSampleType(getContainer(), getUser(), rowId); - if (sampleType != null) - { - sources.add(sampleType); - } - } - return sources; - } - - private boolean ensureCorrectContainer(List sampleTypes) - { - for (ExpSampleType source : sampleTypes) - { - Container sourceContainer = source.getContainer(); - if (!sourceContainer.equals(getContainer())) - { - return false; - } - } - return true; - } - - private List getRuns(List sampleTypes) - { - if (!sampleTypes.isEmpty()) - { - List runArray = ExperimentService.get().getRunsUsingSampleTypes(sampleTypes.toArray(new ExpSampleType[0])); - return ExperimentService.get().runsDeletedWithInput(runArray); - } - else - { - return Collections.emptyList(); - } - } - } - - private DataRegion getSampleTypeRegion(ViewContext model) - { - TableInfo tableInfo = ExperimentServiceImpl.get().getTinfoSampleType(); - - QuerySettings settings = new QuerySettings(model, "SampleType"); - settings.setSelectionKey(DataRegionSelection.getSelectionKey(tableInfo.getSchema().getName(), tableInfo.getName(), "SampleType", settings.getDataRegionName())); - - DataRegion dr = new DataRegion(); - dr.setSettings(settings); - dr.addColumns(tableInfo.getUserEditableColumns()); - dr.removeColumns("lastindexed"); - dr.getDisplayColumn(0).setVisible(false); - - dr.getDisplayColumn("idcol1").setVisible(false); - dr.getDisplayColumn("idcol2").setVisible(false); - dr.getDisplayColumn("idcol3").setVisible(false); - dr.getDisplayColumn("lsid").setVisible(false); - dr.getDisplayColumn("materiallsidprefix").setVisible(false); - dr.getDisplayColumn("parentcol").setVisible(false); - - ActionURL url = new ActionURL(ShowSampleTypeAction.class, model.getContainer()); - dr.getDisplayColumn(1).setURL(url.addParameter("rowId", "${RowId}")); - dr.setShowRecordSelectors(getContainer().hasOneOf(getUser(), DeletePermission.class, UpdatePermission.class)); - - return dr; - } - - @RequiresPermission(ReadPermission.class) - @ActionNames("getSampleType,getSampleTypeApi") // Referenced in labkey-ui-components components/samples/actions.ts TODO: migrate getSampleTypeApi -> getSampleType - public static class GetSampleTypeAction extends ReadOnlyApiAction - { - @Override - public void validateForm(SampleTypeForm form, Errors errors) - { - if (form.getRowId() == null && form.getLSID() == null) - errors.reject(ERROR_REQUIRED, "RowId or LSID must be provided"); - } - - @Override - public Object execute(SampleTypeForm form, BindException errors) throws Exception - { - ExpSampleTypeImpl st = form.getSampleType(getContainer()); - - return getSampleTypeResponse(st); - } - } - - @NotNull - private static ApiSimpleResponse getSampleTypeResponse(ExpSampleType st) throws IOException - { - Map sampleType = new HashMap<>(); - sampleType.put("name", st.getName()); - sampleType.put("nameExpression", st.getNameExpression()); - sampleType.put("labelColor", st.getLabelColor()); - sampleType.put("metricUnit", st.getMetricUnit()); - sampleType.put("description", st.getDescription()); - sampleType.put("importAliases", st.getImportAliasMap()); - sampleType.put("lsid", st.getLSID()); - sampleType.put("rowId", st.getRowId()); - sampleType.put("domainId", st.getDomain().getTypeId()); - sampleType.put("category", st.getCategory()); - - return new ApiSimpleResponse(Map.of("sampleSet", sampleType, "success", true)); - } - - public static class DataTypesWithRequiredLineageForm - { - private Integer _parentDataTypeRowId; - private boolean _sampleParent; - - public Integer getParentDataTypeRowId() - { - return _parentDataTypeRowId; - } - - public void setParentDataTypeRowId(Integer parentDataTypeRowId) - { - this._parentDataTypeRowId = parentDataTypeRowId; - } - - public boolean isSampleParent() - { - return _sampleParent; - } - - public void setSampleParent(boolean sampleParent) - { - _sampleParent = sampleParent; - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetDataTypesWithRequiredLineageAction extends ReadOnlyApiAction - { - @Override - public void validateForm(DataTypesWithRequiredLineageForm form, Errors errors) - { - if (form.getParentDataTypeRowId() == null) - errors.reject(ERROR_REQUIRED, "ParentDataTypeRowId must be provided"); - } - - @Override - public Object execute(DataTypesWithRequiredLineageForm form, BindException errors) throws Exception - { - return getDataTypesWithRequiredLineageResponse(form.getParentDataTypeRowId(), form.isSampleParent(), getContainer(), getUser()); - } - } - @NotNull - private static ApiSimpleResponse getDataTypesWithRequiredLineageResponse(Integer parentDataType, boolean isSampleParent, Container container, User user) - { - Pair, Set> requiredLineages = ExperimentServiceImpl.get().getDataTypesWithRequiredLineage(parentDataType, isSampleParent, container, user); - return new ApiSimpleResponse(Map.of("sampleTypes", requiredLineages.first, "dataClasses", requiredLineages.second,"success", true)); - } - - @RequiresPermission(DesignSampleTypePermission.class) - public static class EditSampleTypeAction extends SimpleViewAction - { - private ExpSampleTypeImpl _sampleType; - - @Override - public ModelAndView getView(SampleTypeForm form, BindException errors) - { - boolean create = form.getLSID() == null && form.getRowId() == null; - if (!create) - _sampleType = form.getSampleType(getContainer()); - - return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("sampleTypeDesigner")); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("sampleSets"); - - root.addChild("Sample Types", ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer())); - if (_sampleType == null) - { - root.addChild("Create Sample Type"); - } - else - { - root.addChild(_sampleType.getName(), ExperimentUrlsImpl.get().getShowSampleTypeURL(_sampleType)); - root.addChild("Update Sample Type"); - } - } - } - - public static class SampleTypeForm extends ReturnUrlForm - { - private Integer rowId; - private String lsid; - - public Integer getRowId() - { - return rowId; - } - - public void setRowId(Integer rowId) - { - this.rowId = rowId; - } - - public String getLSID() - { - return this.lsid; - } - - public void setLSID(String lsid) - { - this.lsid = lsid; - } - - public ExpSampleTypeImpl getSampleType(Container container) throws NotFoundException - { - ExpSampleTypeImpl sampleType = SampleTypeServiceImpl.get().getSampleType(getLSID()); - if (sampleType == null) - sampleType = SampleTypeServiceImpl.get().getSampleType(getRowId()); - - if (sampleType == null) - { - throw new NotFoundException("Sample type not found: " + (getLSID() != null ? getLSID() : getRowId())); - } - - if (!container.equals(sampleType.getContainer())) - { - throw new NotFoundException("Sample type is not defined in the given container."); - } - - return sampleType; - } - } - - @RequiresPermission(InsertPermission.class) - public static class ImportSamplesAction extends AbstractExpDataImportAction - { - @Override - public void validateForm(QueryForm queryForm, Errors errors) - { - _form = queryForm; - _insertOption = queryForm.getInsertOption(); - boolean crossTypeImport = getOptionParamValue(Params.crossTypeImport); - _form.setSchemaName(getTargetSchemaName()); - if (crossTypeImport) - { - _form.setQueryName(getPipelineTargetQueryName()); - } - super.validateForm(queryForm, errors); - if (queryForm.getQueryName() == null) - errors.reject(ERROR_REQUIRED, "Sample type name is required"); - else - { - if (!crossTypeImport) - { - ExpSampleTypeImpl sampleType = SampleTypeServiceImpl.get().getSampleType(getContainer(), getUser(), queryForm.getQueryName()); - if (sampleType == null) - { - errors.reject(ERROR_GENERIC, "Sample type '" + queryForm.getQueryName() + " not found."); - } - } - } - } - - private String getTargetSchemaName() - { - return getOptionParamValue(Params.crossTypeImport) ? ExpSchema.SCHEMA_NAME : "samples"; - } - - @Override - protected UserSchema getTargetSchema() - { - return getOptionParamValue(Params.crossTypeImport) ? QueryService.get().getUserSchema(getUser(), getContainer(), getTargetSchemaName()) : super.getTargetSchema(); - } - - @Override - protected String getPipelineTargetQueryName() - { - return getOptionParamValue(Params.crossTypeImport) ? "materials" : super.getPipelineTargetQueryName(); - } - - @Override - protected Map getRenamedColumns() - { - Map renamedColumns = super.getRenamedColumns(); - renamedColumns.putAll(SampleTypeUpdateServiceDI.SAMPLE_ALT_IMPORT_NAME_COLS); - return renamedColumns; - } - - @Override - protected @Nullable Set getLineageImportAliases() throws IOException - { - Set aliases = new CaseInsensitiveHashSet(); - // Issue 53419: Aliquot parent with number like names that starts with leading zeroes aren't resolved during import - aliases.add(ExpMaterial.ALIQUOTED_FROM_INPUT); - boolean crossTypeImport = getOptionParamValue(AbstractQueryImportAction.Params.crossTypeImport); - // Issue 51894: We need to stop conversion to numbers for alias fields for all type - // If there are aliases defined for one type that are number fields in another type, this will prevent - // conversion to numbers during the initial partitioning, but the conversion will happen when the partition - // file is loaded. - if (crossTypeImport) - { - List sampleTypes = SampleTypeServiceImpl.get().getSampleTypes(getContainer(), getUser(), true); - for (ExpSampleTypeImpl sampleType : sampleTypes) - aliases.addAll(sampleType.getImportAliases().keySet()); - } - else - { - ExpSampleTypeImpl sampleType = SampleTypeServiceImpl.get().getSampleType(getContainer(), getUser(), _form.getQueryName()); - aliases.addAll(sampleType.getImportAliases().keySet()); - } - return aliases; - } - - @Override - protected int importData( - DataLoader dl, - FileStream file, - String originalName, - BatchValidationException errors, - @Nullable AuditBehaviorType auditBehaviorType, - TransactionAuditProvider.@Nullable TransactionAuditEvent auditEvent, - @Nullable String auditUserComment - ) throws IOException - { - initContext(dl, errors, auditBehaviorType, auditUserComment); - - TableInfo tInfo = _target; - QueryUpdateService updateService = _updateService; - if (getOptionParamValue(Params.crossTypeImport)) - { - tInfo = ExperimentService.get().createMaterialTable(new SamplesSchema(getUser(), getContainer()), ContainerFilter.current(this), null); - updateService = tInfo.getUpdateService(); - } - - int count = importData(dl, tInfo, updateService, _context, auditEvent, getUser(), getContainer()); - - if (getOptionParamValue(Params.crossTypeImport)) - { - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "sampleImport", "crossTypeImport"); - if (_context.getInsertOption() == QueryUpdateService.InsertOption.UPDATE) - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "sampleImport", "crossTypeUpdate"); - else if (_context.getInsertOption() == QueryUpdateService.InsertOption.MERGE) - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "sampleImport", "crossTypeMerge"); - } - - return count; - } - - @Override - public ModelAndView getView(QueryForm form, BindException errors) throws Exception - { - initRequest(form); - setHelpTopic("importSampleSets"); // page-wide help topic - setImportHelpTopic("importSampleSets"); // importOptions help topic - setTypeName("samples"); - return getDefaultImportView(form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Sample Types", ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer())); - ActionURL url = _form.urlFor(QueryAction.executeQuery); - if (_form.getQueryName() != null && url != null) - root.addChild(_form.getQueryName(), url); - root.addChild("Import Data"); - } - - @Override - protected JSONObject createSuccessResponse(int rowCount) - { - JSONObject json = super.createSuccessResponse(rowCount); - if (!_context.getResponseInfo().isEmpty()) - { - for (String key : _context.getResponseInfo().keySet()) - json.put(key, _context.getResponseInfo().get(key)); - } - return json; - } - - @Override - protected void configureLoader(DataLoader loader) throws IOException - { - if (getOptionParamValue(Params.crossTypeImport)) - loader.setInferTypes(false); - configureLoader(loader, _target, getRenamedColumns(), allowLineageColumns(), getLineageImportAliases()); - } - } - - public abstract static class AbstractExpDataImportAction extends AbstractQueryImportAction - { - protected QueryForm _form; - protected DataIteratorContext _context; - - @Override - public void validateForm(QueryForm form, Errors errors) - { - QueryDefinition query = form.getQueryDef(); - if (query.getContainerFilter() != null && query.getContainerFilter().getType() != null) - { - // cross folder import not supported - if (query.getContainerFilter().getType() != ContainerFilter.Type.Current) - errors.reject(ERROR_GENERIC, "ContainerFilter is not supported for import actions."); - } - } - - @Override - protected void initRequest(QueryForm form) throws ServletException - { - QueryDefinition query = form.getQueryDef(); - setContainerFilterForImport(query, getContainer(), getUser()); - List qpe = new ArrayList<>(); - TableInfo t = query.getTable(form.getSchema(), qpe, true); - - if (!qpe.isEmpty()) - throw qpe.get(0); - if (!getOptionParamValue(Params.crossTypeImport) && null != t) - { - setTarget(t); - setShowMergeOption(t.supportsInsertOption(QueryUpdateService.InsertOption.MERGE)); - setShowUpdateOption(t.supportsInsertOption(QueryUpdateService.InsertOption.UPDATE)); - } - - _auditBehaviorType = form.getAuditBehavior(); - _auditUserComment = form.getAuditUserComment(); - } - - @Override - protected Map getRenamedColumns() - { - final String renameParamPrefix = "importAlias."; - Map renameColumns = new CaseInsensitiveHashMap<>(); - PropertyValue[] pvs = _form.getInitParameters().getPropertyValues(); - for (PropertyValue pv : pvs) - { - String paramName = pv.getName(); - if (!paramName.startsWith(renameParamPrefix) || pv.getValue() == null) - continue; - - renameColumns.put(paramName.substring(renameParamPrefix.length()), (String) pv.getValue()); - } - return renameColumns; - } - - @Override - protected Set getLineageImportAliases() throws IOException - { - ExpDataClass dataClass = ExperimentServiceImpl.get().getDataClass(getContainer(), getUser(), _form.getQueryName()); - return new CaseInsensitiveHashSet(dataClass.getImportAliases().keySet()); - } - - protected void initContext(DataLoader dl, BatchValidationException errors, @Nullable AuditBehaviorType auditBehaviorType, @Nullable String auditUserComment) - { - _context = createDataIteratorContext(_insertOption, getOptionParamsMap(), getLookupResolutionType(), auditBehaviorType, auditUserComment, errors, null, getContainer()); - - if (_context.isCrossFolderImport() && !getContainer().hasProductFolders()) - _context.setCrossFolderImport(false); - } - - @Override - protected int importData(DataLoader dl, FileStream file, String originalName, BatchValidationException errors, @Nullable AuditBehaviorType auditBehaviorType, TransactionAuditProvider.@Nullable TransactionAuditEvent auditEvent, @Nullable String auditUserComment) throws IOException - { - initContext(dl, errors, auditBehaviorType, auditUserComment); - return importData(dl, _target, _updateService, _context, auditEvent, getUser(), getContainer()); - } - - @Override - protected String getQueryImportProviderName() - { - PropertyValue pv = _form.getInitParameters().getPropertyValue(QUERY_IMPORT_PIPELINE_PROVIDER_PARAM); - return pv == null ? null : (String) pv.getValue(); - } - - @Override - protected String getQueryImportDescription() - { - PropertyValue pv = _form.getInitParameters().getPropertyValue(QUERY_IMPORT_PIPELINE_DESCRIPTION_PARAM); - return pv == null ? null : (String) pv.getValue(); - } - - @Override - protected String getQueryImportJobNotificationProviderName() - { - PropertyValue pv = _form.getInitParameters().getPropertyValue(QUERY_IMPORT_NOTIFICATION_PROVIDER_PARAM); - return pv == null ? null : (String) pv.getValue(); - } - - @Override - protected boolean isBackgroundImportSupported() - { - return true; - } - - @Override - protected boolean allowLineageColumns() - { - return true; - } - - } - - @RequiresPermission(InsertPermission.class) - public static class ImportDataAction extends AbstractExpDataImportAction - { - @Override - public void validateForm(QueryForm queryForm, Errors errors) - { - _form = queryForm; - _form.setSchemaName("exp.data"); - _insertOption = queryForm.getInsertOption(); - super.validateForm(queryForm, errors); - if (queryForm.getQueryName() == null) - errors.reject(ERROR_REQUIRED, "Data class name is required"); - else - { - ExpDataClass dataClass = ExperimentService.get().getDataClass(getContainer(), getUser(), queryForm.getQueryName()); - if (dataClass == null) - { - errors.reject(ERROR_GENERIC, "Data class '" + queryForm.getQueryName() + " not found."); - } - } - } - - @Override - public ModelAndView getView(QueryForm form, BindException errors) throws Exception - { - initRequest(form); - setHelpTopic("dataClass"); // page wide help topic - setImportHelpTopic("dataClass#ui"); // importOptions help topic - setTypeName("data"); - return getDefaultImportView(form, errors); - } - - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Data Classes", ExperimentUrlsImpl.get().getDataClassListURL(getContainer())); - ActionURL url = _form.urlFor(QueryAction.executeQuery); - if (_form.getQueryName() != null && url != null) - root.addChild(_form.getQueryName(), url); - root.addChild("Import Data"); - } - - @Override - protected void configureLoader(DataLoader loader) throws IOException - { - configureLoader(loader, _target, getRenamedColumns(), allowLineageColumns(), getLineageImportAliases()); - } - - } - - @RequiresPermission(UpdatePermission.class) - public class ShowUpdateAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ExperimentForm form, BindException errors) - { - form.refreshFromDb(); - Experiment exp = form.getBean(); - if (exp == null) - { - throw new NotFoundException(); - } - ensureCorrectContainer(getContainer(), ExperimentService.get().getExpExperiment(exp.getRowId()), getViewContext()); - - return new ExperimentUpdateView(new DataRegion(), form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("runGroups"); - addRootNavTrail(root); - root.addChild("Update Run Group"); - } - } - - @RequiresPermission(UpdatePermission.class) - public static class UpdateAction extends FormHandlerAction - { - private Experiment _exp; - - @Override - public void validateCommand(ExperimentForm target, Errors errors) - { - } - - @Override - public boolean handlePost(ExperimentForm form, BindException errors) throws Exception - { - form.doUpdate(); - form.refreshFromDb(); - _exp = form.getBean(); - return true; - } - - @Override - public ActionURL getSuccessURL(ExperimentForm experimentForm) - { - return ExperimentUrlsImpl.get().getExperimentDetailsURL(getContainer(), ExperimentService.get().getExpExperiment(_exp.getRowId())); - } - } - - public static class ExportBean - { - private final LSIDRelativizer _selectedRelativizer; - private final XarExportType _selectedExportType; - private final String _fileName; - private final String _dataRegionSelectionKey; - private final String _error; - private final Long _expRowId; - private final Long _protocolId; - private final ActionURL _postURL; - private final Set _roles; - - public ExportBean(LSIDRelativizer selectedRelativizer, XarExportType selectedExportType, String fileName, ExportOptionsForm form, Set roles, ActionURL postURL) - { - _selectedRelativizer = selectedRelativizer; - _selectedExportType = selectedExportType; - _fileName = fileName; - _dataRegionSelectionKey = form.getDataRegionSelectionKey(); - _error = form.getError(); - _expRowId = form.getExpRowId(); - _postURL = postURL; - _roles = roles; - _protocolId = form.getProtocolId(); - } - - public LSIDRelativizer getSelectedRelativizer() - { - return _selectedRelativizer; - } - - public XarExportType getSelectedExportType() - { - return _selectedExportType; - } - - public String getError() - { - return _error; - } - - public String getFileName() - { - return _fileName; - } - - public Set getRoles() - { - return _roles; - } - - public String getDataRegionSelectionKey() - { - return _dataRegionSelectionKey; - } - - public ActionURL getPostURL() - { - return _postURL; - } - - public Long getProtocolId() - { - return _protocolId; - } - - public Long getExpRowId() - { - return _expRowId; - } - } - - - private String fixupExportName(String runName) - { - runName = runName.replace('/', '-'); - runName = runName.replace('\\', '-'); - return runName; - } - - public static class ExportOptionsForm extends ExperimentRunListForm - { - private String _error; - private XarExportType _exportType; - private LSIDRelativizer _lsidOutputType; - private String _xarFileName; - private String _zipFileName; - private String _fileExportType; - private Long _protocolId; - private Integer _sampleTypeId; - private long[] _dataIds; - private String[] _roles = new String[0]; - - public String getError() - { - return _error; - } - - public void setError(String error) - { - _error = error; - } - - public XarExportType getExportType() - { - return _exportType; - } - - public LSIDRelativizer getLsidOutputType() - { - return _lsidOutputType; - } - - public String getFileExportType() - { - return _fileExportType; - } - - public void setFileExportType(String fileExportType) - { - _fileExportType = fileExportType; - } - - public String getXarFileName() - { - return _xarFileName; - } - - public void setXarFileName(String xarFileName) - { - _xarFileName = xarFileName; - } - - public String getZipFileName() - { - return _zipFileName; - } - - public void setZipFileName(String zipFileName) - { - _zipFileName = zipFileName; - } - - public void setExportType(XarExportType exportType) - { - _exportType = exportType; - } - - public void setLsidOutputType(LSIDRelativizer lsidOutputType) - { - _lsidOutputType = lsidOutputType; - } - - public Long getProtocolId() - { - return _protocolId; - } - - public void setProtocolId(Long protocolId) - { - _protocolId = protocolId; - } - - public String[] getRoles() - { - return _roles; - } - - public void setRoles(String[] roles) - { - _roles = roles; - } - - public Integer getSampleTypeId() - { - return _sampleTypeId; - } - - public void setSampleTypeId(Integer sampleTypeId) - { - _sampleTypeId = sampleTypeId; - } - - public long[] getDataIds() - { - return _dataIds; - } - - public void setDataIds(long[] dataIds) - { - _dataIds = dataIds; - } - - public List lookupProtocols(ViewContext context, boolean clearSelection) - { - List protocols = new ArrayList<>(); - - if (_protocolId != null) - { - ExpProtocol protocol = ExperimentService.get().getExpProtocol(_protocolId.intValue()); - if (protocol == null || !protocol.getContainer().equals(context.getContainer())) - { - throw new NotFoundException(); - } - protocols.add(protocol); - return protocols; - } - - for (Long protocolId : DataRegionSelection.getSelectedIntegers(context, clearSelection)) - { - try - { - ExpProtocol protocol = ExperimentService.get().getExpProtocol(protocolId); - if (protocol == null || !protocol.getContainer().equals(context.getContainer())) - { - throw new NotFoundException(); - } - protocols.add(protocol); - } - catch (NumberFormatException e) - { - throw new NotFoundException("Invalid protocol id: " + protocolId); - } - } - if (protocols.isEmpty()) - { - throw new NotFoundException("No protocols selected"); - } - return protocols; - } - } - - private ActionURL exportXAR(@NotNull XarExportSelection selection, @Nullable String fileName) - throws ExperimentException, IOException, PipelineValidationException - { - return exportXAR(selection, null, null, fileName); - } - - private ActionURL exportXAR(@NotNull XarExportSelection selection, @Nullable LSIDRelativizer lsidRelativizer, @Nullable XarExportType exportType, @Nullable String fileName) - throws ExperimentException, IOException, PipelineValidationException - { - if (lsidRelativizer == null) - lsidRelativizer = LSIDRelativizer.FOLDER_RELATIVE; - - if (exportType == null) - exportType = XarExportType.BROWSER_DOWNLOAD; - - if (fileName == null || fileName.isEmpty()) - fileName = "export.xar"; - - fileName = fixupExportName(fileName); - String xarXmlFileName = null; - if (StringUtils.endsWithIgnoreCase(fileName, ".xar")) - xarXmlFileName = fileName + ".xml"; - - switch (exportType) - { - case BROWSER_DOWNLOAD: - XarExporter exporter = new XarExporter(lsidRelativizer, selection, getUser(), xarXmlFileName, null, getContainer()); - - getViewContext().getResponse().setContentType("application/zip"); - ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, fileName); - ResponseHelper.setPrivate(getViewContext().getResponse()); - - exporter.writeAsArchive(getViewContext().getResponse().getOutputStream()); - return null; - case PIPELINE_FILE: - if (!PipelineService.get().hasValidPipelineRoot(getContainer())) - { - throw new IllegalStateException("You must set a valid pipeline root before you can export a XAR to it."); - } - PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(getContainer()); - XarExportPipelineJob job = new XarExportPipelineJob(getViewBackgroundInfo(), pipeRoot, fileName, lsidRelativizer, selection, xarXmlFileName); - PipelineService.get().queueJob(job); - PipelineStatusFile status = PipelineService.get().getStatusFile(job.getJobGUID()); - return PageFlowUtil.urlProvider(PipelineUrls.class).statusDetails(getContainer(), status.getRowId()); - default: - throw new IllegalArgumentException("Unknown export type: " + exportType); - } - } - - @RequiresPermission(ReadPermission.class) - public class ExportProtocolsAction extends AbstractExportAction - { - @Override - public boolean handlePost(ExportOptionsForm form, BindException errors) throws Exception - { - List protocols = form.lookupProtocols(getViewContext(), false); - - long[] ids = new long[protocols.size()]; - for (int i = 0; i < ids.length; i++) - { - ids[i] = protocols.get(i).getRowId(); - } - XarExportSelection selection = new XarExportSelection(); - selection.addProtocolIds(ids); - - exportXAR(selection, form.getLsidOutputType(), form.getExportType(), form.getXarFileName()); - - if (form.getDataRegionSelectionKey() != null) - { - // Clear the selection - form.lookupProtocols(getViewContext(), true); - } - return true; - } - } - - public abstract static class AbstractExportAction extends FormViewAction - { - protected ActionURL _resultURL; - - @Override - public void validateCommand(ExportOptionsForm target, Errors errors) - { - } - - @Override - public ActionURL getSuccessURL(ExportOptionsForm exportOptionsForm) - { - return _resultURL; - } - - @Override - public ModelAndView getSuccessView(ExportOptionsForm exportOptionsForm) - { - return null; - } - - @Override - public ModelAndView getView(ExportOptionsForm form, boolean reshow, BindException errors) throws Exception - { - // FormViewAction can reinvoke getView() in response to a POST if we're not redirecting the browser, - // so avoid double-creating the export - if ("get".equalsIgnoreCase(getViewContext().getRequest().getMethod())) - handlePost(form, errors); - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - } - - public List lookupRuns(ExportOptionsForm form) - { - Set runIds; - if (form.getRunIds() != null && form.getRunIds().length > 0) - runIds = new HashSet<>(Arrays.asList(form.getRunIds())); - else - runIds = DataRegionSelection.getSelectedIntegers(getViewContext(), form.getDataRegionSelectionKey(), false); - - if (runIds.isEmpty()) - { - throw new NotFoundException(); - } - List result = new ArrayList<>(); - - for (long id : runIds) - { - ExpRun run = ExperimentService.get().getExpRun(id); - if (run == null || !run.getContainer().hasPermission(getUser(), ReadPermission.class)) - { - throw new NotFoundException("Could not find run " + id); - } - result.add(run); - } - return result; - } - } - - @RequiresPermission(ReadPermission.class) - public class ExportRunsAction extends AbstractExportAction - { - @Override - public boolean handlePost(ExportOptionsForm form, BindException errors) throws Exception - { - XarExportSelection selection = new XarExportSelection(); - if (form.getExpRowId() != null) - { - ExpExperiment experiment = ExperimentService.get().getExpExperiment(form.getExpRowId()); - if (experiment != null && !experiment.getContainer().hasPermission(getUser(), ReadPermission.class)) - { - throw new NotFoundException("Run group " + form.getExpRowId()); - } - selection.addExperimentIds(experiment.getRowId()); - } - selection.addRuns(lookupRuns(form)); - - _resultURL = exportXAR(selection, form.getLsidOutputType(), form.getExportType(), form.getXarFileName()); - if (form.getDataRegionSelectionKey() != null) - DataRegionSelection.clearAll(getViewContext(), form.getDataRegionSelectionKey()); - return true; - } - } - - @RequiresPermission(ReadPermission.class) - public class ExportSampleTypeAction extends AbstractExportAction - { - @Override - public boolean handlePost(ExportOptionsForm form, BindException errors) throws Exception - { - Integer rowId = form.getSampleTypeId(); - if (rowId == null) - { - throw new NotFoundException("No sampleTypeId parameter specified"); - } - ExpSampleType sampleType = SampleTypeService.get().getSampleType(getContainer(), getUser(), rowId.intValue()); - if (sampleType == null) - { - throw new NotFoundException("No such sample type with RowId " + rowId); - } - if (!sampleType.getContainer().hasPermission(getUser(), ReadPermission.class)) - { - throw new UnauthorizedException(); - } - - XarExportSelection selection = new XarExportSelection(); - selection.addSampleType(sampleType); - - _resultURL = exportXAR(selection, form.getLsidOutputType(), form.getExportType(), FileUtil.makeLegalName(sampleType.getName() + ".xar")); - return true; - } - } - - @RequiresPermission(ReadPermission.class) - public class ExportRunFilesAction extends AbstractExportAction - { - @Override - public boolean handlePost(ExportOptionsForm form, BindException errors) throws Exception - { - XarExportSelection selection = new XarExportSelection(); - selection.setIncludeXarXml(false); - if ("role".equalsIgnoreCase(form.getFileExportType())) - { - selection.addRoles(form.getRoles()); - } - selection.addRuns(lookupRuns(form)); - - _resultURL = exportXAR(selection, form.getZipFileName()); - if (form.getDataRegionSelectionKey() != null) - DataRegionSelection.clearAll(getViewContext(), form.getDataRegionSelectionKey()); - return true; - } - } - - @RequiresPermission(ReadPermission.class) - public class ExportFilesAction extends AbstractExportAction - { - @Override - public boolean handlePost(ExportOptionsForm form, BindException errors) throws Exception - { - long[] dataIds = form.getDataIds(); - if (dataIds == null || dataIds.length == 0) - { - throw new NotFoundException(); - } - - try - { - for (long id : dataIds) - { - ExpData data = ExperimentService.get().getExpData(id); - if (data == null || !data.getContainer().hasPermission(getUser(), ReadPermission.class)) - { - throw new NotFoundException("Could not find file " + id); - } - } - - XarExportSelection selection = new XarExportSelection(); - selection.setIncludeXarXml(false); - selection.addDataIds(dataIds); - - _resultURL = exportXAR(selection, form.getZipFileName()); - return true; - } - catch (NumberFormatException e) - { - throw new NotFoundException(Arrays.toString(dataIds)); - } - } - } - - public static class ExperimentRunListForm implements DataRegionSelection.DataSelectionKeyForm - { - private String _dataRegionSelectionKey; - private Long _expRowId; - private Long[] _runIds; - - @Override - public String getDataRegionSelectionKey() - { - return _dataRegionSelectionKey; - } - - @Override - public void setDataRegionSelectionKey(String key) - { - _dataRegionSelectionKey = key; - } - - public Long getExpRowId() - { - return _expRowId; - } - - public void setExpRowId(Long expRowId) - { - _expRowId = expRowId; - } - - public Long[] getRunIds() - { - return _runIds; - } - - public void setRunIds(Long[] runIds) - { - _runIds = runIds; - } - - public ExpExperiment lookupExperiment() - { - return getExpRowId() == null ? null : ExperimentService.get().getExpExperiment(getExpRowId().intValue()); - } - } - - private void addSelectedRunsToExperiment(ExpExperiment exp, String dataRegionSelectionKey) - { - Collection runIds = DataRegionSelection.getSelectedIntegers(getViewContext(), dataRegionSelectionKey, true); - List runs = new ArrayList<>(); - for (long runId : runIds) - { - ExpRun run = ExperimentServiceImpl.get().getExpRun(runId); - if (run != null) - { - runs.add(run); - } - } - exp.addRuns(getUser(), runs.toArray(new ExpRun[0])); - } - - - @RequiresPermission(InsertPermission.class) - public class AddRunsToExperimentAction extends FormHandlerAction - { - @Override - public void validateCommand(ExperimentRunListForm target, Errors errors) - { - } - - @Override - public boolean handlePost(ExperimentRunListForm form, BindException errors) - { - addSelectedRunsToExperiment(form.lookupExperiment(), form.getDataRegionSelectionKey()); - return true; - } - - @Override - public ActionURL getSuccessURL(ExperimentRunListForm form) - { - return ExperimentUrlsImpl.get().getExperimentDetailsURL(getContainer(), form.lookupExperiment()); - } - } - - @RequiresPermission(DeletePermission.class) - public static class RemoveSelectedExpRunsAction extends FormHandlerAction - { - @Override - public void validateCommand(ExperimentRunListForm target, Errors errors) - { - } - - @Override - public boolean handlePost(ExperimentRunListForm form, BindException errors) - { - ExpExperiment exp = form.lookupExperiment(); - if (exp == null || !exp.getContainer().hasPermission(getUser(), DeletePermission.class)) - { - throw new NotFoundException("Could not find run group with RowId " + form.getExpRowId()); - } - - for (long runId : DataRegionSelection.getSelectedIntegers(getViewContext(), form.getDataRegionSelectionKey(), false)) - { - ExpRun run = ExperimentService.get().getExpRun(runId); - if (run == null || !run.getContainer().hasPermission(getUser(), DeletePermission.class)) - { - throw new NotFoundException("Could not find run with RowId " + runId); - } - exp.removeRun(getUser(), run); - } - if (form.getDataRegionSelectionKey() != null) - DataRegionSelection.clearAll(getViewContext(), form.getDataRegionSelectionKey()); - return true; - } - - @Override - public ActionURL getSuccessURL(ExperimentRunListForm form) - { - return ExperimentUrlsImpl.get().getExperimentDetailsURL(getContainer(), form.lookupExperiment()); - } - } - - public static ActionURL getResolveLsidURL(Container c, @NotNull String type, @NotNull String lsid) - { - ActionURL url = new ActionURL(ResolveLSIDAction.class, c); - url.addParameter("type", type); - url.addParameter("lsid", lsid); - - return url; - } - - - @RequiresPermission(ReadPermission.class) - public static class ResolveLSIDAction extends SimpleViewAction - { - @Override - public ModelAndView getView(LsidForm form, BindException errors) - { - String message = ""; - if (!PageFlowUtil.empty(form.getLsid())) - { - try - { - String lsid = Lsid.canonical(form.getLsid().trim()); - ActionURL url = LsidManager.get().getDisplayURL(lsid); - if (url == null && form.getType() != null) - { - url = switch (form.getType().toLowerCase()) - { - case "data" -> LsidType.Data.getDisplayURL(new Lsid(lsid)); - case "material" -> LsidType.Material.getDisplayURL(new Lsid(lsid)); - default -> url; - }; - } - if (null != url) - { - throw new RedirectException(url); - } - message = "Could not map LSID to URL"; - } - catch (IllegalArgumentException e) - { - message = "Invalid LSID"; - } - } - - return new HtmlView("Enter LSID", - DOM.createHtmlFragment( - message, - DOM.FORM(at(action, getViewContext().cloneActionURL().setAction(ResolveLSIDAction.class)), - "LSID: ", - DOM.INPUT(at(type, "text", name, "lsid", size, "80", value, form.getLsid())), - PageFlowUtil.button("Go").submit(true)))); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Resolve LSID"); - } - } - - public static class LsidForm - { - private String _lsid; - - public String getType() - { - return _type; - } - - public void setType(String type) - { - _type = type; - } - - private String _type; - - public void setLsid(String lsid) - { - _lsid = lsid; - } - - public String getLsid() - { - return _lsid; - } - } - - public static class SetFlagForm extends LsidForm - { - private String _comment; - private boolean _redirect = true; - - public String getComment() - { - return _comment; - } - - public void setComment(String comment) - { - _comment = comment; - } - - public boolean isRedirect() - { - return _redirect; - } - - public void setRedirect(boolean redirect) - { - _redirect = redirect; - } - } - - /** - * Check for update on the object itself - */ - @RequiresNoPermission - public static class SetFlagAction extends FormHandlerAction - { - @Override - public void validateCommand(SetFlagForm target, Errors errors) - { - } - - @Override - public boolean handlePost(SetFlagForm form, BindException errors) throws Exception - { - String lsid = form.getLsid(); - if (lsid == null) - throw new NotFoundException(); - ExpObject obj = ExperimentService.get().findObjectFromLSID(lsid); - if (obj == null) - throw new NotFoundException(); - Container container = obj.getContainer(); - if (!container.hasPermission(getUser(), UpdatePermission.class)) - { - throw new UnauthorizedException(); - } - - obj.setComment(getUser(), form.getComment()); - return true; - } - - @Override - public URLHelper getSuccessURL(SetFlagForm form) - { - return null; - } - } - - @RequiresPermission(InsertPermission.class) - public class DeriveSamplesChooseTargetAction extends SimpleViewAction - { - private List _materials; - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("sampleSets"); - addRootNavTrail(root); - root.addChild("Sample Types", ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer())); - ExpSampleType sampleType = _materials != null && !_materials.isEmpty() ? _materials.get(0).getSampleType() : null; - if (sampleType != null) - { - root.addChild(sampleType.getName(), ExperimentUrlsImpl.get().getShowSampleTypeURL(sampleType)); - } - root.addChild("Derive Samples"); - } - - @Override - public void validate(DeriveMaterialForm form, BindException errors) - { - _materials = form.lookupMaterials(); - if (_materials.isEmpty()) - { - throw new NotFoundException("Could not find any matching materials"); - } - } - - @Override - public ModelAndView getView(DeriveMaterialForm form, BindException errors) - { - Container c = getContainer(); - PipeRoot root = PipelineService.get().findPipelineRoot(c); - - if (root == null || !root.isValid()) - { - ActionURL pipelineURL = urlProvider(PipelineUrls.class).urlSetup(c); - return new HtmlView(DIV("You must ", - DOM.A(DOM.at(href, pipelineURL), "configure a valid pipeline root for this folder"), - " before deriving samples.")); - } - else - { - Set materialInputRoles = new TreeSet<>(ExperimentService.get().getMaterialInputRoles(getContainer(), getUser())); - Map materialsWithRoles = new LinkedHashMap<>(); - for (ExpMaterial material : _materials) - { - materialsWithRoles.put(material, null); - } - - List sampleTypes = getUploadableSampleTypes(); - - DeriveSamplesChooseTargetBean bean = new DeriveSamplesChooseTargetBean(form.getDataRegionSelectionKey(), form.getTargetSampleTypeId(), sampleTypes, materialsWithRoles, form.getOutputCount(), materialInputRoles, null); - return new JspView<>("/org/labkey/experiment/deriveSamplesChooseTarget.jsp", bean); - } - } - } - - public static class DeriveSamplesChooseTargetBean implements DataRegionSelection.DataSelectionKeyForm - { - private String _dataRegionSelectionKey; - - private final Integer _targetSampleTypeId; - private final List _sampleTypes; - private final Map _sourceMaterials; - private final int _sampleCount; - private final Collection _inputRoles; - private final DerivedSamplePropertyHelper _propertyHelper; - - public static final String CUSTOM_ROLE = "--CUSTOM--"; - - public DeriveSamplesChooseTargetBean(String dataRegionSelectionKey, Integer targetSampleTypeId, List sampleTypes, Map sourceMaterials, int sampleCount, Collection inputRoles, DerivedSamplePropertyHelper helper) - { - _dataRegionSelectionKey = dataRegionSelectionKey; - _targetSampleTypeId = targetSampleTypeId; - _sampleTypes = sampleTypes; - _sourceMaterials = sourceMaterials; - _sampleCount = sampleCount; - _inputRoles = inputRoles; - _propertyHelper = helper; - } - - public Integer getTargetSampleTypeId() - { - return _targetSampleTypeId; - } - - public DerivedSamplePropertyHelper getPropertyHelper() - { - return _propertyHelper; - } - - public int getSampleCount() - { - return _sampleCount; - } - - public Map getSourceMaterials() - { - return _sourceMaterials; - } - - public List getSampleTypes() - { - return _sampleTypes; - } - - public Collection getInputRoles() - { - return _inputRoles; - } - - @Override - public String getDataRegionSelectionKey() - { - return _dataRegionSelectionKey; - } - - @Override - public void setDataRegionSelectionKey(String key) - { - _dataRegionSelectionKey = key; - } - } - - private List getUploadableSampleTypes() - { - // Make a copy so we can modify it - List sampleTypes = new ArrayList<>(SampleTypeService.get().getSampleTypes(getContainer(), getUser(), true)); - sampleTypes.removeIf(sampleType -> !sampleType.canImportMoreSamples()); - return sampleTypes; - } - - @RequiresPermission(InsertPermission.class) - public class DeriveSamplesAction extends FormViewAction - { - private List _materials; - private ActionURL _successUrl; - private final Map _inputMaterials = new LinkedHashMap<>(); - - @Override - public ModelAndView getView(DeriveMaterialForm form, boolean reshow, BindException errors) - { - _materials = form.lookupMaterials(); - if (_materials.isEmpty()) - { - throw new NotFoundException("Could not find any matching materials"); - } - - Container c = getContainer(); - - if (form.getOutputCount() <= 0) - { - form.setOutputCount(1); - } - - if (form.getTargetSampleTypeId() == 0) - throw new NotFoundException("Target sample type required for the derived samples"); - - ExpSampleTypeImpl sampleType = SampleTypeServiceImpl.get().getSampleType(getContainer(), getUser(), form.getTargetSampleTypeId()); - if (sampleType == null) - throw new NotFoundException("Could not find sample type with rowId " + form.getTargetSampleTypeId()); - - InsertView insertView = new InsertView(new DataRegion(), errors); - - DerivedSamplePropertyHelper helper = new DerivedSamplePropertyHelper(sampleType, form.getOutputCount(), c, getUser()); - helper.addSampleColumns(insertView, getUser()); - - int[] rowIds = form.getRowIds(); - for (int i = 0; i < rowIds.length; i++) - { - insertView.getDataRegion().addHiddenFormField("rowIds", Integer.toString(rowIds[i])); - insertView.getDataRegion().addHiddenFormField("inputRole" + i, form.getInputRole(i) == null ? "" : form.getInputRole(i)); - insertView.getDataRegion().addHiddenFormField("customRole" + i, form.getCustomRole(i) == null ? "" : form.getCustomRole(i)); - } - - insertView.getDataRegion().addHiddenFormField("targetSampleTypeId", Integer.toString(form.getTargetSampleTypeId())); - insertView.getDataRegion().addHiddenFormField("outputCount", Integer.toString(form.getOutputCount())); - if (form.getDataRegionSelectionKey() != null) - insertView.getDataRegion().addHiddenFormField(DataRegionSelection.DATA_REGION_SELECTION_KEY, form.getDataRegionSelectionKey()); - insertView.setInitialValues(ViewServlet.adaptParameterMap(getViewContext().getRequest().getParameterMap())); - ButtonBar bar = new ButtonBar(); - bar.setStyle(ButtonBar.Style.separateButtons); - ActionButton submitButton = new ActionButton(DeriveSamplesAction.class, "Submit"); - submitButton.setActionType(ActionButton.Action.POST); - bar.add(submitButton); - insertView.getDataRegion().setButtonBar(bar); - insertView.setTitle("Output Samples"); - - Map materialsWithRoles = new LinkedHashMap<>(); - List materials = form.lookupMaterials(); - for (int i = 0; i < materials.size(); i++) - { - materialsWithRoles.put(materials.get(i), form.determineLabel(i)); - } - - DeriveSamplesChooseTargetBean bean = new DeriveSamplesChooseTargetBean(form.getDataRegionSelectionKey(), form.getTargetSampleTypeId(), getUploadableSampleTypes(), materialsWithRoles, form.getOutputCount(), Collections.emptyList(), helper); - JspView view = new JspView<>("/org/labkey/experiment/summarizeMaterialInputs.jsp", bean); - view.setTitle("Input Samples"); - - return new VBox(view, insertView); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("sampleSets"); - addRootNavTrail(root); - root.addChild("Sample Types", ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer())); - ExpSampleType sampleType = _materials != null && !_materials.isEmpty() ? _materials.get(0).getSampleType() : null; - if (sampleType != null) - { - root.addChild(sampleType.getName(), ExperimentUrlsImpl.get().getShowSampleTypeURL(sampleType)); - } - root.addChild("Derive Samples"); - } - - @Override - public void validateCommand(DeriveMaterialForm form, Errors errors) - { - List materials = form.lookupMaterials(); - - List lockedSamples = new ArrayList<>(); - for (int i = 0; i < materials.size(); i++) - { - ExpMaterial m = materials.get(i); - if (!m.isOperationPermitted(SampleTypeService.SampleOperations.EditLineage)) - { - lockedSamples.add(m); - } - String inputRole = form.determineLabel(i); - if (inputRole == null || inputRole.isEmpty()) - { - ExpSampleType st = m.getSampleType(); - inputRole = st != null ? st.getName() : ExpMaterialRunInput.DEFAULT_ROLE; - } - _inputMaterials.put(materials.get(i), inputRole); - } - - if (!lockedSamples.isEmpty()) - { - errors.reject(ERROR_MSG, SampleTypeService.get().getOperationNotPermittedMessage(lockedSamples, SampleTypeService.SampleOperations.EditLineage)); - } - } - - @Override - public boolean handlePost(DeriveMaterialForm form, BindException errors) - { - ExpSampleTypeImpl sampleType = SampleTypeServiceImpl.get().getSampleType(getContainer(), getUser(), form.getTargetSampleTypeId()); - - DerivedSamplePropertyHelper helper = new DerivedSamplePropertyHelper(sampleType, form.getOutputCount(), getContainer(), getUser()); - - Map, Map> allProperties; - try - { - boolean valid = true; - for (Map.Entry> entry : helper.getPostedPropertyValues(getViewContext().getRequest()).entrySet()) - valid = UploadWizardAction.validatePostedProperties(getViewContext(), entry.getValue(), errors) && valid; - if (!valid) - return false; - - allProperties = helper.getSampleProperties(getViewContext().getRequest(), _inputMaterials.keySet()); - } - catch (DuplicateMaterialException e) - { - errors.addError(new ObjectError(e.getColName(), null, null, e.getMessage())); - return false; - } - catch (ExperimentException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - return false; - } - - try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) - { - Map outputMaterials = new HashMap<>(); - int i = 0; - for (Map.Entry, Map> entry : allProperties.entrySet()) - { - Lsid lsid = entry.getKey().first; - String name = entry.getKey().second; - assert name != null; - - ExpMaterialImpl outputMaterial = ExperimentServiceImpl.get().createExpMaterial(getContainer(), lsid.toString(), name); - if (sampleType != null) - { - outputMaterial.setCpasType(sampleType.getLSID()); - } - outputMaterial.save(getUser()); - - if (sampleType != null) - { - Map pvs = new HashMap<>(); - for (Map.Entry propertyEntry : entry.getValue().entrySet()) - pvs.put(propertyEntry.getKey().getName(), propertyEntry.getValue()); - outputMaterial.setProperties(getUser(), pvs, false); - } - - outputMaterials.put(outputMaterial, helper.getSampleNames().get(i++)); - } - - ExperimentService.get().deriveSamples(_inputMaterials, outputMaterials, getViewBackgroundInfo(), _log); - - tx.commit(); - - // automatically link samples to study, if configured - StudyPublishService.get().autoLinkDerivedSamples(sampleType, outputMaterials.keySet().stream().map(ExpObject::getRowId).collect(toList()), getContainer(), getUser()); - - _successUrl = ExperimentUrlsImpl.get().getShowSampleURL(getContainer(), outputMaterials.keySet().iterator().next()); - - if (form.getDataRegionSelectionKey() != null) - DataRegionSelection.clearAll(getViewContext(), form.getDataRegionSelectionKey()); - } - catch (Exception e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - return false; - } - - return true; - } - - @Override - public URLHelper getSuccessURL(DeriveMaterialForm deriveMaterialForm) - { - return _successUrl; - } - } - - public static class DeriveMaterialForm implements HasViewContext, DataRegionSelection.DataSelectionKeyForm - { - private String _dataRegionSelectionKey; - private int _outputCount = 1; - private int _targetSampleTypeId; - private int[] _rowIds; - private String _name; - - private ViewContext _context; - - @Override - public void setViewContext(ViewContext context) - { - _context = context; - } - - @Override - public ViewContext getViewContext() - { - return _context; - } - - public List lookupMaterials() - { - List result = new ArrayList<>(); - for (int rowId : getRowIds()) - { - ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); - if (material != null) - { - if (material.getContainer().hasPermission(_context.getUser(), ReadPermission.class)) - { - result.add(material); - } - else - { - throw new UnauthorizedException(); - } - } - else - { - throw new NotFoundException("No material with RowId " + rowId); - } - } - result.sort(Comparator.comparing(Identifiable::getName)); - return result; - } - - public String getName() - { - return _name; - } - - public void setName(String name) - { - _name = name; - } - - @Override - public String getDataRegionSelectionKey() - { - return _dataRegionSelectionKey; - } - - @Override - public void setDataRegionSelectionKey(String dataRegionSelectionKey) - { - _dataRegionSelectionKey = dataRegionSelectionKey; - } - - public int[] getRowIds() - { - if (_rowIds == null) - { - _rowIds = PageFlowUtil.toInts(DataRegionSelection.getSelected(getViewContext(), getDataRegionSelectionKey(), false)); - } - return _rowIds; - } - - public void setRowIds(int[] rowIds) - { - _rowIds = rowIds; - } - - public int getOutputCount() - { - return _outputCount; - } - - public void setOutputCount(int outputCount) - { - _outputCount = outputCount; - } - - public int getTargetSampleTypeId() - { - return _targetSampleTypeId; - } - - public void setTargetSampleTypeId(int targetSampleTypeId) - { - _targetSampleTypeId = targetSampleTypeId; - } - - public String getInputRole(int i) - { - return _context.getRequest().getParameter("inputRole" + i); - } - - public String getCustomRole(int i) - { - return _context.getRequest().getParameter("customRole" + i); - } - - public String determineLabel(int index) - { - String result = getInputRole(index); - if (DeriveSamplesChooseTargetBean.CUSTOM_ROLE.equals(result)) - { - result = getCustomRole(index); - } - if (result != null) - { - result = result.trim(); - } - return result; - } - } - - - public static class ExpInput - { - public String role; - public int rowId; - public Lsid lsid; - } - - public static class DerivationSpec - { - public String role; - public Map values; - } - - public static class DerivationForm - { - public List dataInputs; - public List materialInputs; - - public int dataOutputCount; - public Lsid targetDataClass; - public Map dataDefault; - public List dataOutputs; - - public int materialOutputCount; - public Lsid targetSampleType; - public Map materialDefault; - public List materialOutputs; - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(InsertPermission.class) - public static class DeriveAction extends MutatingApiAction - { - @Override - public void validateForm(DerivationForm form, Errors errors) - { - if (errors.hasErrors()) - return; - - if (form.materialOutputCount > 0 && form.materialOutputs != null && !form.materialOutputs.isEmpty()) - errors.reject(ERROR_MSG, "Either 'materialOutputCount' or 'materialOutputs' property can be specified, but not both."); - - if (form.dataOutputCount > 0 && form.dataOutputs != null && !form.dataOutputs.isEmpty()) - errors.reject(ERROR_MSG, "Either 'dataOutputCount' or 'dataOutputs' property can be specified, but not both."); - - boolean hasMaterialOutputs = form.materialOutputCount > 0 || form.materialOutputs != null && !form.materialOutputs.isEmpty(); - boolean hasDataOutputs = form.dataOutputCount > 0 || form.dataOutputs != null && !form.dataOutputs.isEmpty(); - - if (!hasMaterialOutputs && !hasDataOutputs) - errors.reject(ERROR_MSG, "At least one data output or material output is required"); - - if (hasMaterialOutputs && form.targetSampleType == null) - errors.reject(ERROR_MSG, "targetSampleType lsid required for material outputs"); - - if (hasDataOutputs && form.targetDataClass == null) - errors.reject(ERROR_MSG, "targetDataClass lsid required for data outputs"); - } - - @Override - public Object execute(DerivationForm form, BindException errors) throws Exception - { - // Find material inputs - Map materialInputs = new LinkedHashMap<>(); - if (form.materialInputs != null) - { - for (ExpInput in : form.materialInputs) - { - ExpMaterial m = null; - if (in.lsid != null) - { - m = ExperimentService.get().getExpMaterial(in.lsid.toString()); - if (m == null) - errors.reject(ERROR_MSG, "Can't resolve sample '" + in.lsid + "'"); - } - else if (in.rowId > 0) - { - m = ExperimentService.get().getExpMaterial(in.rowId); - if (m == null) - errors.reject(ERROR_MSG, "Can't resolve sample '" + in.rowId + "'"); - } - - if (m == null) - { - errors.reject(ERROR_MSG, "Material input lsid or rowId required"); - continue; - } - - ExpSampleType st = m.getSampleType(); - if (st == null) - { - errors.reject(ERROR_MSG, "Material input is not a member of a SampleType"); - continue; - } - - String role = in.role; - if (role == null || role.isEmpty()) - { - role = st.getName(); - } - materialInputs.put(m, role); - } - } - - // Find input data - Map dataInputs = new LinkedHashMap<>(); - if (form.dataInputs != null) - { - for (ExpInput in : form.dataInputs) - { - ExpData d = null; - if (in.lsid != null) - { - d = ExperimentService.get().getExpData(in.lsid.toString()); - if (d == null) - errors.reject(ERROR_MSG, "Can't resolve data '" + in.lsid + "'"); - } - else if (in.rowId > 0) - { - d = ExperimentService.get().getExpData(in.rowId); - if (d == null) - errors.reject(ERROR_MSG, "Can't resolve data '" + in.rowId + "'"); - } - - if (d == null) - { - errors.reject(ERROR_MSG, "Data input lsid or rowId required"); - continue; - } - - ExpDataClass dc = d.getDataClass(getUser()); - if (dc == null) - { - errors.reject(ERROR_MSG, "Data input is not a member of a DataClass"); - continue; - } - - String role = in.role; - if (role == null || role.isEmpty()) - { - role = dc.getName(); - } - dataInputs.put(d, role); - } - } - - ExpSampleType outSampleType; - if (form.targetSampleType != null) - { - // TODO: check in scope and has permission - outSampleType = SampleTypeService.get().getSampleType(form.targetSampleType.toString()); - if (outSampleType == null) - errors.reject(ERROR_MSG, "Sample type not found: " + form.targetSampleType.toString()); - } - else - { - outSampleType = null; - } - - ExpDataClass outDataClass; - if (form.targetDataClass != null) - { - // TODO: check in scope and has permission - outDataClass = ExperimentServiceImpl.get().getDataClass(form.targetDataClass.toString()); - if (outDataClass == null) - errors.reject(ERROR_MSG, "DataClass not found: " + form.targetDataClass.toString()); - } - else - { - outDataClass = null; - } - - if (errors.hasErrors()) - return null; - - // TODO: support list of resolved ExpData or ExpMaterial instead of string concatenated names - // Create "MaterialInputs/" columns with a value containing a comma-separated list of Material names - final Map> parentInputNames = new HashMap<>(); - Set inputTypes = new CaseInsensitiveHashSet(); - for (ExpMaterial material : materialInputs.keySet()) - { - ExpSampleType st = material.getSampleType(); - String keyName = ExpMaterial.MATERIAL_INPUT_PARENT + "/" + st.getName(); - inputTypes.add(keyName); - parentInputNames.computeIfAbsent(keyName, (x) -> new LinkedHashSet<>()).add(material.getName()); - } - - // TODO: support list of resolved ExpData or ExpMaterial instead of string concatenated names - // Create "DataInputs/" columns with a value containing a comma-separated list of ExpData names - for (ExpData d : dataInputs.keySet()) - { - ExpDataClass dc = d.getDataClass(getUser()); - String keyName = ExpData.DATA_INPUT_PARENT + "/" + dc.getName(); - inputTypes.add(keyName); - parentInputNames.computeIfAbsent(keyName, (x) -> new LinkedHashSet<>()).add(d.getName()); - } - - - try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) - { - Set requiredParentTypes = new CaseInsensitiveHashSet(); - - // output materials - Map outputMaterials = new HashMap<>(); - int materialOutputCount = Math.max(form.materialOutputCount, form.materialOutputs != null ? form.materialOutputs.size() : 0); - if (materialOutputCount > 0 && outSampleType != null) - { - requiredParentTypes.addAll(outSampleType.getRequiredImportAliases().values()); - DerivedOutputs derived = new DerivedOutputs<>(parentInputNames, form.materialDefault, form.materialOutputs, materialOutputCount, ExpMaterial.DEFAULT_CPAS_TYPE) - { - @Override - protected TableInfo createTable() - { - SamplesSchema schema = new SamplesSchema(getUser(), getContainer()); - return schema.getTable(outSampleType.getName()); - } - - @Override - protected List getExpObject(List> insertedRows) - { - List rowIds = insertedRows.stream().map(r -> MapUtils.getLong(r,"rowid")).collect(toList()); - return ExperimentService.get().getExpMaterials(rowIds); - } - }; - - outputMaterials = derived.createOutputs(); - } - - - // create output data - Map outputData = new HashMap<>(); - int dataOutputCount = Math.max(form.dataOutputCount, form.dataOutputs != null ? form.dataOutputs.size() : 0); - if (dataOutputCount > 0 && outDataClass != null) - { - requiredParentTypes.addAll(outDataClass.getRequiredImportAliases().values()); - DerivedOutputs derived = new DerivedOutputs<>(parentInputNames, form.dataDefault, form.dataOutputs, dataOutputCount, ExpData.DEFAULT_CPAS_TYPE) - { - @Override - protected TableInfo createTable() - { - ExpSchema expSchema = new ExpSchema(getUser(), getContainer()); - UserSchema dataSchema = expSchema.getUserSchema(ExpSchema.NestedSchemas.data.name()); - return dataSchema.getTable(outDataClass.getName()); - } - - @Override - protected List getExpObject(List> insertedRows) - { - List lsids = insertedRows.stream().map(r -> (String) r.get("lsid")).collect(toList()); - return ExperimentService.get().getExpDatasByLSID(lsids); - } - }; - - outputData = derived.createOutputs(); - } - - if (outputMaterials.isEmpty() && outputData.isEmpty()) - throw new IllegalStateException("Expected to create " + materialOutputCount + " materials and " + dataOutputCount + " datas"); - - boolean hasMissingRequiredParent = false; - for (String required : requiredParentTypes) - { - if (!inputTypes.contains(required)) - { - hasMissingRequiredParent = true; - break; - } - } - if (hasMissingRequiredParent) - throw new IllegalStateException("Inputs are required: " + String.join(",", requiredParentTypes)); - - // finally, create the derived run if there are any parents - ExpRun run = null; - if (!materialInputs.isEmpty() || !dataInputs.isEmpty()) - run = ExperimentService.get().derive(materialInputs, dataInputs, outputMaterials, outputData, new ViewBackgroundInfo(getContainer(), getUser(), null), _log); - tx.commit(); - - StringBuilder successMessage = new StringBuilder("Created "); - if (!outputMaterials.isEmpty()) - successMessage.append(outputMaterials.size()).append(" materials"); - if (!outputData.isEmpty()) - successMessage.append(outputData.size()).append(" data"); - - JSONObject ret; - if (run != null) - ret = ExperimentJSONConverter.serializeRun(run, null, getUser(), ExperimentJSONConverter.DEFAULT_SETTINGS); - else - ret = ExperimentJSONConverter.serializeRunOutputs(outputData.keySet(), outputMaterials.keySet(), getUser(), ExperimentJSONConverter.DEFAULT_SETTINGS); - - return success(successMessage.toString(), ret); - } - } - - // Helper class that prepares and executes the QueryUpdateService.insertRows() on the data or material table. - private abstract class DerivedOutputs - { - private final @NotNull Map> _parentInputNames; - private final @Nullable Map _defaultValues; - private final @Nullable List _values; - private final int _outputCount; - private final String _rolePrefix; - - - public DerivedOutputs(@NotNull Map> parentInputNames, @Nullable Map defaultValues, @Nullable List values, int outputCount, String rolePrefix) - { - _parentInputNames = parentInputNames; - _defaultValues = defaultValues; - _values = values; - _outputCount = outputCount; - _rolePrefix = rolePrefix; - } - - public Pair>, List> prepareRows() - { - List> rows = new ArrayList<>(); - List roles = new ArrayList<>(); - int unknownOutputDataCount = 0; - - for (int i = 0; i < _outputCount; i++) - { - Map row = new CaseInsensitiveHashMap<>(); - if (_defaultValues != null) - row.putAll(_defaultValues); - DerivationSpec spec = _values != null && i < _values.size() ? _values.get(i) : null; - String role = null; - if (spec != null) - { - row.putAll(spec.values); - role = spec.role; - } - - // NOTE: Input parents are added to each row, but are only used for name generation and not for derivation. - // NOTE: We will derive the inserted samples in a single derivation run after the sample/date have been inserted. - row.putAll(_parentInputNames); - - rows.add(row); - - if (StringUtils.trimToNull(role) == null) - { - role = _rolePrefix + (unknownOutputDataCount == 0 ? "" : Integer.toString(unknownOutputDataCount + 1)); - unknownOutputDataCount++; - } - roles.add(role); - } - return Pair.of(rows, roles); - } - - protected abstract TableInfo createTable(); - - protected abstract List getExpObject(List> insertedRows); - - public Map createOutputs() throws BatchValidationException, DuplicateKeyException, SQLException, QueryUpdateServiceException - { - Pair>, List> pair = prepareRows(); - List> rows = pair.first; - List roles = pair.second; - - TableInfo table = createTable(); - QueryUpdateService qus = table.getUpdateService(); - if (qus == null) - throw new IllegalStateException(); - - Map configParams = new HashMap<>(); - // Skip derivation during insert -- DeriveAction will call ExperimentService.get().derive() after samples are inserted - configParams.put(SampleTypeUpdateServiceDI.Options.SkipDerivation, true); - - BatchValidationException qusErrors = new BatchValidationException(); - List> insertedRows = qus.insertRows(getUser(), getContainer(), rows, qusErrors, configParams, null); - if (qusErrors.hasErrors()) - throw qusErrors; - - if (insertedRows.size() != roles.size()) - throw new IllegalStateException("Expected to create " + roles.size() + " new exp objects for derivation"); - - List outputs = getExpObject(insertedRows); - if (outputs.size() != roles.size()) - throw new IllegalStateException("Expected to create " + roles.size() + " new exp objects for derivation"); - - Map outputMap = new HashMap<>(); - for (int i = 0; i < outputs.size(); i++) - { - String role = roles.get(i); - T data = outputs.get(i); - outputMap.put(data, role); - } - - return outputMap; - } - } - } - - public static class CreateExperimentForm extends ExperimentForm implements DataRegionSelection.DataSelectionKeyForm - { - private boolean _addSelectedRuns; - private String _dataRegionSelectionKey; - - public boolean isAddSelectedRuns() - { - return _addSelectedRuns; - } - - public void setAddSelectedRuns(boolean addSelectedRuns) - { - _addSelectedRuns = addSelectedRuns; - } - - @Override - public String getDataRegionSelectionKey() - { - return _dataRegionSelectionKey; - } - - @Override - public void setDataRegionSelectionKey(String dataRegionSelectionKey) - { - _dataRegionSelectionKey = dataRegionSelectionKey; - } - } - - @RequiresPermission(InsertPermission.class) - @ActionNames("createRunGroup, createExperiment") - public class CreateRunGroupAction extends FormViewAction - { - @Override - public ModelAndView getView(CreateExperimentForm form, boolean reshow, BindException errors) - { - // HACK - convert ExperimentForm to not be a BeanViewForm - form.setAddSelectedRuns("true".equals(getViewContext().getRequest().getParameter("addSelectedRuns"))); - form.setDataRegionSelectionKey(getViewContext().getRequest().getParameter(DataRegionSelection.DATA_REGION_SELECTION_KEY)); - - DataRegion drg = new DataRegion(); - - drg.addHiddenFormField(ActionURL.Param.returnUrl, getViewContext().getRequest().getParameter(ActionURL.Param.returnUrl.name())); - drg.addHiddenFormField("addSelectedRuns", Boolean.toString("true".equals(getViewContext().getRequest().getParameter("addSelectedRuns")))); - form.setDataRegionSelectionKey(getViewContext().getRequest().getParameter(DataRegionSelection.DATA_REGION_SELECTION_KEY)); - // Fix issue 27562 - include session-stored selection - if (form.getDataRegionSelectionKey() != null) - { - for (String rowId : DataRegionSelection.getSelected(getViewContext(), form.getDataRegionSelectionKey(), false)) - { - drg.addHiddenFormField(DataRegion.SELECT_CHECKBOX_NAME, rowId); - } - } - drg.addHiddenFormField(DataRegionSelection.DATA_REGION_SELECTION_KEY, getViewContext().getRequest().getParameter(DataRegionSelection.DATA_REGION_SELECTION_KEY)); - - drg.addColumns(ExperimentServiceImpl.get().getTinfoExperiment(), "RowId,Name,LSID,ContactId,ExperimentDescriptionURL,Hypothesis,Comments,Created"); - - DisplayColumn col = drg.getDisplayColumn("RowId"); - col.setVisible(false); - drg.getDisplayColumn("LSID").setVisible(false); - drg.getDisplayColumn("Created").setVisible(false); - - ButtonBar bb = new ButtonBar(); - bb.setStyle(ButtonBar.Style.separateButtons); - ActionButton insertButton = new ActionButton(new ActionURL(CreateRunGroupAction.class, getContainer()), "Submit", ActionButton.Action.POST); - bb.add(insertButton); - - drg.setButtonBar(bb); - - return new InsertView(drg, errors); - } - - - @Override - public boolean handlePost(CreateExperimentForm form, BindException errors) throws Exception - { - // This is strange... but the "Create new run group..." menu item on the run grid always POSTs, probably to - // allow for long lists of run IDs. This "noPost" parameter on the initial POST is used to inform the action - // that it wants to display the form, not try to save anything yet. - if (!"true".equals(getViewContext().getRequest().getParameter("noPost"))) - { - form.setAddSelectedRuns("true".equals(getViewContext().getRequest().getParameter("addSelectedRuns"))); - form.setDataRegionSelectionKey(getViewContext().getRequest().getParameter(DataRegionSelection.DATA_REGION_SELECTION_KEY)); - - Experiment exp = form.getBean(); - if (exp.getName() == null || exp.getName().trim().isEmpty()) - { - errors.reject(ERROR_MSG, "You must specify a name for the experiment"); - } - else - { - int maxNameLength = ExperimentService.get().getTinfoExperimentRun().getColumn("Name").getScale(); - if (exp.getName().length() > maxNameLength) - { - errors.reject(ERROR_MSG, "Name of the experiment must be " + maxNameLength + " characters or less."); - } - } - - String lsid; - int suffix = 1; - do - { - String template = "urn:lsid:" + XarContext.LSID_AUTHORITY_SUBSTITUTION + ":Experiment.Folder-" + XarContext.CONTAINER_ID_SUBSTITUTION + ":" + exp.getName(); - if (suffix > 1) - { - template = template + suffix; - } - suffix++; - lsid = LsidUtils.resolveLsidFromTemplate(template, new XarContext("Experiment Creation", getContainer(), getUser()), ExpExperiment.DEFAULT_CPAS_TYPE); - } - while (ExperimentService.get().getExpExperiment(lsid) != null); - exp.setLSID(lsid); - exp.setContainer(getContainer()); - - if (errors.getErrorCount() == 0) - { - ExpExperimentImpl wrapper = new ExpExperimentImpl(exp); - wrapper.save(getUser()); - - if (form.isAddSelectedRuns()) - { - addSelectedRunsToExperiment(wrapper, form.getDataRegionSelectionKey()); - } - - if (form.getReturnUrl() != null) - { - throw new RedirectException(form.getReturnUrl()); - } - throw new RedirectException(ExperimentUrlsImpl.get().getShowExperimentsURL(getContainer())); - } - } - return true; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("runGroups"); - root.addChild("Create Run Group"); - } - - @Override - public URLHelper getSuccessURL(CreateExperimentForm createExperimentForm) - { - return null; // null is used to show the form in the case where IDs are POSTed from the grid - } - - @Override - public void validateCommand(CreateExperimentForm target, Errors errors) { } - } - - public static class MoveRunsForm implements DataRegionSelection.DataSelectionKeyForm - { - private String _targetContainerId; - private String _dataRegionSelectionKey; - - @Override - public String getDataRegionSelectionKey() - { - return _dataRegionSelectionKey; - } - - @Override - public void setDataRegionSelectionKey(String key) - { - _dataRegionSelectionKey = key; - } - - public String getTargetContainerId() - { - return _targetContainerId; - } - - public void setTargetContainerId(String targetContainerId) - { - _targetContainerId = targetContainerId; - } - } - - @RequiresPermission(DeletePermission.class) - public class MoveRunsLocationAction extends SimpleViewAction - { - @Override - public ModelAndView getView(MoveRunsForm form, BindException errors) - { - ActionURL moveURL = new ActionURL(MoveRunsAction.class, getContainer()); - PipelineRootContainerTree ct = new PipelineRootContainerTree(getUser(), moveURL) - { - private boolean _clickHandlerRegistered = false; - - @Override - protected void renderCellContents(StringBuilder html, Container c, ActionURL url, boolean hasRoot) - { - boolean renderLink = hasRoot && !c.equals(getContainer()); - - if (renderLink) - { - html.append(""); - } - html.append(PageFlowUtil.filter(c.getName())); - if (renderLink) - { - html.append(""); - } - - if (!_clickHandlerRegistered) - { - HttpView.currentPageConfig().addHandlerForQuerySelector("a.move-target-container", "click", "moveTo(this.attributes.getNamedItem('data-objectid').value);" ); - _clickHandlerRegistered = true; - } - } - }; - ct.setInitialLevel(1); - - MoveRunsBean bean = new MoveRunsBean(ct, form.getDataRegionSelectionKey()); - JspView result = new JspView<>("/org/labkey/experiment/moveRunsLocation.jsp", bean); - result.setTitle("Choose Destination Folder"); - result.setFrame(WebPartView.FrameType.PORTAL); - return result; - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Move Runs"); - } - } - - - @RequiresPermission(DeletePermission.class) - public class MoveRunsAction extends FormHandlerAction - { - private Container _targetContainer; - - @Override - public void validateCommand(MoveRunsForm target, Errors errors) - { - } - - @Override - public boolean handlePost(MoveRunsForm form, BindException errors) - { - _targetContainer = ContainerManager.getForId(form.getTargetContainerId()); - if (_targetContainer == null || !_targetContainer.hasPermission(getUser(), InsertPermission.class)) - { - throw new UnauthorizedException(); - } - - Set runIds = DataRegionSelection.getSelectedIntegers(getViewContext(), form.getDataRegionSelectionKey(), false); - List runs = new ArrayList<>(); - for (Long runId : runIds) - { - ExpRun run = ExperimentService.get().getExpRun(runId); - if (run != null) - { - runs.add(run); - } - } - - ViewBackgroundInfo info = getViewBackgroundInfo(); - info.setContainer(_targetContainer); - - try - { - ExperimentService.get().moveRuns(info, getContainer(), runs); - if (form.getDataRegionSelectionKey() != null) - DataRegionSelection.clearAll(getViewContext(), form.getDataRegionSelectionKey()); - } - catch (IOException e) - { - throw new NotFoundException("Failed to initialize move. Check that the pipeline root is configured correctly. " + e); - } - return true; - } - - @Override - public ActionURL getSuccessURL(MoveRunsForm form) - { - return urlProvider(PipelineUrls.class).urlBegin(_targetContainer); - } - } - - public static class ShowExternalDocsForm - { - private String _objectURI; - private String _propertyURI; - - public String getObjectURI() - { - return _objectURI; - } - - public void setObjectURI(String objectURI) - { - _objectURI = objectURI; - } - - public String getPropertyURI() - { - return _propertyURI; - } - - public void setPropertyURI(String propertyURI) - { - _propertyURI = propertyURI; - } - } - - @RequiresPermission(ReadPermission.class) - public static class ShowExternalDocsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ShowExternalDocsForm form, BindException errors) throws Exception - { - Map props = OntologyManager.getPropertyObjects(getContainer(), form.getObjectURI()); - ObjectProperty prop = props.get(form.getPropertyURI()); - if (prop == null || !getContainer().equals(prop.getContainer())) - { - throw new NotFoundException(); - } - URI uri = new URI(prop.getStringValue()); - File f = new File(uri); - if (!f.exists()) - { - throw new NotFoundException(); - } - - PageFlowUtil.streamFile(getViewContext().getResponse(), new File(f.getAbsolutePath()), false); - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - - // TODO: DotGraph has been adding a "runId" parameter, but ShowGraphMoreListAction - public static ActionURL getShowGraphMoreListURL(Container c, @Nullable Long runId, @NotNull String objtype) - { - ActionURL url = new ActionURL(ShowGraphMoreListAction.class, c); - - if (null != runId) - url.addParameter("runId", runId); - - url.addParameter("objtype", objtype); - - return url; - } - - - @RequiresPermission(ReadPermission.class) - public static class ShowGraphMoreListAction extends SimpleViewAction - { - private ExperimentRunForm _form; - - @Override - public ModelAndView getView(ExperimentRunForm form, BindException errors) - { - _form = form; - return new GraphMoreGrid(getContainer(), errors, getViewContext().getActionURL()); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild(new NavTree("Experiments", ExperimentUrlsImpl.get().getShowExperimentsURL(getContainer()))); - ExpRun run = ExperimentService.get().getExpRun(_form.getRowId()); - if (run != null) - { - root.addChild(new NavTree("Experiment Run", ExperimentUrlsImpl.get().getRunGraphURL(_form.lookupRun()))); - } - root.addChild(new NavTree("Selected Protocol Applications")); - } - } - - @RequiresPermission(DesignAssayPermission.class) - public class AssayXarFileAction extends MutatingApiAction - { - - @Override - public Object execute(Object o, BindException errors) throws Exception - { - ApiSimpleResponse response = new ApiSimpleResponse(); - - if (!(getViewContext().getRequest() instanceof MultipartHttpServletRequest)) - throw new BadRequestException("Expected MultipartHttpServletRequest when posting files."); - - if (!PipelineService.get().hasValidPipelineRoot(getContainer())) - { - return false; - } - - MultipartFile formFile = getFileMap().get("file"); - if (formFile == null) - { - errors.reject(ERROR_MSG, "No file was posted by the browser."); - return false; - } - - byte[] bytes = formFile.getBytes(); - if (bytes.length == 0) - { - errors.reject(ERROR_MSG, "No file was posted by the browser."); - return false; - } - - PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(getContainer()); - Path systemDir = pipeRoot.ensureSystemDirectoryPath(); - Path uploadDir = systemDir.resolve("UploadedXARs"); - FileUtil.createDirectories(uploadDir); - if (!Files.isDirectory(uploadDir)) - { - errors.reject(ERROR_MSG, "Unable to create a 'system/UploadedXARs' directory under the pipeline root"); - return false; - } - String userDirName = getUser().getEmail(); - if (userDirName == null || userDirName.isEmpty()) - { - userDirName = GUEST_DIRECTORY_NAME; - } - Path userDir = uploadDir.resolve(userDirName); - FileUtil.createDirectories(userDir); - if (!Files.isDirectory(userDir)) - { - errors.reject(ERROR_MSG, "Unable to create an 'UploadedXARs/" + userDirName + "' directory under the pipeline root"); - return false; - } - - Path xarFile = userDir.resolve(formFile.getOriginalFilename()); - - // As this is multi-part will need to use finally to close, to prevent a stream closure exception - try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(xarFile))) - { - out.write(bytes); - } - catch (IOException e) - { - errors.reject(ERROR_MSG, "Unable to write uploaded XAR file to " + xarFile); - return false; - } - //noinspection EmptyCatchBlock - - ExperimentPipelineJob job = new ExperimentPipelineJob(getViewBackgroundInfo(), xarFile, - "Uploaded file", true, pipeRoot); - PipelineService.get().queueJob(job); - - response.put("success", true); - return response; - } - } - - @RequiresPermission(InsertPermission.class) - public class ImportXarFileAction extends FormHandlerAction - { - @Override - public void validateCommand(ImportXarForm target, Errors errors) - { - } - - @Override - public boolean handlePost(ImportXarForm form, BindException errors) throws Exception - { - for (File f : form.getValidatedFiles(getContainer())) - { - if (f.isFile()) - { - ExperimentPipelineJob job = new ExperimentPipelineJob(getViewBackgroundInfo(), f.toPath(), "Experiment Import", false, form.getPipeRoot(getContainer())); - - // TODO: Configure module resources with the appropriate log location per container - if (form.getModule() != null) - { - FileLike logFile = form.getPipeRoot(getContainer()).getLogDirectoryFileLike(true).resolveChild("module-resource-xar.log"); - job.setLogFile(logFile.toNioPathForWrite()); - } - - PipelineService.get().queueJob(job); - } - else - { - throw new NotFoundException("Expected a file but found a directory: " + f.getName()); - } - } - - return true; - } - - @Override - public URLHelper getSuccessURL(ImportXarForm importXarForm) - { - return getContainer().getStartURL(getUser()); - } - } - - - @RequiresPermission(InsertPermission.class) - public class ImportXarAction extends MutatingApiAction - { - @Override - public Object execute(ImportXarForm form, BindException errors) throws Exception - { - ApiSimpleResponse response = new ApiSimpleResponse(); - - List> archives = new ArrayList<>(); - for (File f : form.getValidatedFiles(getContainer())) - { - Map archive = new HashMap<>(); - ExperimentPipelineJob job = new ExperimentPipelineJob(getViewBackgroundInfo(), f.toPath(), "Experiment Import", false, form.getPipeRoot(getContainer())); - - // TODO: Configure module resources with the appropriate log location per container - if (form.getModule() != null) - { - FileLike logFile = form.getPipeRoot(getContainer()).getLogDirectoryFileLike(true).resolveChild("module-resource-xar.log"); - job.setLogFile(logFile.toNioPathForWrite()); - } - - PipelineService.get().queueJob(job); - - archive.put("file", f.getName()); - archive.put("job", job.getJobGUID()); - archive.put("path", form.getPath()); // echo back the public path - - archives.add(archive); - } - - response.put("success", true); - response.put("archives", archives); - - return response; - } - } - - - /** - * User: jeckels - * Date: Jan 27, 2008 - */ - public static class ExperimentUrlsImpl implements ExperimentUrls - { - public ActionURL getOverviewURL(Container c) - { - return new ActionURL(BeginAction.class, c); - } - - @Override - public ActionURL getExperimentDetailsURL(Container c, ExpExperiment expExperiment) - { - return new ActionURL(DetailsAction.class, c).addParameter("rowId", expExperiment.getRowId()); - } - - public ActionURL getShowSampleURL(Container c, ExpMaterial material) - { - return getMaterialDetailsBaseURL(c, null).addParameter("rowId", material.getRowId()); - } - - @Override - public ActionURL getExportProtocolURL(Container container, ExpProtocol protocol) - { - return new ActionURL(ExperimentController.ExportProtocolsAction.class, container). - addParameter("protocolId", protocol.getRowId()). - addParameter("xarFileName", protocol.getName() + ".xar"); - } - - @Override - public ActionURL getMoveRunsLocationURL(Container container) - { - return new ActionURL(ExperimentController.MoveRunsLocationAction.class, container); - } - - @Override - public ActionURL getProtocolDetailsURL(ExpProtocol protocol) - { - return new ActionURL(ProtocolDetailsAction.class, protocol.getContainer()).addParameter("rowId", protocol.getRowId()); - } - - @Override - public ActionURL getProtocolApplicationDetailsURL(ExpProtocolApplication app) - { - return getShowApplicationURL(app.getContainer(), app.getRowId()); - } - - public ActionURL getProtocolGridURL(Container c) - { - return new ActionURL(ShowProtocolGridAction.class, c); - } - - public ActionURL getRunGraphDetailURL(ExpRun run) - { - return getShowRunGraphDetailURL(run.getContainer(), run.getRowId()); - } - - @Override - public ActionURL getRunGraphDetailURL(ExpRun run, @Nullable ExpData focus) - { - return getRunGraphDetailURL(run, focus, DotGraph.TYPECODE_DATA); - } - - public ActionURL getRunGraphDetailURL(ExpRun run, @Nullable ExpMaterial focus) - { - return getRunGraphDetailURL(run, focus, DotGraph.TYPECODE_MATERIAL); - } - - public ActionURL getRunGraphDetailURL(ExpRun run, @Nullable ExpProtocolApplication focus) - { - return getRunGraphDetailURL(run, focus, DotGraph.TYPECODE_PROT_APP); - } - - private ActionURL getRunGraphDetailURL(ExpRun run, @Nullable ExpObject focus, String typeCode) - { - ActionURL result = getShowRunGraphDetailURL(run.getContainer(), run.getRowId()); - result.addParameter("detail", "true"); - if (focus != null) - { - result.addParameter("focus", typeCode + focus.getRowId()); - } - return result; - } - - @Override - public ActionURL getRunGraphURL(Container container, long runId) - { - return ExperimentController.getRunGraphURL(container, runId); - } - - @Override - public ActionURL getRunGraphURL(ExpRun run) - { - return getRunGraphURL(run.getContainer(), run.getRowId()); - } - - @Override - public ActionURL getRunTextURL(Container c, long runId) - { - return new ActionURL(ShowRunTextAction.class, c).addParameter("rowId", runId); - } - - @Override - public ActionURL getRunTextURL(ExpRun run) - { - return getRunTextURL(run.getContainer(), run.getRowId()); - } - - @Override - public ActionURL getDeleteExperimentsURL(Container container, URLHelper returnUrl) - { - return new ActionURL(DeleteSelectedExperimentsAction.class, container).addReturnUrl(returnUrl); - } - - @Override - public ActionURL getDeleteProtocolURL(@NotNull ExpProtocol protocol, URLHelper returnUrl) - { - ActionURL result = new ActionURL(DeleteProtocolByRowIdsAction.class, protocol.getContainer()); - result.addParameter("singleObjectRowId", protocol.getRowId()); - if (returnUrl != null) - { - result.addReturnUrl(returnUrl); - } - return result; - } - - @Override - public ActionURL getAddRunsToExperimentURL(Container c, ExpExperiment exp) - { - return new ActionURL(AddRunsToExperimentAction.class, c).addParameter("expRowId", exp.getRowId()); - } - - @Override - public ActionURL getShowRunsURL(Container c, ExperimentRunType type) - { - ActionURL result = new ActionURL(ShowRunsAction.class, c); - result.addParameter("experimentRunFilter", type.getDescription()); - return result; - } - - public ActionURL getShowExperimentsURL(Container c) - { - return new ActionURL(ShowRunGroupsAction.class, c); - } - - @Override - public ActionURL getShowSampleTypeListURL(Container c) - { - return getShowSampleTypeListURL(c, null); - } - - @Override - public ActionURL getShowSampleTypeURL(ExpSampleType sampleType) - { - return getShowSampleTypeURL(sampleType, sampleType.getContainer()); - } - - @Override - public ActionURL getShowSampleTypeURL(ExpSampleType sampleType, Container container) - { - return new ActionURL(ShowSampleTypeAction.class, container).addParameter("rowId", sampleType.getRowId()); - } - - public ActionURL getExperimentListURL(Container container) - { - return new ActionURL(ShowRunGroupsAction.class, container); - } - - public ActionURL getShowSampleTypeListURL(Container c, String errorMessage) - { - ActionURL url = new ActionURL(ListSampleTypesAction.class, c); - if (errorMessage != null) - { - url.addParameter("errorMessage", errorMessage); - } - return url; - } - - @Override - public ActionURL getDataClassListURL(Container c) - { - return getDataClassListURL(c, null); - } - - public ActionURL getDataClassListURL(Container c, String errorMessage) - { - ActionURL url = new ActionURL(ListDataClassAction.class, c); - if (errorMessage != null) - { - url.addParameter("errorMessage", errorMessage); - } - return url; - } - - @Override - public ActionURL getDeleteDatasURL(Container c, URLHelper returnUrl) - { - ActionURL url = new ActionURL(DeleteSelectedDataAction.class, c); - if (returnUrl != null) - url.addReturnUrl(returnUrl); - return url; - } - - public ActionURL getDeleteSelectedExperimentsURL(Container c, URLHelper returnUrl) - { - ActionURL result = new ActionURL(DeleteSelectedExperimentsAction.class, c); - if (returnUrl != null) - result.addReturnUrl(returnUrl); - return result; - } - - @Override - public ActionURL getDeleteSelectedExpRunsURL(Container container, URLHelper returnUrl) - { - return new ActionURL(DeleteSelectedExpRunsAction.class, container).addReturnUrl(returnUrl); - } - - public ActionURL getShowUpdateURL(ExpExperiment experiment) - { - return new ActionURL(ShowUpdateAction.class, experiment.getContainer()).addParameter("rowId", experiment.getRowId()); - } - - @Override - public ActionURL getRemoveSelectedExpRunsURL(Container container, URLHelper returnUrl, ExpExperiment exp) - { - return new ActionURL(RemoveSelectedExpRunsAction.class, container).addReturnUrl(returnUrl).addParameter("expRowId", exp.getRowId()); - } - - @Override - public ActionURL getCreateRunGroupURL(Container container, URLHelper returnUrl, boolean addSelectedRuns) - { - ActionURL result = new ActionURL(CreateRunGroupAction.class, container); - if (returnUrl != null) - { - result.addReturnUrl(returnUrl); - } - if (addSelectedRuns) - { - result.addParameter("addSelectedRuns", "true"); - } - return result; - } - - - public static ExperimentUrlsImpl get() - { - return (ExperimentUrlsImpl) urlProvider(ExperimentUrls.class); - } - - public ActionURL getDownloadGraphURL(ExpRun run, boolean detail, String focus, String focusType) - { - ActionURL result = new ActionURL(DownloadGraphAction.class, run.getContainer()); - result.addParameter("rowId", run.getRowId()).addParameter("detail", detail); - if (focus != null) - { - result.addParameter("focus", focus); - } - if (focusType != null) - { - result.addParameter("focusType", focusType); - } - return result; - } - - public ActionURL getBeginURL(Container container) - { - return new ActionURL(BeginAction.class, container); - } - - @Override - public ActionURL getDomainEditorURL(Container container, String domainURI, boolean createOrEdit) - { - Domain domain = PropertyService.get().getDomain(container, domainURI); - if (domain != null) - return getDomainEditorURL(container, domain); - - ActionURL url = new ActionURL(PropertyController.EditDomainAction.class, container); - url.addParameter("domainURI", domainURI); - if (createOrEdit) - url.addParameter("createOrEdit", true); - return url; - } - - @Override - public ActionURL getDomainEditorURL(Container container, Domain domain) - { - ActionURL url = new ActionURL(PropertyController.EditDomainAction.class, container); - url.addParameter("domainId", domain.getTypeId()); - return url; - } - - @Override - public ActionURL getCreateDataClassURL(Container container) - { - return new ActionURL(EditDataClassAction.class, container); - } - - @Override - public ActionURL getShowDataClassURL(Container container, long rowId) - { - ActionURL url = new ActionURL(ShowDataClassAction.class, container); - url.addParameter("rowId", rowId); - return url; - } - - @Override - public ActionURL getShowFileURL(ExpData data, boolean inline) - { - ActionURL result = getShowFileURL(data.getContainer()).addParameter("rowId", data.getRowId()); - if (inline) - { - result.addParameter("inline", inline); - } - return result; - } - - @Override - public ActionURL getMaterialDetailsURL(ExpMaterial material) - { - return getMaterialDetailsURL(material.getContainer(), material.getRowId()); - } - - @Override - public ActionURL getMaterialDetailsURL(Container c, long materialRowId) - { - return getMaterialDetailsBaseURL(c, null).addParameter("rowId", materialRowId); - } - - @Override - public ActionURL getMaterialDetailsBaseURL(Container c, @Nullable String materialIdFieldKey) - { - return new ActionURL(ShowMaterialAction.class, c); - } - - @Override - public ActionURL getCreateSampleTypeURL(Container container) - { - return new ActionURL(EditSampleTypeAction.class, container); - } - - @Override - public ActionURL getImportSamplesURL(Container container, String sampleTypeName) - { - ActionURL url = new ActionURL(ImportSamplesAction.class, container); - url.addParameter("query.queryName", sampleTypeName); - url.addParameter("schemaName", "exp.materials"); - return url; - } - - @Override - public ActionURL getImportDataURL(Container container, String dataClassName) - { - ActionURL url = new ActionURL(ImportDataAction.class, container); - url.addParameter("query.queryName", dataClassName); - url.addParameter("schemaName", "exp.data"); - return url; - } - - @Override - public ActionURL getDataDetailsURL(ExpData data) - { - return new ActionURL(ShowDataAction.class, data.getContainer()).addParameter("rowId", data.getRowId()); - } - - @Override - public ActionURL getShowFileURL(Container c) - { - return new ActionURL(ShowFileAction.class, c); - } - - @Override - public ActionURL getSetFlagURL(Container container) - { - return new ActionURL(SetFlagAction.class, container); - } - - @Override - public ActionURL getShowRunGraphURL(ExpRun run) - { - return ExperimentController.getRunGraphURL(run.getContainer(), run.getRowId()); - } - - @Override - public ActionURL getRepairTypeURL(Container container) - { - return new ActionURL(TypesController.RepairAction.class, container); - } - - @Override - public ActionURL getUpdateMaterialQueryRowAction(Container c, TableInfo table) - { - ActionURL url = new ActionURL(UpdateMaterialQueryRowAction.class, c); - url.addParameter("schemaName", "samples"); - url.addParameter(QueryView.DATAREGIONNAME_DEFAULT + "." + QueryParam.queryName, table.getName()); - - return url; - } - - @Override - public ActionURL getInsertMaterialQueryRowAction(Container c, TableInfo table) - { - ActionURL url = new ActionURL(InsertMaterialQueryRowAction.class, c); - url.addParameter("schemaName", "samples"); - url.addParameter(QueryView.DATAREGIONNAME_DEFAULT + "." + QueryParam.queryName, table.getName()); - - return url; - } - - @Override - public ActionURL getDataClassAttachmentDownloadAction(Container c) - { - return new ActionURL(ExperimentController.DataClassAttachmentDownloadAction.class, c); - } - - } - - private static abstract class BaseResolveLsidApiAction extends ReadOnlyApiAction - { - protected Set _seeds; - - @Override - public void validateForm(F form, Errors errors) - { - if (null != form.getLsids()) - { - _seeds = new LinkedHashSet<>(form.getLsids().size()); - for (String lsid : form.getLsids()) - { - Identifiable id = LsidManager.get().getObject(lsid); - if (id == null) - throw new NotFoundException("Unable to resolve object: " + lsid); - - // ensure the user has read permission in the seed container - if (!getContainer().equals(id.getContainer())) - { - if (!id.getContainer().hasPermission(getUser(), ReadPermission.class)) - throw new UnauthorizedException("User does not have permission to read object: " + lsid); - } - - _seeds.add(id); - } - } - else - { - throw new ApiUsageException("Starting lsids required"); - } - } - } - - @RequiresPermission(ReadPermission.class) - public static class ResolveAction extends BaseResolveLsidApiAction - { - @Override - public Object execute(ResolveLsidsForm form, BindException errors) - { - var settings = new ExperimentJSONConverter.Settings(form.isIncludeProperties(), form.isIncludeInputsAndOutputs(), form.isIncludeRunSteps()); - var data = _seeds.stream().map(n -> ExperimentJSONConverter.serialize(n, getUser(), settings)).collect(toList()); - return new ApiSimpleResponse("data", data); - } - } - - @RequiresPermission(ReadPermission.class) - public static class LineageAction extends BaseResolveLsidApiAction - { - @Override - public Object execute(ExpLineageOptions options, BindException errors) throws Exception - { - ExpLineageServiceImpl.get().streamLineage(getContainer(), getUser(), getViewContext().getResponse(), _seeds, options); - return null; - } - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(AdminPermission.class) - public static class RebuildEdgesAction extends MutatingApiAction - { - @Override - public Object execute(ExperimentRunForm form, BindException errors) - { - if (form.getRowId() != 0 || form.getLsid() != null) - { - ExpRun run = form.lookupRun(); - if (!run.getContainer().hasPermission(getUser(), ReadPermission.class)) - throw new UnauthorizedException("Not permitted"); - - ExperimentServiceImpl.get().syncRunEdges(run); - } - else - { - // should this require site admin permissions? - ExperimentServiceImpl.get().rebuildAllRunEdges(); - } - return success(); - } - } - - private static class VerifyEdgesForm extends ExperimentRunForm - { - private Integer _limit; - - public Integer getLimit() - { - return _limit; - } - - public void setLimit(Integer limit) - { - _limit = limit; - } - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(AdminPermission.class) - public static class VerifyEdgesAction extends ReadOnlyApiAction - { - @Override - public Object execute(VerifyEdgesForm form, BindException errors) - { - if (form.getRowId() != 0 || form.getLsid() != null) - { - ExpRun run = form.lookupRun(); - if (!run.getContainer().hasPermission(getUser(), ReadPermission.class)) - throw new UnauthorizedException("Not permitted"); - - ExperimentServiceImpl.get().verifyRunEdges(run); - } - else - { - ExperimentServiceImpl.get().verifyAllEdges(getContainer(), form.getLimit()); - } - return success(); - } - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(AdminPermission.class) - public static class RebuildAncestorsAction extends MutatingApiAction - { - @Override - public Object execute(Object form, BindException errors) - { - ClosureQueryHelper.truncateAndRecreate(); - return success(); - } - } - - - @Marshal(Marshaller.Jackson) - @RequiresPermission(AdminPermission.class) - public static class CheckDataClassesIndexedAction extends ReadOnlyApiAction - { - @Override - public Object execute(Object o, BindException errors) throws Exception - { - SearchService search = SearchService.get(); - if (search == null) - return null; - - List> notInIndex = new ArrayList<>(100); - - List list = ExperimentService.get().getDataClasses(getContainer(), getUser(), false); - for (ExpDataClass dc : list) - { - for (ExpData d : dc.getDatas()) - { - String docId = d.getDocumentId(); - if (docId != null) - { - SearchService.SearchHit hit = search.find(docId); - if (hit == null) - { - JSONObject props = ExperimentJSONConverter.serializeData(d, getUser(), ExperimentJSONConverter.DEFAULT_SETTINGS); - props.put("docid", docId); - notInIndex.add(props.toMap()); - } - } - } - } - - return success(notInIndex); - } - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(AdminPermission.class) - public static class CheckEdgesAction extends ReadOnlyApiAction - { - @Override - public Object execute(Object o, BindException errors) throws Exception - { - List result; - DbSchema schema = ExperimentService.get().getSchema(); - TableInfo edgeTable = schema.getTable("Edge"); - - if (null != edgeTable.getColumn("fromObjectId")) - { - var edges = new SqlSelector(ExperimentService.get().getSchema(), "SELECT fromObjectId, toObjectId FROM exp.Edge") - .resultSetStream() - .map(r -> { try { return new Pair<>(r.getInt(1), r.getInt(2)); } catch (SQLException x) { throw new RuntimeException(x); } }) - .collect(toList()); - var cycles = (new GraphAlgorithms()).detectCycleInDirectedGraph(edges); - result = cycles.stream().map(e -> new Integer[]{e.first, e.second}).collect(toList()); - } - else - { - var edges = new SqlSelector(ExperimentService.get().getSchema(), "SELECT fromLsid, toLsid FROM exp.Edge") - .resultSetStream() - .map(r -> { try { return new Pair<>(r.getString(1), r.getString(2)); } catch (SQLException x) { throw new RuntimeException(x); } }) - .collect(toList()); - var cycles = (new GraphAlgorithms()).detectCycleInDirectedGraph(edges); - result = cycles.stream().map(e -> new String[]{e.first, e.second}).collect(toList()); - } - - JSONObject ret = new JSONObject(); - ret.put("result", result); - ret.put("success", true); - return ret; - } - } - - @RequiresPermission(UpdatePermission.class) - public static class UpdateMaterialQueryRowAction extends UserSchemaAction - { - @Override - public BindException bindParameters(PropertyValues m) throws Exception - { - BindException bind = super.bindParameters(m); - - QueryUpdateForm tableForm = (QueryUpdateForm)bind.getTarget(); - - int sampleId; - try - { - sampleId = Integer.parseInt((String) tableForm.getPkVal()); - } - catch (NumberFormatException e) - { - throw new NotFoundException("Invalid RowId: " + tableForm.getPkVal()); - } - - ExpMaterial material = ExperimentService.get().getExpMaterial(sampleId); - if (material == null) - throw new NotFoundException("Invalid material: " + tableForm.getPkVal()); - - return bind; - } - - @Override - public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) - { - int sampleId = Integer.parseInt((String) tableForm.getPkVal()); - - ExpMaterial material = ExperimentService.get().getExpMaterial(sampleId); - if (material == null) - throw new NotFoundException("Invalid material: " + tableForm.getPkVal()); - - boolean isAliquot = !StringUtils.isEmpty(material.getAliquotedFromLSID()); - - TableInfo tableInfo = tableForm.getTable(); - Map scopedFields = new CaseInsensitiveHashMap<>(); - for (DomainProperty dp : tableInfo.getDomain().getProperties()) - { - if (!ExpSchema.DerivationDataScopeType.All.name().equalsIgnoreCase(dp.getDerivationDataScope())) - scopedFields.put(dp.getName(), ExpSchema.DerivationDataScopeType.ChildOnly.name().equalsIgnoreCase(dp.getDerivationDataScope())); - } - - for (var column : tableInfo.getColumns()) - { - String columnName = column.getName(); - if (scopedFields.containsKey(columnName)) - { - boolean isAliquotField = scopedFields.get(columnName); - boolean show = (isAliquot && isAliquotField) || (!isAliquot && !isAliquotField); - ((BaseColumnInfo)column).setUserEditable(show); - ((BaseColumnInfo)column).setHidden(!show); - } - } - - ButtonBar bb = createSubmitCancelButtonBar(tableForm); - UpdateView view = new UpdateView(tableForm, errors); - view.getDataRegion().setButtonBar(bb); - return view; - } - - @Override - public boolean handlePost(QueryUpdateForm tableForm, BindException errors) - { - doInsertUpdate(tableForm, errors, false); - return 0 == errors.getErrorCount(); - } - - @Override - public void addNavTrail(NavTree root) - { - super.addNavTrail(root); - root.addChild("Edit " + _form.getQueryName()); - } - } - - @RequiresPermission(InsertPermission.class) - public static class InsertMaterialQueryRowAction extends UserSchemaAction - { - @Override - public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) - { - TableInfo tableInfo = tableForm.getTable(); - Map propertyFields = new CaseInsensitiveHashMap<>(); - for (DomainProperty dp : tableInfo.getDomain().getProperties()) - { - propertyFields.put(dp.getName(), ExpSchema.DerivationDataScopeType.ChildOnly.name().equalsIgnoreCase(dp.getDerivationDataScope())); - } - - for (var column : tableInfo.getColumns()) - { - String columnName = column.getName(); - if (propertyFields.containsKey(columnName)) - { - boolean isAliquotField = propertyFields.get(columnName); - ((BaseColumnInfo)column).setUserEditable(!isAliquotField); - ((BaseColumnInfo)column).setHidden(isAliquotField); - } - } - - InsertView view = new InsertView(tableForm, errors); - view.getDataRegion().setButtonBar(createSubmitCancelButtonBar(tableForm)); - return view; - } - - @Override - public boolean handlePost(QueryUpdateForm tableForm, BindException errors) - { - doInsertUpdate(tableForm, errors, true); - return 0 == errors.getErrorCount(); - } - - @Override - public void addNavTrail(NavTree root) - { - super.addNavTrail(root); - root.addChild("Insert " + _form.getQueryName()); - } - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(ReadPermission.class) - public static class SaveFindIdsAction extends ReadOnlyApiAction - { - - public static final String FIND_BY_IDS_SESSION_KEY_PREFIX = "findByIds"; - - @Override - public Object execute(FindByIdsForm form, BindException errors) throws Exception - { - HttpServletRequest request = getViewContext().getRequest(); - String key = form.getSessionKey(); - boolean removePrevious = false; - - if (key == null) - { - removePrevious = true; - key = FIND_BY_IDS_SESSION_KEY_PREFIX + "_" + UniqueID.getServerSessionScopedUID(); - } - - if (request != null) - { - if (removePrevious) - SessionHelper.clearAttributesWithPrefix(request, FIND_BY_IDS_SESSION_KEY_PREFIX); - HttpSession session = request.getSession(false); - if (session != null) - { - @SuppressWarnings("unchecked") - List existingIds = (List) session.getAttribute(key); - - // deduplicate from existing ids - if (existingIds != null && form.getSessionKey() != null) - { - existingIds.addAll(form.getIds().stream().filter(id -> !existingIds.contains(id)).toList()); - session.setAttribute(key, existingIds); - } - else - { - session.setAttribute(key, form.getIds()); - } - return success("Saved ids to session key", key); - } - } - - return new SimpleResponse<>(false, "Unable to save to session. Session or request may be null."); - } - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(ReadPermission.class) - public static class SaveOrderedSamplesQueryAction extends ReadOnlyApiAction - { - private static final String SAMPLE_ID_PREFIX = "s:"; - private static final String UNIQUE_ID_PREFIX = "u:"; - - private List _ids; - private Map> _uniqueIdLsids; - - @Override - public void validateForm(FindByIdsForm form, Errors errors) - { - if (form.getSessionKey() == null) - errors.reject(ERROR_REQUIRED, "sessionKey must be provided"); - else - { - _ids = getFindIdsFromSession(form.getSessionKey()); - if (_ids == null || _ids.isEmpty()) - errors.reject(ERROR_REQUIRED, "No ids found corresponding to session key " + form.getSessionKey()); - } - } - - private void ensureUniqueIdLsids() - { - boolean hasUniqueId = _ids.stream().anyMatch(s -> s.startsWith(UNIQUE_ID_PREFIX)); - if (hasUniqueId && _uniqueIdLsids == null) - { - List uniqueIds = _ids.stream().map(s -> s.substring(UNIQUE_ID_PREFIX.length())).toList(); - _uniqueIdLsids = ExperimentService.get().getUniqueIdLsids(uniqueIds, getUser(), getContainer()); - } - } - - @Override - public Object execute(FindByIdsForm form, BindException errors) throws Exception - { - ensureUniqueIdLsids(); - - SQLFragment select = getOrderedRowsSql(); - // need to set the key field so selections are possible - // need the SampleTypeUnits so we will display using that unit - String metadata = - """ - - - - - true - true - - - true - - - true - - -
    -     """; - QueryDefinition def = QueryService.get().saveSessionQuery(getViewContext(), getContainer(), ExperimentServiceImpl.getExpSchema().getName(), select.getSQL(), metadata); - return success("Session query created", Map.of("queryName", def.getName(), "ids", _ids)); - } - - - private List getFindIdsFromSession(String sessionKey) - { - HttpServletRequest request = getViewContext().getRequest(); - List ids = new ArrayList<>(); - if (request != null) - { - HttpSession session = request.getSession(false); - if (session != null) - { - ids = (List) session.getAttribute(sessionKey); - } - } - return ids; - } - - private SQLFragment getOrderedRowsSql() - { - boolean isFMEnabled = InventoryService.isFreezerManagementEnabled(getContainer()); - String samplesTable = isFMEnabled ? "inventory.SampleItems" : "exp.materials"; - List orderedIdCols = new ArrayList<>(Arrays.asList("Id AS ProvidedID", "RowId", "Ordinal")); - List sampleColumns = new ArrayList<>(); - if (!isFMEnabled) - { - sampleColumns.addAll(Arrays.asList( - "S.Name AS SampleID", - "S.MaterialExpDate AS ExpirationDate", - "S.SampleSet as SampleType", - "S.SampleState", - "S.isAliquot", - "S.Created", - "S.CreatedBy" - )); - } - else - { - sampleColumns.addAll(Arrays.asList( - "S.Name AS SampleID", - "S.MaterialExpDate AS ExpirationDate", - "S.LabelColor", - "S.SampleSet", - "S.SampleState", - "S.StoredAmount", - "S.Units", - "S.SampleTypeUnits", - "S.FreezeThawCount", - "S.StorageStatus", - "S.CheckedOutBy", - "S.StorageLocation", - "S.StorageRow", - "S.StorageCol", - "S.StoragePositionNumber", - "S.IsAliquot", - "S.Created", - "S.CreatedBy" - )); - } - - - String sampleIdComma = ""; - String uniqueIdComma = ""; - int index = 1; - SQLFragment sampleIdValuesSql = new SQLFragment(); - SQLFragment uniqueIdValuesSql = new SQLFragment(); - for (String id : _ids) - { - if (id.startsWith(SAMPLE_ID_PREFIX)) - { - sampleIdValuesSql.append(sampleIdComma).append("\t(").appendValue(index); - sampleIdValuesSql.append(", "); - sampleIdValuesSql.append(LabKeySql.quoteString(id.substring(SAMPLE_ID_PREFIX.length()))); - sampleIdValuesSql.append(", "); - sampleIdValuesSql.append(LabKeySql.quoteString("null")); - sampleIdValuesSql.append(")"); - sampleIdComma = "\n,"; - } - else if (id.startsWith(UNIQUE_ID_PREFIX)) - { - String idClean = id.substring(UNIQUE_ID_PREFIX.length()); - - List lsids = _uniqueIdLsids.get(idClean); - if (lsids != null) - { - for (String lsid : lsids) - { - uniqueIdValuesSql.append(uniqueIdComma).append("\t(").appendValue(index); - uniqueIdValuesSql.append(", "); - uniqueIdValuesSql.append(LabKeySql.quoteString(idClean)); - uniqueIdValuesSql.append(", "); - uniqueIdValuesSql.append(LabKeySql.quoteString(lsid)); - uniqueIdValuesSql.append(")"); - uniqueIdComma = "\n,"; - } - } - } - index++; - } - - boolean haveData = !sampleIdValuesSql.isEmpty() || !_uniqueIdLsids.isEmpty(); - SQLFragment sql = new SQLFragment(); - if (!sampleIdValuesSql.isEmpty()) - { - sql.append("WITH _ordered_ids_ AS (\nSELECT * FROM (VALUES\n"); - sql.append(sampleIdValuesSql); - sql.append("\n) AS _values_ )\n"); // name of the alias here doesn't matter - } - if (!uniqueIdValuesSql.isEmpty()) - { - if (!sampleIdValuesSql.isEmpty()) - sql.append(",\n"); - else - sql.append("WITH "); - - sql.append("_ordered_unique_ids_ AS (\nSELECT * FROM (VALUES\n"); - sql.append(uniqueIdValuesSql); - sql.append("\n) AS _values_ )\n"); // name of the alias here doesn't matter - } - - sql.append("SELECT "); - sql.append("\n\tOID.").append(StringUtils.join(orderedIdCols, ",\n\tOID.")); - sql.append(",\n\t").append(StringUtils.join( sampleColumns, ",\n\t")); - sql.append("\nFROM\n("); - if (!sampleIdValuesSql.isEmpty()) - { - sql.append("SELECT\n\tM.RowId,\n\t_ordered_ids_.column1 as Ordinal,\n\t_ordered_ids_.column2 as Id,\n\t_ordered_ids_.column2 as lsid"); - sql.append("\nFROM _ordered_ids_\n"); - sql.append("INNER JOIN exp.materials M ON _ordered_ids_.column2 = M.Name"); - sql.append("\n"); - } - if (!uniqueIdValuesSql.isEmpty()) - { - if (!sampleIdValuesSql.isEmpty()) - sql.append("\nUNION ALL\n\n"); - - sql.append("SELECT\n\tM.RowId,\n\t_ordered_unique_ids_.column1 as Ordinal,\n\t_ordered_unique_ids_.column2 as Id,\n\t_ordered_unique_ids_.column3 as lsid"); - sql.append("\nFROM _ordered_unique_ids_\n"); - sql.append("INNER JOIN exp.materials M ON _ordered_unique_ids_.column3 = M.lsid"); - sql.append("\n"); - } - if (!haveData) // no data to return but return data in the expected shape. - { - sql = new SQLFragment("SELECT\n"); - sql.append(orderedIdCols.stream() - .map(col -> { - int asIndex = col.indexOf("AS"); - if (asIndex > 0) - return "NULL AS " + col.substring(asIndex+ 3); - else - return "NULL AS " + col; - }) - .collect(Collectors.joining(",\t\n"))); - sql.append(",\t\n").append(StringUtils.join(sampleColumns, ",\t\n")); - sql.append("\nFROM ").append(samplesTable).append(" S WHERE 1 = 2"); - return sql; - } - else - { - sql.append(") OID"); - if (isFMEnabled) - sql.append("\nLEFT JOIN inventory.SampleItems S on S.RowId = OID.RowId"); - else - sql.append("\nINNER JOIN exp.materials S on S.RowId = OID.RowId"); - sql.append("\n\nORDER BY Ordinal"); - return sql; - } - } - } - - public static class FindByIdsForm extends FindSessionKeyForm - { - List _ids; - - public List getIds() - { - return _ids; - } - - public void setIds(List ids) - { - _ids = ids; - } - } - - - public static class FindSessionKeyForm - { - private String _sessionKey; - - public String getSessionKey() - { - return _sessionKey; - } - - public void setSessionKey(String sessionKey) - { - _sessionKey = sessionKey; - } - } - - static void validateEntitySequenceForm(EntitySequenceForm form, Errors errors) - { - String kindName = form.getKindName(); - if (StringUtils.isEmpty(kindName) || form.getSeqType() == null) - { - errors.reject(ERROR_REQUIRED, "KindName and SeqType must be provided"); - return; - } - - if (form.getSeqType() == NameGenerator.EntityCounter.genId) - { - if (form.getRowId() == null) - errors.reject(ERROR_REQUIRED, "Data type RowId must be provided for genId"); - } - else if (!SampleTypeDomainKind.NAME.equalsIgnoreCase(kindName)) - { - errors.reject(ERROR_MSG, form.getSeqType() + " is not supported for " + kindName); - } - - if (!SampleTypeDomainKind.NAME.equalsIgnoreCase(kindName) && !DataClassDomainKind.NAME.equalsIgnoreCase(kindName)) - errors.reject(ERROR_MSG, "Invalid KindName. Should be either " + SampleTypeDomainKind.NAME + " or " + DataClassDomainKind.NAME + "."); - - } - - @RequiresPermission(ReadPermission.class) - public static class GetEntitySequenceAction extends ReadOnlyApiAction - { - @Override - public void validateForm(EntitySequenceForm form, Errors errors) - { - validateEntitySequenceForm(form, errors); - } - - @Override - public Object execute(EntitySequenceForm form, BindException errors) throws Exception - { - long value = -1; - if (SampleTypeDomainKind.NAME.equalsIgnoreCase(form.getKindName())) - { - if (form.getSeqType() == NameGenerator.EntityCounter.genId) - { - ExpSampleType sampleType = SampleTypeService.get().getSampleType(form.getRowId()); - if (sampleType != null) - value = sampleType.getCurrentGenId(); - } - else - { - value = SampleTypeService.get().getCurrentCount(form.getSeqType(), getContainer()); - } - - } - else if (DataClassDomainKind.NAME.equalsIgnoreCase(form.getKindName())) - { - ExpDataClass dataClass = ExperimentService.get().getDataClass(form.getRowId()); - if (dataClass != null) - value = dataClass.getCurrentGenId(); - } - - ApiSimpleResponse resp = new ApiSimpleResponse(); - resp.put("success", value > -1); - resp.put("value", value); - return resp; - } - } - - @RequiresPermission(ReadPermission.class) // actual permission checked later - public static class SetEntitySequenceAction extends MutatingApiAction - { - @Override - public void validateForm(EntitySequenceForm form, Errors errors) - { - validateEntitySequenceForm(form, errors); - - if (form.getNewValue() == null || form.getNewValue() < 0) - errors.reject(ERROR_MSG, "Invalid newValue."); - } - - @Override - public Object execute(EntitySequenceForm form, BindException errors) - { - ApiSimpleResponse resp = new ApiSimpleResponse(); - resp.put("success", true); - - try - { - Domain domain = null; - if (SampleTypeDomainKind.NAME.equalsIgnoreCase(form.getKindName())) - { - if (form.getSeqType() == NameGenerator.EntityCounter.genId) - { - if (!getContainer().hasPermission(getUser(), DesignSampleTypePermission.class)) - throw new UnauthorizedException("Insufficient permissions."); - - ExpSampleType sampleType = SampleTypeService.get().getSampleType(form.getRowId()); - if (sampleType != null) - { - sampleType.ensureMinGenId(form.getNewValue()); - domain = sampleType.getDomain(); - } - else - { - resp.put("success", false); - resp.put("error", "Sample type does not exist."); - } - } - else - { - if (!getContainer().hasPermission(getUser(), AdminPermission.class)) - throw new UnauthorizedException("Insufficient permissions."); - - SampleTypeService.get().ensureMinSampleCount(form.getNewValue(), form.getSeqType(), getContainer()); - } - } - else if (DataClassDomainKind.NAME.equalsIgnoreCase(form.getKindName())) - { - if (!getContainer().hasPermission(getUser(), DesignDataClassPermission.class)) - throw new BadRequestException("Insufficient permissions."); - - ExpDataClass dataClass = ExperimentService.get().getDataClass(form.getRowId()); - if (dataClass != null) - { - dataClass.ensureMinGenId(form.getNewValue(), getContainer()); - domain = dataClass.getDomain(); - } - else - { - resp.put("success", false); - resp.put("error", "DataClass does not exist."); - } - } - - if (domain != null) - { - DomainAuditProvider.DomainAuditEvent event = new DomainAuditProvider.DomainAuditEvent(getContainer(), "The genId for domain " + domain.getName() + " has been updated to " + form.getNewValue() + "."); - event.setDomainUri(domain.getTypeURI()); - event.setDomainName(domain.getName()); - AuditLogService.get().addEvent(getUser(), event); - } - } - catch (ExperimentException e) - { - resp.put("success", false); - resp.put("error", e.getMessage()); - } - - return resp; - } - } - - public static class EntitySequenceForm - { - private String _kindName; - private NameGenerator.EntityCounter _seqType; - private Integer _rowId; - private Long _newValue; - - public Integer getRowId() - { - return _rowId; - } - - public void setRowId(Integer rowId) - { - _rowId = rowId; - } - - public String getKindName() - { - return _kindName; - } - - public void setKindName(String kindName) - { - _kindName = kindName; - } - - public Long getNewValue() - { - return _newValue; - } - - public void setNewValue(Long newValue) - { - this._newValue = newValue; - } - - public NameGenerator.EntityCounter getSeqType() - { - return _seqType; - } - - public void setSeqType(String seqType) - { - _seqType = NameGenerator.EntityCounter.valueOf(seqType); - } - - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(ReadPermission.class) - public static class GetCrossFolderDataSelectionAction extends ReadOnlyApiAction - { - @Override - public void validateForm(CrossFolderSelectionForm form, Errors errors) - { - if (form.getDataRegionSelectionKey() == null && form.getRowIds() == null) - errors.reject(ERROR_REQUIRED, "You must provide either a set of rowIds or a dataRegionSelectionKey."); - if (!"samples".equalsIgnoreCase(form.getDataType()) && !"exp.data".equalsIgnoreCase(form.getDataType())&& !"assay".equalsIgnoreCase(form.getDataType())) - errors.reject(ERROR_REQUIRED, "Data type (sample, data or assayrun) must be specified."); - } - - @Override - public Object execute(CrossFolderSelectionForm form, BindException errors) - { - Pair result = ExperimentServiceImpl.getCurrentAndCrossFolderDataCount(form.getIds(false), form.getDataType(), getContainer()); - - ApiSimpleResponse resp = new ApiSimpleResponse(); - resp.put("success", true); - resp.put("currentFolderSelectionCount", result.first); - resp.put("crossFolderSelectionCount", result.second); - - return success(resp); - } - } - - public static class CrossFolderSelectionForm extends DataViewSnapshotSelectionForm - { - private String _dataType; - private String _picklistName; - - public String getDataType() - { - return _dataType; - } - - public void setDataType(String dataType) - { - _dataType = dataType; - } - - public String getPicklistName() - { - return _picklistName; - } - - public void setPicklistName(String picklistName) - { - _picklistName = picklistName; - } - - @Override - public Set getIds(boolean clear) - { - Set selectedIds; - - if (_rowIds != null) - selectedIds = _rowIds; - else if (isUseSnapshotSelection()) - selectedIds = new HashSet<>(DataRegionSelection.getSnapshotSelectedIntegers(getViewContext(), getDataRegionSelectionKey())); - else - selectedIds = DataRegionSelection.getSelectedIntegers(getViewContext(), getDataRegionSelectionKey(), clear); - - if (_picklistName != null) - { - User user = getViewContext().getUser(); - Container container = getViewContext().getContainer(); - UserSchema schema = ListService.get().getUserSchema(user, container); - TableInfo tInfo = schema.getTable(_picklistName); - if (tInfo != null) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addInClause(FieldKey.fromParts("id"), selectedIds); - TableSelector selector = new TableSelector(tInfo, Collections.singleton("SampleID"), filter, null); - return new HashSet<>(selector.getArrayList(Long.class)); - } - } - return selectedIds; - } - } - - @RequiresPermission(AdminPermission.class) - public static class RecomputeAliquotRollup extends SimpleViewAction - { - @Override - public void addNavTrail(NavTree root) - { - } - - @Override - public ModelAndView getView(Object o, BindException errors) throws SQLException - { - try (var ignore = SpringActionController.ignoreSqlUpdates()) - { - Container container = getContainer(); - User user = getUser(); - - List sampleTypes = SampleTypeService.get() - .getSampleTypes(container, user, true); - - HtmlStringBuilder builder = HtmlStringBuilder.of(); - builder.unsafeAppend(""); - - SampleTypeService service = SampleTypeService.get(); - for (ExpSampleType sampleType : sampleTypes) - { - int updatedCount; - updatedCount = service.recomputeSampleTypeRollup(sampleType, container); - // we could check "if (0 < updatedCount) refresh(rollup)", but since this is a "manual" usage lets just always refresh - SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, update); - builder.unsafeAppend(""); - } - - builder.unsafeAppend("
    Sample Type#Recomputed
    ") - .append(sampleType.getName()) - .unsafeAppend("") - .append(updatedCount) - .unsafeAppend("
    "); - return new HtmlView("Aliquot Rollup Recalculation Result", builder); - } - } - } - - /* Also see API CheckEdgesAction */ - @RequiresPermission(TroubleshooterPermission.class) - public static class CycleCheckAction extends FormViewAction - { - List cycleObjectIds = null; - - @Override - public void validateCommand(Object target, Errors errors) - { - - } - - @Override - public ModelAndView getView(Object o, boolean reshow, BindException errors) - { - if (!reshow) - { - return new HtmlView( - DIV("This operation can use a lot of memory.", - LK.FORM(at(method,"POST"), - PageFlowUtil.button("Continue").submit(true))) - ); - } - - if (null == cycleObjectIds) - return new HtmlView(HtmlString.of("No cycles found")); - - Map map = new LongHashMap<>(); - var cf = new ContainerFilter.AllFolders(getUser()); - var materials = ExperimentServiceImpl.get().getExpMaterialsByObjectId(cf, cycleObjectIds); - materials.forEach( (m) -> map.put(m.getObjectId(), m)); - var datas = ExperimentServiceImpl.get().getExpDatasByObjectId(cf, cycleObjectIds); - datas.forEach( (d) -> map.put(d.getObjectId(), d)); - var runs = ExperimentServiceImpl.get().getRunsByObjectId(cf, cycleObjectIds); - runs.forEach( (r) -> map.put(r.getObjectId(), r)); - - ExperimentUrls urls = ExperimentUrls.get(); - return new HtmlView( - DIV("Cycle found involving these objects.", - UL(cycleObjectIds.stream().map((objectid) -> - { - ExpObject exp = map.get(objectid); - if (exp instanceof ExpMaterial mat) - return LI(A(at(target, "_blank", href, urls.getMaterialDetailsURL(mat)), objectid + " : material - " + mat.getName())); - else if (exp instanceof ExpRun run) - return LI(A(at(target, "_blank", href, urls.getRunTextURL(run)), objectid + " : run - " + run.getName())); - else if (exp instanceof ExpData data) - return LI(A(at(target, "_blank", href, urls.getDataDetailsURL(data)), objectid + " : run - " + data.getName())); - else - return LI(String.valueOf(objectid)); - })) - ) - ); - } - - @Override - public boolean handlePost(Object o, BindException errors) - { - var edges = new SqlSelector(ExperimentService.get().getSchema(), "SELECT fromObjectId, toObjectId FROM exp.Edge") - .resultSetStream() - .map(r -> { try { return new Pair<>(r.getInt(1), r.getInt(2)); } catch (SQLException x) { throw new RuntimeException(x); } }) - .collect(toList()); - var cyclesEdges = (new GraphAlgorithms()).detectCycleInDirectedGraph(edges); - - var set = new LinkedHashSet(); - cyclesEdges.forEach( (edge) -> { - set.add(edge.first); - set.add(edge.second); - }); - cycleObjectIds = set.stream().toList(); - return false; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - - } - } - - @RequiresPermission(AdminPermission.class) - public static class MissingFilesCheckAction extends ReadOnlyApiAction - { - @Override - public Object execute(Object o, BindException errors) throws Exception - { - Map> info = ExperimentServiceImpl.get().doMissingFilesCheck(getUser(), getContainer(), true); - JSONObject results = new JSONObject(); - for (String containerId : info.keySet()) - { - JSONObject containerResults = new JSONObject(); - for (String sourceName : info.get(containerId).keySet()) - containerResults.put(sourceName, info.get(containerId).get(sourceName).toJSON()); - results.put(containerId, containerResults); - } - - ApiSimpleResponse response = new ApiSimpleResponse(); - response.put("success", true); - response.put("result", results); - return response; - } - } - -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.controllers.exp; + +import au.com.bytecode.opencsv.CSVWriter; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpSession; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.poi.openxml4j.exceptions.InvalidFormatException; +import org.apache.poi.ss.usermodel.Workbook; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.labkey.api.action.ApiJsonWriter; +import org.labkey.api.action.ApiResponse; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.action.ExportAction; +import org.labkey.api.action.FormHandlerAction; +import org.labkey.api.action.FormViewAction; +import org.labkey.api.action.HasViewContext; +import org.labkey.api.action.Marshal; +import org.labkey.api.action.Marshaller; +import org.labkey.api.action.MutatingApiAction; +import org.labkey.api.action.QueryViewAction; +import org.labkey.api.action.ReadOnlyApiAction; +import org.labkey.api.action.ReturnUrlForm; +import org.labkey.api.action.SimpleApiJsonForm; +import org.labkey.api.action.SimpleErrorView; +import org.labkey.api.action.SimpleResponse; +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.assay.AssayFileWriter; +import org.labkey.api.assay.AssayProtocolSchema; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayService; +import org.labkey.api.assay.actions.UploadWizardAction; +import org.labkey.api.assay.security.DesignAssayPermission; +import org.labkey.api.attachments.AttachmentParent; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.attachments.BaseDownloadAction; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.data.ActionButton; +import org.labkey.api.data.BaseColumnInfo; +import org.labkey.api.data.ButtonBar; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DataRegion; +import org.labkey.api.data.DataRegionSelection; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DisplayColumn; +import org.labkey.api.data.ExcelWriter; +import org.labkey.api.data.MenuButton; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.ShowRows; +import org.labkey.api.data.SimpleDisplayColumn; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TSVWriter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.exp.AbstractParameter; +import org.labkey.api.exp.DeleteForm; +import org.labkey.api.exp.DuplicateMaterialException; +import org.labkey.api.exp.ExperimentDataHandler; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.ExperimentRunForm; +import org.labkey.api.exp.ExperimentRunListView; +import org.labkey.api.exp.ExperimentRunType; +import org.labkey.api.exp.Identifiable; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.LsidManager; +import org.labkey.api.exp.LsidType; +import org.labkey.api.exp.ObjectProperty; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.ProtocolApplicationParameter; +import org.labkey.api.exp.XarContext; +import org.labkey.api.exp.api.DataClassDomainKindProperties; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpExperiment; +import org.labkey.api.exp.api.ExpLineageOptions; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpMaterialRunInput; +import org.labkey.api.exp.api.ExpObject; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolApplication; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpRunAttachmentParent; +import org.labkey.api.exp.api.ExpRunEditor; +import org.labkey.api.exp.api.ExpRunItem; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.ExperimentUrls; +import org.labkey.api.exp.api.NameExpressionOptionService; +import org.labkey.api.exp.api.ResolveLsidsForm; +import org.labkey.api.exp.api.SampleTypeDomainKind; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainTemplate; +import org.labkey.api.exp.property.DomainTemplateGroup; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.query.ExpDataProtocolInputTable; +import org.labkey.api.exp.query.ExpInputTable; +import org.labkey.api.exp.query.ExpMaterialProtocolInputTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.exp.xar.LSIDRelativizer; +import org.labkey.api.exp.xar.LsidUtils; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.inventory.InventoryService; +import org.labkey.api.module.ModuleHtmlView; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineStatusFile; +import org.labkey.api.pipeline.PipelineUrls; +import org.labkey.api.pipeline.PipelineValidationException; +import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AbstractQueryImportAction; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryAction; +import org.labkey.api.query.QueryDefinition; +import org.labkey.api.query.QueryException; +import org.labkey.api.query.QueryForm; +import org.labkey.api.query.QueryParam; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryUpdateForm; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.UserSchemaAction; +import org.labkey.api.reader.ColumnDescriptor; +import org.labkey.api.reader.DataLoader; +import org.labkey.api.reader.DataLoaderFactory; +import org.labkey.api.reader.ExcelFactory; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.ActionNames; +import org.labkey.api.security.RequiresAnyOf; +import org.labkey.api.security.RequiresNoPermission; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.SecurableResource; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.DesignDataClassPermission; +import org.labkey.api.security.permissions.DesignSampleTypePermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.SampleWorkflowDeletePermission; +import org.labkey.api.security.permissions.SiteAdminPermission; +import org.labkey.api.security.permissions.TroubleshooterPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.ConceptURIProperties; +import org.labkey.api.sql.LabKeySql; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.StudyUrls; +import org.labkey.api.study.publish.StudyPublishService; +import org.labkey.api.usageMetrics.SimpleMetricsService; +import org.labkey.api.util.DOM; +import org.labkey.api.util.DOM.LK; +import org.labkey.api.util.ErrorRenderer; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.FileStream; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.HtmlStringBuilder; +import org.labkey.api.util.ImageUtil; +import org.labkey.api.util.JSoupUtil; +import org.labkey.api.util.LinkBuilder; +import org.labkey.api.util.NetworkDrive; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.ResponseHelper; +import org.labkey.api.util.SafeToRender; +import org.labkey.api.util.SessionHelper; +import org.labkey.api.util.StringExpression; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.UniqueID; +import org.labkey.api.util.CsrfInput; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.BadRequestException; +import org.labkey.api.view.DataView; +import org.labkey.api.view.DataViewSnapshotSelectionForm; +import org.labkey.api.view.DetailsView; +import org.labkey.api.view.HBox; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.InsertView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.RedirectException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.UpdateView; +import org.labkey.api.view.VBox; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.ViewServlet; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.ClientDependency; +import org.labkey.api.view.template.PageConfig; +import org.labkey.experiment.ChooseExperimentTypeBean; +import org.labkey.experiment.ConfirmDeleteView; +import org.labkey.experiment.CustomPropertiesView; +import org.labkey.experiment.DataClassWebPart; +import org.labkey.experiment.DerivedSamplePropertyHelper; +import org.labkey.experiment.DotGraph; +import org.labkey.experiment.ExpDataFileListener; +import org.labkey.experiment.ExperimentRunDisplayColumn; +import org.labkey.experiment.ExperimentRunGraph; +import org.labkey.experiment.LineageGraphDisplayColumn; +import org.labkey.experiment.MissingFilesCheckInfo; +import org.labkey.experiment.MoveRunsBean; +import org.labkey.experiment.ParentChildView; +import org.labkey.experiment.ProtocolApplicationDisplayColumn; +import org.labkey.experiment.ProtocolDisplayColumn; +import org.labkey.experiment.ProtocolWebPart; +import org.labkey.experiment.RunGroupWebPart; +import org.labkey.experiment.SampleTypeDisplayColumn; +import org.labkey.experiment.SampleTypeWebPart; +import org.labkey.experiment.StandardAndCustomPropertiesView; +import org.labkey.experiment.XarExportPipelineJob; +import org.labkey.experiment.XarExportType; +import org.labkey.experiment.XarExporter; +import org.labkey.experiment.api.ClosureQueryHelper; +import org.labkey.experiment.api.DataClass; +import org.labkey.experiment.api.DataClassDomainKind; +import org.labkey.experiment.api.ExpDataClassAttachmentParent; +import org.labkey.experiment.api.ExpDataClassImpl; +import org.labkey.experiment.api.ExpDataImpl; +import org.labkey.experiment.api.ExpExperimentImpl; +import org.labkey.experiment.api.ExpMaterialImpl; +import org.labkey.experiment.api.ExpProtocolApplicationImpl; +import org.labkey.experiment.api.ExpProtocolImpl; +import org.labkey.experiment.api.ExpRunImpl; +import org.labkey.experiment.api.ExpSampleTypeImpl; +import org.labkey.experiment.api.Experiment; +import org.labkey.experiment.api.ExperimentServiceImpl; +import org.labkey.experiment.api.GraphAlgorithms; +import org.labkey.experiment.api.ProtocolActionStepDetail; +import org.labkey.experiment.api.SampleTypeServiceImpl; +import org.labkey.experiment.api.SampleTypeUpdateServiceDI; +import org.labkey.experiment.controllers.property.PropertyController; +import org.labkey.experiment.lineage.ExpLineageServiceImpl; +import org.labkey.experiment.pipeline.ExperimentPipelineJob; +import org.labkey.experiment.types.TypesController; +import org.labkey.experiment.xar.XarExportSelection; +import org.labkey.vfs.FileLike; +import org.springframework.beans.PropertyValue; +import org.springframework.beans.PropertyValues; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.validation.ObjectError; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.MultipartHttpServletRequest; +import org.springframework.web.servlet.ModelAndView; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.toList; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; +import static org.labkey.api.exp.query.ExpSchema.TableType.DataInputs; +import static org.labkey.api.query.QueryImportPipelineJob.QUERY_IMPORT_NOTIFICATION_PROVIDER_PARAM; +import static org.labkey.api.query.QueryImportPipelineJob.QUERY_IMPORT_PIPELINE_DESCRIPTION_PARAM; +import static org.labkey.api.query.QueryImportPipelineJob.QUERY_IMPORT_PIPELINE_PROVIDER_PARAM; +import static org.labkey.api.util.DOM.A; +import static org.labkey.api.util.DOM.Attribute.action; +import static org.labkey.api.util.DOM.Attribute.href; +import static org.labkey.api.util.DOM.Attribute.id; +import static org.labkey.api.util.DOM.Attribute.method; +import static org.labkey.api.util.DOM.Attribute.name; +import static org.labkey.api.util.DOM.Attribute.size; +import static org.labkey.api.util.DOM.Attribute.src; +import static org.labkey.api.util.DOM.Attribute.target; +import static org.labkey.api.util.DOM.Attribute.type; +import static org.labkey.api.util.DOM.Attribute.value; +import static org.labkey.api.util.DOM.Attribute.width; +import static org.labkey.api.util.DOM.DIV; +import static org.labkey.api.util.DOM.IMG; +import static org.labkey.api.util.DOM.INPUT; +import static org.labkey.api.util.DOM.LI; +import static org.labkey.api.util.DOM.TABLE; +import static org.labkey.api.util.DOM.TD; +import static org.labkey.api.util.DOM.TR; +import static org.labkey.api.util.DOM.UL; +import static org.labkey.api.util.DOM.at; +import static org.labkey.api.util.DOM.cl; +import static org.labkey.experiment.ExpDataIterators.setContainerFilterForImport; +import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.update; + +public class ExperimentController extends SpringActionController +{ + private static final Logger _log = LogManager.getLogger(ExperimentController.class); + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver( + ExperimentController.class + ); + private static final String GUEST_DIRECTORY_NAME = "guest"; + + public ExperimentController() + { + setActionResolver(_actionResolver); + } + + public static void ensureCorrectContainer(Container requestContainer, ExpObject object, ViewContext viewContext) + { + Container objectContainer = object.getContainer(); + if (!requestContainer.equals(objectContainer)) + { + ActionURL url = viewContext.cloneActionURL(); + url.setContainer(objectContainer); + throw new RedirectException(url); + } + } + + // Complete no-op, but leave in place in case we decide to adjust the base nav trail + private void addRootNavTrail(NavTree root) + { + // Intentionally don't add an "Experiment" node to the list because it's too overloaded. All content on the + // default action can be added to a portal page if desired. + } + + @Override + public PageConfig defaultPageConfig() + { + // set default help topic for controller + PageConfig config = super.defaultPageConfig(); + config.setHelpTopic("experiment"); + return config; + } + + @ActionNames("begin,gridView") + @RequiresPermission(ReadPermission.class) + public class BeginAction extends SimpleViewAction + { + @Override + public VBox getView(Object o, BindException errors) + { + VBox result = new VBox(); + + VBox runListView = createRunListView(20); + result.addView(runListView); + + RunGroupWebPart runGroups = new RunGroupWebPart(getViewContext(), false); + runGroups.showHeader(); + result.addView(runGroups); + + result.addView(new ProtocolWebPart(false, getViewContext())); + result.addView(new SampleTypeWebPart(false, getViewContext())); + result.addView(new DataClassWebPart(false, getViewContext(), null)); + + return result; + } + + @Override + public void addNavTrail(NavTree root) + { + addRootNavTrail(root); + root.addChild("Experiment"); + } + } + + @RequiresPermission(ReadPermission.class) + public class ShowRunsAction extends SimpleViewAction + { + @Override + public VBox getView(Object o, BindException errors) + { + return createRunListView(100); + } + + @Override + public void addNavTrail(NavTree root) + { + addRootNavTrail(root); + root.addChild("Experiment Runs"); + } + } + + private VBox createRunListView(int defaultMaxRows) + { + Set types = ExperimentService.get().getExperimentRunTypes(getContainer()); + ChooseExperimentTypeBean bean = new ChooseExperimentTypeBean(types, ExperimentRunType.getSelectedFilter(types, getViewContext().getRequest().getParameter("experimentRunFilter")), getViewContext().getActionURL().clone(), Collections.emptyList()); + JspView chooserView = new JspView<>("/org/labkey/experiment/experimentRunQueryHeader.jsp", bean); + + ExperimentRunListView view = ExperimentService.get().createExperimentRunWebPart(getViewContext(), bean.getSelectedFilter()); + view.setFrame(WebPartView.FrameType.NONE); + + // When paginated and the user hasn't explicitly set a maxRows, use the default maxRows size. + QuerySettings settings = view.getSettings(); + if (!settings.isMaxRowsSet() && settings.getShowRows() == ShowRows.PAGINATED) + { + settings.setMaxRows(defaultMaxRows); + } + + VBox result = new VBox(chooserView, view); + result.setFrame(WebPartView.FrameType.PORTAL); + return result; + } + + @RequiresPermission(ReadPermission.class) + @ActionNames("showRunGroups, showExperiments") + public class ShowRunGroupsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + RunGroupWebPart webPart = new RunGroupWebPart(getViewContext(), false); + webPart.setFrame(WebPartView.FrameType.NONE); + return webPart; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("runGroups"); + addRootNavTrail(root); + root.addChild("Run Groups"); + } + } + + public record Field(String domainURI, String domainName, String name, Container container) {} + public record MiniExpObject(Object rowId, String name) {} + public record TimelineSummary(MiniExpObject miniExpObject, String mostRecentValue) {} + public record ProblemType(String tableName, String fieldName, String pkName) { + public Object toHtml(List summaries) + { + return DOM.DIV( + DOM.H4(tableName), + DOM.TABLE(at(cl("table-condensed", "labkey-data-region", "table-bordered")), + DOM.THEAD(DOM.TH(pkName), DOM.TH(fieldName)), + summaries.stream().map(summary -> + DOM.TR(DOM.TD(summary.miniExpObject.name), DOM.TD(summary.mostRecentValue))) + )); + } + } + + @RequiresPermission(SiteAdminPermission.class) + public static class ReportLostFieldValuesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + // Find all the fields that could have lost data due to issue 52666 + TableInfo t = new ExpSchema(getUser(), ContainerManager.getRoot()).getTable(ExpSchema.TableType.Fields.name(), ContainerFilter.getUnsafeEverythingFilter()); + List fields = new TableSelector(t, + new SimpleFilter(FieldKey.fromParts("StorageColumnNameMatch"), false). + addCondition(FieldKey.fromParts("DomainURI"), ":AssayDomain-Data.", CompareType.DOES_NOT_CONTAIN), + null). + getArrayList(Field.class); + + // Prep audit table for querying + UserSchema auditSchema = AuditLogService.get().createSchema(getUser(), ContainerManager.getRoot()); + + Map> sampleTypeSummaries = new HashMap<>(); + Map> dataClassSummaries = new HashMap<>(); + Map> listSummaries = new HashMap<>(); + + Map> problematicFields = new LinkedHashMap<>(); + + for (Field field : fields) + { + String domainURI = field.domainURI; + String fieldName = field.name; + Container container = field.container; + Domain domain = PropertyService.get().getDomain(container, domainURI); + if (domain != null && domain.getDomainKind() != null) + { + TableInfo table = domain.getDomainKind().getTableInfo(getUser(), container, domain, ContainerFilter.getUnsafeEverythingFilter()); + + if (table != null) + { + // Drill into sample types + if (domain.getDomainKind().getClass().equals(SampleTypeDomainKind.class)) + { + // rows that currently have no value for the field with potential for data loss + List rowsWithNull = new TableSelector(table, + new HashSet<>(List.of("RowId", "Name")), + new SimpleFilter(new CompareType.CompareClause(FieldKey.fromParts(fieldName), CompareType.ISBLANK, null)), + null). + getArrayList(MiniExpObject.class); + + List fixupsNeeded = checkData( + rowsWithNull, + fieldName, + obj -> new SimpleFilter(FieldKey.fromParts("SampleId"), obj.rowId), + auditSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE, ContainerFilter.getUnsafeEverythingFilter())); + if (!fixupsNeeded.isEmpty()) + { + sampleTypeSummaries.put(new ProblemType(table.getName(), fieldName, "SampleID"), fixupsNeeded); + } + } + // and data classes/sample sources + if (domain.getDomainKind().getClass().equals(DataClassDomainKind.class)) + { + // rows samples that current have no value for the field with potential for data loss + List rowsWithNull = new TableSelector(table, + new HashSet<>(List.of("RowId", "Name")), + new SimpleFilter(new CompareType.CompareClause(FieldKey.fromParts(fieldName), CompareType.ISBLANK, null)), + null). + getArrayList(MiniExpObject.class); + + List fixupsNeeded = checkData( + rowsWithNull, + fieldName, + obj -> new SimpleFilter(FieldKey.fromParts("RowPk"), Objects.toString(obj.rowId)). + addCondition(FieldKey.fromParts("SchemaName"), "exp.data"). + addCondition(FieldKey.fromParts("QueryName"), domain.getName()), + auditSchema.getTable("QueryUpdateAuditEvent", ContainerFilter.getUnsafeEverythingFilter())); + + if (!fixupsNeeded.isEmpty()) + { + dataClassSummaries.put(new ProblemType(table.getName(), fieldName, "SourceID"), fixupsNeeded); + } + } + // and lists + if ("lists".equals(table.getUserSchema().getName())) + { + // rows samples that current have no value for the field with potential for data loss + List rowsWithNull = new ArrayList<>(); + + ColumnInfo entityIdCol = table.getColumn("EntityId"); + ColumnInfo pkCol = table.getPkColumns().get(0); + + new TableSelector(table, + List.of(entityIdCol, pkCol), + new SimpleFilter(new CompareType.CompareClause(FieldKey.fromParts(fieldName), CompareType.ISBLANK, null)), + null). + forEachResults(r -> + { + Object entityId = entityIdCol.getValue(r); + Object pk = pkCol.getValue(r); + rowsWithNull.add(new MiniExpObject(entityId, pk.toString())); + }); + + + List fixupsNeeded = checkData( + rowsWithNull, + fieldName, + obj -> new SimpleFilter(FieldKey.fromParts("ListItemEntityId"), obj.rowId), + auditSchema.getTable("ListAuditEvent", ContainerFilter.getUnsafeEverythingFilter())); + + if (!fixupsNeeded.isEmpty()) + { + listSummaries.put(new ProblemType(table.getName(), fieldName, table.getPkColumnNames().get(0)), fixupsNeeded); + } + } + + long totalRows = new TableSelector(table).getRowCount(); + long emptyRows = new TableSelector(table, new SimpleFilter(new CompareType.CompareClause(FieldKey.fromParts(fieldName), CompareType.ISBLANK, null)), null).getRowCount(); + problematicFields.put(field, Pair.of(totalRows, emptyRows)); + } + else + { + problematicFields.put(field, Pair.of(null, null)); + } + } + } + + return new HtmlView("Fixups Needed", + DOM.createHtmlFragment( + DOM.H2("Potentially Problematic Fields"), + problematicFields.isEmpty() ? "No problematic fields detected!" : + DOM.TABLE(at(cl("table-condensed", "labkey-data-region", "table-bordered")), + DOM.THEAD(DOM.TH("Domain Name"), DOM.TH("Domain URI"), DOM.TH("Field Name"), DOM.TH("Container"), DOM.TH("Total Rows"), DOM.TH("Rows with Nulls")), + problematicFields.entrySet().stream().map(e -> { + Field f = e.getKey(); + Pair counts = e.getValue(); + return DOM.TR( + DOM.TD(f.domainName), + DOM.TD(f.domainURI), + DOM.TD(f.name), + DOM.TD(f.container.getPath()), + DOM.TD(counts.first), + DOM.TD(counts.second) + ); + } + )), + + DOM.H2("Sample Types"), + sampleTypeSummaries.isEmpty() ? "No problems detected!" : + sampleTypeSummaries.entrySet().stream().map(e -> + e.getKey().toHtml(e.getValue())), + + DOM.H2("Data Classes"), + dataClassSummaries.isEmpty() ? "No problems detected!" : + dataClassSummaries.entrySet().stream().map(e -> + e.getKey().toHtml(e.getValue())), + + DOM.H2("Lists"), + listSummaries.isEmpty() ? "No problems detected!" : + listSummaries.entrySet().stream().map(e -> + e.getKey().toHtml(e.getValue())) + )); + } + + @NotNull + private List checkData(List rowsWithNull, String fieldName, Function filterGenerator, TableInfo auditTable) + { + List fixupsNeeded = new ArrayList<>(); + + // For each sample without a value today, check the audit history + for (MiniExpObject row : rowsWithNull) + { + // Order by RowId to get them in the sequence they happened in + var events = new TableSelector(auditTable, filterGenerator.apply(row), new Sort("RowId")).getArrayList(DetailedAuditTypeEvent.class); + // Remember the most recently set value + String mostRecentValue = null; + for (DetailedAuditTypeEvent event : events) + { + Map newValues = new CaseInsensitiveHashMap<>(AbstractAuditTypeProvider.decodeFromDataMap(event.getNewRecordMap())); + if (newValues.containsKey(fieldName)) + { + // Will be the empty string if the value was intentionally set to blank + mostRecentValue = newValues.get(fieldName); + } + } + // If the value had been set before, and its most recent insert/update wasn't setting it blank, + // it's most likely a lost value + if (mostRecentValue != null && !mostRecentValue.isEmpty()) + { + fixupsNeeded.add(new TimelineSummary(row, mostRecentValue)); + } + } + return fixupsNeeded; + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Accidentally Nulled Field Report"); + } + } + + @RequiresPermission(ReadPermission.class) + public static class CreateHiddenRunGroupAction extends MutatingApiAction + { + @Override + public ApiResponse execute(SimpleApiJsonForm form, BindException errors) throws Exception + { + JSONObject json = form.getJsonObject(); + String selectionKey = json.optString("selectionKey", null); + List runs = new ArrayList<>(); + + // Accept either an explicit list of run IDs + if (json.has("runIds")) + { + JSONArray runIds = json.getJSONArray("runIds"); + for (int i = 0; i < runIds.length(); i++) + { + ExpRunImpl run = ExperimentServiceImpl.get().getExpRun(runIds.getInt(i)); + if (run != null) + { + runs.add(run); + } + } + } + // Or a reference to a DataRegion selection key + else if (selectionKey != null) + { + Set ids = DataRegionSelection.getSelectedIntegers(getViewContext(), selectionKey, false); + for (Long id : ids) + { + ExpRunImpl run = ExperimentServiceImpl.get().getExpRun(id); + if (run != null) + { + runs.add(run); + } + } + } + if (runs.isEmpty()) + { + throw new NotFoundException(); + } + + ExpExperiment group = ExperimentService.get().createHiddenRunGroup(getContainer(), getUser(), runs.toArray(new ExpRun[0])); + if (selectionKey != null) + DataRegionSelection.clearAll(getViewContext(), selectionKey); + + ApiSimpleResponse response = new ApiSimpleResponse(); + response.putBean(group, "rowId", "LSID", "name", "hidden"); + return response; + } + } + + + @RequiresPermission(ReadPermission.class) + public class DetailsAction extends QueryViewAction + { + private ExpExperimentImpl _experiment; + + public DetailsAction() + { + super(ExpObjectForm.class); + } + + private Pair> createViews(ExpObjectForm form, BindException errors) + { + _experiment = ExperimentServiceImpl.get().getExpExperiment(form.getRowId()); + if (_experiment == null) + { + throw new NotFoundException("Could not find an experiment with RowId " + form.getRowId()); + } + + if (!_experiment.getContainer().equals(getContainer())) + { + throw new RedirectException(getViewContext().cloneActionURL().setContainer(_experiment.getContainer())); + } + + List protocols = _experiment.getAllProtocols(); + + Set types = new TreeSet<>(ExperimentService.get().getExperimentRunTypes(getContainer())); + ExperimentRunType selectedType = ExperimentRunType.getSelectedFilter(types, getViewContext().getRequest().getParameter("experimentRunFilter")); + + ChooseExperimentTypeBean bean = new ChooseExperimentTypeBean(types, selectedType, getViewContext().getActionURL().clone(), protocols); + JspView chooserView = new JspView<>("/org/labkey/experiment/experimentRunQueryHeader.jsp", bean, errors); + + ExperimentRunListView runListView = ExperimentRunListView.createView(getViewContext(), bean.getSelectedFilter(), true); + runListView.getRunTable().setExperiment(_experiment); + runListView.setShowRemoveFromExperimentButton(true); + runListView.setShowDeleteButton(true); + runListView.setShowAddToRunGroupButton(true); + runListView.setShowExportButtons(true); + runListView.setShowMoveRunsButton(true); + return new Pair<>(runListView, chooserView); + } + + @Override + protected ModelAndView getHtmlView(ExpObjectForm form, BindException errors) throws Exception + { + Pair> views = createViews(form, errors); + + CustomPropertiesView customPropertiesView = new CustomPropertiesView(_experiment.getLSID(), getContainer()); + + TableInfo runGroupsTable = new ExpSchema(getUser(), getContainer()).getTable(ExpSchema.TableType.RunGroups); + + DetailsView detailsView = new DetailsView(new DataRegion(), _experiment.getRowId()); + detailsView.getDataRegion().setTable(runGroupsTable); + detailsView.getDataRegion().addColumns(runGroupsTable, "RowId,Name,Created,Modified,Contact,ExperimentDescriptionURL,Hypothesis,Comments"); + detailsView.getDataRegion().getDisplayColumn(0).setVisible(false); + detailsView.getDataRegion().getDisplayColumn(2).setWidth("60%"); + + ButtonBar bb = new ButtonBar(); + bb.setStyle(ButtonBar.Style.separateButtons); + ActionButton b = new ActionButton(ExperimentUrlsImpl.get().getShowUpdateURL(_experiment), "Edit"); + b.setDisplayPermission(UpdatePermission.class); + bb.add(b); + detailsView.getDataRegion().setButtonBar(bb); + if (_experiment.getBatchProtocol() != null) + { + detailsView.setTitle("Batch Details"); + detailsView.getDataRegion().addColumns(runGroupsTable, "BatchProtocolId"); + } + else + { + detailsView.setTitle("Run Group Details"); + } + + VBox runsVBox = new VBox(views.second, createInitializedQueryView(form, errors, false, null)); + runsVBox.setTitle("Experiment Runs"); + runsVBox.setFrame(WebPartView.FrameType.PORTAL); + + return new VBox(new StandardAndCustomPropertiesView(detailsView, customPropertiesView), runsVBox); + } + + @Override + protected ExperimentRunListView createQueryView(ExpObjectForm form, BindException errors, boolean forExport, String dataRegion) + { + return createViews(form, errors).first; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("runGroups"); + addRootNavTrail(root); + root.addChild("Run Groups", ExperimentUrlsImpl.get().getShowExperimentsURL(getContainer())); + root.addChild(_experiment.getName()); + } + } + + @RequiresPermission(ReadPermission.class) + public class ListSampleTypesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + SampleTypeWebPart view = new SampleTypeWebPart(false, getViewContext()); + view.setFrame(WebPartView.FrameType.NONE); + view.setErrorMessage(getViewContext().getRequest().getParameter("errorMessage")); + + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("sampleSets"); + addRootNavTrail(root); + root.addChild("Sample Types"); + } + } + + @RequiresPermission(ReadPermission.class) + public class ShowSampleTypeAction extends SimpleViewAction + { + private ExpSampleTypeImpl _sampleType; + + @Override + public ModelAndView getView(ExpObjectForm form, BindException errors) + { + _sampleType = SampleTypeServiceImpl.get().getSampleType(getContainer(), getUser(), form.getRowId()); + if (_sampleType == null && form.getLsid() != null) + { + if (form.getLsid().equalsIgnoreCase("Material") || form.getLsid().equalsIgnoreCase("Sample")) + { + // Not a real sample type - just show all the materials instead + throw new RedirectException(new ActionURL(ShowAllMaterialsAction.class, getContainer())); + } + // Check if the URL specifies the LSID, and stick the bean back into the form + _sampleType = SampleTypeServiceImpl.get().getSampleType(form.getLsid()); + } + + if (_sampleType == null) + { + throw new NotFoundException("No matching sample type found"); + } + + List allScopedSampleTypes = (List) SampleTypeService.get().getSampleTypes(getContainer(), getUser(), true); + if (!allScopedSampleTypes.contains(_sampleType)) + { + ensureCorrectContainer(getContainer(), _sampleType, getViewContext()); + } + + SamplesSchema schema = new SamplesSchema(getUser(), getContainer()); + QuerySettings settings = schema.getSettings(getViewContext(), "Material", _sampleType.getName()); + QueryView queryView = new SampleTypeContentsView(_sampleType, schema, settings, errors); + + DetailsView detailsView = new DetailsView(getSampleTypeRegion(getViewContext()), _sampleType.getRowId()); + detailsView.getDataRegion().getDisplayColumn("Name").setURL((ActionURL)null); + detailsView.getDataRegion().getDisplayColumn("LSID").setVisible(false); + detailsView.getDataRegion().getDisplayColumn("MaterialLSIDPrefix").setVisible(false); + detailsView.getDataRegion().getDisplayColumn("LabelColor").setVisible(false); + detailsView.getDataRegion().getDisplayColumn("MetricUnit").setVisible(false); + detailsView.getDataRegion().getDisplayColumn("Category").setVisible(false); + + detailsView.setTitle("Sample Type Properties"); + detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).setStyle(ButtonBar.Style.separateButtons); + + Container autoLinkContainer = _sampleType.getAutoLinkTargetContainer(); + if (null != autoLinkContainer) + { + DisplayColumn autoLinkTargetColumn = detailsView.getDataRegion().getDisplayColumn("autoLinkTargetContainer"); + autoLinkTargetColumn.setVisible(false); + + SimpleDisplayColumn displayAutoLinkTargetColumn = new SimpleDisplayColumn(); + displayAutoLinkTargetColumn.setCaption("Auto Link Target Container:"); + String path = autoLinkContainer.getPath(); + displayAutoLinkTargetColumn.setDisplayHtml(path.equals("/") ? "" : path); + detailsView.getDataRegion().addDisplayColumn(displayAutoLinkTargetColumn); + } + + DisplayColumn autoLinkCategoryColumn = detailsView.getDataRegion().getDisplayColumn("autoLinkCategory"); + autoLinkCategoryColumn.setVisible(false); + SimpleDisplayColumn displayAutoLinkCategoryColumn = new SimpleDisplayColumn(); + displayAutoLinkCategoryColumn.setCaption("Auto Link Category:"); + displayAutoLinkCategoryColumn.setDisplayHtml(_sampleType.getAutoLinkCategory()); + detailsView.getDataRegion().addDisplayColumn(displayAutoLinkCategoryColumn); + + if (_sampleType.hasNameAsIdCol()) + { + SimpleDisplayColumn nameIdCol = new SimpleDisplayColumn(); + nameIdCol.setCaption("Has Name Id Column:"); + nameIdCol.setDisplayHtml("true"); + detailsView.getDataRegion().addDisplayColumn(nameIdCol); + } + + if (_sampleType.hasIdColumns()) + { + SimpleDisplayColumn idCols = new SimpleDisplayColumn(); + idCols.setCaption("Id Column(s):"); + String names = _sampleType.getIdCols().stream() + .filter(Objects::nonNull) + .map(DomainProperty::getName) + .collect(Collectors.joining(", ")); + if (!names.isEmpty()) + { + idCols.setDisplayHtml(PageFlowUtil.filter(names)); + detailsView.getDataRegion().addDisplayColumn(idCols); + } + } + + if (_sampleType.getParentCol() != null) + { + SimpleDisplayColumn parentCol = new SimpleDisplayColumn(PageFlowUtil.filter(_sampleType.getParentCol().getName())); + parentCol.setCaption("Parent Column:"); + detailsView.getDataRegion().addDisplayColumn(parentCol); + } + + try + { + SimpleDisplayColumn importAliasCol = new SimpleDisplayColumn(); + importAliasCol.setCaption("Parent Import Alias(es):"); + if (!_sampleType.getImportAliases().isEmpty()) + importAliasCol.setDisplayHtml(PageFlowUtil.filter(StringUtils.join(_sampleType.getImportAliases().keySet(), ", "))); + detailsView.getDataRegion().addDisplayColumn(importAliasCol); + } + catch (IOException e) + { + // unable to parse import alias map from JSON + } + + if (!getContainer().equals(_sampleType.getContainer())) + { + ActionURL definitionURL = urlProvider(ExperimentUrls.class).getShowSampleTypeURL(_sampleType); + SimpleDisplayColumn definedInCol = new SimpleDisplayColumn("" + + PageFlowUtil.filter(_sampleType.getContainer().getPath()) + + ""); + definedInCol.setCaption("Defined In:"); + detailsView.getDataRegion().addDisplayColumn(definedInCol); + } + + // Not all sample types can be edited + DomainKind domainKind = _sampleType.getDomain().getDomainKind(); + if (domainKind != null && domainKind.canEditDefinition(getUser(), _sampleType.getDomain())) + { + if (domainKind instanceof SampleTypeDomainKind) + { + ActionURL updateURL = new ActionURL(EditSampleTypeAction.class, _sampleType.getContainer()); + updateURL.addParameter("RowId", _sampleType.getRowId()); + updateURL.addReturnUrl(getViewContext().getActionURL()); + + if (!getContainer().equals(_sampleType.getContainer())) + { + String editLink = updateURL.toString(); + ActionButton updateButton = new ActionButton("Edit Type"); + updateButton.setActionType(ActionButton.Action.SCRIPT); + updateButton.setScript("if (window.confirm('This sample type is defined in the " + _sampleType.getContainer().getPath() + " folder. Would you still like to edit it?')) { window.location = '" + editLink + "' }"); + detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).add(updateButton); + } + else + { + ActionButton updateButton = new ActionButton(updateURL, "Edit Type", ActionButton.Action.LINK); + updateButton.setDisplayPermission(DesignSampleTypePermission.class); + updateButton.setPrimary(true); + detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).add(updateButton); + } + + ActionURL deleteURL = new ActionURL(DeleteSampleTypesAction.class, _sampleType.getContainer()); + deleteURL.addParameter("singleObjectRowId", _sampleType.getRowId()); + deleteURL.addReturnUrl(ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer())); + ActionButton deleteButton = new ActionButton(deleteURL, "Delete Type", ActionButton.Action.LINK); + deleteButton.setDisplayPermission(DesignSampleTypePermission.class); + detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).add(deleteButton); + } + else + { + ActionURL editURL = domainKind.urlEditDefinition(_sampleType.getDomain(), new ViewBackgroundInfo(_sampleType.getContainer(), getUser(), getViewContext().getActionURL())); + if (editURL != null) + { + editURL.addReturnUrl(getViewContext().getActionURL()); + ActionButton editTypeButton = new ActionButton(editURL, "Edit Fields"); + editTypeButton.setDisplayPermission(UpdatePermission.class); + detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).add(editTypeButton); + } + } + } + + if (_sampleType.canImportMoreSamples()) + { + TableInfo table = queryView.getTable(); + if (table != null) + { + ActionURL importURL = table.getImportDataURL(getContainer()); + if (importURL != null) + { + importURL = importURL.clone(); + importURL.addReturnUrl(getViewContext().getActionURL()); + ActionButton uploadButton = new ActionButton(importURL, "Import More Samples", ActionButton.Action.LINK); + uploadButton.setDisplayPermission(UpdatePermission.class); + detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).add(uploadButton); + } + } + } + + var publish = StudyPublishService.get(); + if (AuditLogService.get().isViewable() && publish != null) + { + ContainerFilter cf = ContainerFilter.Type.CurrentAndSubfolders.create(getContainer(), getUser()); + ActionURL linkToStudyHistoryURL = publish.getPublishHistory(getContainer(), Dataset.PublishSource.SampleType, _sampleType.getRowId(), cf); + ActionButton linkToStudyHistoryButton = new ActionButton(linkToStudyHistoryURL, "Link to Study History", ActionButton.Action.LINK); + linkToStudyHistoryButton.setDisplayPermission(InsertPermission.class); + detailsView.getDataRegion().getButtonBar(DataRegion.MODE_DETAILS).add(linkToStudyHistoryButton); + } + + return new VBox(detailsView, queryView); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("sampleSets"); + ActionURL url = ExperimentUrls.get().getShowSampleTypeListURL(getContainer()); + addRootNavTrail(root); + root.addChild("Sample Types", url); + root.addChild("Sample Type " + _sampleType.getName()); + } + } + + @RequiresPermission(ReadPermission.class) + public class ShowAllMaterialsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + ExpSchema schema = new ExpSchema(getUser(), getContainer()); + QuerySettings settings = schema.getSettings(getViewContext(), "Materials", ExpSchema.TableType.Materials.toString()); + QueryView view = new QueryView(schema, settings, errors) + { + @Override + protected void populateButtonBar(DataView view, ButtonBar bar) + { + super.populateButtonBar(view, bar); + bar.add(SampleTypeContentsView.getDeriveSamplesButton(getContainer(),null)); + } + }; + view.setShowDetailsColumn(false); + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("sampleSets"); + addRootNavTrail(root); + root.addChild("All Materials"); + } + } + + /** + * Only shows standard and custom properties, not parent and child samples. Used for indexing + */ + @RequiresPermission(ReadPermission.class) + public class ShowMaterialSimpleAction extends SimpleViewAction + { + protected ExpMaterialImpl _material; + + @Override + public VBox getView(ExpObjectForm form, BindException errors) throws Exception + { + Container c = getContainer(); + _material = ExperimentServiceImpl.get().getExpMaterial(form.getRowId()); + if (_material == null && form.getLsid() != null) + { + _material = ExperimentServiceImpl.get().getExpMaterial(form.getLsid()); + } + if (_material == null) + { + throw new NotFoundException("Could not find a material with RowId " + form.getRowId()); + } + + ensureCorrectContainer(getContainer(), _material, getViewContext()); + + ExpRunImpl run = _material.getRun(); + ExpProtocol sourceProtocol = _material.getSourceProtocol(); + ExpProtocolApplication sourceProtocolApplication = _material.getSourceApplication(); + + DataRegion dr = new DataRegion(); + dr.addColumns(ExperimentServiceImpl.get().getTinfoMaterial().getUserEditableColumns()); + dr.removeColumns("RowId", "RunId", "LastIndexed", "LSID", "SourceApplicationId", "CpasType"); + + //dr.addColumns(extraProps); + dr.addDisplayColumn(new ExperimentRunDisplayColumn(run, "Source Experiment Run")); + dr.addDisplayColumn(new ProtocolDisplayColumn(sourceProtocol, "Source Protocol")); + dr.addDisplayColumn(new ProtocolApplicationDisplayColumn(sourceProtocolApplication, "Source Protocol Application")); + dr.addDisplayColumn(new LineageGraphDisplayColumn(_material, run)); + dr.addDisplayColumn(new SampleTypeDisplayColumn(_material)); + + //TODO: Can't yet edit materials uploaded from a material source + dr.setButtonBar(new ButtonBar()); + DetailsView detailsView = new DetailsView(dr, _material.getRowId()); + detailsView.setTitle("Standard Properties"); + detailsView.setFrame(WebPartView.FrameType.PORTAL); + + CustomPropertiesView cpv = new CustomPropertiesView(_material, c, getUser()); + + return new VBox(new StandardAndCustomPropertiesView(detailsView, cpv)); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("sampleSets"); + addRootNavTrail(root); + root.addChild("Sample Types", ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer())); + ExpSampleType sampleType = _material.getSampleType(); + if (sampleType != null) + { + root.addChild(sampleType.getName(), ExperimentUrlsImpl.get().getShowSampleTypeURL(sampleType)); + } + root.addChild("Sample " + _material.getName()); + } + } + + @RequiresPermission(ReadPermission.class) + public class ShowMaterialAction extends ShowMaterialSimpleAction + { + @Override + public VBox getView(ExpObjectForm form, BindException errors) throws Exception + { + VBox vbox = super.getView(form, errors); + + List materialsToInvestigate = new ArrayList<>(); + final Set successorRuns = new HashSet<>(); + materialsToInvestigate.add(_material); + Set investigatedMaterials = new HashSet<>(); + do + { + // Query for all the next tier of materials at once - issue 45402 + List followupRuns = ExperimentService.get().getRunsUsingMaterials(materialsToInvestigate); + + // Mark this set as investigated and reset for the next cycle + investigatedMaterials.addAll(materialsToInvestigate); + materialsToInvestigate = new ArrayList<>(); + + for (ExpRun r : followupRuns) + { + // Only expand the material outputs of the run if it's our first time visiting it + if (successorRuns.add(r)) + { + materialsToInvestigate.addAll(r.getMaterialOutputs()); + } + } + + if (successorRuns.size() > 1000) + { + // Give up - there may be a cycle or other problematic data + break; + } + + // Cull the ones we've already looked up + materialsToInvestigate.removeAll(investigatedMaterials); + } + while (!materialsToInvestigate.isEmpty()); + + HtmlStringBuilder updateLinks = HtmlStringBuilder.of(); + ExpSampleType st = _material.getSampleType(); + if (st != null && st.getContainer() != null && st.getContainer().hasPermission(getUser(), UpdatePermission.class)) + { + // XXX: ridiculous amount of work to get a update url expression for the sample type's table. + UserSchema samplesSchema = QueryService.get().getUserSchema(getUser(), st.getContainer(), "Samples"); + QueryDefinition queryDef = samplesSchema.getQueryDefForTable(st.getName()); + StringExpression expr = queryDef.urlExpr(QueryAction.updateQueryRow, null); + if (expr != null) + { + // Since we're building a detailsURL outside the context of a "row" need to set the correct + // container context on the generated expr. + ((DetailsURL) expr).setContainerContext(st.getContainer()); + String url = expr.eval(Collections.singletonMap(new FieldKey(null, "RowId"), _material.getRowId())); + updateLinks.append(LinkBuilder.labkeyLink("edit", url)).append(" "); + } + } + + if (getContainer().hasPermission(getUser(), InsertPermission.class)) + { + ActionURL deriveURL = new ActionURL(DeriveSamplesChooseTargetAction.class, getContainer()); + deriveURL.addParameter("rowIds", _material.getRowId()); + if (st != null) + deriveURL.addParameter("targetSampleTypeId", st.getRowId()); + + updateLinks.append(LinkBuilder.labkeyLink("derive samples from this sample", deriveURL)).append(" "); + } + + vbox.addView(new HtmlView(updateLinks)); + + ExperimentRunListView runListView = ExperimentRunListView.createView(getViewContext(), ExperimentRunType.ALL_RUNS_TYPE, true); + runListView.setShowRecordSelectors(false); + runListView.getRunTable().setRuns(successorRuns); + runListView.getRunTable().setContainerFilter(new ContainerFilter.AllFolders(getUser())); + runListView.setAllowableContainerFilterTypes(ContainerFilter.Type.Current, ContainerFilter.Type.CurrentAndSubfolders, ContainerFilter.Type.AllFolders); + runListView.setTitle("Runs associated with this material or a derived material"); + + ParentChildView pv = new ParentChildView(_material, getViewContext()); + vbox.addView(pv); + vbox.addView(runListView); + + return vbox; + } + } + + + // + // DataClass + // + + @RequiresPermission(ReadPermission.class) + public class ListDataClassAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + DataClassWebPart view = new DataClassWebPart(false, getViewContext(), null); + view.setFrame(WebPartView.FrameType.NONE); + view.setErrorMessage(getViewContext().getRequest().getParameter("errorMessage")); + + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("dataClass"); + addRootNavTrail(root); + root.addChild("Data Classes"); + } + } + + public static class DataClassForm extends ExpObjectForm + { + private String _name; + + public String getName() + { + return _name; + } + + public void setName(String name) + { + _name = name; + } + + public ExpDataClassImpl getDataClass(@Nullable Container container) + { + ExpDataClassImpl dataClass = null; + + if (getName() != null) + { + dataClass = ExperimentServiceImpl.get().getDataClass(getContainer(), getUser(), getName()); + if (dataClass == null) + throw new NotFoundException("No data class found for name '" + getName() + "'."); + } + else if (getRowId() > 0) + { + dataClass = ExperimentServiceImpl.get().getDataClass(getContainer(), getUser(), getRowId()); + } + + if (dataClass == null) + throw new NotFoundException("No data class found."); + else if (container != null && !container.equals(dataClass.getContainer())) + throw new NotFoundException("Data class is not defined in the given container."); + + return dataClass; + } + } + + @RequiresPermission(ReadPermission.class) + public class ShowDataClassAction extends SimpleViewAction + { + private ExpDataClassImpl _dataClass; + + @Override + public ModelAndView getView(DataClassForm form, BindException errors) + { + _dataClass = form.getDataClass(null); + return new VBox(getDataClassPropertiesView(), getDataClassContentsView(errors)); + } + + private DetailsView getDataClassPropertiesView() + { + ExpSchema expSchema = new ExpSchema(getUser(), _dataClass.getContainer()); + + TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, null); + QueryUpdateForm tvf = new QueryUpdateForm(table, getViewContext(), null); + tvf.setPkVal(_dataClass.getRowId()); + DetailsView detailsView = new DetailsView(tvf); + detailsView.setTitle("Data Class Properties"); + + ButtonBar bb = new ButtonBar(); + bb.setStyle(ButtonBar.Style.separateButtons); + boolean inDefinitionContainer = getContainer().equals(_dataClass.getContainer()); + + DomainKind domainKind = _dataClass.getDomain().getDomainKind(); + if (domainKind != null && domainKind.canEditDefinition(getUser(), _dataClass.getDomain())) + { + ActionURL updateURL = new ActionURL(EditDataClassAction.class, _dataClass.getContainer()); + updateURL.addParameter("rowId", _dataClass.getRowId()); + updateURL.addReturnUrl(urlProvider(ExperimentUrls.class).getShowDataClassURL(_dataClass.getContainer(), _dataClass.getRowId())); + + if (inDefinitionContainer) + { + ActionButton updateButton = new ActionButton(updateURL, "Edit Data Class", ActionButton.Action.LINK); + updateButton.setDisplayPermission(DesignDataClassPermission.class); + updateButton.setPrimary(true); + bb.add(updateButton); + } + else if (_dataClass.getContainer().hasPermission(getUser(), DesignDataClassPermission.class)) + { + ActionButton updateButton = new ActionButton("Edit Data Class"); + updateButton.setActionType(ActionButton.Action.SCRIPT); + updateButton.setScript("if (window.confirm('This data class is defined in the " + _dataClass.getContainer().getPath() + " folder. Would you still like to edit it?')) { window.location = '" + updateURL + "' }"); + updateButton.setPrimary(true); + bb.add(updateButton); + } + + ActionURL deleteURL = new ActionURL(DeleteDataClassAction.class, _dataClass.getContainer()); + deleteURL.addParameter("singleObjectRowId", _dataClass.getRowId()); + deleteURL.addReturnUrl(ExperimentUrlsImpl.get().getDataClassListURL(getContainer())); + ActionButton deleteButton = new ActionButton(deleteURL, "Delete Data Class", ActionButton.Action.LINK); + + if (inDefinitionContainer) + { + deleteButton.setDisplayPermission(DesignDataClassPermission.class); + bb.add(deleteButton); + } + else if (_dataClass.getContainer().hasPermission(getUser(), DesignDataClassPermission.class)) + { + bb.add(deleteButton); + } + } + detailsView.getDataRegion().setButtonBar(bb); + + if (!inDefinitionContainer) + { + ActionURL definitionURL = urlProvider(ExperimentUrls.class).getShowDataClassURL(_dataClass.getContainer(), _dataClass.getRowId()); + LinkBuilder link = LinkBuilder.simpleLink(_dataClass.getContainer().getPath(), definitionURL); + SimpleDisplayColumn definedInCol = new SimpleDisplayColumn(link.toString()); + definedInCol.setCaption("Defined In:"); + detailsView.getDataRegion().addDisplayColumn(definedInCol); + } + + return detailsView; + } + + private QueryView getDataClassContentsView(BindException errors) + { + UserSchema dataClassSchema = QueryService.get().getUserSchema(getUser(), getContainer(), ExpSchema.SCHEMA_EXP_DATA); + QuerySettings settings = dataClassSchema.getSettings(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, _dataClass.getName()); + + return new QueryView(dataClassSchema, settings, errors) + { + @Override + public @NotNull LinkedHashSet getClientDependencies() + { + LinkedHashSet resources = super.getClientDependencies(); + resources.add(ClientDependency.fromPath("Ext4")); + resources.add(ClientDependency.fromPath("dataregion/confirmDelete.js")); + return resources; + } + + @Override + public ActionButton createDeleteButton() + { + ActionButton button = super.createDeleteButton(); + if (button != null) + { + String dependencyText = ExperimentService.get() + .getObjectReferencers() + .stream() + .map(referencer -> referencer.getObjectReferenceDescription(ExpData.class)) + .collect(Collectors.joining(" or ")); + + button.setScript("LABKEY.dataregion.confirmDelete(" + + PageFlowUtil.jsString(getDataRegionName()) + ", " + + PageFlowUtil.jsString(ExpSchema.SCHEMA_EXP_DATA.toString()) + ", " + + PageFlowUtil.jsString(getQueryDef().getName()) + ", " + + "'experiment', 'getDataOperationConfirmationData.api', " + + PageFlowUtil.jsString(getSelectionKey()) + ", " + + "'data object', 'data objects', '" + dependencyText + "', {dataOperation: 'Delete'})"); + button.setRequiresSelection(true); + } + return button; + } + }; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("dataClass"); + addRootNavTrail(root); + root.addChild("Data Classes", ExperimentUrlsImpl.get().getDataClassListURL(getContainer())); + root.addChild(_dataClass.getName()); + } + } + + @RequiresPermission(DesignDataClassPermission.class) + public class DeleteDataClassAction extends AbstractDeleteAction + { + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("dataClass"); + super.addNavTrail(root); + } + + @Override + protected void deleteObjects(DeleteForm deleteForm) + { + List dataClasses = getDataClasses(deleteForm); + if (!ensureCorrectContainer(dataClasses)) + { + throw new UnauthorizedException(); + } + for (ExpRun run : getRuns(dataClasses)) + { + if (!run.getContainer().hasPermission(getUser(), DeletePermission.class)) + { + throw new UnauthorizedException(); + } + } + for (ExpDataClass dataClass : dataClasses) + { + dataClass.delete(getUser(), deleteForm.getUserComment()); + } + } + + @Override + public ModelAndView getView(DeleteForm deleteForm, boolean reshow, BindException errors) + { + List dataClasses = getDataClasses(deleteForm); + + if (!ensureCorrectContainer(dataClasses)) + { + throw new RedirectException(ExperimentUrlsImpl.get().getDataClassListURL(getContainer(), "To delete a data class, you must be in its folder or project.")); + } + + return new ConfirmDeleteView("Data Class", ShowDataClassAction.class, dataClasses, deleteForm, getRuns(dataClasses)); + } + + private List getDataClasses(DeleteForm deleteForm) + { + List dataClasses = new ArrayList<>(); + for (long rowId : deleteForm.getIds(false)) + { + ExpDataClass dataClass = ExperimentServiceImpl.get().getDataClass(getContainer(), getUser(), rowId); + if (dataClass != null) + { + dataClasses.add(dataClass); + } + } + return dataClasses; + } + + private boolean ensureCorrectContainer(List dataClasses) + { + for (ExpDataClass dataClass : dataClasses) + { + Container sourceContainer = dataClass.getContainer(); + if (!sourceContainer.equals(getContainer())) + { + return false; + } + } + return true; + } + + private List getRuns(List dataClasses) + { + if (!dataClasses.isEmpty()) + { + List runArray = ExperimentService.get().getRunsUsingDataClasses(dataClasses); + return ExperimentService.get().runsDeletedWithInput(runArray); + } + else + { + return Collections.emptyList(); + } + } + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(ReadPermission.class) + public static class GetDataClassPropertiesAction extends ReadOnlyApiAction + { + @Override + public Object execute(DataClassForm form, BindException errors) throws Exception + { + ExpDataClass dataClass = form.getDataClass(getContainer()); + if (dataClass != null) + return new DataClassDomainKindProperties(dataClass); + else + throw new NotFoundException("Data class does not exist in this container for rowId " + form.getRowId() + "."); + } + } + + @RequiresPermission(DesignDataClassPermission.class) + public static class EditDataClassAction extends SimpleViewAction + { + private ExpDataClassImpl _dataClass; + + @Override + public ModelAndView getView(DataClassForm form, BindException errors) + { + boolean create = form.getLSID() == null && form.getRowId() == 0 && form.getName() == null; + if (!create) + _dataClass = form.getDataClass(getContainer()); + + return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("dataClassDesigner")); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("dataClass"); + + root.addChild("Data Classes", ExperimentUrlsImpl.get().getDataClassListURL(getContainer())); + if (_dataClass == null) + { + root.addChild("Create Data Class"); + } + else + { + root.addChild(_dataClass.getName(), ExperimentUrlsImpl.get().getShowDataClassURL(getContainer(), _dataClass.getRowId())); + root.addChild("Update Data Class"); + } + } + } + + @RequiresPermission(DesignDataClassPermission.class) + public static class CreateDataClassFromTemplateAction extends FormViewAction + { + private ActionURL _successUrl; + private Map _domainTemplates; + + @Override + public void validateCommand(CreateDataClassFromTemplateForm form, Errors errors) + { + String name = null; + _domainTemplates = DomainTemplateGroup.getAllTemplates(getContainer()); + + if (!_domainTemplates.containsKey(form.getDomainTemplate())) + { + errors.reject(ERROR_MSG, "Unknown template selected: " + form.getDomainTemplate()); + } + else + { + DomainTemplate template = _domainTemplates.get(form.getDomainTemplate()); + name = template.getTemplateName(); + + // Issue 40230: if template includes sample type option, verify that it exists + if (template.getOptions().containsKey("sampleSet")) + { + String sampleTypeName = template.getOptions().get("sampleSet").toString(); + ExpSampleType sampleType = SampleTypeServiceImpl.get().getSampleType(getContainer(), getUser(), sampleTypeName); + if (sampleType == null) + errors.reject(ERROR_MSG, "Unable to find a sample type in this container with name: " + sampleTypeName + "."); + } + } + + if (StringUtils.isBlank(name)) + errors.reject(ERROR_MSG, "DataClass template selection is required."); + else if (ExperimentService.get().getDataClass(getContainer(), getUser(), name) != null) + errors.reject(ERROR_MSG, "DataClass '" + name + "' already exists."); + + } + + @Override + public ModelAndView getView(CreateDataClassFromTemplateForm form, boolean reshow, BindException errors) + { + Set templates = DomainTemplateGroup.getTemplatesForDomainKind(getContainer(), DataClassDomainKind.NAME); + form.setAvailableDomainTemplateNames(templates); + + Set messages = new HashSet<>(); + Map groups = DomainTemplateGroup.getAllGroups(getContainer()); + for (DomainTemplateGroup g : groups.values()) + messages.addAll(g.getErrors()); + form.setXmlParseErrors(messages); + + return new JspView<>("/org/labkey/experiment/createDataClassFromTemplate.jsp", form, errors); + } + + @Override + public boolean handlePost(CreateDataClassFromTemplateForm form, BindException errors) throws Exception + { + DomainTemplate template = _domainTemplates.get(form.getDomainTemplate()); + Domain domain = DomainUtil.createDomain(template, getContainer(), getUser(), form.getName()); + + _successUrl = domain.getDomainKind().urlEditDefinition(domain, getViewContext()); + return true; + } + + @Override + public URLHelper getSuccessURL(CreateDataClassFromTemplateForm form) + { + return _successUrl; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("dataClass"); + root.addChild("Create Data Class from Template"); + } + } + + public static class CreateDataClassFromTemplateForm extends DataClass + { + private String _domainTemplate; + private Set _availableDomainTemplateNames; + private Set _xmlParseErrors; + private final ReturnUrlForm _returnUrlForm = new ReturnUrlForm(); + + public String getDomainTemplate() + { + return _domainTemplate; + } + + public void setDomainTemplate(String domainTemplate) + { + _domainTemplate = domainTemplate; + } + + public Set getAvailableDomainTemplateNames() + { + return _availableDomainTemplateNames; + } + + public void setAvailableDomainTemplateNames(Set availableDomainTemplateNames) + { + _availableDomainTemplateNames = availableDomainTemplateNames; + } + + public Set getXmlParseErrors() + { + return _xmlParseErrors; + } + + public void setXmlParseErrors(Set xmlParseErrors) + { + _xmlParseErrors = xmlParseErrors; + } + + @Nullable + public String getReturnUrl() + { + return _returnUrlForm.getReturnUrl(); + } + + public void setReturnUrl(String s) + { + _returnUrlForm.setReturnUrl(s); + } + } + + public static class ConceptURIForm + { + private String _conceptURI; + + public String getConceptURI() + { + return _conceptURI; + } + + public void setConceptURI(String conceptURI) + { + _conceptURI = conceptURI; + } + } + + @RequiresPermission(AdminPermission.class) + public static class RemoveConceptMappingAction extends MutatingApiAction + { + @Override + public void validateForm(ConceptURIForm form, Errors errors) + { + if (form.getConceptURI() == null || ConceptURIProperties.getLookup(getContainer(), form.getConceptURI()) == null) + errors.reject(ERROR_MSG, "Concept URI not found: " + form.getConceptURI()); + } + + @Override + public Object execute(ConceptURIForm form, BindException errors) + { + ConceptURIProperties.removeLookup(getContainer(), form.getConceptURI()); + return new ApiSimpleResponse("success", true); + } + } + + @RequiresPermission(ReadPermission.class) + public static class RunAttachmentDownloadAction extends BaseDownloadAction + { + @Nullable + @Override + public Pair getAttachment(AttachmentForm form) + { + if (form.getLsid() == null || form.getName() == null) + throw new NotFoundException("Error: missing required param 'lsid' or 'name'."); + + ExpRun run = ExperimentService.get().getExpRun(form.getLsid()); + if (run == null) + throw new NotFoundException("Run not found: " + form.getLsid()); + + if (!run.getContainer().equals(getContainer())) + { + if (run.getContainer().hasPermission(getUser(), ReadPermission.class)) + throw new RedirectException(getViewContext().cloneActionURL().setContainer(run.getContainer())); + else + throw new NotFoundException("Run not found"); + } + + AttachmentParent parent = new ExpRunAttachmentParent(run); + return new Pair<>(parent, form.getName()); + } + } + + @RequiresPermission(ReadPermission.class) + public static class DataClassAttachmentDownloadAction extends BaseDownloadAction + { + @Nullable + @Override + public Pair getAttachment(AttachmentForm form) + { + if (form.getLsid() == null || form.getName() == null) + throw new NotFoundException("Error: missing required param 'lsid' or 'name'."); + + Lsid lsid = new Lsid(form.getLsid()); + ExpData data = ExperimentServiceImpl.get().getExpData(lsid.toString()); + if (data == null) + throw new NotFoundException("Error: Data object not found for the given LSID: " + lsid); + AttachmentParent parent = new ExpDataClassAttachmentParent(data.getContainer(), lsid); + + return new Pair<>(parent, form.getName()); + } + } + + public static class AttachmentForm extends LsidForm implements BaseDownloadAction.InlineDownloader + { + private String _name; + private boolean _inline = true; + + public String getName() + { + return _name; + } + + public void setName(String name) + { + _name = name; + } + + @Override + public boolean isInline() + { + return _inline; + } + + public void setInline(boolean inline) + { + _inline = inline; + } + } + + // + // END DataClass actions + // + + public static ActionURL getRunGraphURL(Container c, long runId) + { + return new ActionURL(ShowRunGraphAction.class, c).addParameter("rowId", runId); + } + + + @RequiresPermission(ReadPermission.class) + public class ShowRunGraphAction extends AbstractShowRunAction + { + @Override + protected VBox createLowerView(ExpRunImpl experimentRun, BindException errors) + { + return new VBox( + createRunViewTabs(experimentRun, false, true, true), + new ExperimentRunGraphView(experimentRun, false)); + } + } + + + @RequiresPermission(ReadPermission.class) + public static class DownloadGraphAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ExperimentRunForm form, BindException errors) throws Exception + { + boolean detail = form.isDetail(); + String focus = form.getFocus(); + String focusType = form.getFocusType(); + + ExpRunImpl experimentRun = (ExpRunImpl) form.lookupRun(); + ensureCorrectContainer(getContainer(), experimentRun, getViewContext()); + + ExperimentRunGraph.RunGraphFiles files; + try + { + files = ExperimentRunGraph.generateRunGraph(getViewContext(), experimentRun, detail, focus, focusType); + } + catch (ExperimentException e) + { + PageFlowUtil.streamTextAsImage(getViewContext().getResponse(), "ERROR: " + e.getMessage(), 600, 150, Color.RED); + return null; + } + + try + { + PageFlowUtil.streamFile(getViewContext().getResponse(), new File(files.getImageFile().getAbsolutePath()), false); + } + catch (FileNotFoundException e) + { + getViewContext().getResponse().sendRedirect(getViewContext().getRequest().getContextPath() + "/experiment/ExperimentRunNotFound.gif"); + } + finally + { + files.release(); + } + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + throw new UnsupportedOperationException(); + } + } + + private abstract class AbstractShowRunAction extends SimpleViewAction + { + private ExpRunImpl _experimentRun; + + @Override + public ModelAndView getView(ExperimentRunForm form, BindException errors) + { + _experimentRun = (ExpRunImpl) form.lookupRun(); + ensureCorrectContainer(getContainer(), _experimentRun, getViewContext()); + + VBox vbox = new VBox(); + + JspView detailsView = new JspView<>("/org/labkey/experiment/ExperimentRunDetails.jsp", _experimentRun); + detailsView.setTitle("Standard Properties"); + + var attachmentParent = new ExpRunAttachmentParent(_experimentRun); + var attachments = AttachmentService.get().getAttachments(attachmentParent) + .stream() + .map(att -> Pair.of(att.getName(), new ActionURL(RunAttachmentDownloadAction.class, _experimentRun.getContainer()).addParameter("name", att.getName()).addParameter("lsid", _experimentRun.getLSID()))) + .collect(toList()); + CustomPropertiesView cpv = new CustomPropertiesView(_experimentRun.getLSID(), getContainer(), attachments); + + vbox.addView(new StandardAndCustomPropertiesView(detailsView, cpv)); + + HtmlStringBuilder updateLinks = HtmlStringBuilder.of(); + List runEditors = ExperimentService.get().getRunEditors(); + for (ExpRunEditor editor : runEditors) + { + if (editor.isProtocolEditor(form.lookupRun().getProtocol())) + { + updateLinks.append(LinkBuilder.labkeyLink("edit " + editor.getDisplayName() + " run", editor.getEditUrl(getContainer()).addParameter("rowId", form.getRowId()))); + } + } + + if (!updateLinks.isEmpty()) + { + HtmlView view = new HtmlView(updateLinks); + vbox.addView(view); + } + + VBox lowerView = createLowerView(_experimentRun, errors); + lowerView.setFrame(WebPartView.FrameType.PORTAL); + lowerView.setTitle("Run Details"); + NavTree tree = new NavTree(""); + File runRoot = _experimentRun.getFilePathRoot(); + if (NetworkDrive.exists(runRoot)) + { + if (!runRoot.isDirectory()) + { + runRoot = runRoot.getParentFile(); + } + PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(_experimentRun.getContainer()); + if (pipelineRoot != null) + { + if (pipelineRoot.isUnderRoot(runRoot)) + { + String path = pipelineRoot.relativePath(runRoot); + tree.addChild("View Files", urlProvider(PipelineUrls.class).urlBrowse(_experimentRun.getContainer(), null, path)); + } + } + } + + final String exportFilesFormId = "exportFilesForm"; + NavTree downloadFiles = new NavTree("Download all files"); + downloadFiles.setScript("document.getElementById('" + exportFilesFormId + "').submit();"); + tree.addChild(downloadFiles); + + // CONSIDER: Show modal dialog using ExperimentService.get().createRunExportView() + NavTree exportXarFiles = new NavTree("Export XAR"); + exportXarFiles.setScript("LABKEY.Experiment.exportRuns({runIds: [" + _experimentRun.getRowId() + "] });"); + tree.addChild(exportXarFiles); + + lowerView.setNavMenu(tree); + lowerView.setIsWebPart(false); + + vbox.addView(lowerView); + vbox.addView(new ExperimentRunGroupsView(getUser(), getContainer(), _experimentRun, getViewContext().getActionURL(), errors)); + + DOM.Renderable exportFilesForm = LK.FORM(at( + id, exportFilesFormId, + method, "POST", + action, new ActionURL(ExportRunFilesAction.class, _experimentRun.getContainer())), + INPUT(at(type, "hidden", + name, DataRegionSelection.DATA_REGION_SELECTION_KEY, + value, "ExportSingleRun")), + INPUT(at(type, "hidden", + name, DataRegion.SELECT_CHECKBOX_NAME, + value, _experimentRun.getRowId())), + INPUT(at(type, "hidden", + name, "zipFileName", + value, _experimentRun.getName() + ".zip"))); + + HtmlView hiddenFormView = new HtmlView(exportFilesForm); + vbox.addView(hiddenFormView); + + return vbox; + } + + protected abstract VBox createLowerView(ExpRunImpl experimentRun, BindException errors); + + @Override + public void addNavTrail(NavTree root) + { + root.addChild(_experimentRun.getName()); + } + } + + public static class ToggleRunExperimentMembershipForm + { + private int _runId; + private int _experimentId; + private boolean _included; + + public int getRunId() + { + return _runId; + } + + public void setRunId(int runId) + { + _runId = runId; + } + + public int getExperimentId() + { + return _experimentId; + } + + public void setExperimentId(int experimentId) + { + _experimentId = experimentId; + } + + public boolean isIncluded() + { + return _included; + } + + public void setIncluded(boolean included) + { + _included = included; + } + } + + @RequiresPermission(UpdatePermission.class) + public static class ToggleRunExperimentMembershipAction extends FormHandlerAction + { + @Override + public boolean handlePost(ToggleRunExperimentMembershipForm form, BindException errors) + { + ExpRun run = ExperimentService.get().getExpRun(form.getRunId()); + // Check if the user has permission to update this run + if (run == null || !run.getContainer().hasPermission(getUser(), UpdatePermission.class)) + { + throw new NotFoundException(); + } + + ExpExperiment exp = ExperimentService.get().getExpExperiment(form.getExperimentId()); + if (exp == null) + { + throw new NotFoundException(); + } + // Check if this + if (!ExperimentService.get().getExperiments(run.getContainer(), getUser(), true, false).contains(exp)) + { + throw new NotFoundException(); + } + // Users must have permission to view, but not necessarily update, the container the holds the run group + if (!exp.getContainer().hasPermission(getUser(), ReadPermission.class)) + { + throw new UnauthorizedException(); + } + + if (form.isIncluded()) + { + exp.addRuns(getUser(), run); + } + else + { + exp.removeRun(getUser(), run); + } + + return true; + } + + @Override + public URLHelper getSuccessURL(ToggleRunExperimentMembershipForm form) + { + return null; + } + + @Override + public void validateCommand(ToggleRunExperimentMembershipForm target, Errors errors) + { + } + } + + private HtmlView createRunViewTabs(ExpRun expRun, boolean showGraphSummary, boolean showGraphDetail, boolean showText) + { + return new HtmlView( + TABLE(cl("labkey-tab-strip"), + TR( + createTabSpacer(false), + createTab("Graph Summary View", ExperimentUrlsImpl.get().getRunGraphURL(expRun), !showGraphSummary), + createTabSpacer(false), + createTab("Graph Detail View", ExperimentUrlsImpl.get().getRunGraphDetailURL(expRun), !showGraphDetail), + createTabSpacer(false), + createTab("Text View", ExperimentUrlsImpl.get().getRunTextURL(expRun), !showText), + createTabSpacer(true)))); + } + + private DOM.Renderable createTab(String text, ActionURL url, boolean selected) + { + return TD(cl(selected,"labkey-tab-selected", "labkey-tab"), + A(at(href, url), text)); + } + + private DOM.Renderable createTabSpacer(boolean fullWidth) + { + return TD(cl("labkey-tab-space").at(fullWidth, width, "100%"), + IMG(at(src, AppProps.getInstance().getContextPath() + "/_.gif", width, "5"))); + } + + @RequiresPermission(ReadPermission.class) + public class ShowRunTextAction extends AbstractShowRunAction + { + @Override + protected VBox createLowerView(ExpRunImpl expRun, BindException errors) + { + JspView applicationsView = new JspView<>("/org/labkey/experiment/ProtocolApplications.jsp", expRun); + applicationsView.setFrame(WebPartView.FrameType.TITLE); + applicationsView.setTitle("Protocol Applications"); + + HtmlView toggleView = createRunViewTabs(expRun, true, true, false); + + QuerySettings runDataInputsSettings = new QuerySettings(getViewContext(), "RunDataInputs", DataInputs.name()); + UsageQueryView runDataInputsView = new UsageQueryView("Data Inputs", getViewContext(), expRun, ExpProtocol.ApplicationType.ExperimentRun, runDataInputsSettings, errors); + runDataInputsView.setButtonBarPosition(DataRegion.ButtonBarPosition.TOP); + + QuerySettings runDataOutputsSettings = new QuerySettings(getViewContext(), "RunDataOutputs", DataInputs.name()); + UsageQueryView runDataOutputsView = new UsageQueryView("Data Outputs", getViewContext(), expRun, ExpProtocol.ApplicationType.ExperimentRunOutput, runDataOutputsSettings, errors); + runDataOutputsView.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); + + QuerySettings runMaterialInputsSetting = new QuerySettings(getViewContext(), "RunMaterialInputs", ExpSchema.TableType.MaterialInputs.name()); + UsageQueryView runMaterialInputsView = new UsageQueryView("Material Inputs", getViewContext(), expRun, ExpProtocol.ApplicationType.ExperimentRun, runMaterialInputsSetting, errors); + runMaterialInputsView.setButtonBarPosition(DataRegion.ButtonBarPosition.TOP); + + QuerySettings runMaterialOutputsSettings = new QuerySettings(getViewContext(), "RunMaterialOutputs", ExpSchema.TableType.MaterialInputs.name()); + UsageQueryView runMaterialOutputsView = new UsageQueryView("Material Outputs", getViewContext(), expRun, ExpProtocol.ApplicationType.ExperimentRunOutput, runMaterialOutputsSettings, errors); + runMaterialOutputsView.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); + + HBox inputsView = new HBox(runDataInputsView, runMaterialInputsView); + HBox registeredInputsView = new HBox(); + + var expService = ExperimentService.get(); + expService.getRunInputsViewProviders().forEach(provider -> + { + var queryView = provider.createView(getViewContext(), expRun, errors); + if (queryView != null) + { + registeredInputsView.addView(queryView); + } + }); + HBox outputsView = new HBox(runDataOutputsView, runMaterialOutputsView); + HBox registeredOutputsView = new HBox(); + expService.getRunOutputsViewProviders().forEach(provider -> + { + var queryView = provider.createView(getViewContext(), expRun, errors); + if (queryView != null) + { + registeredOutputsView.addView(queryView); + } + }); + + var vBox = new VBox(); + vBox.addView(toggleView); + vBox.addView(inputsView); + if (!registeredInputsView.isEmpty()) + vBox.addView(registeredInputsView); + vBox.addView(outputsView); + if (!registeredOutputsView.isEmpty()) + vBox.addView(registeredOutputsView); + vBox.addView(applicationsView); + + return vBox; + } + } + + private static class UsageQueryView extends QueryView + { + private final ExpRun _run; + private final ExpProtocol.ApplicationType _type; + + public UsageQueryView(String title, ViewContext context, ExpRun run, ExpProtocol.ApplicationType type, + QuerySettings settings, BindException errors) + { + super(new ExpSchema(context.getUser(), context.getContainer()), settings, errors); + setTitle(title); + setFrame(FrameType.TITLE); + _run = run; + _type = type; + setShowBorders(true); + setShadeAlternatingRows(true); + setShowExportButtons(false); + setShowPagination(false); + disableContainerFilterSelection(); + } + + @Override + protected TableInfo createTable() + { + String tableName = getSettings().getQueryName(); + ExpInputTable tableInfo = (ExpInputTable) getSchema().getTable(tableName, new ContainerFilter.AllFolders(getUser()), true, true); + tableInfo.setRun(_run, _type); + tableInfo.setLocked(true); + return tableInfo; + } + } + + + public static ActionURL getShowRunGraphDetailURL(Container c, long rowId) + { + ActionURL url = new ActionURL(ShowRunGraphDetailAction.class, c); + url.addParameter("rowId", rowId); + return url; + } + + + @RequiresPermission(ReadPermission.class) + public class ShowRunGraphDetailAction extends AbstractShowRunAction + { + @Override + protected VBox createLowerView(ExpRunImpl run, BindException errors) + { + ExperimentRunGraphView gw = new ExperimentRunGraphView(run, true); + if (null != getViewContext().getActionURL().getParameter("focus")) + gw.setFocus(getViewContext().getActionURL().getParameter("focus")); + if (null != getViewContext().getActionURL().getParameter("focusType")) + gw.setFocusType(getViewContext().getActionURL().getParameter("focusType")); + return new VBox(createRunViewTabs(run, true, false, true), gw); + } + } + + private abstract class AbstractDataAction extends SimpleViewAction + { + protected ExpDataImpl _data; + + @Override + public final ModelAndView getView(DataForm form, BindException errors) throws Exception + { + _data = form.lookupData(); + if (_data == null) + { + throw new NotFoundException("Could not find a data with RowId " + form.getRowId()); + } + + ensureCorrectContainer(getContainer(), _data, getViewContext()); + return getDataView(form, errors); + } + + protected abstract ModelAndView getDataView(DataForm form, BindException errors) throws Exception; + + @Override + public void addNavTrail(NavTree root) + { + addRootNavTrail(root); + root.addChild("Data " + _data.getName()); + } + } + + @RequiresPermission(ReadPermission.class) + public class ShowDataAction extends AbstractDataAction + { + @Override + public ModelAndView getDataView(DataForm form, BindException errors) + { + ExpRun run = _data.getRun(); + ExpProtocol sourceProtocol = _data.getSourceProtocol(); + ExpProtocolApplication sourceProtocolApplication = _data.getSourceApplication(); + ExpDataClass dataClass = _data.getDataClass(getUser()); + + ExpSchema schema = new ExpSchema(getUser(), getContainer()); + TableInfo table; + long pk; + if (dataClass == null) + { + table = schema.getDatasTable(); + pk = _data.getRowId(); + } + else + { + table = schema.getSchema(ExpSchema.NestedSchemas.data).getTable(dataClass.getName()); + pk = new TableSelector(table, Collections.singleton("rowId"), new SimpleFilter(FieldKey.fromParts("lsid"), _data.getLSID()), null).getObject(Integer.class); + } + + DataRegion dr = new DataRegion(); + dr.setTable(table); + List cols = table.getColumns().stream().filter(ColumnInfo::isShownInDetailsView).collect(toList()); + dr.addColumns(cols); + dr.removeColumns("RowId", "Created", "CreatedBy", "Modified", "ModifiedBy", "DataFileUrl", "Run", "LSID", "CpasType", "SourceApplicationId", "Folder", "Generated"); + dr.addDisplayColumn(new ExperimentRunDisplayColumn(run, "Source Experiment Run")); + dr.addDisplayColumn(new ProtocolDisplayColumn(sourceProtocol, "Source Protocol")); + dr.addDisplayColumn(new ProtocolApplicationDisplayColumn(sourceProtocolApplication, "Source Protocol Application")); + dr.addDisplayColumn(new LineageGraphDisplayColumn(_data, run)); + DetailsView detailsView = new DetailsView(dr, pk); + detailsView.setTitle("Standard Properties"); + detailsView.setFrame(WebPartView.FrameType.PORTAL); + ButtonBar bb = new ButtonBar(); + bb.setStyle(ButtonBar.Style.separateButtons); + + ExperimentDataHandler handler = _data.findDataHandler(); + ActionURL viewDataURL = handler == null ? null : handler.getContentURL(_data); + if (viewDataURL != null) + { + bb.add(new ActionButton("View data", viewDataURL)); + } + + if (_data.isPathAccessible()) + { + bb.add(new ActionButton("View file", ExperimentUrlsImpl.get().getShowFileURL(_data, true))); + bb.add(new ActionButton("Download file", ExperimentUrlsImpl.get().getShowFileURL(_data, false))); + + if (getContainer().hasPermission(getUser(), InsertPermission.class)) + { + String relativePath = null; + PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); + if (root != null) + { + Path rootFile = root.getRootNioPath(); + Path dataFile = _data.getFilePath(); + if (dataFile != null) + { + Path pathRelative; + try + { + pathRelative = rootFile.relativize(dataFile); + if (null != pathRelative) + relativePath = pathRelative.toString(); + } + catch (IllegalArgumentException e) + { + // dataFile not relative to root + } + } + } + ActionURL browseURL = urlProvider(PipelineUrls.class).urlBrowse(getContainer(), getViewContext().getActionURL(), relativePath); + bb.add(new ActionButton("Browse in pipeline", browseURL)); + } + } + + // add links to any other exp.data that share the same dataFileUrl path + var altDataList = ExperimentService.get().getAllExpDataByURL(_data.getDataFileUrl(), getContainer()); + altDataList.removeIf(_data::equals); + if (!altDataList.isEmpty()) + { + MenuButton menu = new MenuButton("Alternate Data"); + for (ExpData altData : altDataList) + { + ExpRun altDataRun = altData.getRun(); + StringBuilder sb = new StringBuilder(altData.getName()); + if (altDataRun != null) + sb.append(" created by run '").append(altDataRun.getName()).append("' (").append(altDataRun.getProtocol().getName()).append(")"); + menu.addMenuItem(sb.toString(), altData.detailsURL()); + } + bb.add(menu); + } + + dr.setButtonBarPosition(DataRegion.ButtonBarPosition.TOP); + dr.setButtonBar(bb); + + CustomPropertiesView cpv = new CustomPropertiesView(_data.getLSID(), getContainer()); + HBox hbox = new StandardAndCustomPropertiesView(detailsView, cpv); + + VBox vbox = new VBox(hbox); + + ParentChildView pv = new ParentChildView(_data, getViewContext()); + vbox.addView(pv); + + ExperimentRunListView runListView = ExperimentRunListView.createView(getViewContext(), ExperimentRunType.ALL_RUNS_TYPE, true); + runListView.getRunTable().setInputData(_data); + runListView.getRunTable().setContainerFilter(new ContainerFilter.AllFolders(getUser())); + runListView.getRunTable().setLocked(true); + runListView.setTitle("Runs using this data as an input"); + vbox.addView(runListView); + + if (_data.isInlineImage() && _data.isFileOnDisk()) + { + ActionURL showFileURL = new ActionURL(ShowFileAction.class, getContainer()).addParameter("rowId", _data.getRowId()); + HtmlView imageView = new HtmlView(IMG(at(src, showFileURL))); + return new VBox(vbox, imageView); + } + return vbox; + } + } + + @RequiresPermission(AdminPermission.class) + public static class CheckDataFileAction extends MutatingApiAction + { + private ExpDataImpl _data; + + @Override + public void validateForm(DataFileForm form, Errors errors) + { + _data = form.lookupData(); + if (_data == null) + { + errors.reject(ERROR_MSG, "No ExpData found for id: " + form.getRowId()); + } + } + + @Override + public ApiResponse execute(DataFileForm form, BindException errors) + { + File dataFile = _data.getFile(); + Container dataContainer = _data.getContainer(); + boolean fileExists = _data.isFileOnDisk(); + boolean fileExistsAtCurrent = false; + File newDataFile = null; + + ApiSimpleResponse response = new ApiSimpleResponse(); + response.put("dataFileUrl", _data.getDataFileUrl()); + response.put("fileExists", fileExists); + response.put("containerPath", dataContainer.getPath()); + + if (!fileExists) + { + PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(dataContainer); + if (pipelineRoot != null && pipelineRoot.isValid() && dataFile != null) + { + newDataFile = pipelineRoot.resolvePath("/" + AssayFileWriter.DIR_NAME + "/" + dataFile.getName()); + fileExistsAtCurrent = NetworkDrive.exists(newDataFile); + response.put("fileExistsAtCurrent", fileExistsAtCurrent); + } + } + + // if the current dataFileUrl does not exist on disk and we have the file at the current + // pipeline root /assaydata dir, fix the dataFileUrl value + if (form.isAttemptFilePathFix()) + { + if (fileExistsAtCurrent) + { + ExpDataFileListener fileListener = new ExpDataFileListener(); + fileListener.fileMoved(dataFile, newDataFile, getUser(), dataContainer); + response.put("filePathFixed", true); + + // update the ExpData object so that we can get the new dataFileUrl + _data = form.lookupData(); + response.put("newDataFileUrl", _data.getDataFileUrl()); + } + else + { + response.put("filePathFixed", false); + } + } + + response.put("success", true); + return response; + } + } + + public static class DataFileForm extends DataForm + { + private boolean _attemptFilePathFix; + + public boolean isAttemptFilePathFix() + { + return _attemptFilePathFix; + } + + public void setAttemptFilePathFix(boolean attemptFilePathFix) + { + _attemptFilePathFix = attemptFilePathFix; + } + } + + @RequiresPermission(ReadPermission.class) + public class ShowFileAction extends AbstractDataAction + { + @Override + protected ModelAndView getDataView(DataForm form, BindException errors) throws IOException + { + if (!_data.isPathAccessible()) + { + throw new NotFoundException("Data file " + _data.getDataFileUrl() + " does not exist on disk"); + } + + PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); + if (root != null && !root.isUnderRoot(_data.getFilePath())) + { + // Issue 35649: ImmPort module "publish" creates exp.data object in this container for paths that originate in a different container + FileContentService fileSvc = FileContentService.get(); + if (fileSvc == null) + throw new UnauthorizedException("Data file is not under the pipeline root for this folder"); + + List containers = fileSvc.getContainersForFilePath(_data.getFilePath()); + if (containers.isEmpty() || containers.stream().noneMatch(c -> c.hasPermission(getUser(), ReadPermission.class))) + throw new UnauthorizedException("Data file is not under the pipeline root for this folder"); + } + + //Issues 25667 and 31152 + if (form.isInline()) + { + ExperimentDataHandler h = _data.findDataHandler(); + if (h != null) + { + URLHelper url = h.getShowFileURL(_data); + if (url != null) + { + throw new RedirectException(url, false); + } + } + } + + try + { + Path realContent = _data.getFilePath(); + if (null == realContent) + throw new IllegalStateException("Path not found."); + + boolean inline = _data.isInlineImage() || form.isInline() || "inlineImage".equalsIgnoreCase(form.getFormat()); + if (_data.isInlineImage() && form.getMaxDimension() != null) + { + try (InputStream inputStream = Files.newInputStream(realContent)) + { + BufferedImage image = ImageIO.read(inputStream); + // If image, create a thumbnail, otherwise fall through as a regular download attempt + if (image != null) + { + int imageMax = Math.max(image.getHeight(), image.getWidth()); + if (imageMax > form.getMaxDimension().intValue()) + { + double scale = (double) form.getMaxDimension().intValue() / (double) imageMax; + ByteArrayOutputStream bOut = new ByteArrayOutputStream(); + ImageUtil.resizeImage(image, bOut, scale, 1); + PageFlowUtil.streamFileBytes(getViewContext().getResponse(), FileUtil.getFileName(realContent) + ".png", bOut.toByteArray(), !inline); + return null; + } + } + } + } + + boolean extended = "jsonTSVExtended".equalsIgnoreCase(form.getFormat()); + boolean ignoreTypes = "jsonTSVIgnoreTypes".equalsIgnoreCase(form.getFormat()); + if ("jsonTSV".equalsIgnoreCase(form.getFormat()) || extended || ignoreTypes) + { + if (!FileUtil.hasCloudScheme(realContent)) // TODO: handle streaming from S3 to JSON + streamToJSON(realContent.toFile(), form.getFormat(), -1, null); + return null; + } + + try (InputStream inputStream = Files.newInputStream(realContent)) + { + PageFlowUtil.streamFile(getViewContext().getResponse(), Collections.emptyMap(), FileUtil.getFileName(realContent), inputStream, !inline); + } + } + catch (IOException e) + { + try + { + // Try to write the exception back to the caller if we haven't already flushed the buffer + ApiJsonWriter writer = new ApiJsonWriter(getViewContext().getResponse()); + writer.writeResponse(e); + } + catch (IllegalStateException ise) + { + // Most likely that a disconnected client caused the IOException writing back the response + } + } + + return null; + } + } + + + public static class ParseForm + { + String format = "jsonTSV"; + int maxRows = -1; + + public String getFormat() + { + return format; + } + + public void setFormat(String format) + { + this.format = format; + } + + public int getMaxRows() + { + return maxRows; + } + + public void setMaxRows(int maxRow) + { + this.maxRows = maxRow; + } + } + + @RequiresNoPermission + public class ParseFileAction extends MutatingApiAction + { + @Override + public Object execute(ParseForm form, BindException errors) throws Exception + { + if (!(getViewContext().getRequest() instanceof MultipartHttpServletRequest)) + throw new BadRequestException("Expected MultipartHttpServletRequest when posting files."); + + MultipartFile formFile = getFileMap().get("file"); + if (formFile == null) + { + return true; + } + + File tempFile = null; + try + { + tempFile = FileUtil.createTempFile("parse", formFile.getOriginalFilename()); + FileUtil.copyData(formFile.getInputStream(), tempFile); + streamToJSON(tempFile, form.getFormat(), form.getMaxRows(), formFile.getOriginalFilename()); + } + finally + { + if (null != tempFile) + tempFile.delete(); + } + return null; + } + } + + + // SampleTypeTest + private void streamToJSON(File realContent, String format, int maxRow, String originalFileName) throws IOException + { + String lowerCaseFileName = realContent.getName().toLowerCase(); + boolean extended = "jsonTSVExtended".equalsIgnoreCase(format); + boolean ignoreTypes = "jsonTSVIgnoreTypes".equalsIgnoreCase(format); + + JSONArray sheetsArray; + if (lowerCaseFileName.endsWith(".xls") || lowerCaseFileName.endsWith(".xlsx")) + { + try + { + sheetsArray = ExcelFactory.convertExcelToJSON(realContent, extended, maxRow); + } + catch (InvalidFormatException e) + { + throw new NotFoundException("Could not open " + realContent.getName(), e); + } + } + else + { + DataLoaderFactory dlf = DataLoader.get().findFactory(realContent, null); + if (null == dlf) + { + throw new ApiUsageException("Unable to parse file " + realContent + ", it is likely of an unsupported file type"); + } + + try (DataLoader tabLoader = dlf.createLoader(realContent, true)) + { + tabLoader.setScanAheadLineCount(5000); + ColumnDescriptor[] cols = tabLoader.getColumns(); + + if (ignoreTypes) + for (ColumnDescriptor col : cols) + col.clazz = String.class; + + JSONArray rowsArray = new JSONArray(); + JSONArray headerArray = new JSONArray(); + for (ColumnDescriptor col : cols) + { + if (extended) + { + JSONObject valueObject = new JSONObject(); + valueObject.put("value", col.name); + headerArray.put(valueObject); + } + else + { + headerArray.put(col.name); + } + } + rowsArray.put(headerArray); + for (Map rowMap : tabLoader) + { + // headers count as a row to be consistent + if (maxRow > -1 && maxRow <= rowsArray.length() + 1) + break; + + JSONArray rowArray = new JSONArray(); + for (ColumnDescriptor col : cols) + { + Object value = rowMap.get(col.name); + if (extended) + { + JSONObject valueObject = new JSONObject(); + valueObject.put("value", value); + rowArray.put(valueObject); + } + else + { + rowArray.put(value); + } + } + rowsArray.put(rowArray); + } + + JSONObject sheetJSON = new JSONObject(); + sheetJSON.put("name", "flat"); + sheetJSON.put("data", rowsArray); + sheetsArray = new JSONArray(); + sheetsArray.put(sheetJSON); + } + } + + try (ApiJsonWriter writer = new ApiJsonWriter(getViewContext().getResponse())) + { + JSONObject workbookJSON = new JSONObject(); + workbookJSON.put("fileName", realContent.getName()); + workbookJSON.put("sheets", sheetsArray); + if (originalFileName != null) + workbookJSON.put("originalFileName", originalFileName); + writer.writeResponse(new ApiSimpleResponse(workbookJSON)); + } + } + + + public static class ConvertArraysToExcelForm + { + private String _json; + + public String getJson() + { + return _json; + } + + public void setJson(String json) + { + _json = json; + } + } + + @RequiresPermission(ReadPermission.class) + public static class ConvertArraysToExcelAction extends ExportAction + { + @Override + public void validate(ConvertArraysToExcelForm form, BindException errors) + { + if (form.getJson() == null) + { + errors.reject(ERROR_MSG, "Unable to convert to Excel - no spreadsheet data given"); + } + } + + @Override + public void export(ConvertArraysToExcelForm form, HttpServletResponse response, BindException errors) throws Exception + { + try + { + JSONObject rootObject; + JSONArray sheetsArray; + if (form.getJson() == null || form.getJson().trim().isEmpty()) + { + // Create JSON so that we return an empty file + rootObject = new JSONObject(); + sheetsArray = new JSONArray(); + JSONObject sheetObject = new JSONObject(); + sheetsArray.put(sheetObject); + } + else + { + rootObject = new JSONObject(form.getJson()); + sheetsArray = rootObject.getJSONArray("sheets"); + } + String filename = rootObject.has("fileName") ? rootObject.getString("fileName") : "ExcelExport.xls"; + ExcelWriter.ExcelDocumentType docType = filename.toLowerCase().endsWith(".xlsx") ? ExcelWriter.ExcelDocumentType.xlsx : ExcelWriter.ExcelDocumentType.xls; + + try (Workbook workbook = ExcelFactory.createFromArray(sheetsArray, docType)) + { + response.setContentType(docType.getMimeType()); + ResponseHelper.setContentDisposition(response, ResponseHelper.ContentDispositionType.attachment, filename); + ResponseHelper.setPrivate(response); + workbook.write(response.getOutputStream()); + + JSONObject qInfo = rootObject.has("queryinfo") ? rootObject.getJSONObject("queryinfo") : null; + if (qInfo != null) + { + QueryService.get().addAuditEvent(getUser(), getContainer(), qInfo.getString("schema"), + qInfo.getString("query"), getViewContext().getActionURL(), + rootObject.getString("auditMessage") + filename, + null); + SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "convertTable", "asExcel"); + } + } + } + catch (JSONException | ClassCastException e) + { + // We can get a ClassCastException if we expect an array and get a simple String, for example + ExceptionUtil.renderErrorView(getViewContext(), getPageConfig(), ErrorRenderer.ErrorType.notFound, HttpServletResponse.SC_BAD_REQUEST, "Failed to convert to Excel - invalid input", e, false, false); + } + } + } + + @RequiresPermission(ReadPermission.class) + public static class ConvertArraysToTableAction extends ExportAction + { + @Override + public void validate(ConvertArraysToExcelForm form, BindException errors) + { + if (form.getJson() == null) + { + errors.reject(ERROR_MSG, "Unable to convert to table - no data given"); + } + } + + @Override + public void export(ConvertArraysToExcelForm form, HttpServletResponse response, BindException errors) throws Exception + { + try + { + JSONObject rootObject; + JSONArray rowsArray; + if (form.getJson() == null || form.getJson().trim().isEmpty()) + { + // Create JSON so that we return an empty file + rootObject = new JSONObject(); + rowsArray = new JSONArray(); + } + else + { + rootObject = new JSONObject(form.getJson()); + rowsArray = rootObject.getJSONArray("rows"); + } + + TSVWriter.DELIM delimType = (!rootObject.isNull("delim") ? TSVWriter.DELIM.valueOf(rootObject.getString("delim")) : TSVWriter.DELIM.TAB); + TSVWriter.QUOTE quoteType = (!rootObject.isNull("quoteChar") ? TSVWriter.QUOTE.valueOf(rootObject.getString("quoteChar")) : TSVWriter.QUOTE.NONE); + String filenamePrefix = (!rootObject.isNull("fileNamePrefix") ? rootObject.getString("fileNamePrefix") : "Export"); + String filename = filenamePrefix + "." + delimType.extension; + String newlineChar = !rootObject.isNull("newlineChar") ? rootObject.getString("newlineChar") : "\n"; + + PageFlowUtil.prepareResponseForFile(response, Collections.emptyMap(), filename, true); + response.setContentType(delimType.contentType); + + //NOTE: we could also have used TSVWriter; however, this is in use elsewhere and we dont need a custom subclass + try (CSVWriter writer = new CSVWriter(response.getWriter(), delimType.delim, quoteType.quoteChar, newlineChar)) + { + for (int i = 0; i < rowsArray.length(); i++) + { + List objectList = rowsArray.getJSONArray(i).toList(); + Iterator it = objectList.iterator(); + List list = new ArrayList<>(); + + while (it.hasNext()) + { + Object o = it.next(); + if (o != null) + list.add(o.toString()); + else + list.add(""); + } + + writer.writeNext(list.toArray(new String[0])); + } + } + + JSONObject qInfo = rootObject.optJSONObject("queryinfo"); + if (qInfo != null) + { + QueryService.get().addAuditEvent(getUser(), getContainer(), qInfo.getString("schema"), qInfo.getString("query"), + getViewContext().getActionURL(), + rootObject.getString("auditMessage") + filename, + rowsArray.length()); + SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "convertTable", "asDelimited"); + } + } + catch (JSONException e) + { + ExceptionUtil.renderErrorView(getViewContext(), getPageConfig(), ErrorRenderer.ErrorType.notFound, HttpServletResponse.SC_BAD_REQUEST, "Failed to convert to table - invalid input", e, false, false); + } + } + } + + + public static class ConvertHtmlToExcelForm + { + private String _baseUrl; + private String _htmlFragment; + private String _name = "workbook.xls"; + + + public String getName() + { + return _name; + } + + public void setName(String name) + { + _name = name; + } + + public String getBaseUrl() + { + return _baseUrl; + } + + public void setBaseUrl(String baseUrl) + { + _baseUrl = baseUrl; + } + + public String getHtmlFragment() + { + return _htmlFragment; + } + + public void setHtmlFragment(String htmlFragment) + { + _htmlFragment = htmlFragment; + } + } + + + @RequiresPermission(ReadPermission.class) + public static class ConvertHtmlToExcelAction extends FormViewAction + { + String _responseHtml = null; + + @Override + public void validateCommand(ConvertHtmlToExcelForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(ConvertHtmlToExcelForm form, boolean reshow, BindException errors) + { + String html = + "

    " + + "" + + new CsrfInput(getViewContext()) + + "
    "; + return HtmlView.unsafe(html); + } + + @Override + public boolean handlePost(ConvertHtmlToExcelForm form, BindException errors) + { + ActionURL url = getViewContext().getActionURL(); + String base = url.getBaseServerURI(); + if (!base.endsWith("/")) base += "/"; + + String baseTag = ""; + SafeToRender css = PageFlowUtil.getStylesheetIncludes(getContainer()); + String htmlFragment = StringUtils.trimToEmpty(form.getHtmlFragment()); + String html = "" + baseTag + css + "" + htmlFragment + ""; + + // UNDONE: strip script + List tidyErrors = new ArrayList<>(); + String tidy = JSoupUtil.tidyHTML(html, false, tidyErrors); + + if (!tidyErrors.isEmpty()) + { + for (String err : tidyErrors) + { + errors.reject(ERROR_MSG, err); + } + return false; + } + + _responseHtml = tidy; + return true; + } + + @Override + public ModelAndView getSuccessView(ConvertHtmlToExcelForm form) + { + ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, form.getName()); + getPageConfig().setTemplate(PageConfig.Template.None); + HtmlView v = HtmlView.unsafe(_responseHtml); + v.setContentType("application/vnd.ms-excel"); + v.setFrame(WebPartView.FrameType.NONE); + return v; + } + + @Override + public URLHelper getSuccessURL(ConvertHtmlToExcelForm convertHtmlToExcelForm) + { + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + + public static ActionURL getShowApplicationURL(Container c, long rowId) + { + ActionURL url = new ActionURL(ShowApplicationAction.class, c); + url.addParameter("rowId", rowId); + + return url; + } + + + @RequiresPermission(ReadPermission.class) + public class ShowApplicationAction extends SimpleViewAction + { + private ExpProtocolApplicationImpl _app; + private ExpRun _run; + + @Override + public ModelAndView getView(ExpObjectForm form, BindException errors) + { + _app = ExperimentServiceImpl.get().getExpProtocolApplication(form.getRowId()); + if (_app == null) + { + throw new NotFoundException("Could not find Protocol Application"); + } + _run = _app.getRun(); + if (_run == null) + { + throw new NotFoundException("No experiment run associated with Protocol Application"); + } + ensureCorrectContainer(getContainer(), _app, getViewContext()); + + ExpProtocol protocol = _app.getProtocol(); + + DataRegion dr = new DataRegion(); + dr.addColumns(ExperimentServiceImpl.get().getTinfoProtocolApplication().getUserEditableColumns()); + DetailsView detailsView = new DetailsView(dr, form.getRowId()); + dr.removeColumns("RunId", "ProtocolLSID", "RowId", "LSID"); + dr.addDisplayColumn(new ExperimentRunDisplayColumn(_run)); + dr.addDisplayColumn(new ProtocolDisplayColumn(protocol)); + dr.addDisplayColumn(new LineageGraphDisplayColumn(_app, _run)); + detailsView.setTitle("Protocol Application"); + + Container c = getContainer(); + ApplicationOutputGrid outMGrid = new ApplicationOutputGrid(c, _app.getRowId(), ExperimentServiceImpl.get().getTinfoMaterial()); + ApplicationOutputGrid outDGrid = new ApplicationOutputGrid(c, _app.getRowId(), ExperimentServiceImpl.get().getTinfoData()); + Map map = new HashMap<>(); + for (ProtocolApplicationParameter param : ExperimentService.get().getProtocolApplicationParameters(_app.getRowId())) + { + map.put(param.getOntologyEntryURI(), param); + } + + JspView> paramsView = new JspView<>("/org/labkey/experiment/Parameters.jsp", map); + paramsView.setTitle("Protocol Application Parameters"); + CustomPropertiesView cpv = new CustomPropertiesView(_app.getLSID(), c); + return new VBox(new StandardAndCustomPropertiesView(detailsView, cpv), paramsView, outMGrid, outDGrid); + } + + @Override + public void addNavTrail(NavTree root) + { + addRootNavTrail(root); + root.addChild("Experiment Run", ExperimentUrlsImpl.get().getRunGraphDetailURL(_run)); + root.addChild("Protocol Application " + _app.getName()); + } + } + + @RequiresPermission(ReadPermission.class) + public class ShowProtocolGridAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new ProtocolWebPart(false, getViewContext()); + } + + @Override + public void addNavTrail(NavTree root) + { + addRootNavTrail(root); + root.addChild("Protocols"); + } + } + + @RequiresPermission(ReadPermission.class) + public class ProtocolDetailsAction extends SimpleViewAction + { + private ExpProtocolImpl _protocol; + + @Override + public ModelAndView getView(ExpObjectForm form, BindException errors) + { + _protocol = ExperimentServiceImpl.get().getExpProtocol(form.getRowId()); + if (_protocol == null) + { + _protocol = ExperimentServiceImpl.get().getExpProtocol(form.getLSID()); + } + + if (_protocol == null) + { + throw new NotFoundException("Unable to find a matching protocol"); + } + ensureCorrectContainer(getContainer(), _protocol, getViewContext()); + + JspView detailsView = new JspView<>("/org/labkey/experiment/ProtocolDetails.jsp", _protocol); + detailsView.setTitle("Standard Properties"); + + CustomPropertiesView cpv = new CustomPropertiesView(_protocol.getLSID(), getContainer()); + ProtocolParametersView parametersView = new ProtocolParametersView(_protocol); + + VBox protocolDetails = new VBox(); + protocolDetails.setFrame(WebPartView.FrameType.PORTAL); + protocolDetails.setTitle("Protocol Details"); + protocolDetails.addView(new ProtocolInputOutputsView(_protocol, errors)); + + JspView stepsView = new JspView<>("/org/labkey/experiment/ProtocolSteps.jsp", _protocol); + stepsView.setTitle("Protocol Steps"); + stepsView.setFrame(WebPartView.FrameType.TITLE); + protocolDetails.addView(stepsView); + + ExpSchema schema = new ExpSchema(getUser(), getContainer()); + ExperimentRunListView runView = new ExperimentRunListView(schema, ExperimentRunListView.getRunListQuerySettings(schema, getViewContext(), ExpSchema.TableType.Runs.name(), true), ExperimentRunType.ALL_RUNS_TYPE) + { + @Override + public DataView createDataView() + { + DataView result = super.createDataView(); + result.getRenderContext().setBaseFilter(new SimpleFilter(FieldKey.fromParts("Protocol", "LSID"), _protocol.getLSID())); + return result; + } + }; + + runView.setTitle("Runs Using This Protocol"); + + return new VBox(new StandardAndCustomPropertiesView(detailsView, cpv), parametersView, protocolDetails, runView); + } + + @Override + public void addNavTrail(NavTree root) + { + addRootNavTrail(root); + root.addChild("Protocols", ExperimentUrlsImpl.get().getProtocolGridURL(getContainer())); + root.addChild("Protocol: " + _protocol.getName()); + } + } + + public class ProtocolInputOutputsView extends VBox + { + ProtocolInputOutputsView(ExpProtocol protocol, Errors errors) + { + HBox inputsView = new HBox(); + addView(inputsView); + + HBox outputsView = new HBox(); + addView(outputsView); + + UserSchema expSchema = QueryService.get().getUserSchema(getUser(), getContainer(), ExpSchema.SCHEMA_NAME); + + class ProtocolInputGrid extends QueryView + { + public ProtocolInputGrid(String title, QuerySettings settings, @Nullable Errors errors) + { + super(expSchema, settings, errors); + + setFrame(FrameType.TITLE); + setTitle(title); + setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); + setShowBorders(true); + setShadeAlternatingRows(true); + setShowExportButtons(false); + setShowPagination(false); + disableContainerFilterSelection(); + } + } + + // INPUTS + + QuerySettings materialInputsSettings = expSchema.getSettings("mpi", ExpSchema.TableType.MaterialProtocolInputs.toString()); + materialInputsSettings.getBaseFilter().addCondition(FieldKey.fromParts(ExpMaterialProtocolInputTable.Column.Protocol.toString()), protocol.getRowId()); + materialInputsSettings.setFieldKeys(Arrays.asList( + FieldKey.fromParts(ExpMaterialProtocolInputTable.Column.Name.toString()), + FieldKey.fromParts(ExpMaterialProtocolInputTable.Column.SampleSet.toString()) + )); + QueryView materialInputsView = new ProtocolInputGrid("Material Inputs", materialInputsSettings, errors); + inputsView.addView(materialInputsView); + + QuerySettings dataInputsSettings = expSchema.getSettings("dpi", ExpSchema.TableType.DataProtocolInputs.toString()); + dataInputsSettings.getBaseFilter().addCondition(FieldKey.fromParts(ExpDataProtocolInputTable.Column.Protocol.toString()), protocol.getRowId()); + dataInputsSettings.setFieldKeys(Arrays.asList( + FieldKey.fromParts(ExpDataProtocolInputTable.Column.Name.toString()), + FieldKey.fromParts(ExpDataProtocolInputTable.Column.DataClass.toString()) + )); + QueryView dataInputsView = new ProtocolInputGrid("Data Inputs", dataInputsSettings, errors); + inputsView.addView(dataInputsView); + + // OUTPUTS + + QuerySettings materialOutputsSettings = expSchema.getSettings("mpo", ExpSchema.TableType.MaterialProtocolInputs.toString()); + materialOutputsSettings.getBaseFilter().addCondition(FieldKey.fromParts(ExpMaterialProtocolInputTable.Column.Protocol.toString()), protocol.getRowId()); + materialOutputsSettings.setFieldKeys(Arrays.asList( + FieldKey.fromParts(ExpMaterialProtocolInputTable.Column.Name.toString()), + FieldKey.fromParts(ExpMaterialProtocolInputTable.Column.SampleSet.toString()) + )); + QueryView materialOutputsView = new ProtocolInputGrid("Material Outputs", materialOutputsSettings, errors); + outputsView.addView(materialOutputsView); + + QuerySettings dataOutputsSettings = expSchema.getSettings("dpo", ExpSchema.TableType.DataProtocolInputs.toString()); + dataOutputsSettings.getBaseFilter().addCondition(FieldKey.fromParts(ExpDataProtocolInputTable.Column.Protocol.toString()), protocol.getRowId()); + dataOutputsSettings.setFieldKeys(Arrays.asList( + FieldKey.fromParts(ExpDataProtocolInputTable.Column.Name.toString()), + FieldKey.fromParts(ExpDataProtocolInputTable.Column.DataClass.toString()) + )); + QueryView dataOutputsView = new ProtocolInputGrid("Data Outputs", dataOutputsSettings, errors); + outputsView.addView(dataOutputsView); + } + } + + + @RequiresPermission(ReadPermission.class) + public class ProtocolPredecessorsAction extends SimpleViewAction + { + private ExpProtocol _parentProtocol; + private ProtocolActionStepDetail _actionStep; + + @Override + public ModelAndView getView(Object o, BindException errors) + { + ActionURL url = getViewContext().getActionURL(); + + String parentProtocolLSID = url.getParameter("ParentLSID"); + int actionSequence; + try + { + actionSequence = Integer.parseInt(url.getParameter("Sequence")); + } + catch (NumberFormatException e) + { + throw new NotFoundException("Could not find SequenceId " + url.getParameter("Sequence")); + } + + _parentProtocol = ExperimentService.get().getExpProtocol(parentProtocolLSID); + if (_parentProtocol == null) + { + throw new NotFoundException("Unable to find a matching protocol"); + } + + ensureCorrectContainer(getContainer(), _parentProtocol, getViewContext()); + + _actionStep = ExperimentServiceImpl.get().getProtocolActionStepDetail(parentProtocolLSID, actionSequence); + + if (_actionStep == null) + { + throw new NotFoundException("Unable to find a matching protocol action step"); + } + + ExpProtocol childProtocol = ExperimentService.get().getExpProtocol(_actionStep.getChildProtocolLSID()); + + JspView detailsView = new JspView<>("/org/labkey/experiment/ProtocolDetails.jsp", childProtocol); + detailsView.setTitle("Standard Properties"); + + CustomPropertiesView cpv = new CustomPropertiesView(childProtocol.getLSID(), getContainer()); + + ProtocolParametersView parametersView = new ProtocolParametersView(childProtocol); + + VBox protocolDetails = new VBox(); + protocolDetails.setFrame(WebPartView.FrameType.PORTAL); + protocolDetails.setTitle("Protocol Details"); + protocolDetails.addView(new ProtocolInputOutputsView(childProtocol, errors)); + protocolDetails.addView(new ProtocolSuccessorPredecessorView(parentProtocolLSID, actionSequence, getContainer(), "PredecessorChildLSID", "PredecessorSequence", "ActionSequence", "Protocol Predecessors")); + protocolDetails.addView(new ProtocolSuccessorPredecessorView(parentProtocolLSID, actionSequence, getContainer(), "ChildProtocolLSID", "ActionSequence", "PredecessorSequence", "Protocol Successors")); + + return new VBox(new StandardAndCustomPropertiesView(detailsView, cpv), parametersView, protocolDetails); + } + + @Override + public void addNavTrail(NavTree root) + { + addRootNavTrail(root); + root.addChild("Protocols", ExperimentUrlsImpl.get().getProtocolGridURL(getContainer())); + root.addChild("Parent Protocol '" + _parentProtocol.getName() + "'", ExperimentUrlsImpl.get().getProtocolDetailsURL(_parentProtocol)); + root.addChild("Protocol Step: " + _actionStep.getName()); + } + } + + public static class DataForm + { + private boolean _inline; + private long _rowId; + private String _lsid; + private Integer _maxDimension; + private String _format; + + public boolean isInline() + { + return _inline; + } + + public void setInline(boolean inline) + { + _inline = inline; + } + + public long getRowId() + { + return _rowId; + } + + public void setRowId(long rowId) + { + _rowId = rowId; + } + + public String getLsid() + { + return _lsid; + } + + public void setLsid(String lsid) + { + _lsid = lsid; + } + + public ExpDataImpl lookupData() + { + ExpDataImpl result = ExperimentServiceImpl.get().getExpData(getRowId()); + if (result == null && getLsid() != null) + { + result = ExperimentServiceImpl.get().getExpData(getLsid()); + } + return result; + } + + public Integer getMaxDimension() + { + return _maxDimension; + } + + public void setMaxDimension(Integer maxDimension) + { + _maxDimension = maxDimension; + } + + public String getFormat() + { + return _format; + } + + public void setFormat(String format) + { + _format = format; + } + } + + public static class ExpObjectForm extends QueryViewAction.QueryExportForm + { + private long _rowId; + private String _lsid; + + public String getLsid() + { + return _lsid; + } + + public void setLsid(String lsid) + { + _lsid = lsid; + } + + public String getLSID() + { + return getLsid(); + } + + public void setLSID(String lsid) + { + setLsid(lsid); + } + + public long getRowId() + { + return _rowId; + } + + public void setRowId(long rowId) + { + _rowId = rowId; + } + } + + @RequiresAnyOf({DeletePermission.class, SampleWorkflowDeletePermission.class}) + public class DeleteSelectedExpRunsAction extends AbstractDeleteAction + { + @Override + public void addNavTrail(NavTree root) + { + // UNDONE: Need help topic on Runs + setHelpTopic("experiment"); + super.addNavTrail(root); + } + + @Override + public ModelAndView getView(DeleteForm deleteForm, boolean reshow, BindException errors) + { + List runs = new ArrayList<>(); + + Map idToRunMap = new LongHashMap<>(); + for (long runId : deleteForm.getIds(false)) + { + ExpRun run = ExperimentService.get().getExpRun(runId); + if (run != null) + { + if (!run.canDelete(getUser())) + throw new UnauthorizedException("You do not have permission to delete " + + (ExpProtocol.isSampleWorkflowProtocol(run.getProtocol().getLSID()) ? "jobs" : "runs") + + " in " + run.getContainer()); + + runs.add(run); + idToRunMap.put(run.getRowId(), run); + } + } + + Map referencedItems = new LongHashMap<>(); + List referenceDescriptions = new ArrayList<>(); + AssayService assayService = AssayService.get(); + if (!idToRunMap.isEmpty() && assayService != null ) + { + // using the first run as a representative, since all interactions here are (I believe) using the same protocol. + ExpProtocol protocol = runs.get(0).getProtocol(); + AssayProvider provider = assayService.getProvider(protocol); + if (provider != null) + { + SchemaKey key = AssayProtocolSchema.schemaName(provider, protocol); + ExperimentService.get().getObjectReferencers() + .forEach(referencer -> { + Collection referenced = referencer.getItemsWithReferences( + idToRunMap.keySet(), + key.toString(), + "Runs" + ); + referenced.forEach(id -> referencedItems.put(id, idToRunMap.get(id))); + referenceDescriptions.add(referencer.getObjectReferenceDescription(ExpRun.class)); + } + ); + } + + } + + List> permissionDatasetRows = new ArrayList<>(); + List> noPermissionDatasetRows = new ArrayList<>(); + if (StudyPublishService.get() != null) + { + for (Dataset dataset : StudyPublishService.get().getDatasetsForAssayRuns(runs, getUser())) + { + ActionURL url = urlProvider(StudyUrls.class).getDatasetURL(dataset.getContainer(), dataset.getDatasetId()); + TableInfo t = dataset.getTableInfo(getUser()); + if (null != t && t.hasPermission(getUser(),DeletePermission.class)) + { + permissionDatasetRows.add(new Pair<>(dataset, url)); + } + else + { + noPermissionDatasetRows.add(new Pair<>(dataset, url)); + } + } + } + + return new ConfirmDeleteView( + "run", + ShowRunGraphAction.class, + runs.stream().filter(run -> !referencedItems.containsKey(run.getRowId())).toList(), + deleteForm, + Collections.emptyList(), + "dataset(s) have one or more rows which", + permissionDatasetRows, + noPermissionDatasetRows, + referencedItems.values().stream().toList(), + referenceDescriptions.stream().filter(Objects::nonNull).collect(Collectors.joining(", or "))); + } + + @Override + protected void deleteObjects(DeleteForm deleteForm) + { + ExperimentServiceImpl.get().deleteExperimentRunsByRowIds(getContainer(), getUser(), deleteForm.getUserComment(), deleteForm.getIds(false)); + } + } + + public static class DeleteRunForm + { + private int _runId; + + public int getRunId() + { + return _runId; + } + + public void setRunId(int runId) + { + _runId = runId; + } + } + + /** + * Separate delete action from the client API + */ + @RequiresAnyOf({DeletePermission.class, SampleWorkflowDeletePermission.class}) + public static class DeleteRunAction extends MutatingApiAction + { + @Override + public ApiResponse execute(DeleteRunForm form, BindException errors) + { + ExpRun run = ExperimentService.get().getExpRun(form.getRunId()); + if (run == null) + { + throw new NotFoundException("Could not find run with ID " + form.getRunId()); + } + if (!run.canDelete(getUser())) + throw new UnauthorizedException("You do not have permission to delete " + + (ExpProtocol.isSampleWorkflowProtocol(run.getProtocol().getLSID()) ? "jobs" : "runs") + " in this container."); + + run.delete(getUser()); + return new ApiSimpleResponse("success", true); + } + } + + + @RequiresAnyOf({DeletePermission.class, SampleWorkflowDeletePermission.class}) + public static class DeleteRunsAction extends AbstractDeleteAPIAction + { + @Override + protected ApiSimpleResponse deleteObjects(CascadeDeleteForm form) + { + Set runIdsToDelete = new HashSet<>(form.getIds(true)); + Set runIdsCascadeDeleted = new HashSet<>(); + + if (form.isCascade()) + { + for (long runId : runIdsToDelete) + { + ExpRun run = ExperimentService.get().getExpRun(runId); + if (run != null) + addReplacesRuns(run, runIdsCascadeDeleted); + } + + if (!runIdsCascadeDeleted.isEmpty()) + runIdsToDelete.addAll(runIdsCascadeDeleted); + } + + ExperimentService.get().deleteExperimentRunsByRowIds(getContainer(), getUser(), form.getUserComment(), runIdsToDelete); + + ApiSimpleResponse response = new ApiSimpleResponse("success", true); + response.put("runIdsDeleted", runIdsToDelete); + if (!runIdsCascadeDeleted.isEmpty()) + response.put("runIdsCascadeDeleted", runIdsCascadeDeleted); + return response; + } + + private void addReplacesRuns(ExpRun run, Set runIds) + { + for (ExpRun replacedRun : run.getReplacesRuns()) + { + runIds.add(replacedRun.getRowId()); + addReplacesRuns(replacedRun, runIds); + } + } + } + + private abstract static class AbstractDeleteAPIAction extends MutatingApiAction + { + @Override + public void validateForm(CascadeDeleteForm form, Errors errors) + { + if (form.getSingleObjectRowId() == null && form.getDataRegionSelectionKey() == null && form.getRowIds() == null) + errors.reject(ERROR_REQUIRED, "Either singleObjectRowId, dataRegionSelectionKey, or rowIds is required"); + } + + @Override + public ApiResponse execute(CascadeDeleteForm form, BindException errors) throws Exception + { + ApiSimpleResponse response; + + try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) + { + tx.addCommitTask(form::clearSelected, POSTCOMMIT); + + response = deleteObjects(form); + tx.commit(); + } + + if (null != response.get("success")) + response.put("success", !errors.hasErrors()); + + return response; + } + + protected abstract ApiSimpleResponse deleteObjects(CascadeDeleteForm form) throws Exception; + } + + public static class CascadeDeleteForm extends DeleteForm + { + private boolean _cascade; + + public boolean isCascade() + { + return _cascade; + } + + public void setCascade(boolean cascade) + { + _cascade = cascade; + } + } + + private abstract static class AbstractDeleteAction extends FormViewAction + { + @Override + public void validateCommand(DeleteForm target, Errors errors) + { + } + + @Override + public boolean handlePost(DeleteForm deleteForm, BindException errors) throws Exception + { + if (!deleteForm.isForceDelete()) + { + return false; + } + else + { + try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) + { + tx.addCommitTask(deleteForm::clearSelected, POSTCOMMIT); + + deleteObjects(deleteForm); + tx.commit(); + } + catch (BatchValidationException v) + { + v.addToErrors(errors); + } + + return !errors.hasErrors(); + } + } + + @Override + public ActionURL getSuccessURL(DeleteForm form) + { + return form.getSuccessActionURL(ExperimentUrlsImpl.get().getOverviewURL(getContainer())); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Confirm Deletion"); + } + + protected abstract void deleteObjects(DeleteForm form) throws Exception; + } + + @RequiresPermission(DesignAssayPermission.class) + public static class DeleteProtocolByRowIdsAPIAction extends AbstractDeleteAPIAction + { + @Override + protected ApiSimpleResponse deleteObjects(CascadeDeleteForm form) + { + for (ExpProtocol protocol : getProtocolsForDeletion(form)) + { + if (!protocol.getContainer().hasPermission(getUser(), DesignAssayPermission.class)) + throw new UnauthorizedException("You do not have sufficient permissions to delete this assay design."); + + protocol.delete(getUser(), form.getUserComment()); + } + + return new ApiSimpleResponse(); + } + } + + public static List getProtocolsForDeletion(DeleteForm form) + { + List protocols = new ArrayList<>(); + for (long protocolId : form.getIds(false)) + { + ExpProtocol protocol = ExperimentService.get().getExpProtocol(protocolId); + if (protocol != null) + { + protocols.add(protocol); + } + } + return protocols; + } + + @RequiresPermission(DesignAssayPermission.class) + public class DeleteProtocolByRowIdsAction extends AbstractDeleteAction + { + @Override + public void addNavTrail(NavTree root) + { + // UNDONE: Need help topic on protocols + setHelpTopic("experiment"); + super.addNavTrail(root); + } + + @Override + public ModelAndView getView(DeleteForm form, boolean reshow, BindException errors) + { + List runs = ExperimentService.get().getExpRunsForProtocolIds(false, form.getIds(false)); + List protocols = getProtocolsForDeletion(form); + String noun = "Assay Design"; + List> deleteableDatasets = new ArrayList<>(); + List> noPermissionDatasets = new ArrayList<>(); + if (AssayService.get() != null && StudyService.get() != null) + { + for (ExpProtocol protocol : protocols) + { + if (!protocol.getContainer().hasPermission(getUser(), DesignAssayPermission.class)) + throw new UnauthorizedException("You do not have sufficient permissions to delete this assay design."); + + if (AssayService.get().getProvider(protocol) == null) + { + noun = "Protocol"; + } + for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(protocol.getRowId(), Dataset.PublishSource.Assay)) + { + Pair entry = new Pair<>(dataset, urlProvider(StudyUrls.class).getDatasetURL(dataset.getContainer(), dataset.getDatasetId())); + if (dataset.canDeleteDefinition(getUser())) + { + deleteableDatasets.add(entry); + } + else + { + noPermissionDatasets.add(entry); + } + } + } + } + + return new ConfirmDeleteView(noun, ProtocolDetailsAction.class, protocols, form, runs, "Dataset", deleteableDatasets, noPermissionDatasets, Collections.emptyList(), null); + } + + @Override + protected void deleteObjects(DeleteForm form) + { + for (ExpProtocol protocol : getProtocolsForDeletion(form)) + { + protocol.delete(getUser(), form.getUserComment()); + } + } + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(ReadPermission.class) + public static class GetDataOperationConfirmationDataAction extends ReadOnlyApiAction + { + @Override + public void validateForm(DataOperationConfirmationForm form, Errors errors) + { + if (form.getDataRegionSelectionKey() == null && form.getRowIds() == null) + errors.reject(ERROR_REQUIRED, "You must provide either a set of rowIds or a dataRegionSelectionKey"); + if (form.getDataOperation() == null) + errors.reject(ERROR_REQUIRED, "An operation type must be provided."); + } + + @Override + public Object execute(DataOperationConfirmationForm form, BindException errors) + { + Collection requestIds = form.getIds(false); + ExperimentServiceImpl service = ExperimentServiceImpl.get(); + List allData = service.getExpDatas(requestIds); + + Set notAllowedIds = new HashSet<>(); + if (form.getDataOperation() == ExpDataImpl.DataOperations.Delete) + service.getObjectReferencers().forEach(referencer -> + notAllowedIds.addAll(referencer.getItemsWithReferences(requestIds, "exp.data"))); + + Map>> response = ExperimentServiceImpl.partitionRequestedOperationObjects(getUser(), requestIds, notAllowedIds, allData); + + Collection containers = new HashSet<>(); + Collection notPermittedIds = new ArrayList<>(); + Class permClass = form.getDataOperation().getPermissionClass(); + for (ExpDataImpl expData : allData) + { + Container c = expData.getContainer(); + if (c.hasPermission(getUser(), ReadPermission.class)) + containers.add(c); + if (permClass != null && !c.hasPermission(getUser(), permClass)) + notPermittedIds.add(expData.getRowId()); + } + + NameExpressionOptionService svc = NameExpressionOptionService.get(); + response.put("containers", containers.stream().map(c -> Map.of( + "id", c.getEntityId(), + "path", (Object) c.getPath(), + "permitted", permClass == null || c.hasPermission(getUser(), permClass), + "canEditName", svc.getAllowUserSpecificNamesValue(c) + )).toList()); + + response.put("notPermitted", notPermittedIds.stream().map(id -> Map.of("RowId", (Object) id)).toList()); + + return success(response); + } + } + + + public static class DataOperationConfirmationForm extends DataViewSnapshotSelectionForm + { + private ExpDataImpl.DataOperations _dataOperation; + + public ExpDataImpl.DataOperations getDataOperation() + { + return _dataOperation; + } + + public void setDataOperation(ExpDataImpl.DataOperations dataOperation) + { + _dataOperation = dataOperation; + } + + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(ReadPermission.class) + public static class GetMaterialOperationConfirmationDataAction extends ReadOnlyApiAction + { + @Override + public void validateForm(MaterialOperationConfirmationForm form, Errors errors) + { + if (form.getDataRegionSelectionKey() == null && form.getRowIds() == null) + errors.reject(ERROR_REQUIRED, "You must provide either a set of rowIds or a dataRegionSelectionKey."); + if (form.getSampleOperation() == null) + errors.reject(ERROR_REQUIRED, "An operation type must be provided."); + } + + @Override + public Object execute(MaterialOperationConfirmationForm form, BindException errors) + { + Set requestIds = form.getIds(false); + ExperimentServiceImpl service = ExperimentServiceImpl.get(); + List allMaterials = service.getExpMaterials(requestIds); + + Set notAllowedIds = new HashSet<>(); + // We prevent deletion if a sample is used as a parent, has assay data, is used in a job, etc. + if (form.getSampleOperation() == SampleTypeService.SampleOperations.Delete) + service.getObjectReferencers().forEach(referencer -> + notAllowedIds.addAll(referencer.getItemsWithReferences(requestIds, "samples"))); + + if (SampleStatusService.get().supportsSampleStatus()) + notAllowedIds.addAll(service.findIdsNotPermittedForOperation(allMaterials, form.getSampleOperation())); + + Map>> response = ExperimentServiceImpl.partitionRequestedOperationObjects(getUser(), requestIds, notAllowedIds, allMaterials); + + Collection containers = new HashSet<>(); + Collection notPermittedIds = new ArrayList<>(); + Class permClass = form.getSampleOperation().getPermissionClass(); + for (ExpMaterial material : allMaterials) + { + Container c = material.getContainer(); + if (c.hasPermission(getUser(), ReadPermission.class)) + containers.add(c); + if (permClass != null && !c.hasPermission(getUser(), permClass)) + notPermittedIds.add(material.getRowId()); + } + + NameExpressionOptionService svc = NameExpressionOptionService.get(); + + response.put("containers", containers.stream().map(c -> Map.of( + "id", c.getEntityId(), + "path", (Object) c.getPath(), + "permitted", permClass == null || c.hasPermission(getUser(), permClass), + "canEditName", svc.getAllowUserSpecificNamesValue(c) + )).toList()); + + response.put("notPermitted", notPermittedIds.stream().map(id -> Map.of("RowId", (Object) id)).toList()); + + if (form.getSampleOperation() == SampleTypeService.SampleOperations.Delete) + // String 'associatedDatasets' must be synced to its handling in confirmDelete.js, confirmDelete() + response.put("associatedDatasets", ExperimentServiceImpl.includeLinkedToStudyText(allMaterials, requestIds, getUser(), getContainer())); + + return success(response); + } + } + + public static class MaterialOperationConfirmationForm extends DataViewSnapshotSelectionForm + { + private SampleTypeService.SampleOperations _sampleOperation; + + public SampleTypeService.SampleOperations getSampleOperation() + { + return _sampleOperation; + } + + public void setSampleOperation(SampleTypeService.SampleOperations sampleOperation) + { + _sampleOperation = sampleOperation; + } + } + + @RequiresPermission(DeletePermission.class) + public class DeleteSelectedDataAction extends AbstractDeleteAction + { + @Override + public void addNavTrail(NavTree root) + { + // UNDONE: Need help topic on Datas + setHelpTopic("experiment"); + super.addNavTrail(root); + } + + @Override + protected void deleteObjects(DeleteForm deleteForm) throws Exception + { + List datas = getDatas(deleteForm, false); + + for (ExpRun run : getRuns(datas)) + { + if (!run.getContainer().hasPermission(getUser(), DeletePermission.class)) + throw new UnauthorizedException(); + } + + // Issue 32076: Delete the exp.Data objects using QueryUpdateService so trigger scripts will be executed + Map, List> byDataClass = datas.stream().collect(Collectors.groupingBy(d -> Optional.ofNullable(d.getDataClass(null)))); + for (Optional opt : byDataClass.keySet()) + { + SchemaKey schemaKey; + String queryName; + ExpDataClass dc = opt.orElse(null); + List ds = byDataClass.get(opt); + if (dc == null) + { + // Reference to exp.Data table + schemaKey = ExpSchema.SCHEMA_EXP; + queryName = ExpSchema.TableType.Data.name(); + } + else + { + // Reference to exp.data. table + schemaKey = ExpSchema.SCHEMA_EXP_DATA; + queryName = dc.getName(); + } + + UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaKey); + if (schema == null) + throw new IllegalStateException("Failed to get schema '" + schemaKey + "'"); + + TableInfo table = schema.getTable(queryName); + if (table == null) + throw new IllegalStateException("Failed to get table '" + queryName + "' in schema '" + schemaKey + "'"); + + QueryUpdateService qus = table.getUpdateService(); + if (qus == null) + throw new IllegalStateException(); + + qus.deleteRows(getUser(), getContainer(), toKeys(ds), null, null); + } + } + + protected List> toKeys(List datas) + { + return datas.stream().map(d -> CaseInsensitiveHashMap.of("rowId", d.getRowId())).collect(toList()); + } + + @Override + public ModelAndView getView(DeleteForm deleteForm, boolean reshow, BindException errors) + { + if (errors.hasErrors()) + return new SimpleErrorView(errors, false); + + List datas = getDatas(deleteForm, false); + List runs = getRuns(datas); + + return new ConfirmDeleteView("Data", ShowDataAction.class, datas, deleteForm, runs); + } + + private List getRuns(List datas) + { + List runArray = ExperimentService.get().getRunsUsingDatas(datas); + return new ArrayList<>(ExperimentService.get().runsDeletedWithInput(runArray)); + } + + private List getDatas(DeleteForm deleteForm, boolean clear) + { + List datas = new ArrayList<>(); + for (long dataId : deleteForm.getIds(clear)) + { + ExpData data = ExperimentService.get().getExpData(dataId); + if (data != null) + { + datas.add(data); + } + } + return datas; + } + } + + @RequiresPermission(DeletePermission.class) + public class DeleteSelectedExperimentsAction extends AbstractDeleteAction + { + @Override + protected void deleteObjects(DeleteForm deleteForm) + { + for (ExpExperiment exp : lookupExperiments(deleteForm)) + { + exp.delete(getUser()); + } + } + + @Override + public ModelAndView getView(DeleteForm deleteForm, boolean reshow, BindException errors) + { + List experiments = lookupExperiments(deleteForm); + + List runs = new ArrayList<>(); + boolean allBatches = true; + for (ExpExperiment experiment : experiments) + { + // Deleting a batch also deletes all of its runs + if (experiment.getBatchProtocol() != null) + { + runs.addAll(experiment.getRuns()); + } + else + { + allBatches = false; + } + } + + return new ConfirmDeleteView(allBatches ? "batch" : "run group", DetailsAction.class, experiments, deleteForm, runs); + } + + private List lookupExperiments(DeleteForm deleteForm) + { + List experiments = new ArrayList<>(); + for (long experimentId : deleteForm.getIds(false)) + { + ExpExperiment experiment = ExperimentService.get().getExpExperiment(experimentId); + if (experiment != null) + { + experiments.add(experiment); + } + } + return experiments; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("runGroups"); + super.addNavTrail(root); + } + } + + @RequiresPermission(DesignSampleTypePermission.class) + public class DeleteSampleTypesAction extends AbstractDeleteAction + { + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("sampleSets"); + super.addNavTrail(root); + } + + @Override + protected void deleteObjects(DeleteForm deleteForm) + { + List sampleTypes = getSampleTypes(deleteForm); + if (sampleTypes.isEmpty()) + { + throw new NotFoundException("No sample types found for ids provided."); + } + if (!ensureCorrectContainer(sampleTypes)) + { + throw new UnauthorizedException(); + } + + for (ExpRun run : getRuns(sampleTypes)) + { + if (!run.getContainer().hasPermission(getUser(), DeletePermission.class)) + { + throw new UnauthorizedException(); + } + } + + for (ExpSampleType source : sampleTypes) + { + Domain domain = source.getDomain(); + if (!domain.getDomainKind().canDeleteDefinition(getUser(), domain)) + { + throw new UnauthorizedException(); + } + + source.delete(getUser(), deleteForm.getUserComment()); + } + } + + @Override + public ModelAndView getView(DeleteForm deleteForm, boolean reshow, BindException errors) + { + List sampleTypes = getSampleTypes(deleteForm); + if (!ensureCorrectContainer(sampleTypes)) + { + throw new RedirectException(ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer(), "To delete a sample type, you must be in its folder or project.")); + } + + List> deleteableDatasets = new ArrayList<>(); + List> noPermissionDatasets = new ArrayList<>(); + if (StudyService.get() != null && StudyPublishService.get() != null) + { + for (ExpSampleType sampleType: sampleTypes) + { + for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(sampleType.getRowId(), Dataset.PublishSource.SampleType)) + { + ActionURL datasetURL = StudyService.get().getDatasetURL(getContainer(), dataset.getDatasetId()); + Pair entry = new Pair<>(dataset, datasetURL); + if (dataset.canDeleteDefinition(getUser())) + { + deleteableDatasets.add(entry); + } + else + { + noPermissionDatasets.add(entry); + } + } + } + } + return new ConfirmDeleteView("Sample Type", ShowSampleTypeAction.class, sampleTypes, deleteForm, getRuns(sampleTypes), "Dataset", deleteableDatasets, noPermissionDatasets, Collections.emptyList(), null); + } + + private List getSampleTypes(DeleteForm deleteForm) + { + List sources = new ArrayList<>(); + for (long rowId : deleteForm.getIds(false)) + { + ExpSampleType sampleType = SampleTypeService.get().getSampleType(getContainer(), getUser(), rowId); + if (sampleType != null) + { + sources.add(sampleType); + } + } + return sources; + } + + private boolean ensureCorrectContainer(List sampleTypes) + { + for (ExpSampleType source : sampleTypes) + { + Container sourceContainer = source.getContainer(); + if (!sourceContainer.equals(getContainer())) + { + return false; + } + } + return true; + } + + private List getRuns(List sampleTypes) + { + if (!sampleTypes.isEmpty()) + { + List runArray = ExperimentService.get().getRunsUsingSampleTypes(sampleTypes.toArray(new ExpSampleType[0])); + return ExperimentService.get().runsDeletedWithInput(runArray); + } + else + { + return Collections.emptyList(); + } + } + } + + private DataRegion getSampleTypeRegion(ViewContext model) + { + TableInfo tableInfo = ExperimentServiceImpl.get().getTinfoSampleType(); + + QuerySettings settings = new QuerySettings(model, "SampleType"); + settings.setSelectionKey(DataRegionSelection.getSelectionKey(tableInfo.getSchema().getName(), tableInfo.getName(), "SampleType", settings.getDataRegionName())); + + DataRegion dr = new DataRegion(); + dr.setSettings(settings); + dr.addColumns(tableInfo.getUserEditableColumns()); + dr.removeColumns("lastindexed"); + dr.getDisplayColumn(0).setVisible(false); + + dr.getDisplayColumn("idcol1").setVisible(false); + dr.getDisplayColumn("idcol2").setVisible(false); + dr.getDisplayColumn("idcol3").setVisible(false); + dr.getDisplayColumn("lsid").setVisible(false); + dr.getDisplayColumn("materiallsidprefix").setVisible(false); + dr.getDisplayColumn("parentcol").setVisible(false); + + ActionURL url = new ActionURL(ShowSampleTypeAction.class, model.getContainer()); + dr.getDisplayColumn(1).setURL(url.addParameter("rowId", "${RowId}")); + dr.setShowRecordSelectors(getContainer().hasOneOf(getUser(), DeletePermission.class, UpdatePermission.class)); + + return dr; + } + + @RequiresPermission(ReadPermission.class) + @ActionNames("getSampleType,getSampleTypeApi") // Referenced in labkey-ui-components components/samples/actions.ts TODO: migrate getSampleTypeApi -> getSampleType + public static class GetSampleTypeAction extends ReadOnlyApiAction + { + @Override + public void validateForm(SampleTypeForm form, Errors errors) + { + if (form.getRowId() == null && form.getLSID() == null) + errors.reject(ERROR_REQUIRED, "RowId or LSID must be provided"); + } + + @Override + public Object execute(SampleTypeForm form, BindException errors) throws Exception + { + ExpSampleTypeImpl st = form.getSampleType(getContainer()); + + return getSampleTypeResponse(st); + } + } + + @NotNull + private static ApiSimpleResponse getSampleTypeResponse(ExpSampleType st) throws IOException + { + Map sampleType = new HashMap<>(); + sampleType.put("name", st.getName()); + sampleType.put("nameExpression", st.getNameExpression()); + sampleType.put("labelColor", st.getLabelColor()); + sampleType.put("metricUnit", st.getMetricUnit()); + sampleType.put("description", st.getDescription()); + sampleType.put("importAliases", st.getImportAliasMap()); + sampleType.put("lsid", st.getLSID()); + sampleType.put("rowId", st.getRowId()); + sampleType.put("domainId", st.getDomain().getTypeId()); + sampleType.put("category", st.getCategory()); + + return new ApiSimpleResponse(Map.of("sampleSet", sampleType, "success", true)); + } + + public static class DataTypesWithRequiredLineageForm + { + private Integer _parentDataTypeRowId; + private boolean _sampleParent; + + public Integer getParentDataTypeRowId() + { + return _parentDataTypeRowId; + } + + public void setParentDataTypeRowId(Integer parentDataTypeRowId) + { + this._parentDataTypeRowId = parentDataTypeRowId; + } + + public boolean isSampleParent() + { + return _sampleParent; + } + + public void setSampleParent(boolean sampleParent) + { + _sampleParent = sampleParent; + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetDataTypesWithRequiredLineageAction extends ReadOnlyApiAction + { + @Override + public void validateForm(DataTypesWithRequiredLineageForm form, Errors errors) + { + if (form.getParentDataTypeRowId() == null) + errors.reject(ERROR_REQUIRED, "ParentDataTypeRowId must be provided"); + } + + @Override + public Object execute(DataTypesWithRequiredLineageForm form, BindException errors) throws Exception + { + return getDataTypesWithRequiredLineageResponse(form.getParentDataTypeRowId(), form.isSampleParent(), getContainer(), getUser()); + } + } + @NotNull + private static ApiSimpleResponse getDataTypesWithRequiredLineageResponse(Integer parentDataType, boolean isSampleParent, Container container, User user) + { + Pair, Set> requiredLineages = ExperimentServiceImpl.get().getDataTypesWithRequiredLineage(parentDataType, isSampleParent, container, user); + return new ApiSimpleResponse(Map.of("sampleTypes", requiredLineages.first, "dataClasses", requiredLineages.second,"success", true)); + } + + @RequiresPermission(DesignSampleTypePermission.class) + public static class EditSampleTypeAction extends SimpleViewAction + { + private ExpSampleTypeImpl _sampleType; + + @Override + public ModelAndView getView(SampleTypeForm form, BindException errors) + { + boolean create = form.getLSID() == null && form.getRowId() == null; + if (!create) + _sampleType = form.getSampleType(getContainer()); + + return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("sampleTypeDesigner")); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("sampleSets"); + + root.addChild("Sample Types", ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer())); + if (_sampleType == null) + { + root.addChild("Create Sample Type"); + } + else + { + root.addChild(_sampleType.getName(), ExperimentUrlsImpl.get().getShowSampleTypeURL(_sampleType)); + root.addChild("Update Sample Type"); + } + } + } + + public static class SampleTypeForm extends ReturnUrlForm + { + private Integer rowId; + private String lsid; + + public Integer getRowId() + { + return rowId; + } + + public void setRowId(Integer rowId) + { + this.rowId = rowId; + } + + public String getLSID() + { + return this.lsid; + } + + public void setLSID(String lsid) + { + this.lsid = lsid; + } + + public ExpSampleTypeImpl getSampleType(Container container) throws NotFoundException + { + ExpSampleTypeImpl sampleType = SampleTypeServiceImpl.get().getSampleType(getLSID()); + if (sampleType == null) + sampleType = SampleTypeServiceImpl.get().getSampleType(getRowId()); + + if (sampleType == null) + { + throw new NotFoundException("Sample type not found: " + (getLSID() != null ? getLSID() : getRowId())); + } + + if (!container.equals(sampleType.getContainer())) + { + throw new NotFoundException("Sample type is not defined in the given container."); + } + + return sampleType; + } + } + + @RequiresPermission(InsertPermission.class) + public static class ImportSamplesAction extends AbstractExpDataImportAction + { + @Override + public void validateForm(QueryForm queryForm, Errors errors) + { + _form = queryForm; + _insertOption = queryForm.getInsertOption(); + boolean crossTypeImport = getOptionParamValue(Params.crossTypeImport); + _form.setSchemaName(getTargetSchemaName()); + if (crossTypeImport) + { + _form.setQueryName(getPipelineTargetQueryName()); + } + super.validateForm(queryForm, errors); + if (queryForm.getQueryName() == null) + errors.reject(ERROR_REQUIRED, "Sample type name is required"); + else + { + if (!crossTypeImport) + { + ExpSampleTypeImpl sampleType = SampleTypeServiceImpl.get().getSampleType(getContainer(), getUser(), queryForm.getQueryName()); + if (sampleType == null) + { + errors.reject(ERROR_GENERIC, "Sample type '" + queryForm.getQueryName() + " not found."); + } + } + } + } + + private String getTargetSchemaName() + { + return getOptionParamValue(Params.crossTypeImport) ? ExpSchema.SCHEMA_NAME : "samples"; + } + + @Override + protected UserSchema getTargetSchema() + { + return getOptionParamValue(Params.crossTypeImport) ? QueryService.get().getUserSchema(getUser(), getContainer(), getTargetSchemaName()) : super.getTargetSchema(); + } + + @Override + protected String getPipelineTargetQueryName() + { + return getOptionParamValue(Params.crossTypeImport) ? "materials" : super.getPipelineTargetQueryName(); + } + + @Override + protected Map getRenamedColumns() + { + Map renamedColumns = super.getRenamedColumns(); + renamedColumns.putAll(SampleTypeUpdateServiceDI.SAMPLE_ALT_IMPORT_NAME_COLS); + return renamedColumns; + } + + @Override + protected @Nullable Set getLineageImportAliases() throws IOException + { + Set aliases = new CaseInsensitiveHashSet(); + // Issue 53419: Aliquot parent with number like names that starts with leading zeroes aren't resolved during import + aliases.add(ExpMaterial.ALIQUOTED_FROM_INPUT); + boolean crossTypeImport = getOptionParamValue(AbstractQueryImportAction.Params.crossTypeImport); + // Issue 51894: We need to stop conversion to numbers for alias fields for all type + // If there are aliases defined for one type that are number fields in another type, this will prevent + // conversion to numbers during the initial partitioning, but the conversion will happen when the partition + // file is loaded. + if (crossTypeImport) + { + List sampleTypes = SampleTypeServiceImpl.get().getSampleTypes(getContainer(), getUser(), true); + for (ExpSampleTypeImpl sampleType : sampleTypes) + aliases.addAll(sampleType.getImportAliases().keySet()); + } + else + { + ExpSampleTypeImpl sampleType = SampleTypeServiceImpl.get().getSampleType(getContainer(), getUser(), _form.getQueryName()); + aliases.addAll(sampleType.getImportAliases().keySet()); + } + return aliases; + } + + @Override + protected int importData( + DataLoader dl, + FileStream file, + String originalName, + BatchValidationException errors, + @Nullable AuditBehaviorType auditBehaviorType, + TransactionAuditProvider.@Nullable TransactionAuditEvent auditEvent, + @Nullable String auditUserComment + ) throws IOException + { + initContext(dl, errors, auditBehaviorType, auditUserComment); + + TableInfo tInfo = _target; + QueryUpdateService updateService = _updateService; + if (getOptionParamValue(Params.crossTypeImport)) + { + tInfo = ExperimentService.get().createMaterialTable(new SamplesSchema(getUser(), getContainer()), ContainerFilter.current(this), null); + updateService = tInfo.getUpdateService(); + } + + int count = importData(dl, tInfo, updateService, _context, auditEvent, getUser(), getContainer()); + + if (getOptionParamValue(Params.crossTypeImport)) + { + SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "sampleImport", "crossTypeImport"); + if (_context.getInsertOption() == QueryUpdateService.InsertOption.UPDATE) + SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "sampleImport", "crossTypeUpdate"); + else if (_context.getInsertOption() == QueryUpdateService.InsertOption.MERGE) + SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "sampleImport", "crossTypeMerge"); + } + + return count; + } + + @Override + public ModelAndView getView(QueryForm form, BindException errors) throws Exception + { + initRequest(form); + setHelpTopic("importSampleSets"); // page-wide help topic + setImportHelpTopic("importSampleSets"); // importOptions help topic + setTypeName("samples"); + return getDefaultImportView(form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Sample Types", ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer())); + ActionURL url = _form.urlFor(QueryAction.executeQuery); + if (_form.getQueryName() != null && url != null) + root.addChild(_form.getQueryName(), url); + root.addChild("Import Data"); + } + + @Override + protected JSONObject createSuccessResponse(int rowCount) + { + JSONObject json = super.createSuccessResponse(rowCount); + if (!_context.getResponseInfo().isEmpty()) + { + for (String key : _context.getResponseInfo().keySet()) + json.put(key, _context.getResponseInfo().get(key)); + } + return json; + } + + @Override + protected void configureLoader(DataLoader loader) throws IOException + { + if (getOptionParamValue(Params.crossTypeImport)) + loader.setInferTypes(false); + configureLoader(loader, _target, getRenamedColumns(), allowLineageColumns(), getLineageImportAliases()); + } + } + + public abstract static class AbstractExpDataImportAction extends AbstractQueryImportAction + { + protected QueryForm _form; + protected DataIteratorContext _context; + + @Override + public void validateForm(QueryForm form, Errors errors) + { + QueryDefinition query = form.getQueryDef(); + if (query.getContainerFilter() != null && query.getContainerFilter().getType() != null) + { + // cross folder import not supported + if (query.getContainerFilter().getType() != ContainerFilter.Type.Current) + errors.reject(ERROR_GENERIC, "ContainerFilter is not supported for import actions."); + } + } + + @Override + protected void initRequest(QueryForm form) throws ServletException + { + QueryDefinition query = form.getQueryDef(); + setContainerFilterForImport(query, getContainer(), getUser()); + List qpe = new ArrayList<>(); + TableInfo t = query.getTable(form.getSchema(), qpe, true); + + if (!qpe.isEmpty()) + throw qpe.get(0); + if (!getOptionParamValue(Params.crossTypeImport) && null != t) + { + setTarget(t); + setShowMergeOption(t.supportsInsertOption(QueryUpdateService.InsertOption.MERGE)); + setShowUpdateOption(t.supportsInsertOption(QueryUpdateService.InsertOption.UPDATE)); + } + + _auditBehaviorType = form.getAuditBehavior(); + _auditUserComment = form.getAuditUserComment(); + } + + @Override + protected Map getRenamedColumns() + { + final String renameParamPrefix = "importAlias."; + Map renameColumns = new CaseInsensitiveHashMap<>(); + PropertyValue[] pvs = _form.getInitParameters().getPropertyValues(); + for (PropertyValue pv : pvs) + { + String paramName = pv.getName(); + if (!paramName.startsWith(renameParamPrefix) || pv.getValue() == null) + continue; + + renameColumns.put(paramName.substring(renameParamPrefix.length()), (String) pv.getValue()); + } + return renameColumns; + } + + @Override + protected Set getLineageImportAliases() throws IOException + { + ExpDataClass dataClass = ExperimentServiceImpl.get().getDataClass(getContainer(), getUser(), _form.getQueryName()); + return new CaseInsensitiveHashSet(dataClass.getImportAliases().keySet()); + } + + protected void initContext(DataLoader dl, BatchValidationException errors, @Nullable AuditBehaviorType auditBehaviorType, @Nullable String auditUserComment) + { + _context = createDataIteratorContext(_insertOption, getOptionParamsMap(), getLookupResolutionType(), auditBehaviorType, auditUserComment, errors, null, getContainer()); + + if (_context.isCrossFolderImport() && !getContainer().hasProductFolders()) + _context.setCrossFolderImport(false); + } + + @Override + protected int importData(DataLoader dl, FileStream file, String originalName, BatchValidationException errors, @Nullable AuditBehaviorType auditBehaviorType, TransactionAuditProvider.@Nullable TransactionAuditEvent auditEvent, @Nullable String auditUserComment) throws IOException + { + initContext(dl, errors, auditBehaviorType, auditUserComment); + return importData(dl, _target, _updateService, _context, auditEvent, getUser(), getContainer()); + } + + @Override + protected String getQueryImportProviderName() + { + PropertyValue pv = _form.getInitParameters().getPropertyValue(QUERY_IMPORT_PIPELINE_PROVIDER_PARAM); + return pv == null ? null : (String) pv.getValue(); + } + + @Override + protected String getQueryImportDescription() + { + PropertyValue pv = _form.getInitParameters().getPropertyValue(QUERY_IMPORT_PIPELINE_DESCRIPTION_PARAM); + return pv == null ? null : (String) pv.getValue(); + } + + @Override + protected String getQueryImportJobNotificationProviderName() + { + PropertyValue pv = _form.getInitParameters().getPropertyValue(QUERY_IMPORT_NOTIFICATION_PROVIDER_PARAM); + return pv == null ? null : (String) pv.getValue(); + } + + @Override + protected boolean isBackgroundImportSupported() + { + return true; + } + + @Override + protected boolean allowLineageColumns() + { + return true; + } + + } + + @RequiresPermission(InsertPermission.class) + public static class ImportDataAction extends AbstractExpDataImportAction + { + @Override + public void validateForm(QueryForm queryForm, Errors errors) + { + _form = queryForm; + _form.setSchemaName("exp.data"); + _insertOption = queryForm.getInsertOption(); + super.validateForm(queryForm, errors); + if (queryForm.getQueryName() == null) + errors.reject(ERROR_REQUIRED, "Data class name is required"); + else + { + ExpDataClass dataClass = ExperimentService.get().getDataClass(getContainer(), getUser(), queryForm.getQueryName()); + if (dataClass == null) + { + errors.reject(ERROR_GENERIC, "Data class '" + queryForm.getQueryName() + " not found."); + } + } + } + + @Override + public ModelAndView getView(QueryForm form, BindException errors) throws Exception + { + initRequest(form); + setHelpTopic("dataClass"); // page wide help topic + setImportHelpTopic("dataClass#ui"); // importOptions help topic + setTypeName("data"); + return getDefaultImportView(form, errors); + } + + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Data Classes", ExperimentUrlsImpl.get().getDataClassListURL(getContainer())); + ActionURL url = _form.urlFor(QueryAction.executeQuery); + if (_form.getQueryName() != null && url != null) + root.addChild(_form.getQueryName(), url); + root.addChild("Import Data"); + } + + @Override + protected void configureLoader(DataLoader loader) throws IOException + { + configureLoader(loader, _target, getRenamedColumns(), allowLineageColumns(), getLineageImportAliases()); + } + + } + + @RequiresPermission(UpdatePermission.class) + public class ShowUpdateAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ExperimentForm form, BindException errors) + { + form.refreshFromDb(); + Experiment exp = form.getBean(); + if (exp == null) + { + throw new NotFoundException(); + } + ensureCorrectContainer(getContainer(), ExperimentService.get().getExpExperiment(exp.getRowId()), getViewContext()); + + return new ExperimentUpdateView(new DataRegion(), form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("runGroups"); + addRootNavTrail(root); + root.addChild("Update Run Group"); + } + } + + @RequiresPermission(UpdatePermission.class) + public static class UpdateAction extends FormHandlerAction + { + private Experiment _exp; + + @Override + public void validateCommand(ExperimentForm target, Errors errors) + { + } + + @Override + public boolean handlePost(ExperimentForm form, BindException errors) throws Exception + { + form.doUpdate(); + form.refreshFromDb(); + _exp = form.getBean(); + return true; + } + + @Override + public ActionURL getSuccessURL(ExperimentForm experimentForm) + { + return ExperimentUrlsImpl.get().getExperimentDetailsURL(getContainer(), ExperimentService.get().getExpExperiment(_exp.getRowId())); + } + } + + public static class ExportBean + { + private final LSIDRelativizer _selectedRelativizer; + private final XarExportType _selectedExportType; + private final String _fileName; + private final String _dataRegionSelectionKey; + private final String _error; + private final Long _expRowId; + private final Long _protocolId; + private final ActionURL _postURL; + private final Set _roles; + + public ExportBean(LSIDRelativizer selectedRelativizer, XarExportType selectedExportType, String fileName, ExportOptionsForm form, Set roles, ActionURL postURL) + { + _selectedRelativizer = selectedRelativizer; + _selectedExportType = selectedExportType; + _fileName = fileName; + _dataRegionSelectionKey = form.getDataRegionSelectionKey(); + _error = form.getError(); + _expRowId = form.getExpRowId(); + _postURL = postURL; + _roles = roles; + _protocolId = form.getProtocolId(); + } + + public LSIDRelativizer getSelectedRelativizer() + { + return _selectedRelativizer; + } + + public XarExportType getSelectedExportType() + { + return _selectedExportType; + } + + public String getError() + { + return _error; + } + + public String getFileName() + { + return _fileName; + } + + public Set getRoles() + { + return _roles; + } + + public String getDataRegionSelectionKey() + { + return _dataRegionSelectionKey; + } + + public ActionURL getPostURL() + { + return _postURL; + } + + public Long getProtocolId() + { + return _protocolId; + } + + public Long getExpRowId() + { + return _expRowId; + } + } + + + private String fixupExportName(String runName) + { + runName = runName.replace('/', '-'); + runName = runName.replace('\\', '-'); + return runName; + } + + public static class ExportOptionsForm extends ExperimentRunListForm + { + private String _error; + private XarExportType _exportType; + private LSIDRelativizer _lsidOutputType; + private String _xarFileName; + private String _zipFileName; + private String _fileExportType; + private Long _protocolId; + private Integer _sampleTypeId; + private long[] _dataIds; + private String[] _roles = new String[0]; + + public String getError() + { + return _error; + } + + public void setError(String error) + { + _error = error; + } + + public XarExportType getExportType() + { + return _exportType; + } + + public LSIDRelativizer getLsidOutputType() + { + return _lsidOutputType; + } + + public String getFileExportType() + { + return _fileExportType; + } + + public void setFileExportType(String fileExportType) + { + _fileExportType = fileExportType; + } + + public String getXarFileName() + { + return _xarFileName; + } + + public void setXarFileName(String xarFileName) + { + _xarFileName = xarFileName; + } + + public String getZipFileName() + { + return _zipFileName; + } + + public void setZipFileName(String zipFileName) + { + _zipFileName = zipFileName; + } + + public void setExportType(XarExportType exportType) + { + _exportType = exportType; + } + + public void setLsidOutputType(LSIDRelativizer lsidOutputType) + { + _lsidOutputType = lsidOutputType; + } + + public Long getProtocolId() + { + return _protocolId; + } + + public void setProtocolId(Long protocolId) + { + _protocolId = protocolId; + } + + public String[] getRoles() + { + return _roles; + } + + public void setRoles(String[] roles) + { + _roles = roles; + } + + public Integer getSampleTypeId() + { + return _sampleTypeId; + } + + public void setSampleTypeId(Integer sampleTypeId) + { + _sampleTypeId = sampleTypeId; + } + + public long[] getDataIds() + { + return _dataIds; + } + + public void setDataIds(long[] dataIds) + { + _dataIds = dataIds; + } + + public List lookupProtocols(ViewContext context, boolean clearSelection) + { + List protocols = new ArrayList<>(); + + if (_protocolId != null) + { + ExpProtocol protocol = ExperimentService.get().getExpProtocol(_protocolId.intValue()); + if (protocol == null || !protocol.getContainer().equals(context.getContainer())) + { + throw new NotFoundException(); + } + protocols.add(protocol); + return protocols; + } + + for (Long protocolId : DataRegionSelection.getSelectedIntegers(context, clearSelection)) + { + try + { + ExpProtocol protocol = ExperimentService.get().getExpProtocol(protocolId); + if (protocol == null || !protocol.getContainer().equals(context.getContainer())) + { + throw new NotFoundException(); + } + protocols.add(protocol); + } + catch (NumberFormatException e) + { + throw new NotFoundException("Invalid protocol id: " + protocolId); + } + } + if (protocols.isEmpty()) + { + throw new NotFoundException("No protocols selected"); + } + return protocols; + } + } + + private ActionURL exportXAR(@NotNull XarExportSelection selection, @Nullable String fileName) + throws ExperimentException, IOException, PipelineValidationException + { + return exportXAR(selection, null, null, fileName); + } + + private ActionURL exportXAR(@NotNull XarExportSelection selection, @Nullable LSIDRelativizer lsidRelativizer, @Nullable XarExportType exportType, @Nullable String fileName) + throws ExperimentException, IOException, PipelineValidationException + { + if (lsidRelativizer == null) + lsidRelativizer = LSIDRelativizer.FOLDER_RELATIVE; + + if (exportType == null) + exportType = XarExportType.BROWSER_DOWNLOAD; + + if (fileName == null || fileName.isEmpty()) + fileName = "export.xar"; + + fileName = fixupExportName(fileName); + String xarXmlFileName = null; + if (StringUtils.endsWithIgnoreCase(fileName, ".xar")) + xarXmlFileName = fileName + ".xml"; + + switch (exportType) + { + case BROWSER_DOWNLOAD: + XarExporter exporter = new XarExporter(lsidRelativizer, selection, getUser(), xarXmlFileName, null, getContainer()); + + getViewContext().getResponse().setContentType("application/zip"); + ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, fileName); + ResponseHelper.setPrivate(getViewContext().getResponse()); + + exporter.writeAsArchive(getViewContext().getResponse().getOutputStream()); + return null; + case PIPELINE_FILE: + if (!PipelineService.get().hasValidPipelineRoot(getContainer())) + { + throw new IllegalStateException("You must set a valid pipeline root before you can export a XAR to it."); + } + PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(getContainer()); + XarExportPipelineJob job = new XarExportPipelineJob(getViewBackgroundInfo(), pipeRoot, fileName, lsidRelativizer, selection, xarXmlFileName); + PipelineService.get().queueJob(job); + PipelineStatusFile status = PipelineService.get().getStatusFile(job.getJobGUID()); + return PageFlowUtil.urlProvider(PipelineUrls.class).statusDetails(getContainer(), status.getRowId()); + default: + throw new IllegalArgumentException("Unknown export type: " + exportType); + } + } + + @RequiresPermission(ReadPermission.class) + public class ExportProtocolsAction extends AbstractExportAction + { + @Override + public boolean handlePost(ExportOptionsForm form, BindException errors) throws Exception + { + List protocols = form.lookupProtocols(getViewContext(), false); + + long[] ids = new long[protocols.size()]; + for (int i = 0; i < ids.length; i++) + { + ids[i] = protocols.get(i).getRowId(); + } + XarExportSelection selection = new XarExportSelection(); + selection.addProtocolIds(ids); + + exportXAR(selection, form.getLsidOutputType(), form.getExportType(), form.getXarFileName()); + + if (form.getDataRegionSelectionKey() != null) + { + // Clear the selection + form.lookupProtocols(getViewContext(), true); + } + return true; + } + } + + public abstract static class AbstractExportAction extends FormViewAction + { + protected ActionURL _resultURL; + + @Override + public void validateCommand(ExportOptionsForm target, Errors errors) + { + } + + @Override + public ActionURL getSuccessURL(ExportOptionsForm exportOptionsForm) + { + return _resultURL; + } + + @Override + public ModelAndView getSuccessView(ExportOptionsForm exportOptionsForm) + { + return null; + } + + @Override + public ModelAndView getView(ExportOptionsForm form, boolean reshow, BindException errors) throws Exception + { + // FormViewAction can reinvoke getView() in response to a POST if we're not redirecting the browser, + // so avoid double-creating the export + if ("get".equalsIgnoreCase(getViewContext().getRequest().getMethod())) + handlePost(form, errors); + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + } + + public List lookupRuns(ExportOptionsForm form) + { + Set runIds; + if (form.getRunIds() != null && form.getRunIds().length > 0) + runIds = new HashSet<>(Arrays.asList(form.getRunIds())); + else + runIds = DataRegionSelection.getSelectedIntegers(getViewContext(), form.getDataRegionSelectionKey(), false); + + if (runIds.isEmpty()) + { + throw new NotFoundException(); + } + List result = new ArrayList<>(); + + for (long id : runIds) + { + ExpRun run = ExperimentService.get().getExpRun(id); + if (run == null || !run.getContainer().hasPermission(getUser(), ReadPermission.class)) + { + throw new NotFoundException("Could not find run " + id); + } + result.add(run); + } + return result; + } + } + + @RequiresPermission(ReadPermission.class) + public class ExportRunsAction extends AbstractExportAction + { + @Override + public boolean handlePost(ExportOptionsForm form, BindException errors) throws Exception + { + XarExportSelection selection = new XarExportSelection(); + if (form.getExpRowId() != null) + { + ExpExperiment experiment = ExperimentService.get().getExpExperiment(form.getExpRowId()); + if (experiment != null && !experiment.getContainer().hasPermission(getUser(), ReadPermission.class)) + { + throw new NotFoundException("Run group " + form.getExpRowId()); + } + selection.addExperimentIds(experiment.getRowId()); + } + selection.addRuns(lookupRuns(form)); + + _resultURL = exportXAR(selection, form.getLsidOutputType(), form.getExportType(), form.getXarFileName()); + if (form.getDataRegionSelectionKey() != null) + DataRegionSelection.clearAll(getViewContext(), form.getDataRegionSelectionKey()); + return true; + } + } + + @RequiresPermission(ReadPermission.class) + public class ExportSampleTypeAction extends AbstractExportAction + { + @Override + public boolean handlePost(ExportOptionsForm form, BindException errors) throws Exception + { + Integer rowId = form.getSampleTypeId(); + if (rowId == null) + { + throw new NotFoundException("No sampleTypeId parameter specified"); + } + ExpSampleType sampleType = SampleTypeService.get().getSampleType(getContainer(), getUser(), rowId.intValue()); + if (sampleType == null) + { + throw new NotFoundException("No such sample type with RowId " + rowId); + } + if (!sampleType.getContainer().hasPermission(getUser(), ReadPermission.class)) + { + throw new UnauthorizedException(); + } + + XarExportSelection selection = new XarExportSelection(); + selection.addSampleType(sampleType); + + _resultURL = exportXAR(selection, form.getLsidOutputType(), form.getExportType(), FileUtil.makeLegalName(sampleType.getName() + ".xar")); + return true; + } + } + + @RequiresPermission(ReadPermission.class) + public class ExportRunFilesAction extends AbstractExportAction + { + @Override + public boolean handlePost(ExportOptionsForm form, BindException errors) throws Exception + { + XarExportSelection selection = new XarExportSelection(); + selection.setIncludeXarXml(false); + if ("role".equalsIgnoreCase(form.getFileExportType())) + { + selection.addRoles(form.getRoles()); + } + selection.addRuns(lookupRuns(form)); + + _resultURL = exportXAR(selection, form.getZipFileName()); + if (form.getDataRegionSelectionKey() != null) + DataRegionSelection.clearAll(getViewContext(), form.getDataRegionSelectionKey()); + return true; + } + } + + @RequiresPermission(ReadPermission.class) + public class ExportFilesAction extends AbstractExportAction + { + @Override + public boolean handlePost(ExportOptionsForm form, BindException errors) throws Exception + { + long[] dataIds = form.getDataIds(); + if (dataIds == null || dataIds.length == 0) + { + throw new NotFoundException(); + } + + try + { + for (long id : dataIds) + { + ExpData data = ExperimentService.get().getExpData(id); + if (data == null || !data.getContainer().hasPermission(getUser(), ReadPermission.class)) + { + throw new NotFoundException("Could not find file " + id); + } + } + + XarExportSelection selection = new XarExportSelection(); + selection.setIncludeXarXml(false); + selection.addDataIds(dataIds); + + _resultURL = exportXAR(selection, form.getZipFileName()); + return true; + } + catch (NumberFormatException e) + { + throw new NotFoundException(Arrays.toString(dataIds)); + } + } + } + + public static class ExperimentRunListForm implements DataRegionSelection.DataSelectionKeyForm + { + private String _dataRegionSelectionKey; + private Long _expRowId; + private Long[] _runIds; + + @Override + public String getDataRegionSelectionKey() + { + return _dataRegionSelectionKey; + } + + @Override + public void setDataRegionSelectionKey(String key) + { + _dataRegionSelectionKey = key; + } + + public Long getExpRowId() + { + return _expRowId; + } + + public void setExpRowId(Long expRowId) + { + _expRowId = expRowId; + } + + public Long[] getRunIds() + { + return _runIds; + } + + public void setRunIds(Long[] runIds) + { + _runIds = runIds; + } + + public ExpExperiment lookupExperiment() + { + return getExpRowId() == null ? null : ExperimentService.get().getExpExperiment(getExpRowId().intValue()); + } + } + + private void addSelectedRunsToExperiment(ExpExperiment exp, String dataRegionSelectionKey) + { + Collection runIds = DataRegionSelection.getSelectedIntegers(getViewContext(), dataRegionSelectionKey, true); + List runs = new ArrayList<>(); + for (long runId : runIds) + { + ExpRun run = ExperimentServiceImpl.get().getExpRun(runId); + if (run != null) + { + runs.add(run); + } + } + exp.addRuns(getUser(), runs.toArray(new ExpRun[0])); + } + + + @RequiresPermission(InsertPermission.class) + public class AddRunsToExperimentAction extends FormHandlerAction + { + @Override + public void validateCommand(ExperimentRunListForm target, Errors errors) + { + } + + @Override + public boolean handlePost(ExperimentRunListForm form, BindException errors) + { + addSelectedRunsToExperiment(form.lookupExperiment(), form.getDataRegionSelectionKey()); + return true; + } + + @Override + public ActionURL getSuccessURL(ExperimentRunListForm form) + { + return ExperimentUrlsImpl.get().getExperimentDetailsURL(getContainer(), form.lookupExperiment()); + } + } + + @RequiresPermission(DeletePermission.class) + public static class RemoveSelectedExpRunsAction extends FormHandlerAction + { + @Override + public void validateCommand(ExperimentRunListForm target, Errors errors) + { + } + + @Override + public boolean handlePost(ExperimentRunListForm form, BindException errors) + { + ExpExperiment exp = form.lookupExperiment(); + if (exp == null || !exp.getContainer().hasPermission(getUser(), DeletePermission.class)) + { + throw new NotFoundException("Could not find run group with RowId " + form.getExpRowId()); + } + + for (long runId : DataRegionSelection.getSelectedIntegers(getViewContext(), form.getDataRegionSelectionKey(), false)) + { + ExpRun run = ExperimentService.get().getExpRun(runId); + if (run == null || !run.getContainer().hasPermission(getUser(), DeletePermission.class)) + { + throw new NotFoundException("Could not find run with RowId " + runId); + } + exp.removeRun(getUser(), run); + } + if (form.getDataRegionSelectionKey() != null) + DataRegionSelection.clearAll(getViewContext(), form.getDataRegionSelectionKey()); + return true; + } + + @Override + public ActionURL getSuccessURL(ExperimentRunListForm form) + { + return ExperimentUrlsImpl.get().getExperimentDetailsURL(getContainer(), form.lookupExperiment()); + } + } + + public static ActionURL getResolveLsidURL(Container c, @NotNull String type, @NotNull String lsid) + { + ActionURL url = new ActionURL(ResolveLSIDAction.class, c); + url.addParameter("type", type); + url.addParameter("lsid", lsid); + + return url; + } + + + @RequiresPermission(ReadPermission.class) + public static class ResolveLSIDAction extends SimpleViewAction + { + @Override + public ModelAndView getView(LsidForm form, BindException errors) + { + String message = ""; + if (!PageFlowUtil.empty(form.getLsid())) + { + try + { + String lsid = Lsid.canonical(form.getLsid().trim()); + ActionURL url = LsidManager.get().getDisplayURL(lsid); + if (url == null && form.getType() != null) + { + url = switch (form.getType().toLowerCase()) + { + case "data" -> LsidType.Data.getDisplayURL(new Lsid(lsid)); + case "material" -> LsidType.Material.getDisplayURL(new Lsid(lsid)); + default -> url; + }; + } + if (null != url) + { + throw new RedirectException(url); + } + message = "Could not map LSID to URL"; + } + catch (IllegalArgumentException e) + { + message = "Invalid LSID"; + } + } + + return new HtmlView("Enter LSID", + DOM.createHtmlFragment( + message, + DOM.FORM(at(action, getViewContext().cloneActionURL().setAction(ResolveLSIDAction.class)), + "LSID: ", + DOM.INPUT(at(type, "text", name, "lsid", size, "80", value, form.getLsid())), + PageFlowUtil.button("Go").submit(true)))); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Resolve LSID"); + } + } + + public static class LsidForm + { + private String _lsid; + + public String getType() + { + return _type; + } + + public void setType(String type) + { + _type = type; + } + + private String _type; + + public void setLsid(String lsid) + { + _lsid = lsid; + } + + public String getLsid() + { + return _lsid; + } + } + + public static class SetFlagForm extends LsidForm + { + private String _comment; + private boolean _redirect = true; + + public String getComment() + { + return _comment; + } + + public void setComment(String comment) + { + _comment = comment; + } + + public boolean isRedirect() + { + return _redirect; + } + + public void setRedirect(boolean redirect) + { + _redirect = redirect; + } + } + + /** + * Check for update on the object itself + */ + @RequiresNoPermission + public static class SetFlagAction extends FormHandlerAction + { + @Override + public void validateCommand(SetFlagForm target, Errors errors) + { + } + + @Override + public boolean handlePost(SetFlagForm form, BindException errors) throws Exception + { + String lsid = form.getLsid(); + if (lsid == null) + throw new NotFoundException(); + ExpObject obj = ExperimentService.get().findObjectFromLSID(lsid); + if (obj == null) + throw new NotFoundException(); + Container container = obj.getContainer(); + if (!container.hasPermission(getUser(), UpdatePermission.class)) + { + throw new UnauthorizedException(); + } + + obj.setComment(getUser(), form.getComment()); + return true; + } + + @Override + public URLHelper getSuccessURL(SetFlagForm form) + { + return null; + } + } + + @RequiresPermission(InsertPermission.class) + public class DeriveSamplesChooseTargetAction extends SimpleViewAction + { + private List _materials; + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("sampleSets"); + addRootNavTrail(root); + root.addChild("Sample Types", ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer())); + ExpSampleType sampleType = _materials != null && !_materials.isEmpty() ? _materials.get(0).getSampleType() : null; + if (sampleType != null) + { + root.addChild(sampleType.getName(), ExperimentUrlsImpl.get().getShowSampleTypeURL(sampleType)); + } + root.addChild("Derive Samples"); + } + + @Override + public void validate(DeriveMaterialForm form, BindException errors) + { + _materials = form.lookupMaterials(); + if (_materials.isEmpty()) + { + throw new NotFoundException("Could not find any matching materials"); + } + } + + @Override + public ModelAndView getView(DeriveMaterialForm form, BindException errors) + { + Container c = getContainer(); + PipeRoot root = PipelineService.get().findPipelineRoot(c); + + if (root == null || !root.isValid()) + { + ActionURL pipelineURL = urlProvider(PipelineUrls.class).urlSetup(c); + return new HtmlView(DIV("You must ", + DOM.A(DOM.at(href, pipelineURL), "configure a valid pipeline root for this folder"), + " before deriving samples.")); + } + else + { + Set materialInputRoles = new TreeSet<>(ExperimentService.get().getMaterialInputRoles(getContainer(), getUser())); + Map materialsWithRoles = new LinkedHashMap<>(); + for (ExpMaterial material : _materials) + { + materialsWithRoles.put(material, null); + } + + List sampleTypes = getUploadableSampleTypes(); + + DeriveSamplesChooseTargetBean bean = new DeriveSamplesChooseTargetBean(form.getDataRegionSelectionKey(), form.getTargetSampleTypeId(), sampleTypes, materialsWithRoles, form.getOutputCount(), materialInputRoles, null); + return new JspView<>("/org/labkey/experiment/deriveSamplesChooseTarget.jsp", bean); + } + } + } + + public static class DeriveSamplesChooseTargetBean implements DataRegionSelection.DataSelectionKeyForm + { + private String _dataRegionSelectionKey; + + private final Integer _targetSampleTypeId; + private final List _sampleTypes; + private final Map _sourceMaterials; + private final int _sampleCount; + private final Collection _inputRoles; + private final DerivedSamplePropertyHelper _propertyHelper; + + public static final String CUSTOM_ROLE = "--CUSTOM--"; + + public DeriveSamplesChooseTargetBean(String dataRegionSelectionKey, Integer targetSampleTypeId, List sampleTypes, Map sourceMaterials, int sampleCount, Collection inputRoles, DerivedSamplePropertyHelper helper) + { + _dataRegionSelectionKey = dataRegionSelectionKey; + _targetSampleTypeId = targetSampleTypeId; + _sampleTypes = sampleTypes; + _sourceMaterials = sourceMaterials; + _sampleCount = sampleCount; + _inputRoles = inputRoles; + _propertyHelper = helper; + } + + public Integer getTargetSampleTypeId() + { + return _targetSampleTypeId; + } + + public DerivedSamplePropertyHelper getPropertyHelper() + { + return _propertyHelper; + } + + public int getSampleCount() + { + return _sampleCount; + } + + public Map getSourceMaterials() + { + return _sourceMaterials; + } + + public List getSampleTypes() + { + return _sampleTypes; + } + + public Collection getInputRoles() + { + return _inputRoles; + } + + @Override + public String getDataRegionSelectionKey() + { + return _dataRegionSelectionKey; + } + + @Override + public void setDataRegionSelectionKey(String key) + { + _dataRegionSelectionKey = key; + } + } + + private List getUploadableSampleTypes() + { + // Make a copy so we can modify it + List sampleTypes = new ArrayList<>(SampleTypeService.get().getSampleTypes(getContainer(), getUser(), true)); + sampleTypes.removeIf(sampleType -> !sampleType.canImportMoreSamples()); + return sampleTypes; + } + + @RequiresPermission(InsertPermission.class) + public class DeriveSamplesAction extends FormViewAction + { + private List _materials; + private ActionURL _successUrl; + private final Map _inputMaterials = new LinkedHashMap<>(); + + @Override + public ModelAndView getView(DeriveMaterialForm form, boolean reshow, BindException errors) + { + _materials = form.lookupMaterials(); + if (_materials.isEmpty()) + { + throw new NotFoundException("Could not find any matching materials"); + } + + Container c = getContainer(); + + if (form.getOutputCount() <= 0) + { + form.setOutputCount(1); + } + + if (form.getTargetSampleTypeId() == 0) + throw new NotFoundException("Target sample type required for the derived samples"); + + ExpSampleTypeImpl sampleType = SampleTypeServiceImpl.get().getSampleType(getContainer(), getUser(), form.getTargetSampleTypeId()); + if (sampleType == null) + throw new NotFoundException("Could not find sample type with rowId " + form.getTargetSampleTypeId()); + + InsertView insertView = new InsertView(new DataRegion(), errors); + + DerivedSamplePropertyHelper helper = new DerivedSamplePropertyHelper(sampleType, form.getOutputCount(), c, getUser()); + helper.addSampleColumns(insertView, getUser()); + + int[] rowIds = form.getRowIds(); + for (int i = 0; i < rowIds.length; i++) + { + insertView.getDataRegion().addHiddenFormField("rowIds", Integer.toString(rowIds[i])); + insertView.getDataRegion().addHiddenFormField("inputRole" + i, form.getInputRole(i) == null ? "" : form.getInputRole(i)); + insertView.getDataRegion().addHiddenFormField("customRole" + i, form.getCustomRole(i) == null ? "" : form.getCustomRole(i)); + } + + insertView.getDataRegion().addHiddenFormField("targetSampleTypeId", Integer.toString(form.getTargetSampleTypeId())); + insertView.getDataRegion().addHiddenFormField("outputCount", Integer.toString(form.getOutputCount())); + if (form.getDataRegionSelectionKey() != null) + insertView.getDataRegion().addHiddenFormField(DataRegionSelection.DATA_REGION_SELECTION_KEY, form.getDataRegionSelectionKey()); + insertView.setInitialValues(ViewServlet.adaptParameterMap(getViewContext().getRequest().getParameterMap())); + ButtonBar bar = new ButtonBar(); + bar.setStyle(ButtonBar.Style.separateButtons); + ActionButton submitButton = new ActionButton(DeriveSamplesAction.class, "Submit"); + submitButton.setActionType(ActionButton.Action.POST); + bar.add(submitButton); + insertView.getDataRegion().setButtonBar(bar); + insertView.setTitle("Output Samples"); + + Map materialsWithRoles = new LinkedHashMap<>(); + List materials = form.lookupMaterials(); + for (int i = 0; i < materials.size(); i++) + { + materialsWithRoles.put(materials.get(i), form.determineLabel(i)); + } + + DeriveSamplesChooseTargetBean bean = new DeriveSamplesChooseTargetBean(form.getDataRegionSelectionKey(), form.getTargetSampleTypeId(), getUploadableSampleTypes(), materialsWithRoles, form.getOutputCount(), Collections.emptyList(), helper); + JspView view = new JspView<>("/org/labkey/experiment/summarizeMaterialInputs.jsp", bean); + view.setTitle("Input Samples"); + + return new VBox(view, insertView); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("sampleSets"); + addRootNavTrail(root); + root.addChild("Sample Types", ExperimentUrlsImpl.get().getShowSampleTypeListURL(getContainer())); + ExpSampleType sampleType = _materials != null && !_materials.isEmpty() ? _materials.get(0).getSampleType() : null; + if (sampleType != null) + { + root.addChild(sampleType.getName(), ExperimentUrlsImpl.get().getShowSampleTypeURL(sampleType)); + } + root.addChild("Derive Samples"); + } + + @Override + public void validateCommand(DeriveMaterialForm form, Errors errors) + { + List materials = form.lookupMaterials(); + + List lockedSamples = new ArrayList<>(); + for (int i = 0; i < materials.size(); i++) + { + ExpMaterial m = materials.get(i); + if (!m.isOperationPermitted(SampleTypeService.SampleOperations.EditLineage)) + { + lockedSamples.add(m); + } + String inputRole = form.determineLabel(i); + if (inputRole == null || inputRole.isEmpty()) + { + ExpSampleType st = m.getSampleType(); + inputRole = st != null ? st.getName() : ExpMaterialRunInput.DEFAULT_ROLE; + } + _inputMaterials.put(materials.get(i), inputRole); + } + + if (!lockedSamples.isEmpty()) + { + errors.reject(ERROR_MSG, SampleTypeService.get().getOperationNotPermittedMessage(lockedSamples, SampleTypeService.SampleOperations.EditLineage)); + } + } + + @Override + public boolean handlePost(DeriveMaterialForm form, BindException errors) + { + ExpSampleTypeImpl sampleType = SampleTypeServiceImpl.get().getSampleType(getContainer(), getUser(), form.getTargetSampleTypeId()); + + DerivedSamplePropertyHelper helper = new DerivedSamplePropertyHelper(sampleType, form.getOutputCount(), getContainer(), getUser()); + + Map, Map> allProperties; + try + { + boolean valid = true; + for (Map.Entry> entry : helper.getPostedPropertyValues(getViewContext().getRequest()).entrySet()) + valid = UploadWizardAction.validatePostedProperties(getViewContext(), entry.getValue(), errors) && valid; + if (!valid) + return false; + + allProperties = helper.getSampleProperties(getViewContext().getRequest(), _inputMaterials.keySet()); + } + catch (DuplicateMaterialException e) + { + errors.addError(new ObjectError(e.getColName(), null, null, e.getMessage())); + return false; + } + catch (ExperimentException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + return false; + } + + try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) + { + Map outputMaterials = new HashMap<>(); + int i = 0; + for (Map.Entry, Map> entry : allProperties.entrySet()) + { + Lsid lsid = entry.getKey().first; + String name = entry.getKey().second; + assert name != null; + + ExpMaterialImpl outputMaterial = ExperimentServiceImpl.get().createExpMaterial(getContainer(), lsid.toString(), name); + if (sampleType != null) + { + outputMaterial.setCpasType(sampleType.getLSID()); + } + outputMaterial.save(getUser()); + + if (sampleType != null) + { + Map pvs = new HashMap<>(); + for (Map.Entry propertyEntry : entry.getValue().entrySet()) + pvs.put(propertyEntry.getKey().getName(), propertyEntry.getValue()); + outputMaterial.setProperties(getUser(), pvs, false); + } + + outputMaterials.put(outputMaterial, helper.getSampleNames().get(i++)); + } + + ExperimentService.get().deriveSamples(_inputMaterials, outputMaterials, getViewBackgroundInfo(), _log); + + tx.commit(); + + // automatically link samples to study, if configured + StudyPublishService.get().autoLinkDerivedSamples(sampleType, outputMaterials.keySet().stream().map(ExpObject::getRowId).collect(toList()), getContainer(), getUser()); + + _successUrl = ExperimentUrlsImpl.get().getShowSampleURL(getContainer(), outputMaterials.keySet().iterator().next()); + + if (form.getDataRegionSelectionKey() != null) + DataRegionSelection.clearAll(getViewContext(), form.getDataRegionSelectionKey()); + } + catch (Exception e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + return false; + } + + return true; + } + + @Override + public URLHelper getSuccessURL(DeriveMaterialForm deriveMaterialForm) + { + return _successUrl; + } + } + + public static class DeriveMaterialForm implements HasViewContext, DataRegionSelection.DataSelectionKeyForm + { + private String _dataRegionSelectionKey; + private int _outputCount = 1; + private int _targetSampleTypeId; + private int[] _rowIds; + private String _name; + + private ViewContext _context; + + @Override + public void setViewContext(ViewContext context) + { + _context = context; + } + + @Override + public ViewContext getViewContext() + { + return _context; + } + + public List lookupMaterials() + { + List result = new ArrayList<>(); + for (int rowId : getRowIds()) + { + ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); + if (material != null) + { + if (material.getContainer().hasPermission(_context.getUser(), ReadPermission.class)) + { + result.add(material); + } + else + { + throw new UnauthorizedException(); + } + } + else + { + throw new NotFoundException("No material with RowId " + rowId); + } + } + result.sort(Comparator.comparing(Identifiable::getName)); + return result; + } + + public String getName() + { + return _name; + } + + public void setName(String name) + { + _name = name; + } + + @Override + public String getDataRegionSelectionKey() + { + return _dataRegionSelectionKey; + } + + @Override + public void setDataRegionSelectionKey(String dataRegionSelectionKey) + { + _dataRegionSelectionKey = dataRegionSelectionKey; + } + + public int[] getRowIds() + { + if (_rowIds == null) + { + _rowIds = PageFlowUtil.toInts(DataRegionSelection.getSelected(getViewContext(), getDataRegionSelectionKey(), false)); + } + return _rowIds; + } + + public void setRowIds(int[] rowIds) + { + _rowIds = rowIds; + } + + public int getOutputCount() + { + return _outputCount; + } + + public void setOutputCount(int outputCount) + { + _outputCount = outputCount; + } + + public int getTargetSampleTypeId() + { + return _targetSampleTypeId; + } + + public void setTargetSampleTypeId(int targetSampleTypeId) + { + _targetSampleTypeId = targetSampleTypeId; + } + + public String getInputRole(int i) + { + return _context.getRequest().getParameter("inputRole" + i); + } + + public String getCustomRole(int i) + { + return _context.getRequest().getParameter("customRole" + i); + } + + public String determineLabel(int index) + { + String result = getInputRole(index); + if (DeriveSamplesChooseTargetBean.CUSTOM_ROLE.equals(result)) + { + result = getCustomRole(index); + } + if (result != null) + { + result = result.trim(); + } + return result; + } + } + + + public static class ExpInput + { + public String role; + public int rowId; + public Lsid lsid; + } + + public static class DerivationSpec + { + public String role; + public Map values; + } + + public static class DerivationForm + { + public List dataInputs; + public List materialInputs; + + public int dataOutputCount; + public Lsid targetDataClass; + public Map dataDefault; + public List dataOutputs; + + public int materialOutputCount; + public Lsid targetSampleType; + public Map materialDefault; + public List materialOutputs; + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(InsertPermission.class) + public static class DeriveAction extends MutatingApiAction + { + @Override + public void validateForm(DerivationForm form, Errors errors) + { + if (errors.hasErrors()) + return; + + if (form.materialOutputCount > 0 && form.materialOutputs != null && !form.materialOutputs.isEmpty()) + errors.reject(ERROR_MSG, "Either 'materialOutputCount' or 'materialOutputs' property can be specified, but not both."); + + if (form.dataOutputCount > 0 && form.dataOutputs != null && !form.dataOutputs.isEmpty()) + errors.reject(ERROR_MSG, "Either 'dataOutputCount' or 'dataOutputs' property can be specified, but not both."); + + boolean hasMaterialOutputs = form.materialOutputCount > 0 || form.materialOutputs != null && !form.materialOutputs.isEmpty(); + boolean hasDataOutputs = form.dataOutputCount > 0 || form.dataOutputs != null && !form.dataOutputs.isEmpty(); + + if (!hasMaterialOutputs && !hasDataOutputs) + errors.reject(ERROR_MSG, "At least one data output or material output is required"); + + if (hasMaterialOutputs && form.targetSampleType == null) + errors.reject(ERROR_MSG, "targetSampleType lsid required for material outputs"); + + if (hasDataOutputs && form.targetDataClass == null) + errors.reject(ERROR_MSG, "targetDataClass lsid required for data outputs"); + } + + @Override + public Object execute(DerivationForm form, BindException errors) throws Exception + { + // Find material inputs + Map materialInputs = new LinkedHashMap<>(); + if (form.materialInputs != null) + { + for (ExpInput in : form.materialInputs) + { + ExpMaterial m = null; + if (in.lsid != null) + { + m = ExperimentService.get().getExpMaterial(in.lsid.toString()); + if (m == null) + errors.reject(ERROR_MSG, "Can't resolve sample '" + in.lsid + "'"); + } + else if (in.rowId > 0) + { + m = ExperimentService.get().getExpMaterial(in.rowId); + if (m == null) + errors.reject(ERROR_MSG, "Can't resolve sample '" + in.rowId + "'"); + } + + if (m == null) + { + errors.reject(ERROR_MSG, "Material input lsid or rowId required"); + continue; + } + + ExpSampleType st = m.getSampleType(); + if (st == null) + { + errors.reject(ERROR_MSG, "Material input is not a member of a SampleType"); + continue; + } + + String role = in.role; + if (role == null || role.isEmpty()) + { + role = st.getName(); + } + materialInputs.put(m, role); + } + } + + // Find input data + Map dataInputs = new LinkedHashMap<>(); + if (form.dataInputs != null) + { + for (ExpInput in : form.dataInputs) + { + ExpData d = null; + if (in.lsid != null) + { + d = ExperimentService.get().getExpData(in.lsid.toString()); + if (d == null) + errors.reject(ERROR_MSG, "Can't resolve data '" + in.lsid + "'"); + } + else if (in.rowId > 0) + { + d = ExperimentService.get().getExpData(in.rowId); + if (d == null) + errors.reject(ERROR_MSG, "Can't resolve data '" + in.rowId + "'"); + } + + if (d == null) + { + errors.reject(ERROR_MSG, "Data input lsid or rowId required"); + continue; + } + + ExpDataClass dc = d.getDataClass(getUser()); + if (dc == null) + { + errors.reject(ERROR_MSG, "Data input is not a member of a DataClass"); + continue; + } + + String role = in.role; + if (role == null || role.isEmpty()) + { + role = dc.getName(); + } + dataInputs.put(d, role); + } + } + + ExpSampleType outSampleType; + if (form.targetSampleType != null) + { + // TODO: check in scope and has permission + outSampleType = SampleTypeService.get().getSampleType(form.targetSampleType.toString()); + if (outSampleType == null) + errors.reject(ERROR_MSG, "Sample type not found: " + form.targetSampleType.toString()); + } + else + { + outSampleType = null; + } + + ExpDataClass outDataClass; + if (form.targetDataClass != null) + { + // TODO: check in scope and has permission + outDataClass = ExperimentServiceImpl.get().getDataClass(form.targetDataClass.toString()); + if (outDataClass == null) + errors.reject(ERROR_MSG, "DataClass not found: " + form.targetDataClass.toString()); + } + else + { + outDataClass = null; + } + + if (errors.hasErrors()) + return null; + + // TODO: support list of resolved ExpData or ExpMaterial instead of string concatenated names + // Create "MaterialInputs/" columns with a value containing a comma-separated list of Material names + final Map> parentInputNames = new HashMap<>(); + Set inputTypes = new CaseInsensitiveHashSet(); + for (ExpMaterial material : materialInputs.keySet()) + { + ExpSampleType st = material.getSampleType(); + String keyName = ExpMaterial.MATERIAL_INPUT_PARENT + "/" + st.getName(); + inputTypes.add(keyName); + parentInputNames.computeIfAbsent(keyName, (x) -> new LinkedHashSet<>()).add(material.getName()); + } + + // TODO: support list of resolved ExpData or ExpMaterial instead of string concatenated names + // Create "DataInputs/" columns with a value containing a comma-separated list of ExpData names + for (ExpData d : dataInputs.keySet()) + { + ExpDataClass dc = d.getDataClass(getUser()); + String keyName = ExpData.DATA_INPUT_PARENT + "/" + dc.getName(); + inputTypes.add(keyName); + parentInputNames.computeIfAbsent(keyName, (x) -> new LinkedHashSet<>()).add(d.getName()); + } + + + try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) + { + Set requiredParentTypes = new CaseInsensitiveHashSet(); + + // output materials + Map outputMaterials = new HashMap<>(); + int materialOutputCount = Math.max(form.materialOutputCount, form.materialOutputs != null ? form.materialOutputs.size() : 0); + if (materialOutputCount > 0 && outSampleType != null) + { + requiredParentTypes.addAll(outSampleType.getRequiredImportAliases().values()); + DerivedOutputs derived = new DerivedOutputs<>(parentInputNames, form.materialDefault, form.materialOutputs, materialOutputCount, ExpMaterial.DEFAULT_CPAS_TYPE) + { + @Override + protected TableInfo createTable() + { + SamplesSchema schema = new SamplesSchema(getUser(), getContainer()); + return schema.getTable(outSampleType.getName()); + } + + @Override + protected List getExpObject(List> insertedRows) + { + List rowIds = insertedRows.stream().map(r -> MapUtils.getLong(r,"rowid")).collect(toList()); + return ExperimentService.get().getExpMaterials(rowIds); + } + }; + + outputMaterials = derived.createOutputs(); + } + + + // create output data + Map outputData = new HashMap<>(); + int dataOutputCount = Math.max(form.dataOutputCount, form.dataOutputs != null ? form.dataOutputs.size() : 0); + if (dataOutputCount > 0 && outDataClass != null) + { + requiredParentTypes.addAll(outDataClass.getRequiredImportAliases().values()); + DerivedOutputs derived = new DerivedOutputs<>(parentInputNames, form.dataDefault, form.dataOutputs, dataOutputCount, ExpData.DEFAULT_CPAS_TYPE) + { + @Override + protected TableInfo createTable() + { + ExpSchema expSchema = new ExpSchema(getUser(), getContainer()); + UserSchema dataSchema = expSchema.getUserSchema(ExpSchema.NestedSchemas.data.name()); + return dataSchema.getTable(outDataClass.getName()); + } + + @Override + protected List getExpObject(List> insertedRows) + { + List lsids = insertedRows.stream().map(r -> (String) r.get("lsid")).collect(toList()); + return ExperimentService.get().getExpDatasByLSID(lsids); + } + }; + + outputData = derived.createOutputs(); + } + + if (outputMaterials.isEmpty() && outputData.isEmpty()) + throw new IllegalStateException("Expected to create " + materialOutputCount + " materials and " + dataOutputCount + " datas"); + + boolean hasMissingRequiredParent = false; + for (String required : requiredParentTypes) + { + if (!inputTypes.contains(required)) + { + hasMissingRequiredParent = true; + break; + } + } + if (hasMissingRequiredParent) + throw new IllegalStateException("Inputs are required: " + String.join(",", requiredParentTypes)); + + // finally, create the derived run if there are any parents + ExpRun run = null; + if (!materialInputs.isEmpty() || !dataInputs.isEmpty()) + run = ExperimentService.get().derive(materialInputs, dataInputs, outputMaterials, outputData, new ViewBackgroundInfo(getContainer(), getUser(), null), _log); + tx.commit(); + + StringBuilder successMessage = new StringBuilder("Created "); + if (!outputMaterials.isEmpty()) + successMessage.append(outputMaterials.size()).append(" materials"); + if (!outputData.isEmpty()) + successMessage.append(outputData.size()).append(" data"); + + JSONObject ret; + if (run != null) + ret = ExperimentJSONConverter.serializeRun(run, null, getUser(), ExperimentJSONConverter.DEFAULT_SETTINGS); + else + ret = ExperimentJSONConverter.serializeRunOutputs(outputData.keySet(), outputMaterials.keySet(), getUser(), ExperimentJSONConverter.DEFAULT_SETTINGS); + + return success(successMessage.toString(), ret); + } + } + + // Helper class that prepares and executes the QueryUpdateService.insertRows() on the data or material table. + private abstract class DerivedOutputs + { + private final @NotNull Map> _parentInputNames; + private final @Nullable Map _defaultValues; + private final @Nullable List _values; + private final int _outputCount; + private final String _rolePrefix; + + + public DerivedOutputs(@NotNull Map> parentInputNames, @Nullable Map defaultValues, @Nullable List values, int outputCount, String rolePrefix) + { + _parentInputNames = parentInputNames; + _defaultValues = defaultValues; + _values = values; + _outputCount = outputCount; + _rolePrefix = rolePrefix; + } + + public Pair>, List> prepareRows() + { + List> rows = new ArrayList<>(); + List roles = new ArrayList<>(); + int unknownOutputDataCount = 0; + + for (int i = 0; i < _outputCount; i++) + { + Map row = new CaseInsensitiveHashMap<>(); + if (_defaultValues != null) + row.putAll(_defaultValues); + DerivationSpec spec = _values != null && i < _values.size() ? _values.get(i) : null; + String role = null; + if (spec != null) + { + row.putAll(spec.values); + role = spec.role; + } + + // NOTE: Input parents are added to each row, but are only used for name generation and not for derivation. + // NOTE: We will derive the inserted samples in a single derivation run after the sample/date have been inserted. + row.putAll(_parentInputNames); + + rows.add(row); + + if (StringUtils.trimToNull(role) == null) + { + role = _rolePrefix + (unknownOutputDataCount == 0 ? "" : Integer.toString(unknownOutputDataCount + 1)); + unknownOutputDataCount++; + } + roles.add(role); + } + return Pair.of(rows, roles); + } + + protected abstract TableInfo createTable(); + + protected abstract List getExpObject(List> insertedRows); + + public Map createOutputs() throws BatchValidationException, DuplicateKeyException, SQLException, QueryUpdateServiceException + { + Pair>, List> pair = prepareRows(); + List> rows = pair.first; + List roles = pair.second; + + TableInfo table = createTable(); + QueryUpdateService qus = table.getUpdateService(); + if (qus == null) + throw new IllegalStateException(); + + Map configParams = new HashMap<>(); + // Skip derivation during insert -- DeriveAction will call ExperimentService.get().derive() after samples are inserted + configParams.put(SampleTypeUpdateServiceDI.Options.SkipDerivation, true); + + BatchValidationException qusErrors = new BatchValidationException(); + List> insertedRows = qus.insertRows(getUser(), getContainer(), rows, qusErrors, configParams, null); + if (qusErrors.hasErrors()) + throw qusErrors; + + if (insertedRows.size() != roles.size()) + throw new IllegalStateException("Expected to create " + roles.size() + " new exp objects for derivation"); + + List outputs = getExpObject(insertedRows); + if (outputs.size() != roles.size()) + throw new IllegalStateException("Expected to create " + roles.size() + " new exp objects for derivation"); + + Map outputMap = new HashMap<>(); + for (int i = 0; i < outputs.size(); i++) + { + String role = roles.get(i); + T data = outputs.get(i); + outputMap.put(data, role); + } + + return outputMap; + } + } + } + + public static class CreateExperimentForm extends ExperimentForm implements DataRegionSelection.DataSelectionKeyForm + { + private boolean _addSelectedRuns; + private String _dataRegionSelectionKey; + + public boolean isAddSelectedRuns() + { + return _addSelectedRuns; + } + + public void setAddSelectedRuns(boolean addSelectedRuns) + { + _addSelectedRuns = addSelectedRuns; + } + + @Override + public String getDataRegionSelectionKey() + { + return _dataRegionSelectionKey; + } + + @Override + public void setDataRegionSelectionKey(String dataRegionSelectionKey) + { + _dataRegionSelectionKey = dataRegionSelectionKey; + } + } + + @RequiresPermission(InsertPermission.class) + @ActionNames("createRunGroup, createExperiment") + public class CreateRunGroupAction extends FormViewAction + { + @Override + public ModelAndView getView(CreateExperimentForm form, boolean reshow, BindException errors) + { + // HACK - convert ExperimentForm to not be a BeanViewForm + form.setAddSelectedRuns("true".equals(getViewContext().getRequest().getParameter("addSelectedRuns"))); + form.setDataRegionSelectionKey(getViewContext().getRequest().getParameter(DataRegionSelection.DATA_REGION_SELECTION_KEY)); + + DataRegion drg = new DataRegion(); + + drg.addHiddenFormField(ActionURL.Param.returnUrl, getViewContext().getRequest().getParameter(ActionURL.Param.returnUrl.name())); + drg.addHiddenFormField("addSelectedRuns", Boolean.toString("true".equals(getViewContext().getRequest().getParameter("addSelectedRuns")))); + form.setDataRegionSelectionKey(getViewContext().getRequest().getParameter(DataRegionSelection.DATA_REGION_SELECTION_KEY)); + // Fix issue 27562 - include session-stored selection + if (form.getDataRegionSelectionKey() != null) + { + for (String rowId : DataRegionSelection.getSelected(getViewContext(), form.getDataRegionSelectionKey(), false)) + { + drg.addHiddenFormField(DataRegion.SELECT_CHECKBOX_NAME, rowId); + } + } + drg.addHiddenFormField(DataRegionSelection.DATA_REGION_SELECTION_KEY, getViewContext().getRequest().getParameter(DataRegionSelection.DATA_REGION_SELECTION_KEY)); + + drg.addColumns(ExperimentServiceImpl.get().getTinfoExperiment(), "RowId,Name,LSID,ContactId,ExperimentDescriptionURL,Hypothesis,Comments,Created"); + + DisplayColumn col = drg.getDisplayColumn("RowId"); + col.setVisible(false); + drg.getDisplayColumn("LSID").setVisible(false); + drg.getDisplayColumn("Created").setVisible(false); + + ButtonBar bb = new ButtonBar(); + bb.setStyle(ButtonBar.Style.separateButtons); + ActionButton insertButton = new ActionButton(new ActionURL(CreateRunGroupAction.class, getContainer()), "Submit", ActionButton.Action.POST); + bb.add(insertButton); + + drg.setButtonBar(bb); + + return new InsertView(drg, errors); + } + + + @Override + public boolean handlePost(CreateExperimentForm form, BindException errors) throws Exception + { + // This is strange... but the "Create new run group..." menu item on the run grid always POSTs, probably to + // allow for long lists of run IDs. This "noPost" parameter on the initial POST is used to inform the action + // that it wants to display the form, not try to save anything yet. + if (!"true".equals(getViewContext().getRequest().getParameter("noPost"))) + { + form.setAddSelectedRuns("true".equals(getViewContext().getRequest().getParameter("addSelectedRuns"))); + form.setDataRegionSelectionKey(getViewContext().getRequest().getParameter(DataRegionSelection.DATA_REGION_SELECTION_KEY)); + + Experiment exp = form.getBean(); + if (exp.getName() == null || exp.getName().trim().isEmpty()) + { + errors.reject(ERROR_MSG, "You must specify a name for the experiment"); + } + else + { + int maxNameLength = ExperimentService.get().getTinfoExperimentRun().getColumn("Name").getScale(); + if (exp.getName().length() > maxNameLength) + { + errors.reject(ERROR_MSG, "Name of the experiment must be " + maxNameLength + " characters or less."); + } + } + + String lsid; + int suffix = 1; + do + { + String template = "urn:lsid:" + XarContext.LSID_AUTHORITY_SUBSTITUTION + ":Experiment.Folder-" + XarContext.CONTAINER_ID_SUBSTITUTION + ":" + exp.getName(); + if (suffix > 1) + { + template = template + suffix; + } + suffix++; + lsid = LsidUtils.resolveLsidFromTemplate(template, new XarContext("Experiment Creation", getContainer(), getUser()), ExpExperiment.DEFAULT_CPAS_TYPE); + } + while (ExperimentService.get().getExpExperiment(lsid) != null); + exp.setLSID(lsid); + exp.setContainer(getContainer()); + + if (errors.getErrorCount() == 0) + { + ExpExperimentImpl wrapper = new ExpExperimentImpl(exp); + wrapper.save(getUser()); + + if (form.isAddSelectedRuns()) + { + addSelectedRunsToExperiment(wrapper, form.getDataRegionSelectionKey()); + } + + if (form.getReturnUrl() != null) + { + throw new RedirectException(form.getReturnUrl()); + } + throw new RedirectException(ExperimentUrlsImpl.get().getShowExperimentsURL(getContainer())); + } + } + return true; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("runGroups"); + root.addChild("Create Run Group"); + } + + @Override + public URLHelper getSuccessURL(CreateExperimentForm createExperimentForm) + { + return null; // null is used to show the form in the case where IDs are POSTed from the grid + } + + @Override + public void validateCommand(CreateExperimentForm target, Errors errors) { } + } + + public static class MoveRunsForm implements DataRegionSelection.DataSelectionKeyForm + { + private String _targetContainerId; + private String _dataRegionSelectionKey; + + @Override + public String getDataRegionSelectionKey() + { + return _dataRegionSelectionKey; + } + + @Override + public void setDataRegionSelectionKey(String key) + { + _dataRegionSelectionKey = key; + } + + public String getTargetContainerId() + { + return _targetContainerId; + } + + public void setTargetContainerId(String targetContainerId) + { + _targetContainerId = targetContainerId; + } + } + + @RequiresPermission(DeletePermission.class) + public class MoveRunsLocationAction extends SimpleViewAction + { + @Override + public ModelAndView getView(MoveRunsForm form, BindException errors) + { + ActionURL moveURL = new ActionURL(MoveRunsAction.class, getContainer()); + PipelineRootContainerTree ct = new PipelineRootContainerTree(getUser(), moveURL) + { + private boolean _clickHandlerRegistered = false; + + @Override + protected void renderCellContents(StringBuilder html, Container c, ActionURL url, boolean hasRoot) + { + boolean renderLink = hasRoot && !c.equals(getContainer()); + + if (renderLink) + { + html.append(""); + } + html.append(PageFlowUtil.filter(c.getName())); + if (renderLink) + { + html.append(""); + } + + if (!_clickHandlerRegistered) + { + HttpView.currentPageConfig().addHandlerForQuerySelector("a.move-target-container", "click", "moveTo(this.attributes.getNamedItem('data-objectid').value);" ); + _clickHandlerRegistered = true; + } + } + }; + ct.setInitialLevel(1); + + MoveRunsBean bean = new MoveRunsBean(ct, form.getDataRegionSelectionKey()); + JspView result = new JspView<>("/org/labkey/experiment/moveRunsLocation.jsp", bean); + result.setTitle("Choose Destination Folder"); + result.setFrame(WebPartView.FrameType.PORTAL); + return result; + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Move Runs"); + } + } + + + @RequiresPermission(DeletePermission.class) + public class MoveRunsAction extends FormHandlerAction + { + private Container _targetContainer; + + @Override + public void validateCommand(MoveRunsForm target, Errors errors) + { + } + + @Override + public boolean handlePost(MoveRunsForm form, BindException errors) + { + _targetContainer = ContainerManager.getForId(form.getTargetContainerId()); + if (_targetContainer == null || !_targetContainer.hasPermission(getUser(), InsertPermission.class)) + { + throw new UnauthorizedException(); + } + + Set runIds = DataRegionSelection.getSelectedIntegers(getViewContext(), form.getDataRegionSelectionKey(), false); + List runs = new ArrayList<>(); + for (Long runId : runIds) + { + ExpRun run = ExperimentService.get().getExpRun(runId); + if (run != null) + { + runs.add(run); + } + } + + ViewBackgroundInfo info = getViewBackgroundInfo(); + info.setContainer(_targetContainer); + + try + { + ExperimentService.get().moveRuns(info, getContainer(), runs); + if (form.getDataRegionSelectionKey() != null) + DataRegionSelection.clearAll(getViewContext(), form.getDataRegionSelectionKey()); + } + catch (IOException e) + { + throw new NotFoundException("Failed to initialize move. Check that the pipeline root is configured correctly. " + e); + } + return true; + } + + @Override + public ActionURL getSuccessURL(MoveRunsForm form) + { + return urlProvider(PipelineUrls.class).urlBegin(_targetContainer); + } + } + + public static class ShowExternalDocsForm + { + private String _objectURI; + private String _propertyURI; + + public String getObjectURI() + { + return _objectURI; + } + + public void setObjectURI(String objectURI) + { + _objectURI = objectURI; + } + + public String getPropertyURI() + { + return _propertyURI; + } + + public void setPropertyURI(String propertyURI) + { + _propertyURI = propertyURI; + } + } + + @RequiresPermission(ReadPermission.class) + public static class ShowExternalDocsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ShowExternalDocsForm form, BindException errors) throws Exception + { + Map props = OntologyManager.getPropertyObjects(getContainer(), form.getObjectURI()); + ObjectProperty prop = props.get(form.getPropertyURI()); + if (prop == null || !getContainer().equals(prop.getContainer())) + { + throw new NotFoundException(); + } + URI uri = new URI(prop.getStringValue()); + File f = new File(uri); + if (!f.exists()) + { + throw new NotFoundException(); + } + + PageFlowUtil.streamFile(getViewContext().getResponse(), new File(f.getAbsolutePath()), false); + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + + // TODO: DotGraph has been adding a "runId" parameter, but ShowGraphMoreListAction + public static ActionURL getShowGraphMoreListURL(Container c, @Nullable Long runId, @NotNull String objtype) + { + ActionURL url = new ActionURL(ShowGraphMoreListAction.class, c); + + if (null != runId) + url.addParameter("runId", runId); + + url.addParameter("objtype", objtype); + + return url; + } + + + @RequiresPermission(ReadPermission.class) + public static class ShowGraphMoreListAction extends SimpleViewAction + { + private ExperimentRunForm _form; + + @Override + public ModelAndView getView(ExperimentRunForm form, BindException errors) + { + _form = form; + return new GraphMoreGrid(getContainer(), errors, getViewContext().getActionURL()); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild(new NavTree("Experiments", ExperimentUrlsImpl.get().getShowExperimentsURL(getContainer()))); + ExpRun run = ExperimentService.get().getExpRun(_form.getRowId()); + if (run != null) + { + root.addChild(new NavTree("Experiment Run", ExperimentUrlsImpl.get().getRunGraphURL(_form.lookupRun()))); + } + root.addChild(new NavTree("Selected Protocol Applications")); + } + } + + @RequiresPermission(DesignAssayPermission.class) + public class AssayXarFileAction extends MutatingApiAction + { + + @Override + public Object execute(Object o, BindException errors) throws Exception + { + ApiSimpleResponse response = new ApiSimpleResponse(); + + if (!(getViewContext().getRequest() instanceof MultipartHttpServletRequest)) + throw new BadRequestException("Expected MultipartHttpServletRequest when posting files."); + + if (!PipelineService.get().hasValidPipelineRoot(getContainer())) + { + return false; + } + + MultipartFile formFile = getFileMap().get("file"); + if (formFile == null) + { + errors.reject(ERROR_MSG, "No file was posted by the browser."); + return false; + } + + byte[] bytes = formFile.getBytes(); + if (bytes.length == 0) + { + errors.reject(ERROR_MSG, "No file was posted by the browser."); + return false; + } + + PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(getContainer()); + Path systemDir = pipeRoot.ensureSystemDirectoryPath(); + Path uploadDir = systemDir.resolve("UploadedXARs"); + FileUtil.createDirectories(uploadDir); + if (!Files.isDirectory(uploadDir)) + { + errors.reject(ERROR_MSG, "Unable to create a 'system/UploadedXARs' directory under the pipeline root"); + return false; + } + String userDirName = getUser().getEmail(); + if (userDirName == null || userDirName.isEmpty()) + { + userDirName = GUEST_DIRECTORY_NAME; + } + Path userDir = uploadDir.resolve(userDirName); + FileUtil.createDirectories(userDir); + if (!Files.isDirectory(userDir)) + { + errors.reject(ERROR_MSG, "Unable to create an 'UploadedXARs/" + userDirName + "' directory under the pipeline root"); + return false; + } + + Path xarFile = userDir.resolve(formFile.getOriginalFilename()); + + // As this is multi-part will need to use finally to close, to prevent a stream closure exception + try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(xarFile))) + { + out.write(bytes); + } + catch (IOException e) + { + errors.reject(ERROR_MSG, "Unable to write uploaded XAR file to " + xarFile); + return false; + } + //noinspection EmptyCatchBlock + + ExperimentPipelineJob job = new ExperimentPipelineJob(getViewBackgroundInfo(), xarFile, + "Uploaded file", true, pipeRoot); + PipelineService.get().queueJob(job); + + response.put("success", true); + return response; + } + } + + @RequiresPermission(InsertPermission.class) + public class ImportXarFileAction extends FormHandlerAction + { + @Override + public void validateCommand(ImportXarForm target, Errors errors) + { + } + + @Override + public boolean handlePost(ImportXarForm form, BindException errors) throws Exception + { + for (File f : form.getValidatedFiles(getContainer())) + { + if (f.isFile()) + { + ExperimentPipelineJob job = new ExperimentPipelineJob(getViewBackgroundInfo(), f.toPath(), "Experiment Import", false, form.getPipeRoot(getContainer())); + + // TODO: Configure module resources with the appropriate log location per container + if (form.getModule() != null) + { + FileLike logFile = form.getPipeRoot(getContainer()).getLogDirectoryFileLike(true).resolveChild("module-resource-xar.log"); + job.setLogFile(logFile.toNioPathForWrite()); + } + + PipelineService.get().queueJob(job); + } + else + { + throw new NotFoundException("Expected a file but found a directory: " + f.getName()); + } + } + + return true; + } + + @Override + public URLHelper getSuccessURL(ImportXarForm importXarForm) + { + return getContainer().getStartURL(getUser()); + } + } + + + @RequiresPermission(InsertPermission.class) + public class ImportXarAction extends MutatingApiAction + { + @Override + public Object execute(ImportXarForm form, BindException errors) throws Exception + { + ApiSimpleResponse response = new ApiSimpleResponse(); + + List> archives = new ArrayList<>(); + for (File f : form.getValidatedFiles(getContainer())) + { + Map archive = new HashMap<>(); + ExperimentPipelineJob job = new ExperimentPipelineJob(getViewBackgroundInfo(), f.toPath(), "Experiment Import", false, form.getPipeRoot(getContainer())); + + // TODO: Configure module resources with the appropriate log location per container + if (form.getModule() != null) + { + FileLike logFile = form.getPipeRoot(getContainer()).getLogDirectoryFileLike(true).resolveChild("module-resource-xar.log"); + job.setLogFile(logFile.toNioPathForWrite()); + } + + PipelineService.get().queueJob(job); + + archive.put("file", f.getName()); + archive.put("job", job.getJobGUID()); + archive.put("path", form.getPath()); // echo back the public path + + archives.add(archive); + } + + response.put("success", true); + response.put("archives", archives); + + return response; + } + } + + + /** + * User: jeckels + * Date: Jan 27, 2008 + */ + public static class ExperimentUrlsImpl implements ExperimentUrls + { + public ActionURL getOverviewURL(Container c) + { + return new ActionURL(BeginAction.class, c); + } + + @Override + public ActionURL getExperimentDetailsURL(Container c, ExpExperiment expExperiment) + { + return new ActionURL(DetailsAction.class, c).addParameter("rowId", expExperiment.getRowId()); + } + + public ActionURL getShowSampleURL(Container c, ExpMaterial material) + { + return getMaterialDetailsBaseURL(c, null).addParameter("rowId", material.getRowId()); + } + + @Override + public ActionURL getExportProtocolURL(Container container, ExpProtocol protocol) + { + return new ActionURL(ExperimentController.ExportProtocolsAction.class, container). + addParameter("protocolId", protocol.getRowId()). + addParameter("xarFileName", protocol.getName() + ".xar"); + } + + @Override + public ActionURL getMoveRunsLocationURL(Container container) + { + return new ActionURL(ExperimentController.MoveRunsLocationAction.class, container); + } + + @Override + public ActionURL getProtocolDetailsURL(ExpProtocol protocol) + { + return new ActionURL(ProtocolDetailsAction.class, protocol.getContainer()).addParameter("rowId", protocol.getRowId()); + } + + @Override + public ActionURL getProtocolApplicationDetailsURL(ExpProtocolApplication app) + { + return getShowApplicationURL(app.getContainer(), app.getRowId()); + } + + public ActionURL getProtocolGridURL(Container c) + { + return new ActionURL(ShowProtocolGridAction.class, c); + } + + public ActionURL getRunGraphDetailURL(ExpRun run) + { + return getShowRunGraphDetailURL(run.getContainer(), run.getRowId()); + } + + @Override + public ActionURL getRunGraphDetailURL(ExpRun run, @Nullable ExpData focus) + { + return getRunGraphDetailURL(run, focus, DotGraph.TYPECODE_DATA); + } + + public ActionURL getRunGraphDetailURL(ExpRun run, @Nullable ExpMaterial focus) + { + return getRunGraphDetailURL(run, focus, DotGraph.TYPECODE_MATERIAL); + } + + public ActionURL getRunGraphDetailURL(ExpRun run, @Nullable ExpProtocolApplication focus) + { + return getRunGraphDetailURL(run, focus, DotGraph.TYPECODE_PROT_APP); + } + + private ActionURL getRunGraphDetailURL(ExpRun run, @Nullable ExpObject focus, String typeCode) + { + ActionURL result = getShowRunGraphDetailURL(run.getContainer(), run.getRowId()); + result.addParameter("detail", "true"); + if (focus != null) + { + result.addParameter("focus", typeCode + focus.getRowId()); + } + return result; + } + + @Override + public ActionURL getRunGraphURL(Container container, long runId) + { + return ExperimentController.getRunGraphURL(container, runId); + } + + @Override + public ActionURL getRunGraphURL(ExpRun run) + { + return getRunGraphURL(run.getContainer(), run.getRowId()); + } + + @Override + public ActionURL getRunTextURL(Container c, long runId) + { + return new ActionURL(ShowRunTextAction.class, c).addParameter("rowId", runId); + } + + @Override + public ActionURL getRunTextURL(ExpRun run) + { + return getRunTextURL(run.getContainer(), run.getRowId()); + } + + @Override + public ActionURL getDeleteExperimentsURL(Container container, URLHelper returnUrl) + { + return new ActionURL(DeleteSelectedExperimentsAction.class, container).addReturnUrl(returnUrl); + } + + @Override + public ActionURL getDeleteProtocolURL(@NotNull ExpProtocol protocol, URLHelper returnUrl) + { + ActionURL result = new ActionURL(DeleteProtocolByRowIdsAction.class, protocol.getContainer()); + result.addParameter("singleObjectRowId", protocol.getRowId()); + if (returnUrl != null) + { + result.addReturnUrl(returnUrl); + } + return result; + } + + @Override + public ActionURL getAddRunsToExperimentURL(Container c, ExpExperiment exp) + { + return new ActionURL(AddRunsToExperimentAction.class, c).addParameter("expRowId", exp.getRowId()); + } + + @Override + public ActionURL getShowRunsURL(Container c, ExperimentRunType type) + { + ActionURL result = new ActionURL(ShowRunsAction.class, c); + result.addParameter("experimentRunFilter", type.getDescription()); + return result; + } + + public ActionURL getShowExperimentsURL(Container c) + { + return new ActionURL(ShowRunGroupsAction.class, c); + } + + @Override + public ActionURL getShowSampleTypeListURL(Container c) + { + return getShowSampleTypeListURL(c, null); + } + + @Override + public ActionURL getShowSampleTypeURL(ExpSampleType sampleType) + { + return getShowSampleTypeURL(sampleType, sampleType.getContainer()); + } + + @Override + public ActionURL getShowSampleTypeURL(ExpSampleType sampleType, Container container) + { + return new ActionURL(ShowSampleTypeAction.class, container).addParameter("rowId", sampleType.getRowId()); + } + + public ActionURL getExperimentListURL(Container container) + { + return new ActionURL(ShowRunGroupsAction.class, container); + } + + public ActionURL getShowSampleTypeListURL(Container c, String errorMessage) + { + ActionURL url = new ActionURL(ListSampleTypesAction.class, c); + if (errorMessage != null) + { + url.addParameter("errorMessage", errorMessage); + } + return url; + } + + @Override + public ActionURL getDataClassListURL(Container c) + { + return getDataClassListURL(c, null); + } + + public ActionURL getDataClassListURL(Container c, String errorMessage) + { + ActionURL url = new ActionURL(ListDataClassAction.class, c); + if (errorMessage != null) + { + url.addParameter("errorMessage", errorMessage); + } + return url; + } + + @Override + public ActionURL getDeleteDatasURL(Container c, URLHelper returnUrl) + { + ActionURL url = new ActionURL(DeleteSelectedDataAction.class, c); + if (returnUrl != null) + url.addReturnUrl(returnUrl); + return url; + } + + public ActionURL getDeleteSelectedExperimentsURL(Container c, URLHelper returnUrl) + { + ActionURL result = new ActionURL(DeleteSelectedExperimentsAction.class, c); + if (returnUrl != null) + result.addReturnUrl(returnUrl); + return result; + } + + @Override + public ActionURL getDeleteSelectedExpRunsURL(Container container, URLHelper returnUrl) + { + return new ActionURL(DeleteSelectedExpRunsAction.class, container).addReturnUrl(returnUrl); + } + + public ActionURL getShowUpdateURL(ExpExperiment experiment) + { + return new ActionURL(ShowUpdateAction.class, experiment.getContainer()).addParameter("rowId", experiment.getRowId()); + } + + @Override + public ActionURL getRemoveSelectedExpRunsURL(Container container, URLHelper returnUrl, ExpExperiment exp) + { + return new ActionURL(RemoveSelectedExpRunsAction.class, container).addReturnUrl(returnUrl).addParameter("expRowId", exp.getRowId()); + } + + @Override + public ActionURL getCreateRunGroupURL(Container container, URLHelper returnUrl, boolean addSelectedRuns) + { + ActionURL result = new ActionURL(CreateRunGroupAction.class, container); + if (returnUrl != null) + { + result.addReturnUrl(returnUrl); + } + if (addSelectedRuns) + { + result.addParameter("addSelectedRuns", "true"); + } + return result; + } + + + public static ExperimentUrlsImpl get() + { + return (ExperimentUrlsImpl) urlProvider(ExperimentUrls.class); + } + + public ActionURL getDownloadGraphURL(ExpRun run, boolean detail, String focus, String focusType) + { + ActionURL result = new ActionURL(DownloadGraphAction.class, run.getContainer()); + result.addParameter("rowId", run.getRowId()).addParameter("detail", detail); + if (focus != null) + { + result.addParameter("focus", focus); + } + if (focusType != null) + { + result.addParameter("focusType", focusType); + } + return result; + } + + public ActionURL getBeginURL(Container container) + { + return new ActionURL(BeginAction.class, container); + } + + @Override + public ActionURL getDomainEditorURL(Container container, String domainURI, boolean createOrEdit) + { + Domain domain = PropertyService.get().getDomain(container, domainURI); + if (domain != null) + return getDomainEditorURL(container, domain); + + ActionURL url = new ActionURL(PropertyController.EditDomainAction.class, container); + url.addParameter("domainURI", domainURI); + if (createOrEdit) + url.addParameter("createOrEdit", true); + return url; + } + + @Override + public ActionURL getDomainEditorURL(Container container, Domain domain) + { + ActionURL url = new ActionURL(PropertyController.EditDomainAction.class, container); + url.addParameter("domainId", domain.getTypeId()); + return url; + } + + @Override + public ActionURL getCreateDataClassURL(Container container) + { + return new ActionURL(EditDataClassAction.class, container); + } + + @Override + public ActionURL getShowDataClassURL(Container container, long rowId) + { + ActionURL url = new ActionURL(ShowDataClassAction.class, container); + url.addParameter("rowId", rowId); + return url; + } + + @Override + public ActionURL getShowFileURL(ExpData data, boolean inline) + { + ActionURL result = getShowFileURL(data.getContainer()).addParameter("rowId", data.getRowId()); + if (inline) + { + result.addParameter("inline", inline); + } + return result; + } + + @Override + public ActionURL getMaterialDetailsURL(ExpMaterial material) + { + return getMaterialDetailsURL(material.getContainer(), material.getRowId()); + } + + @Override + public ActionURL getMaterialDetailsURL(Container c, long materialRowId) + { + return getMaterialDetailsBaseURL(c, null).addParameter("rowId", materialRowId); + } + + @Override + public ActionURL getMaterialDetailsBaseURL(Container c, @Nullable String materialIdFieldKey) + { + return new ActionURL(ShowMaterialAction.class, c); + } + + @Override + public ActionURL getCreateSampleTypeURL(Container container) + { + return new ActionURL(EditSampleTypeAction.class, container); + } + + @Override + public ActionURL getImportSamplesURL(Container container, String sampleTypeName) + { + ActionURL url = new ActionURL(ImportSamplesAction.class, container); + url.addParameter("query.queryName", sampleTypeName); + url.addParameter("schemaName", "exp.materials"); + return url; + } + + @Override + public ActionURL getImportDataURL(Container container, String dataClassName) + { + ActionURL url = new ActionURL(ImportDataAction.class, container); + url.addParameter("query.queryName", dataClassName); + url.addParameter("schemaName", "exp.data"); + return url; + } + + @Override + public ActionURL getDataDetailsURL(ExpData data) + { + return new ActionURL(ShowDataAction.class, data.getContainer()).addParameter("rowId", data.getRowId()); + } + + @Override + public ActionURL getShowFileURL(Container c) + { + return new ActionURL(ShowFileAction.class, c); + } + + @Override + public ActionURL getSetFlagURL(Container container) + { + return new ActionURL(SetFlagAction.class, container); + } + + @Override + public ActionURL getShowRunGraphURL(ExpRun run) + { + return ExperimentController.getRunGraphURL(run.getContainer(), run.getRowId()); + } + + @Override + public ActionURL getRepairTypeURL(Container container) + { + return new ActionURL(TypesController.RepairAction.class, container); + } + + @Override + public ActionURL getUpdateMaterialQueryRowAction(Container c, TableInfo table) + { + ActionURL url = new ActionURL(UpdateMaterialQueryRowAction.class, c); + url.addParameter("schemaName", "samples"); + url.addParameter(QueryView.DATAREGIONNAME_DEFAULT + "." + QueryParam.queryName, table.getName()); + + return url; + } + + @Override + public ActionURL getInsertMaterialQueryRowAction(Container c, TableInfo table) + { + ActionURL url = new ActionURL(InsertMaterialQueryRowAction.class, c); + url.addParameter("schemaName", "samples"); + url.addParameter(QueryView.DATAREGIONNAME_DEFAULT + "." + QueryParam.queryName, table.getName()); + + return url; + } + + @Override + public ActionURL getDataClassAttachmentDownloadAction(Container c) + { + return new ActionURL(ExperimentController.DataClassAttachmentDownloadAction.class, c); + } + + } + + private static abstract class BaseResolveLsidApiAction extends ReadOnlyApiAction + { + protected Set _seeds; + + @Override + public void validateForm(F form, Errors errors) + { + if (null != form.getLsids()) + { + _seeds = new LinkedHashSet<>(form.getLsids().size()); + for (String lsid : form.getLsids()) + { + Identifiable id = LsidManager.get().getObject(lsid); + if (id == null) + throw new NotFoundException("Unable to resolve object: " + lsid); + + // ensure the user has read permission in the seed container + if (!getContainer().equals(id.getContainer())) + { + if (!id.getContainer().hasPermission(getUser(), ReadPermission.class)) + throw new UnauthorizedException("User does not have permission to read object: " + lsid); + } + + _seeds.add(id); + } + } + else + { + throw new ApiUsageException("Starting lsids required"); + } + } + } + + @RequiresPermission(ReadPermission.class) + public static class ResolveAction extends BaseResolveLsidApiAction + { + @Override + public Object execute(ResolveLsidsForm form, BindException errors) + { + var settings = new ExperimentJSONConverter.Settings(form.isIncludeProperties(), form.isIncludeInputsAndOutputs(), form.isIncludeRunSteps()); + var data = _seeds.stream().map(n -> ExperimentJSONConverter.serialize(n, getUser(), settings)).collect(toList()); + return new ApiSimpleResponse("data", data); + } + } + + @RequiresPermission(ReadPermission.class) + public static class LineageAction extends BaseResolveLsidApiAction + { + @Override + public Object execute(ExpLineageOptions options, BindException errors) throws Exception + { + ExpLineageServiceImpl.get().streamLineage(getContainer(), getUser(), getViewContext().getResponse(), _seeds, options); + return null; + } + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(AdminPermission.class) + public static class RebuildEdgesAction extends MutatingApiAction + { + @Override + public Object execute(ExperimentRunForm form, BindException errors) + { + if (form.getRowId() != 0 || form.getLsid() != null) + { + ExpRun run = form.lookupRun(); + if (!run.getContainer().hasPermission(getUser(), ReadPermission.class)) + throw new UnauthorizedException("Not permitted"); + + ExperimentServiceImpl.get().syncRunEdges(run); + } + else + { + // should this require site admin permissions? + ExperimentServiceImpl.get().rebuildAllRunEdges(); + } + return success(); + } + } + + private static class VerifyEdgesForm extends ExperimentRunForm + { + private Integer _limit; + + public Integer getLimit() + { + return _limit; + } + + public void setLimit(Integer limit) + { + _limit = limit; + } + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(AdminPermission.class) + public static class VerifyEdgesAction extends ReadOnlyApiAction + { + @Override + public Object execute(VerifyEdgesForm form, BindException errors) + { + if (form.getRowId() != 0 || form.getLsid() != null) + { + ExpRun run = form.lookupRun(); + if (!run.getContainer().hasPermission(getUser(), ReadPermission.class)) + throw new UnauthorizedException("Not permitted"); + + ExperimentServiceImpl.get().verifyRunEdges(run); + } + else + { + ExperimentServiceImpl.get().verifyAllEdges(getContainer(), form.getLimit()); + } + return success(); + } + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(AdminPermission.class) + public static class RebuildAncestorsAction extends MutatingApiAction + { + @Override + public Object execute(Object form, BindException errors) + { + ClosureQueryHelper.truncateAndRecreate(); + return success(); + } + } + + + @Marshal(Marshaller.Jackson) + @RequiresPermission(AdminPermission.class) + public static class CheckDataClassesIndexedAction extends ReadOnlyApiAction + { + @Override + public Object execute(Object o, BindException errors) throws Exception + { + SearchService search = SearchService.get(); + + List> notInIndex = new ArrayList<>(100); + + List list = ExperimentService.get().getDataClasses(getContainer(), getUser(), false); + for (ExpDataClass dc : list) + { + for (ExpData d : dc.getDatas()) + { + String docId = d.getDocumentId(); + if (docId != null) + { + SearchService.SearchHit hit = search.find(docId); + if (hit == null) + { + JSONObject props = ExperimentJSONConverter.serializeData(d, getUser(), ExperimentJSONConverter.DEFAULT_SETTINGS); + props.put("docid", docId); + notInIndex.add(props.toMap()); + } + } + } + } + + return success(notInIndex); + } + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(AdminPermission.class) + public static class CheckEdgesAction extends ReadOnlyApiAction + { + @Override + public Object execute(Object o, BindException errors) throws Exception + { + List result; + DbSchema schema = ExperimentService.get().getSchema(); + TableInfo edgeTable = schema.getTable("Edge"); + + if (null != edgeTable.getColumn("fromObjectId")) + { + var edges = new SqlSelector(ExperimentService.get().getSchema(), "SELECT fromObjectId, toObjectId FROM exp.Edge") + .resultSetStream() + .map(r -> { try { return new Pair<>(r.getInt(1), r.getInt(2)); } catch (SQLException x) { throw new RuntimeException(x); } }) + .collect(toList()); + var cycles = (new GraphAlgorithms()).detectCycleInDirectedGraph(edges); + result = cycles.stream().map(e -> new Integer[]{e.first, e.second}).collect(toList()); + } + else + { + var edges = new SqlSelector(ExperimentService.get().getSchema(), "SELECT fromLsid, toLsid FROM exp.Edge") + .resultSetStream() + .map(r -> { try { return new Pair<>(r.getString(1), r.getString(2)); } catch (SQLException x) { throw new RuntimeException(x); } }) + .collect(toList()); + var cycles = (new GraphAlgorithms()).detectCycleInDirectedGraph(edges); + result = cycles.stream().map(e -> new String[]{e.first, e.second}).collect(toList()); + } + + JSONObject ret = new JSONObject(); + ret.put("result", result); + ret.put("success", true); + return ret; + } + } + + @RequiresPermission(UpdatePermission.class) + public static class UpdateMaterialQueryRowAction extends UserSchemaAction + { + @Override + public BindException bindParameters(PropertyValues m) throws Exception + { + BindException bind = super.bindParameters(m); + + QueryUpdateForm tableForm = (QueryUpdateForm)bind.getTarget(); + + int sampleId; + try + { + sampleId = Integer.parseInt((String) tableForm.getPkVal()); + } + catch (NumberFormatException e) + { + throw new NotFoundException("Invalid RowId: " + tableForm.getPkVal()); + } + + ExpMaterial material = ExperimentService.get().getExpMaterial(sampleId); + if (material == null) + throw new NotFoundException("Invalid material: " + tableForm.getPkVal()); + + return bind; + } + + @Override + public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) + { + int sampleId = Integer.parseInt((String) tableForm.getPkVal()); + + ExpMaterial material = ExperimentService.get().getExpMaterial(sampleId); + if (material == null) + throw new NotFoundException("Invalid material: " + tableForm.getPkVal()); + + boolean isAliquot = !StringUtils.isEmpty(material.getAliquotedFromLSID()); + + TableInfo tableInfo = tableForm.getTable(); + Map scopedFields = new CaseInsensitiveHashMap<>(); + for (DomainProperty dp : tableInfo.getDomain().getProperties()) + { + if (!ExpSchema.DerivationDataScopeType.All.name().equalsIgnoreCase(dp.getDerivationDataScope())) + scopedFields.put(dp.getName(), ExpSchema.DerivationDataScopeType.ChildOnly.name().equalsIgnoreCase(dp.getDerivationDataScope())); + } + + for (var column : tableInfo.getColumns()) + { + String columnName = column.getName(); + if (scopedFields.containsKey(columnName)) + { + boolean isAliquotField = scopedFields.get(columnName); + boolean show = (isAliquot && isAliquotField) || (!isAliquot && !isAliquotField); + ((BaseColumnInfo)column).setUserEditable(show); + ((BaseColumnInfo)column).setHidden(!show); + } + } + + ButtonBar bb = createSubmitCancelButtonBar(tableForm); + UpdateView view = new UpdateView(tableForm, errors); + view.getDataRegion().setButtonBar(bb); + return view; + } + + @Override + public boolean handlePost(QueryUpdateForm tableForm, BindException errors) + { + doInsertUpdate(tableForm, errors, false); + return 0 == errors.getErrorCount(); + } + + @Override + public void addNavTrail(NavTree root) + { + super.addNavTrail(root); + root.addChild("Edit " + _form.getQueryName()); + } + } + + @RequiresPermission(InsertPermission.class) + public static class InsertMaterialQueryRowAction extends UserSchemaAction + { + @Override + public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) + { + TableInfo tableInfo = tableForm.getTable(); + Map propertyFields = new CaseInsensitiveHashMap<>(); + for (DomainProperty dp : tableInfo.getDomain().getProperties()) + { + propertyFields.put(dp.getName(), ExpSchema.DerivationDataScopeType.ChildOnly.name().equalsIgnoreCase(dp.getDerivationDataScope())); + } + + for (var column : tableInfo.getColumns()) + { + String columnName = column.getName(); + if (propertyFields.containsKey(columnName)) + { + boolean isAliquotField = propertyFields.get(columnName); + ((BaseColumnInfo)column).setUserEditable(!isAliquotField); + ((BaseColumnInfo)column).setHidden(isAliquotField); + } + } + + InsertView view = new InsertView(tableForm, errors); + view.getDataRegion().setButtonBar(createSubmitCancelButtonBar(tableForm)); + return view; + } + + @Override + public boolean handlePost(QueryUpdateForm tableForm, BindException errors) + { + doInsertUpdate(tableForm, errors, true); + return 0 == errors.getErrorCount(); + } + + @Override + public void addNavTrail(NavTree root) + { + super.addNavTrail(root); + root.addChild("Insert " + _form.getQueryName()); + } + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(ReadPermission.class) + public static class SaveFindIdsAction extends ReadOnlyApiAction + { + + public static final String FIND_BY_IDS_SESSION_KEY_PREFIX = "findByIds"; + + @Override + public Object execute(FindByIdsForm form, BindException errors) throws Exception + { + HttpServletRequest request = getViewContext().getRequest(); + String key = form.getSessionKey(); + boolean removePrevious = false; + + if (key == null) + { + removePrevious = true; + key = FIND_BY_IDS_SESSION_KEY_PREFIX + "_" + UniqueID.getServerSessionScopedUID(); + } + + if (request != null) + { + if (removePrevious) + SessionHelper.clearAttributesWithPrefix(request, FIND_BY_IDS_SESSION_KEY_PREFIX); + HttpSession session = request.getSession(false); + if (session != null) + { + @SuppressWarnings("unchecked") + List existingIds = (List) session.getAttribute(key); + + // deduplicate from existing ids + if (existingIds != null && form.getSessionKey() != null) + { + existingIds.addAll(form.getIds().stream().filter(id -> !existingIds.contains(id)).toList()); + session.setAttribute(key, existingIds); + } + else + { + session.setAttribute(key, form.getIds()); + } + return success("Saved ids to session key", key); + } + } + + return new SimpleResponse<>(false, "Unable to save to session. Session or request may be null."); + } + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(ReadPermission.class) + public static class SaveOrderedSamplesQueryAction extends ReadOnlyApiAction + { + private static final String SAMPLE_ID_PREFIX = "s:"; + private static final String UNIQUE_ID_PREFIX = "u:"; + + private List _ids; + private Map> _uniqueIdLsids; + + @Override + public void validateForm(FindByIdsForm form, Errors errors) + { + if (form.getSessionKey() == null) + errors.reject(ERROR_REQUIRED, "sessionKey must be provided"); + else + { + _ids = getFindIdsFromSession(form.getSessionKey()); + if (_ids == null || _ids.isEmpty()) + errors.reject(ERROR_REQUIRED, "No ids found corresponding to session key " + form.getSessionKey()); + } + } + + private void ensureUniqueIdLsids() + { + boolean hasUniqueId = _ids.stream().anyMatch(s -> s.startsWith(UNIQUE_ID_PREFIX)); + if (hasUniqueId && _uniqueIdLsids == null) + { + List uniqueIds = _ids.stream().map(s -> s.substring(UNIQUE_ID_PREFIX.length())).toList(); + _uniqueIdLsids = ExperimentService.get().getUniqueIdLsids(uniqueIds, getUser(), getContainer()); + } + } + + @Override + public Object execute(FindByIdsForm form, BindException errors) throws Exception + { + ensureUniqueIdLsids(); + + SQLFragment select = getOrderedRowsSql(); + // need to set the key field so selections are possible + // need the SampleTypeUnits so we will display using that unit + String metadata = + """ + + + + + true + true + + + true + + + true + + +
    +     """; + QueryDefinition def = QueryService.get().saveSessionQuery(getViewContext(), getContainer(), ExperimentServiceImpl.getExpSchema().getName(), select.getSQL(), metadata); + return success("Session query created", Map.of("queryName", def.getName(), "ids", _ids)); + } + + + private List getFindIdsFromSession(String sessionKey) + { + HttpServletRequest request = getViewContext().getRequest(); + List ids = new ArrayList<>(); + if (request != null) + { + HttpSession session = request.getSession(false); + if (session != null) + { + ids = (List) session.getAttribute(sessionKey); + } + } + return ids; + } + + private SQLFragment getOrderedRowsSql() + { + boolean isFMEnabled = InventoryService.isFreezerManagementEnabled(getContainer()); + String samplesTable = isFMEnabled ? "inventory.SampleItems" : "exp.materials"; + List orderedIdCols = new ArrayList<>(Arrays.asList("Id AS ProvidedID", "RowId", "Ordinal")); + List sampleColumns = new ArrayList<>(); + if (!isFMEnabled) + { + sampleColumns.addAll(Arrays.asList( + "S.Name AS SampleID", + "S.MaterialExpDate AS ExpirationDate", + "S.SampleSet as SampleType", + "S.SampleState", + "S.isAliquot", + "S.Created", + "S.CreatedBy" + )); + } + else + { + sampleColumns.addAll(Arrays.asList( + "S.Name AS SampleID", + "S.MaterialExpDate AS ExpirationDate", + "S.LabelColor", + "S.SampleSet", + "S.SampleState", + "S.StoredAmount", + "S.Units", + "S.SampleTypeUnits", + "S.FreezeThawCount", + "S.StorageStatus", + "S.CheckedOutBy", + "S.StorageLocation", + "S.StorageRow", + "S.StorageCol", + "S.StoragePositionNumber", + "S.IsAliquot", + "S.Created", + "S.CreatedBy" + )); + } + + + String sampleIdComma = ""; + String uniqueIdComma = ""; + int index = 1; + SQLFragment sampleIdValuesSql = new SQLFragment(); + SQLFragment uniqueIdValuesSql = new SQLFragment(); + for (String id : _ids) + { + if (id.startsWith(SAMPLE_ID_PREFIX)) + { + sampleIdValuesSql.append(sampleIdComma).append("\t(").appendValue(index); + sampleIdValuesSql.append(", "); + sampleIdValuesSql.append(LabKeySql.quoteString(id.substring(SAMPLE_ID_PREFIX.length()))); + sampleIdValuesSql.append(", "); + sampleIdValuesSql.append(LabKeySql.quoteString("null")); + sampleIdValuesSql.append(")"); + sampleIdComma = "\n,"; + } + else if (id.startsWith(UNIQUE_ID_PREFIX)) + { + String idClean = id.substring(UNIQUE_ID_PREFIX.length()); + + List lsids = _uniqueIdLsids.get(idClean); + if (lsids != null) + { + for (String lsid : lsids) + { + uniqueIdValuesSql.append(uniqueIdComma).append("\t(").appendValue(index); + uniqueIdValuesSql.append(", "); + uniqueIdValuesSql.append(LabKeySql.quoteString(idClean)); + uniqueIdValuesSql.append(", "); + uniqueIdValuesSql.append(LabKeySql.quoteString(lsid)); + uniqueIdValuesSql.append(")"); + uniqueIdComma = "\n,"; + } + } + } + index++; + } + + boolean haveData = !sampleIdValuesSql.isEmpty() || !_uniqueIdLsids.isEmpty(); + SQLFragment sql = new SQLFragment(); + if (!sampleIdValuesSql.isEmpty()) + { + sql.append("WITH _ordered_ids_ AS (\nSELECT * FROM (VALUES\n"); + sql.append(sampleIdValuesSql); + sql.append("\n) AS _values_ )\n"); // name of the alias here doesn't matter + } + if (!uniqueIdValuesSql.isEmpty()) + { + if (!sampleIdValuesSql.isEmpty()) + sql.append(",\n"); + else + sql.append("WITH "); + + sql.append("_ordered_unique_ids_ AS (\nSELECT * FROM (VALUES\n"); + sql.append(uniqueIdValuesSql); + sql.append("\n) AS _values_ )\n"); // name of the alias here doesn't matter + } + + sql.append("SELECT "); + sql.append("\n\tOID.").append(StringUtils.join(orderedIdCols, ",\n\tOID.")); + sql.append(",\n\t").append(StringUtils.join( sampleColumns, ",\n\t")); + sql.append("\nFROM\n("); + if (!sampleIdValuesSql.isEmpty()) + { + sql.append("SELECT\n\tM.RowId,\n\t_ordered_ids_.column1 as Ordinal,\n\t_ordered_ids_.column2 as Id,\n\t_ordered_ids_.column2 as lsid"); + sql.append("\nFROM _ordered_ids_\n"); + sql.append("INNER JOIN exp.materials M ON _ordered_ids_.column2 = M.Name"); + sql.append("\n"); + } + if (!uniqueIdValuesSql.isEmpty()) + { + if (!sampleIdValuesSql.isEmpty()) + sql.append("\nUNION ALL\n\n"); + + sql.append("SELECT\n\tM.RowId,\n\t_ordered_unique_ids_.column1 as Ordinal,\n\t_ordered_unique_ids_.column2 as Id,\n\t_ordered_unique_ids_.column3 as lsid"); + sql.append("\nFROM _ordered_unique_ids_\n"); + sql.append("INNER JOIN exp.materials M ON _ordered_unique_ids_.column3 = M.lsid"); + sql.append("\n"); + } + if (!haveData) // no data to return but return data in the expected shape. + { + sql = new SQLFragment("SELECT\n"); + sql.append(orderedIdCols.stream() + .map(col -> { + int asIndex = col.indexOf("AS"); + if (asIndex > 0) + return "NULL AS " + col.substring(asIndex+ 3); + else + return "NULL AS " + col; + }) + .collect(Collectors.joining(",\t\n"))); + sql.append(",\t\n").append(StringUtils.join(sampleColumns, ",\t\n")); + sql.append("\nFROM ").append(samplesTable).append(" S WHERE 1 = 2"); + return sql; + } + else + { + sql.append(") OID"); + if (isFMEnabled) + sql.append("\nLEFT JOIN inventory.SampleItems S on S.RowId = OID.RowId"); + else + sql.append("\nINNER JOIN exp.materials S on S.RowId = OID.RowId"); + sql.append("\n\nORDER BY Ordinal"); + return sql; + } + } + } + + public static class FindByIdsForm extends FindSessionKeyForm + { + List _ids; + + public List getIds() + { + return _ids; + } + + public void setIds(List ids) + { + _ids = ids; + } + } + + + public static class FindSessionKeyForm + { + private String _sessionKey; + + public String getSessionKey() + { + return _sessionKey; + } + + public void setSessionKey(String sessionKey) + { + _sessionKey = sessionKey; + } + } + + static void validateEntitySequenceForm(EntitySequenceForm form, Errors errors) + { + String kindName = form.getKindName(); + if (StringUtils.isEmpty(kindName) || form.getSeqType() == null) + { + errors.reject(ERROR_REQUIRED, "KindName and SeqType must be provided"); + return; + } + + if (form.getSeqType() == NameGenerator.EntityCounter.genId) + { + if (form.getRowId() == null) + errors.reject(ERROR_REQUIRED, "Data type RowId must be provided for genId"); + } + else if (!SampleTypeDomainKind.NAME.equalsIgnoreCase(kindName)) + { + errors.reject(ERROR_MSG, form.getSeqType() + " is not supported for " + kindName); + } + + if (!SampleTypeDomainKind.NAME.equalsIgnoreCase(kindName) && !DataClassDomainKind.NAME.equalsIgnoreCase(kindName)) + errors.reject(ERROR_MSG, "Invalid KindName. Should be either " + SampleTypeDomainKind.NAME + " or " + DataClassDomainKind.NAME + "."); + + } + + @RequiresPermission(ReadPermission.class) + public static class GetEntitySequenceAction extends ReadOnlyApiAction + { + @Override + public void validateForm(EntitySequenceForm form, Errors errors) + { + validateEntitySequenceForm(form, errors); + } + + @Override + public Object execute(EntitySequenceForm form, BindException errors) throws Exception + { + long value = -1; + if (SampleTypeDomainKind.NAME.equalsIgnoreCase(form.getKindName())) + { + if (form.getSeqType() == NameGenerator.EntityCounter.genId) + { + ExpSampleType sampleType = SampleTypeService.get().getSampleType(form.getRowId()); + if (sampleType != null) + value = sampleType.getCurrentGenId(); + } + else + { + value = SampleTypeService.get().getCurrentCount(form.getSeqType(), getContainer()); + } + + } + else if (DataClassDomainKind.NAME.equalsIgnoreCase(form.getKindName())) + { + ExpDataClass dataClass = ExperimentService.get().getDataClass(form.getRowId()); + if (dataClass != null) + value = dataClass.getCurrentGenId(); + } + + ApiSimpleResponse resp = new ApiSimpleResponse(); + resp.put("success", value > -1); + resp.put("value", value); + return resp; + } + } + + @RequiresPermission(ReadPermission.class) // actual permission checked later + public static class SetEntitySequenceAction extends MutatingApiAction + { + @Override + public void validateForm(EntitySequenceForm form, Errors errors) + { + validateEntitySequenceForm(form, errors); + + if (form.getNewValue() == null || form.getNewValue() < 0) + errors.reject(ERROR_MSG, "Invalid newValue."); + } + + @Override + public Object execute(EntitySequenceForm form, BindException errors) + { + ApiSimpleResponse resp = new ApiSimpleResponse(); + resp.put("success", true); + + try + { + Domain domain = null; + if (SampleTypeDomainKind.NAME.equalsIgnoreCase(form.getKindName())) + { + if (form.getSeqType() == NameGenerator.EntityCounter.genId) + { + if (!getContainer().hasPermission(getUser(), DesignSampleTypePermission.class)) + throw new UnauthorizedException("Insufficient permissions."); + + ExpSampleType sampleType = SampleTypeService.get().getSampleType(form.getRowId()); + if (sampleType != null) + { + sampleType.ensureMinGenId(form.getNewValue()); + domain = sampleType.getDomain(); + } + else + { + resp.put("success", false); + resp.put("error", "Sample type does not exist."); + } + } + else + { + if (!getContainer().hasPermission(getUser(), AdminPermission.class)) + throw new UnauthorizedException("Insufficient permissions."); + + SampleTypeService.get().ensureMinSampleCount(form.getNewValue(), form.getSeqType(), getContainer()); + } + } + else if (DataClassDomainKind.NAME.equalsIgnoreCase(form.getKindName())) + { + if (!getContainer().hasPermission(getUser(), DesignDataClassPermission.class)) + throw new BadRequestException("Insufficient permissions."); + + ExpDataClass dataClass = ExperimentService.get().getDataClass(form.getRowId()); + if (dataClass != null) + { + dataClass.ensureMinGenId(form.getNewValue(), getContainer()); + domain = dataClass.getDomain(); + } + else + { + resp.put("success", false); + resp.put("error", "DataClass does not exist."); + } + } + + if (domain != null) + { + DomainAuditProvider.DomainAuditEvent event = new DomainAuditProvider.DomainAuditEvent(getContainer(), "The genId for domain " + domain.getName() + " has been updated to " + form.getNewValue() + "."); + event.setDomainUri(domain.getTypeURI()); + event.setDomainName(domain.getName()); + AuditLogService.get().addEvent(getUser(), event); + } + } + catch (ExperimentException e) + { + resp.put("success", false); + resp.put("error", e.getMessage()); + } + + return resp; + } + } + + public static class EntitySequenceForm + { + private String _kindName; + private NameGenerator.EntityCounter _seqType; + private Integer _rowId; + private Long _newValue; + + public Integer getRowId() + { + return _rowId; + } + + public void setRowId(Integer rowId) + { + _rowId = rowId; + } + + public String getKindName() + { + return _kindName; + } + + public void setKindName(String kindName) + { + _kindName = kindName; + } + + public Long getNewValue() + { + return _newValue; + } + + public void setNewValue(Long newValue) + { + this._newValue = newValue; + } + + public NameGenerator.EntityCounter getSeqType() + { + return _seqType; + } + + public void setSeqType(String seqType) + { + _seqType = NameGenerator.EntityCounter.valueOf(seqType); + } + + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(ReadPermission.class) + public static class GetCrossFolderDataSelectionAction extends ReadOnlyApiAction + { + @Override + public void validateForm(CrossFolderSelectionForm form, Errors errors) + { + if (form.getDataRegionSelectionKey() == null && form.getRowIds() == null) + errors.reject(ERROR_REQUIRED, "You must provide either a set of rowIds or a dataRegionSelectionKey."); + if (!"samples".equalsIgnoreCase(form.getDataType()) && !"exp.data".equalsIgnoreCase(form.getDataType())&& !"assay".equalsIgnoreCase(form.getDataType())) + errors.reject(ERROR_REQUIRED, "Data type (sample, data or assayrun) must be specified."); + } + + @Override + public Object execute(CrossFolderSelectionForm form, BindException errors) + { + Pair result = ExperimentServiceImpl.getCurrentAndCrossFolderDataCount(form.getIds(false), form.getDataType(), getContainer()); + + ApiSimpleResponse resp = new ApiSimpleResponse(); + resp.put("success", true); + resp.put("currentFolderSelectionCount", result.first); + resp.put("crossFolderSelectionCount", result.second); + + return success(resp); + } + } + + public static class CrossFolderSelectionForm extends DataViewSnapshotSelectionForm + { + private String _dataType; + private String _picklistName; + + public String getDataType() + { + return _dataType; + } + + public void setDataType(String dataType) + { + _dataType = dataType; + } + + public String getPicklistName() + { + return _picklistName; + } + + public void setPicklistName(String picklistName) + { + _picklistName = picklistName; + } + + @Override + public Set getIds(boolean clear) + { + Set selectedIds; + + if (_rowIds != null) + selectedIds = _rowIds; + else if (isUseSnapshotSelection()) + selectedIds = new HashSet<>(DataRegionSelection.getSnapshotSelectedIntegers(getViewContext(), getDataRegionSelectionKey())); + else + selectedIds = DataRegionSelection.getSelectedIntegers(getViewContext(), getDataRegionSelectionKey(), clear); + + if (_picklistName != null) + { + User user = getViewContext().getUser(); + Container container = getViewContext().getContainer(); + UserSchema schema = ListService.get().getUserSchema(user, container); + TableInfo tInfo = schema.getTable(_picklistName); + if (tInfo != null) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addInClause(FieldKey.fromParts("id"), selectedIds); + TableSelector selector = new TableSelector(tInfo, Collections.singleton("SampleID"), filter, null); + return new HashSet<>(selector.getArrayList(Long.class)); + } + } + return selectedIds; + } + } + + @RequiresPermission(AdminPermission.class) + public static class RecomputeAliquotRollup extends SimpleViewAction + { + @Override + public void addNavTrail(NavTree root) + { + } + + @Override + public ModelAndView getView(Object o, BindException errors) throws SQLException + { + try (var ignore = SpringActionController.ignoreSqlUpdates()) + { + Container container = getContainer(); + User user = getUser(); + + List sampleTypes = SampleTypeService.get() + .getSampleTypes(container, user, true); + + HtmlStringBuilder builder = HtmlStringBuilder.of(); + builder.unsafeAppend(""); + + SampleTypeService service = SampleTypeService.get(); + for (ExpSampleType sampleType : sampleTypes) + { + int updatedCount; + updatedCount = service.recomputeSampleTypeRollup(sampleType, container); + // we could check "if (0 < updatedCount) refresh(rollup)", but since this is a "manual" usage lets just always refresh + SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, update); + builder.unsafeAppend(""); + } + + builder.unsafeAppend("
    Sample Type#Recomputed
    ") + .append(sampleType.getName()) + .unsafeAppend("") + .append(updatedCount) + .unsafeAppend("
    "); + return new HtmlView("Aliquot Rollup Recalculation Result", builder); + } + } + } + + /* Also see API CheckEdgesAction */ + @RequiresPermission(TroubleshooterPermission.class) + public static class CycleCheckAction extends FormViewAction + { + List cycleObjectIds = null; + + @Override + public void validateCommand(Object target, Errors errors) + { + + } + + @Override + public ModelAndView getView(Object o, boolean reshow, BindException errors) + { + if (!reshow) + { + return new HtmlView( + DIV("This operation can use a lot of memory.", + LK.FORM(at(method,"POST"), + PageFlowUtil.button("Continue").submit(true))) + ); + } + + if (null == cycleObjectIds) + return new HtmlView(HtmlString.of("No cycles found")); + + Map map = new LongHashMap<>(); + var cf = new ContainerFilter.AllFolders(getUser()); + var materials = ExperimentServiceImpl.get().getExpMaterialsByObjectId(cf, cycleObjectIds); + materials.forEach( (m) -> map.put(m.getObjectId(), m)); + var datas = ExperimentServiceImpl.get().getExpDatasByObjectId(cf, cycleObjectIds); + datas.forEach( (d) -> map.put(d.getObjectId(), d)); + var runs = ExperimentServiceImpl.get().getRunsByObjectId(cf, cycleObjectIds); + runs.forEach( (r) -> map.put(r.getObjectId(), r)); + + ExperimentUrls urls = ExperimentUrls.get(); + return new HtmlView( + DIV("Cycle found involving these objects.", + UL(cycleObjectIds.stream().map((objectid) -> + { + ExpObject exp = map.get(objectid); + if (exp instanceof ExpMaterial mat) + return LI(A(at(target, "_blank", href, urls.getMaterialDetailsURL(mat)), objectid + " : material - " + mat.getName())); + else if (exp instanceof ExpRun run) + return LI(A(at(target, "_blank", href, urls.getRunTextURL(run)), objectid + " : run - " + run.getName())); + else if (exp instanceof ExpData data) + return LI(A(at(target, "_blank", href, urls.getDataDetailsURL(data)), objectid + " : run - " + data.getName())); + else + return LI(String.valueOf(objectid)); + })) + ) + ); + } + + @Override + public boolean handlePost(Object o, BindException errors) + { + var edges = new SqlSelector(ExperimentService.get().getSchema(), "SELECT fromObjectId, toObjectId FROM exp.Edge") + .resultSetStream() + .map(r -> { try { return new Pair<>(r.getInt(1), r.getInt(2)); } catch (SQLException x) { throw new RuntimeException(x); } }) + .collect(toList()); + var cyclesEdges = (new GraphAlgorithms()).detectCycleInDirectedGraph(edges); + + var set = new LinkedHashSet(); + cyclesEdges.forEach( (edge) -> { + set.add(edge.first); + set.add(edge.second); + }); + cycleObjectIds = set.stream().toList(); + return false; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + + } + } + + @RequiresPermission(AdminPermission.class) + public static class MissingFilesCheckAction extends ReadOnlyApiAction + { + @Override + public Object execute(Object o, BindException errors) throws Exception + { + Map> info = ExperimentServiceImpl.get().doMissingFilesCheck(getUser(), getContainer(), true); + JSONObject results = new JSONObject(); + for (String containerId : info.keySet()) + { + JSONObject containerResults = new JSONObject(); + for (String sourceName : info.get(containerId).keySet()) + containerResults.put(sourceName, info.get(containerId).get(sourceName).toJSON()); + results.put(containerId, containerResults); + } + + ApiSimpleResponse response = new ApiSimpleResponse(); + response.put("success", true); + response.put("result", results); + return response; + } + } + +} diff --git a/issues/src/org/labkey/issue/IssuesModule.java b/issues/src/org/labkey/issue/IssuesModule.java index f3f307ebe22..d5b7c549227 100644 --- a/issues/src/org/labkey/issue/IssuesModule.java +++ b/issues/src/org/labkey/issue/IssuesModule.java @@ -1,237 +1,234 @@ -/* - * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.issue; - -import org.jetbrains.annotations.NotNull; -import org.json.JSONObject; -import org.labkey.api.admin.notification.NotificationService; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DataRegion; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.issues.IssueService; -import org.labkey.api.issues.IssuesListDefService; -import org.labkey.api.issues.IssuesSchema; -import org.labkey.api.module.DefaultModule; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.services.ServiceRegistry; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.emailTemplate.EmailTemplateService; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.BaseWebPartFactory; -import org.labkey.api.view.Portal; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.view.WebPartView; -import org.labkey.api.writer.ContainerUser; -import org.labkey.issue.model.GeneralIssuesListDefProvider; -import org.labkey.issue.model.IssueCommentType; -import org.labkey.issue.model.IssueManager; -import org.labkey.issue.model.IssueObject; -import org.labkey.issue.model.IssuesListDefServiceImpl; -import org.labkey.issue.query.IssueDefDomainKind; -import org.labkey.issue.query.IssuesQuerySchema; -import org.labkey.issue.view.IssuesSummaryWebPartFactory; -import org.labkey.issue.view.IssuesWebPartFactory; - -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.labkey.api.issues.IssuesSchema.ISSUE_DEF_SCHEMA_NAME; - -public class IssuesModule extends DefaultModule implements SearchService.DocumentProvider -{ - public static final String NAME = "Issues"; - - @Override - public String getName() - { - return NAME; - } - - @Override - public Double getSchemaVersion() - { - return 25.001; - } - - @Override - protected void init() - { - addController("issues", IssuesController.class); - IssuesQuerySchema.register(this); - - EmailTemplateService.get().registerTemplate(IssueUpdateEmailTemplate.class); - PropertyService.get().registerDomainKind(new IssueDefDomainKind()); - - IssuesListDefService.setInstance(new IssuesListDefServiceImpl()); - IssuesListDefService.get().registerIssuesListDefProvider(new GeneralIssuesListDefProvider()); - - NotificationService.get().registerNotificationType(IssueObject.class.getName(), "Issues", "fa-bug"); - AttachmentService.get().registerAttachmentType(IssueCommentType.get()); - } - - @Override - @NotNull - protected Collection createWebPartFactories() - { - return List.of( - new BaseWebPartFactory("Issue Definitions") - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - UserSchema schema = QueryService.get().getUserSchema(portalCtx.getUser(), portalCtx.getContainer(), IssuesQuerySchema.SCHEMA_NAME); - QuerySettings settings = schema.getSettings(portalCtx, IssuesQuerySchema.TableType.IssueListDef.name(), IssuesQuerySchema.TableType.IssueListDef.name()); - - QueryView view = schema.createView(portalCtx, settings, null); - view.setFrame(WebPartView.FrameType.PORTAL); - view.setTitle("Issue Definitions"); - - return view; - } - }, - new IssuesWebPartFactory(), - new IssuesSummaryWebPartFactory() - ); - } - - @Override - public boolean hasScripts() - { - return true; - } - - - @Override - public void doStartup(ModuleContext moduleContext) - { - ContainerManager.addContainerListener(new IssueContainerListener()); - SecurityManager.addGroupListener(new IssueGroupListener()); - UserManager.addUserListener(new IssueUserListener()); - ServiceRegistry.get().registerService(IssueService.class, new IssueServiceImpl()); - - SearchService ss = SearchService.get(); - - if (null != ss) - { - ss.addSearchCategory(IssueManager.searchCategory); - ss.addResourceResolver("issue", IssueManager.getSearchResolver()); - ss.addDocumentProvider(this); - ss.addSearchResultTemplate(new IssuesController.IssueSearchResultTemplate()); - } - - UsageMetricsService svc = UsageMetricsService.get(); - if (svc != null) - { - svc.registerUsageMetrics(getName(), () -> { - Map metric = new HashMap<>(); - - metric.put("issueDefCount", new SqlSelector(IssuesSchema.getInstance().getSchema(), "SELECT COUNT(*) FROM issues.issueListDef").getObject(Long.class)); - metric.put("issuesCount", new SqlSelector(IssuesSchema.getInstance().getSchema(), "SELECT COUNT(*) FROM issues.issues").getObject(Long.class)); - - return metric; - }); - } - } - - @NotNull - @Override - public Collection getSummary(Container c) - { - Collection list = new LinkedList<>(); - long count = IssueManager.getIssueCount(c); - if (count > 0) - list.add("" + count + " Issue" + (count > 1 ? "s" : "")); - return list; - } - - @Override - public TabDisplayMode getTabDisplayMode() - { - return Module.TabDisplayMode.DISPLAY_USER_PREFERENCE; - } - - @Override - public ActionURL getTabURL(Container c, User user) - { - return new ActionURL(IssuesController.BeginAction.class, c).addParameter(DataRegion.LAST_FILTER_PARAM, true); - } - - @Override - @NotNull - public Set getIntegrationTests() - { - return Collections.singleton(org.labkey.issue.model.IssueManager.TestCase.class); - } - - @Override - @NotNull - public List getSchemaNames() - { - return List.of( - IssuesSchema.getInstance().getSchemaName(), - ISSUE_DEF_SCHEMA_NAME - ); - } - - @Override - public @NotNull Collection getProvisionedSchemaNames() - { - return Set.of( - ISSUE_DEF_SCHEMA_NAME - ); - } - - @Override - public void enumerateDocuments(SearchService.TaskIndexingQueue queue, final Date modifiedSince) - { - queue.addRunnable((q) -> - IssueManager.indexIssues(q, modifiedSince)); - } - - @Override - public void indexDeleted() - { - new SqlExecutor(IssuesSchema.getInstance().getSchema()).execute("UPDATE issues.issues SET lastIndexed=NULL"); - } - - @Override - public JSONObject getPageContextJson(ContainerUser context) - { - JSONObject json = new JSONObject(getDefaultPageContextJson(context.getContainer())); - json.put("hasRestrictedIssueList", IssuesListDefService.get().getRestrictedIssueProvider() != null); - return json; - } -} +/* + * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.issue; + +import org.jetbrains.annotations.NotNull; +import org.json.JSONObject; +import org.labkey.api.admin.notification.NotificationService; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DataRegion; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.issues.IssueService; +import org.labkey.api.issues.IssuesListDefService; +import org.labkey.api.issues.IssuesSchema; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.services.ServiceRegistry; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.emailTemplate.EmailTemplateService; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.BaseWebPartFactory; +import org.labkey.api.view.Portal; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; +import org.labkey.api.writer.ContainerUser; +import org.labkey.issue.model.GeneralIssuesListDefProvider; +import org.labkey.issue.model.IssueCommentType; +import org.labkey.issue.model.IssueManager; +import org.labkey.issue.model.IssueObject; +import org.labkey.issue.model.IssuesListDefServiceImpl; +import org.labkey.issue.query.IssueDefDomainKind; +import org.labkey.issue.query.IssuesQuerySchema; +import org.labkey.issue.view.IssuesSummaryWebPartFactory; +import org.labkey.issue.view.IssuesWebPartFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.labkey.api.issues.IssuesSchema.ISSUE_DEF_SCHEMA_NAME; + +public class IssuesModule extends DefaultModule implements SearchService.DocumentProvider +{ + public static final String NAME = "Issues"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public Double getSchemaVersion() + { + return 25.001; + } + + @Override + protected void init() + { + addController("issues", IssuesController.class); + IssuesQuerySchema.register(this); + + EmailTemplateService.get().registerTemplate(IssueUpdateEmailTemplate.class); + PropertyService.get().registerDomainKind(new IssueDefDomainKind()); + + IssuesListDefService.setInstance(new IssuesListDefServiceImpl()); + IssuesListDefService.get().registerIssuesListDefProvider(new GeneralIssuesListDefProvider()); + + NotificationService.get().registerNotificationType(IssueObject.class.getName(), "Issues", "fa-bug"); + AttachmentService.get().registerAttachmentType(IssueCommentType.get()); + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + return List.of( + new BaseWebPartFactory("Issue Definitions") + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + UserSchema schema = QueryService.get().getUserSchema(portalCtx.getUser(), portalCtx.getContainer(), IssuesQuerySchema.SCHEMA_NAME); + QuerySettings settings = schema.getSettings(portalCtx, IssuesQuerySchema.TableType.IssueListDef.name(), IssuesQuerySchema.TableType.IssueListDef.name()); + + QueryView view = schema.createView(portalCtx, settings, null); + view.setFrame(WebPartView.FrameType.PORTAL); + view.setTitle("Issue Definitions"); + + return view; + } + }, + new IssuesWebPartFactory(), + new IssuesSummaryWebPartFactory() + ); + } + + @Override + public boolean hasScripts() + { + return true; + } + + + @Override + public void doStartup(ModuleContext moduleContext) + { + ContainerManager.addContainerListener(new IssueContainerListener()); + SecurityManager.addGroupListener(new IssueGroupListener()); + UserManager.addUserListener(new IssueUserListener()); + ServiceRegistry.get().registerService(IssueService.class, new IssueServiceImpl()); + + SearchService ss = SearchService.get(); + + ss.addSearchCategory(IssueManager.searchCategory); + ss.addResourceResolver("issue", IssueManager.getSearchResolver()); + ss.addDocumentProvider(this); + ss.addSearchResultTemplate(new IssuesController.IssueSearchResultTemplate()); + + UsageMetricsService svc = UsageMetricsService.get(); + if (svc != null) + { + svc.registerUsageMetrics(getName(), () -> { + Map metric = new HashMap<>(); + + metric.put("issueDefCount", new SqlSelector(IssuesSchema.getInstance().getSchema(), "SELECT COUNT(*) FROM issues.issueListDef").getObject(Long.class)); + metric.put("issuesCount", new SqlSelector(IssuesSchema.getInstance().getSchema(), "SELECT COUNT(*) FROM issues.issues").getObject(Long.class)); + + return metric; + }); + } + } + + @NotNull + @Override + public Collection getSummary(Container c) + { + Collection list = new LinkedList<>(); + long count = IssueManager.getIssueCount(c); + if (count > 0) + list.add("" + count + " Issue" + (count > 1 ? "s" : "")); + return list; + } + + @Override + public TabDisplayMode getTabDisplayMode() + { + return Module.TabDisplayMode.DISPLAY_USER_PREFERENCE; + } + + @Override + public ActionURL getTabURL(Container c, User user) + { + return new ActionURL(IssuesController.BeginAction.class, c).addParameter(DataRegion.LAST_FILTER_PARAM, true); + } + + @Override + @NotNull + public Set getIntegrationTests() + { + return Collections.singleton(org.labkey.issue.model.IssueManager.TestCase.class); + } + + @Override + @NotNull + public List getSchemaNames() + { + return List.of( + IssuesSchema.getInstance().getSchemaName(), + ISSUE_DEF_SCHEMA_NAME + ); + } + + @Override + public @NotNull Collection getProvisionedSchemaNames() + { + return Set.of( + ISSUE_DEF_SCHEMA_NAME + ); + } + + @Override + public void enumerateDocuments(SearchService.TaskIndexingQueue queue, final Date modifiedSince) + { + queue.addRunnable((q) -> + IssueManager.indexIssues(q, modifiedSince)); + } + + @Override + public void indexDeleted() + { + new SqlExecutor(IssuesSchema.getInstance().getSchema()).execute("UPDATE issues.issues SET lastIndexed=NULL"); + } + + @Override + public JSONObject getPageContextJson(ContainerUser context) + { + JSONObject json = new JSONObject(getDefaultPageContextJson(context.getContainer())); + json.put("hasRestrictedIssueList", IssuesListDefService.get().getRestrictedIssueProvider() != null); + return json; + } +} diff --git a/list/src/org/labkey/list/ListModule.java b/list/src/org/labkey/list/ListModule.java index 6ca75a7db52..6d5a7bfc4e7 100644 --- a/list/src/org/labkey/list/ListModule.java +++ b/list/src/org/labkey/list/ListModule.java @@ -1,227 +1,224 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.list; - -import org.jetbrains.annotations.NotNull; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.data.Container; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.lists.permissions.DesignListPermission; -import org.labkey.api.lists.permissions.ManagePicklistsPermission; -import org.labkey.api.module.AdminLinkManager; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.SpringModule; -import org.labkey.api.module.Summary; -import org.labkey.api.query.QueryService; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.PlatformDeveloperPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.WebPartFactory; -import org.labkey.list.controllers.ListController; -import org.labkey.list.model.FolderListImporter; -import org.labkey.list.model.FolderListWriter; -import org.labkey.list.model.IntegerListDomainKind; -import org.labkey.list.model.ListAuditProvider; -import org.labkey.list.model.ListDef; -import org.labkey.list.model.ListManager; -import org.labkey.list.model.ListManagerSchema; -import org.labkey.list.model.ListQuerySchema; -import org.labkey.list.model.ListSchema; -import org.labkey.list.model.ListServiceImpl; -import org.labkey.list.model.ListWriter; -import org.labkey.list.model.PicklistDomainKind; -import org.labkey.list.model.VarcharListDomainKind; -import org.labkey.list.view.ListItemType; -import org.labkey.list.view.ListsWebPart; -import org.labkey.list.view.SingleListWebPartFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class ListModule extends SpringModule -{ - @Override - public String getName() - { - return "List"; - } - - // Note: ExperimentModule handles the list schema - @Override - public Double getSchemaVersion() - { - return 25.000; - } - - // Note: ExperimentModule handles the list schema - @Override - public boolean hasScripts() - { - return true; - } - - @Override - @NotNull - protected Collection createWebPartFactories() - { - return List.of( - ListsWebPart.FACTORY, - new SingleListWebPartFactory() - ); - } - - @Override - protected void init() - { - addController("list", ListController.class); - ListService.setInstance(new ListServiceImpl()); - ListQuerySchema.register(this); - ListManagerSchema.register(this); - - PropertyService.get().registerDomainKind(new IntegerListDomainKind()); - PropertyService.get().registerDomainKind(new VarcharListDomainKind()); - PropertyService.get().registerDomainKind(new PicklistDomainKind()); - - RoleManager.registerPermission(new DesignListPermission()); - RoleManager.registerPermission(new ManagePicklistsPermission()); - - AttachmentService.get().registerAttachmentType(ListItemType.get()); - ExperimentService.get().addExperimentListener(new PicklistMaterialListener()); - - QueryService.get().addCompareType(new PicklistSampleCompareType()); - } - - @Override - public void startupAfterSpringConfig(ModuleContext moduleContext) - { - AuditLogService.get().registerAuditType(new ListAuditProvider()); - - FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); - if (null != folderRegistry) - { - folderRegistry.addWriterFactory(new FolderListWriter.ListDesignWriter.Factory()); - folderRegistry.addWriterFactory(new FolderListWriter.ListDataWriter.Factory()); - folderRegistry.addImportFactory(new FolderListImporter.Factory()); - } - - SearchService ss = SearchService.get(); - if (null != ss) - { - ss.addDocumentProvider(ListManager.get()); - ss.addSearchCategory(ListManager.listCategory); - } - - AdminLinkManager.getInstance().addListener((adminNavTree, container, user) -> - { - // Only need read permissions to view manage lists page - if (container.hasPermission(user, ReadPermission.class)) - adminNavTree.addChild(new NavTree("Manage Lists", ListService.get().getManageListsURL(container))); - }); - - UsageMetricsService svc = UsageMetricsService.get(); - if (null != svc) - { - svc.registerUsageMetrics(getName(), () -> { - Map metric = new HashMap<>(); - DbSchema dbSchema = DbSchema.get("exp", DbSchemaType.Module); - metric.put("listCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list").getObject(Long.class)); - metric.put("publicPicklistCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list WHERE category ='" + ListDefinition.Category.PublicPicklist + "'").getObject(Long.class)); - metric.put("privatePicklistCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list WHERE category ='" + ListDefinition.Category.PrivatePicklist + "'").getObject(Long.class)); - return metric; - }); - } - } - - @NotNull - @Override - public Collection getSummary(Container c) - { - Collection results = new ArrayList<>(); - Collection lists = ListManager.get().getLists(c); - if (!lists.isEmpty()) - { - results.add(lists.size() + " lists"); - } - return results; - } - - @Override - public @NotNull List getDetailedSummary(Container c, User user) - { - ArrayList summary = new ArrayList<>(); - int picklistCount = ListManager.get().getPicklists(c, false).size(); - if (picklistCount > 0) - summary.add(new Summary(picklistCount, "Picklist")); - - return summary; - } - - @Override - public ActionURL getTabURL(Container c, User user) - { - // Don't show full List nav trails to users that aren't admins or developers since they almost certainly don't - // want to go to those links - if (c.hasOneOf(user, AdminPermission.class, PlatformDeveloperPermission.class)) - { - return super.getTabURL(c, user); - } - return null; - } - - @NotNull - @Override - public Set getSchemaNames() - { - return PageFlowUtil.set(ListSchema.getInstance().getSchemaName()); - } - - @NotNull - @Override - public Collection getProvisionedSchemaNames() - { - return PageFlowUtil.set(ListSchema.getInstance().getSchemaName()); - } - - @NotNull - @Override - public Set getUnitTests() - { - return Set.of( - ListManager.TestCase.class, - ListWriter.TestCase.class - ); - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.list; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.data.Container; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.lists.permissions.DesignListPermission; +import org.labkey.api.lists.permissions.ManagePicklistsPermission; +import org.labkey.api.module.AdminLinkManager; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.SpringModule; +import org.labkey.api.module.Summary; +import org.labkey.api.query.QueryService; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.PlatformDeveloperPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.WebPartFactory; +import org.labkey.list.controllers.ListController; +import org.labkey.list.model.FolderListImporter; +import org.labkey.list.model.FolderListWriter; +import org.labkey.list.model.IntegerListDomainKind; +import org.labkey.list.model.ListAuditProvider; +import org.labkey.list.model.ListDef; +import org.labkey.list.model.ListManager; +import org.labkey.list.model.ListManagerSchema; +import org.labkey.list.model.ListQuerySchema; +import org.labkey.list.model.ListSchema; +import org.labkey.list.model.ListServiceImpl; +import org.labkey.list.model.ListWriter; +import org.labkey.list.model.PicklistDomainKind; +import org.labkey.list.model.VarcharListDomainKind; +import org.labkey.list.view.ListItemType; +import org.labkey.list.view.ListsWebPart; +import org.labkey.list.view.SingleListWebPartFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ListModule extends SpringModule +{ + @Override + public String getName() + { + return "List"; + } + + // Note: ExperimentModule handles the list schema + @Override + public Double getSchemaVersion() + { + return 25.000; + } + + // Note: ExperimentModule handles the list schema + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + return List.of( + ListsWebPart.FACTORY, + new SingleListWebPartFactory() + ); + } + + @Override + protected void init() + { + addController("list", ListController.class); + ListService.setInstance(new ListServiceImpl()); + ListQuerySchema.register(this); + ListManagerSchema.register(this); + + PropertyService.get().registerDomainKind(new IntegerListDomainKind()); + PropertyService.get().registerDomainKind(new VarcharListDomainKind()); + PropertyService.get().registerDomainKind(new PicklistDomainKind()); + + RoleManager.registerPermission(new DesignListPermission()); + RoleManager.registerPermission(new ManagePicklistsPermission()); + + AttachmentService.get().registerAttachmentType(ListItemType.get()); + ExperimentService.get().addExperimentListener(new PicklistMaterialListener()); + + QueryService.get().addCompareType(new PicklistSampleCompareType()); + } + + @Override + public void startupAfterSpringConfig(ModuleContext moduleContext) + { + AuditLogService.get().registerAuditType(new ListAuditProvider()); + + FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); + if (null != folderRegistry) + { + folderRegistry.addWriterFactory(new FolderListWriter.ListDesignWriter.Factory()); + folderRegistry.addWriterFactory(new FolderListWriter.ListDataWriter.Factory()); + folderRegistry.addImportFactory(new FolderListImporter.Factory()); + } + + SearchService ss = SearchService.get(); + ss.addDocumentProvider(ListManager.get()); + ss.addSearchCategory(ListManager.listCategory); + + AdminLinkManager.getInstance().addListener((adminNavTree, container, user) -> + { + // Only need read permissions to view manage lists page + if (container.hasPermission(user, ReadPermission.class)) + adminNavTree.addChild(new NavTree("Manage Lists", ListService.get().getManageListsURL(container))); + }); + + UsageMetricsService svc = UsageMetricsService.get(); + if (null != svc) + { + svc.registerUsageMetrics(getName(), () -> { + Map metric = new HashMap<>(); + DbSchema dbSchema = DbSchema.get("exp", DbSchemaType.Module); + metric.put("listCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list").getObject(Long.class)); + metric.put("publicPicklistCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list WHERE category ='" + ListDefinition.Category.PublicPicklist + "'").getObject(Long.class)); + metric.put("privatePicklistCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list WHERE category ='" + ListDefinition.Category.PrivatePicklist + "'").getObject(Long.class)); + return metric; + }); + } + } + + @NotNull + @Override + public Collection getSummary(Container c) + { + Collection results = new ArrayList<>(); + Collection lists = ListManager.get().getLists(c); + if (!lists.isEmpty()) + { + results.add(lists.size() + " lists"); + } + return results; + } + + @Override + public @NotNull List getDetailedSummary(Container c, User user) + { + ArrayList summary = new ArrayList<>(); + int picklistCount = ListManager.get().getPicklists(c, false).size(); + if (picklistCount > 0) + summary.add(new Summary(picklistCount, "Picklist")); + + return summary; + } + + @Override + public ActionURL getTabURL(Container c, User user) + { + // Don't show full List nav trails to users that aren't admins or developers since they almost certainly don't + // want to go to those links + if (c.hasOneOf(user, AdminPermission.class, PlatformDeveloperPermission.class)) + { + return super.getTabURL(c, user); + } + return null; + } + + @NotNull + @Override + public Set getSchemaNames() + { + return PageFlowUtil.set(ListSchema.getInstance().getSchemaName()); + } + + @NotNull + @Override + public Collection getProvisionedSchemaNames() + { + return PageFlowUtil.set(ListSchema.getInstance().getSchemaName()); + } + + @NotNull + @Override + public Set getUnitTests() + { + return Set.of( + ListManager.TestCase.class, + ListWriter.TestCase.class + ); + } +} diff --git a/list/src/org/labkey/list/model/ListManager.java b/list/src/org/labkey/list/model/ListManager.java index 50447942a5b..842b0cd75b1 100644 --- a/list/src/org/labkey/list/model/ListManager.java +++ b/list/src/org/labkey/list/model/ListManager.java @@ -1,1374 +1,1368 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.list.model; - -import org.apache.commons.collections4.MultiValuedMap; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheLoader; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.LabKeyCollectors; -import org.labkey.api.data.*; -import org.labkey.api.data.Selector.ForEachBlock; -import org.labkey.api.exceptions.OptimisticConflictException; -import org.labkey.api.exp.DomainURIFactory; -import org.labkey.api.exp.ImportTypesHelper; -import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptor; -import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptorsList; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListDefinition.BodySetting; -import org.labkey.api.exp.list.ListDefinition.IndexSetting; -import org.labkey.api.exp.list.ListItem; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.JunitUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.Path; -import org.labkey.api.util.StringExpression; -import org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior; -import org.labkey.api.util.StringExpressionFactory.FieldKeyStringExpression; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NavTree; -import org.labkey.api.webdav.SimpleDocumentResource; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.list.controllers.ListController; -import org.labkey.list.model.ListImporter.ValidatorImporter; -import org.labkey.list.view.ListItemAttachmentParent; -import org.springframework.jdbc.BadSqlGrammarException; - -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import static org.labkey.api.util.IntegerUtils.asInteger; - -public class ListManager implements SearchService.DocumentProvider -{ - private static final Logger LOG = LogHelper.getLogger(ListManager.class, "List indexing events"); - private static final String LIST_SEQUENCE_NAME = "org.labkey.list.Lists"; - private static final ListManager INSTANCE = new ListManager(); - - public static final String LIST_AUDIT_EVENT = "ListAuditEvent"; - public static final String LISTID_FIELD_NAME = "listId"; - - - private final Cache> _listDefCache = DatabaseCache.get(CoreSchema.getInstance().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "List definitions", new ListDefCacheLoader()) ; - - private class ListDefCacheLoader implements CacheLoader> - { - @Override - public List load(@NotNull String entityId, @Nullable Object argument) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Container"), entityId); - ArrayList ownLists = new TableSelector(getListMetadataTable(), filter, null).getArrayList(ListDef.class); - return ownLists.isEmpty() ? Collections.emptyList() : Collections.unmodifiableList(ownLists); - } - } - - public static ListManager get() - { - return INSTANCE; - } - - DbSchema getListMetadataSchema() - { - return ExperimentService.get().getSchema(); - } - - TableInfo getListMetadataTable() - { - return getListMetadataSchema().getTable("list"); - } - - public Collection getPicklists(Container container) - { - return getLists(container, true).stream().filter(ListDef::isPicklist).collect(Collectors.toList()); - } - - public Collection getPicklists(Container container, boolean includeProjectAndShared) - { - return getLists(container, includeProjectAndShared).stream().filter(ListDef::isPicklist).collect(Collectors.toList()); - } - - public Collection getLists(Container container) - { - return getLists(container, false); - } - - public Collection getLists(Container container, boolean includeProjectAndShared) - { - return getLists(container, null, false, true, includeProjectAndShared); - } - - public Collection getLists( - @NotNull Container container, - @Nullable User user, - boolean checkVisibility, - boolean includePicklists, - boolean includeProjectAndShared - ) - { - Collection scopedLists = getAllScopedLists(container, includeProjectAndShared); - if (!includePicklists) - scopedLists = scopedLists.stream().filter(listDef -> !listDef.isPicklist()).collect(Collectors.toList()); - if (checkVisibility) - return scopedLists.stream().filter(listDef -> listDef.isVisible(user)).collect(Collectors.toList()); - else - return scopedLists; - } - - /** - * Returns all list definitions defined within the scope of the container. This can optionally include list - * definitions from the container's project as well as the Shared folder. In the event of a name collision the - * closest container's list definition will be returned (i.e. container > project > Shared). - */ - private Collection getAllScopedLists(@NotNull Container container, boolean includeProjectAndShared) - { - List ownLists = _listDefCache.get(container.getId()); - Map listDefMap = new CaseInsensitiveHashMap<>(); - - if (includeProjectAndShared) - { - for (ListDef sharedList : _listDefCache.get(ContainerManager.getSharedContainer().getId())) - listDefMap.put(sharedList.getName(), sharedList); - - Container project = container.getProject(); - if (project != null) - { - for (ListDef projectList : _listDefCache.get(project.getId())) - listDefMap.put(projectList.getName(), projectList); - } - } - - // Workbooks can see parent lists. - if (container.isWorkbook()) - { - Container parent = container.getParent(); - if (parent != null) - { - for (ListDef parentList : _listDefCache.get(parent.getId())) - listDefMap.put(parentList.getName(), parentList); - } - } - - for (ListDef ownList : ownLists) - listDefMap.put(ownList.getName(), ownList); - - return listDefMap.values(); - } - - /** - * Utility method now that ListTable is ContainerFilter aware; TableInfo.getSelectName() returns now returns null - */ - String getListTableName(TableInfo ti) - { - if (ti instanceof ListTable lti) - return lti.getRealTable().getSelectName(); - return ti.getSelectName(); // if db is being upgraded from <= 13.1, lists are still SchemaTableInfo instances - } - - @Nullable - public ListDef getList(Container container, int listId) - { - SimpleFilter filter = new PkFilter(getListMetadataTable(), new Object[]{container, listId}); - ListDef list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDef.class); - - // Workbooks can see their parent's lists, so check that container if we didn't find the list the first time - if (list == null && container.isWorkbook()) - { - filter = new PkFilter(getListMetadataTable(), new Object[]{container.getParent(), listId}); - list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDef.class); - } - return list; - } - - public ListDomainKindProperties getListDomainKindProperties(Container container, @Nullable Integer listId) - { - if (null == listId) - { - return new ListDomainKindProperties(); - } - else - { - SimpleFilter filter = new PkFilter(getListMetadataTable(), new Object[]{container, listId}); - ListDomainKindProperties list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); - - // Workbooks can see their parent's lists, so check that container if we didn't find the list the first time - if (list == null && container.isWorkbook()) - { - filter = new PkFilter(getListMetadataTable(), new Object[]{container.getParent(), listId}); - list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); - } - return list; - } - } - - // Note: callers must invoke indexer (can't invoke here since we may be in a transaction) - public ListDef insert(User user, final ListDef def, Collection preferredListIds) - { - Container c = def.lookupContainer(); - if (null == c) - throw OptimisticConflictException.create(Table.ERROR_DELETED); - - TableInfo tinfo = getListMetadataTable(); - DbSequence sequence = DbSequenceManager.get(c, LIST_SEQUENCE_NAME); - ListDef.ListDefBuilder builder = new ListDef.ListDefBuilder(def); - - builder.setListId(-1); - - for (Integer preferredListId : preferredListIds) - { - SimpleFilter filter = new SimpleFilter(tinfo.getColumn("Container").getFieldKey(), c).addCondition(tinfo.getColumn("ListId"), preferredListId); - - // Need to check proactively... unfortunately, calling insert and handling the constraint violation will cancel the current transaction - if (!new TableSelector(getListMetadataTable().getColumn("ListId"), filter, null).exists()) - { - builder.setListId(preferredListId); - sequence.ensureMinimum(preferredListId); // Ensure sequence is at or above the preferred ID we just used - break; - } - } - - // If none of the preferred IDs is available then use the next sequence value - if (builder.getListId() == -1) - builder.setListId((int)sequence.next()); - - ListDef ret = Table.insert(user, tinfo, builder.build()); - _listDefCache.remove(c.getId()); - return ret; - } - - - // Note: callers must invoke indexer (can't invoke here since we may already be in a transaction) - ListDef update(User user, final ListDef def) - { - Container c = def.lookupContainer(); - if (null == c) - throw OptimisticConflictException.create(Table.ERROR_DELETED); - - DbScope scope = getListMetadataSchema().getScope(); - ListDef ret; - - try (DbScope.Transaction transaction = scope.ensureTransaction()) - { - ListDef old = getList(c, def.getListId()); - ret = Table.update(user, getListMetadataTable(), def, new Object[]{c, def.getListId()}); - handleIndexSettingChanges(scope, def, old, ret); - - String oldName = old.getName(); - String updatedName = ret.getName(); - queryChangeUpdate(user, c, oldName, updatedName); - transaction.commit(); - } - - return ret; - } - - //Note: this is sort of a dupe of above update() which returns ListDef - ListDomainKindProperties update(User user, Container c, final ListDomainKindProperties listProps) - { - if (null == c) - throw OptimisticConflictException.create(Table.ERROR_DELETED); - - DbScope scope = getListMetadataSchema().getScope(); - ListDomainKindProperties updated; - - try (DbScope.Transaction transaction = scope.ensureTransaction()) - { - ListDomainKindProperties old = getListDomainKindProperties(c, listProps.getListId()); - updated = Table.update(user, getListMetadataTable(), listProps, new Object[]{c, listProps.getListId()}); - ListDef listDef = getList(c, listProps.getListId()); - handleIndexSettingChanges(scope, listDef, old, listProps); - String oldName = old.getName(); - String updatedName = updated.getName(); - queryChangeUpdate(user, c, oldName, updatedName); - - transaction.commit(); - } - - return updated; - } - - // Queue up one-time operations related to turning indexing on or off - private void handleIndexSettingChanges(DbScope scope, ListDef listDef, ListIndexingSettings old, ListIndexingSettings updated) - { - boolean oldEachItemIndex = old.isEachItemIndex(); - boolean newEachItemIndex = updated.isEachItemIndex(); - - String oldEachItemTitleTemplate = old.getEachItemTitleTemplate(); - String newEachItemTitleTemplate = updated.getEachItemTitleTemplate(); - - int oldEachItemBodySetting = old.getEachItemBodySetting(); - int newEachItemBodySetting = updated.getEachItemBodySetting(); - - String oldEachItemBodyTemplate = old.getEachItemBodyTemplate(); - String newEachItemBodyTemplate = updated.getEachItemBodyTemplate(); - - boolean oldEntireListIndex = old.isEntireListIndex(); - boolean newEntireListIndex = updated.isEntireListIndex(); - - boolean oldFileAttachmentIndex = old.isFileAttachmentIndex(); - boolean newFileAttachmentIndex = updated.isFileAttachmentIndex(); - - String oldEntireListTitleTemplate = old.getEntireListTitleTemplate(); - String newEntireListTitleTemplate = updated.getEntireListTitleTemplate(); - - int oldEntireListIndexSetting = old.getEntireListIndexSetting(); - int newEntireListIndexSetting = updated.getEntireListIndexSetting(); - - int oldEntireListBodySetting = old.getEntireListBodySetting(); - int newEntireListBodySetting = updated.getEntireListBodySetting(); - - String oldEntireListBodyTemplate = old.getEntireListBodyTemplate(); - String newEntireListBodyTemplate = updated.getEntireListBodyTemplate(); - - scope.addCommitTask(() -> { - ListDefinition list = ListDefinitionImpl.of(listDef); - - // Is each-item indexing turned on? - if (newEachItemIndex) - { - // Turning on each-item indexing, or changing document title template, body template, - // or body setting -> clear this list's LastIndexed column - if - ( - !oldEachItemIndex || - !Objects.equals(newEachItemTitleTemplate, oldEachItemTitleTemplate) || - !Objects.equals(newEachItemBodyTemplate, oldEachItemBodyTemplate) || - newEachItemBodySetting != oldEachItemBodySetting - ) - { - clearLastIndexed(scope, ListSchema.getInstance().getSchemaName(), listDef); - } - } - else - { - // Turning off each-item indexing -> clear item docs from the index - if (oldEachItemIndex) - deleteIndexedItems(list); - } - - // Is attachment indexing turned on? - if (newFileAttachmentIndex) - { - // Turning on attachment indexing or changing title template -> clear attachment LastIndexed column - if - ( - !oldFileAttachmentIndex || - !Objects.equals(newEachItemTitleTemplate, oldEachItemTitleTemplate) // Attachment indexing uses the each-item title template - ) - { - clearAttachmentLastIndexed(list); - } - } - else - { - // Turning off attachment indexing -> clear attachment docs from the index - if (oldFileAttachmentIndex) - deleteIndexedAttachments(list); - } - - // Is entire-list indexing turned on? - if (newEntireListIndex) - { - // Turning on entire-list indexing, or changing the title template, body template, indexing settings, or - // body settings -> clear this list's last indexed column - if - ( - !oldEntireListIndex || - !Objects.equals(newEntireListTitleTemplate, oldEntireListTitleTemplate) || - !Objects.equals(newEntireListBodyTemplate, oldEntireListBodyTemplate) || - newEntireListIndexSetting != oldEntireListIndexSetting || - newEntireListBodySetting != oldEntireListBodySetting - ) - { - SQLFragment sql = new SQLFragment("UPDATE ") - .append(getListMetadataTable().getSelectName()) - .append(" SET LastIndexed = NULL WHERE ListId = ? AND LastIndexed IS NOT NULL") - .add(list.getListId()); - - new SqlExecutor(scope).execute(sql); - } - } - else - { - // Turning off entire-list indexing -> clear entire-list doc from the index - if (oldEntireListIndex) - deleteIndexedEntireListDoc(list); - } - }, DbScope.CommitTaskOption.POSTCOMMIT); - } - - private void queryChangeUpdate(User user, Container c, String oldName, String updatedName) - { - _listDefCache.remove(c.getId()); - QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldName, updatedName, new SchemaKey(null, ListQuerySchema.NAME), user, c); - } - - // CONSIDER: move "list delete" from ListDefinitionImpl.delete() implementation to ListManager for consistency - void deleteListDef(Container c, int listid) - { - DbScope scope = getListMetadataSchema().getScope(); - assert scope.isTransactionActive(); - try - { - Table.delete(ListManager.get().getListMetadataTable(), new Object[]{c, listid}); - } - catch (OptimisticConflictException x) - { - // ok - } - _listDefCache.remove(c.getId()); - } - - public static final SearchService.SearchCategory listCategory = new SearchService.SearchCategory("list", "Lists"); - - // Index all lists in this container - @Override - public void enumerateDocuments(SearchService.TaskIndexingQueue queue, @Nullable Date since) - { - Consumer r = (q) -> { - Map lists = ListService.get().getLists(q.getContainer(), null, false); - - try - { - QueryService.get().setEnvironment(QueryService.Environment.USER, User.getSearchUser()); - QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, q.getContainer()); - for (ListDefinition list : lists.values()) - { - try - { - boolean reindex = since == null; - indexList(q, list, reindex); - } - catch (Exception ex) - { - LOG.error("Error indexing list '" + list.getName() + "' in container '" + q.getContainer().getPath() + "'.", ex); - } - } - } - finally - { - QueryService.get().clearEnvironment(); - } - }; - - queue.addRunnable(r); - } - - public void indexList(final ListDefinition def) - { - indexList(((ListDefinitionImpl) def)._def); - } - - // Index a single list - public void indexList(final ListDef def) - { - SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(def.lookupContainer(), SearchService.PRIORITY.modified); - Consumer r = (q) -> - { - Container c = def.lookupContainer(); - if (!ContainerManager.exists(c)) - { - LOG.info("List container has been deleted or is being deleted; not indexing list \"" + def.getName() + "\""); - } - else - { - //Refresh list definition -- Issue #42207 - MSSQL server returns entityId as uppercase string - ListDefinition list = ListService.get().getList(c, def.getListId()); - if (null != list) // Could have just been deleted - indexList(q, list, false); - } - }; - - Container c = def.lookupContainer(); - if (!ContainerManager.exists(c)) - { - LOG.info("List container has been deleted or is being deleted; not indexing list \"" + def.getName() + "\""); - } - else - { - queue.addRunnable(r); - } - } - - private void indexList(SearchService.TaskIndexingQueue queue, ListDefinition list, final boolean reindex) - { - Domain domain = list.getDomain(); - - // List might have just been deleted - if (null != domain) - { - // indexing methods turn off JDBC driver caching and use a side connection, so we must not be in a transaction - assert !DbScope.getLabKeyScope().isTransactionActive() : "Should not be in a transaction since this code path disables JDBC driver caching"; - - indexEntireList(queue, list, reindex); - indexModifiedItems(queue, list, reindex); - indexAttachments(queue, list, reindex); - } - } - - // Delete a single list item from the index after item delete - public void deleteItemIndex(final ListDefinition list, @NotNull final String entityId) - { - // Transaction-aware is good practice. But it happens to be critical in the case of calling indexEntireList() - // because it turns off JDBC caching, using a non-transacted connection (bad news if we call it mid-transaction). - getListMetadataSchema().getScope().addCommitTask(() -> - { - SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(list.getContainer(), SearchService.PRIORITY.modified); - if (list.getEachItemIndex()) - { - SearchService.get().deleteResource(getDocumentId(list, entityId)); - } - - // Reindex the entire list document iff data is being indexed - if (list.getEntireListIndex() && list.getEntireListIndexSetting().indexItemData()) - { - indexEntireList(queue, list, true); - } - }, DbScope.CommitTaskOption.POSTCOMMIT); - } - - private String getDocumentId(ListDefinition list) - { - return "list:" + ((ListDefinitionImpl)list).getEntityId(); - } - - // Use each item's EntityId since PKs are mutable. ObjectIds maybe be the better choice (they're shorter) but - // that would require adding this column to the query definition. Consider: a private TableInfo just for indexing. - private String getDocumentId(ListDefinition list, @Nullable String entityId) - { - return getDocumentId(list) + ":" + (null != entityId ? entityId : ""); - } - - private static boolean hasAttachmentColumns(@NotNull TableInfo listTable) - { - return listTable.getColumns().stream().anyMatch(ci -> ci.getPropertyType() == PropertyType.ATTACHMENT); - } - - // Index all modified items in this list - private void indexModifiedItems(@NotNull SearchService.TaskIndexingQueue queue, final ListDefinition list, final boolean reindex) - { - if (list.getEachItemIndex()) - { - String lastIndexClause = reindex ? "(1=1) OR " : ""; //Prepend TRUE if we want to force a reindexing - - // Index all items that have never been indexed OR where either the list definition or list item itself has changed since last indexed - lastIndexClause += "LastIndexed IS NULL OR LastIndexed < ? OR (Modified IS NOT NULL AND LastIndexed < Modified)"; - SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause(lastIndexClause, new Object[]{list.getModified()})); - - indexItems(queue, list, filter); - } - } - - // Reindex items specified by filter - private void indexItems(@NotNull SearchService.TaskIndexingQueue queue, final ListDefinition list, SimpleFilter filter) - { - TableInfo listTable = list.getTable(User.getSearchUser()); - - if (null != listTable) - { - FieldKeyStringExpression titleTemplate = createEachItemTitleTemplate(list, listTable); - FieldKeyStringExpression bodyTemplate = createBodyTemplate(list, "\"each item as a separate document\" custom indexing template", list.getEachItemBodySetting(), list.getEachItemBodyTemplate(), listTable); - - FieldKey keyKey = new FieldKey(null, list.getKeyName()); - FieldKey entityIdKey = new FieldKey(null, "EntityId"); - - FieldKey createdKey = new FieldKey(null, "created"); - FieldKey createdByKey = new FieldKey(null, "createdBy"); - FieldKey modifiedKey = new FieldKey(null, "modified"); - FieldKey modifiedByKey = new FieldKey(null, "modifiedBy"); - - // TODO: Attempting to respect tableUrl for details link... but this doesn't actually work. See #28747. - StringExpression se = listTable.getDetailsURL(null, list.getContainer()); - - new TableSelector(listTable, filter, null).setJdbcCaching(false).setForDisplay(true).forEachResults(results -> { - Map map = results.getFieldKeyRowMap(); - final Object pk = map.get(keyKey); - String entityId = (String) map.get(entityIdKey); - - String documentId = getDocumentId(list, entityId); - Map props = new HashMap<>(); - props.put(SearchService.PROPERTY.categories.toString(), listCategory.toString()); - String displayTitle = titleTemplate.eval(map); - props.put(SearchService.PROPERTY.title.toString(), displayTitle); - - Date created = null; - if (map.get(createdKey) instanceof Date) - created = (Date) map.get(createdKey); - - Date modified = null; - if (map.get(modifiedKey) instanceof Date) - modified = (Date) map.get(modifiedKey); - - String body = bodyTemplate.eval(map); - - ActionURL itemURL; - - try - { - itemURL = new ActionURL(se.eval(map)); - } - catch (Exception e) - { - itemURL = list.urlDetails(pk); - } - - itemURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames - - SimpleDocumentResource r = new SimpleDocumentResource( - new Path(documentId), - documentId, - list.getContainer().getEntityId(), - "text/plain", - body, - itemURL, - UserManager.getUser(asInteger( map.get(createdByKey))), created, - UserManager.getUser(asInteger( map.get(modifiedByKey))), modified, - props) - { - @Override - public void setLastIndexed(long ms, long modified) - { - try - { - ListManager.get().setItemLastIndexed(list, pk, listTable, ms, modified); - } - catch (BadSqlGrammarException e) - { - // This may occur due to a race condition between enumeration and list deletion. Issue #48878 - // expected P-sql expected MS-sql - if (e.getCause().getMessage().contains("does not exist") || e.getCause().getMessage().contains("Invalid object name")) - LOG.debug("Attempt to set LastIndexed on list table failed", e); - else - throw e; - } - } - }; - - // Add navtrail that includes link to full list grid - ActionURL gridURL = list.urlShowData(); - gridURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames - NavTree t = new NavTree("list", gridURL); - String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); - r.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); - - queue.addResource(r); - LOG.debug("List \"" + list + "\": Queued indexing of item with PK = " + pk); - }); - } - } - - /** - * Add searchable resources to Indexing task for file attachments - * @param list containing file attachments - */ - private void indexAttachments(@NotNull final SearchService.TaskIndexingQueue queue, ListDefinition list, boolean reindex) - { - TableInfo listTable = list.getTable(User.getSearchUser()); - if (listTable != null && list.getFileAttachmentIndex() && hasAttachmentColumns(listTable)) - { - //Get common objects & properties - AttachmentService as = AttachmentService.get(); - FieldKeyStringExpression titleTemplate = createEachItemTitleTemplate(list, listTable); - - // Breadcrumb link to list is the same for all attachments on all items - ActionURL gridURL = list.urlShowData(); - gridURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames - NavTree t = new NavTree("list", gridURL); - String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); - - // Enumerate all list rows in batches and re-index based on the value of reindex parameter - // For now, enumerate all rows. In the future, pass in a PK filter for the single item change case? - SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause("(1=1)", null)); - - // Need to pass non-null modifiedSince for incremental indexing, otherwise all attachments will be returned - // TODO: Pass modifiedSince into this method? - Date modifiedSince = reindex ? null : new Date(); - - new TableSelector(listTable, filter, null).setJdbcCaching(false).setForDisplay(true).forEachMapBatch(10_000, batch -> { - // RowEntityId -> List item RowMap - Map> lookupMap = batch.stream() - .collect(Collectors.toMap(map -> (String) map.get("EntityId"), map -> map)); - - // RowEntityId -> Document names that need to be indexed - MultiValuedMap documentMultiMap = as.listAttachmentsForIndexing(lookupMap.keySet(), modifiedSince).stream() - .collect(LabKeyCollectors.toMultiValuedMap(stringStringPair -> stringStringPair.first, stringStringPair -> stringStringPair.second)); - - documentMultiMap.asMap().forEach((rowEntityId, documentNames) -> { - Map map = lookupMap.get(rowEntityId); - String title = titleTemplate.eval(map); - - documentNames.forEach(documentName -> { - ActionURL downloadUrl = ListController.getDownloadURL(list, rowEntityId, documentName); - - //Generate searchable resource - String displayTitle = title + " attachment file \"" + documentName + "\""; - WebdavResource attachmentRes = as.getDocumentResource( - new Path(rowEntityId, documentName), - downloadUrl, - displayTitle, - new ListItemAttachmentParent(rowEntityId, list.getContainer()), - documentName, - SearchService.fileCategory - ); - - attachmentRes.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); - queue.addResource(attachmentRes); - LOG.debug("List \"" + list + "\": Queued indexing of attachment \"" + documentName + "\" for item with PK = " + map.get(list.getKeyName())); - }); - }); - }); - } - } - - private void indexEntireList(SearchService.TaskIndexingQueue queue, final ListDefinition list, boolean reindex) - { - if (list.getEntireListIndex()) - { - IndexSetting setting = list.getEntireListIndexSetting(); - String documentId = getDocumentId(list); - - // First check if metadata needs to be indexed: if the setting is enabled and the definition has changed - boolean needToIndex = (setting.indexMetaData() && hasDefinitionChangedSinceLastIndex(list)); - - // If that didn't hold true then check for entire list data indexing: if the definition has changed or any item has been modified - if (!needToIndex && setting.indexItemData()) - needToIndex = hasDefinitionChangedSinceLastIndex(list) || hasModifiedItems(list); - - needToIndex |= reindex; - - if (needToIndex) - { - StringBuilder body = new StringBuilder(); - Map props = new HashMap<>(); - - // Use standard title if template is null/whitespace - String templateString = StringUtils.trimToNull(list.getEntireListTitleTemplate()); - String title = null == templateString ? "List " + list.getName() : templateString; - - props.put(SearchService.PROPERTY.categories.toString(), listCategory.toString()); - props.put(SearchService.PROPERTY.title.toString(), title); - - if (!StringUtils.isEmpty(list.getDescription())) - body.append(list.getDescription()).append("\n"); - - String sep = ""; - - if (setting.indexMetaData()) - { - String comma = ""; - for (DomainProperty property : list.getDomain().getProperties()) - { - String n = StringUtils.trimToEmpty(property.getName()); - String l = StringUtils.trimToEmpty(property.getLabel()); - if (n.equals(l)) - l = ""; - body.append(comma).append(sep).append(StringUtilsLabKey.joinNonBlank(" ", n, l)); - comma = ","; - sep = "\n"; - } - } - - if (setting.indexItemData()) - { - TableInfo ti = list.getTable(User.getSearchUser()); - int fileSizeLimit = (int) (SearchService.get().getFileSizeLimit() * .99); - - if (ti != null) - { - body.append(sep); - FieldKeyStringExpression template = createBodyTemplate(list, "\"entire list as a single document\" custom indexing template", list.getEntireListBodySetting(), list.getEntireListBodyTemplate(), ti); - - // All columns, all rows, no filters, no sorts - new TableSelector(ti).setJdbcCaching(false).setForDisplay(true).forEachResults(new ForEachBlock<>() - { - @Override - public void exec(Results results) throws StopIteratingException - { - body.append(template.eval(results.getFieldKeyRowMap())).append("\n"); - // Issue 25366: Short circuit for very large list - if (body.length() > fileSizeLimit) - { - body.setLength(fileSizeLimit); // indexer also checks size... make sure we're under the limit - stopIterating(); - } - } - }); - } - } - - ActionURL url = list.urlShowData(); - url.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames - - SimpleDocumentResource r = new SimpleDocumentResource( - new Path(documentId), - documentId, - list.getContainer().getEntityId(), - "text/plain", - body.toString(), - url, - props) - { - @Override - public void setLastIndexed(long ms, long modified) - { - ListManager.get().setLastIndexed(list, ms); - } - }; - - queue.addResource(r); - LOG.debug("List \"" + list + "\": Queued indexing of entire list document"); - } - } - } - - void deleteIndexedList(ListDefinition list) - { - if (list.getEntireListIndex()) - deleteIndexedEntireListDoc(list); - - if (list.getEachItemIndex()) - deleteIndexedItems(list); - - if (list.getFileAttachmentIndex()) - deleteIndexedAttachments(list); - } - - private void deleteIndexedAttachments(@NotNull ListDefinition list) - { - handleAttachmentParents(list, AttachmentService::deleteIndexedAttachments); - } - - private void clearAttachmentLastIndexed(@NotNull ListDefinition list) - { - handleAttachmentParents(list, AttachmentService::clearLastIndexed); - } - - private interface AttachmentParentHandler - { - void handle(AttachmentService as, List parentIds); - } - - // If the list has any attachment columns, select all parent IDs and invoke the passed in handler in batches of 10,000 - private void handleAttachmentParents(@NotNull ListDefinition list, AttachmentParentHandler handler) - { - // make sure container still exists (race condition on container delete) - Container listContainer = list.getContainer(); - if (null == listContainer) - return; - TableInfo listTable = new ListQuerySchema(User.getSearchUser(), listContainer).getTable(list.getName()); - if (null == listTable) - return; - - AttachmentService as = AttachmentService.get(); - - if (hasAttachmentColumns(listTable)) - { - new TableSelector(listTable, Collections.singleton("EntityId")).setJdbcCaching(false).forEachBatch(String.class, 10_000, parentIds -> handler.handle(as, parentIds)); - } - } - - // Un-index the entire list doc, but leave the list items alone - private void deleteIndexedEntireListDoc(ListDefinition list) - { - SearchService ss = SearchService.get(); - - if (null != ss) - ss.deleteResource(getDocumentId(list)); - } - - - // Un-index all list items, but leave the entire list doc alone - private void deleteIndexedItems(ListDefinition list) - { - SearchService ss = SearchService.get(); - - if (null != ss) - ss.deleteResourcesForPrefix(getDocumentId(list, null)); - } - - - private FieldKeyStringExpression createEachItemTitleTemplate(ListDefinition list, TableInfo listTable) - { - FieldKeyStringExpression template; - StringBuilder error = new StringBuilder(); - String templateString = StringUtils.trimToNull(list.getEachItemTitleTemplate()); - - if (null != templateString) - { - template = createValidStringExpression(templateString, error); - - if (null != template) - return template; - else - LOG.warn(getTemplateErrorMessage(list, "\"each item as a separate document\" title template", error)); - } - - // Issue 21794: If you're devious enough to put ${ in your list name then we'll just strip it out - String name = list.getName().replaceAll("\\$\\{", "_{"); - template = createValidStringExpression("List " + name + " - ${" + PageFlowUtil.encode(listTable.getTitleColumn()) + "}", error); - - if (null == template) - throw new IllegalStateException(getTemplateErrorMessage(list, "auto-generated title template", error)); - - return template; - } - - - private FieldKeyStringExpression createBodyTemplate(ListDefinition list, String templateType, BodySetting setting, @Nullable String customTemplate, TableInfo listTable) - { - FieldKeyStringExpression template; - StringBuilder error = new StringBuilder(); - - if (setting == BodySetting.Custom && !StringUtils.isBlank(customTemplate)) - { - template = createValidStringExpression(customTemplate, error); - - if (null != template) - return template; - else - LOG.warn(getTemplateErrorMessage(list, templateType, error)); - } - - StringBuilder sb = new StringBuilder(); - String sep = ""; - - for (ColumnInfo column : listTable.getColumns()) - { - if (setting.accept(column)) - { - sb.append(sep); - sb.append("${"); - sb.append(column.getFieldKey().encode()); // Issue 21794: Must encode - sb.append("}"); - sep = " "; - } - } - - template = createValidStringExpression(sb.toString(), error); - - if (null == template) - throw new IllegalStateException(getTemplateErrorMessage(list, "auto-generated indexing template", error)); - - return template; - } - - - // Issue 21726: Perform some simple validation of custom indexing template - private @Nullable FieldKeyStringExpression createValidStringExpression(String template, StringBuilder error) - { - // Don't URL encode and use lenient substitution (replace nulls with blank) - FieldKeyStringExpression se = FieldKeyStringExpression.create(template, false, NullValueBehavior.ReplaceNullWithBlank); - - try - { - // TODO: Is there a more official way to validate a StringExpression? - se.eval(Collections.emptyMap()); - } - catch (IllegalArgumentException e) - { - error.append(e.getMessage()); - se = null; - } - - return se; - } - - - private String getTemplateErrorMessage(ListDefinition list, String templateType, CharSequence message) - { - return "Invalid " + templateType + " for list \"" + list.getName() + "\" in " + list.getContainer().getPath() + ": " + message; - } - - - private boolean hasDefinitionChangedSinceLastIndex(ListDefinition list) - { - return list.getLastIndexed() == null || list.getModified().compareTo(list.getLastIndexed()) > 0; - } - - - // Checks for existence of list items that have been modified since the entire list was last indexed - private boolean hasModifiedItems(ListDefinition list) - { - TableInfo table = list.getTable(User.getSearchUser()); - - if (null != table && null != getListTableName(table)) - { - // Using EXISTS query should be reasonably efficient. - SQLFragment sql = new SQLFragment("SELECT 1 FROM "); - sql.append(getListTableName(table)); - sql.append(" WHERE Modified > (SELECT LastIndexed FROM ").append(getListMetadataTable()); - sql.append(" WHERE ListId = ? AND Container = ?)"); - sql.add(list.getListId()); - sql.add(list.getContainer().getEntityId()); - - return new SqlSelector(getListMetadataSchema(), sql).exists(); - } - - return false; - } - - private void setLastIndexed(ListDefinition list, long ms) - { - // list table does not have an index on listid, so we should include container in the WHERE - SQLFragment update = new SQLFragment("UPDATE ").append(getListMetadataTable()) - .append(" SET LastIndexed = ? WHERE Container = ? AND ListId = ?").addAll(new Timestamp(ms), list.getContainer(), list.getListId()); - new SqlExecutor(getListMetadataSchema()).execute(update); - _listDefCache.remove(list.getContainer().getId()); - list = ListDefinitionImpl.of(getList(list.getContainer(), list.getListId())); - long modified = list.getModified().getTime(); - String warning = ms < modified ? ". WARNING: LastIndexed is less than Modified! " + ms + " vs. " + modified : ""; - LOG.debug("List \"" + list + "\": Set LastIndexed for entire list document" + warning); - } - - - private void setItemLastIndexed(ListDefinition list, Object pk, TableInfo ti, long ms, long modified) - { - // The "search user" might not have access - if (null != ti) - { - // 'unwrap' ListTable to get schema table for update - TableInfo sti = ((ListTable)ti).getSchemaTableInfo(); - ColumnInfo keyColumn = sti.getColumn(list.getKeyName()); - if (null != keyColumn) - { - var keySelectName = keyColumn.getSelectIdentifier(); - SQLFragment sqlf = new SQLFragment("UPDATE ").appendIdentifier(getListTableName(sti)) - .append(" SET LastIndexed = ").appendValue(new Timestamp(ms)) - .append(" WHERE ").appendIdentifier(keySelectName).append(" = ?").add(pk); - new SqlExecutor(sti.getSchema()).execute(sqlf); - } - String warning = ms < modified ? ". WARNING: LastIndexed is less than Modified! " + ms + " vs. " + modified : ""; - LOG.debug("List \"" + list + "\": Set LastIndexed for item with PK = " + pk + warning); - } - } - - - @Override - public void indexDeleted() - { - TableInfo listTable = getListMetadataTable(); - DbScope scope = listTable.getSchema().getScope(); - - // Clear LastIndexed column of the exp.List table, which addresses the "index the entire list as a single document" case - clearLastIndexed(scope, listTable.getSelectName()); - - String listSchemaName = ListSchema.getInstance().getSchemaName(); - - // Now clear LastIndexed column of every underlying list table, which addresses the "index each list item as a separate document" case. See #28748. - new TableSelector(getListMetadataTable()).forEach(ListDef.class, listDef -> clearLastIndexed(scope, listSchemaName, listDef)); - } - - private void clearLastIndexed(DbScope scope, String listSchemaName, ListDef listDef) - { - // Clear LastIndexed column only for lists that are set to index each item, Issue 47998 - if (listDef.isEachItemIndex()) - { - ListDefinition list = new ListDefinitionImpl(listDef); - Domain domain = list.getDomain(); - if (null != domain && null != domain.getStorageTableName()) - { - LOG.info("List " + listDef.getContainerPath() + " - " + listDef.getName() + ": Set to index each item, so clearing last indexed"); - clearLastIndexed(scope, listSchemaName + "." + domain.getStorageTableName()); - } - } - } - - private void clearLastIndexed(DbScope scope, String selectName) - { - try - { - // Yes, that WHERE clause is intentional and makes a big performance improvement in some cases - new SqlExecutor(scope).execute("UPDATE " + selectName + " SET LastIndexed = NULL WHERE LastIndexed IS NOT NULL"); - } - catch (Exception e) - { - // Log the exception, but allow other tables to be cleared - ExceptionUtil.logExceptionToMothership(null, e); - } - } - - void addAuditEvent(ListDefinitionImpl list, User user, String comment) - { - if (null != user) - { - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(list.getContainer(), comment, list); - AuditLogService.get().addEvent(user, event); - } - } - - /** - * Modeled after ListItemImpl.addAuditEvent - */ - void addAuditEvent(ListDefinitionImpl list, User user, Container c, String comment, String entityId, @Nullable String oldRecord, @Nullable String newRecord) - { - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(c, comment, list); - - event.setListItemEntityId(entityId); - if (oldRecord != null) event.setOldRecordMap(oldRecord); - if (newRecord != null) event.setNewRecordMap(newRecord); - - AuditLogService.get().addEvent(user, event); - } - - String formatAuditItem(ListDefinitionImpl list, User user, Map props) - { - String itemRecord = ""; - TableInfo ti = list.getTable(user); - - if (null != ti) - { - Map recordChangedMap = new CaseInsensitiveHashMap<>(); - Set reserved = list.getDomain().getDomainKind().getReservedPropertyNames(list.getDomain(), user); - - // Match props to columns - for (Map.Entry entry : props.entrySet()) - { - String baseKey = entry.getKey(); - - boolean isReserved = false; - for (String res : reserved) - { - if (res.equalsIgnoreCase(baseKey)) - { - isReserved = true; - break; - } - } - - if (isReserved) - continue; - - ColumnInfo col = ti.getColumn(FieldKey.fromParts(baseKey)); - String value = Objects.toString(entry.getValue(), ""); - String key = null; - - if (null != col) - { - // Found the column - key = col.getName(); // best good - } - else - { - // See if there is a match in the domain properties - for (DomainProperty dp : list.getDomain().getProperties()) - { - if (dp.getName().equalsIgnoreCase(baseKey)) - { - key = dp.getName(); // middle good - } - } - - // Try by name - DomainProperty dp = list.getDomain().getPropertyByName(baseKey); - if (null != dp) - key = dp.getName(); - } - - if (null != key && null != value) - recordChangedMap.put(key, value); - } - - if (!recordChangedMap.isEmpty()) - itemRecord = ListAuditProvider.encodeForDataMap(recordChangedMap); - } - - return itemRecord; - } - - boolean importListSchema( - ListDefinition unsavedList, - ImportTypesHelper importHelper, - User user, - Collection validatorImporters, - List errors - ) throws Exception - { - if (!errors.isEmpty()) - return false; - - final Container container = unsavedList.getContainer(); - final Domain domain = unsavedList.getDomain(); - final String typeURI = domain.getTypeURI(); - - DomainURIFactory factory = name -> new Pair<>(typeURI, container); - - ImportPropertyDescriptorsList pds = importHelper.getImportPropertyDescriptors(factory, errors, container); - - if (!errors.isEmpty()) - return false; - - for (ImportPropertyDescriptor ipd : pds.properties) - { - if (null == ipd.domainName || null == ipd.domainURI) - errors.add("List not specified for property: " + ipd.pd.getName()); - } - - if (!errors.isEmpty()) - return false; - - for (ImportPropertyDescriptor ipd : pds.properties) - { - DomainProperty domainProperty = domain.addPropertyOfPropertyDescriptor(ipd.pd); - domainProperty.setConditionalFormats(ipd.formats); - domainProperty.setDefaultValue(ipd.defaultValue); - } - - unsavedList.save(user); - - // Save validators later, after all the lists are imported, #40343 - validatorImporters.add(new ValidatorImporter(domain.getTypeId(), pds.properties, user)); - - return true; - } - - public static class TestCase extends Assert - { - private static final String LIST_NAME = "Unit Test list"; - private static final String WORKBOOK1_NAME = "Unit Test Workbook 1"; - private static final String WORKBOOK2_NAME = "Unit Test Workbook 2"; - private static final String FIELD_NAME = "field"; - private static final String PARENT_LIST_ITEM = "parentItem"; - private static final String WORKBOOK1_LIST_ITEM = "workbook1Item"; - private static final String WORKBOOK2_LIST_ITEM = "workbook2Item"; - private static final Integer PARENT_LI_KEY = 1; - private static final Integer WB1_LI_KEY = 2; - private static final Integer WB2_LI_KEY = 3; - - private ListDefinitionImpl list; - private Container c; - private User u; - private DomainProperty dp; - - @Before - public void setUp() throws Exception - { - JunitUtil.deleteTestContainer(); - cleanup(); - c = JunitUtil.getTestContainer(); - TestContext context = TestContext.get(); - u = context.getUser(); - list = (ListDefinitionImpl)ListService.get().createList(c, LIST_NAME, ListDefinition.KeyType.AutoIncrementInteger); - list.setKeyName("Unit test list Key"); - - dp = list.getDomain().addProperty(); - dp.setName(FIELD_NAME); - dp.setType(PropertyService.get().getType(c, PropertyType.STRING.getXmlName())); - dp.setPropertyURI(ListDomainKind.createPropertyURI(list.getName(), FIELD_NAME, c, list.getKeyType()).toString()); - list.save(u); - - addListItem(c, list, PARENT_LIST_ITEM); - } - - private void addListItem(Container scopedContainer, ListDefinition scopedList, String value) - { - List lis = new ArrayList<>(); - ListItem li = scopedList.createListItem(); - li.setProperty(dp, value); - lis.add(li); - list.insertListItems(u, scopedContainer, lis); - } - - @After - public void tearDown() throws Exception - { - cleanup(); - } - - private void cleanup() throws Exception - { - //TestContext context = TestContext.get(); - ExperimentService.get().deleteAllExpObjInContainer(c, u); - } - - @Test - public void testListServiceInOwnFolder() - { - Map lists = ListService.get().getLists(c); - assertTrue("Test List not found in own container", lists.containsKey(LIST_NAME)); - ListItem li = lists.get(LIST_NAME).getListItem(1, u, c); - assertEquals("Item not found in own container", PARENT_LIST_ITEM, li.getProperty(dp)); - } - - @Test - public void testListServiceInWorkbook() - { - Container workbook1 = setupWorkbook(WORKBOOK1_NAME); - Container workbook2 = setupWorkbook(WORKBOOK2_NAME); - Map lists = ListService.get().getLists(workbook1); - assertTrue("Test List not found in workbook", lists.containsKey(LIST_NAME)); - - checkListItemScoping(workbook1, workbook2); - } - - private Container setupWorkbook(String title) - { - return ContainerManager.createContainer(c, null, title, null, WorkbookContainerType.NAME, u); - } - - private void checkListItemScoping(Container wb1, Container wb2) - { - ListDefinition wbList1 = ListService.get().getLists(wb1).get(LIST_NAME); - ListDefinition wbList2 = ListService.get().getLists(wb2).get(LIST_NAME); - - assertEquals("Lists available to each workbook are not the same", wbList1.toString(), wbList2.toString()); - addListItem(wb1, wbList1, WORKBOOK1_LIST_ITEM); - addListItem(wb2, wbList2, WORKBOOK2_LIST_ITEM); - - assertNull("Parent item visible in workbook", wbList1.getListItem(PARENT_LI_KEY, u, wb1)); - assertNull("Sibling workbook item visible in another workbook", wbList1.getListItem(WB2_LI_KEY, u, wb1)); - assertEquals("Parent container can not see child workbook item", WORKBOOK1_LIST_ITEM, wbList1.getListItem(WB1_LI_KEY, u, c).getProperty(dp)); - assertEquals("Workbook can not see its own list item", WORKBOOK1_LIST_ITEM, wbList1.getListItem(WB1_LI_KEY, u, wb1).getProperty(dp)); - } - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.list.model; + +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheLoader; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.LabKeyCollectors; +import org.labkey.api.data.*; +import org.labkey.api.data.Selector.ForEachBlock; +import org.labkey.api.exceptions.OptimisticConflictException; +import org.labkey.api.exp.DomainURIFactory; +import org.labkey.api.exp.ImportTypesHelper; +import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptor; +import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptorsList; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListDefinition.BodySetting; +import org.labkey.api.exp.list.ListDefinition.IndexSetting; +import org.labkey.api.exp.list.ListItem; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.Path; +import org.labkey.api.util.StringExpression; +import org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior; +import org.labkey.api.util.StringExpressionFactory.FieldKeyStringExpression; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NavTree; +import org.labkey.api.webdav.SimpleDocumentResource; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.list.controllers.ListController; +import org.labkey.list.model.ListImporter.ValidatorImporter; +import org.labkey.list.view.ListItemAttachmentParent; +import org.springframework.jdbc.BadSqlGrammarException; + +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.labkey.api.util.IntegerUtils.asInteger; + +public class ListManager implements SearchService.DocumentProvider +{ + private static final Logger LOG = LogHelper.getLogger(ListManager.class, "List indexing events"); + private static final String LIST_SEQUENCE_NAME = "org.labkey.list.Lists"; + private static final ListManager INSTANCE = new ListManager(); + + public static final String LIST_AUDIT_EVENT = "ListAuditEvent"; + public static final String LISTID_FIELD_NAME = "listId"; + + + private final Cache> _listDefCache = DatabaseCache.get(CoreSchema.getInstance().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "List definitions", new ListDefCacheLoader()) ; + + private class ListDefCacheLoader implements CacheLoader> + { + @Override + public List load(@NotNull String entityId, @Nullable Object argument) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Container"), entityId); + ArrayList ownLists = new TableSelector(getListMetadataTable(), filter, null).getArrayList(ListDef.class); + return ownLists.isEmpty() ? Collections.emptyList() : Collections.unmodifiableList(ownLists); + } + } + + public static ListManager get() + { + return INSTANCE; + } + + DbSchema getListMetadataSchema() + { + return ExperimentService.get().getSchema(); + } + + TableInfo getListMetadataTable() + { + return getListMetadataSchema().getTable("list"); + } + + public Collection getPicklists(Container container) + { + return getLists(container, true).stream().filter(ListDef::isPicklist).collect(Collectors.toList()); + } + + public Collection getPicklists(Container container, boolean includeProjectAndShared) + { + return getLists(container, includeProjectAndShared).stream().filter(ListDef::isPicklist).collect(Collectors.toList()); + } + + public Collection getLists(Container container) + { + return getLists(container, false); + } + + public Collection getLists(Container container, boolean includeProjectAndShared) + { + return getLists(container, null, false, true, includeProjectAndShared); + } + + public Collection getLists( + @NotNull Container container, + @Nullable User user, + boolean checkVisibility, + boolean includePicklists, + boolean includeProjectAndShared + ) + { + Collection scopedLists = getAllScopedLists(container, includeProjectAndShared); + if (!includePicklists) + scopedLists = scopedLists.stream().filter(listDef -> !listDef.isPicklist()).collect(Collectors.toList()); + if (checkVisibility) + return scopedLists.stream().filter(listDef -> listDef.isVisible(user)).collect(Collectors.toList()); + else + return scopedLists; + } + + /** + * Returns all list definitions defined within the scope of the container. This can optionally include list + * definitions from the container's project as well as the Shared folder. In the event of a name collision the + * closest container's list definition will be returned (i.e. container > project > Shared). + */ + private Collection getAllScopedLists(@NotNull Container container, boolean includeProjectAndShared) + { + List ownLists = _listDefCache.get(container.getId()); + Map listDefMap = new CaseInsensitiveHashMap<>(); + + if (includeProjectAndShared) + { + for (ListDef sharedList : _listDefCache.get(ContainerManager.getSharedContainer().getId())) + listDefMap.put(sharedList.getName(), sharedList); + + Container project = container.getProject(); + if (project != null) + { + for (ListDef projectList : _listDefCache.get(project.getId())) + listDefMap.put(projectList.getName(), projectList); + } + } + + // Workbooks can see parent lists. + if (container.isWorkbook()) + { + Container parent = container.getParent(); + if (parent != null) + { + for (ListDef parentList : _listDefCache.get(parent.getId())) + listDefMap.put(parentList.getName(), parentList); + } + } + + for (ListDef ownList : ownLists) + listDefMap.put(ownList.getName(), ownList); + + return listDefMap.values(); + } + + /** + * Utility method now that ListTable is ContainerFilter aware; TableInfo.getSelectName() returns now returns null + */ + String getListTableName(TableInfo ti) + { + if (ti instanceof ListTable lti) + return lti.getRealTable().getSelectName(); + return ti.getSelectName(); // if db is being upgraded from <= 13.1, lists are still SchemaTableInfo instances + } + + @Nullable + public ListDef getList(Container container, int listId) + { + SimpleFilter filter = new PkFilter(getListMetadataTable(), new Object[]{container, listId}); + ListDef list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDef.class); + + // Workbooks can see their parent's lists, so check that container if we didn't find the list the first time + if (list == null && container.isWorkbook()) + { + filter = new PkFilter(getListMetadataTable(), new Object[]{container.getParent(), listId}); + list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDef.class); + } + return list; + } + + public ListDomainKindProperties getListDomainKindProperties(Container container, @Nullable Integer listId) + { + if (null == listId) + { + return new ListDomainKindProperties(); + } + else + { + SimpleFilter filter = new PkFilter(getListMetadataTable(), new Object[]{container, listId}); + ListDomainKindProperties list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); + + // Workbooks can see their parent's lists, so check that container if we didn't find the list the first time + if (list == null && container.isWorkbook()) + { + filter = new PkFilter(getListMetadataTable(), new Object[]{container.getParent(), listId}); + list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); + } + return list; + } + } + + // Note: callers must invoke indexer (can't invoke here since we may be in a transaction) + public ListDef insert(User user, final ListDef def, Collection preferredListIds) + { + Container c = def.lookupContainer(); + if (null == c) + throw OptimisticConflictException.create(Table.ERROR_DELETED); + + TableInfo tinfo = getListMetadataTable(); + DbSequence sequence = DbSequenceManager.get(c, LIST_SEQUENCE_NAME); + ListDef.ListDefBuilder builder = new ListDef.ListDefBuilder(def); + + builder.setListId(-1); + + for (Integer preferredListId : preferredListIds) + { + SimpleFilter filter = new SimpleFilter(tinfo.getColumn("Container").getFieldKey(), c).addCondition(tinfo.getColumn("ListId"), preferredListId); + + // Need to check proactively... unfortunately, calling insert and handling the constraint violation will cancel the current transaction + if (!new TableSelector(getListMetadataTable().getColumn("ListId"), filter, null).exists()) + { + builder.setListId(preferredListId); + sequence.ensureMinimum(preferredListId); // Ensure sequence is at or above the preferred ID we just used + break; + } + } + + // If none of the preferred IDs is available then use the next sequence value + if (builder.getListId() == -1) + builder.setListId((int)sequence.next()); + + ListDef ret = Table.insert(user, tinfo, builder.build()); + _listDefCache.remove(c.getId()); + return ret; + } + + + // Note: callers must invoke indexer (can't invoke here since we may already be in a transaction) + ListDef update(User user, final ListDef def) + { + Container c = def.lookupContainer(); + if (null == c) + throw OptimisticConflictException.create(Table.ERROR_DELETED); + + DbScope scope = getListMetadataSchema().getScope(); + ListDef ret; + + try (DbScope.Transaction transaction = scope.ensureTransaction()) + { + ListDef old = getList(c, def.getListId()); + ret = Table.update(user, getListMetadataTable(), def, new Object[]{c, def.getListId()}); + handleIndexSettingChanges(scope, def, old, ret); + + String oldName = old.getName(); + String updatedName = ret.getName(); + queryChangeUpdate(user, c, oldName, updatedName); + transaction.commit(); + } + + return ret; + } + + //Note: this is sort of a dupe of above update() which returns ListDef + ListDomainKindProperties update(User user, Container c, final ListDomainKindProperties listProps) + { + if (null == c) + throw OptimisticConflictException.create(Table.ERROR_DELETED); + + DbScope scope = getListMetadataSchema().getScope(); + ListDomainKindProperties updated; + + try (DbScope.Transaction transaction = scope.ensureTransaction()) + { + ListDomainKindProperties old = getListDomainKindProperties(c, listProps.getListId()); + updated = Table.update(user, getListMetadataTable(), listProps, new Object[]{c, listProps.getListId()}); + ListDef listDef = getList(c, listProps.getListId()); + handleIndexSettingChanges(scope, listDef, old, listProps); + String oldName = old.getName(); + String updatedName = updated.getName(); + queryChangeUpdate(user, c, oldName, updatedName); + + transaction.commit(); + } + + return updated; + } + + // Queue up one-time operations related to turning indexing on or off + private void handleIndexSettingChanges(DbScope scope, ListDef listDef, ListIndexingSettings old, ListIndexingSettings updated) + { + boolean oldEachItemIndex = old.isEachItemIndex(); + boolean newEachItemIndex = updated.isEachItemIndex(); + + String oldEachItemTitleTemplate = old.getEachItemTitleTemplate(); + String newEachItemTitleTemplate = updated.getEachItemTitleTemplate(); + + int oldEachItemBodySetting = old.getEachItemBodySetting(); + int newEachItemBodySetting = updated.getEachItemBodySetting(); + + String oldEachItemBodyTemplate = old.getEachItemBodyTemplate(); + String newEachItemBodyTemplate = updated.getEachItemBodyTemplate(); + + boolean oldEntireListIndex = old.isEntireListIndex(); + boolean newEntireListIndex = updated.isEntireListIndex(); + + boolean oldFileAttachmentIndex = old.isFileAttachmentIndex(); + boolean newFileAttachmentIndex = updated.isFileAttachmentIndex(); + + String oldEntireListTitleTemplate = old.getEntireListTitleTemplate(); + String newEntireListTitleTemplate = updated.getEntireListTitleTemplate(); + + int oldEntireListIndexSetting = old.getEntireListIndexSetting(); + int newEntireListIndexSetting = updated.getEntireListIndexSetting(); + + int oldEntireListBodySetting = old.getEntireListBodySetting(); + int newEntireListBodySetting = updated.getEntireListBodySetting(); + + String oldEntireListBodyTemplate = old.getEntireListBodyTemplate(); + String newEntireListBodyTemplate = updated.getEntireListBodyTemplate(); + + scope.addCommitTask(() -> { + ListDefinition list = ListDefinitionImpl.of(listDef); + + // Is each-item indexing turned on? + if (newEachItemIndex) + { + // Turning on each-item indexing, or changing document title template, body template, + // or body setting -> clear this list's LastIndexed column + if + ( + !oldEachItemIndex || + !Objects.equals(newEachItemTitleTemplate, oldEachItemTitleTemplate) || + !Objects.equals(newEachItemBodyTemplate, oldEachItemBodyTemplate) || + newEachItemBodySetting != oldEachItemBodySetting + ) + { + clearLastIndexed(scope, ListSchema.getInstance().getSchemaName(), listDef); + } + } + else + { + // Turning off each-item indexing -> clear item docs from the index + if (oldEachItemIndex) + deleteIndexedItems(list); + } + + // Is attachment indexing turned on? + if (newFileAttachmentIndex) + { + // Turning on attachment indexing or changing title template -> clear attachment LastIndexed column + if + ( + !oldFileAttachmentIndex || + !Objects.equals(newEachItemTitleTemplate, oldEachItemTitleTemplate) // Attachment indexing uses the each-item title template + ) + { + clearAttachmentLastIndexed(list); + } + } + else + { + // Turning off attachment indexing -> clear attachment docs from the index + if (oldFileAttachmentIndex) + deleteIndexedAttachments(list); + } + + // Is entire-list indexing turned on? + if (newEntireListIndex) + { + // Turning on entire-list indexing, or changing the title template, body template, indexing settings, or + // body settings -> clear this list's last indexed column + if + ( + !oldEntireListIndex || + !Objects.equals(newEntireListTitleTemplate, oldEntireListTitleTemplate) || + !Objects.equals(newEntireListBodyTemplate, oldEntireListBodyTemplate) || + newEntireListIndexSetting != oldEntireListIndexSetting || + newEntireListBodySetting != oldEntireListBodySetting + ) + { + SQLFragment sql = new SQLFragment("UPDATE ") + .append(getListMetadataTable().getSelectName()) + .append(" SET LastIndexed = NULL WHERE ListId = ? AND LastIndexed IS NOT NULL") + .add(list.getListId()); + + new SqlExecutor(scope).execute(sql); + } + } + else + { + // Turning off entire-list indexing -> clear entire-list doc from the index + if (oldEntireListIndex) + deleteIndexedEntireListDoc(list); + } + }, DbScope.CommitTaskOption.POSTCOMMIT); + } + + private void queryChangeUpdate(User user, Container c, String oldName, String updatedName) + { + _listDefCache.remove(c.getId()); + QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldName, updatedName, new SchemaKey(null, ListQuerySchema.NAME), user, c); + } + + // CONSIDER: move "list delete" from ListDefinitionImpl.delete() implementation to ListManager for consistency + void deleteListDef(Container c, int listid) + { + DbScope scope = getListMetadataSchema().getScope(); + assert scope.isTransactionActive(); + try + { + Table.delete(ListManager.get().getListMetadataTable(), new Object[]{c, listid}); + } + catch (OptimisticConflictException x) + { + // ok + } + _listDefCache.remove(c.getId()); + } + + public static final SearchService.SearchCategory listCategory = new SearchService.SearchCategory("list", "Lists"); + + // Index all lists in this container + @Override + public void enumerateDocuments(SearchService.TaskIndexingQueue queue, @Nullable Date since) + { + Consumer r = (q) -> { + Map lists = ListService.get().getLists(q.getContainer(), null, false); + + try + { + QueryService.get().setEnvironment(QueryService.Environment.USER, User.getSearchUser()); + QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, q.getContainer()); + for (ListDefinition list : lists.values()) + { + try + { + boolean reindex = since == null; + indexList(q, list, reindex); + } + catch (Exception ex) + { + LOG.error("Error indexing list '" + list.getName() + "' in container '" + q.getContainer().getPath() + "'.", ex); + } + } + } + finally + { + QueryService.get().clearEnvironment(); + } + }; + + queue.addRunnable(r); + } + + public void indexList(final ListDefinition def) + { + indexList(((ListDefinitionImpl) def)._def); + } + + // Index a single list + public void indexList(final ListDef def) + { + SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(def.lookupContainer(), SearchService.PRIORITY.modified); + Consumer r = (q) -> + { + Container c = def.lookupContainer(); + if (!ContainerManager.exists(c)) + { + LOG.info("List container has been deleted or is being deleted; not indexing list \"" + def.getName() + "\""); + } + else + { + //Refresh list definition -- Issue #42207 - MSSQL server returns entityId as uppercase string + ListDefinition list = ListService.get().getList(c, def.getListId()); + if (null != list) // Could have just been deleted + indexList(q, list, false); + } + }; + + Container c = def.lookupContainer(); + if (!ContainerManager.exists(c)) + { + LOG.info("List container has been deleted or is being deleted; not indexing list \"" + def.getName() + "\""); + } + else + { + queue.addRunnable(r); + } + } + + private void indexList(SearchService.TaskIndexingQueue queue, ListDefinition list, final boolean reindex) + { + Domain domain = list.getDomain(); + + // List might have just been deleted + if (null != domain) + { + // indexing methods turn off JDBC driver caching and use a side connection, so we must not be in a transaction + assert !DbScope.getLabKeyScope().isTransactionActive() : "Should not be in a transaction since this code path disables JDBC driver caching"; + + indexEntireList(queue, list, reindex); + indexModifiedItems(queue, list, reindex); + indexAttachments(queue, list, reindex); + } + } + + // Delete a single list item from the index after item delete + public void deleteItemIndex(final ListDefinition list, @NotNull final String entityId) + { + // Transaction-aware is good practice. But it happens to be critical in the case of calling indexEntireList() + // because it turns off JDBC caching, using a non-transacted connection (bad news if we call it mid-transaction). + getListMetadataSchema().getScope().addCommitTask(() -> + { + SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(list.getContainer(), SearchService.PRIORITY.modified); + if (list.getEachItemIndex()) + { + SearchService.get().deleteResource(getDocumentId(list, entityId)); + } + + // Reindex the entire list document iff data is being indexed + if (list.getEntireListIndex() && list.getEntireListIndexSetting().indexItemData()) + { + indexEntireList(queue, list, true); + } + }, DbScope.CommitTaskOption.POSTCOMMIT); + } + + private String getDocumentId(ListDefinition list) + { + return "list:" + ((ListDefinitionImpl)list).getEntityId(); + } + + // Use each item's EntityId since PKs are mutable. ObjectIds maybe be the better choice (they're shorter) but + // that would require adding this column to the query definition. Consider: a private TableInfo just for indexing. + private String getDocumentId(ListDefinition list, @Nullable String entityId) + { + return getDocumentId(list) + ":" + (null != entityId ? entityId : ""); + } + + private static boolean hasAttachmentColumns(@NotNull TableInfo listTable) + { + return listTable.getColumns().stream().anyMatch(ci -> ci.getPropertyType() == PropertyType.ATTACHMENT); + } + + // Index all modified items in this list + private void indexModifiedItems(@NotNull SearchService.TaskIndexingQueue queue, final ListDefinition list, final boolean reindex) + { + if (list.getEachItemIndex()) + { + String lastIndexClause = reindex ? "(1=1) OR " : ""; //Prepend TRUE if we want to force a reindexing + + // Index all items that have never been indexed OR where either the list definition or list item itself has changed since last indexed + lastIndexClause += "LastIndexed IS NULL OR LastIndexed < ? OR (Modified IS NOT NULL AND LastIndexed < Modified)"; + SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause(lastIndexClause, new Object[]{list.getModified()})); + + indexItems(queue, list, filter); + } + } + + // Reindex items specified by filter + private void indexItems(@NotNull SearchService.TaskIndexingQueue queue, final ListDefinition list, SimpleFilter filter) + { + TableInfo listTable = list.getTable(User.getSearchUser()); + + if (null != listTable) + { + FieldKeyStringExpression titleTemplate = createEachItemTitleTemplate(list, listTable); + FieldKeyStringExpression bodyTemplate = createBodyTemplate(list, "\"each item as a separate document\" custom indexing template", list.getEachItemBodySetting(), list.getEachItemBodyTemplate(), listTable); + + FieldKey keyKey = new FieldKey(null, list.getKeyName()); + FieldKey entityIdKey = new FieldKey(null, "EntityId"); + + FieldKey createdKey = new FieldKey(null, "created"); + FieldKey createdByKey = new FieldKey(null, "createdBy"); + FieldKey modifiedKey = new FieldKey(null, "modified"); + FieldKey modifiedByKey = new FieldKey(null, "modifiedBy"); + + // TODO: Attempting to respect tableUrl for details link... but this doesn't actually work. See #28747. + StringExpression se = listTable.getDetailsURL(null, list.getContainer()); + + new TableSelector(listTable, filter, null).setJdbcCaching(false).setForDisplay(true).forEachResults(results -> { + Map map = results.getFieldKeyRowMap(); + final Object pk = map.get(keyKey); + String entityId = (String) map.get(entityIdKey); + + String documentId = getDocumentId(list, entityId); + Map props = new HashMap<>(); + props.put(SearchService.PROPERTY.categories.toString(), listCategory.toString()); + String displayTitle = titleTemplate.eval(map); + props.put(SearchService.PROPERTY.title.toString(), displayTitle); + + Date created = null; + if (map.get(createdKey) instanceof Date) + created = (Date) map.get(createdKey); + + Date modified = null; + if (map.get(modifiedKey) instanceof Date) + modified = (Date) map.get(modifiedKey); + + String body = bodyTemplate.eval(map); + + ActionURL itemURL; + + try + { + itemURL = new ActionURL(se.eval(map)); + } + catch (Exception e) + { + itemURL = list.urlDetails(pk); + } + + itemURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames + + SimpleDocumentResource r = new SimpleDocumentResource( + new Path(documentId), + documentId, + list.getContainer().getEntityId(), + "text/plain", + body, + itemURL, + UserManager.getUser(asInteger( map.get(createdByKey))), created, + UserManager.getUser(asInteger( map.get(modifiedByKey))), modified, + props) + { + @Override + public void setLastIndexed(long ms, long modified) + { + try + { + ListManager.get().setItemLastIndexed(list, pk, listTable, ms, modified); + } + catch (BadSqlGrammarException e) + { + // This may occur due to a race condition between enumeration and list deletion. Issue #48878 + // expected P-sql expected MS-sql + if (e.getCause().getMessage().contains("does not exist") || e.getCause().getMessage().contains("Invalid object name")) + LOG.debug("Attempt to set LastIndexed on list table failed", e); + else + throw e; + } + } + }; + + // Add navtrail that includes link to full list grid + ActionURL gridURL = list.urlShowData(); + gridURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames + NavTree t = new NavTree("list", gridURL); + String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); + r.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); + + queue.addResource(r); + LOG.debug("List \"" + list + "\": Queued indexing of item with PK = " + pk); + }); + } + } + + /** + * Add searchable resources to Indexing task for file attachments + * @param list containing file attachments + */ + private void indexAttachments(@NotNull final SearchService.TaskIndexingQueue queue, ListDefinition list, boolean reindex) + { + TableInfo listTable = list.getTable(User.getSearchUser()); + if (listTable != null && list.getFileAttachmentIndex() && hasAttachmentColumns(listTable)) + { + //Get common objects & properties + AttachmentService as = AttachmentService.get(); + FieldKeyStringExpression titleTemplate = createEachItemTitleTemplate(list, listTable); + + // Breadcrumb link to list is the same for all attachments on all items + ActionURL gridURL = list.urlShowData(); + gridURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames + NavTree t = new NavTree("list", gridURL); + String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); + + // Enumerate all list rows in batches and re-index based on the value of reindex parameter + // For now, enumerate all rows. In the future, pass in a PK filter for the single item change case? + SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause("(1=1)", null)); + + // Need to pass non-null modifiedSince for incremental indexing, otherwise all attachments will be returned + // TODO: Pass modifiedSince into this method? + Date modifiedSince = reindex ? null : new Date(); + + new TableSelector(listTable, filter, null).setJdbcCaching(false).setForDisplay(true).forEachMapBatch(10_000, batch -> { + // RowEntityId -> List item RowMap + Map> lookupMap = batch.stream() + .collect(Collectors.toMap(map -> (String) map.get("EntityId"), map -> map)); + + // RowEntityId -> Document names that need to be indexed + MultiValuedMap documentMultiMap = as.listAttachmentsForIndexing(lookupMap.keySet(), modifiedSince).stream() + .collect(LabKeyCollectors.toMultiValuedMap(stringStringPair -> stringStringPair.first, stringStringPair -> stringStringPair.second)); + + documentMultiMap.asMap().forEach((rowEntityId, documentNames) -> { + Map map = lookupMap.get(rowEntityId); + String title = titleTemplate.eval(map); + + documentNames.forEach(documentName -> { + ActionURL downloadUrl = ListController.getDownloadURL(list, rowEntityId, documentName); + + //Generate searchable resource + String displayTitle = title + " attachment file \"" + documentName + "\""; + WebdavResource attachmentRes = as.getDocumentResource( + new Path(rowEntityId, documentName), + downloadUrl, + displayTitle, + new ListItemAttachmentParent(rowEntityId, list.getContainer()), + documentName, + SearchService.fileCategory + ); + + attachmentRes.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); + queue.addResource(attachmentRes); + LOG.debug("List \"" + list + "\": Queued indexing of attachment \"" + documentName + "\" for item with PK = " + map.get(list.getKeyName())); + }); + }); + }); + } + } + + private void indexEntireList(SearchService.TaskIndexingQueue queue, final ListDefinition list, boolean reindex) + { + if (list.getEntireListIndex()) + { + IndexSetting setting = list.getEntireListIndexSetting(); + String documentId = getDocumentId(list); + + // First check if metadata needs to be indexed: if the setting is enabled and the definition has changed + boolean needToIndex = (setting.indexMetaData() && hasDefinitionChangedSinceLastIndex(list)); + + // If that didn't hold true then check for entire list data indexing: if the definition has changed or any item has been modified + if (!needToIndex && setting.indexItemData()) + needToIndex = hasDefinitionChangedSinceLastIndex(list) || hasModifiedItems(list); + + needToIndex |= reindex; + + if (needToIndex) + { + StringBuilder body = new StringBuilder(); + Map props = new HashMap<>(); + + // Use standard title if template is null/whitespace + String templateString = StringUtils.trimToNull(list.getEntireListTitleTemplate()); + String title = null == templateString ? "List " + list.getName() : templateString; + + props.put(SearchService.PROPERTY.categories.toString(), listCategory.toString()); + props.put(SearchService.PROPERTY.title.toString(), title); + + if (!StringUtils.isEmpty(list.getDescription())) + body.append(list.getDescription()).append("\n"); + + String sep = ""; + + if (setting.indexMetaData()) + { + String comma = ""; + for (DomainProperty property : list.getDomain().getProperties()) + { + String n = StringUtils.trimToEmpty(property.getName()); + String l = StringUtils.trimToEmpty(property.getLabel()); + if (n.equals(l)) + l = ""; + body.append(comma).append(sep).append(StringUtilsLabKey.joinNonBlank(" ", n, l)); + comma = ","; + sep = "\n"; + } + } + + if (setting.indexItemData()) + { + TableInfo ti = list.getTable(User.getSearchUser()); + int fileSizeLimit = (int) (SearchService.get().getFileSizeLimit() * .99); + + if (ti != null) + { + body.append(sep); + FieldKeyStringExpression template = createBodyTemplate(list, "\"entire list as a single document\" custom indexing template", list.getEntireListBodySetting(), list.getEntireListBodyTemplate(), ti); + + // All columns, all rows, no filters, no sorts + new TableSelector(ti).setJdbcCaching(false).setForDisplay(true).forEachResults(new ForEachBlock<>() + { + @Override + public void exec(Results results) throws StopIteratingException + { + body.append(template.eval(results.getFieldKeyRowMap())).append("\n"); + // Issue 25366: Short circuit for very large list + if (body.length() > fileSizeLimit) + { + body.setLength(fileSizeLimit); // indexer also checks size... make sure we're under the limit + stopIterating(); + } + } + }); + } + } + + ActionURL url = list.urlShowData(); + url.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames + + SimpleDocumentResource r = new SimpleDocumentResource( + new Path(documentId), + documentId, + list.getContainer().getEntityId(), + "text/plain", + body.toString(), + url, + props) + { + @Override + public void setLastIndexed(long ms, long modified) + { + ListManager.get().setLastIndexed(list, ms); + } + }; + + queue.addResource(r); + LOG.debug("List \"" + list + "\": Queued indexing of entire list document"); + } + } + } + + void deleteIndexedList(ListDefinition list) + { + if (list.getEntireListIndex()) + deleteIndexedEntireListDoc(list); + + if (list.getEachItemIndex()) + deleteIndexedItems(list); + + if (list.getFileAttachmentIndex()) + deleteIndexedAttachments(list); + } + + private void deleteIndexedAttachments(@NotNull ListDefinition list) + { + handleAttachmentParents(list, AttachmentService::deleteIndexedAttachments); + } + + private void clearAttachmentLastIndexed(@NotNull ListDefinition list) + { + handleAttachmentParents(list, AttachmentService::clearLastIndexed); + } + + private interface AttachmentParentHandler + { + void handle(AttachmentService as, List parentIds); + } + + // If the list has any attachment columns, select all parent IDs and invoke the passed in handler in batches of 10,000 + private void handleAttachmentParents(@NotNull ListDefinition list, AttachmentParentHandler handler) + { + // make sure container still exists (race condition on container delete) + Container listContainer = list.getContainer(); + if (null == listContainer) + return; + TableInfo listTable = new ListQuerySchema(User.getSearchUser(), listContainer).getTable(list.getName()); + if (null == listTable) + return; + + AttachmentService as = AttachmentService.get(); + + if (hasAttachmentColumns(listTable)) + { + new TableSelector(listTable, Collections.singleton("EntityId")).setJdbcCaching(false).forEachBatch(String.class, 10_000, parentIds -> handler.handle(as, parentIds)); + } + } + + // Un-index the entire list doc, but leave the list items alone + private void deleteIndexedEntireListDoc(ListDefinition list) + { + SearchService.get().deleteResource(getDocumentId(list)); + } + + + // Un-index all list items, but leave the entire list doc alone + private void deleteIndexedItems(ListDefinition list) + { + SearchService.get().deleteResourcesForPrefix(getDocumentId(list, null)); + } + + + private FieldKeyStringExpression createEachItemTitleTemplate(ListDefinition list, TableInfo listTable) + { + FieldKeyStringExpression template; + StringBuilder error = new StringBuilder(); + String templateString = StringUtils.trimToNull(list.getEachItemTitleTemplate()); + + if (null != templateString) + { + template = createValidStringExpression(templateString, error); + + if (null != template) + return template; + else + LOG.warn(getTemplateErrorMessage(list, "\"each item as a separate document\" title template", error)); + } + + // Issue 21794: If you're devious enough to put ${ in your list name then we'll just strip it out + String name = list.getName().replaceAll("\\$\\{", "_{"); + template = createValidStringExpression("List " + name + " - ${" + PageFlowUtil.encode(listTable.getTitleColumn()) + "}", error); + + if (null == template) + throw new IllegalStateException(getTemplateErrorMessage(list, "auto-generated title template", error)); + + return template; + } + + + private FieldKeyStringExpression createBodyTemplate(ListDefinition list, String templateType, BodySetting setting, @Nullable String customTemplate, TableInfo listTable) + { + FieldKeyStringExpression template; + StringBuilder error = new StringBuilder(); + + if (setting == BodySetting.Custom && !StringUtils.isBlank(customTemplate)) + { + template = createValidStringExpression(customTemplate, error); + + if (null != template) + return template; + else + LOG.warn(getTemplateErrorMessage(list, templateType, error)); + } + + StringBuilder sb = new StringBuilder(); + String sep = ""; + + for (ColumnInfo column : listTable.getColumns()) + { + if (setting.accept(column)) + { + sb.append(sep); + sb.append("${"); + sb.append(column.getFieldKey().encode()); // Issue 21794: Must encode + sb.append("}"); + sep = " "; + } + } + + template = createValidStringExpression(sb.toString(), error); + + if (null == template) + throw new IllegalStateException(getTemplateErrorMessage(list, "auto-generated indexing template", error)); + + return template; + } + + + // Issue 21726: Perform some simple validation of custom indexing template + private @Nullable FieldKeyStringExpression createValidStringExpression(String template, StringBuilder error) + { + // Don't URL encode and use lenient substitution (replace nulls with blank) + FieldKeyStringExpression se = FieldKeyStringExpression.create(template, false, NullValueBehavior.ReplaceNullWithBlank); + + try + { + // TODO: Is there a more official way to validate a StringExpression? + se.eval(Collections.emptyMap()); + } + catch (IllegalArgumentException e) + { + error.append(e.getMessage()); + se = null; + } + + return se; + } + + + private String getTemplateErrorMessage(ListDefinition list, String templateType, CharSequence message) + { + return "Invalid " + templateType + " for list \"" + list.getName() + "\" in " + list.getContainer().getPath() + ": " + message; + } + + + private boolean hasDefinitionChangedSinceLastIndex(ListDefinition list) + { + return list.getLastIndexed() == null || list.getModified().compareTo(list.getLastIndexed()) > 0; + } + + + // Checks for existence of list items that have been modified since the entire list was last indexed + private boolean hasModifiedItems(ListDefinition list) + { + TableInfo table = list.getTable(User.getSearchUser()); + + if (null != table && null != getListTableName(table)) + { + // Using EXISTS query should be reasonably efficient. + SQLFragment sql = new SQLFragment("SELECT 1 FROM "); + sql.append(getListTableName(table)); + sql.append(" WHERE Modified > (SELECT LastIndexed FROM ").append(getListMetadataTable()); + sql.append(" WHERE ListId = ? AND Container = ?)"); + sql.add(list.getListId()); + sql.add(list.getContainer().getEntityId()); + + return new SqlSelector(getListMetadataSchema(), sql).exists(); + } + + return false; + } + + private void setLastIndexed(ListDefinition list, long ms) + { + // list table does not have an index on listid, so we should include container in the WHERE + SQLFragment update = new SQLFragment("UPDATE ").append(getListMetadataTable()) + .append(" SET LastIndexed = ? WHERE Container = ? AND ListId = ?").addAll(new Timestamp(ms), list.getContainer(), list.getListId()); + new SqlExecutor(getListMetadataSchema()).execute(update); + _listDefCache.remove(list.getContainer().getId()); + list = ListDefinitionImpl.of(getList(list.getContainer(), list.getListId())); + long modified = list.getModified().getTime(); + String warning = ms < modified ? ". WARNING: LastIndexed is less than Modified! " + ms + " vs. " + modified : ""; + LOG.debug("List \"" + list + "\": Set LastIndexed for entire list document" + warning); + } + + + private void setItemLastIndexed(ListDefinition list, Object pk, TableInfo ti, long ms, long modified) + { + // The "search user" might not have access + if (null != ti) + { + // 'unwrap' ListTable to get schema table for update + TableInfo sti = ((ListTable)ti).getSchemaTableInfo(); + ColumnInfo keyColumn = sti.getColumn(list.getKeyName()); + if (null != keyColumn) + { + var keySelectName = keyColumn.getSelectIdentifier(); + SQLFragment sqlf = new SQLFragment("UPDATE ").appendIdentifier(getListTableName(sti)) + .append(" SET LastIndexed = ").appendValue(new Timestamp(ms)) + .append(" WHERE ").appendIdentifier(keySelectName).append(" = ?").add(pk); + new SqlExecutor(sti.getSchema()).execute(sqlf); + } + String warning = ms < modified ? ". WARNING: LastIndexed is less than Modified! " + ms + " vs. " + modified : ""; + LOG.debug("List \"" + list + "\": Set LastIndexed for item with PK = " + pk + warning); + } + } + + + @Override + public void indexDeleted() + { + TableInfo listTable = getListMetadataTable(); + DbScope scope = listTable.getSchema().getScope(); + + // Clear LastIndexed column of the exp.List table, which addresses the "index the entire list as a single document" case + clearLastIndexed(scope, listTable.getSelectName()); + + String listSchemaName = ListSchema.getInstance().getSchemaName(); + + // Now clear LastIndexed column of every underlying list table, which addresses the "index each list item as a separate document" case. See #28748. + new TableSelector(getListMetadataTable()).forEach(ListDef.class, listDef -> clearLastIndexed(scope, listSchemaName, listDef)); + } + + private void clearLastIndexed(DbScope scope, String listSchemaName, ListDef listDef) + { + // Clear LastIndexed column only for lists that are set to index each item, Issue 47998 + if (listDef.isEachItemIndex()) + { + ListDefinition list = new ListDefinitionImpl(listDef); + Domain domain = list.getDomain(); + if (null != domain && null != domain.getStorageTableName()) + { + LOG.info("List " + listDef.getContainerPath() + " - " + listDef.getName() + ": Set to index each item, so clearing last indexed"); + clearLastIndexed(scope, listSchemaName + "." + domain.getStorageTableName()); + } + } + } + + private void clearLastIndexed(DbScope scope, String selectName) + { + try + { + // Yes, that WHERE clause is intentional and makes a big performance improvement in some cases + new SqlExecutor(scope).execute("UPDATE " + selectName + " SET LastIndexed = NULL WHERE LastIndexed IS NOT NULL"); + } + catch (Exception e) + { + // Log the exception, but allow other tables to be cleared + ExceptionUtil.logExceptionToMothership(null, e); + } + } + + void addAuditEvent(ListDefinitionImpl list, User user, String comment) + { + if (null != user) + { + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(list.getContainer(), comment, list); + AuditLogService.get().addEvent(user, event); + } + } + + /** + * Modeled after ListItemImpl.addAuditEvent + */ + void addAuditEvent(ListDefinitionImpl list, User user, Container c, String comment, String entityId, @Nullable String oldRecord, @Nullable String newRecord) + { + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(c, comment, list); + + event.setListItemEntityId(entityId); + if (oldRecord != null) event.setOldRecordMap(oldRecord); + if (newRecord != null) event.setNewRecordMap(newRecord); + + AuditLogService.get().addEvent(user, event); + } + + String formatAuditItem(ListDefinitionImpl list, User user, Map props) + { + String itemRecord = ""; + TableInfo ti = list.getTable(user); + + if (null != ti) + { + Map recordChangedMap = new CaseInsensitiveHashMap<>(); + Set reserved = list.getDomain().getDomainKind().getReservedPropertyNames(list.getDomain(), user); + + // Match props to columns + for (Map.Entry entry : props.entrySet()) + { + String baseKey = entry.getKey(); + + boolean isReserved = false; + for (String res : reserved) + { + if (res.equalsIgnoreCase(baseKey)) + { + isReserved = true; + break; + } + } + + if (isReserved) + continue; + + ColumnInfo col = ti.getColumn(FieldKey.fromParts(baseKey)); + String value = Objects.toString(entry.getValue(), ""); + String key = null; + + if (null != col) + { + // Found the column + key = col.getName(); // best good + } + else + { + // See if there is a match in the domain properties + for (DomainProperty dp : list.getDomain().getProperties()) + { + if (dp.getName().equalsIgnoreCase(baseKey)) + { + key = dp.getName(); // middle good + } + } + + // Try by name + DomainProperty dp = list.getDomain().getPropertyByName(baseKey); + if (null != dp) + key = dp.getName(); + } + + if (null != key && null != value) + recordChangedMap.put(key, value); + } + + if (!recordChangedMap.isEmpty()) + itemRecord = ListAuditProvider.encodeForDataMap(recordChangedMap); + } + + return itemRecord; + } + + boolean importListSchema( + ListDefinition unsavedList, + ImportTypesHelper importHelper, + User user, + Collection validatorImporters, + List errors + ) throws Exception + { + if (!errors.isEmpty()) + return false; + + final Container container = unsavedList.getContainer(); + final Domain domain = unsavedList.getDomain(); + final String typeURI = domain.getTypeURI(); + + DomainURIFactory factory = name -> new Pair<>(typeURI, container); + + ImportPropertyDescriptorsList pds = importHelper.getImportPropertyDescriptors(factory, errors, container); + + if (!errors.isEmpty()) + return false; + + for (ImportPropertyDescriptor ipd : pds.properties) + { + if (null == ipd.domainName || null == ipd.domainURI) + errors.add("List not specified for property: " + ipd.pd.getName()); + } + + if (!errors.isEmpty()) + return false; + + for (ImportPropertyDescriptor ipd : pds.properties) + { + DomainProperty domainProperty = domain.addPropertyOfPropertyDescriptor(ipd.pd); + domainProperty.setConditionalFormats(ipd.formats); + domainProperty.setDefaultValue(ipd.defaultValue); + } + + unsavedList.save(user); + + // Save validators later, after all the lists are imported, #40343 + validatorImporters.add(new ValidatorImporter(domain.getTypeId(), pds.properties, user)); + + return true; + } + + public static class TestCase extends Assert + { + private static final String LIST_NAME = "Unit Test list"; + private static final String WORKBOOK1_NAME = "Unit Test Workbook 1"; + private static final String WORKBOOK2_NAME = "Unit Test Workbook 2"; + private static final String FIELD_NAME = "field"; + private static final String PARENT_LIST_ITEM = "parentItem"; + private static final String WORKBOOK1_LIST_ITEM = "workbook1Item"; + private static final String WORKBOOK2_LIST_ITEM = "workbook2Item"; + private static final Integer PARENT_LI_KEY = 1; + private static final Integer WB1_LI_KEY = 2; + private static final Integer WB2_LI_KEY = 3; + + private ListDefinitionImpl list; + private Container c; + private User u; + private DomainProperty dp; + + @Before + public void setUp() throws Exception + { + JunitUtil.deleteTestContainer(); + cleanup(); + c = JunitUtil.getTestContainer(); + TestContext context = TestContext.get(); + u = context.getUser(); + list = (ListDefinitionImpl)ListService.get().createList(c, LIST_NAME, ListDefinition.KeyType.AutoIncrementInteger); + list.setKeyName("Unit test list Key"); + + dp = list.getDomain().addProperty(); + dp.setName(FIELD_NAME); + dp.setType(PropertyService.get().getType(c, PropertyType.STRING.getXmlName())); + dp.setPropertyURI(ListDomainKind.createPropertyURI(list.getName(), FIELD_NAME, c, list.getKeyType()).toString()); + list.save(u); + + addListItem(c, list, PARENT_LIST_ITEM); + } + + private void addListItem(Container scopedContainer, ListDefinition scopedList, String value) + { + List lis = new ArrayList<>(); + ListItem li = scopedList.createListItem(); + li.setProperty(dp, value); + lis.add(li); + list.insertListItems(u, scopedContainer, lis); + } + + @After + public void tearDown() throws Exception + { + cleanup(); + } + + private void cleanup() throws Exception + { + //TestContext context = TestContext.get(); + ExperimentService.get().deleteAllExpObjInContainer(c, u); + } + + @Test + public void testListServiceInOwnFolder() + { + Map lists = ListService.get().getLists(c); + assertTrue("Test List not found in own container", lists.containsKey(LIST_NAME)); + ListItem li = lists.get(LIST_NAME).getListItem(1, u, c); + assertEquals("Item not found in own container", PARENT_LIST_ITEM, li.getProperty(dp)); + } + + @Test + public void testListServiceInWorkbook() + { + Container workbook1 = setupWorkbook(WORKBOOK1_NAME); + Container workbook2 = setupWorkbook(WORKBOOK2_NAME); + Map lists = ListService.get().getLists(workbook1); + assertTrue("Test List not found in workbook", lists.containsKey(LIST_NAME)); + + checkListItemScoping(workbook1, workbook2); + } + + private Container setupWorkbook(String title) + { + return ContainerManager.createContainer(c, null, title, null, WorkbookContainerType.NAME, u); + } + + private void checkListItemScoping(Container wb1, Container wb2) + { + ListDefinition wbList1 = ListService.get().getLists(wb1).get(LIST_NAME); + ListDefinition wbList2 = ListService.get().getLists(wb2).get(LIST_NAME); + + assertEquals("Lists available to each workbook are not the same", wbList1.toString(), wbList2.toString()); + addListItem(wb1, wbList1, WORKBOOK1_LIST_ITEM); + addListItem(wb2, wbList2, WORKBOOK2_LIST_ITEM); + + assertNull("Parent item visible in workbook", wbList1.getListItem(PARENT_LI_KEY, u, wb1)); + assertNull("Sibling workbook item visible in another workbook", wbList1.getListItem(WB2_LI_KEY, u, wb1)); + assertEquals("Parent container can not see child workbook item", WORKBOOK1_LIST_ITEM, wbList1.getListItem(WB1_LI_KEY, u, c).getProperty(dp)); + assertEquals("Workbook can not see its own list item", WORKBOOK1_LIST_ITEM, wbList1.getListItem(WB1_LI_KEY, u, wb1).getProperty(dp)); + } + } +} diff --git a/pipeline/src/org/labkey/pipeline/api/PipelineManager.java b/pipeline/src/org/labkey/pipeline/api/PipelineManager.java index 357735cb5dc..5cbfb521925 100644 --- a/pipeline/src/org/labkey/pipeline/api/PipelineManager.java +++ b/pipeline/src/org/labkey/pipeline/api/PipelineManager.java @@ -1,919 +1,917 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.pipeline.api; - -import jakarta.mail.Address; -import jakarta.mail.Message; -import jakarta.mail.internet.MimeMessage; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONException; -import org.json.JSONObject; -import org.labkey.api.admin.InvalidFileException; -import org.labkey.api.cache.BlockingCache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.Filter; -import org.labkey.api.data.ObjectFactory; -import org.labkey.api.data.PropertyManager; -import org.labkey.api.data.PropertyManager.WritablePropertyMap; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.files.FileContentService; -import org.labkey.api.pipeline.DirectoryNotDeletedException; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineJobService; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineUrls; -import org.labkey.api.pipeline.trigger.PipelineTriggerConfig; -import org.labkey.api.pipeline.trigger.PipelineTriggerRegistry; -import org.labkey.api.pipeline.trigger.PipelineTriggerType; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.ValidEmail; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.trigger.TriggerConfiguration; -import org.labkey.api.util.ConfigurationException; -import org.labkey.api.util.ContainerUtil; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.MailHelper; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.util.XmlBeansUtil; -import org.labkey.api.util.emailTemplate.EmailTemplate; -import org.labkey.api.util.emailTemplate.EmailTemplateService; -import org.labkey.api.util.emailTemplate.UserOriginatedEmailTemplate; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.webdav.WebdavService; -import org.labkey.api.writer.ZipUtil; -import org.labkey.folder.xml.FolderDocument; -import org.labkey.pipeline.query.TriggerConfigurationsTable; -import org.labkey.pipeline.status.StatusController; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.nio.file.FileSystemAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - -import static org.labkey.api.action.SpringActionController.ERROR_MSG; - - -/** - * Manages pipeline root configurations and notification emails for job success and failures. - */ -public class PipelineManager -{ - private static final Logger _log = LogManager.getLogger(PipelineManager.class); - private static final PipelineSchema pipeline = PipelineSchema.getInstance(); - private static final BlockingCache CACHE = CacheManager.getBlockingStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "Pipeline roots", - (key, argument) -> new TableSelector(pipeline.getTableInfoPipelineRoots(), (Filter)argument, null).getObject(PipelineRoot.class)); - - protected static PipelineRoot getPipelineRootObject(Container container, String type) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("Type"), type); - - return CACHE.get(getCacheKey(container, type), filter); - } - - private static String getCacheKey(Container c, @Nullable String type) - { - return c.getId() + "/" + StringUtils.trimToEmpty(type); - } - - @Nullable - public static PipelineRoot findPipelineRoot(@NotNull Container container) - { - return findPipelineRoot(container, PipelineService.PRIMARY_ROOT); - } - - @Nullable - public static PipelineRoot findPipelineRoot(@NotNull Container container, String type) - { - while (container != null && !container.isRoot()) - { - PipelineRoot pipelineRoot = getPipelineRootObject(container, type); - if (null != pipelineRoot) - return pipelineRoot; - container = container.getParent(); - } - return null; - } - - - static public PipelineRoot[] getPipelineRoots(String type) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Type"), type); - - return new TableSelector(pipeline.getTableInfoPipelineRoots(), filter, null).getArray(PipelineRoot.class); - } - - static public void setPipelineRoot(User user, Container container, URI[] roots, String type, - boolean searchable) - { - PipelineRoot oldValue = getPipelineRootObject(container, type); - PipelineRoot newValue = null; - - try - { - if (roots == null || roots.length == 0 || (roots.length == 1 && roots[0] == null)) - { - if (oldValue != null) - { - Table.delete(PipelineSchema.getInstance().getTableInfoPipelineRoots(), oldValue.getPipelineRootId()); - } - } - else - { - if (oldValue == null) - { - newValue = new PipelineRoot(); - } - else - { - newValue = new PipelineRoot(oldValue); - } - newValue.setPath(roots[0].toString()); - newValue.setSupplementalPath(roots.length > 1 ? roots[1].toString() : null); - newValue.setContainerId(container.getId()); - newValue.setType(type); - newValue.setSearchable(searchable); - if (oldValue == null) - { - Table.insert(user, pipeline.getTableInfoPipelineRoots(), newValue); - } - else - { - Table.update(user, pipeline.getTableInfoPipelineRoots(), newValue, newValue.getPipelineRootId()); - } - - org.labkey.api.util.Path davPath = WebdavService.getPath().append(container.getParsedPath()).append(FileContentService.PIPELINE_LINK); - SearchService ss = SearchService.get(); - if (null != ss) - ss.addPathToCrawl(davPath, null); - } - } - finally - { - CACHE.remove(getCacheKey(container, type)); - } - - ContainerManager.firePropertyChangeEvent(new ContainerManager.ContainerPropertyChangeEvent( - container, user, ContainerManager.Property.PipelineRoot, oldValue, newValue)); - } - - static public void purge(Container container, User user) - { - SQLFragment sql = new SQLFragment(); - sql.append("UPDATE ").append(ExperimentService.get().getTinfoExperimentRun()). - append(" SET JobId = NULL WHERE JobId IN (SELECT RowId FROM "). - append(pipeline.getTableInfoStatusFiles(), "p"). - append(" WHERE container = ?)"); - sql.add(container.getId()); - - try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) - { - new SqlExecutor(PipelineSchema.getInstance().getSchema()).execute(sql); - ExperimentService.get().clearCaches(); - - ContainerUtil.purgeTable(pipeline.getTableInfoStatusFiles(), container, "Container"); - - transaction.commit(); - } - - try - { - ContainerUtil.purgeTable(pipeline.getTableInfoPipelineRoots(), container, "Container"); - } - finally - { - CACHE.remove(getCacheKey(container, null)); - } - - // Delete trigger configurations through the UserSchema so that we stop any associated listeners. See issue 33986 - try - { - PipelineQuerySchema schema = new PipelineQuerySchema(user, container); - TriggerConfigurationsTable table = schema.createTriggerConfigurationsTable(null); // bypass security check since this is internal, see issue 36249 - table.getUpdateService().truncateRows(user, container); - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - catch (QueryUpdateServiceException e) - { - throw UnexpectedException.wrap(e); - } - } - - static void setPipelineProperty(Container container, String name, String value) - { - WritablePropertyMap props = PropertyManager.getWritableProperties(container, "pipelineRoots", true); - if (value == null) - props.remove(name); - else - props.put(name, value); - props.save(); - } - - static String getPipelineProperty(Container container, String name) - { - Map props = PropertyManager.getProperties(container, "pipelineRoots"); - return props.get(name); - } - - public static void sendNotificationEmail(PipelineStatusFileImpl statusFile, Container c, User user) - { - PipelineMessage message; - if (PipelineJob.TaskStatus.complete.matches(statusFile.getStatus())) - { - String interval = PipelineEmailPreferences.get().getSuccessNotificationInterval(c); - if (!"0".equals(interval) && interval != null) return; - - message = createPipelineMessage(c, statusFile, - EmailTemplateService.get().getEmailTemplate(PipelineJobSuccess.class), - PipelineEmailPreferences.get().getNotifyOwnerOnSuccess(c), - PipelineEmailPreferences.get().getNotifyUsersOnSuccess(c)); - } - else - { - String interval = PipelineEmailPreferences.get().getFailureNotificationInterval(c); - if (!"0".equals(interval) && interval != null) - { - _log.info("Deciding not to send error notification email based on interval " + interval); - return; - } - - _log.info("Creating error notification email"); - message = createPipelineMessage(c, statusFile, - EmailTemplateService.get().getEmailTemplate(PipelineJobFailed.class), - PipelineEmailPreferences.get().getNotifyOwnerOnError(c), - PipelineEmailPreferences.get().getNotifyUsersOnError(c)); - if (message == null) - { - _log.info("Did not create a message for error notification email"); - } - } - - try - { - if (message != null) - { - Message m = message.createMessage(user); - MailHelper.send(m, null, c); - } - } - catch (ConfigurationException me) - { - _log.error("Failed sending an email notification message for a pipeline job", me); - } - } - - public static void sendNotificationEmail(PipelineStatusFileImpl[] statusFiles, Container c, Date min, Date max, boolean isSuccess) - { - PipelineDigestTemplate template = isSuccess ? - EmailTemplateService.get().getEmailTemplate(PipelineDigestJobSuccess.class) : - EmailTemplateService.get().getEmailTemplate(PipelineDigestJobFailed.class); - - PipelineDigestMessage[] messages = createPipelineDigestMessage(c, statusFiles, template, - PipelineEmailPreferences.get().getNotifyOwnerOnSuccess(c), - PipelineEmailPreferences.get().getNotifyUsersOnSuccess(c), - min, max); - - if (messages != null) - { - for (PipelineDigestMessage msg : messages) - { - try - { - Message m = msg.createMessage(); - MailHelper.send(m, null, c); - } - catch (ConfigurationException me) - { - // Stop trying if email is misconfigured - _log.error("Failed sending an email notification message for a pipeline job", me); - return; - } - catch (Exception e) - { - // Keep trying to send to other recipients - ExceptionUtil.logExceptionToMothership(null, e); - } - } - } - } - - private static PipelineMessage createPipelineMessage(Container c, PipelineStatusFileImpl statusFile, - PipelineEmailTemplate template, - boolean notifyOwner, String notifyUsers) - { - if (notifyOwner || !StringUtils.isEmpty(notifyUsers)) - { - StringBuilder sb = new StringBuilder(); - - if (notifyOwner && !StringUtils.isEmpty(statusFile.getEmail())) - { - try - { - ValidEmail ve = new ValidEmail(statusFile.getEmail()); - sb.append(ve.getEmailAddress()); - sb.append(';'); - } - catch (ValidEmail.InvalidEmailException e) - { - _log.warn("Pipeline job status file uses an invalid email: " + statusFile.getEmail() + ". RowId: " + statusFile.getRowId()); - } - } - - if (!StringUtils.isEmpty(notifyUsers)) - sb.append(notifyUsers); - - if (!sb.isEmpty()) - { - PipelineMessage message = new PipelineMessage(c, template, statusFile); - message.setRecipients(sb.toString()); - return message; - } - } - return null; - } - - private static PipelineDigestMessage[] createPipelineDigestMessage(Container c, PipelineStatusFileImpl[] statusFiles, - PipelineDigestTemplate template, - boolean notifyOwner, String notifyUsers, - Date min, Date max) - { - if (notifyOwner || !StringUtils.isEmpty(notifyUsers)) - { - Map recipients = new HashMap<>(); - for (PipelineStatusFileImpl sf : statusFiles) - { - if (notifyOwner && !StringUtils.isEmpty(sf.getEmail())) - { - if (!recipients.containsKey(sf.getEmail())) - { - StringBuilder sb = new StringBuilder(); - sb.append(sf.getEmail()); - sb.append(';'); - - if (!StringUtils.isEmpty(notifyUsers)) - sb.append(notifyUsers); - - recipients.put(sf.getEmail(), sb); - } - } - } - - if (recipients.isEmpty() && !StringUtils.isEmpty(notifyUsers)) - { - StringBuilder sb = new StringBuilder(); - sb.append(notifyUsers); - - recipients.put("notifyUsers", sb); - } - - List messages = new ArrayList<>(); - for (StringBuilder sb : recipients.values()) - { - PipelineDigestMessage message = new PipelineDigestMessage(c, template, statusFiles, min, max, sb.toString()); - messages.add(message); - } - return messages.toArray(new PipelineDigestMessage[0]); - } - return null; - } - - private static class PipelineMessage - { - private final Container _c; - private final PipelineEmailTemplate _template; - private final PipelineStatusFileImpl _statusFile; - private String _recipients; - - public PipelineMessage(Container c, PipelineEmailTemplate template, PipelineStatusFileImpl statusFile) - { - _c = c; - _template = template; - _statusFile = statusFile; - } - //public void setTemplate(PipelineEmailTemplate template){_template = template;} - //public void setStatusFiles(PipelineStatusFileImpl[] statusFiles){_statusFiles = statusFiles;} - public void setRecipients(String recipients){_recipients = recipients;} - - public MimeMessage createMessage(User user) - { - try - { - MailHelper.MultipartMessage m = MailHelper.createMultipartMessage(); - - ActionURL url = StatusController.urlDetails(_statusFile); - - _template.setOriginatingUser(user); - _template.setDataUrl(url.getURIString()); - _template.setJobDescription(_statusFile.getDescription()); - _template.setStatus(_statusFile.getStatus()); - _template.setTimeCreated(_statusFile.getCreated()); - - m.setTemplate(_template, _c); - m.addFrom(new Address[]{_template.renderFrom(_c, LookAndFeelProperties.getInstance(_c).getSystemEmailAddress())}); - - m.addRecipients(Message.RecipientType.TO, MailHelper.createAddressArray(_recipients)); - - return m; - } - catch (Exception e) - { - _log.error("Failed creating an email notification message for a pipeline job", e); - } - return null; - } - } - - private static class PipelineDigestMessage - { - private final Container _c; - private final PipelineDigestTemplate _template; - private final PipelineStatusFileImpl[] _statusFiles; - private final String _recipients; - private final Date _min; - private final Date _max; - - public PipelineDigestMessage(Container c, PipelineDigestTemplate template, PipelineStatusFileImpl[] statusFiles, - Date min, Date max, String recipients) - { - _c = c; - _template = template; - _statusFiles = statusFiles; - _min = min; - _max = max; - _recipients = recipients; - } - - public MimeMessage createMessage() - { - try - { - MailHelper.MultipartMessage m = MailHelper.createMultipartMessage(); - - _template.setStatusFiles(_statusFiles); - _template.setStartTime(_min); - _template.setEndTime(_max); - - _template.renderAllToMessage(m, _c); - - m.addFrom(new Address[]{_template.renderFrom(_c, LookAndFeelProperties.getInstance(_c).getSystemEmailAddress())}); - m.addRecipients(Message.RecipientType.TO, MailHelper.createAddressArray(_recipients)); - - return m; - } - catch (Exception e) - { - _log.error("Failed creating an email notification message for a pipeline job", e); - } - return null; - } - } - - public static abstract class PipelineEmailTemplate extends UserOriginatedEmailTemplate - { - protected String _dataUrl; - protected String _jobDescription; - protected Date _timeCreated; - protected String _status; - - protected static final String DEFAULT_BODY = "Job description: ^jobDescription^\n" + - "Created: ^timeCreated^\n" + - "Status: ^status^\n" + - "Additional details for this job can be obtained by navigating to this link:\n\n^dataURL^\n\n" + - "Manage your email notifications at\n" + - "^setupURL^\n"; - - protected PipelineEmailTemplate(@NotNull String name, String description, String subject, String body) - { - super(name, description, subject, body, ContentType.Plain, Scope.Site); - } - - public void setDataUrl(String dataUrl){_dataUrl = dataUrl;} - public void setJobDescription(String description){_jobDescription = description;} - public void setTimeCreated(Date timeCreated){_timeCreated = timeCreated;} - public void setStatus(String status){_status = status;} - - @Override - protected void addCustomReplacements(Replacements replacements) - { - super.addCustomReplacements(replacements); - replacements.add("dataURL", String.class, "Link to the job details for this pipeline job", ContentType.Plain, c -> _dataUrl); - replacements.add("jobDescription", String.class, "The job description", ContentType.Plain, c -> _jobDescription); - replacements.add("timeCreated", Date.class, "The date and time this job was created", ContentType.Plain, c -> _timeCreated); - replacements.add("status", String.class, "The job status", ContentType.Plain, c -> _status); - replacements.add("setupURL", String.class, "URL to configure the pipeline, including email notifications", ContentType.Plain, c -> PageFlowUtil.urlProvider(PipelineUrls.class).urlSetup(c).getURIString()); - } - } - - public static class PipelineJobSuccess extends PipelineEmailTemplate - { - public PipelineJobSuccess() - { - super("Pipeline job succeeded", "Sent to users who have been configured to receive notifications when a pipeline job completes successfully", "The pipeline job: ^jobDescription^ has completed successfully", DEFAULT_BODY); - } - } - - public static class PipelineJobFailed extends PipelineEmailTemplate - { - public PipelineJobFailed() - { - super("Pipeline job failed", "Sent to users who have been configured to receive notifications when a pipeline job fails", "The pipeline job: ^jobDescription^ did not complete successfully", DEFAULT_BODY); - } - } - - public static abstract class PipelineDigestTemplate extends EmailTemplate - { - private PipelineStatusFileImpl[] _statusFiles; - private Date _startTime; - private Date _endTime; - - protected static final String DEFAULT_BODY = "The following jobs have completed between the time of: ^startTime^ " + - "and the end time of: ^endTime^:\n\n^pipelineJobs^"; - - protected PipelineDigestTemplate(String name, String description, String subject, String body) - { - super(name, description, subject, body, ContentType.HTML, Scope.Site); - } - - @Override - protected void addCustomReplacements(Replacements replacements) - { - replacements.add("pipelineJobs", String.class, "The list of all pipeline jobs that have completed for this notification period", ContentType.HTML, c -> getJobStatus()); - replacements.add("startTime", Date.class, "The start of the time period for job completion", ContentType.HTML, c -> _startTime); - replacements.add("endTime", Date.class, "The end of the time period for job completion", ContentType.HTML, c -> _endTime); - } - - public void setStatusFiles(PipelineStatusFileImpl[] statusFiles){_statusFiles = statusFiles;} - public void setStartTime(Date startTime){_startTime = startTime;} - public void setEndTime(Date endTime){_endTime = endTime;} - - private String getJobStatus() - { - if (_statusFiles != null) - { - StringBuilder sb = new StringBuilder(); - sb.append(""); - sb.append(""); - for (PipelineStatusFileImpl sf : _statusFiles) - { - ActionURL url = StatusController.urlDetails(sf); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - } - sb.append("
    DescriptionCreatedStatusDetails
    ").append(PageFlowUtil.filter(sf.getDescription())).append("").append(PageFlowUtil.filter(sf.getCreated())).append("").append(PageFlowUtil.filter(sf.getStatus())).append("").append(url.getURIString()).append("
    "); - return sb.toString(); - } - return null; - } - } - - public static class PipelineDigestJobSuccess extends PipelineDigestTemplate - { - public PipelineDigestJobSuccess() - { - super("Pipeline jobs succeeded (digest)", - "Sent for pipeline jobs that have completed successfully during a configured time period", "The pipeline jobs have completed successfully", - DEFAULT_BODY - ); - } - } - - public static class PipelineDigestJobFailed extends PipelineDigestTemplate - { - public PipelineDigestJobFailed() - { - super("Pipeline jobs failed (digest)", - "Sent for pipeline jobs that have not completed successfully during a configured time period", "The pipeline jobs did not complete successfully", - DEFAULT_BODY - ); - } - } - - public static TriggerConfiguration getTriggerConfiguration(Container container, String name) - { - TableInfo tinfo = PipelineSchema.getInstance().getTableInfoTriggerConfigurations(); - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("Name"), name); - - return new TableSelector(tinfo, filter, null).getObject(TriggerConfiguration.class); - } - - public static boolean insertOrUpdateTriggerConfiguration(User user, Container container, TriggerConfiguration config) throws Exception - { - UserSchema schema = QueryService.get().getUserSchema(user, container, PipelineSchema.getInstance().getSchemaName()); - if (schema != null) - { - TableInfo tableInfo = schema.getTable(PipelineQuerySchema.TRIGGER_CONFIGURATIONS_TABLE_NAME); - if (tableInfo != null) - { - if (config.getRowId() != null) - config.beforeUpdate(user); - else - config.beforeInsert(user, container.getId()); - ObjectFactory factory = ObjectFactory.Registry.getFactory(TriggerConfiguration.class); - Map row = factory.toMap(config, null); - - QueryUpdateService qus = tableInfo.getUpdateService(); - List> rowList = new LinkedList<>(); - rowList.add(row); - if (qus != null) - { - BatchValidationException errors = new BatchValidationException(); - if (row.get("RowId") != null) - rowList = qus.updateRows(user, container, rowList, null, errors, null, null); - else - rowList = qus.insertRows(user, container, rowList, errors, null, null); - if (errors.hasErrors()) - throw errors; - } - return !rowList.isEmpty(); - } - } - return false; - } - - public static void validateTriggerConfiguration(TriggerConfiguration config, Container container, User user, Errors errors) - { - Integer rowId = config.getRowId(); - String name = config.getName(); - String type = config.getType(); - String pipelineId = config.getPipelineId(); - boolean isEnabled = config.isEnabled(); - - // validate that the config name is unique for this container - if (StringUtils.isNotEmpty(name)) - { - if (name.length() > 255) - errors.rejectValue("Name", null, "Name must be less than 256 characters"); - - Collection existingConfigs = PipelineTriggerRegistry.get().getConfigs(container, null, name, false); - if (!existingConfigs.isEmpty()) - { - for (PipelineTriggerConfig existingConfig : existingConfigs) - { - if (rowId == null || !rowId.equals(existingConfig.getRowId())) - { - errors.rejectValue("Name", null, "A pipeline trigger configuration already exists in this container for the given name: " + name); - break; - } - } - } - } - else - { - errors.rejectValue("Name", null, "A name is required for trigger configurations."); - } - - // validate that the type is a valid registered PipelineTriggerType - PipelineTriggerType triggerType = PipelineTriggerRegistry.get().getTypeByName(type); - if (triggerType == null) - { - errors.rejectValue("Type", null, "Invalid pipeline trigger type:" + type); - return; - } - - // validate that the pipelineId is a valid TaskPipeline - if (pipelineId != null) - { - try - { - PipelineJobService.get().getTaskPipeline(pipelineId); - } - catch (NotFoundException e) - { - errors.rejectValue("PipelineId", null, "Invalid pipeline task id: " + pipelineId); - } - } - else - { - errors.reject(null, null, "Pipeline Task ID required."); - return; - } - - // validate that the configuration values parse as valid JSON - validateConfigJson(triggerType, config.getConfiguration(), pipelineId, isEnabled, errors, container, user); - - Object customConfiguration = config.getCustomConfiguration(); - if (customConfiguration != null && !customConfiguration.toString().isEmpty()) - validateConfigJson(triggerType, customConfiguration, pipelineId, isEnabled, errors, true, container, user); - } - - private static void validateConfigJson(PipelineTriggerType triggerType, Object configuration, String pipelineId, boolean isEnabled, Errors errors, Container sourceContainer, User user) - { - validateConfigJson(triggerType, configuration, pipelineId, isEnabled, errors, false, sourceContainer, user); - } - - private static void validateConfigJson(PipelineTriggerType triggerType, Object configuration, String pipelineId, boolean isEnabled, Errors errors, boolean jsonValidityOnly, Container sourceContainer, User user) - { - JSONObject json = null; - if (configuration != null) - { - try - { - json = new JSONObject(configuration.toString()); - } - catch (JSONException e) - { - errors.reject(ERROR_MSG, "Invalid JSON object for the configuration field: " + e); - } - } - - // give the PipelineTriggerType a chance to validate the configuration JSON object - if (triggerType != null && !jsonValidityOnly) - { - List> configErrors = triggerType.validateConfiguration(pipelineId, isEnabled, json, sourceContainer, user); - for (Pair msg : configErrors) - errors.rejectValue(msg.first, null, msg.second); - } - } - - public static Path validateFolderImportFileNioPath(String archiveFilePath, PipeRoot pipeRoot, Errors errors) - { - Path archiveFile = Path.of(archiveFilePath); - - if (!archiveFile.isAbsolute()) - { - // Resolve the relative path to an absolute path under the current container's root - archiveFile = archiveFilePath.contains("://") ? pipeRoot.resolveToNioPathFromUrl(archiveFilePath) : pipeRoot.resolveToNioPath(archiveFilePath); - } - - // Be sure that the referenced file exists and is under the pipeline root - if (archiveFile == null || !Files.exists(archiveFile)) - { - errors.reject(ERROR_MSG, "Could not find file at path: " + archiveFilePath); - } - else if (!pipeRoot.isCloudRoot() && !pipeRoot.isUnderRoot(archiveFile)) // TODO: check for isCloud, then file should be in temp - { - errors.reject(ERROR_MSG, "Cannot access file " + archiveFilePath); - } - - return archiveFile; - } - - @Nullable - private static Path expandZipLocally(PipeRoot pipelineRoot, Path archiveFile, BindException errors) - { - try - { - // check if the archive file already exists in the unzip dir of this pipeline root - Path importDir = pipelineRoot.getImportDirectory().toPath(); - if (!archiveFile.getParent().toAbsolutePath().toString().equalsIgnoreCase(importDir.toAbsolutePath().toString())) - importDir = pipelineRoot.deleteImportDirectory(null); - - boolean shouldUnzip = Files.notExists(importDir); - - if (!shouldUnzip) - { - try (Stream pathStream = Files.list(importDir)) - { - shouldUnzip = pathStream.noneMatch(s -> s.getFileName().toString().equalsIgnoreCase(archiveFile.getFileName().toString())); - } - } - - if (shouldUnzip) - { - // Only unzip once - try (InputStream is = Files.newInputStream(archiveFile)) - { - ZipUtil.unzipToDirectory(is, importDir); - } - } - - return importDir; - } - catch (FileNotFoundException e) - { - errors.reject(ERROR_MSG, "File not found."); - } - catch (FileSystemAlreadyExistsException | DirectoryNotDeletedException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - catch (IOException e) - { - errors.reject(ERROR_MSG, "This file does not appear to be a valid .zip file."); - } - - // Return null if errors were observed - return null; - } - - private static Path getImportXmlFile(@NotNull PipeRoot pipelineRoot, @NotNull Path archiveFile, @NotNull String xmlFileName, BindException errors) throws InvalidFileException - { - Path xmlFile = archiveFile; - - if (archiveFile.getFileName().toString().toLowerCase().endsWith(".zip")) - { - Path importDir = expandZipLocally(pipelineRoot, archiveFile, errors); - if (importDir != null) - { - xmlFile = getXmlFilePathFromArchive(importDir, archiveFile, xmlFileName); - } - } - //Downloading expanded archive will be handled later in the archive processing... - //We don't really have the job context here - - return xmlFile; - } - - public static @NotNull Path getXmlFilePathFromArchive(@NotNull Path importDir, Path archiveFile, @NotNull String xmlFileName) throws InvalidFileException - { - // when importing a folder archive for a study, the study.xml file may not be at the root - if ("study.xml".equalsIgnoreCase(xmlFileName) && archiveFile.getFileName().toString().toLowerCase().endsWith(".folder.zip")) - { - File folderXml = new File(importDir.toFile(), "folder.xml"); - FolderDocument folderDoc; - try - { - folderDoc = FolderDocument.Factory.parse(folderXml, XmlBeansUtil.getDefaultParseOptions()); - XmlBeansUtil.validateXmlDocument(folderDoc, xmlFileName); - } - catch (Exception e) - { - throw new InvalidFileException(folderXml.getParentFile().toPath(), folderXml.toPath(), e); - } - - if (folderDoc.getFolder().isSetStudy()) - { - importDir = importDir.resolve(folderDoc.getFolder().getStudy().getDir()); - } - } - - return importDir.toAbsolutePath().resolve(xmlFileName); - } - - public static Path getArchiveXmlFile(Container container, Path archiveFile, String xmlFileName, BindException errors) throws InvalidFileException - { - PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(container); - Path xmlFile = getImportXmlFile(pipelineRoot, archiveFile, xmlFileName, errors); - - // if this is an import from a source template folder that has been previously implicitly exported - // to the unzip dir (without ever creating a zip file) then just look there for the xmlFile. - if (pipelineRoot != null && Files.isDirectory(archiveFile)) - { - xmlFile = java.nio.file.Path.of(archiveFile.toString(), xmlFileName); - } - - return xmlFile; - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.pipeline.api; + +import jakarta.mail.Address; +import jakarta.mail.Message; +import jakarta.mail.internet.MimeMessage; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONException; +import org.json.JSONObject; +import org.labkey.api.admin.InvalidFileException; +import org.labkey.api.cache.BlockingCache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.Filter; +import org.labkey.api.data.ObjectFactory; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.PropertyManager.WritablePropertyMap; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.files.FileContentService; +import org.labkey.api.pipeline.DirectoryNotDeletedException; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobService; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineUrls; +import org.labkey.api.pipeline.trigger.PipelineTriggerConfig; +import org.labkey.api.pipeline.trigger.PipelineTriggerRegistry; +import org.labkey.api.pipeline.trigger.PipelineTriggerType; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.ValidEmail; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.trigger.TriggerConfiguration; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.ContainerUtil; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.MailHelper; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.util.XmlBeansUtil; +import org.labkey.api.util.emailTemplate.EmailTemplate; +import org.labkey.api.util.emailTemplate.EmailTemplateService; +import org.labkey.api.util.emailTemplate.UserOriginatedEmailTemplate; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.webdav.WebdavService; +import org.labkey.api.writer.ZipUtil; +import org.labkey.folder.xml.FolderDocument; +import org.labkey.pipeline.query.TriggerConfigurationsTable; +import org.labkey.pipeline.status.StatusController; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.FileSystemAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static org.labkey.api.action.SpringActionController.ERROR_MSG; + + +/** + * Manages pipeline root configurations and notification emails for job success and failures. + */ +public class PipelineManager +{ + private static final Logger _log = LogManager.getLogger(PipelineManager.class); + private static final PipelineSchema pipeline = PipelineSchema.getInstance(); + private static final BlockingCache CACHE = CacheManager.getBlockingStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "Pipeline roots", + (key, argument) -> new TableSelector(pipeline.getTableInfoPipelineRoots(), (Filter)argument, null).getObject(PipelineRoot.class)); + + protected static PipelineRoot getPipelineRootObject(Container container, String type) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("Type"), type); + + return CACHE.get(getCacheKey(container, type), filter); + } + + private static String getCacheKey(Container c, @Nullable String type) + { + return c.getId() + "/" + StringUtils.trimToEmpty(type); + } + + @Nullable + public static PipelineRoot findPipelineRoot(@NotNull Container container) + { + return findPipelineRoot(container, PipelineService.PRIMARY_ROOT); + } + + @Nullable + public static PipelineRoot findPipelineRoot(@NotNull Container container, String type) + { + while (container != null && !container.isRoot()) + { + PipelineRoot pipelineRoot = getPipelineRootObject(container, type); + if (null != pipelineRoot) + return pipelineRoot; + container = container.getParent(); + } + return null; + } + + + static public PipelineRoot[] getPipelineRoots(String type) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Type"), type); + + return new TableSelector(pipeline.getTableInfoPipelineRoots(), filter, null).getArray(PipelineRoot.class); + } + + static public void setPipelineRoot(User user, Container container, URI[] roots, String type, + boolean searchable) + { + PipelineRoot oldValue = getPipelineRootObject(container, type); + PipelineRoot newValue = null; + + try + { + if (roots == null || roots.length == 0 || (roots.length == 1 && roots[0] == null)) + { + if (oldValue != null) + { + Table.delete(PipelineSchema.getInstance().getTableInfoPipelineRoots(), oldValue.getPipelineRootId()); + } + } + else + { + if (oldValue == null) + { + newValue = new PipelineRoot(); + } + else + { + newValue = new PipelineRoot(oldValue); + } + newValue.setPath(roots[0].toString()); + newValue.setSupplementalPath(roots.length > 1 ? roots[1].toString() : null); + newValue.setContainerId(container.getId()); + newValue.setType(type); + newValue.setSearchable(searchable); + if (oldValue == null) + { + Table.insert(user, pipeline.getTableInfoPipelineRoots(), newValue); + } + else + { + Table.update(user, pipeline.getTableInfoPipelineRoots(), newValue, newValue.getPipelineRootId()); + } + + org.labkey.api.util.Path davPath = WebdavService.getPath().append(container.getParsedPath()).append(FileContentService.PIPELINE_LINK); + SearchService.get().addPathToCrawl(davPath, null); + } + } + finally + { + CACHE.remove(getCacheKey(container, type)); + } + + ContainerManager.firePropertyChangeEvent(new ContainerManager.ContainerPropertyChangeEvent( + container, user, ContainerManager.Property.PipelineRoot, oldValue, newValue)); + } + + static public void purge(Container container, User user) + { + SQLFragment sql = new SQLFragment(); + sql.append("UPDATE ").append(ExperimentService.get().getTinfoExperimentRun()). + append(" SET JobId = NULL WHERE JobId IN (SELECT RowId FROM "). + append(pipeline.getTableInfoStatusFiles(), "p"). + append(" WHERE container = ?)"); + sql.add(container.getId()); + + try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) + { + new SqlExecutor(PipelineSchema.getInstance().getSchema()).execute(sql); + ExperimentService.get().clearCaches(); + + ContainerUtil.purgeTable(pipeline.getTableInfoStatusFiles(), container, "Container"); + + transaction.commit(); + } + + try + { + ContainerUtil.purgeTable(pipeline.getTableInfoPipelineRoots(), container, "Container"); + } + finally + { + CACHE.remove(getCacheKey(container, null)); + } + + // Delete trigger configurations through the UserSchema so that we stop any associated listeners. See issue 33986 + try + { + PipelineQuerySchema schema = new PipelineQuerySchema(user, container); + TriggerConfigurationsTable table = schema.createTriggerConfigurationsTable(null); // bypass security check since this is internal, see issue 36249 + table.getUpdateService().truncateRows(user, container); + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + catch (QueryUpdateServiceException e) + { + throw UnexpectedException.wrap(e); + } + } + + static void setPipelineProperty(Container container, String name, String value) + { + WritablePropertyMap props = PropertyManager.getWritableProperties(container, "pipelineRoots", true); + if (value == null) + props.remove(name); + else + props.put(name, value); + props.save(); + } + + static String getPipelineProperty(Container container, String name) + { + Map props = PropertyManager.getProperties(container, "pipelineRoots"); + return props.get(name); + } + + public static void sendNotificationEmail(PipelineStatusFileImpl statusFile, Container c, User user) + { + PipelineMessage message; + if (PipelineJob.TaskStatus.complete.matches(statusFile.getStatus())) + { + String interval = PipelineEmailPreferences.get().getSuccessNotificationInterval(c); + if (!"0".equals(interval) && interval != null) return; + + message = createPipelineMessage(c, statusFile, + EmailTemplateService.get().getEmailTemplate(PipelineJobSuccess.class), + PipelineEmailPreferences.get().getNotifyOwnerOnSuccess(c), + PipelineEmailPreferences.get().getNotifyUsersOnSuccess(c)); + } + else + { + String interval = PipelineEmailPreferences.get().getFailureNotificationInterval(c); + if (!"0".equals(interval) && interval != null) + { + _log.info("Deciding not to send error notification email based on interval " + interval); + return; + } + + _log.info("Creating error notification email"); + message = createPipelineMessage(c, statusFile, + EmailTemplateService.get().getEmailTemplate(PipelineJobFailed.class), + PipelineEmailPreferences.get().getNotifyOwnerOnError(c), + PipelineEmailPreferences.get().getNotifyUsersOnError(c)); + if (message == null) + { + _log.info("Did not create a message for error notification email"); + } + } + + try + { + if (message != null) + { + Message m = message.createMessage(user); + MailHelper.send(m, null, c); + } + } + catch (ConfigurationException me) + { + _log.error("Failed sending an email notification message for a pipeline job", me); + } + } + + public static void sendNotificationEmail(PipelineStatusFileImpl[] statusFiles, Container c, Date min, Date max, boolean isSuccess) + { + PipelineDigestTemplate template = isSuccess ? + EmailTemplateService.get().getEmailTemplate(PipelineDigestJobSuccess.class) : + EmailTemplateService.get().getEmailTemplate(PipelineDigestJobFailed.class); + + PipelineDigestMessage[] messages = createPipelineDigestMessage(c, statusFiles, template, + PipelineEmailPreferences.get().getNotifyOwnerOnSuccess(c), + PipelineEmailPreferences.get().getNotifyUsersOnSuccess(c), + min, max); + + if (messages != null) + { + for (PipelineDigestMessage msg : messages) + { + try + { + Message m = msg.createMessage(); + MailHelper.send(m, null, c); + } + catch (ConfigurationException me) + { + // Stop trying if email is misconfigured + _log.error("Failed sending an email notification message for a pipeline job", me); + return; + } + catch (Exception e) + { + // Keep trying to send to other recipients + ExceptionUtil.logExceptionToMothership(null, e); + } + } + } + } + + private static PipelineMessage createPipelineMessage(Container c, PipelineStatusFileImpl statusFile, + PipelineEmailTemplate template, + boolean notifyOwner, String notifyUsers) + { + if (notifyOwner || !StringUtils.isEmpty(notifyUsers)) + { + StringBuilder sb = new StringBuilder(); + + if (notifyOwner && !StringUtils.isEmpty(statusFile.getEmail())) + { + try + { + ValidEmail ve = new ValidEmail(statusFile.getEmail()); + sb.append(ve.getEmailAddress()); + sb.append(';'); + } + catch (ValidEmail.InvalidEmailException e) + { + _log.warn("Pipeline job status file uses an invalid email: " + statusFile.getEmail() + ". RowId: " + statusFile.getRowId()); + } + } + + if (!StringUtils.isEmpty(notifyUsers)) + sb.append(notifyUsers); + + if (!sb.isEmpty()) + { + PipelineMessage message = new PipelineMessage(c, template, statusFile); + message.setRecipients(sb.toString()); + return message; + } + } + return null; + } + + private static PipelineDigestMessage[] createPipelineDigestMessage(Container c, PipelineStatusFileImpl[] statusFiles, + PipelineDigestTemplate template, + boolean notifyOwner, String notifyUsers, + Date min, Date max) + { + if (notifyOwner || !StringUtils.isEmpty(notifyUsers)) + { + Map recipients = new HashMap<>(); + for (PipelineStatusFileImpl sf : statusFiles) + { + if (notifyOwner && !StringUtils.isEmpty(sf.getEmail())) + { + if (!recipients.containsKey(sf.getEmail())) + { + StringBuilder sb = new StringBuilder(); + sb.append(sf.getEmail()); + sb.append(';'); + + if (!StringUtils.isEmpty(notifyUsers)) + sb.append(notifyUsers); + + recipients.put(sf.getEmail(), sb); + } + } + } + + if (recipients.isEmpty() && !StringUtils.isEmpty(notifyUsers)) + { + StringBuilder sb = new StringBuilder(); + sb.append(notifyUsers); + + recipients.put("notifyUsers", sb); + } + + List messages = new ArrayList<>(); + for (StringBuilder sb : recipients.values()) + { + PipelineDigestMessage message = new PipelineDigestMessage(c, template, statusFiles, min, max, sb.toString()); + messages.add(message); + } + return messages.toArray(new PipelineDigestMessage[0]); + } + return null; + } + + private static class PipelineMessage + { + private final Container _c; + private final PipelineEmailTemplate _template; + private final PipelineStatusFileImpl _statusFile; + private String _recipients; + + public PipelineMessage(Container c, PipelineEmailTemplate template, PipelineStatusFileImpl statusFile) + { + _c = c; + _template = template; + _statusFile = statusFile; + } + //public void setTemplate(PipelineEmailTemplate template){_template = template;} + //public void setStatusFiles(PipelineStatusFileImpl[] statusFiles){_statusFiles = statusFiles;} + public void setRecipients(String recipients){_recipients = recipients;} + + public MimeMessage createMessage(User user) + { + try + { + MailHelper.MultipartMessage m = MailHelper.createMultipartMessage(); + + ActionURL url = StatusController.urlDetails(_statusFile); + + _template.setOriginatingUser(user); + _template.setDataUrl(url.getURIString()); + _template.setJobDescription(_statusFile.getDescription()); + _template.setStatus(_statusFile.getStatus()); + _template.setTimeCreated(_statusFile.getCreated()); + + m.setTemplate(_template, _c); + m.addFrom(new Address[]{_template.renderFrom(_c, LookAndFeelProperties.getInstance(_c).getSystemEmailAddress())}); + + m.addRecipients(Message.RecipientType.TO, MailHelper.createAddressArray(_recipients)); + + return m; + } + catch (Exception e) + { + _log.error("Failed creating an email notification message for a pipeline job", e); + } + return null; + } + } + + private static class PipelineDigestMessage + { + private final Container _c; + private final PipelineDigestTemplate _template; + private final PipelineStatusFileImpl[] _statusFiles; + private final String _recipients; + private final Date _min; + private final Date _max; + + public PipelineDigestMessage(Container c, PipelineDigestTemplate template, PipelineStatusFileImpl[] statusFiles, + Date min, Date max, String recipients) + { + _c = c; + _template = template; + _statusFiles = statusFiles; + _min = min; + _max = max; + _recipients = recipients; + } + + public MimeMessage createMessage() + { + try + { + MailHelper.MultipartMessage m = MailHelper.createMultipartMessage(); + + _template.setStatusFiles(_statusFiles); + _template.setStartTime(_min); + _template.setEndTime(_max); + + _template.renderAllToMessage(m, _c); + + m.addFrom(new Address[]{_template.renderFrom(_c, LookAndFeelProperties.getInstance(_c).getSystemEmailAddress())}); + m.addRecipients(Message.RecipientType.TO, MailHelper.createAddressArray(_recipients)); + + return m; + } + catch (Exception e) + { + _log.error("Failed creating an email notification message for a pipeline job", e); + } + return null; + } + } + + public static abstract class PipelineEmailTemplate extends UserOriginatedEmailTemplate + { + protected String _dataUrl; + protected String _jobDescription; + protected Date _timeCreated; + protected String _status; + + protected static final String DEFAULT_BODY = "Job description: ^jobDescription^\n" + + "Created: ^timeCreated^\n" + + "Status: ^status^\n" + + "Additional details for this job can be obtained by navigating to this link:\n\n^dataURL^\n\n" + + "Manage your email notifications at\n" + + "^setupURL^\n"; + + protected PipelineEmailTemplate(@NotNull String name, String description, String subject, String body) + { + super(name, description, subject, body, ContentType.Plain, Scope.Site); + } + + public void setDataUrl(String dataUrl){_dataUrl = dataUrl;} + public void setJobDescription(String description){_jobDescription = description;} + public void setTimeCreated(Date timeCreated){_timeCreated = timeCreated;} + public void setStatus(String status){_status = status;} + + @Override + protected void addCustomReplacements(Replacements replacements) + { + super.addCustomReplacements(replacements); + replacements.add("dataURL", String.class, "Link to the job details for this pipeline job", ContentType.Plain, c -> _dataUrl); + replacements.add("jobDescription", String.class, "The job description", ContentType.Plain, c -> _jobDescription); + replacements.add("timeCreated", Date.class, "The date and time this job was created", ContentType.Plain, c -> _timeCreated); + replacements.add("status", String.class, "The job status", ContentType.Plain, c -> _status); + replacements.add("setupURL", String.class, "URL to configure the pipeline, including email notifications", ContentType.Plain, c -> PageFlowUtil.urlProvider(PipelineUrls.class).urlSetup(c).getURIString()); + } + } + + public static class PipelineJobSuccess extends PipelineEmailTemplate + { + public PipelineJobSuccess() + { + super("Pipeline job succeeded", "Sent to users who have been configured to receive notifications when a pipeline job completes successfully", "The pipeline job: ^jobDescription^ has completed successfully", DEFAULT_BODY); + } + } + + public static class PipelineJobFailed extends PipelineEmailTemplate + { + public PipelineJobFailed() + { + super("Pipeline job failed", "Sent to users who have been configured to receive notifications when a pipeline job fails", "The pipeline job: ^jobDescription^ did not complete successfully", DEFAULT_BODY); + } + } + + public static abstract class PipelineDigestTemplate extends EmailTemplate + { + private PipelineStatusFileImpl[] _statusFiles; + private Date _startTime; + private Date _endTime; + + protected static final String DEFAULT_BODY = "The following jobs have completed between the time of: ^startTime^ " + + "and the end time of: ^endTime^:\n\n^pipelineJobs^"; + + protected PipelineDigestTemplate(String name, String description, String subject, String body) + { + super(name, description, subject, body, ContentType.HTML, Scope.Site); + } + + @Override + protected void addCustomReplacements(Replacements replacements) + { + replacements.add("pipelineJobs", String.class, "The list of all pipeline jobs that have completed for this notification period", ContentType.HTML, c -> getJobStatus()); + replacements.add("startTime", Date.class, "The start of the time period for job completion", ContentType.HTML, c -> _startTime); + replacements.add("endTime", Date.class, "The end of the time period for job completion", ContentType.HTML, c -> _endTime); + } + + public void setStatusFiles(PipelineStatusFileImpl[] statusFiles){_statusFiles = statusFiles;} + public void setStartTime(Date startTime){_startTime = startTime;} + public void setEndTime(Date endTime){_endTime = endTime;} + + private String getJobStatus() + { + if (_statusFiles != null) + { + StringBuilder sb = new StringBuilder(); + sb.append(""); + sb.append(""); + for (PipelineStatusFileImpl sf : _statusFiles) + { + ActionURL url = StatusController.urlDetails(sf); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + } + sb.append("
    DescriptionCreatedStatusDetails
    ").append(PageFlowUtil.filter(sf.getDescription())).append("").append(PageFlowUtil.filter(sf.getCreated())).append("").append(PageFlowUtil.filter(sf.getStatus())).append("").append(url.getURIString()).append("
    "); + return sb.toString(); + } + return null; + } + } + + public static class PipelineDigestJobSuccess extends PipelineDigestTemplate + { + public PipelineDigestJobSuccess() + { + super("Pipeline jobs succeeded (digest)", + "Sent for pipeline jobs that have completed successfully during a configured time period", "The pipeline jobs have completed successfully", + DEFAULT_BODY + ); + } + } + + public static class PipelineDigestJobFailed extends PipelineDigestTemplate + { + public PipelineDigestJobFailed() + { + super("Pipeline jobs failed (digest)", + "Sent for pipeline jobs that have not completed successfully during a configured time period", "The pipeline jobs did not complete successfully", + DEFAULT_BODY + ); + } + } + + public static TriggerConfiguration getTriggerConfiguration(Container container, String name) + { + TableInfo tinfo = PipelineSchema.getInstance().getTableInfoTriggerConfigurations(); + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("Name"), name); + + return new TableSelector(tinfo, filter, null).getObject(TriggerConfiguration.class); + } + + public static boolean insertOrUpdateTriggerConfiguration(User user, Container container, TriggerConfiguration config) throws Exception + { + UserSchema schema = QueryService.get().getUserSchema(user, container, PipelineSchema.getInstance().getSchemaName()); + if (schema != null) + { + TableInfo tableInfo = schema.getTable(PipelineQuerySchema.TRIGGER_CONFIGURATIONS_TABLE_NAME); + if (tableInfo != null) + { + if (config.getRowId() != null) + config.beforeUpdate(user); + else + config.beforeInsert(user, container.getId()); + ObjectFactory factory = ObjectFactory.Registry.getFactory(TriggerConfiguration.class); + Map row = factory.toMap(config, null); + + QueryUpdateService qus = tableInfo.getUpdateService(); + List> rowList = new LinkedList<>(); + rowList.add(row); + if (qus != null) + { + BatchValidationException errors = new BatchValidationException(); + if (row.get("RowId") != null) + rowList = qus.updateRows(user, container, rowList, null, errors, null, null); + else + rowList = qus.insertRows(user, container, rowList, errors, null, null); + if (errors.hasErrors()) + throw errors; + } + return !rowList.isEmpty(); + } + } + return false; + } + + public static void validateTriggerConfiguration(TriggerConfiguration config, Container container, User user, Errors errors) + { + Integer rowId = config.getRowId(); + String name = config.getName(); + String type = config.getType(); + String pipelineId = config.getPipelineId(); + boolean isEnabled = config.isEnabled(); + + // validate that the config name is unique for this container + if (StringUtils.isNotEmpty(name)) + { + if (name.length() > 255) + errors.rejectValue("Name", null, "Name must be less than 256 characters"); + + Collection existingConfigs = PipelineTriggerRegistry.get().getConfigs(container, null, name, false); + if (!existingConfigs.isEmpty()) + { + for (PipelineTriggerConfig existingConfig : existingConfigs) + { + if (rowId == null || !rowId.equals(existingConfig.getRowId())) + { + errors.rejectValue("Name", null, "A pipeline trigger configuration already exists in this container for the given name: " + name); + break; + } + } + } + } + else + { + errors.rejectValue("Name", null, "A name is required for trigger configurations."); + } + + // validate that the type is a valid registered PipelineTriggerType + PipelineTriggerType triggerType = PipelineTriggerRegistry.get().getTypeByName(type); + if (triggerType == null) + { + errors.rejectValue("Type", null, "Invalid pipeline trigger type:" + type); + return; + } + + // validate that the pipelineId is a valid TaskPipeline + if (pipelineId != null) + { + try + { + PipelineJobService.get().getTaskPipeline(pipelineId); + } + catch (NotFoundException e) + { + errors.rejectValue("PipelineId", null, "Invalid pipeline task id: " + pipelineId); + } + } + else + { + errors.reject(null, null, "Pipeline Task ID required."); + return; + } + + // validate that the configuration values parse as valid JSON + validateConfigJson(triggerType, config.getConfiguration(), pipelineId, isEnabled, errors, container, user); + + Object customConfiguration = config.getCustomConfiguration(); + if (customConfiguration != null && !customConfiguration.toString().isEmpty()) + validateConfigJson(triggerType, customConfiguration, pipelineId, isEnabled, errors, true, container, user); + } + + private static void validateConfigJson(PipelineTriggerType triggerType, Object configuration, String pipelineId, boolean isEnabled, Errors errors, Container sourceContainer, User user) + { + validateConfigJson(triggerType, configuration, pipelineId, isEnabled, errors, false, sourceContainer, user); + } + + private static void validateConfigJson(PipelineTriggerType triggerType, Object configuration, String pipelineId, boolean isEnabled, Errors errors, boolean jsonValidityOnly, Container sourceContainer, User user) + { + JSONObject json = null; + if (configuration != null) + { + try + { + json = new JSONObject(configuration.toString()); + } + catch (JSONException e) + { + errors.reject(ERROR_MSG, "Invalid JSON object for the configuration field: " + e); + } + } + + // give the PipelineTriggerType a chance to validate the configuration JSON object + if (triggerType != null && !jsonValidityOnly) + { + List> configErrors = triggerType.validateConfiguration(pipelineId, isEnabled, json, sourceContainer, user); + for (Pair msg : configErrors) + errors.rejectValue(msg.first, null, msg.second); + } + } + + public static Path validateFolderImportFileNioPath(String archiveFilePath, PipeRoot pipeRoot, Errors errors) + { + Path archiveFile = Path.of(archiveFilePath); + + if (!archiveFile.isAbsolute()) + { + // Resolve the relative path to an absolute path under the current container's root + archiveFile = archiveFilePath.contains("://") ? pipeRoot.resolveToNioPathFromUrl(archiveFilePath) : pipeRoot.resolveToNioPath(archiveFilePath); + } + + // Be sure that the referenced file exists and is under the pipeline root + if (archiveFile == null || !Files.exists(archiveFile)) + { + errors.reject(ERROR_MSG, "Could not find file at path: " + archiveFilePath); + } + else if (!pipeRoot.isCloudRoot() && !pipeRoot.isUnderRoot(archiveFile)) // TODO: check for isCloud, then file should be in temp + { + errors.reject(ERROR_MSG, "Cannot access file " + archiveFilePath); + } + + return archiveFile; + } + + @Nullable + private static Path expandZipLocally(PipeRoot pipelineRoot, Path archiveFile, BindException errors) + { + try + { + // check if the archive file already exists in the unzip dir of this pipeline root + Path importDir = pipelineRoot.getImportDirectory().toPath(); + if (!archiveFile.getParent().toAbsolutePath().toString().equalsIgnoreCase(importDir.toAbsolutePath().toString())) + importDir = pipelineRoot.deleteImportDirectory(null); + + boolean shouldUnzip = Files.notExists(importDir); + + if (!shouldUnzip) + { + try (Stream pathStream = Files.list(importDir)) + { + shouldUnzip = pathStream.noneMatch(s -> s.getFileName().toString().equalsIgnoreCase(archiveFile.getFileName().toString())); + } + } + + if (shouldUnzip) + { + // Only unzip once + try (InputStream is = Files.newInputStream(archiveFile)) + { + ZipUtil.unzipToDirectory(is, importDir); + } + } + + return importDir; + } + catch (FileNotFoundException e) + { + errors.reject(ERROR_MSG, "File not found."); + } + catch (FileSystemAlreadyExistsException | DirectoryNotDeletedException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + catch (IOException e) + { + errors.reject(ERROR_MSG, "This file does not appear to be a valid .zip file."); + } + + // Return null if errors were observed + return null; + } + + private static Path getImportXmlFile(@NotNull PipeRoot pipelineRoot, @NotNull Path archiveFile, @NotNull String xmlFileName, BindException errors) throws InvalidFileException + { + Path xmlFile = archiveFile; + + if (archiveFile.getFileName().toString().toLowerCase().endsWith(".zip")) + { + Path importDir = expandZipLocally(pipelineRoot, archiveFile, errors); + if (importDir != null) + { + xmlFile = getXmlFilePathFromArchive(importDir, archiveFile, xmlFileName); + } + } + //Downloading expanded archive will be handled later in the archive processing... + //We don't really have the job context here + + return xmlFile; + } + + public static @NotNull Path getXmlFilePathFromArchive(@NotNull Path importDir, Path archiveFile, @NotNull String xmlFileName) throws InvalidFileException + { + // when importing a folder archive for a study, the study.xml file may not be at the root + if ("study.xml".equalsIgnoreCase(xmlFileName) && archiveFile.getFileName().toString().toLowerCase().endsWith(".folder.zip")) + { + File folderXml = new File(importDir.toFile(), "folder.xml"); + FolderDocument folderDoc; + try + { + folderDoc = FolderDocument.Factory.parse(folderXml, XmlBeansUtil.getDefaultParseOptions()); + XmlBeansUtil.validateXmlDocument(folderDoc, xmlFileName); + } + catch (Exception e) + { + throw new InvalidFileException(folderXml.getParentFile().toPath(), folderXml.toPath(), e); + } + + if (folderDoc.getFolder().isSetStudy()) + { + importDir = importDir.resolve(folderDoc.getFolder().getStudy().getDir()); + } + } + + return importDir.toAbsolutePath().resolve(xmlFileName); + } + + public static Path getArchiveXmlFile(Container container, Path archiveFile, String xmlFileName, BindException errors) throws InvalidFileException + { + PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(container); + Path xmlFile = getImportXmlFile(pipelineRoot, archiveFile, xmlFileName, errors); + + // if this is an import from a source template folder that has been previously implicitly exported + // to the unzip dir (without ever creating a zip file) then just look there for the xmlFile. + if (pipelineRoot != null && Files.isDirectory(archiveFile)) + { + xmlFile = java.nio.file.Path.of(archiveFile.toString(), xmlFileName); + } + + return xmlFile; + } +} diff --git a/query/src/org/labkey/query/QueryModule.java b/query/src/org/labkey/query/QueryModule.java index 4ae5a44f6d6..af9130cccf8 100644 --- a/query/src/org/labkey/query/QueryModule.java +++ b/query/src/org/labkey/query/QueryModule.java @@ -1,424 +1,421 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.query; - -import org.jetbrains.annotations.NotNull; -import org.json.JSONObject; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.DefaultAuditProvider; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.data.Aggregate; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.views.DataViewService; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.message.digest.DailyMessageDigest; -import org.labkey.api.message.digest.ReportAndDatasetChangeDigestProvider; -import org.labkey.api.module.AdminLinkManager; -import org.labkey.api.module.DefaultModule; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.query.DefaultSchema; -import org.labkey.api.query.JavaExportScriptFactory; -import org.labkey.api.query.JavaScriptExportScriptFactory; -import org.labkey.api.query.PerlExportScriptFactory; -import org.labkey.api.query.PythonExportScriptFactory; -import org.labkey.api.query.QuerySchema; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.RExportScriptFactory; -import org.labkey.api.query.SasExportScriptFactory; -import org.labkey.api.query.SimpleTableDomainKind; -import org.labkey.api.query.URLExportScriptFactory; -import org.labkey.api.query.column.BuiltInColumnTypes; -import org.labkey.api.query.snapshot.QuerySnapshotService; -import org.labkey.api.reports.ReportService; -import org.labkey.api.reports.report.ExternalScriptEngineReport; -import org.labkey.api.reports.report.InternalScriptEngineReport; -import org.labkey.api.reports.report.JavaScriptReport; -import org.labkey.api.reports.report.JavaScriptReportDescriptor; -import org.labkey.api.reports.report.QueryReport; -import org.labkey.api.reports.report.QueryReportDescriptor; -import org.labkey.api.reports.report.ReportDescriptor; -import org.labkey.api.reports.report.ReportUrls; -import org.labkey.api.reports.report.python.IpynbReport; -import org.labkey.api.reports.report.python.IpynbReportDescriptor; -import org.labkey.api.reports.report.r.RReport; -import org.labkey.api.reports.report.r.RReportDescriptor; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.PlatformDeveloperPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.roles.PlatformDeveloperRole; -import org.labkey.api.security.roles.Role; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.stats.AnalyticsProviderRegistry; -import org.labkey.api.stats.SummaryStatisticRegistry; -import org.labkey.api.util.JspTestCase; -import org.labkey.api.util.JunitUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.emailTemplate.EmailTemplateService; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.writer.ContainerUser; -import org.labkey.query.analytics.AggregatesCountNonBlankAnalyticsProvider; -import org.labkey.query.analytics.AggregatesMaxAnalyticsProvider; -import org.labkey.query.analytics.AggregatesMeanAnalyticsProvider; -import org.labkey.query.analytics.AggregatesMinAnalyticsProvider; -import org.labkey.query.analytics.AggregatesSumAnalyticsProvider; -import org.labkey.query.analytics.RemoveColumnAnalyticsProvider; -import org.labkey.query.analytics.SummaryStatisticsAnalyticsProvider; -import org.labkey.query.audit.QueryExportAuditProvider; -import org.labkey.query.audit.QueryUpdateAuditProvider; -import org.labkey.query.controllers.OlapController; -import org.labkey.query.controllers.QueryController; -import org.labkey.query.controllers.SqlController; -import org.labkey.query.jdbc.QueryDriver; -import org.labkey.query.olap.MemberSet; -import org.labkey.query.olap.ServerManager; -import org.labkey.query.olap.metadata.MetadataElementBase; -import org.labkey.query.olap.rolap.RolapReader; -import org.labkey.query.olap.rolap.RolapTestCase; -import org.labkey.query.olap.rolap.RolapTestSchema; -import org.labkey.query.persist.QueryManager; -import org.labkey.query.reports.AttachmentReport; -import org.labkey.query.reports.LinkReport; -import org.labkey.query.reports.ModuleReportCache; -import org.labkey.query.reports.ReportAndDatasetChangeDigestProviderImpl; -import org.labkey.query.reports.ReportAuditProvider; -import org.labkey.query.reports.ReportImporter; -import org.labkey.query.reports.ReportNotificationInfoProvider; -import org.labkey.query.reports.ReportServiceImpl; -import org.labkey.query.reports.ReportViewProvider; -import org.labkey.query.reports.ReportWriter; -import org.labkey.query.reports.ReportsController; -import org.labkey.query.reports.ReportsPipelineProvider; -import org.labkey.query.reports.ReportsWebPartFactory; -import org.labkey.query.reports.ViewCategoryImporter; -import org.labkey.query.reports.ViewCategoryWriter; -import org.labkey.query.reports.getdata.AggregateQueryDataTransform; -import org.labkey.query.reports.getdata.FilterClauseBuilder; -import org.labkey.query.reports.view.ReportAndDatasetChangeDigestEmailTemplate; -import org.labkey.query.reports.view.ReportUIProvider; -import org.labkey.query.sql.Method; -import org.labkey.query.sql.QNode; -import org.labkey.query.sql.Query; -import org.labkey.query.sql.SqlParser; -import org.labkey.query.view.InheritedQueryDataViewProvider; -import org.labkey.query.view.QueryDataViewProvider; -import org.labkey.query.view.QueryWebPartFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.function.Supplier; - -import static org.labkey.api.query.QueryService.USE_ROW_BY_ROW_UPDATE; - -public class QueryModule extends DefaultModule -{ - public QueryModule() - { - QueryService.setInstance(new QueryServiceImpl()); - BuiltInColumnTypes.registerStandardColumnTransformers(); - - QueryDriver.register(); - ReportAndDatasetChangeDigestProvider.set(new ReportAndDatasetChangeDigestProviderImpl()); - } - - @Override - public String getName() - { - return "Query"; - } - - @Override - public Double getSchemaVersion() - { - return 25.001; - } - - @Override - protected void init() - { - DefaultSchema.registerProvider("rolap_test", new DefaultSchema.SchemaProvider(this) - { - @Override - public boolean isAvailable(DefaultSchema schema, Module module) - { - return schema.getContainer().getParsedPath().equals(JunitUtil.getTestContainerPath()); - } - - @Override - public QuerySchema createSchema(DefaultSchema schema, Module module) - { - return new RolapTestSchema(schema.getUser(), schema.getContainer()); - } - }); - - addController("query", QueryController.class); - addController("sql", SqlController.class); - addController("reports", ReportsController.class); - addController("olap", OlapController.class); - - ExternalSchema.register(); - LinkedSchema.register(); - - QueryService.get().addQueryListener(new CustomViewQueryChangeListener()); - QueryService.get().addQueryListener(new QuerySnapshotQueryChangeListener()); - QueryService.get().addQueryListener(new QueryDefQueryChangeListener()); - - ReportService.registerProvider(ReportServiceImpl.getInstance()); - ReportService.get().addUIProvider(new ReportUIProvider()); - ReportService.get().addGlobalItemFilterType(JavaScriptReport.TYPE); - ReportService.get().addGlobalItemFilterType(QuerySnapshotService.TYPE); - ReportService.get().addGlobalItemFilterType(IpynbReport.TYPE); - - ReportService.get().registerDescriptor(new IpynbReportDescriptor()); - ReportService.get().registerDescriptor(new ReportDescriptor()); - ReportService.get().registerDescriptor(new QueryReportDescriptor()); - ReportService.get().registerDescriptor(new RReportDescriptor()); - ReportService.get().registerDescriptor(new JavaScriptReportDescriptor()); - - ReportService.get().registerReport(new IpynbReport()); - ReportService.get().registerReport(new QueryReport()); - ReportService.get().registerReport(new RReport()); - ReportService.get().registerReport(new ExternalScriptEngineReport()); - ReportService.get().registerReport(new InternalScriptEngineReport()); - ReportService.get().registerReport(new JavaScriptReport()); - ReportService.get().registerReport(new AttachmentReport()); - ReportService.get().registerReport(new LinkReport()); - EmailTemplateService.get().registerTemplate(ReportAndDatasetChangeDigestEmailTemplate.class); - - QueryView.register(new RExportScriptFactory()); - QueryView.register(new JavaScriptExportScriptFactory()); - QueryView.register(new PerlExportScriptFactory()); - QueryView.register(new JavaExportScriptFactory()); - QueryView.register(new URLExportScriptFactory()); - QueryView.register(new PythonExportScriptFactory()); - QueryView.register(new SasExportScriptFactory()); - - DataViewService.get().registerProvider(ReportViewProvider.TYPE, new ReportViewProvider()); - - DataViewService.get().registerProvider(QueryDataViewProvider.TYPE, new QueryDataViewProvider()); - DataViewService.get().registerProvider(InheritedQueryDataViewProvider.TYPE, new InheritedQueryDataViewProvider()); - - OptionalFeatureService.get().addExperimentalFeatureFlag(QueryView.EXPERIMENTAL_GENERIC_DETAILS_URL, "Generic [details] link in grids/queries", - "This feature will turn on generating a generic [details] URL link in most grids.", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(QueryServiceImpl.EXPERIMENTAL_LAST_MODIFIED, "Include Last-Modified header on query metadata requests", - "For schema, query, and view metadata requests include a Last-Modified header such that the browser can cache the response. " + - "The metadata is invalidated when performing actions such as creating a new List or modifying the columns on a custom view", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(USE_ROW_BY_ROW_UPDATE, "Use row-by-row update", - "For Query.updateRows api, do row-by-row update, instead of using a prepared statement that updates rows in batches.", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(QueryServiceImpl.EXPERIMENTAL_PRODUCT_ALL_FOLDER_LOOKUPS, "Less restrictive product folder lookups", - "Allow for lookup fields in product folders to query across all folders within the top-level folder.", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(QueryServiceImpl.EXPERIMENTAL_PRODUCT_PROJECT_DATA_LISTING_SCOPED, "Product folders display folder-specific data", - "Only list folder-specific data within product folders.", false); - } - - - @Override - @NotNull - protected Collection createWebPartFactories() - { - return List.of( - new DataViewsWebPartFactory(), - new QueryWebPartFactory(), - new ReportsWebPartFactory() -// new QueryBrowserWebPartFactory() - ); - } - - @Override - public boolean hasScripts() - { - return true; - } - - @Override - public void doStartup(ModuleContext moduleContext) - { - ContainerManager.addContainerListener(QueryManager.CONTAINER_LISTENER, ContainerManager.ContainerListener.Order.Last); - - if (null != PipelineService.get()) - PipelineService.get().registerPipelineProvider(new ReportsPipelineProvider(this)); - QueryController.registerAdminConsoleLinks(); - - FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); - if (null != folderRegistry) - { - folderRegistry.addFactories(new QueryWriter.Factory(), new QueryImporter.Factory()); - folderRegistry.addFactories(new CustomViewWriter.Factory(), new CustomViewImporter.Factory()); - folderRegistry.addFactories(new ReportWriter.Factory(), new ReportImporter.Factory()); - folderRegistry.addFactories(new ViewCategoryWriter.Factory(), new ViewCategoryImporter.Factory()); - folderRegistry.addFactories(new ExternalSchemaDefWriterFactory(), new ExternalSchemaDefImporterFactory()); - } - - SearchService ss = SearchService.get(); - - if (null != ss) - { - ss.addDocumentProvider(ExternalSchemaDocumentProvider.getInstance()); - ss.addSearchCategory(ExternalSchemaDocumentProvider.externalTableCategory); - } - if (null != PropertyService.get()) - PropertyService.get().registerDomainKind(new SimpleTableDomainKind()); - - if (null != AuditLogService.get() && AuditLogService.get().getClass() != DefaultAuditProvider.class) - { - AuditLogService.get().registerAuditType(new QueryExportAuditProvider()); - AuditLogService.get().registerAuditType(new QueryUpdateAuditProvider()); - } - AuditLogService.get().registerAuditType(new ReportAuditProvider()); - - ReportAndDatasetChangeDigestProvider.get().addNotificationInfoProvider(new ReportNotificationInfoProvider()); - DailyMessageDigest.getInstance().addProvider(ReportAndDatasetChangeDigestProvider.get()); - // Note: DailyMessageDigest timer is initialized by the AnnouncementModule - - CacheManager.addListener(new ServerManager.CacheListener()); - CacheManager.addListener(new QueryServiceImpl.CacheListener()); - - AdminLinkManager.getInstance().addListener((adminNavTree, container, user) -> { - if (container.hasPermission(user, ReadPermission.class)) - adminNavTree.addChild(new NavTree("Manage Views", PageFlowUtil.urlProvider(ReportUrls.class).urlManageViews(container))); - }); - - AnalyticsProviderRegistry analyticsProviderRegistry = AnalyticsProviderRegistry.get(); - if (null != analyticsProviderRegistry) - { - analyticsProviderRegistry.registerProvider(new AggregatesCountNonBlankAnalyticsProvider()); - analyticsProviderRegistry.registerProvider(new AggregatesSumAnalyticsProvider()); - analyticsProviderRegistry.registerProvider(new AggregatesMeanAnalyticsProvider()); - analyticsProviderRegistry.registerProvider(new AggregatesMinAnalyticsProvider()); - analyticsProviderRegistry.registerProvider(new AggregatesMaxAnalyticsProvider()); - analyticsProviderRegistry.registerProvider(new SummaryStatisticsAnalyticsProvider()); - analyticsProviderRegistry.registerProvider(new RemoveColumnAnalyticsProvider()); - } - - SummaryStatisticRegistry summaryStatisticRegistry = SummaryStatisticRegistry.get(); - if (null != summaryStatisticRegistry) - { - summaryStatisticRegistry.register(Aggregate.BaseType.SUM); - summaryStatisticRegistry.register(Aggregate.BaseType.MEAN); - summaryStatisticRegistry.register(Aggregate.BaseType.COUNT); - summaryStatisticRegistry.register(Aggregate.BaseType.MIN); - summaryStatisticRegistry.register(Aggregate.BaseType.MAX); - } - - QueryManager.registerUsageMetrics(getName()); - ReportServiceImpl.registerUsageMetrics(getName()); - - // Administrators, Platform Developers, and Trusted Analysts can edit queries, if they also have edit permissions in the current folder - RoleManager.registerPermission(new EditQueriesPermission()); - Role platformDeveloperRole = RoleManager.getRole(PlatformDeveloperRole.class); - platformDeveloperRole.addPermission(EditQueriesPermission.class); - Role trustedAnalystRole = RoleManager.getRole("org.labkey.api.security.roles.TrustedAnalystRole"); - if (null != trustedAnalystRole) - trustedAnalystRole.addPermission(EditQueriesPermission.class); - } - - @Override - @NotNull - public Set getSchemaNames() - { - return PageFlowUtil.set(QueryManager.get().getDbSchemaName(), "junit"); - } - - @Override - @NotNull - public Set getIntegrationTests() - { - return Set.of( - ModuleReportCache.TestCase.class, - OlapController.TestCase.class, - QueryController.TestCase.class, - QueryController.SaveRowsTestCase.class, - QueryServiceImpl.TestCase.class, - RolapReader.RolapTest.class, - RolapTestCase.class, - ServerManager.TestCase.class - ); - } - - @Override - public @NotNull Collection>> getIntegrationTestFactories() - { - List>> ret = new ArrayList<>(super.getIntegrationTestFactories()); - ret.add(new JspTestCase("/org/labkey/query/MultiValueTest.jsp")); - ret.add(new JspTestCase("/org/labkey/query/olap/OlapTestCase.jsp")); - ret.add(new JspTestCase("/org/labkey/query/QueryServiceImplTestCase.jsp")); - ret.add(new JspTestCase("/org/labkey/query/QueryTestCase.jsp")); - ret.add(new JspTestCase("/org/labkey/query/sql/CalculatedColumnTestCase.jsp")); - - return ret; - } - - - @Override - @NotNull - public Set getUnitTests() - { - return Set.of( - AggregateQueryDataTransform.TestCase.class, - AttachmentReport.TestCase.class, - FilterClauseBuilder.TestCase.class, - JdbcType.TestCase.class, - MemberSet.TestCase.class, - MetadataElementBase.TestCase.class, - Method.TestCase.class, - QNode.TestCase.class, - Query.TestCase.class, - ReportsController.SerializationTest.class, - SqlParser.SqlParserTestCase.class, - TableWriter.TestCase.class - ); - } - - @Override - public ActionURL getTabURL(Container c, User user) - { - // Don't show Query nav trails to users who aren't admins or developers since they almost certainly don't want - // to go to those links - if (c.hasOneOf(user, AdminPermission.class, PlatformDeveloperPermission.class)) - { - return super.getTabURL(c, user); - } - return null; - } - - @Override - public JSONObject getPageContextJson(ContainerUser context) - { - JSONObject json = super.getPageContextJson(context); - boolean hasEditQueriesPermission = context.getContainer().hasPermission(context.getUser(), EditQueriesPermission.class); - json.put("hasEditQueriesPermission", hasEditQueriesPermission); - Container container = context.getContainer(); - boolean isProductFoldersEnabled = container != null && container.isProductFoldersEnabled(); // TODO: should these be moved to CoreModule? - json.put(QueryService.PRODUCT_FOLDERS_ENABLED, isProductFoldersEnabled); - json.put(QueryService.PRODUCT_FOLDERS_EXIST, isProductFoldersEnabled && container.hasProductFolders()); - json.put(QueryService.EXPERIMENTAL_PRODUCT_ALL_FOLDER_LOOKUPS, QueryService.get().isProductFoldersAllFolderScopeEnabled()); - json.put(QueryService.EXPERIMENTAL_PRODUCT_PROJECT_DATA_LISTING_SCOPED, QueryService.get().isProductFoldersDataListingScopedToProject()); - return json; - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.query; + +import org.jetbrains.annotations.NotNull; +import org.json.JSONObject; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.DefaultAuditProvider; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.data.Aggregate; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.views.DataViewService; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.message.digest.DailyMessageDigest; +import org.labkey.api.message.digest.ReportAndDatasetChangeDigestProvider; +import org.labkey.api.module.AdminLinkManager; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.JavaExportScriptFactory; +import org.labkey.api.query.JavaScriptExportScriptFactory; +import org.labkey.api.query.PerlExportScriptFactory; +import org.labkey.api.query.PythonExportScriptFactory; +import org.labkey.api.query.QuerySchema; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.RExportScriptFactory; +import org.labkey.api.query.SasExportScriptFactory; +import org.labkey.api.query.SimpleTableDomainKind; +import org.labkey.api.query.URLExportScriptFactory; +import org.labkey.api.query.column.BuiltInColumnTypes; +import org.labkey.api.query.snapshot.QuerySnapshotService; +import org.labkey.api.reports.ReportService; +import org.labkey.api.reports.report.ExternalScriptEngineReport; +import org.labkey.api.reports.report.InternalScriptEngineReport; +import org.labkey.api.reports.report.JavaScriptReport; +import org.labkey.api.reports.report.JavaScriptReportDescriptor; +import org.labkey.api.reports.report.QueryReport; +import org.labkey.api.reports.report.QueryReportDescriptor; +import org.labkey.api.reports.report.ReportDescriptor; +import org.labkey.api.reports.report.ReportUrls; +import org.labkey.api.reports.report.python.IpynbReport; +import org.labkey.api.reports.report.python.IpynbReportDescriptor; +import org.labkey.api.reports.report.r.RReport; +import org.labkey.api.reports.report.r.RReportDescriptor; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.PlatformDeveloperPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.roles.PlatformDeveloperRole; +import org.labkey.api.security.roles.Role; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.stats.AnalyticsProviderRegistry; +import org.labkey.api.stats.SummaryStatisticRegistry; +import org.labkey.api.util.JspTestCase; +import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.emailTemplate.EmailTemplateService; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.writer.ContainerUser; +import org.labkey.query.analytics.AggregatesCountNonBlankAnalyticsProvider; +import org.labkey.query.analytics.AggregatesMaxAnalyticsProvider; +import org.labkey.query.analytics.AggregatesMeanAnalyticsProvider; +import org.labkey.query.analytics.AggregatesMinAnalyticsProvider; +import org.labkey.query.analytics.AggregatesSumAnalyticsProvider; +import org.labkey.query.analytics.RemoveColumnAnalyticsProvider; +import org.labkey.query.analytics.SummaryStatisticsAnalyticsProvider; +import org.labkey.query.audit.QueryExportAuditProvider; +import org.labkey.query.audit.QueryUpdateAuditProvider; +import org.labkey.query.controllers.OlapController; +import org.labkey.query.controllers.QueryController; +import org.labkey.query.controllers.SqlController; +import org.labkey.query.jdbc.QueryDriver; +import org.labkey.query.olap.MemberSet; +import org.labkey.query.olap.ServerManager; +import org.labkey.query.olap.metadata.MetadataElementBase; +import org.labkey.query.olap.rolap.RolapReader; +import org.labkey.query.olap.rolap.RolapTestCase; +import org.labkey.query.olap.rolap.RolapTestSchema; +import org.labkey.query.persist.QueryManager; +import org.labkey.query.reports.AttachmentReport; +import org.labkey.query.reports.LinkReport; +import org.labkey.query.reports.ModuleReportCache; +import org.labkey.query.reports.ReportAndDatasetChangeDigestProviderImpl; +import org.labkey.query.reports.ReportAuditProvider; +import org.labkey.query.reports.ReportImporter; +import org.labkey.query.reports.ReportNotificationInfoProvider; +import org.labkey.query.reports.ReportServiceImpl; +import org.labkey.query.reports.ReportViewProvider; +import org.labkey.query.reports.ReportWriter; +import org.labkey.query.reports.ReportsController; +import org.labkey.query.reports.ReportsPipelineProvider; +import org.labkey.query.reports.ReportsWebPartFactory; +import org.labkey.query.reports.ViewCategoryImporter; +import org.labkey.query.reports.ViewCategoryWriter; +import org.labkey.query.reports.getdata.AggregateQueryDataTransform; +import org.labkey.query.reports.getdata.FilterClauseBuilder; +import org.labkey.query.reports.view.ReportAndDatasetChangeDigestEmailTemplate; +import org.labkey.query.reports.view.ReportUIProvider; +import org.labkey.query.sql.Method; +import org.labkey.query.sql.QNode; +import org.labkey.query.sql.Query; +import org.labkey.query.sql.SqlParser; +import org.labkey.query.view.InheritedQueryDataViewProvider; +import org.labkey.query.view.QueryDataViewProvider; +import org.labkey.query.view.QueryWebPartFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.function.Supplier; + +import static org.labkey.api.query.QueryService.USE_ROW_BY_ROW_UPDATE; + +public class QueryModule extends DefaultModule +{ + public QueryModule() + { + QueryService.setInstance(new QueryServiceImpl()); + BuiltInColumnTypes.registerStandardColumnTransformers(); + + QueryDriver.register(); + ReportAndDatasetChangeDigestProvider.set(new ReportAndDatasetChangeDigestProviderImpl()); + } + + @Override + public String getName() + { + return "Query"; + } + + @Override + public Double getSchemaVersion() + { + return 25.001; + } + + @Override + protected void init() + { + DefaultSchema.registerProvider("rolap_test", new DefaultSchema.SchemaProvider(this) + { + @Override + public boolean isAvailable(DefaultSchema schema, Module module) + { + return schema.getContainer().getParsedPath().equals(JunitUtil.getTestContainerPath()); + } + + @Override + public QuerySchema createSchema(DefaultSchema schema, Module module) + { + return new RolapTestSchema(schema.getUser(), schema.getContainer()); + } + }); + + addController("query", QueryController.class); + addController("sql", SqlController.class); + addController("reports", ReportsController.class); + addController("olap", OlapController.class); + + ExternalSchema.register(); + LinkedSchema.register(); + + QueryService.get().addQueryListener(new CustomViewQueryChangeListener()); + QueryService.get().addQueryListener(new QuerySnapshotQueryChangeListener()); + QueryService.get().addQueryListener(new QueryDefQueryChangeListener()); + + ReportService.registerProvider(ReportServiceImpl.getInstance()); + ReportService.get().addUIProvider(new ReportUIProvider()); + ReportService.get().addGlobalItemFilterType(JavaScriptReport.TYPE); + ReportService.get().addGlobalItemFilterType(QuerySnapshotService.TYPE); + ReportService.get().addGlobalItemFilterType(IpynbReport.TYPE); + + ReportService.get().registerDescriptor(new IpynbReportDescriptor()); + ReportService.get().registerDescriptor(new ReportDescriptor()); + ReportService.get().registerDescriptor(new QueryReportDescriptor()); + ReportService.get().registerDescriptor(new RReportDescriptor()); + ReportService.get().registerDescriptor(new JavaScriptReportDescriptor()); + + ReportService.get().registerReport(new IpynbReport()); + ReportService.get().registerReport(new QueryReport()); + ReportService.get().registerReport(new RReport()); + ReportService.get().registerReport(new ExternalScriptEngineReport()); + ReportService.get().registerReport(new InternalScriptEngineReport()); + ReportService.get().registerReport(new JavaScriptReport()); + ReportService.get().registerReport(new AttachmentReport()); + ReportService.get().registerReport(new LinkReport()); + EmailTemplateService.get().registerTemplate(ReportAndDatasetChangeDigestEmailTemplate.class); + + QueryView.register(new RExportScriptFactory()); + QueryView.register(new JavaScriptExportScriptFactory()); + QueryView.register(new PerlExportScriptFactory()); + QueryView.register(new JavaExportScriptFactory()); + QueryView.register(new URLExportScriptFactory()); + QueryView.register(new PythonExportScriptFactory()); + QueryView.register(new SasExportScriptFactory()); + + DataViewService.get().registerProvider(ReportViewProvider.TYPE, new ReportViewProvider()); + + DataViewService.get().registerProvider(QueryDataViewProvider.TYPE, new QueryDataViewProvider()); + DataViewService.get().registerProvider(InheritedQueryDataViewProvider.TYPE, new InheritedQueryDataViewProvider()); + + OptionalFeatureService.get().addExperimentalFeatureFlag(QueryView.EXPERIMENTAL_GENERIC_DETAILS_URL, "Generic [details] link in grids/queries", + "This feature will turn on generating a generic [details] URL link in most grids.", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(QueryServiceImpl.EXPERIMENTAL_LAST_MODIFIED, "Include Last-Modified header on query metadata requests", + "For schema, query, and view metadata requests include a Last-Modified header such that the browser can cache the response. " + + "The metadata is invalidated when performing actions such as creating a new List or modifying the columns on a custom view", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(USE_ROW_BY_ROW_UPDATE, "Use row-by-row update", + "For Query.updateRows api, do row-by-row update, instead of using a prepared statement that updates rows in batches.", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(QueryServiceImpl.EXPERIMENTAL_PRODUCT_ALL_FOLDER_LOOKUPS, "Less restrictive product folder lookups", + "Allow for lookup fields in product folders to query across all folders within the top-level folder.", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(QueryServiceImpl.EXPERIMENTAL_PRODUCT_PROJECT_DATA_LISTING_SCOPED, "Product folders display folder-specific data", + "Only list folder-specific data within product folders.", false); + } + + + @Override + @NotNull + protected Collection createWebPartFactories() + { + return List.of( + new DataViewsWebPartFactory(), + new QueryWebPartFactory(), + new ReportsWebPartFactory() +// new QueryBrowserWebPartFactory() + ); + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + public void doStartup(ModuleContext moduleContext) + { + ContainerManager.addContainerListener(QueryManager.CONTAINER_LISTENER, ContainerManager.ContainerListener.Order.Last); + + if (null != PipelineService.get()) + PipelineService.get().registerPipelineProvider(new ReportsPipelineProvider(this)); + QueryController.registerAdminConsoleLinks(); + + FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); + if (null != folderRegistry) + { + folderRegistry.addFactories(new QueryWriter.Factory(), new QueryImporter.Factory()); + folderRegistry.addFactories(new CustomViewWriter.Factory(), new CustomViewImporter.Factory()); + folderRegistry.addFactories(new ReportWriter.Factory(), new ReportImporter.Factory()); + folderRegistry.addFactories(new ViewCategoryWriter.Factory(), new ViewCategoryImporter.Factory()); + folderRegistry.addFactories(new ExternalSchemaDefWriterFactory(), new ExternalSchemaDefImporterFactory()); + } + + SearchService ss = SearchService.get(); + ss.addDocumentProvider(ExternalSchemaDocumentProvider.getInstance()); + ss.addSearchCategory(ExternalSchemaDocumentProvider.externalTableCategory); + + if (null != PropertyService.get()) + PropertyService.get().registerDomainKind(new SimpleTableDomainKind()); + + if (null != AuditLogService.get() && AuditLogService.get().getClass() != DefaultAuditProvider.class) + { + AuditLogService.get().registerAuditType(new QueryExportAuditProvider()); + AuditLogService.get().registerAuditType(new QueryUpdateAuditProvider()); + } + AuditLogService.get().registerAuditType(new ReportAuditProvider()); + + ReportAndDatasetChangeDigestProvider.get().addNotificationInfoProvider(new ReportNotificationInfoProvider()); + DailyMessageDigest.getInstance().addProvider(ReportAndDatasetChangeDigestProvider.get()); + // Note: DailyMessageDigest timer is initialized by the AnnouncementModule + + CacheManager.addListener(new ServerManager.CacheListener()); + CacheManager.addListener(new QueryServiceImpl.CacheListener()); + + AdminLinkManager.getInstance().addListener((adminNavTree, container, user) -> { + if (container.hasPermission(user, ReadPermission.class)) + adminNavTree.addChild(new NavTree("Manage Views", PageFlowUtil.urlProvider(ReportUrls.class).urlManageViews(container))); + }); + + AnalyticsProviderRegistry analyticsProviderRegistry = AnalyticsProviderRegistry.get(); + if (null != analyticsProviderRegistry) + { + analyticsProviderRegistry.registerProvider(new AggregatesCountNonBlankAnalyticsProvider()); + analyticsProviderRegistry.registerProvider(new AggregatesSumAnalyticsProvider()); + analyticsProviderRegistry.registerProvider(new AggregatesMeanAnalyticsProvider()); + analyticsProviderRegistry.registerProvider(new AggregatesMinAnalyticsProvider()); + analyticsProviderRegistry.registerProvider(new AggregatesMaxAnalyticsProvider()); + analyticsProviderRegistry.registerProvider(new SummaryStatisticsAnalyticsProvider()); + analyticsProviderRegistry.registerProvider(new RemoveColumnAnalyticsProvider()); + } + + SummaryStatisticRegistry summaryStatisticRegistry = SummaryStatisticRegistry.get(); + if (null != summaryStatisticRegistry) + { + summaryStatisticRegistry.register(Aggregate.BaseType.SUM); + summaryStatisticRegistry.register(Aggregate.BaseType.MEAN); + summaryStatisticRegistry.register(Aggregate.BaseType.COUNT); + summaryStatisticRegistry.register(Aggregate.BaseType.MIN); + summaryStatisticRegistry.register(Aggregate.BaseType.MAX); + } + + QueryManager.registerUsageMetrics(getName()); + ReportServiceImpl.registerUsageMetrics(getName()); + + // Administrators, Platform Developers, and Trusted Analysts can edit queries, if they also have edit permissions in the current folder + RoleManager.registerPermission(new EditQueriesPermission()); + Role platformDeveloperRole = RoleManager.getRole(PlatformDeveloperRole.class); + platformDeveloperRole.addPermission(EditQueriesPermission.class); + Role trustedAnalystRole = RoleManager.getRole("org.labkey.api.security.roles.TrustedAnalystRole"); + if (null != trustedAnalystRole) + trustedAnalystRole.addPermission(EditQueriesPermission.class); + } + + @Override + @NotNull + public Set getSchemaNames() + { + return PageFlowUtil.set(QueryManager.get().getDbSchemaName(), "junit"); + } + + @Override + @NotNull + public Set getIntegrationTests() + { + return Set.of( + ModuleReportCache.TestCase.class, + OlapController.TestCase.class, + QueryController.TestCase.class, + QueryController.SaveRowsTestCase.class, + QueryServiceImpl.TestCase.class, + RolapReader.RolapTest.class, + RolapTestCase.class, + ServerManager.TestCase.class + ); + } + + @Override + public @NotNull Collection>> getIntegrationTestFactories() + { + List>> ret = new ArrayList<>(super.getIntegrationTestFactories()); + ret.add(new JspTestCase("/org/labkey/query/MultiValueTest.jsp")); + ret.add(new JspTestCase("/org/labkey/query/olap/OlapTestCase.jsp")); + ret.add(new JspTestCase("/org/labkey/query/QueryServiceImplTestCase.jsp")); + ret.add(new JspTestCase("/org/labkey/query/QueryTestCase.jsp")); + ret.add(new JspTestCase("/org/labkey/query/sql/CalculatedColumnTestCase.jsp")); + + return ret; + } + + + @Override + @NotNull + public Set getUnitTests() + { + return Set.of( + AggregateQueryDataTransform.TestCase.class, + AttachmentReport.TestCase.class, + FilterClauseBuilder.TestCase.class, + JdbcType.TestCase.class, + MemberSet.TestCase.class, + MetadataElementBase.TestCase.class, + Method.TestCase.class, + QNode.TestCase.class, + Query.TestCase.class, + ReportsController.SerializationTest.class, + SqlParser.SqlParserTestCase.class, + TableWriter.TestCase.class + ); + } + + @Override + public ActionURL getTabURL(Container c, User user) + { + // Don't show Query nav trails to users who aren't admins or developers since they almost certainly don't want + // to go to those links + if (c.hasOneOf(user, AdminPermission.class, PlatformDeveloperPermission.class)) + { + return super.getTabURL(c, user); + } + return null; + } + + @Override + public JSONObject getPageContextJson(ContainerUser context) + { + JSONObject json = super.getPageContextJson(context); + boolean hasEditQueriesPermission = context.getContainer().hasPermission(context.getUser(), EditQueriesPermission.class); + json.put("hasEditQueriesPermission", hasEditQueriesPermission); + Container container = context.getContainer(); + boolean isProductFoldersEnabled = container != null && container.isProductFoldersEnabled(); // TODO: should these be moved to CoreModule? + json.put(QueryService.PRODUCT_FOLDERS_ENABLED, isProductFoldersEnabled); + json.put(QueryService.PRODUCT_FOLDERS_EXIST, isProductFoldersEnabled && container.hasProductFolders()); + json.put(QueryService.EXPERIMENTAL_PRODUCT_ALL_FOLDER_LOOKUPS, QueryService.get().isProductFoldersAllFolderScopeEnabled()); + json.put(QueryService.EXPERIMENTAL_PRODUCT_PROJECT_DATA_LISTING_SCOPED, QueryService.get().isProductFoldersDataListingScopedToProject()); + return json; + } +} diff --git a/search/src/org/labkey/search/SearchContainerListener.java b/search/src/org/labkey/search/SearchContainerListener.java index 79ef1082492..79bc97af6df 100644 --- a/search/src/org/labkey/search/SearchContainerListener.java +++ b/search/src/org/labkey/search/SearchContainerListener.java @@ -1,59 +1,55 @@ -/* - * Copyright (c) 2009-2017 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.search; - -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.webdav.WebdavService; -import org.labkey.search.model.DavCrawler; - -import java.beans.PropertyChangeEvent; - -public class SearchContainerListener extends ContainerManager.AbstractContainerListener -{ - @Override - public void containerCreated(Container c, User user) - { - SearchService ss = SearchService.get(); - if (null != ss) - { - DavCrawler.getInstance().addPathToCrawl(WebdavService.getPath().append(c.getParsedPath()), null); - } - } - - @Override - public void containerDeleted(Container c, User user) - { - SearchService.get().deleteContainer(c.getId()); - } - - @Override - public void propertyChange(PropertyChangeEvent propertyChangeEvent) - { - ContainerManager.ContainerPropertyChangeEvent evt = (ContainerManager.ContainerPropertyChangeEvent) propertyChangeEvent; - - switch (evt.property) - { - case Name: // Rename - case Parent: // Move - SearchService.get().reindexContainerFiles(evt.container); - break; - } - } +/* + * Copyright (c) 2009-2017 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.search; + +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.webdav.WebdavService; +import org.labkey.search.model.DavCrawler; + +import java.beans.PropertyChangeEvent; + +public class SearchContainerListener extends ContainerManager.AbstractContainerListener +{ + @Override + public void containerCreated(Container c, User user) + { + DavCrawler.getInstance().addPathToCrawl(WebdavService.getPath().append(c.getParsedPath()), null); + } + + @Override + public void containerDeleted(Container c, User user) + { + SearchService.get().deleteContainer(c.getId()); + } + + @Override + public void propertyChange(PropertyChangeEvent propertyChangeEvent) + { + ContainerManager.ContainerPropertyChangeEvent evt = (ContainerManager.ContainerPropertyChangeEvent) propertyChangeEvent; + + switch (evt.property) + { + case Name: // Rename + case Parent: // Move + SearchService.get().reindexContainerFiles(evt.container); + break; + } + } } \ No newline at end of file diff --git a/search/src/org/labkey/search/SearchController.java b/search/src/org/labkey/search/SearchController.java index bb3b183bc67..d5e4e33e029 100644 --- a/search/src/org/labkey/search/SearchController.java +++ b/search/src/org/labkey/search/SearchController.java @@ -1,1199 +1,1187 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.search; - -import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.hc.core5.http.HttpStatus; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.action.ApiResponse; -import org.labkey.api.action.ApiSimpleResponse; -import org.labkey.api.action.ExportAction; -import org.labkey.api.action.FormHandlerAction; -import org.labkey.api.action.FormViewAction; -import org.labkey.api.action.ReadOnlyApiAction; -import org.labkey.api.action.ReturnUrlForm; -import org.labkey.api.action.SimpleRedirectAction; -import org.labkey.api.action.SimpleViewAction; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AdminUrls; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.search.SearchResultTemplate; -import org.labkey.api.search.SearchScope; -import org.labkey.api.search.SearchService; -import org.labkey.api.search.SearchService.SearchResult; -import org.labkey.api.search.SearchUrls; -import org.labkey.api.security.AdminConsoleAction; -import org.labkey.api.security.RequiresPermission; -import org.labkey.api.security.RequiresSiteAdmin; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminOperationsPermission; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.ApplicationAdminPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.util.ConfigurationException; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.HtmlStringBuilder; -import org.labkey.api.util.Path; -import org.labkey.api.util.ResponseHelper; -import org.labkey.api.util.URLHelper; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.FolderManagement.FolderManagementViewPostAction; -import org.labkey.api.view.HtmlView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.VBox; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.PageConfig; -import org.labkey.api.webdav.WebdavService; -import org.labkey.search.audit.SearchAuditProvider; -import org.labkey.search.model.AbstractSearchService; -import org.labkey.search.model.CrawlerRunningState; -import org.labkey.search.model.IndexInspector; -import org.labkey.search.model.LuceneDirectoryType; -import org.labkey.search.model.SearchPropertyManager; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.servlet.ModelAndView; - -import java.io.IOException; -import java.sql.Date; -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -public class SearchController extends SpringActionController -{ - private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(SearchController.class); - - private static final Logger LOG = LogHelper.getLogger(SearchController.class, "Search UI and admin"); - - public SearchController() - { - setActionResolver(_actionResolver); - } - - - @SuppressWarnings("unused") - public static class SearchUrlsImpl implements SearchUrls - { - @Override - public ActionURL getSearchURL(Container c, @Nullable String query) - { - return SearchController.getSearchURL(c, query); - } - - @Override - public ActionURL getSearchURL(String query, String category) - { - return SearchController.getSearchURL(ContainerManager.getRoot(), query, category, null); - } - - @Override - public ActionURL getSearchURL(Container c, @Nullable String query, @NotNull String template) - { - return SearchController.getSearchURL(c, query, null, template); - } - } - - - @RequiresPermission(ReadPermission.class) - public class BeginAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(Object o) - { - return getSearchURL(); - } - } - - - public static class AdminForm - { - public String[] _messages = {"", "Index deleted", "Index path changed", "Directory type changed", "File size limit changed"}; - private int msg = 0; - private boolean pause; - private boolean start; - private boolean delete; - private String indexPath; - - private boolean limit; - private int fileLimitMB; - - private boolean _path; - - private boolean _directory; - private String _directoryType; - - public String getMessage() - { - return msg >= 0 && msg < _messages.length ? _messages[msg] : ""; - } - - public void setMsg(int m) - { - msg = m; - } - - public boolean isDelete() - { - return delete; - } - - public void setDelete(boolean delete) - { - this.delete = delete; - } - - public boolean isStart() - { - return start; - } - - public void setStart(boolean start) - { - this.start = start; - } - - public boolean isPause() - { - return pause; - } - - public void setPause(boolean pause) - { - this.pause = pause; - } - - public String getIndexPath() - { - return indexPath; - } - - public void setIndexPath(String indexPath) - { - this.indexPath = indexPath; - } - - public boolean isPath() - { - return _path; - } - - public void setPath(boolean path) - { - _path = path; - } - - public boolean isDirectory() - { - return _directory; - } - - public void setDirectory(boolean directory) - { - _directory = directory; - } - - public String getDirectoryType() - { - return _directoryType; - } - - public void setDirectoryType(String directoryType) - { - _directoryType = directoryType; - } - - public boolean isLimit() - { - return limit; - } - - public void setLimit(boolean limit) - { - this.limit = limit; - } - - public int getFileLimitMB() - { - return fileLimitMB; - } - - public int setFileLimitMB(int fileLimitMB) - { - return this.fileLimitMB = fileLimitMB; - } - } - - - @AdminConsoleAction(AdminOperationsPermission.class) - public static class AdminAction extends FormViewAction - { - @SuppressWarnings("UnusedDeclaration") - public AdminAction() - { - } - - public AdminAction(ViewContext ctx, PageConfig pageConfig) - { - setViewContext(ctx); - setPageConfig(pageConfig); - } - - private int _msgid = 0; - - @Override - public void validateCommand(AdminForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(AdminForm form, boolean reshow, BindException errors) - { - SearchService ss = SearchService.get(); - - if (null == ss) - throw new ConfigurationException("Search is misconfigured"); - - @SuppressWarnings({"ThrowableResultOfMethodCallIgnored"}) - Throwable t = ss.getConfigurationError(); - - VBox vbox = new VBox(); - - if (null != t) - { - HtmlStringBuilder builder = HtmlStringBuilder.of(HtmlString.unsafe("Your search index is misconfigured. Search is disabled and documents are not being indexed, pending resolution of this issue. See below for details about the cause of the problem.

    ")); - builder.append(ExceptionUtil.renderException(t)); - WebPartView configErrorView = new HtmlView(builder); - configErrorView.setTitle("Search Configuration Error"); - configErrorView.setFrame(WebPartView.FrameType.PORTAL); - vbox.addView(configErrorView); - } - - // Spring errors get displayed in the "Index Configuration" pane - WebPartView indexerView = new JspView<>("/org/labkey/search/view/indexerAdmin.jsp", form, errors); - indexerView.setTitle("Index Configuration"); - vbox.addView(indexerView); - - // Won't be able to gather statistics if the search index is misconfigured - if (null == t) - { - WebPartView indexerStatsView = new JspView<>("/org/labkey/search/view/indexerStats.jsp", form); - indexerStatsView.setTitle("Index Statistics"); - vbox.addView(indexerStatsView); - } - - WebPartView searchStatsView = new JspView<>("/org/labkey/search/view/searchStats.jsp", form); - searchStatsView.setTitle("Search Statistics"); - vbox.addView(searchStatsView); - - return vbox; - } - - @Override - public boolean handlePost(AdminForm form, BindException errors) - { - SearchService ss = SearchService.get(); - if (null == ss) - { - errors.reject(ERROR_MSG, "Indexing service is not running"); - return false; - } - - if (form.isStart()) - { - SearchPropertyManager.setCrawlerRunningState(getUser(), CrawlerRunningState.Start); - ss.startCrawler(); - } - else if (form.isPause()) - { - SearchPropertyManager.setCrawlerRunningState(getUser(), CrawlerRunningState.Pause); - ss.pauseCrawler(); - } - else if (form.isDelete()) - { - ss.deleteIndex("a site admin requested it"); - ss.start(); - SearchPropertyManager.audit(getUser(), "Index Deleted"); - _msgid = 1; - } - else if (form.isPath()) - { - SearchPropertyManager.setIndexPath(getUser(), form.getIndexPath()); - ss.updateIndex(); - _msgid = 2; - } - else if (form.isDirectory()) - { - LuceneDirectoryType type = EnumUtils.getEnum(LuceneDirectoryType.class, form.getDirectoryType()); - if (null == type) - { - errors.reject(ERROR_MSG, "Unrecognized value for \"directoryType\": \"" + form.getDirectoryType() + "\""); - return false; - } - SearchPropertyManager.setDirectoryType(getUser(), type); - ss.resetIndex(); - _msgid = 3; - } - else if (form.isLimit()) - { - SearchPropertyManager.setFileSizeLimitMB(getUser(), form.getFileLimitMB()); - ss.resetIndex(); - _msgid = 4; - } - - return true; - } - - @Override - public URLHelper getSuccessURL(AdminForm o) - { - ActionURL success = new ActionURL(AdminAction.class, getContainer()); - if (0 != _msgid) - success.addParameter("msg", String.valueOf(_msgid)); - return success; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("searchAdmin"); - urlProvider(AdminUrls.class).addAdminNavTrail(root, "Full-Text Search Configuration", getClass(), getContainer()); - } - } - - - @AdminConsoleAction - public class IndexContentsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/search/view/exportContents.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - new AdminAction(getViewContext(), getPageConfig()).addNavTrail(root); - root.addChild("Index Contents"); - } - } - - - public static class ExportForm - { - private String _format = "Text"; - - public String getFormat() - { - return _format; - } - - public void setFormat(String format) - { - _format = format; - } - } - - - @AdminConsoleAction - public static class ExportIndexContentsAction extends ExportAction - { - @Override - public void export(ExportForm form, HttpServletResponse response, BindException errors) throws Exception - { - new IndexInspector().export(response, form.getFormat()); - } - } - - - /** for selenium testing */ - @RequiresSiteAdmin - public class WaitForIdleAction extends SimpleRedirectAction - { - @Override - public URLHelper getRedirectURL(Object o) throws Exception - { - SearchService ss = AbstractSearchService.get(); - ss.waitForIdle(); - return new ActionURL(AdminAction.class, getContainer()); - } - } - - // UNDONE: remove; for testing only - @RequiresSiteAdmin - public class CancelAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(ReturnUrlForm form) - { - SearchService ss = SearchService.get(); - SearchService.IndexTask defaultTask = ss.defaultTask(); - for (SearchService.IndexTask task : ss.getTasks()) - { - if (task != defaultTask && !task.isCancelled()) - task.cancel(true); - } - - return form.getReturnActionURL(getSearchURL()); - } - } - - - // UNDONE: remove; for testing only - // cause the current directory to be crawled soon - @RequiresSiteAdmin - public class CrawlAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(ReturnUrlForm form) - { - SearchService ss = SearchService.get(); - - ss.addPathToCrawl( - WebdavService.getPath().append(getContainer().getParsedPath()), - new Date(System.currentTimeMillis())); - - return form.getReturnActionURL(getSearchURL()); - } - } - - public static class IndexForm extends ReturnUrlForm - { - boolean _full = false; - boolean _wait = false; - boolean _since = false; - - public boolean isFull() - { - return _full; - } - - @SuppressWarnings("unused") - public void setFull(boolean full) - { - _full = full; - } - - public boolean isWait() - { - return _wait; - } - - @SuppressWarnings("unused") - public void setWait(boolean wait) - { - _wait = wait; - } - - public boolean isSince() - { - return _since; - } - - @SuppressWarnings("unused") - public void setSince(boolean since) - { - _since = since; - } - } - - // for testing only - @RequiresSiteAdmin - public class IndexAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(IndexForm form) throws Exception - { - SearchService ss = SearchService.get(); - - SearchService.IndexTask task = null; - - try (var ignored = SpringActionController.ignoreSqlUpdates()) - { - if (form.isFull()) - { - ss.indexFull(true, "a site admin requested it"); - } - else if (form.isSince()) - { - task = ss.indexContainer(null, getContainer(), new Date(System.currentTimeMillis()- TimeUnit.DAYS.toMillis(1))); - } - else - { - task = ss.indexContainer(null, getContainer(), null); - } - } - - if (form.isWait() && null != task) - { - task.get(); // wait for completion - if (ss instanceof AbstractSearchService) - ((AbstractSearchService)ss).commit(); - } - - return form.getReturnActionURL(getSearchURL()); - } - } - - @RequiresPermission(ReadPermission.class) - public class JsonAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(SearchForm form, BindException errors) - { - SearchService ss = SearchService.get(); - - audit(form); - - final Path contextPath = Path.parse(getViewContext().getContextPath()); - - final String query = form.getQueryString() - .replaceAll("(? hits = result.hits; - totalHits = result.totalHits; - - arr = new Object[hits.size()]; - - int i = 0; - int batchSize = 1000; - Map> docDataMap = new HashMap<>(); - for (int ind = 0; ind < hits.size(); ind++) - { - SearchService.SearchHit hit = hits.get(ind); - JSONObject o = new JSONObject(); - String id = StringUtils.isEmpty(hit.docid) ? String.valueOf(i) : hit.docid; - - o.put("id", id); - o.put("title", hit.title); - o.put("container", hit.container); - o.put("url", form.isNormalizeUrls() ? hit.normalizeHref(contextPath) : hit.url); - o.put("summary", StringUtils.trimToEmpty(hit.summary)); - o.put("score", hit.score); - o.put("identifiers", hit.identifiers); - o.put("category", StringUtils.trimToEmpty(hit.category)); - - if (form.isExperimentalCustomJson()) - { - o.put("jsonData", hit.jsonData); - - if (ind % batchSize == 0) - { - int batchEnd = Math.min(hits.size(), ind + batchSize); - List docIds = new ArrayList<>(); - for (int j = ind; j < batchEnd; j++) - docIds.add(hits.get(j).docid); - - docDataMap = ss.getCustomSearchJsonMap(getUser(), docIds); - } - - Map custom = docDataMap.get(hit.docid); - if (custom != null) - o.put("data", custom); - } - - arr[i++] = o; - } - } - - JSONObject metaData = new JSONObject(); - metaData.put("idProperty","id"); - metaData.put("root", "hits"); - metaData.put("successProperty", "success"); - - response.put("metaData", metaData); - response.put("success",true); - response.put("hits", arr); - response.put("totalHits", totalHits); - response.put("q", query); - - return new ApiSimpleResponse(response); - } - } - - - @RequiresPermission(ReadPermission.class) - public static class TestJson extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/search/view/testJson.jsp", null, null); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - - public static ActionURL getSearchURL(Container c) - { - return new ActionURL(SearchAction.class, c); - } - - private ActionURL getSearchURL() - { - return getSearchURL(getContainer()); - } - - private static ActionURL getSearchURL(Container c, @Nullable String queryString) - { - return getSearchURL(c, queryString, null, null); - } - - private static ActionURL getSearchURL(Container c, @Nullable String queryString, @Nullable String category, @Nullable String template) - { - ActionURL url = getSearchURL(c); - - if (null != queryString) - url.addParameter("q", queryString); - - if (null != category) - url.addParameter("category", category); - - if (null != template) - url.addParameter("template", template); - - return url; - } - - // This interface used to be used to hide all the specifics of internal vs. external index search, but we no longer support external indexes. This interface could be removed. - public interface SearchConfiguration - { - ActionURL getPostURL(Container c); // Search does not actually post - String getDescription(Container c); - SearchResult getSearchResult(String queryString, @Nullable String category, User user, Container currentContainer, SearchScope scope, @Nullable String sortField, int offset, int limit, boolean invertSort) throws IOException; - boolean includeAdvancedUI(); - boolean includeNavigationLinks(); - } - - - public static class InternalSearchConfiguration implements SearchConfiguration - { - private final SearchService _ss = SearchService.get(); - - private InternalSearchConfiguration() - { - } - - @Override - public ActionURL getPostURL(Container c) - { - return getSearchURL(c); - } - - @Override - public String getDescription(Container c) - { - return LookAndFeelProperties.getInstance(c).getShortName(); - } - - @Override - public SearchResult getSearchResult(String queryString, @Nullable String category, User user, Container currentContainer, SearchScope scope, String sortField, int offset, int limit, boolean invertSort) throws IOException - { - SearchService.SearchOptions.Builder options = new SearchService.SearchOptions.Builder(queryString, user, currentContainer); - options.categories = _ss.getCategories(category); - options.invertResults = invertSort; - options.limit = limit; - options.offset = offset; - options.scope = scope; - options.sortField = sortField; - - return _ss.search(options.build()); - } - - @Override - public boolean includeAdvancedUI() - { - return true; - } - - @Override - public boolean includeNavigationLinks() - { - return true; - } - } - - - @RequiresPermission(ReadPermission.class) - public class SearchAction extends SimpleViewAction - { - private String _category = null; - private SearchScope _scope = null; - private SearchForm _form = null; - - @Override - public ModelAndView getView(SearchForm form, BindException errors) - { - _category = form.getCategory(); - _scope = form.getSearchScope(); - _form = form; - - SearchService ss = SearchService.get(); - - if (null == _scope || null == _scope.getRoot(getContainer())) - { - throw new NotFoundException(); - } - - form.setPrint(isPrint()); - - audit(form); - - // reenable caching for search results page (fast browser back button) - HttpServletResponse response = getViewContext().getResponse(); - ResponseHelper.setPrivate(response, Duration.ofMinutes(5)); - getPageConfig().setNoIndex(); - setHelpTopic("luceneSearch"); - - return new JspView<>("/org/labkey/search/view/search.jsp", form); - } - - @Override - public void addNavTrail(NavTree root) - { - _form.getSearchResultTemplate().addNavTrail(root, getViewContext(), _scope, _category); - } - } - - public static class PriorityForm - { - SearchService.PRIORITY priority = SearchService.PRIORITY.modified; - - public SearchService.PRIORITY getPriority() - { - return priority; - } - - public void setPriority(SearchService.PRIORITY priority) - { - this.priority = Objects.requireNonNullElse(priority, SearchService.PRIORITY.modified); - } - } - - // This is intended to help test search indexing. This action sticks a special runnable in the indexer queue - // and then returns when that runnable is executed (or if five minutes goes by without the runnable executing). - // The tests can invoke this action to ensure that the indexer has executed all previous indexing tasks. It - // does not guarantee that all indexed content has been committed... but that may not be required in practice. - - @RequiresPermission(ApplicationAdminPermission.class) - public static class WaitForIndexerAction extends ExportAction - { - @Override - public void export(PriorityForm form, HttpServletResponse response, BindException errors) throws Exception - { - SearchService ss = SearchService.get(); - long startTime = System.currentTimeMillis(); - boolean success = ss.drainQueue(form.getPriority(), 5, TimeUnit.MINUTES); - - LOG.info("Spent {}ms draining the search indexer queue at priority {}. Success: {}", System.currentTimeMillis() - startTime, form.getPriority(), success); - - // Return an error if we time out - if (!success) - response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR); - } - } - - @RequiresPermission(ReadPermission.class) - public class CommentAction extends FormHandlerAction - { - @Override - public void validateCommand(SearchForm target, Errors errors) - { - } - - @Override - public boolean handlePost(SearchForm searchForm, BindException errors) - { - audit(searchForm); - return true; - } - - @Override - public URLHelper getSuccessURL(SearchForm searchForm) - { - return getSearchURL(); - } - } - - - public static class SearchForm - { - private String[] _query; - private String _sortField; - private boolean _print = false; - private int _offset = 0; - private int _limit = 1000; - private String _category = null; - private String _comment = null; - private int _textBoxWidth = 50; // default size - private List _fields; - private boolean _includeHelpLink = true; - private boolean _webpart = false; - private boolean _showAdvanced = false; - private boolean _invertSort = false; - private SearchConfiguration _config = new InternalSearchConfiguration(); // Assume internal search (for webparts, etc.) - private String _template = null; - private SearchScope _scope = SearchScope.All; - private boolean _normalizeUrls = false; - private boolean _experimentalCustomJson = false; - - public void setConfiguration(SearchConfiguration config) - { - _config = config; - } - - public SearchConfiguration getConfig() - { - return _config; - } - - public String[] getQ() - { - return null == _query ? new String[0] : _query; - } - - public String getQueryString() - { - if (null == _query || _query.length == 0) - return ""; - - return StringUtils.join(_query, " "); - } - - public void setQ(String[] query) - { - _query = query; - } - - public String getSortField() - { - return _sortField; - } - - public void setSortField(String sortField) - { - _sortField = sortField; - } - - public boolean isPrint() - { - return _print; - } - - public void setPrint(boolean print) - { - _print = print; - } - - public int getOffset() - { - return _offset; - } - - public void setOffset(int o) - { - _offset = o; - } - - public int getLimit() - { - return _limit; - } - - public void setLimit(int o) - { - _limit = o; - } - - public String getScope() - { - return _scope.name(); - } - - public void setScope(String scope) - { - try - { - _scope = SearchScope.valueOf(scope); - } - catch (IllegalArgumentException e) - { - _scope = SearchScope.All; - } - } - - public SearchScope getSearchScope() - { - return _scope; - } - - public String getCategory() - { - return _category; - } - - public void setCategory(String category) - { - _category = category; - } - - public String getComment() - { - return _comment; - } - - public void setComment(String comment) - { - _comment = comment; - } - - public int getTextBoxWidth() - { - return _textBoxWidth; - } - - public void setTextBoxWidth(int textBoxWidth) - { - _textBoxWidth = textBoxWidth; - } - - public boolean getIncludeHelpLink() - { - return _includeHelpLink; - } - - public void setIncludeHelpLink(boolean includeHelpLink) - { - _includeHelpLink = includeHelpLink; - } - - public boolean isWebPart() - { - return _webpart; - } - - public void setWebPart(boolean webpart) - { - _webpart = webpart; - } - - public boolean isShowAdvanced() - { - return _showAdvanced; - } - - public void setShowAdvanced(boolean showAdvanced) - { - _showAdvanced = showAdvanced; - } - - public boolean isInvertSort() - { - return _invertSort; - } - - public void setInvertSort(boolean invertSort) - { - _invertSort = invertSort; - } - - public String getTemplate() - { - return _template; - } - - public void setTemplate(String template) - { - _template = template; - } - - public SearchResultTemplate getSearchResultTemplate() - { - SearchService ss = AbstractSearchService.get(); - return ss.getSearchResultTemplate(getTemplate()); - } - - public boolean isNormalizeUrls() - { - return _normalizeUrls; - } - - public void setNormalizeUrls(boolean normalizeUrls) - { - _normalizeUrls = normalizeUrls; - } - - public boolean isExperimentalCustomJson() - { - return _experimentalCustomJson; - } - - public void setExperimentalCustomJson(boolean experimentalCustomJson) - { - _experimentalCustomJson = experimentalCustomJson; - } - - public List getFields() - { - return _fields; - } - - public void setFields(List fields) - { - _fields = fields; - } - } - - - protected void audit(SearchForm form) - { - ViewContext ctx = getViewContext(); - String comment = form.getComment(); - - audit(ctx.getUser(), ctx.getContainer(), form.getQueryString(), comment); - } - - - public static void audit(@Nullable User user, @Nullable Container c, String query, String comment) - { - if ((null != user && user.isSearchUser()) || StringUtils.isEmpty(query)) - return; - - AuditLogService audit = AuditLogService.get(); - if (null == audit) - return; - - if (null == c) - c = ContainerManager.getRoot(); - - if (query.length() > 200) - query = query.substring(0, 197) + "..."; - - SearchAuditProvider.SearchAuditEvent event = new SearchAuditProvider.SearchAuditEvent(c, comment); - event.setQuery(query); - - AuditLogService.get().addEvent(user, event); - } - - - public static class SearchSettingsForm - { - private boolean _searchable; - - public boolean isSearchable() - { - return _searchable; - } - - @SuppressWarnings("unused") - public void setSearchable(boolean searchable) - { - _searchable = searchable; - } - } - - - @RequiresPermission(AdminPermission.class) - public static class SearchSettingsAction extends FolderManagementViewPostAction - { - @Override - protected JspView getTabView(SearchSettingsForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/search/view/fullTextSearch.jsp", form, errors); - } - - @Override - public void validateCommand(SearchSettingsForm form, Errors errors) - { - } - - @Override - public boolean handlePost(SearchSettingsForm form, BindException errors) - { - Container container = getContainer(); - if (container.isRoot()) - { - throw new NotFoundException(); - } - - ContainerManager.updateSearchable(container, form.isSearchable(), getUser()); - - return true; - } - - @Override - public URLHelper getSuccessURL(SearchSettingsForm searchForm) - { - // In this case, must redirect back to view so Container is reloaded (simple reshow will continue to show the old value) - return getViewContext().getActionURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - super.addNavTrail(root); - setHelpTopic("searchAdmin"); - } - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.search; + +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.action.ApiResponse; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.ExportAction; +import org.labkey.api.action.FormHandlerAction; +import org.labkey.api.action.FormViewAction; +import org.labkey.api.action.ReadOnlyApiAction; +import org.labkey.api.action.ReturnUrlForm; +import org.labkey.api.action.SimpleRedirectAction; +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AdminUrls; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.search.SearchResultTemplate; +import org.labkey.api.search.SearchScope; +import org.labkey.api.search.SearchService; +import org.labkey.api.search.SearchService.SearchResult; +import org.labkey.api.search.SearchUrls; +import org.labkey.api.security.AdminConsoleAction; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.RequiresSiteAdmin; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminOperationsPermission; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.ApplicationAdminPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.HtmlStringBuilder; +import org.labkey.api.util.Path; +import org.labkey.api.util.ResponseHelper; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.FolderManagement.FolderManagementViewPostAction; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.VBox; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.PageConfig; +import org.labkey.api.webdav.WebdavService; +import org.labkey.search.audit.SearchAuditProvider; +import org.labkey.search.model.AbstractSearchService; +import org.labkey.search.model.CrawlerRunningState; +import org.labkey.search.model.IndexInspector; +import org.labkey.search.model.LuceneDirectoryType; +import org.labkey.search.model.SearchPropertyManager; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.servlet.ModelAndView; + +import java.io.IOException; +import java.sql.Date; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +public class SearchController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(SearchController.class); + + private static final Logger LOG = LogHelper.getLogger(SearchController.class, "Search UI and admin"); + + public SearchController() + { + setActionResolver(_actionResolver); + } + + + @SuppressWarnings("unused") + public static class SearchUrlsImpl implements SearchUrls + { + @Override + public ActionURL getSearchURL(Container c, @Nullable String query) + { + return SearchController.getSearchURL(c, query); + } + + @Override + public ActionURL getSearchURL(String query, String category) + { + return SearchController.getSearchURL(ContainerManager.getRoot(), query, category, null); + } + + @Override + public ActionURL getSearchURL(Container c, @Nullable String query, @NotNull String template) + { + return SearchController.getSearchURL(c, query, null, template); + } + } + + + @RequiresPermission(ReadPermission.class) + public class BeginAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(Object o) + { + return getSearchURL(); + } + } + + + public static class AdminForm + { + public String[] _messages = {"", "Index deleted", "Index path changed", "Directory type changed", "File size limit changed"}; + private int msg = 0; + private boolean pause; + private boolean start; + private boolean delete; + private String indexPath; + + private boolean limit; + private int fileLimitMB; + + private boolean _path; + + private boolean _directory; + private String _directoryType; + + public String getMessage() + { + return msg >= 0 && msg < _messages.length ? _messages[msg] : ""; + } + + public void setMsg(int m) + { + msg = m; + } + + public boolean isDelete() + { + return delete; + } + + public void setDelete(boolean delete) + { + this.delete = delete; + } + + public boolean isStart() + { + return start; + } + + public void setStart(boolean start) + { + this.start = start; + } + + public boolean isPause() + { + return pause; + } + + public void setPause(boolean pause) + { + this.pause = pause; + } + + public String getIndexPath() + { + return indexPath; + } + + public void setIndexPath(String indexPath) + { + this.indexPath = indexPath; + } + + public boolean isPath() + { + return _path; + } + + public void setPath(boolean path) + { + _path = path; + } + + public boolean isDirectory() + { + return _directory; + } + + public void setDirectory(boolean directory) + { + _directory = directory; + } + + public String getDirectoryType() + { + return _directoryType; + } + + public void setDirectoryType(String directoryType) + { + _directoryType = directoryType; + } + + public boolean isLimit() + { + return limit; + } + + public void setLimit(boolean limit) + { + this.limit = limit; + } + + public int getFileLimitMB() + { + return fileLimitMB; + } + + public int setFileLimitMB(int fileLimitMB) + { + return this.fileLimitMB = fileLimitMB; + } + } + + + @AdminConsoleAction(AdminOperationsPermission.class) + public static class AdminAction extends FormViewAction + { + @SuppressWarnings("UnusedDeclaration") + public AdminAction() + { + } + + public AdminAction(ViewContext ctx, PageConfig pageConfig) + { + setViewContext(ctx); + setPageConfig(pageConfig); + } + + private int _msgid = 0; + + @Override + public void validateCommand(AdminForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(AdminForm form, boolean reshow, BindException errors) + { + SearchService ss = SearchService.get(); + @SuppressWarnings({"ThrowableResultOfMethodCallIgnored"}) + Throwable t = ss.getConfigurationError(); + + VBox vbox = new VBox(); + + if (null != t) + { + HtmlStringBuilder builder = HtmlStringBuilder.of(HtmlString.unsafe("Your search index is misconfigured. Search is disabled and documents are not being indexed, pending resolution of this issue. See below for details about the cause of the problem.

    ")); + builder.append(ExceptionUtil.renderException(t)); + WebPartView configErrorView = new HtmlView(builder); + configErrorView.setTitle("Search Configuration Error"); + configErrorView.setFrame(WebPartView.FrameType.PORTAL); + vbox.addView(configErrorView); + } + + // Spring errors get displayed in the "Index Configuration" pane + WebPartView indexerView = new JspView<>("/org/labkey/search/view/indexerAdmin.jsp", form, errors); + indexerView.setTitle("Index Configuration"); + vbox.addView(indexerView); + + // Won't be able to gather statistics if the search index is misconfigured + if (null == t) + { + WebPartView indexerStatsView = new JspView<>("/org/labkey/search/view/indexerStats.jsp", form); + indexerStatsView.setTitle("Index Statistics"); + vbox.addView(indexerStatsView); + } + + WebPartView searchStatsView = new JspView<>("/org/labkey/search/view/searchStats.jsp", form); + searchStatsView.setTitle("Search Statistics"); + vbox.addView(searchStatsView); + + return vbox; + } + + @Override + public boolean handlePost(AdminForm form, BindException errors) + { + SearchService ss = SearchService.get(); + + if (form.isStart()) + { + SearchPropertyManager.setCrawlerRunningState(getUser(), CrawlerRunningState.Start); + ss.startCrawler(); + } + else if (form.isPause()) + { + SearchPropertyManager.setCrawlerRunningState(getUser(), CrawlerRunningState.Pause); + ss.pauseCrawler(); + } + else if (form.isDelete()) + { + ss.deleteIndex("a site admin requested it"); + ss.start(); + SearchPropertyManager.audit(getUser(), "Index Deleted"); + _msgid = 1; + } + else if (form.isPath()) + { + SearchPropertyManager.setIndexPath(getUser(), form.getIndexPath()); + ss.updateIndex(); + _msgid = 2; + } + else if (form.isDirectory()) + { + LuceneDirectoryType type = EnumUtils.getEnum(LuceneDirectoryType.class, form.getDirectoryType()); + if (null == type) + { + errors.reject(ERROR_MSG, "Unrecognized value for \"directoryType\": \"" + form.getDirectoryType() + "\""); + return false; + } + SearchPropertyManager.setDirectoryType(getUser(), type); + ss.resetIndex(); + _msgid = 3; + } + else if (form.isLimit()) + { + SearchPropertyManager.setFileSizeLimitMB(getUser(), form.getFileLimitMB()); + ss.resetIndex(); + _msgid = 4; + } + + return true; + } + + @Override + public URLHelper getSuccessURL(AdminForm o) + { + ActionURL success = new ActionURL(AdminAction.class, getContainer()); + if (0 != _msgid) + success.addParameter("msg", String.valueOf(_msgid)); + return success; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("searchAdmin"); + urlProvider(AdminUrls.class).addAdminNavTrail(root, "Full-Text Search Configuration", getClass(), getContainer()); + } + } + + + @AdminConsoleAction + public class IndexContentsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/search/view/exportContents.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + new AdminAction(getViewContext(), getPageConfig()).addNavTrail(root); + root.addChild("Index Contents"); + } + } + + + public static class ExportForm + { + private String _format = "Text"; + + public String getFormat() + { + return _format; + } + + public void setFormat(String format) + { + _format = format; + } + } + + + @AdminConsoleAction + public static class ExportIndexContentsAction extends ExportAction + { + @Override + public void export(ExportForm form, HttpServletResponse response, BindException errors) throws Exception + { + new IndexInspector().export(response, form.getFormat()); + } + } + + + /** for selenium testing */ + @RequiresSiteAdmin + public class WaitForIdleAction extends SimpleRedirectAction + { + @Override + public URLHelper getRedirectURL(Object o) throws Exception + { + SearchService ss = AbstractSearchService.get(); + ss.waitForIdle(); + return new ActionURL(AdminAction.class, getContainer()); + } + } + + // UNDONE: remove; for testing only + @RequiresSiteAdmin + public class CancelAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(ReturnUrlForm form) + { + SearchService ss = SearchService.get(); + SearchService.IndexTask defaultTask = ss.defaultTask(); + for (SearchService.IndexTask task : ss.getTasks()) + { + if (task != defaultTask && !task.isCancelled()) + task.cancel(true); + } + + return form.getReturnActionURL(getSearchURL()); + } + } + + + // UNDONE: remove; for testing only + // cause the current directory to be crawled soon + @RequiresSiteAdmin + public class CrawlAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(ReturnUrlForm form) + { + SearchService ss = SearchService.get(); + + ss.addPathToCrawl( + WebdavService.getPath().append(getContainer().getParsedPath()), + new Date(System.currentTimeMillis())); + + return form.getReturnActionURL(getSearchURL()); + } + } + + public static class IndexForm extends ReturnUrlForm + { + boolean _full = false; + boolean _wait = false; + boolean _since = false; + + public boolean isFull() + { + return _full; + } + + @SuppressWarnings("unused") + public void setFull(boolean full) + { + _full = full; + } + + public boolean isWait() + { + return _wait; + } + + @SuppressWarnings("unused") + public void setWait(boolean wait) + { + _wait = wait; + } + + public boolean isSince() + { + return _since; + } + + @SuppressWarnings("unused") + public void setSince(boolean since) + { + _since = since; + } + } + + // for testing only + @RequiresSiteAdmin + public class IndexAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(IndexForm form) throws Exception + { + SearchService ss = SearchService.get(); + + SearchService.IndexTask task = null; + + try (var ignored = SpringActionController.ignoreSqlUpdates()) + { + if (form.isFull()) + { + ss.indexFull(true, "a site admin requested it"); + } + else if (form.isSince()) + { + task = ss.indexContainer(null, getContainer(), new Date(System.currentTimeMillis()- TimeUnit.DAYS.toMillis(1))); + } + else + { + task = ss.indexContainer(null, getContainer(), null); + } + } + + if (form.isWait() && null != task) + { + task.get(); // wait for completion + if (ss instanceof AbstractSearchService) + ((AbstractSearchService)ss).commit(); + } + + return form.getReturnActionURL(getSearchURL()); + } + } + + @RequiresPermission(ReadPermission.class) + public class JsonAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(SearchForm form, BindException errors) + { + SearchService ss = SearchService.get(); + + audit(form); + + final Path contextPath = Path.parse(getViewContext().getContextPath()); + + final String query = form.getQueryString() + .replaceAll("(? hits = result.hits; + totalHits = result.totalHits; + + arr = new Object[hits.size()]; + + int i = 0; + int batchSize = 1000; + Map> docDataMap = new HashMap<>(); + for (int ind = 0; ind < hits.size(); ind++) + { + SearchService.SearchHit hit = hits.get(ind); + JSONObject o = new JSONObject(); + String id = StringUtils.isEmpty(hit.docid) ? String.valueOf(i) : hit.docid; + + o.put("id", id); + o.put("title", hit.title); + o.put("container", hit.container); + o.put("url", form.isNormalizeUrls() ? hit.normalizeHref(contextPath) : hit.url); + o.put("summary", StringUtils.trimToEmpty(hit.summary)); + o.put("score", hit.score); + o.put("identifiers", hit.identifiers); + o.put("category", StringUtils.trimToEmpty(hit.category)); + + if (form.isExperimentalCustomJson()) + { + o.put("jsonData", hit.jsonData); + + if (ind % batchSize == 0) + { + int batchEnd = Math.min(hits.size(), ind + batchSize); + List docIds = new ArrayList<>(); + for (int j = ind; j < batchEnd; j++) + docIds.add(hits.get(j).docid); + + docDataMap = ss.getCustomSearchJsonMap(getUser(), docIds); + } + + Map custom = docDataMap.get(hit.docid); + if (custom != null) + o.put("data", custom); + } + + arr[i++] = o; + } + } + + JSONObject metaData = new JSONObject(); + metaData.put("idProperty","id"); + metaData.put("root", "hits"); + metaData.put("successProperty", "success"); + + response.put("metaData", metaData); + response.put("success",true); + response.put("hits", arr); + response.put("totalHits", totalHits); + response.put("q", query); + + return new ApiSimpleResponse(response); + } + } + + + @RequiresPermission(ReadPermission.class) + public static class TestJson extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/search/view/testJson.jsp", null, null); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + + public static ActionURL getSearchURL(Container c) + { + return new ActionURL(SearchAction.class, c); + } + + private ActionURL getSearchURL() + { + return getSearchURL(getContainer()); + } + + private static ActionURL getSearchURL(Container c, @Nullable String queryString) + { + return getSearchURL(c, queryString, null, null); + } + + private static ActionURL getSearchURL(Container c, @Nullable String queryString, @Nullable String category, @Nullable String template) + { + ActionURL url = getSearchURL(c); + + if (null != queryString) + url.addParameter("q", queryString); + + if (null != category) + url.addParameter("category", category); + + if (null != template) + url.addParameter("template", template); + + return url; + } + + // This interface used to be used to hide all the specifics of internal vs. external index search, but we no longer support external indexes. This interface could be removed. + public interface SearchConfiguration + { + ActionURL getPostURL(Container c); // Search does not actually post + String getDescription(Container c); + SearchResult getSearchResult(String queryString, @Nullable String category, User user, Container currentContainer, SearchScope scope, @Nullable String sortField, int offset, int limit, boolean invertSort) throws IOException; + boolean includeAdvancedUI(); + boolean includeNavigationLinks(); + } + + + public static class InternalSearchConfiguration implements SearchConfiguration + { + private final SearchService _ss = SearchService.get(); + + private InternalSearchConfiguration() + { + } + + @Override + public ActionURL getPostURL(Container c) + { + return getSearchURL(c); + } + + @Override + public String getDescription(Container c) + { + return LookAndFeelProperties.getInstance(c).getShortName(); + } + + @Override + public SearchResult getSearchResult(String queryString, @Nullable String category, User user, Container currentContainer, SearchScope scope, String sortField, int offset, int limit, boolean invertSort) throws IOException + { + SearchService.SearchOptions.Builder options = new SearchService.SearchOptions.Builder(queryString, user, currentContainer); + options.categories = _ss.getCategories(category); + options.invertResults = invertSort; + options.limit = limit; + options.offset = offset; + options.scope = scope; + options.sortField = sortField; + + return _ss.search(options.build()); + } + + @Override + public boolean includeAdvancedUI() + { + return true; + } + + @Override + public boolean includeNavigationLinks() + { + return true; + } + } + + + @RequiresPermission(ReadPermission.class) + public class SearchAction extends SimpleViewAction + { + private String _category = null; + private SearchScope _scope = null; + private SearchForm _form = null; + + @Override + public ModelAndView getView(SearchForm form, BindException errors) + { + _category = form.getCategory(); + _scope = form.getSearchScope(); + _form = form; + + if (null == _scope || null == _scope.getRoot(getContainer())) + { + throw new NotFoundException(); + } + + form.setPrint(isPrint()); + + audit(form); + + // reenable caching for search results page (fast browser back button) + HttpServletResponse response = getViewContext().getResponse(); + ResponseHelper.setPrivate(response, Duration.ofMinutes(5)); + getPageConfig().setNoIndex(); + setHelpTopic("luceneSearch"); + + return new JspView<>("/org/labkey/search/view/search.jsp", form); + } + + @Override + public void addNavTrail(NavTree root) + { + _form.getSearchResultTemplate().addNavTrail(root, getViewContext(), _scope, _category); + } + } + + public static class PriorityForm + { + SearchService.PRIORITY priority = SearchService.PRIORITY.modified; + + public SearchService.PRIORITY getPriority() + { + return priority; + } + + public void setPriority(SearchService.PRIORITY priority) + { + this.priority = Objects.requireNonNullElse(priority, SearchService.PRIORITY.modified); + } + } + + // This is intended to help test search indexing. This action sticks a special runnable in the indexer queue + // and then returns when that runnable is executed (or if five minutes goes by without the runnable executing). + // The tests can invoke this action to ensure that the indexer has executed all previous indexing tasks. It + // does not guarantee that all indexed content has been committed... but that may not be required in practice. + + @RequiresPermission(ApplicationAdminPermission.class) + public static class WaitForIndexerAction extends ExportAction + { + @Override + public void export(PriorityForm form, HttpServletResponse response, BindException errors) throws Exception + { + long startTime = System.currentTimeMillis(); + boolean success = SearchService.get().drainQueue(form.getPriority(), 5, TimeUnit.MINUTES); + + LOG.info("Spent {}ms draining the search indexer queue at priority {}. Success: {}", System.currentTimeMillis() - startTime, form.getPriority(), success); + + // Return an error if we time out + if (!success) + response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR); + } + } + + @RequiresPermission(ReadPermission.class) + public class CommentAction extends FormHandlerAction + { + @Override + public void validateCommand(SearchForm target, Errors errors) + { + } + + @Override + public boolean handlePost(SearchForm searchForm, BindException errors) + { + audit(searchForm); + return true; + } + + @Override + public URLHelper getSuccessURL(SearchForm searchForm) + { + return getSearchURL(); + } + } + + + public static class SearchForm + { + private String[] _query; + private String _sortField; + private boolean _print = false; + private int _offset = 0; + private int _limit = 1000; + private String _category = null; + private String _comment = null; + private int _textBoxWidth = 50; // default size + private List _fields; + private boolean _includeHelpLink = true; + private boolean _webpart = false; + private boolean _showAdvanced = false; + private boolean _invertSort = false; + private SearchConfiguration _config = new InternalSearchConfiguration(); // Assume internal search (for webparts, etc.) + private String _template = null; + private SearchScope _scope = SearchScope.All; + private boolean _normalizeUrls = false; + private boolean _experimentalCustomJson = false; + + public void setConfiguration(SearchConfiguration config) + { + _config = config; + } + + public SearchConfiguration getConfig() + { + return _config; + } + + public String[] getQ() + { + return null == _query ? new String[0] : _query; + } + + public String getQueryString() + { + if (null == _query || _query.length == 0) + return ""; + + return StringUtils.join(_query, " "); + } + + public void setQ(String[] query) + { + _query = query; + } + + public String getSortField() + { + return _sortField; + } + + public void setSortField(String sortField) + { + _sortField = sortField; + } + + public boolean isPrint() + { + return _print; + } + + public void setPrint(boolean print) + { + _print = print; + } + + public int getOffset() + { + return _offset; + } + + public void setOffset(int o) + { + _offset = o; + } + + public int getLimit() + { + return _limit; + } + + public void setLimit(int o) + { + _limit = o; + } + + public String getScope() + { + return _scope.name(); + } + + public void setScope(String scope) + { + try + { + _scope = SearchScope.valueOf(scope); + } + catch (IllegalArgumentException e) + { + _scope = SearchScope.All; + } + } + + public SearchScope getSearchScope() + { + return _scope; + } + + public String getCategory() + { + return _category; + } + + public void setCategory(String category) + { + _category = category; + } + + public String getComment() + { + return _comment; + } + + public void setComment(String comment) + { + _comment = comment; + } + + public int getTextBoxWidth() + { + return _textBoxWidth; + } + + public void setTextBoxWidth(int textBoxWidth) + { + _textBoxWidth = textBoxWidth; + } + + public boolean getIncludeHelpLink() + { + return _includeHelpLink; + } + + public void setIncludeHelpLink(boolean includeHelpLink) + { + _includeHelpLink = includeHelpLink; + } + + public boolean isWebPart() + { + return _webpart; + } + + public void setWebPart(boolean webpart) + { + _webpart = webpart; + } + + public boolean isShowAdvanced() + { + return _showAdvanced; + } + + public void setShowAdvanced(boolean showAdvanced) + { + _showAdvanced = showAdvanced; + } + + public boolean isInvertSort() + { + return _invertSort; + } + + public void setInvertSort(boolean invertSort) + { + _invertSort = invertSort; + } + + public String getTemplate() + { + return _template; + } + + public void setTemplate(String template) + { + _template = template; + } + + public SearchResultTemplate getSearchResultTemplate() + { + SearchService ss = AbstractSearchService.get(); + return ss.getSearchResultTemplate(getTemplate()); + } + + public boolean isNormalizeUrls() + { + return _normalizeUrls; + } + + public void setNormalizeUrls(boolean normalizeUrls) + { + _normalizeUrls = normalizeUrls; + } + + public boolean isExperimentalCustomJson() + { + return _experimentalCustomJson; + } + + public void setExperimentalCustomJson(boolean experimentalCustomJson) + { + _experimentalCustomJson = experimentalCustomJson; + } + + public List getFields() + { + return _fields; + } + + public void setFields(List fields) + { + _fields = fields; + } + } + + + protected void audit(SearchForm form) + { + ViewContext ctx = getViewContext(); + String comment = form.getComment(); + + audit(ctx.getUser(), ctx.getContainer(), form.getQueryString(), comment); + } + + + public static void audit(@Nullable User user, @Nullable Container c, String query, String comment) + { + if ((null != user && user.isSearchUser()) || StringUtils.isEmpty(query)) + return; + + AuditLogService audit = AuditLogService.get(); + if (null == audit) + return; + + if (null == c) + c = ContainerManager.getRoot(); + + if (query.length() > 200) + query = query.substring(0, 197) + "..."; + + SearchAuditProvider.SearchAuditEvent event = new SearchAuditProvider.SearchAuditEvent(c, comment); + event.setQuery(query); + + AuditLogService.get().addEvent(user, event); + } + + + public static class SearchSettingsForm + { + private boolean _searchable; + + public boolean isSearchable() + { + return _searchable; + } + + @SuppressWarnings("unused") + public void setSearchable(boolean searchable) + { + _searchable = searchable; + } + } + + + @RequiresPermission(AdminPermission.class) + public static class SearchSettingsAction extends FolderManagementViewPostAction + { + @Override + protected JspView getTabView(SearchSettingsForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/search/view/fullTextSearch.jsp", form, errors); + } + + @Override + public void validateCommand(SearchSettingsForm form, Errors errors) + { + } + + @Override + public boolean handlePost(SearchSettingsForm form, BindException errors) + { + Container container = getContainer(); + if (container.isRoot()) + { + throw new NotFoundException(); + } + + ContainerManager.updateSearchable(container, form.isSearchable(), getUser()); + + return true; + } + + @Override + public URLHelper getSuccessURL(SearchSettingsForm searchForm) + { + // In this case, must redirect back to view so Container is reloaded (simple reshow will continue to show the old value) + return getViewContext().getActionURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + super.addNavTrail(root); + setHelpTopic("searchAdmin"); + } + } +} diff --git a/search/src/org/labkey/search/SearchModule.java b/search/src/org/labkey/search/SearchModule.java index a8572dfcb2f..1bf6a6baa37 100644 --- a/search/src/org/labkey/search/SearchModule.java +++ b/search/src/org/labkey/search/SearchModule.java @@ -1,274 +1,262 @@ -/* - * Copyright (c) 2009-2020 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.search; - -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpgradeCode; -import org.labkey.api.mbean.LabKeyManagement; -import org.labkey.api.mbean.SearchMXBean; -import org.labkey.api.module.DefaultModule; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.LimitedUser; -import org.labkey.api.security.User; -import org.labkey.api.security.roles.CanSeeAuditLogRole; -import org.labkey.api.settings.AdminConsole; -import org.labkey.api.settings.StandardStartupPropertyHandler; -import org.labkey.api.settings.StartupPropertyEntry; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.JobRunner; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.FolderManagement; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.api.webdav.WebdavService; -import org.labkey.search.audit.SearchAuditProvider; -import org.labkey.search.model.AbstractSearchService; -import org.labkey.search.model.DavCrawler; -import org.labkey.search.model.LuceneSearchServiceImpl; -import org.labkey.search.model.PlainTextDocumentParser; -import org.labkey.search.model.SearchStartupProperties; -import org.labkey.search.view.SearchWebPartFactory; - -import javax.management.StandardMBean; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.ConcurrentLinkedQueue; - -public class SearchModule extends DefaultModule -{ - private static final Logger LOG = LogHelper.getLogger(SearchModule.class, "Search module startup issues"); - public static final String NAME = "Search"; - - @Override - public String getName() - { - return NAME; - } - - @Override - public Double getSchemaVersion() - { - return 25.001; - } - - @Override - public boolean hasScripts() - { - return true; - } - - @Override - @NotNull - public Set getSchemaNames() - { - return PageFlowUtil.set("search"); - } - - @Override - @NotNull - public Set getSchemasToTest() - { - // Should test the "search" schema, but it differs between SQL Server & PostgreSQL - return Collections.emptySet(); - } - - @Override - @NotNull - protected Collection createWebPartFactories() - { - return List.of(new SearchWebPartFactory()); - } - - @Override - protected void init() - { - addController("search", SearchController.class); - LuceneSearchServiceImpl ss = new LuceneSearchServiceImpl(); - SearchService.setInstance(ss); - - LabKeyManagement.register(new StandardMBean(ss, SearchMXBean.class, true), "Search"); - - ss.addResourceResolver("dav", new AbstractSearchService.ResourceResolver() - { - @Override - public WebdavResource resolve(@NotNull String path) - { - return WebdavService.get().lookup(path); - } - }); - } - - @Override - public void doStartup(ModuleContext moduleContext) - { - ModuleLoader.getInstance().handleStartupProperties( - new StandardStartupPropertyHandler<>("SearchSettings", SearchStartupProperties.class) - { - @Override - public void handle(Map properties) - { - SearchService ss = SearchService.get(); - if (null == ss) - LOG.error("Search service is not present"); - else - properties.forEach((ssp, sp) -> { - try - { - ssp.setProperty(ss, _searchIndexStartupHandler, sp.getValue()); - } - catch (Exception e) - { - LOG.error("Exception while attempting to set startup property", e); - } - }); - } - } - ); - - final SearchService ss = SearchService.get(); - - if (null != ss) - { - AdminConsole.addLink(AdminConsole.SettingsLinkType.Management, "full-text search", new ActionURL(SearchController.AdminAction.class, null)); - - CacheManager.addListener(() -> { - SearchService._log.info("Purging SearchService queues"); - ss.purgeQueues(); - }); - - ss.addDocumentParser(new PlainTextDocumentParser()); - } - - AuditLogService.get().registerAuditType(new SearchAuditProvider()); - - // add a container listener, so we'll know when containers are deleted - ContainerManager.addContainerListener(new SearchContainerListener()); - - FolderManagement.addTab(FolderManagement.TYPE.FolderManagement, "Search", "fullTextSearch", FolderManagement.NOT_ROOT, SearchController.SearchSettingsAction.class); - - UsageMetricsService.get().registerUsageMetrics(getName(), () -> - { - // Report the total number of search entries in the audit log - User user = new LimitedUser(User.getSearchUser(), CanSeeAuditLogRole.class); - UserSchema auditSchema = AuditLogService.get().createSchema(user, ContainerManager.getRoot()); - TableInfo auditTable = auditSchema.getTableOrThrow(SearchAuditProvider.EVENT_TYPE, ContainerFilter.getUnsafeEverythingFilter()); - - long count = new TableSelector(auditTable).getRowCount(); - return Collections.singletonMap("fullTextSearches", count); - }); - } - - @Override - public void startBackgroundThreads() - { - SearchService ss = SearchService.get(); - - if (null != ss) - { - // Execute any reindexing operations in the background to not block startup, Issue #48960 - JobRunner.getDefault().execute(() -> { - _searchIndexStartupHandler.reindexIfNeeded(ss); - ss.start(); - DavCrawler.getInstance().start(); - }); - } - } - - private final SearchIndexStartupHandler _searchIndexStartupHandler = new SearchIndexStartupHandler(); - - public static class SearchIndexStartupHandler - { - private volatile boolean _deleteIndex = false; - private volatile boolean _indexFull = false; - - private final Queue _deleteIndexReasons = new ConcurrentLinkedQueue<>(); - private final Queue _indexFullReasons = new ConcurrentLinkedQueue<>(); - - public void setDeleteIndex(String reason) - { - _deleteIndex = true; - _deleteIndexReasons.add(reason); - } - - public void setIndexFull(String reason) - { - _indexFull = true; - _indexFullReasons.add(reason); - } - - private void reindexIfNeeded(@NotNull SearchService ss) - { - if (_deleteIndex) - { - ss.deleteIndex(_deleteIndexReasons.toString()); - } - if (_indexFull) - { - ss.indexFull(true, _indexFullReasons.toString()); - } - } - } - - @NotNull - @Override - public Set getIntegrationTests() - { - return Set.of - ( - LuceneSearchServiceImpl.TestCase.class, - LuceneSearchServiceImpl.TikaTestCase.class - ); - } - - @Override - public @Nullable UpgradeCode getUpgradeCode() - { - return new SearchUpgradeCode(); - } - - public class SearchUpgradeCode implements UpgradeCode - { - @SuppressWarnings("unused") - public void reindex(ModuleContext context) - { - if (!context.isNewInstall()) - { - // Delete the index, clear last indexed on all documents, and initiate an aggressive reindexing - _searchIndexStartupHandler.setDeleteIndex("initiated by search upgrade code"); - _searchIndexStartupHandler.setIndexFull("initiated by search upgrade code"); - } - } - } -} +/* + * Copyright (c) 2009-2020 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.search; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpgradeCode; +import org.labkey.api.mbean.LabKeyManagement; +import org.labkey.api.mbean.SearchMXBean; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.LimitedUser; +import org.labkey.api.security.User; +import org.labkey.api.security.roles.CanSeeAuditLogRole; +import org.labkey.api.settings.AdminConsole; +import org.labkey.api.settings.StandardStartupPropertyHandler; +import org.labkey.api.settings.StartupPropertyEntry; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.JobRunner; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.FolderManagement; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; +import org.labkey.search.audit.SearchAuditProvider; +import org.labkey.search.model.AbstractSearchService; +import org.labkey.search.model.DavCrawler; +import org.labkey.search.model.LuceneSearchServiceImpl; +import org.labkey.search.model.PlainTextDocumentParser; +import org.labkey.search.model.SearchStartupProperties; +import org.labkey.search.view.SearchWebPartFactory; + +import javax.management.StandardMBean; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; + +public class SearchModule extends DefaultModule +{ + private static final Logger LOG = LogHelper.getLogger(SearchModule.class, "Search module startup issues"); + public static final String NAME = "Search"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public Double getSchemaVersion() + { + return 25.001; + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + public Set getSchemaNames() + { + return PageFlowUtil.set("search"); + } + + @Override + @NotNull + public Set getSchemasToTest() + { + // Should test the "search" schema, but it differs between SQL Server & PostgreSQL + return Collections.emptySet(); + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + return List.of(new SearchWebPartFactory()); + } + + @Override + protected void init() + { + addController("search", SearchController.class); + LuceneSearchServiceImpl ss = new LuceneSearchServiceImpl(); + SearchService.setInstance(ss); + + LabKeyManagement.register(new StandardMBean(ss, SearchMXBean.class, true), "Search"); + + ss.addResourceResolver("dav", new AbstractSearchService.ResourceResolver() + { + @Override + public WebdavResource resolve(@NotNull String path) + { + return WebdavService.get().lookup(path); + } + }); + } + + @Override + public void doStartup(ModuleContext moduleContext) + { + ModuleLoader.getInstance().handleStartupProperties( + new StandardStartupPropertyHandler<>("SearchSettings", SearchStartupProperties.class) + { + @Override + public void handle(Map properties) + { + properties.forEach((ssp, sp) -> { + try + { + ssp.setProperty(SearchService.get(), _searchIndexStartupHandler, sp.getValue()); + } + catch (Exception e) + { + LOG.error("Exception while attempting to set startup property", e); + } + }); + } + } + ); + + final SearchService ss = SearchService.get(); + + AdminConsole.addLink(AdminConsole.SettingsLinkType.Management, "full-text search", new ActionURL(SearchController.AdminAction.class, null)); + + CacheManager.addListener(() -> { + SearchService._log.info("Purging SearchService queues"); + ss.purgeQueues(); + }); + + ss.addDocumentParser(new PlainTextDocumentParser()); + + AuditLogService.get().registerAuditType(new SearchAuditProvider()); + + // add a container listener, so we'll know when containers are deleted + ContainerManager.addContainerListener(new SearchContainerListener()); + + FolderManagement.addTab(FolderManagement.TYPE.FolderManagement, "Search", "fullTextSearch", FolderManagement.NOT_ROOT, SearchController.SearchSettingsAction.class); + + UsageMetricsService.get().registerUsageMetrics(getName(), () -> + { + // Report the total number of search entries in the audit log + User user = new LimitedUser(User.getSearchUser(), CanSeeAuditLogRole.class); + UserSchema auditSchema = AuditLogService.get().createSchema(user, ContainerManager.getRoot()); + TableInfo auditTable = auditSchema.getTableOrThrow(SearchAuditProvider.EVENT_TYPE, ContainerFilter.getUnsafeEverythingFilter()); + + long count = new TableSelector(auditTable).getRowCount(); + return Collections.singletonMap("fullTextSearches", count); + }); + } + + @Override + public void startBackgroundThreads() + { + // Execute any reindexing operations in the background to not block startup, Issue #48960 + JobRunner.getDefault().execute(() -> { + _searchIndexStartupHandler.reindexIfNeeded(ss); + SearchService.get().start(); + DavCrawler.getInstance().start(); + }); + } + + private final SearchIndexStartupHandler _searchIndexStartupHandler = new SearchIndexStartupHandler(); + + public static class SearchIndexStartupHandler + { + private volatile boolean _deleteIndex = false; + private volatile boolean _indexFull = false; + + private final Queue _deleteIndexReasons = new ConcurrentLinkedQueue<>(); + private final Queue _indexFullReasons = new ConcurrentLinkedQueue<>(); + + public void setDeleteIndex(String reason) + { + _deleteIndex = true; + _deleteIndexReasons.add(reason); + } + + public void setIndexFull(String reason) + { + _indexFull = true; + _indexFullReasons.add(reason); + } + + private void reindexIfNeeded(@NotNull SearchService ss) + { + if (_deleteIndex) + { + ss.deleteIndex(_deleteIndexReasons.toString()); + } + if (_indexFull) + { + ss.indexFull(true, _indexFullReasons.toString()); + } + } + } + + @NotNull + @Override + public Set getIntegrationTests() + { + return Set.of + ( + LuceneSearchServiceImpl.TestCase.class, + LuceneSearchServiceImpl.TikaTestCase.class + ); + } + + @Override + public @Nullable UpgradeCode getUpgradeCode() + { + return new SearchUpgradeCode(); + } + + public class SearchUpgradeCode implements UpgradeCode + { + @SuppressWarnings("unused") + public void reindex(ModuleContext context) + { + if (!context.isNewInstall()) + { + // Delete the index, clear last indexed on all documents, and initiate an aggressive reindexing + _searchIndexStartupHandler.setDeleteIndex("initiated by search upgrade code"); + _searchIndexStartupHandler.setIndexFull("initiated by search upgrade code"); + } + } + } +} diff --git a/search/src/org/labkey/search/model/LuceneSearchServiceImpl.java b/search/src/org/labkey/search/model/LuceneSearchServiceImpl.java index 95ecd643575..15d6a9c4dc6 100644 --- a/search/src/org/labkey/search/model/LuceneSearchServiceImpl.java +++ b/search/src/org/labkey/search/model/LuceneSearchServiceImpl.java @@ -1546,7 +1546,7 @@ interface FindHandler } @Override - public WebPartView getSearchView(boolean includeSubfolders, int textBoxWidth, boolean includeHelpLink, boolean isWebpart) + public SearchWebPart getSearchView(boolean includeSubfolders, int textBoxWidth, boolean includeHelpLink, boolean isWebpart) { return new SearchWebPart(includeSubfolders, textBoxWidth, includeHelpLink, isWebpart); } diff --git a/search/src/org/labkey/search/model/SavePaths.java b/search/src/org/labkey/search/model/SavePaths.java index 6c7c3d9b8d8..7ff93620012 100644 --- a/search/src/org/labkey/search/model/SavePaths.java +++ b/search/src/org/labkey/search/model/SavePaths.java @@ -467,11 +467,6 @@ public boolean updateFile(@NotNull Path path, @NotNull Date lastIndexed, Date mo private static DbSchema getSearchSchema() { - SearchService ss = SearchService.get(); - - if (null != ss) - return ss.getSchema(); - else - return null; + return SearchService.get().getSchema(); } } diff --git a/search/src/org/labkey/search/view/indexerAdmin.jsp b/search/src/org/labkey/search/view/indexerAdmin.jsp index e4673db94cf..71524317f2f 100644 --- a/search/src/org/labkey/search/view/indexerAdmin.jsp +++ b/search/src/org/labkey/search/view/indexerAdmin.jsp @@ -45,12 +45,6 @@ String indexDirectoryPath = null != indexDirectory ? indexDirectory.getPath() :

    <%=h(form.getMessage())%>
    <% } -if (null == ss) -{ - %>Indexing service is not configured.<% -} -else -{ HtmlString indexPathHelp = HtmlStringBuilder.of("The index path setting supports string substitution of specific server properties. For example, the value:") .unsafeAppend("

      ./temp/${serverGuid}/labkey_full_text_index

    ") .append("will currently result in this path:") @@ -90,7 +84,7 @@ else Current Index Properties: <% - for (Map.Entry e : ss.getIndexFormatProperties().entrySet()) + for (Map.Entry e : ss.getIndexFormatProperties().entrySet()) { %> @@ -179,6 +173,4 @@ else } %> -

    <% -} -%> \ No newline at end of file +

    \ No newline at end of file diff --git a/search/src/org/labkey/search/view/indexerStats.jsp b/search/src/org/labkey/search/view/indexerStats.jsp index a18e2ad81c2..2a8651c4080 100644 --- a/search/src/org/labkey/search/view/indexerStats.jsp +++ b/search/src/org/labkey/search/view/indexerStats.jsp @@ -27,22 +27,15 @@ <% SearchService ss = SearchService.get(); -if (null == ss) -{ - %>Indexing service is not configured.<% -} -else -{ - %><% + %> +
    <% if (ss instanceof AbstractSearchService) { renderMap(out, ((AbstractSearchService) ss).getIndexerStats()); } renderMap(out, DavCrawler.getInstance().getStats()); - %>
    <% -} -%> + %> <%=link("Export index contents", SearchController.IndexContentsAction.class)%> <%! private void renderMap(JspWriter out, Map m) throws IOException diff --git a/search/src/org/labkey/search/view/searchStats.jsp b/search/src/org/labkey/search/view/searchStats.jsp index f0f13e12041..e0f750f95c5 100644 --- a/search/src/org/labkey/search/view/searchStats.jsp +++ b/search/src/org/labkey/search/view/searchStats.jsp @@ -26,12 +26,6 @@ <% SearchService ss = SearchService.get(); -if (null == ss) -{ - %>Indexing service is not configured.<% -} -else -{ %><% if (ss instanceof AbstractSearchService) { @@ -49,6 +43,4 @@ else <% } } - %>
    <%=label%>  <%=v%> 
    <% -} -%> \ No newline at end of file + %> \ No newline at end of file diff --git a/study/src/org/labkey/study/model/StudyManager.java b/study/src/org/labkey/study/model/StudyManager.java index 22391cb4708..540b3650af8 100644 --- a/study/src/org/labkey/study/model/StudyManager.java +++ b/study/src/org/labkey/study/model/StudyManager.java @@ -1,4960 +1,4958 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.study.model; - -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.ListUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.Constants; -import org.labkey.api.annotations.Migrate; -import org.labkey.api.assay.AssayService; -import org.labkey.api.attachments.Attachment; -import org.labkey.api.attachments.AttachmentParent; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.cache.BlockingCache; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheLoader; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.IntHashMap; -import org.labkey.api.collections.LabKeyCollectors; -import org.labkey.api.compliance.ComplianceFolderSettings; -import org.labkey.api.compliance.ComplianceService; -import org.labkey.api.compliance.PhiColumnBehavior; -import org.labkey.api.data.Activity; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DatabaseCache; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DbScope.CommitTaskOption; -import org.labkey.api.data.DbScope.Transaction; -import org.labkey.api.data.Filter; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.LookupResolutionType; -import org.labkey.api.data.PHI; -import org.labkey.api.data.PropertyManager; -import org.labkey.api.data.PropertyManager.WritablePropertyMap; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SimpleFilter.OrClause; -import org.labkey.api.data.SimpleFilter.SQLClause; -import org.labkey.api.data.Sort; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpdateableTableInfo; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.dataiterator.BeanDataIterator; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DataIteratorUtil; -import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; -import org.labkey.api.dataiterator.Pump; -import org.labkey.api.dataiterator.StandardDataIteratorBuilder; -import org.labkey.api.exceptions.OptimisticConflictException; -import org.labkey.api.exp.ChangePropertyDescriptorException; -import org.labkey.api.exp.DomainDescriptor; -import org.labkey.api.exp.DomainNotFoundException; -import org.labkey.api.exp.DomainURIFactory; -import org.labkey.api.exp.ObjectProperty; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptor; -import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptorsList; -import org.labkey.api.exp.OntologyObject; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.ProvenanceService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.SystemProperty; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleHtmlView; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.portal.ProjectUrls; -import org.labkey.api.qc.DataState; -import org.labkey.api.qc.QCStateManager; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.FilteredTable; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.query.snapshot.QuerySnapshotDefinition; -import org.labkey.api.reader.ColumnDescriptor; -import org.labkey.api.reader.DataLoader; -import org.labkey.api.reader.MapLoader; -import org.labkey.api.reports.model.ReportPropsManager; -import org.labkey.api.reports.model.ViewCategory; -import org.labkey.api.reports.model.ViewCategoryListener; -import org.labkey.api.reports.model.ViewCategoryManager; -import org.labkey.api.search.SearchService; -import org.labkey.api.search.SearchService.LastIndexedClause; -import org.labkey.api.security.RoleAssignment; -import org.labkey.api.security.SecurableResource; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.SecurityPolicy; -import org.labkey.api.security.SecurityPolicyManager; -import org.labkey.api.security.User; -import org.labkey.api.security.UserPrincipal; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.roles.ReaderRole; -import org.labkey.api.security.roles.RestrictedReaderRole; -import org.labkey.api.security.roles.Role; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.specimen.SpecimenManager; -import org.labkey.api.specimen.SpecimenSchema; -import org.labkey.api.specimen.location.LocationCache; -import org.labkey.api.specimen.model.SpecimenTablesProvider; -import org.labkey.api.study.Cohort; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.DataspaceContainerFilter; -import org.labkey.api.study.QueryHelper; -import org.labkey.api.study.QueryHelper.StudyCacheCollections; -import org.labkey.api.study.SpecimenService; -import org.labkey.api.study.Study; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.TimepointType; -import org.labkey.api.study.Visit; -import org.labkey.api.study.Visit.Order; -import org.labkey.api.study.model.ParticipantDataset; -import org.labkey.api.study.model.ParticipantInfo; -import org.labkey.api.studydesign.StudyDesignManager; -import org.labkey.api.studydesign.StudyDesignService; -import org.labkey.api.studydesign.query.AbstractStudyDesignDomainKind; -import org.labkey.api.studydesign.query.StudyPersonnelDomainKind; -import org.labkey.api.studydesign.query.StudyProductAntigenDomainKind; -import org.labkey.api.studydesign.query.StudyProductDomainKind; -import org.labkey.api.studydesign.query.StudyTreatmentDomainKind; -import org.labkey.api.studydesign.query.StudyTreatmentProductDomainKind; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JunitUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.Path; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.WebPartView; -import org.labkey.api.webdav.SimpleDocumentResource; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.study.StudySchema; -import org.labkey.study.controllers.BaseStudyController.StudyJspView; -import org.labkey.study.controllers.StudyController; -import org.labkey.study.dataset.DatasetAuditProvider; -import org.labkey.study.importer.SchemaReader; -import org.labkey.study.model.StudySnapshot.SnapshotSettings; -import org.labkey.study.query.DatasetTableImpl; -import org.labkey.study.query.DatasetUpdateService; -import org.labkey.study.query.StudyQuerySchema; -import org.labkey.study.visitmanager.AbsoluteDateVisitManager; -import org.labkey.study.visitmanager.RelativeDateVisitManager; -import org.labkey.study.visitmanager.SequenceVisitManager; -import org.labkey.study.visitmanager.VisitManager; -import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.validation.BindException; - -import java.io.IOException; -import java.math.BigDecimal; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeMap; -import java.util.WeakHashMap; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import static org.labkey.api.action.SpringActionController.ERROR_MSG; -import static org.labkey.api.util.IntegerUtils.asInteger; -import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.PERSONNEL_TABLE_NAME; -import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.PRODUCT_ANTIGEN_TABLE_NAME; -import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.PRODUCT_TABLE_NAME; -import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.TREATMENT_PRODUCT_MAP_TABLE_NAME; -import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.TREATMENT_TABLE_NAME; - -public class StudyManager -{ - public static final SearchService.SearchCategory datasetCategory = new SearchService.SearchCategory("dataset", "Study Datasets"); - public static final SearchService.SearchCategory subjectCategory = new SearchService.SearchCategory("subject", "Study Subjects"); - - private static final Logger _log = LogHelper.getLogger(StudyManager.class, "Dataset operations"); - private static final StudyManager _instance = new StudyManager(); - private static final StudySchema SCHEMA = StudySchema.getInstance(); - private static final String LSID_REQUIRED = "LSID_REQUIRED"; - - private final StudyHelper _studyHelper; - private final VisitHelper _visitHelper; - private final DatasetHelper _datasetHelper; - private final QueryHelper> _cohortHelper; - private final BlockingCache> _sharedProperties; - private final BlockingCache> _participantCache = DatabaseCache.get(StudySchema.getInstance().getScope(), Constants.getMaxContainers(), CacheManager.HOUR, "Participants", (c, argument) -> { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - return Collections.unmodifiableMap( - new TableSelector(StudySchema.getInstance().getTableInfoParticipant(), filter, new Sort("ParticipantId")) - .stream(Participant.class) - .collect(LabKeyCollectors.toLinkedMap(Participant::getParticipantId, participant -> participant)) - ); - }); - - private StudyManager() - { - _studyHelper = new StudyHelper(); - _visitHelper = new VisitHelper(); - _cohortHelper = new QueryHelper<>(() -> StudySchema.getInstance().getTableInfoCohort(), CohortImpl.class, "Label"); - - /* - * Whenever we explicitly invalidate a dataset, unmaterialize it as well. This is probably a little overkill, - * e.g. name change doesn't need to unmaterialize however, this is the best choke point - */ - _datasetHelper = new DatasetHelper(); - - // Cache of PropertyDescriptors found in the Shared container for datasets in the given study Container. - // The shared properties cache will be cleared when the _datasetHelper cache is cleared. - _sharedProperties = CacheManager.getBlockingCache(1000, CacheManager.UNLIMITED, "Study shared properties", - (key, argument) -> - { - Container sharedContainer = ContainerManager.getSharedContainer(); - assert key != sharedContainer; - - Collection defs = _datasetHelper.getCollection(key); - - Set set = new LinkedHashSet<>(); - for (DatasetDefinition def : defs) - { - Domain domain = def.getDomain(); - if (domain == null) - continue; - - for (DomainProperty dp : domain.getProperties()) - if (dp.getContainer().equals(sharedContainer)) - set.add(dp.getPropertyDescriptor()); - } - return Collections.unmodifiableSet(set); - } - ); - - ViewCategoryManager.addCategoryListener(new CategoryListener(this)); - } - - // Study helper is different from other query helpers. There's (at most) one study per container, so we cache a - // single map holding all StudyImpls at the root. Even though we're caching a single object in this cache, it - // provides a fast "has study" check (see Issue 19632) and leverages the DatabaseCache & cache-clearing semantics - // of QueryHelper. - private static class StudyHelper extends QueryHelper> - { - private static final Container ROOT = ContainerManager.getRoot(); - - private StudyHelper() - { - super(() -> StudySchema.getInstance().getTableInfoStudy(), StudyImpl.class, "Label"); - } - - private @NotNull Collection getAllStudies() - { - return getCollections().getCollection(); - } - - private @Nullable StudyImpl getStudy(Container c) - { - return getCollections().get(c.getId()); - } - - @Override - protected TableSelector getTableSelector(Container c) - { - assert c.equals(ROOT); - return new TableSelector(getTableInfo(), null, new Sort(_defaultSortString)); - } - - private StudyCacheCollections getCollections() - { - return getCollections(ROOT); - } - - @Override - public void clearCache(Container c) - { - super.clearCache(ROOT); - } - } - - private static class VisitHelper extends QueryHelper - { - private static final Order DEFAULT_ORDER = Order.DISPLAY; - - private VisitHelper() - { - super(() -> StudySchema.getInstance().getTableInfoVisit(), VisitImpl.class, DEFAULT_ORDER.getSortColumns()); - } - - private Collection getCollection(Container c, Order order) - { - return getCollections(c).getCollection(order); - } - - @Override - protected VisitCollections createCollections(Collection collection) - { - return new VisitCollections(collection); - } - - private static class VisitCollections extends StudyCacheCollections - { - private final Collection _sequenceNumVisits; - private final Collection _chronologicalVisits; - - private VisitCollections(Collection collection) - { - super(collection); - - // I'd prefer to push comparators into Visit.Order, but Visit (in API) doesn't know about the display - // order field. - List sorted = new ArrayList<>(collection); - sorted.sort(Comparator.comparing(VisitImpl::getSequenceNumMin)); - _sequenceNumVisits = Collections.unmodifiableCollection(sorted); - - sorted = new ArrayList<>(collection); - sorted.sort(Comparator.comparing(VisitImpl::getChronologicalOrder).thenComparing(VisitImpl::getSequenceNumMin)); - _chronologicalVisits = Collections.unmodifiableCollection(sorted); - } - - private Collection getCollection(Order order) - { - return order == DEFAULT_ORDER ? getCollection() : order == Order.SEQUENCE_NUM ? _sequenceNumVisits : _chronologicalVisits; - } - } - } - - private class DatasetHelper extends QueryHelper - { - private DatasetHelper() - { - super(() -> StudySchema.getInstance().getTableInfoDataset(), DatasetDefinition.class); - } - - @Override - public void clearCache(Container c) - { - super.clearCache(c); - _sharedProperties.remove(c); - } - - private @Nullable DatasetDefinition getByName(Study study, String name) - { - return getCollections(study)._nameMap.get(name); - } - - private @Nullable DatasetDefinition getByLabel(Study study, String label) - { - return getCollections(study)._labelMap.get(label); - } - - private @Nullable DatasetDefinition getByEntityId(Study study, String entityId) - { - return getCollections(study)._entityIdMap.get(entityId); - } - - private @NotNull List getDatasetsForCategory(Study study, @NotNull ViewCategory category) - { - List ret = getCollections(study)._categoryMap.get(category.getRowId()); - return ret != null ? ret : Collections.emptyList(); - } - - private @NotNull List getDatasetsForCohort(Study study, @NotNull Cohort cohort) - { - return getCollections(study).getDatasetsForCohort(cohort); - } - - @Override - protected DatasetCollections createCollections(Collection collection) - { - return new DatasetCollections(collection); - } - - protected DatasetCollections getCollections(Study study) - { - return super.getCollections(study.getContainer()); - } - - private static class DatasetCollections extends StudyCacheCollections - { - private final Map _nameMap = new CaseInsensitiveHashMap<>(); - private final Map _labelMap = new CaseInsensitiveHashMap<>(); - private final Map _entityIdMap = new HashMap<>(); - - private final Map> _categoryMap; - private final Map> _cohortMap; - private final List _nullCohortDatasets; - - private DatasetCollections(Collection collection) - { - super(collection); - - // study.Dataset has constraints on LOWER(Name) and LOWER(Label), so this code path should never attempt - // to put duplicates into these maps. Use asserts to verify this. - collection.forEach(def -> { - DatasetDefinition old = _nameMap.put(def.getName(), def); assert old == null; - old = _labelMap.put(def.getLabel(), def); assert old == null; - old = _entityIdMap.put(def.getEntityId(), def); assert old == null; - }); - - // Group by (non-null) category ID - _categoryMap = collection.stream() - .filter(def -> def.getCategoryId() != null) - .collect(Collectors.groupingBy(DatasetDefinition::getCategoryId)); - - // Group by cohort - _nullCohortDatasets = collection.stream() - .filter(def -> def.getCohortId() == null) - .toList(); - - _cohortMap = collection.stream() - .filter(def -> def.getCohortId() != null) - .collect(Collectors.groupingBy(DatasetDefinition::getCohortId)); - - // Datasets with null cohort get added to every cohort list. Also, make lists immutable. - if (!_cohortMap.isEmpty()) - { - _cohortMap.keySet().forEach(key -> { - List list = _cohortMap.get(key); - list.addAll(_nullCohortDatasets); - _cohortMap.put(key, Collections.unmodifiableList(list)); - }); - } - } - - private @NotNull List getDatasetsForCohort(Cohort cohort) - { - List ret = _cohortMap.get(cohort.getRowId()); - return ret != null ? ret : _nullCohortDatasets; - } - } - } - - public static StudyManager getInstance() - { - return _instance; - } - - @Nullable - public StudyImpl getStudy(@NotNull Container c) - { - StudyImpl study; - boolean retry = true; - - while (true) - { - study = _studyHelper.getStudy(c); - - // This should be a very fast "has study" check, replacement fix for Issue 19632 - if (null == study) - break; - - assert (study.getContainer().getId().equals(c.getId())); - - // UNDONE: There is a subtle bug in caching, cached objects shouldn't hold onto Container objects - Container freshestContainer = ContainerManager.getForId(c.getId()); - if (study.getContainer() == freshestContainer) - break; - - if (!retry) // we only get one retry - break; - - _log.debug("Clearing cached study for {} as its container reference didn't match the current object from ContainerManager {}", c, freshestContainer); - - _studyHelper.clearCache(c); // Clear the cached "all studies" map - retry = false; - } - - return study; - } - - /** @return all studies in the whole server, unfiltered by permissions and sorted by Label */ - @NotNull - public Collection getAllStudies() - { - return _studyHelper.getAllStudies(); - } - - /** @return all studies under the given root in the container hierarchy (inclusive), unfiltered by permissions */ - @NotNull - public Set getAllStudies(@NotNull Container root) - { - Set result = new LinkedHashSet<>(); - for (StudyImpl study : getAllStudies()) - { - if (study.getContainer().equals(root) || study.getContainer().isDescendant(root)) - { - result.add(study); - } - } - return Collections.unmodifiableSet(result); - } - - /** @return all studies under the given root in the container hierarchy (inclusive), to which the user has at least read permission */ - @NotNull - public Set getAllStudies(@NotNull Container root, @NotNull User user) - { - return getAllStudies(root, user, ReadPermission.class); - } - - /** @return all studies under the given root in the container hierarchy (inclusive), to which the user has at least the specified permission */ - @NotNull - public Set getAllStudies(@NotNull Container root, @NotNull User user, @NotNull Class perm) - { - Set result = new LinkedHashSet<>(); - for (StudyImpl study : getAllStudies()) - { - if (study.getContainer().hasPermission(user, perm) && - (study.getContainer().equals(root) || study.getContainer().isDescendant(root))) - { - result.add(study); - } - } - return Collections.unmodifiableSet(result); - } - - public StudyImpl createStudy(User user, StudyImpl study) - { - Container container = study.getContainer(); - assert null != container; - assert null != user; - if (study.getLsid() == null) - study.initLsid(); - - if (study.getProtocolDocumentEntityId() == null) - study.setProtocolDocumentEntityId(GUID.makeGUID()); - - if (study.getAlternateIdDigits() == 0) - study.setAlternateIdDigits(StudyManager.ALTERNATEID_DEFAULT_NUM_DIGITS); - - try (Transaction transaction = StudySchema.getInstance().getScope().ensureTransaction()) - { - SpecimenSchema.get().getTableInfoLocation(container, user); // This provisioned table is needed for creating the study - study = _studyHelper.create(user, study); - clearCaches(container,false); - - //note: we no longer copy the container's policy to the study upon creation - //instead, we let it inherit the container's policy until the security type - //is changed to one of the advanced options. - - // Force provisioned specimen tables to be created - SpecimenSchema.get().getTableInfoSpecimenPrimaryType(container, user); - SpecimenSchema.get().getTableInfoSpecimenDerivative(container, user); - SpecimenSchema.get().getTableInfoSpecimenAdditive(container, user); - SpecimenSchema.get().getTableInfoSpecimen(container, user); - SpecimenSchema.get().getTableInfoVial(container, user); - SpecimenSchema.get().getTableInfoSpecimenEvent(container, user); - transaction.commit(); - } - StudyDesignManager.get().ensureStudyDesignDomains(container, user); - QueryService.get().updateLastModified(); - ContainerManager.notifyContainerChange(container.getId(), ContainerManager.Property.StudyChange); - return study; - } - - public ValidationException updateStudy(@Nullable User user, StudyImpl study) - { - StudyImpl oldStudy = getStudy(study.getContainer()); - Date oldStartDate = oldStudy.getStartDate(); - _studyHelper.update(user, study, study.getContainer()); - ValidationException errors = new ValidationException(); - - if (oldStudy.getTimepointType() == TimepointType.DATE && !Objects.equals(study.getStartDate(), oldStartDate)) - { - // start date has changed, and datasets may use that value. Uncache. - RelativeDateVisitManager visitManager = (RelativeDateVisitManager) getVisitManager(study); - errors = visitManager.recomputeDates(oldStartDate, user); - clearCaches(study.getContainer(), true); - } - else - { - // Need to get rid of any old copies of the study - clearCaches(study.getContainer(), false); - } - - if (oldStudy.getSecurityType() != study.getSecurityType()) - { - String comment = "Dataset security type changed from " + oldStudy.getSecurityType() + " to " + study.getSecurityType(); - StudyService.get().addStudyAuditEvent(study.getContainer(), user, comment); - } - QueryService.get().updateLastModified(); - return errors; - } - - public void updateStudySnapshot(StudySnapshot snapshot, User user) - { - // For now, "refresh" is the only field that can be updated (plus the Modified fields, which get handled automatically) - Map map = new HashMap<>(); - map.put("refresh", snapshot.isRefresh()); - - Table.update(user, StudySchema.getInstance().getTableInfoStudySnapshot(), map, snapshot.getRowId()); - } - - public void createDatasetDefinition(User user, Container container, int datasetId) - { - createDatasetDefinition(user, new DatasetDefinition(getStudy(container), datasetId)); - } - - public void createDatasetDefinition(User user, DatasetDefinition datasetDefinition) - { - if (datasetDefinition.getDatasetId() <= 0) - throw new IllegalArgumentException("datasetId must be greater than zero."); - DbScope scope = StudySchema.getInstance().getScope(); - - try (Transaction transaction = scope.ensureTransaction()) - { - ensureViewCategory(user, datasetDefinition); - _datasetHelper.create(user, datasetDefinition); - // This method call has the side effect of ensuring that we have a domain. If we don't create it here, - // we're open to a race condition if another thread tries to do something with the dataset's table - // and ends up attempting to create the domain as well - datasetDefinition.getStorageTableInfo(true); - - QueryService.get().updateLastModified(); - transaction.commit(); - } - indexDataset(SearchService.get().defaultTask().getQueue(datasetDefinition.getContainer(), SearchService.PRIORITY.modified), datasetDefinition); - } - - /** - * Temporary shim until we can redo the dataset category UI - */ - private void ensureViewCategory(User user, DatasetDefinition def) - { - ViewCategory category = null; - - if (def.getCategoryId() != null) - category = ViewCategoryManager.getInstance().getCategory(def.getContainer(), def.getCategoryId()); - - if (category == null && def.getCategory() != null) - { - // the imported category name may be encoded to contain subcategory info - String[] parts = ViewCategoryManager.getInstance().decode(def.getCategory()); - category = ViewCategoryManager.getInstance().ensureViewCategory(def.getContainer(), user, parts); - } - - if (category != null) - { - def.setCategoryId(category.getRowId()); - def.setCategory(category.getLabel()); - } - } - - public void updateDatasetDefinition(User user, DatasetDefinition datasetDefinition, List errors) - { - try - { - updateDatasetDefinition(user, datasetDefinition); - } - catch (IllegalArgumentException ex) - { - errors.add(ex.getMessage()); - } - } - - public boolean isKeyChanged(final DatasetDefinition datasetDefinition) - { - DatasetDefinition old = getDatasetDefinition(datasetDefinition.getStudy(), datasetDefinition.getDatasetId()); - if (old != null) - { - return old.isDemographicData() != datasetDefinition.isDemographicData() || - !Strings.CS.equals(old.getKeyPropertyName(), datasetDefinition.getKeyPropertyName()) || - old.getUseTimeKeyField() != datasetDefinition.getUseTimeKeyField(); - } - return false; - } - - /* most users should call the List errors version to avoid uncaught exceptions */ - @Deprecated - public boolean updateDatasetDefinition(User user, final DatasetDefinition datasetDefinition) - { - if (datasetDefinition.isShared()) - { - // check if we're updating the dataset property overrides in a sub-folder - if (!datasetDefinition.getContainer().equals(datasetDefinition.getDefinitionContainer())) - { - return updateDatasetPropertyOverrides(user, datasetDefinition); - } - } - - DbScope scope = StudySchema.getInstance().getScope(); - - try (Transaction transaction = scope.ensureTransaction()) - { - DatasetDefinition old = getDatasetDefinition(datasetDefinition.getStudy(), datasetDefinition.getDatasetId()); - if (null == old) - throw OptimisticConflictException.create(Table.ERROR_DELETED); - - // make sure we reload domain and tableinfo - Domain domain = datasetDefinition.refreshDomain(); - - // Check if the extra key field has changed - boolean isProvisioned = domain != null && domain.getStorageTableName() != null; - boolean isKeyChanged = isKeyChanged(datasetDefinition); - boolean isSharedChanged = old.getDataSharingEnum() != datasetDefinition.getDataSharingEnum(); - if (isProvisioned && isSharedChanged) - { - // let's not change the shared setting if there are existing rows - if (new TableSelector(datasetDefinition.getStorageTableInfo(false)).exists()) - { - throw new IllegalArgumentException("Can't change data sharing setting if there are existing data rows."); - } - } - - if (isProvisioned && isKeyChanged) - { - TableInfo storageTableInfo = datasetDefinition.getStorageTableInfo(false); - - // If so, we need to update the _key column and the LSID - - // Set the _key column to be the value of the selected column - // Change how we build up tableName - String tableName = storageTableInfo.toString(); - SQLFragment updateKeySQL = new SQLFragment("UPDATE " + tableName + " SET _key = "); - if (datasetDefinition.getUseTimeKeyField()) - { - ColumnInfo col = storageTableInfo.getColumn("Date"); - if (null == col) - { - throw new IllegalArgumentException("Cannot find 'Date' column in table: " + tableName); - } - SQLFragment colFrag = col.getValueSql(tableName); - updateKeySQL.append(storageTableInfo.getSqlDialect().getISOFormat(colFrag)); - } - else if (datasetDefinition.getKeyPropertyName() == null) - { - // No column selected, so set it to be null - updateKeySQL.append("NULL"); - } - else - { - ColumnInfo col = storageTableInfo.getColumn(datasetDefinition.getKeyPropertyName()); - if (null == col) - { - throw new IllegalArgumentException("Cannot find 'key' column: " + datasetDefinition.getKeyPropertyName() + " in table: " + tableName); - } - SQLFragment colFrag = col.getValueSql(tableName); - if (col.getJdbcType() == JdbcType.TIMESTAMP) - colFrag = storageTableInfo.getSqlDialect().getISOFormat(colFrag); - updateKeySQL.append(colFrag); - } - - try - { - new SqlExecutor(StudySchema.getInstance().getSchema()).setLogLevel(Level.OFF).execute(updateKeySQL); - - // Now update the LSID column. Note - this needs to be the same as DatasetImportHelper.getURI() - SQLFragment updateLSIDSQL = new SQLFragment("UPDATE " + tableName + " SET lsid = "); - updateLSIDSQL.append(datasetDefinition.generateLSIDSQL()); - new SqlExecutor(StudySchema.getInstance().getSchema()).execute(updateLSIDSQL); - } - catch (DataIntegrityViolationException x) - { - _log.debug("Old Dataset: " + old.getName()); - _log.debug(" Demographic: " + old.isDemographicData()); - _log.debug(" Key: " + old.getKeyPropertyName()); - _log.debug("New Dataset: " + datasetDefinition.getName()); - _log.debug(" Demographic: " + datasetDefinition.isDemographicData()); - _log.debug(" Key: " + datasetDefinition.getKeyPropertyName()); - - if (datasetDefinition.isDemographicData()) - throw new IllegalArgumentException("Can not change dataset type to demographic for dataset " + datasetDefinition.getName()); - else - throw new IllegalArgumentException("Changing the dataset key would result in duplicate keys for dataset " + datasetDefinition.getName()); - } - } - Object[] pk = new Object[]{datasetDefinition.getContainer().getId(), datasetDefinition.getDatasetId()}; - ensureViewCategory(user, datasetDefinition); - ensureDatasetDefinitionDomain(user, datasetDefinition); - _datasetHelper.update(user, datasetDefinition, pk); - - QueryChangeListener.QueryPropertyChange nameChange = null; - if (!old.getName().equals(datasetDefinition.getName())) - { - nameChange = new QueryChangeListener.QueryPropertyChange<>( - QueryService.get().getUserSchema(user, datasetDefinition.getContainer(), StudyQuerySchema.SCHEMA_NAME).getQueryDefForTable(datasetDefinition.getName()), - QueryChangeListener.QueryProperty.Name, - old.getName(), - datasetDefinition.getName() - ); - } - final QueryChangeListener.QueryPropertyChange change = nameChange; - - transaction.addCommitTask(() -> - { - uncache(datasetDefinition); - if (null != change) - { - QueryService.get().fireQueryChanged(user, datasetDefinition.getContainer(), null, new SchemaKey(null, StudyQuerySchema.SCHEMA_NAME), - QueryChangeListener.QueryProperty.Name, Collections.singleton(change)); - } - indexDataset(SearchService.get().defaultTask().getQueue(datasetDefinition.getContainer(), SearchService.PRIORITY.modified), datasetDefinition); - }, CommitTaskOption.POSTCOMMIT); - - // NOTE: not redundant with uncache() in commit task, there may be an active outer transaction - uncache(datasetDefinition); - QueryService.get().updateLastModified(); - transaction.commit(); - } - datasetDefinition.refreshDomain(); - return true; - } - - /** - * Shared dataset may save some of its properties in the current container rather than the dataset definition container. - * Currently allowed overrides: - *
      - *
    • isShownByDefault
      • - *
    - * @return true if successful. - */ - private boolean updateDatasetPropertyOverrides(User user, final DatasetDefinition datasetDefinition) - { - if (!datasetDefinition.isShared() || datasetDefinition.getContainer().isProject()) - { - throw new IllegalArgumentException("Dataset property overrides can only be applied to shared datasets and in sub-containers"); - } - - DbScope scope = StudySchema.getInstance().getScope(); - - try (Transaction transaction = scope.ensureTransaction()) - { - DatasetDefinition old = getDatasetDefinition(datasetDefinition.getStudy(), datasetDefinition.getDatasetId()); - if (null == old) - throw OptimisticConflictException.create(Table.ERROR_DELETED); - - DatasetDefinition original = getDatasetDefinition(datasetDefinition.getDefinitionStudy(), datasetDefinition.getDatasetId()); - - // make sure we reload domain and tableinfo - Domain domain = datasetDefinition.refreshDomain(); - - // Error if any other properties have been changed - if (!Objects.equals(old.getLabel(), datasetDefinition.getLabel())) - { - throw new IllegalArgumentException("Shared dataset label can't be changed"); - } - if (!Objects.equals(old.getCategoryId(), datasetDefinition.getCategoryId())) - { - throw new IllegalArgumentException("Shared dataset category can't be changed"); - } - if (!Objects.equals(old.getCohortId(), datasetDefinition.getCohortId())) - { - throw new IllegalArgumentException("Shared dataset cohort can't be changed"); - } - - // track added and removed properties against the shared dataset in the definition container - Map add = new HashMap<>(); - List remove = new LinkedList<>(); - if (datasetDefinition.isShowByDefault() != original.isShowByDefault()) - add.put("showByDefault", String.valueOf(datasetDefinition.isShowByDefault())); - else - remove.add("showByDefault"); - - // update the override map - Container c = datasetDefinition.getContainer(); - String category = "dataset-overrides:" + datasetDefinition.getDatasetId(); - WritablePropertyMap map = null; - - if (!add.isEmpty()) - { - map = PropertyManager.getWritableProperties(c, category, true); - map.putAll(add); - } - - if (!remove.isEmpty()) - { - if (map == null) - map = PropertyManager.getWritableProperties(c, category, false); - - if (map != null) - { - for (String key : remove) - map.remove(key); - } - } - - // persist change -- if overrides are no longer needed, just remove it - if (map != null) - { - if (map.isEmpty()) - map.delete(); - else - map.save(); - } - - transaction.addCommitTask(() -> { - // And post-commit to make sure that no other threads have reloaded the cache in the meantime - uncache(datasetDefinition); - }, CommitTaskOption.POSTCOMMIT, CommitTaskOption.IMMEDIATE); - transaction.commit(); - } - - return true; - } - - public void deleteDatasetPropertyOverrides(User user, Container c, BindException errors) - { - if (c.isProject()) - { - errors.reject(ERROR_MSG, "can't delete dataset property override from project-level study"); - return; - } - - Study study = getStudy(c); - if (study == null) - { - errors.reject(ERROR_MSG, "study not found"); - return; - } - - if (study.isDataspaceStudy()) - { - errors.reject(ERROR_MSG, "can't delete dataset property override from a shared study"); - return; - } - - Study sharedStudy = getSharedStudy(study); - if (sharedStudy == null) - { - errors.reject(ERROR_MSG, "not a sub-study of a shared study"); - return; - } - - DbScope scope = StudySchema.getInstance().getScope(); - try (Transaction transaction = scope.ensureTransaction()) - { - for (DatasetDefinition dataset : getDatasetDefinitions(study)) - { - if (dataset.isInherited()) - deleteDatasetPropertyOverrides(user, dataset); - } - transaction.commit(); - } - } - - private boolean deleteDatasetPropertyOverrides(User user, final DatasetDefinition datasetDefinition) - { - if (!datasetDefinition.isInherited() || datasetDefinition.getContainer().isProject()) - { - throw new IllegalArgumentException("Dataset property overrides can only be applied to shared datasets and in sub-containers"); - } - - DbScope scope = StudySchema.getInstance().getScope(); - try (Transaction transaction = scope.ensureTransaction()) - { - Container c = datasetDefinition.getContainer(); - String category = "dataset-overrides:" + datasetDefinition.getDatasetId(); - PropertyManager.getNormalStore().deletePropertySet(c, category); - - transaction.addCommitTask(() -> { - // And post-commit to make sure that no other threads have reloaded the cache in the meantime - uncache(datasetDefinition); - }, CommitTaskOption.POSTCOMMIT, CommitTaskOption.IMMEDIATE); - transaction.commit(); - } - - return true; - } - - public boolean isDataUniquePerParticipant(DatasetDefinition dataset) - { - // don't use dataset.getTableInfo() since this method is called during updateDatasetDefinition`() and may be in an inconsistent state - TableInfo t = dataset.getStorageTableInfo(false); - SQLFragment sql = new SQLFragment(); - sql.append("SELECT MAX(n) FROM (SELECT COUNT(*) AS n FROM ").append(t.getFromSQL("DS")).append(" GROUP BY ParticipantId) x"); - Integer maxCount = new SqlSelector(StudySchema.getInstance().getSchema(), sql).getObject(Integer.class); - return maxCount == null || maxCount <= 1; - } - - - public static class VisitCreationException extends RuntimeException - { - public VisitCreationException(String message) - { - super(message); - } - } - - - public VisitImpl createVisit(Study study, User user, VisitImpl visit) - { - return createVisit(study, user, visit, null); - } - - - public VisitImpl createVisit(Study study, User user, VisitImpl visit, @Nullable Collection existingVisits) - { - Study visitStudy = getStudyForVisits(study); - - if (visit.getContainer() != null && !visit.getContainer().getId().equals(visitStudy.getContainer().getId())) - throw new VisitCreationException("Visit container does not match study"); - visit.setContainer(visitStudy.getContainer()); - - if (visit.getSequenceNumMin().compareTo(visit.getSequenceNumMax()) > 0) - throw new VisitCreationException("SequenceNumMin must be less than or equal to SequenceNumMax"); - - if (null == existingVisits) - existingVisits = getVisits(study, Order.SEQUENCE_NUM); - - int prevDisplayOrder = 0; - int prevChronologicalOrder = 0; - - for (VisitImpl existingVisit : existingVisits) - { - if (existingVisit.getSequenceNumMin().compareTo(visit.getSequenceNumMin()) < 0) - { - prevChronologicalOrder = existingVisit.getChronologicalOrder(); - prevDisplayOrder = existingVisit.getDisplayOrder(); - } - - if (existingVisit.getSequenceNumMin().compareTo(existingVisit.getSequenceNumMax()) > 0) - throw new VisitCreationException("Corrupt existing visit " + existingVisit + - ": SequenceNumMin must be less than or equal to SequenceNumMax"); - boolean disjoint = (visit.getSequenceNumMax().compareTo(existingVisit.getSequenceNumMin()) < 0) || (visit.getSequenceNumMin().compareTo(existingVisit.getSequenceNumMax()) > 0); - if (!disjoint) - { - throw new VisitCreationException("New visit " + visit + " overlaps existing visit " + existingVisit); - } - } - - // if our visit doesn't have a display order or chronological order set, but the visit before our new visit - // (based on sequencenum) does, then assign the previous visit's order info to our new visit. This won't always - // be exactly right, but it's better than having all newly created visits appear at the beginning of the display - // and chronological lists: - if (visit.getDisplayOrder() == 0 && prevDisplayOrder > 0) - visit.setDisplayOrder(prevDisplayOrder); - if (visit.getChronologicalOrder() == 0 && prevChronologicalOrder > 0) - visit.setChronologicalOrder(prevChronologicalOrder); - - visit = _visitHelper.create(user, visit); - - if (visit.getRowId() == 0) - throw new VisitCreationException("Visit rowId has not been set properly"); - - return visit; - } - - /** - * Return a visit object regardless of whether it exists in the database but will not insert a new - * record into the database. - */ - public VisitImpl getVisit(Study study, User user, BigDecimal sequenceNum, Visit.Type type) - { - Collection visits = getVisits(study, Order.SEQUENCE_NUM); - return ensureVisitWithoutSaving(study, sequenceNum, type, visits); - } - - /** - * Ensures the existence of a visit for the specified sequence numbers and will insert into the database - * if the visit does not yet exist. - * - * @param failForUndefinedVisits If true, new visits will not be created and an error will be added to the returned - * ValidationException object. - * @return ValidationException which will contain any relevant errors. Callers should check hasErrors on the object. - */ - public @NotNull ValidationException ensureVisits(Study study, User user, Set sequencenums, @Nullable Visit.Type type, - boolean failForUndefinedVisits) - { - Collection visits = getVisits(study, Order.SEQUENCE_NUM); - ValidationException errors = new ValidationException(); - List seqNumFailures = new ArrayList<>(); - - for (BigDecimal sequencenum : sequencenums) - { - VisitImpl result = ensureVisitWithoutSaving(study, sequencenum, type, visits); - if (result.getRowId() == 0) - { - if (!failForUndefinedVisits) - { - createVisit(study, user, result, visits); - // Refresh existing visits to avoid constraint violation, see #44425 - visits = getVisits(study, Order.SEQUENCE_NUM); - } - else - seqNumFailures.add(String.valueOf(sequencenum)); - } - } - - if (!seqNumFailures.isEmpty()) - { - String timepointNoun = study.getTimepointType().isVisitBased() ? "visit" : "timepoint"; - errors.addError(new SimpleValidationError(String.format("Creating new %ss is not allowed for this study. The following %s not currently exist : (%s)", - timepointNoun, timepointNoun + (seqNumFailures.size() > 1 ? "s do" : " does"), - String.join(", ", seqNumFailures)))); - } - return errors; - } - - private VisitImpl ensureVisitWithoutSaving(Study study, double seqNumDouble, @Nullable Visit.Type type, Collection existingVisits) - { - return ensureVisitWithoutSaving(study, VisitImpl.getSequenceNum(seqNumDouble), type, existingVisits); - } - - private VisitImpl ensureVisitWithoutSaving(Study study, BigDecimal sequenceNum, @Nullable Visit.Type type, Collection existingVisits) - { - sequenceNum = VisitImpl.normalizeSequenceNum(sequenceNum); - - // Remember the SequenceNums closest to the requested id in case we need to create one - BigDecimal nextVisit = Visit.MAX_SEQUENCE_NUM; - BigDecimal previousVisit = Visit.MIN_SEQUENCE_NUM; - for (VisitImpl visit : existingVisits) - { - if (visit.getSequenceNumMin().compareTo(sequenceNum) <= 0 && visit.getSequenceNumMax().compareTo(sequenceNum) >= 0) - return visit; - // check to see if our new sequencenum is within the range of an existing visit: - // Check if it's the closest to the requested id, either before or after - if (visit.getSequenceNumMin().compareTo(nextVisit) < 0 && visit.getSequenceNumMin().compareTo(sequenceNum) > 0) - { - nextVisit = visit.getSequenceNumMin(); - } - if (visit.getSequenceNumMax().compareTo(previousVisit) > 0 && visit.getSequenceNumMax().compareTo(sequenceNum) < 0) - { - previousVisit = visit.getSequenceNumMax(); - } - } - BigDecimal visitIdMin = sequenceNum; - BigDecimal visitIdMax = sequenceNum; - String label = null; - if (!study.getTimepointType().isVisitBased()) - { - boolean isFloatingPoint = sequenceNum.stripTrailingZeros().scale() > 0; - - // Do special handling for data-based studies - if (study.getDefaultTimepointDuration() == 1 || isFloatingPoint || sequenceNum.compareTo(BigDecimal.ZERO) < 0) - { - // See if there's a fractional part to the number - if (isFloatingPoint) - { - label = "Day " + VisitImpl.formatSequenceNum(sequenceNum); - } - else - { - // If not, drop the decimal from the default name - label = "Day " + sequenceNum.intValue(); - } - } - else - { - // Try to create a timepoint that spans the default number of days - // For example, if duration is 7 days, do timepoints for days 0-6, 7-13, 14-20, etc - int intervalNumber = sequenceNum.intValue() / study.getDefaultTimepointDuration(); - visitIdMin = BigDecimal.valueOf((long)intervalNumber * study.getDefaultTimepointDuration()); - visitIdMax = BigDecimal.valueOf((long)(intervalNumber + 1) * study.getDefaultTimepointDuration() - 1); - - // Scale the timepoint to be smaller if there are existing timepoints that overlap - // on its desired day range - if (!Visit.MIN_SEQUENCE_NUM.equals(previousVisit)) - { - visitIdMin = visitIdMin.max(previousVisit.add(BigDecimal.ONE)); - } - if (!Visit.MAX_SEQUENCE_NUM.equals(nextVisit)) - { - visitIdMax = visitIdMax.min(nextVisit.subtract(BigDecimal.ONE)); - } - - // Default label is "Day X - Y" - label = "Day " + visitIdMin.intValue() + " - " + visitIdMax.intValue(); - if (visitIdMin.compareTo(visitIdMax) == 0) - { - // Single day timepoint, so don't use the range - label = "Day " + visitIdMin.intValue(); - } - else if (visitIdMin.intValue() == intervalNumber * study.getDefaultTimepointDuration() && - visitIdMax.intValue() == (intervalNumber + 1) * study.getDefaultTimepointDuration() - 1) - { - // The timepoint is the full span for the default duration, so see if we - // should call it "Week" or "Month" - if (study.getDefaultTimepointDuration() == 7) - { - label = "Week " + (intervalNumber + 1); - } - else if (study.getDefaultTimepointDuration() == 30 || study.getDefaultTimepointDuration() == 31) - { - label = "Month " + (intervalNumber + 1); - } - } - } - } - - // create visit in shared study - Study visitStudy = getStudyForVisits(study); - return new VisitImpl(visitStudy.getContainer(), visitIdMin, visitIdMax, label, type); - } - - public void importVisitAliases(Study study, User user, List aliases) throws ValidationException - { - DataIteratorBuilder it = new BeanDataIterator.Builder<>(VisitAlias.class, aliases); - importVisitAliases(study, user, it); - } - - public int importVisitAliases(final Study study, User user, DataIteratorBuilder loader) throws ValidationException - { - TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitAliases(); - DbScope scope = tinfo.getSchema().getScope(); - - // We want to delete and bulk insert in the same transaction - try (Transaction transaction = scope.ensureTransaction()) - { - clearVisitAliases(study); - - DataIteratorContext context = new DataIteratorContext(); - context.setInsertOption(QueryUpdateService.InsertOption.IMPORT); - StandardDataIteratorBuilder etl = StandardDataIteratorBuilder.forInsert(tinfo, loader, study.getContainer(), user, context); - DataIteratorBuilder insert = ((UpdateableTableInfo) tinfo).persistRows(etl, context); - Pump p = new Pump(insert, context); - p.run(); - - if (context.getErrors().hasErrors()) - throw context.getErrors().getRowErrors().get(0); - - transaction.commit(); - - return p.getRowCount(); - } - } - - - public void clearVisitAliases(Study study) - { - SimpleFilter containerFilter = SimpleFilter.createContainerFilter(study.getContainer()); - TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitAliases(); - DbScope scope = tinfo.getSchema().getScope(); - - try (Transaction transaction = scope.ensureTransaction()) - { - Table.delete(tinfo, containerFilter); - transaction.commit(); - } - } - - - public Map getVisitImportMap(Study study, boolean includeStandardMapping) - { - Collection customMapping = getCustomVisitImportMapping(study); - Collection visits = includeStandardMapping ? StudyManager.getInstance().getVisits(study, Order.SEQUENCE_NUM) : Collections.emptyList(); - - Map map = new CaseInsensitiveHashMap<>((customMapping.size() + visits.size()) * 3 / 4); - -// // allow prepended "visit" -// for (Visit visit : visits) -// { -// if (null == visit.getLabel()) -// continue; -// String label = "visit " + visit.getLabel(); -// // Use the **first** instance of each label -// if (!map.containsKey(label)) -// map.put(label, visit.getSequenceNumMin()); -// } - - // Load up standard label -> min sequence number mapping first - for (Visit visit : visits) - { - String label = visit.getLabel(); - - // Use the **first** instance of each label - if (null != label && !map.containsKey(label)) - map.put(label, visit.getSequenceNumMin()); - } - - // Now load custom mapping, overwriting any existing standard labels - for (VisitAlias alias : customMapping) - map.put(alias.getName(), alias.getSequenceNum()); - - return map; - } - - - // Return the custom import mapping (optionally provided by the admin), ordered by sequence num then row id (which - // maintains import order in the case where multiple names map to the same sequence number). - public Collection getCustomVisitImportMapping(Study study) - { - SimpleFilter containerFilter = SimpleFilter.createContainerFilter(study.getContainer()); - TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitAliases(); - - return new TableSelector(tinfo, tinfo.getColumns("Name, SequenceNum"), containerFilter, new Sort("SequenceNum,RowId")).getCollection(VisitAlias.class); - } - - - // Return the standard import mapping (generated from Visit.Label -> Visit.SequenceNumMin), ordered by sequence - // num for display purposes. Include VisitAliases that won't be used, but mark them as overridden. - public Collection getStandardVisitImportMapping(Study study) - { - List list = new LinkedList<>(); - Set labels = new CaseInsensitiveHashSet(); - Map customMap = getVisitImportMap(study, false); - - Collection visits = StudyManager.getInstance().getVisits(study, Order.SEQUENCE_NUM); - - for (Visit visit : visits) - { - String label = visit.getLabel(); - - if (null != label) - { - boolean overridden = labels.contains(label) || customMap.containsKey(label); - list.add(new VisitAlias(label, visit.getSequenceNumMin(), visit.getSequenceString(), overridden)); - - if (!overridden) - labels.add(label); - } - } - - return list; - } - - - public static class VisitAlias - { - private String _name; - private BigDecimal _sequenceNum; - private String _sequenceString; - private boolean _overridden; // For display purposes -- we show all visits and gray out the ones that are not used - - @SuppressWarnings({"UnusedDeclaration"}) // Constructed via reflection by the Table layer - public VisitAlias() - { - } - - public VisitAlias(String name, BigDecimal sequenceNum, @Nullable String sequenceString, boolean overridden) - { - _name = name; - _sequenceNum = sequenceNum; - _sequenceString = sequenceString; - _overridden = overridden; - } - - public VisitAlias(String name, BigDecimal sequenceNum) - { - this(name, VisitImpl.normalizeSequenceNum(sequenceNum), null, false); - } - - public String getName() - { - return _name; - } - - public void setName(String name) - { - _name = name; - } - - public BigDecimal getSequenceNum() - { - return _sequenceNum; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSequenceNum(BigDecimal sequenceNum) - { - _sequenceNum = VisitImpl.normalizeSequenceNum(sequenceNum); - } - - public boolean isOverridden() - { - return _overridden; - } - - public String getSequenceNumString() - { - return VisitImpl.formatSequenceNum(_sequenceNum); - } - - public String getSequenceString() - { - if (null == _sequenceString) - return getSequenceNumString(); - else - return _sequenceString; - } - - public String toString() - { - return _name + " (" + VisitImpl.formatSequenceNum(_sequenceNum) + ")"; - } - } - - - public Map importVisitTags(Study study, User user, List visitTags) throws ValidationException - { - // Import, don't overwrite existing - final Map allVisitTagMap = new HashMap<>(); - final Map newVisitTagMap = new HashMap<>(); - for (VisitTag visitTag : visitTags) - { - newVisitTagMap.put(visitTag.getName(), visitTag); - } - - Container container = getStudyForVisitTag(study).getContainer(); - SimpleFilter containerFilter = SimpleFilter.createContainerFilter(container); - TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTag(); - if (null == tinfo) - throw new IllegalStateException("Study Import/Export expected TableInfo."); - - TableSelector selector = new TableSelector(tinfo, containerFilter, null); - selector.forEach(VisitTag.class, visitTag -> { - allVisitTagMap.put(visitTag.getName(), visitTag); - newVisitTagMap.remove(visitTag.getName()); - }); - - List newVisitTags = new ArrayList<>(newVisitTagMap.values()); - DataIteratorBuilder loader = new BeanDataIterator.Builder<>(VisitTag.class, newVisitTags); - DbScope scope = tinfo.getSchema().getScope(); - - try (Transaction transaction = scope.ensureTransaction()) - { - DataIteratorContext context = new DataIteratorContext(); - context.setInsertOption(QueryUpdateService.InsertOption.IMPORT); - StandardDataIteratorBuilder etl = StandardDataIteratorBuilder.forInsert(tinfo, loader, container, user, context); - DataIteratorBuilder insert = ((UpdateableTableInfo) tinfo).persistRows(etl, context); - Pump p = new Pump(insert, context); - p.run(); - - BatchValidationException errors = context.getErrors(); - if (errors.hasErrors()) - throw errors.getRowErrors().get(0); - - transaction.commit(); - } - allVisitTagMap.putAll(newVisitTagMap); - return allVisitTagMap; - } - - - public Integer createVisitTagMapEntry(User user, Container container, String visitTagName, @NotNull Integer visitId, @Nullable Integer cohortId) - { - TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTagMap(); - Map map = new CaseInsensitiveHashMap<>(); - map.put("visitTag", visitTagName); - map.put("visitId", visitId); - map.put("cohortId", cohortId); - map.put("containerId", container.getId()); - map = Table.insert(user, tinfo, map); - return asInteger(map.get("RowId")); - } - - @Nullable - public String checkSingleUseVisitTag(VisitTag visitTag, @Nullable Integer cohortId, @NotNull List visitTagMapEntries, - @Nullable Integer oldRowId, Container container, User user) - { - for (VisitTagMapEntry visitTagMapEntry : visitTagMapEntries) - if ((null == oldRowId || !oldRowId.equals(visitTagMapEntry.getRowId())) && - ((null == cohortId && null == visitTagMapEntry.getCohortId()) || null != cohortId && cohortId.equals(visitTagMapEntry.getCohortId()))) - { - Cohort cohort = null != cohortId ? getCohortForRowId(container, user, cohortId) : null; - return "Single use visit tag '" + visitTag.getCaption() + - "' may not be used for more than one visit for the same cohort '" + (null != cohort ? cohort.getLabel() : "") + "'."; - } - return null; - } - - public Map getVisitTags(Study study) - { - // TODO: Use QueryHelper? - final Map visitTags = new HashMap<>(); - SimpleFilter containerFilter = SimpleFilter.createContainerFilter(study.getContainer()); - TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTag(); - new TableSelector(tinfo, containerFilter, null).forEach(VisitTag.class, visitTag -> visitTags.put(visitTag.getName(), visitTag)); - return visitTags; - } - - public - @Nullable - VisitTag getVisitTag(Study study, String visitTagName) - { - final List visitTags = new ArrayList<>(); - SimpleFilter filter = SimpleFilter.createContainerFilter(study.getContainer()); - filter.addCondition(FieldKey.fromString("Name"), visitTagName); - TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTag(); - new TableSelector(tinfo, filter, null).forEach(VisitTag.class, visitTags::add); - - if (visitTags.isEmpty()) - return null; - if (visitTags.size() > 1) - throw new IllegalStateException("Expected only one visit tag with given name."); - return visitTags.get(0); - } - - public Map> getVisitTagMapMap(Study study) - { - final Map> visitTagMapMap = new IntHashMap<>(); - SimpleFilter containerFilter = SimpleFilter.createContainerFilter(study.getContainer()); - TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTagMap(); - new TableSelector(tinfo, containerFilter, null).forEach(VisitTagMapEntry.class, visitTagMapEntry -> { - if (!visitTagMapMap.containsKey(visitTagMapEntry.getVisitId())) - visitTagMapMap.put(visitTagMapEntry.getVisitId(), new ArrayList<>()); - visitTagMapMap.get(visitTagMapEntry.getVisitId()).add(visitTagMapEntry); - }); - - return visitTagMapMap; - } - - public Map> getVisitTagToVisitTagMapEntries(Study study) - { - final Map> visitTagToVisitTagMapEntries = new HashMap<>(); - SimpleFilter containerFilter = SimpleFilter.createContainerFilter(study.getContainer()); - TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTagMap(); - new TableSelector(tinfo, containerFilter, null).forEach(VisitTagMapEntry.class, visitTagMapEntry -> { - if (!visitTagToVisitTagMapEntries.containsKey(visitTagMapEntry.getVisitTag())) - visitTagToVisitTagMapEntries.put(visitTagMapEntry.getVisitTag(), new ArrayList<>()); - visitTagToVisitTagMapEntries.get(visitTagMapEntry.getVisitTag()).add(visitTagMapEntry); - }); - - return visitTagToVisitTagMapEntries; - } - - public List getVisitTagMapEntries(Study study, String visitTagName) - { - final List visitTagMapEntries = new ArrayList<>(); - SimpleFilter filter = SimpleFilter.createContainerFilter(study.getContainer()); - filter.addCondition(FieldKey.fromString("VisitTag"), visitTagName); - TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTagMap(); - new TableSelector(tinfo, filter, null).forEach(VisitTagMapEntry.class, visitTagMapEntries::add); - - return visitTagMapEntries; - } - - public static String makeVisitTagMapKey(String visitTagName, int visitId, @Nullable Integer cohortId) - { - return visitTagName + "/" + visitId + "/" + cohortId; - } - - public void createCohort(Study study, User user, CohortImpl cohort) - { - if (cohort.getContainer() != null && !cohort.getContainer().equals(study.getContainer())) - throw new IllegalArgumentException("Cohort container does not match study"); - cohort.setContainer(study.getContainer()); - - // Lsid requires the row id, which does not get created until this object has been inserted into the db - if (cohort.getLsid() != null) - throw new IllegalStateException("Attempt to create a new cohort with lsid already set"); - cohort.setLsid(LSID_REQUIRED); - cohort = _cohortHelper.create(user, cohort); - - if (cohort.getRowId() == 0) - throw new IllegalStateException("Cohort rowId has not been set properly"); - - cohort.initLsid(); - _cohortHelper.update(user, cohort); - } - - public void deleteVisit(StudyImpl study, VisitImpl visit, User user) - { - deleteVisits(study, Collections.singleton(visit), user, false); - } - - /* - Delete multiple visits; more efficient than calling deleteVisit() in a loop. - */ - public void deleteVisits(StudyImpl study, Collection visits, User user, boolean unused) - { - // Short circuit on empty - if (visits.isEmpty()) - return; - - // Extract visit rowIds - Collection visitIds = CollectionUtils.collect(visits, VisitImpl::getRowId); - - StudySchema schema = StudySchema.getInstance(); - SQLFragment visitInClause = new SQLFragment(); - schema.getSqlDialect().appendInClauseSql(visitInClause, visitIds); - - try (Transaction transaction = schema.getSchema().getScope().ensureTransaction()) - { - if (!unused) - { - for (DatasetDefinition def : study.getDatasets()) - { - TableInfo t = def.getStorageTableInfo(false); - if (null == t) - continue; - - SQLFragment sqlf = new SQLFragment(); - sqlf.append("DELETE FROM "); - sqlf.append(t); - if (schema.getSqlDialect().isSqlServer()) - sqlf.append(" WITH (UPDLOCK)"); - sqlf.append(" WHERE LSID IN (SELECT LSID FROM "); - sqlf.append(t); - sqlf.append(" d, "); - sqlf.append(StudySchema.getInstance().getTableInfoParticipantVisit(), "pv"); - sqlf.append(" WHERE d.ParticipantId = pv.ParticipantId AND d.SequenceNum = pv.SequenceNum AND pv.Container = ?"); - sqlf.add(study.getContainer()); - sqlf.append(" AND pv.VisitRowId ").append(visitInClause).append(')'); - - int count = new SqlExecutor(schema.getSchema()).execute(sqlf); - if (count > 0) - StudyManager.datasetModified(def, true); - } - - for (VisitImpl visit : visits) - { - // Delete specimens first because we may need ParticipantVisit to figure out which specimens - SpecimenManager.get().deleteSpecimensForVisit(visit); - StudyDesignService svc = StudyDesignService.get(); - if (svc != null) - { - svc.deleteTreatmentVisitMapForVisit(study.getContainer(), visit.getRowId()); - svc.deleteAssaySpecimenVisits(study.getContainer(), visit.getRowId()); - } - } - } - - SQLFragment sqlFragParticipantVisit = new SQLFragment("DELETE FROM " + schema.getTableInfoParticipantVisit() + "\n" + - "WHERE Container = ?").add(study.getContainer()); - sqlFragParticipantVisit.append(" AND VisitRowId ").append(visitInClause); - new SqlExecutor(schema.getSchema()).execute(sqlFragParticipantVisit); - - SQLFragment sqlFragVisitMap = new SQLFragment("DELETE FROM " + schema.getTableInfoVisitMap() + "\n" + - "WHERE Container = ?").add(study.getContainer()); - sqlFragVisitMap.append(" AND VisitRowId ").append(visitInClause); - new SqlExecutor(schema.getSchema()).execute(sqlFragVisitMap); - - // UNDONE broken _visitHelper.delete(visit); - try - { - Study visitStudy = getStudyForVisits(study); - Container c = visitStudy.getContainer(); - - try - { - for (VisitImpl visit : visits) - { - Table.delete(schema.getTableInfoVisit(), new Object[]{c, visit.getRowId()}); - } - } - finally - { - _visitHelper.clearCache(c); - } - } - catch (OptimisticConflictException x) - { - /* ignore */ - } - - transaction.commit(); - - getVisitManager(study).updateParticipantVisits(user, study.getDatasets()); - } - } - - public void updateVisit(User user, VisitImpl visit) - { - _visitHelper.update(user, visit, visit.getContainer().getId(), visit.getRowId()); - } - - public void updateCohort(User user, CohortImpl cohort) - { - _cohortHelper.update(user, cohort); - } - - public void updateParticipant(User user, Participant participant) - { - Container c = participant.getContainer(); - Table.update(user, SCHEMA.getTableInfoParticipant(), participant, new Object[]{c.getId(), participant.getParticipantId()}); - _participantCache.remove(c); - } - - public void createVisitDatasetMapping(User user, Container container, int visitId, int datasetId, boolean isRequired) - { - VisitDataset vds = new VisitDataset(container, datasetId, visitId, isRequired); - Table.insert(user, SCHEMA.getTableInfoVisitMap(), vds); - } - - public VisitDataset getVisitDatasetMapping(Container container, int visitRowId, int datasetId) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("VisitRowId"), visitRowId); - filter.addCondition(FieldKey.fromParts("DataSetId"), datasetId); - - Boolean required = new TableSelector(SCHEMA.getTableInfoVisitMap().getColumn("Required"), filter, null).getObject(Boolean.class); - - return (null != required ? new VisitDataset(container, datasetId, visitRowId, required) : null); - } - - public Collection getVisits(Study study, Order order) - { - return getVisits(study, null, null, order); - } - - public Collection getVisits(Study study, @Nullable Cohort cohort, @Nullable User user, Order order) - { - if (study.getTimepointType() == TimepointType.CONTINUOUS) - return Collections.emptyList(); - - Study visitStudy = getStudyForVisits(study); - Collection visits = _visitHelper.getCollection(visitStudy.getContainer(), order); - if (cohort != null && showCohorts(study.getContainer(), user)) - { - // We could cache all combinations of cohort x order instead of filtering on-the-fly, but this seems fast enough - visits = visits.stream() - .filter(visit -> visit.getCohortId() == null || visit.getCohortId() == cohort.getRowId()) - .toList(); - } - - return visits; - } - - public void clearParticipantVisitCaches(Study study) - { - _visitHelper.clearCache(study.getContainer()); - - // clear shared study - Study visitStudy = getStudyForVisits(study); - if (!study.equals(visitStudy)) - _visitHelper.clearCache(visitStudy.getContainer()); - - _participantCache.remove(study.getContainer()); - } - - public VisitImpl getVisitForRowId(Study study, int rowId) - { - Study visitStudy = getStudyForVisits(study); - - return _visitHelper.get(visitStudy.getContainer(), rowId); - } - - /** - * Helper to insert a new QCState and manage some study specific behavior - */ - public DataState insertQCState(User user, DataState state) - { - boolean isFirst = QCStateManager.getInstance().getStates(state.getContainer()).isEmpty(); - DataState newState = QCStateManager.getInstance().insertState(user, state); - if (isFirst) - // switching from zero to more than zero QC states affects the columns in our materialized datasets - // (adding a QC State column), so we unmaterialize them here: - StudyManager.getInstance().clearCaches(state.getContainer(), true); - - return newState; - } - - @Nullable - public DataState getDefaultQCState(StudyImpl study) - { - Long defaultQcStateId = study.getDefaultDirectEntryQCState(); - DataState defaultQCState = null; - if (defaultQcStateId != null) - defaultQCState = QCStateManager.getInstance().getStateForRowId( - study.getContainer(), defaultQcStateId); - return defaultQCState; - } - - private Map getVisitsForDataRows(DatasetDefinition def, Collection dataLsids) - { - final Map visits = new HashMap<>(); - - if (dataLsids == null || dataLsids.isEmpty()) - return visits; - - final Study study = def.getStudy(); - final Study visitStudy = getStudyForVisits(study); - - TableInfo ds = def.getDatasetSchemaTableInfo(null, false); - - SQLFragment sql = new SQLFragment(); - sql.append("SELECT sd.LSID AS LSID, v.RowId AS RowId FROM ").append(ds.getFromSQL("sd")).append("\n" + - "JOIN study.ParticipantVisit pv ON \n" + - "\tsd.SequenceNum = pv.SequenceNum AND\n" + - "\tsd.ParticipantId = pv.ParticipantId\n" + - "JOIN study.Visit v ON\n" + - "\tpv.VisitRowId = v.RowId AND\n" + - "\tpv.Container = ? AND v.Container = ?\n" + - "WHERE sd.lsid "); - sql.add(def.getContainer().getId()); - // shared visit container - sql.add(visitStudy.getContainer().getId()); - - StudySchema.getInstance().getSqlDialect().appendInClauseSql(sql, dataLsids); - - new SqlSelector(StudySchema.getInstance().getSchema(), sql).forEach(rs -> { - String lsid = rs.getString("LSID"); - int visitId = rs.getInt("RowId"); - visits.put(lsid, getVisitForRowId(study, visitId)); - }); - - return visits; - } - - public List getVisitsForDataset(Container container, int datasetId) - { - List visits = new ArrayList<>(); - - DatasetDefinition def = getDatasetDefinition(getStudy(container), datasetId); - TableInfo ds = def.getDatasetSchemaTableInfo(null, false); - - final Study study = def.getStudy(); - final Study visitStudy = getStudyForVisits(study); - - SQLFragment sql = new SQLFragment(); - sql.append("SELECT DISTINCT v.RowId AS RowId FROM ").append(ds.getFromSQL("sd")).append("\n" + - "JOIN study.ParticipantVisit pv ON \n" + - "\tsd.SequenceNum = pv.SequenceNum AND\n" + - "\tsd.ParticipantId = pv.ParticipantId\n" + - "JOIN study.Visit v ON\n" + - "\tpv.VisitRowId = v.RowId AND\n" + - "\tpv.Container = ? AND v.Container = ?\n"); - sql.add(container.getId()); - // shared visit container - sql.add(visitStudy.getContainer().getId()); - - SqlSelector selector = new SqlSelector(StudySchema.getInstance().getSchema(), sql); - for (Integer rowId : selector.getArray(Integer.class)) - { - visits.add(getVisitForRowId(study, rowId)); - } - return visits; - } - - public void updateDataQCState(Container container, User user, int datasetId, Collection lsids, DataState newState, String comments) - { - DbScope scope = StudySchema.getInstance().getSchema().getScope(); - Study study = getStudy(container); - DatasetDefinition def = getDatasetDefinition(study, datasetId); - - Map lsidVisits = null; - if (!def.isDemographicData()) - lsidVisits = getVisitsForDataRows(def, lsids); - List> rows = def.getDatasetRows(user, lsids); - if (rows.isEmpty()) - return; - - Map oldQCStates = new HashMap<>(); - Map newQCStates = new HashMap<>(); - - Set updateLsids = new HashSet<>(); - for (Map row : rows) - { - String lsid = (String) row.get("lsid"); - - Long oldStateId = MapUtils.getLong(row,DatasetTableImpl.QCSTATE_ID_COLNAME); - DataState oldState = null; - if (oldStateId != null) - oldState = QCStateManager.getInstance().getStateForRowId(container, oldStateId); - - // check to see if we're actually changing state. If not, no-op: - if (safeIntegersEqual(newState != null ? newState.getRowId() : null, oldStateId)) - continue; - - updateLsids.add(lsid); - - StringBuilder auditKey = new StringBuilder(StudyService.get().getSubjectNounSingular(container) + " "); - auditKey.append(row.get(StudyService.get().getSubjectColumnName(container))); - if (!def.isDemographicData()) - { - VisitImpl visit = lsidVisits.get(lsid); - auditKey.append(", Visit ").append(visit != null ? visit.getLabel() : "unknown"); - } - String keyProp = def.getKeyPropertyName(); - if (keyProp != null) - { - auditKey.append(", ").append(keyProp).append(" ").append(row.get(keyProp)); - } - - oldQCStates.put(auditKey.toString(), oldState != null ? oldState.getLabel() : "unspecified"); - newQCStates.put(auditKey.toString(), newState != null ? newState.getLabel() : "unspecified"); - } - - if (updateLsids.isEmpty()) - return; - - try (Transaction transaction = scope.ensureTransaction()) - { - // TODO fix updating across study data - SQLFragment sql = new SQLFragment("UPDATE " ).append(def.getStorageTableInfo(false)); - sql.append(" SET QCState = "); - // do string concatenation, rather that using a parameter, for the new state id because Postgres null - // parameters are typed which causes a cast exception trying to set the value back to null (bug 6370) - sql.appendValue(newState != null ? newState.getRowId() : null); - sql.append(", modified = ?"); - sql.add(new Date()); - sql.append("\nWHERE lsid "); - StudySchema.getInstance().getSqlDialect().appendInClauseSql(sql, updateLsids); - - new SqlExecutor(StudySchema.getInstance().getSchema()).execute(sql); - - //def.deleteFromMaterialized(user, updateLsids); - //def.insertIntoMaterialized(user, updateLsids); - - String auditComment = "QC state was changed for " + updateLsids.size() + " record" + - (updateLsids.size() == 1 ? "" : "s") + ". User comment: " + comments; - - DatasetAuditProvider.DatasetAuditEvent event = new DatasetAuditProvider.DatasetAuditEvent(container, auditComment, datasetId); - event.setHasDetails(true); - event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldQCStates)); - event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newQCStates)); - - AuditLogService.get().addEvent(user, event); - clearCaches(container, false); - - transaction.commit(); - } - } - - public static boolean safeIntegersEqual(Integer first, Integer second) - { - if (first == null && second == null) - return true; - if (first == null) - return false; - return first.equals(second); - } - - public static boolean safeIntegersEqual(Long first, Long second) - { - if (first == null && second == null) - return true; - if (first == null) - return false; - return first.equals(second); - } - - public boolean showCohorts(Container container, @Nullable User user) - { - if (user == null) - return false; - - if (user.hasRootAdminPermission()) - return true; - - StudyImpl study = StudyManager.getInstance().getStudy(container); - - if (study == null) - return false; - - Integer cohortDatasetId = study.getParticipantCohortDatasetId(); - if (study.isManualCohortAssignment() || null == cohortDatasetId || -1 == cohortDatasetId) - { - // If we're not reading from a dataset for cohort definition, - // we use the container's permission - return container.hasPermission(user, ReadPermission.class); - } - - // Automatic cohort assignment -- can the user read the source dataset? - DatasetDefinition def = getDatasetDefinition(study, cohortDatasetId); - - if (def != null) - return def.canReadInternal(user); - - return false; - } - - public void assertCohortsViewable(Container container, User user) - { - if (!showCohorts(container, user)) - throw new UnauthorizedException("User does not have permission to view cohort information"); - } - - public Collection getCohorts(Container container, User user) - { - assertCohortsViewable(container, user); - return _cohortHelper.getCollection(container); - } - - public CohortImpl getCurrentCohortForParticipant(Container container, User user, String participantId) - { - assertCohortsViewable(container, user); - Participant participant = getParticipant(getStudy(container), participantId); - if (participant != null && participant.getCurrentCohortId() != null) - return _cohortHelper.get(container, participant.getCurrentCohortId().intValue()); - return null; - } - - public CohortImpl getCohortForRowId(Container container, User user, int rowId) - { - assertCohortsViewable(container, user); - return _cohortHelper.get(container, rowId); - } - - public CohortImpl getCohortByLabel(Container container, User user, String label) - { - assertCohortsViewable(container, user); - - List cohorts = _cohortHelper.getCollection(container).stream() - .filter(cohort -> cohort.getLabel().equals(label)) - .toList(); - - if (cohorts.size() == 1) - return cohorts.get(0); - - return null; - } - - private boolean isCohortInUse(CohortImpl cohort, Container c, TableInfo table, String... columnNames) - { - List params = new ArrayList<>(); - params.add(c.getId()); - - StringBuilder cols = new StringBuilder("("); - String or = ""; - for (String columnName : columnNames) - { - cols.append(or).append(columnName).append(" = ?"); - params.add(cohort.getRowId()); - or = " OR "; - } - cols.append(")"); - - return new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT * FROM " + - table + " WHERE Container = ? AND " + cols, params).exists(); - } - - public boolean isCohortInUse(CohortImpl cohort) - { - Container c = cohort.getContainer(); - Study visitStudy = getStudyForVisits(getStudy(c)); - - return isCohortInUse(cohort, c, StudySchema.getInstance().getTableInfoDataset(), "CohortId") || - isCohortInUse(cohort, c, StudySchema.getInstance().getTableInfoParticipant(), "CurrentCohortId", "InitialCohortId") || - isCohortInUse(cohort, c, StudySchema.getInstance().getTableInfoParticipantVisit(), "CohortId") || - isCohortInUse(cohort, visitStudy.getContainer(), StudySchema.getInstance().getTableInfoVisit(), "CohortId"); - } - - public void deleteCohort(CohortImpl cohort) - { - StudySchema schema = StudySchema.getInstance(); - - try (Transaction transaction = schema.getSchema().getScope().ensureTransaction()) - { - Container container = cohort.getContainer(); - StudyDesignService svc = StudyDesignService.get(); - if (svc != null) - svc.deleteTreatmentVisitMapForCohort(container, cohort.getRowId()); - _cohortHelper.delete(cohort); - - // delete extended properties - String lsid = cohort.getLsid(); - Map resourceProperties = OntologyManager.getPropertyObjects(container, lsid); - if (resourceProperties != null && !resourceProperties.isEmpty()) - { - OntologyManager.deleteOntologyObject(lsid, container, false); - } - - transaction.commit(); - } - } - - public VisitImpl getVisitForSequence(Study study, BigDecimal seqNum) - { - Collection visits = getVisits(study, Order.SEQUENCE_NUM); - for (VisitImpl v : visits) - { - if (v.isInRange(seqNum)) - return v; - } - return null; - } - - public List getDatasetDefinitions(Study study) - { - return getDatasetDefinitions(study, null); - } - - public List getDatasetDefinitions(Study study, @Nullable Cohort cohort, String... types) - { - List local = getDatasetDefinitionsLocal(study, cohort, types); - List shared = Collections.emptyList(); - List combined; - - Study sharedStudy = getSharedStudy(study); - if (null != sharedStudy) - shared = getDatasetDefinitionsLocal(sharedStudy, cohort, types); - - if (shared.isEmpty()) - combined = local; - else - { - // NOTE: it's confusing that both ID and name are unique, manage page should warn about funny inconsistencies - // NOTE: here we'll have LOCAL datasets hide SHARED datasets by both id and name until I have a better idea - CaseInsensitiveHashSet names = new CaseInsensitiveHashSet(); - HashSet ids = new HashSet<>(); - - combined = new ArrayList<>(local.size() + shared.size()); - for (DatasetDefinition dsd : local) - { - combined.add(dsd); - names.add(dsd.getName()); - ids.add(dsd.getDatasetId()); - } - for (DatasetDefinition dsd : shared) - { - if (!names.contains(dsd.getName()) && !ids.contains(dsd.getDatasetId())) - { - DatasetDefinition wrapped = dsd.createLocalDatasetDefinition((StudyImpl) study); - combined.add(wrapped); - } - } - } - - // sort by display order, category, and dataset ID - combined.sort(DATASET_ORDER_COMPARATOR); - - return Collections.unmodifiableList(combined); - } - - public static final Comparator DATASET_ORDER_COMPARATOR = new Comparator<>() - { - @Override - public int compare(DatasetDefinition o1, DatasetDefinition o2) - { - if (o1.getDisplayOrder() != 0 || o2.getDisplayOrder() != 0) - return o1.getDisplayOrder() - o2.getDisplayOrder(); - - if (Strings.CS.equals(o1.getCategory(), o2.getCategory())) - return o1.getDatasetId() - o2.getDatasetId(); - - if (o1.getCategory() != null && o2.getCategory() == null) - return -1; - if (o1.getCategory() == null && o2.getCategory() != null) - return 1; - if (o1.getCategory() != null && o2.getCategory() != null) - return o1.getCategory().compareTo(o2.getCategory()); - - return o1.getDatasetId() - o2.getDatasetId(); - } - }; - - /** - * Get the list of datasets that are 'shadowed' by the list of local dataset definitions or for any local dataset in the study. - * This is pretty much the inverse of getDatasetDefinitions() - * This can be used in the management/admin UI to warn about shadowed datasets - */ - public List getShadowedDatasets(@NotNull Study study, @Nullable List local) - { - if (study.getContainer().isProject()) - return Collections.emptyList(); - - Study sharedStudy = getSharedStudy(study); - if (null == sharedStudy) - return Collections.emptyList(); - - if (null == local) - local = getDatasetDefinitionsLocal(study); - List shared = getDatasetDefinitionsLocal(sharedStudy); - - if (local.isEmpty() || shared.isEmpty()) - return Collections.emptyList(); - - CaseInsensitiveHashSet names = new CaseInsensitiveHashSet(); - HashSet ids = new HashSet<>(); - - for (DatasetDefinition dsd : local) - { - if (dsd.getDefinitionContainer().equals(dsd.getContainer())) - { - names.add(dsd.getName()); - ids.add(dsd.getDatasetId()); - } - } - Map shadowed = new TreeMap<>(); - for (DatasetDefinition dsd : shared) - { - if (names.contains(dsd.getName()) || ids.contains(dsd.getDatasetId())) - shadowed.put(dsd.getDatasetId(), dsd); - } - - return new ArrayList<>(shadowed.values()); - } - - public List getDatasetDefinitionsLocal(Study study) - { - return getDatasetDefinitionsLocal(study, null); - } - - public List getDatasetDefinitionsLocal(Study study, @Nullable Cohort cohort, String... types) - { - Collection ret = cohort != null ? _datasetHelper.getDatasetsForCohort(study, cohort) : _datasetHelper.getCollection(study.getContainer()); - - if (types != null && types.length > 0) - { - Set typeSet = Set.of(types); - ret = ret.stream().filter(def -> typeSet.contains(def.getType())).toList(); - } - - // Make a copy (it's immutable) so that we can sort it. See issue 17875 - return new ArrayList<>(ret); - } - - public Set getSharedProperties(Study study) - { - return _sharedProperties.get(study.getContainer()); - } - - @Nullable - public DatasetDefinition getDatasetDefinition(Study s, int id) - { - DatasetDefinition ds = _datasetHelper.get(s.getContainer(), id); - if (null != ds) - return ds; - - Study sharedStudy = getSharedStudy(s); - if (null == sharedStudy) - return null; - - ds = getDatasetDefinition(sharedStudy, id); - if (null == ds) - return null; - return ds.createLocalDatasetDefinition((StudyImpl) s); - } - - - @Nullable - public DatasetDefinition getDatasetDefinitionByLabel(Study s, String label) - { - if (label == null) - { - return null; - } - - return _datasetHelper.getByLabel(s, label); - } - - - @Nullable - public DatasetDefinition getDatasetDefinitionByEntityId(Study s, String entityId) - { - return _datasetHelper.getByEntityId(s, entityId); - } - - - @Nullable - public DatasetDefinition getDatasetDefinitionByName(Study s, String name) - { - DatasetDefinition def = _datasetHelper.getByName(s, name); - if (def != null) - return def; - - Study sharedStudy = getSharedStudy(s); - if (null == sharedStudy) - return null; - - def = getDatasetDefinitionByName(sharedStudy, name); - if (null == def) - return null; - return def.createLocalDatasetDefinition((StudyImpl) s); - } - - - @Nullable - public DatasetDefinition getDatasetDefinitionByQueryName(Study study, String queryName) - { - // first try resolving the dataset def by name and then by label - DatasetDefinition def = getDatasetDefinitionByName(study, queryName); - if (null != def) - return def; - def = StudyManager.getInstance().getDatasetDefinitionByLabel(study, queryName); - if (null != def) - return def; - - // try shared study - if (study.getContainer().isProject()) - return null; - Study shared = StudyManager.getInstance().getSharedStudy(study); - if (null == shared) - return null; - - // first try resolving the dataset def by name and then by label - def = StudyManager.getInstance().getDatasetDefinitionByName(shared, queryName); - if (null != def) - return def.createLocalDatasetDefinition((StudyImpl) study); - def = StudyManager.getInstance().getDatasetDefinitionByLabel(shared, queryName); - if (null != def) - return def.createLocalDatasetDefinition((StudyImpl) study); - - return null; - } - - - // domainURI -> - private static final Cache> domainCache = CacheManager.getCache(5000, CacheManager.DAY, "Domain->Dataset map"); - - private static final CacheLoader> loader = (domainURI, argument) -> { - SQLFragment sql = new SQLFragment(); - sql.append("SELECT Container, DatasetId FROM study.Dataset WHERE TypeURI=?"); - sql.add(domainURI); - - Map map = new SqlSelector(StudySchema.getInstance().getSchema(), sql).getMap(); - - if (null == map) - return null; - else - return new Pair<>((String)map.get("Container"), asInteger(map.get("DatasetId"))); - }; - - - @Nullable - DatasetDefinition getDatasetDefinition(String domainURI) - { - for (int retry=0 ; retry < 2 ; retry++) - { - Pair p = domainCache.get(domainURI, null, loader); - if (null == p) - return null; - - Container c = ContainerManager.getForId(p.first); - if (c != null) - { - Study study = StudyManager.getInstance().getStudy(c); - if (null != study) - { - DatasetDefinition ret = StudyManager.getInstance().getDatasetDefinition(study, p.second); - if (null != ret && null != ret.getDomain() && Strings.CI.equals(ret.getDomain().getTypeURI(), domainURI)) - return ret; - } - } - domainCache.remove(domainURI); - } - return null; - } - - - public List getDatasetLSIDs(User user, DatasetDefinition def) - { - TableInfo tInfo = def.getTableInfo(user); - return new TableSelector(tInfo.getColumn("lsid")).getArrayList(String.class); - } - - - public void uncache(DatasetDefinition def) - { - if (null == def) - return; - - _log.debug("Uncaching dataset: " + def.getName(), new Throwable()); - - _datasetHelper.clearCache(def.getContainer()); - String uri = def.getTypeURI(); - if (null != uri) - domainCache.remove(uri); - - // Also clear caches of subjects and visits; changes to this dataset may have affected this data: - clearParticipantVisitCaches(def.getStudy()); - } - - public Map getRequiredMap(Study study) - { - TableInfo tableVisitMap = StudySchema.getInstance().getTableInfoVisitMap(); - final HashMap map = new HashMap<>(); - - new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT DatasetId, VisitRowId, Required FROM " + tableVisitMap + " WHERE Container = ?", - study.getContainer()).forEach(rs -> map.put(new VisitMapKey(rs.getInt(1), rs.getInt(2)), rs.getBoolean(3))); - - return map; - } - - private static final String VISITMAP_JOIN_BY_VISIT = """ - SELECT d.*, vm.Required FROM study.Visit v, study.DataSet d, study.VisitMap vm - WHERE v.RowId = vm.VisitRowId AND vm.DataSetId = d.DataSetId AND v.Container = vm.Container AND - vm.Container = d.Container AND v.Container = ? AND v.RowId = ? - ORDER BY d.DisplayOrder, d.DataSetId"""; - - private static final String VISITMAP_JOIN_BY_DATASET = """ - SELECT vm.VisitRowId, vm.Required - FROM study.VisitMap vm JOIN study.Visit v ON vm.VisitRowId = v.RowId - WHERE vm.Container = ? AND vm.DataSetId = ? - ORDER BY v.DisplayOrder, v.RowId"""; - - List getMapping(final VisitImpl visit) - { - if (visit.getContainer() == null) - throw new IllegalStateException("Visit has no container"); - - final List visitDatasets = new ArrayList<>(); - - new SqlSelector(StudySchema.getInstance().getSchema(), VISITMAP_JOIN_BY_VISIT, - visit.getContainer(), visit.getRowId()).forEach(rs -> { - int datasetId = rs.getInt("DataSetId"); - boolean isRequired = rs.getBoolean("Required"); - visitDatasets.add(new VisitDataset(visit.getContainer(), datasetId, visit.getRowId(), isRequired)); - }); - - return visitDatasets; - } - - - public List getMapping(final Dataset dataset) - { - final List visitDatasets = new ArrayList<>(); - - new SqlSelector(StudySchema.getInstance().getSchema(), VISITMAP_JOIN_BY_DATASET, - dataset.getContainer(), dataset.getDatasetId()).forEach(rs -> { - int visitRowId = rs.getInt("VisitRowId"); - boolean isRequired = rs.getBoolean("Required"); - visitDatasets.add(new VisitDataset(dataset.getContainer(), dataset.getDatasetId(), visitRowId, isRequired)); - }); - - return visitDatasets; - } - - - public void updateVisitDatasetMapping(User user, Container container, int visitId, - int datasetId, VisitDatasetType type) - { - VisitDataset vds = getVisitDatasetMapping(container, visitId, datasetId); - if (vds == null) - { - if (type != VisitDatasetType.NOT_ASSOCIATED) - { - // need to insert a new VisitMap entry: - createVisitDatasetMapping(user, container, visitId, - datasetId, type == VisitDatasetType.REQUIRED); - } - } - else if (type == VisitDatasetType.NOT_ASSOCIATED) - { - // need to remove an existing VisitMap entry: - Table.delete(SCHEMA.getTableInfoVisitMap(), - new Object[] { container.getId(), visitId, datasetId}); - } - else if ((VisitDatasetType.OPTIONAL == type && vds.isRequired()) || - (VisitDatasetType.REQUIRED == type && !vds.isRequired())) - { - Map required = new HashMap<>(1); - required.put("Required", VisitDatasetType.REQUIRED == type ? Boolean.TRUE : Boolean.FALSE); - Table.update(user, SCHEMA.getTableInfoVisitMap(), required, - new Object[]{container.getId(), visitId, datasetId}); - } - } - - public long getNumDatasetRows(User user, Dataset dataset) - { - TableInfo sdTable = dataset.getTableInfo(user); - return new TableSelector(sdTable).getRowCount(); - } - - - /** - * Delete all rows from a dataset or just those newer than the cutoff date. - */ - public int purgeDataset(DatasetDefinition dataset, @Nullable Date cutoff) - { - return dataset.deleteRows(cutoff); - } - - /** - * delete a dataset definition along with associated type, data, visitmap entries - * @param performStudyResync whether to kick off our normal bookkeeping. If the whole study is being deleted, - * we don't need to bother doing this, for example. - */ - public void deleteDataset(StudyImpl study, User user, DatasetDefinition ds, boolean performStudyResync, @Nullable String auditUserComment) - { - try (Transaction transaction = StudySchema.getInstance().getScope().ensureTransaction()) - { - if (!ds.canDeleteDefinition(user)) - throw new UnauthorizedException("Can't delete dataset: " + ds.getName()); - - // When the dataset is deleted, the provenance rows should be cleaned up - ProvenanceService pvs = ProvenanceService.get(); - - Collection allDatasetLsids = pvs.getDatasetProvenanceLsids(user, ds); - - allDatasetLsids.forEach(lsid -> { - Set protocolApplications = pvs.getProtocolApplications(lsid); - - OntologyObject expObject = OntologyManager.getOntologyObject(null, lsid); - if (null != expObject) - { - pvs.deleteObjectProvenance(expObject.getObjectId()); - } - - if (!protocolApplications.isEmpty()) - { - ExperimentService expService = ExperimentService.get(); - protocolApplications.forEach(protocolApp -> { - ExpRun run = expService.getExpProtocolApplication(protocolApp).getRun(); - expService.deleteExperimentRunsByRowIds(study.getContainer(), user, run.getRowId()); - }); - } - }); - - try - { - deleteDatasetType(study, user, ds, auditUserComment); - - QuerySnapshotDefinition def = QueryService.get().getSnapshotDef(study.getContainer(), StudySchema.getInstance().getSchemaName(), ds.getName()); - if (def != null) - def.delete(user); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - new SqlExecutor(StudySchema.getInstance().getSchema()).execute("DELETE FROM " + SCHEMA.getTableInfoVisitMap() + "\n" + - "WHERE Container=? AND DatasetId=?", study.getContainer(), ds.getDatasetId()); - - // UNDONE: This is broken - // _datasetHelper.delete(ds); - new SqlExecutor(StudySchema.getInstance().getSchema()).execute("DELETE FROM " + StudySchema.getInstance().getTableInfoDataset() + "\n" + - "WHERE Container=? AND DatasetId=?", study.getContainer(), ds.getDatasetId()); - _datasetHelper.clearCache(study.getContainer()); - - SecurityPolicyManager.deletePolicy(ds); - - if (safeIntegersEqual(ds.getDatasetId(), study.getParticipantCohortDatasetId())) - CohortManager.getInstance().setManualCohortAssignment(study, user, Collections.emptyMap()); - - if (performStudyResync) - { - // This dataset may have contained the only references to some subjects or visits; as a result, we need - // to re-sync the participant and participant/visit tables. (Issue 12447) - // Don't provide the deleted dataset in the list of modified datasets; deletion doesn't count as a modification - // within VisitManager, and passing in the empty set ensures that all subject/visit info will be recalculated. - getVisitManager(study).updateParticipantVisits(user, Collections.emptySet()); - } - - SchemaKey schemaPath = SchemaKey.fromParts(SCHEMA.getSchemaName()); - QueryService.get().fireQueryDeleted(user, study.getContainer(), null, schemaPath, Collections.singleton(ds.getName())); - new DatasetDefinition.DatasetAuditHandler(ds).addAuditEvent(user, study.getContainer(), AuditBehaviorType.DETAILED, "Dataset deleted: " + ds.getName(), null); - - transaction.addCommitTask(() -> - unindexDataset(ds), - CommitTaskOption.POSTCOMMIT - ); - - transaction.commit(); - } - } - - /** delete a dataset type and data - * does not clear typeURI as we're about to delete the dataset - */ - private void deleteDatasetType(Study study, User user, DatasetDefinition ds, @Nullable String auditUserComment) - { - assert StudySchema.getInstance().getSchema().getScope().isTransactionActive(); - - if (null == ds) - return; - - if (!ds.canDeleteDefinition(user)) - throw new IllegalStateException("Can't delete dataset: " + ds.getName()); - - Domain domain = ds.getDomain(); - if (domain == null) - return; - - DomainAuditProvider.DomainAuditEvent event = new DomainAuditProvider.DomainAuditEvent(study.getContainer(), String.format("The domain %s was deleted", domain.getName())); - event.setUserComment(auditUserComment); - event.setDomainUri(domain.getTypeURI()); - event.setDomainName(domain.getName()); - AuditLogService.get().addEvent(user, event); - - StorageProvisioner.get().drop(domain); - - if (ds.getTypeURI() != null) - { - try - { - OntologyManager.deleteType(ds.getTypeURI(), study.getContainer()); - } - catch (DomainNotFoundException x) - { - // continue - } - } - } - - // Any container can be passed here (whether it contains a study or not). - public void clearCaches(Container c, boolean unmaterializeDatasets) - { - Study study = getStudy(c); - _studyHelper.clearCache(c); - _visitHelper.clearCache(c); - LocationCache.clear(c); - AssayService.get().clearProtocolCache(); - if (unmaterializeDatasets && null != study) - for (DatasetDefinition def : getDatasetDefinitions(study)) - uncache(def); - // Aggressive, but datasets are cached with container objects that might go stale, for example, when moving a - // folder tree to another parent, the datasets in subfolders will be left with invalid paths. See FolderTest. - _datasetHelper.clearCache(); - _cohortHelper.clearCache(c); - _participantCache.remove(c); - } - - public void deleteAllStudyData(Container c, User user) - { - // No need to delete individual participants if the whole study is going away - VisitManager.cancelParticipantPurge(c); - - // Before we delete any data, we need to go fetch the Dataset definitions. - StudyImpl study = StudyManager.getInstance().getStudy(c); - List dsds; - if (study == null) // no study in this folder - dsds = Collections.emptyList(); - else - dsds = study.getDatasets(); - - // get the list of study design tables - List studyDesignTables = getStudyDesignTables(c, user); - - DbScope scope = StudySchema.getInstance().getSchema().getScope(); - - Set deletedTables = new HashSet<>(); - SimpleFilter containerFilter = SimpleFilter.createContainerFilter(c); - - try (Transaction transaction = scope.ensureTransaction()) - { - StudyDesignManager.get().deleteStudyDesignData(c, deletedTables); - - for (DatasetDefinition dsd : dsds) - { - if (dsd.getContainer().equals(dsd.getDefinitionContainer())) - deleteDataset(study, user, dsd, false, null); - else - dsd.deleteAllRows(user); - } - - // - // specimens - // - SpecimenService ss = SpecimenService.get(); - if (null != ss) - ss.deleteAllSpecimenData(c, deletedTables, user); - - // Since study creates these tables, study needs to delete them - new SpecimenTablesProvider(c, null, null).deleteTables(); - LocationCache.clear(c); - - // - // metadata - // - Table.delete(SCHEMA.getTableInfoVisitMap(), containerFilter); - assert deletedTables.add(SCHEMA.getTableInfoVisitMap()); - Table.delete(StudySchema.getInstance().getTableInfoUploadLog(), containerFilter); - assert deletedTables.add(StudySchema.getInstance().getTableInfoUploadLog()); - Table.delete(_datasetHelper.getTableInfo(), containerFilter); - _datasetHelper.clearCache(c); - assert deletedTables.add(_datasetHelper.getTableInfo()); - Table.delete(_visitHelper.getTableInfo(), containerFilter); - _visitHelper.clearCache(c); - assert deletedTables.add(_visitHelper.getTableInfo()); - Table.delete(_studyHelper.getTableInfo(), containerFilter); - _studyHelper.clearCache(c); - assert deletedTables.add(_studyHelper.getTableInfo()); - - // participant lists - Table.delete(ParticipantGroupManager.getTableInfoParticipantGroupMap(), containerFilter); - assert deletedTables.add(ParticipantGroupManager.getTableInfoParticipantGroupMap()); - Table.delete(ParticipantGroupManager.getTableInfoParticipantGroup(), containerFilter); - assert deletedTables.add(ParticipantGroupManager.getTableInfoParticipantGroup()); - Table.delete(StudySchema.getInstance().getTableInfoParticipantCategory(), containerFilter); - assert deletedTables.add(StudySchema.getInstance().getTableInfoParticipantCategory()); - ParticipantGroupManager.getInstance().clearCache(c); - - // - // participant data (OntologyManager will take care of properties) - // - // Table.delete(StudySchema.getInstance().getTableInfoStudyData(null), containerFilter); - //assert deletedTables.add(StudySchema.getInstance().getTableInfoStudyData(null)); - Table.delete(StudySchema.getInstance().getTableInfoParticipantVisit(), containerFilter); - assert deletedTables.add(StudySchema.getInstance().getTableInfoParticipantVisit()); - Table.delete(StudySchema.getInstance().getTableInfoVisitAliases(), containerFilter); - assert deletedTables.add(StudySchema.getInstance().getTableInfoVisitAliases()); - Table.delete(SCHEMA.getTableInfoParticipant(), containerFilter); - _participantCache.remove(c); - assert deletedTables.add(SCHEMA.getTableInfoParticipant()); - Table.delete(_cohortHelper.getTableInfo(), containerFilter); - _cohortHelper.clearCache(c); - assert deletedTables.add(StudySchema.getInstance().getTableInfoCohort()); - Table.delete(StudySchema.getInstance().getTableInfoParticipantView(), containerFilter); - assert deletedTables.add(StudySchema.getInstance().getTableInfoParticipantView()); - - // participant group cohort union view - assert deletedTables.add(StudySchema.getInstance().getSchema().getTable(StudyQuerySchema.PARTICIPANT_GROUP_COHORT_UNION_TABLE_NAME)); - - // Specimen comments - Table.delete(SpecimenSchema.get().getTableInfoSpecimenComment(), containerFilter); - assert deletedTables.add(SpecimenSchema.get().getTableInfoSpecimenComment()); - - deleteStudyDesignData(c, user, studyDesignTables); - - Table.delete(StudySchema.getInstance().getTableInfoVisitTag(), containerFilter); - assert deletedTables.add(StudySchema.getInstance().getTableInfoVisitTag()); - Table.delete(StudySchema.getInstance().getTableInfoVisitTagMap(), containerFilter); - assert deletedTables.add(StudySchema.getInstance().getTableInfoVisitTagMap()); - - // dataset tables - for (DatasetDefinition dsd : dsds) - { - fireDatasetChanged(dsd); - } - - // Clear this container ID from any source and destination columns of study snapshots. Then delete any - // study snapshots that are orphaned (both source and destination are gone). - SqlExecutor executor = new SqlExecutor(StudySchema.getInstance().getSchema()); - executor.execute(getStudySnapshotUpdateSql(c, "Source")); - executor.execute(getStudySnapshotUpdateSql(c, "Destination")); - - Filter orphanedFilter = new SimpleFilter - ( - new CompareType.CompareClause(FieldKey.fromParts("Source"), CompareType.ISBLANK, null), - new CompareType.CompareClause(FieldKey.fromParts("Destination"), CompareType.ISBLANK, null) - ); - Table.delete(StudySchema.getInstance().getTableInfoStudySnapshot(), orphanedFilter); - - assert deletedTables.add(StudySchema.getInstance().getTableInfoStudySnapshot()); - - transaction.commit(); - } - - ContainerManager.notifyContainerChange(c.getId(), ContainerManager.Property.StudyChange); - - // - // trust and verify... but only when asserts are on - // - - assert verifyAllTablesWereDeleted(deletedTables); - } - - private List getStudyDesignTables(Container c, User user) - { - List studyDesignTables = new ArrayList<>(); - UserSchema schema = QueryService.get().getUserSchema(user, c, StudyQuerySchema.SCHEMA_NAME); - - addIfProvisioned(studyDesignTables, schema, new StudyProductDomainKind(), PRODUCT_TABLE_NAME); - addIfProvisioned(studyDesignTables, schema, new StudyProductAntigenDomainKind(), PRODUCT_ANTIGEN_TABLE_NAME); - addIfProvisioned(studyDesignTables, schema, new StudyTreatmentProductDomainKind(), TREATMENT_PRODUCT_MAP_TABLE_NAME); - addIfProvisioned(studyDesignTables, schema, new StudyTreatmentDomainKind(), TREATMENT_TABLE_NAME); - addIfProvisioned(studyDesignTables, schema, new StudyPersonnelDomainKind(), PERSONNEL_TABLE_NAME); - - return studyDesignTables; - } - - private void addIfProvisioned(List studyDesignTables, UserSchema schema, AbstractStudyDesignDomainKind domainKind, String tableName) - { - // Might not be provisioned (e.g., if this isn't a study) - Domain domain = domainKind.getDomain(schema.getContainer(), tableName); - - if (null != domain) - studyDesignTables.add(schema.getTable(tableName)); - } - - private void deleteStudyDesignData(Container c, User user, List studyDesignTables) - { - for (TableInfo tinfo : studyDesignTables) - { - if (tinfo instanceof FilteredTable) - { - Table.delete(((FilteredTable)tinfo).getRealTable(), new SimpleFilter(FieldKey.fromParts("Container"), c)); - } - } - } - - private SQLFragment getStudySnapshotUpdateSql(Container c, String columnName) - { - SQLFragment sql = new SQLFragment(); - sql.append("UPDATE "); - sql.append(StudySchema.getInstance().getTableInfoStudySnapshot()); - sql.append(" SET "); - sql.append(columnName); - sql.append(" = NULL WHERE "); - sql.append(columnName); - sql.append(" = ?"); - sql.add(c); - - return sql; - } - - // TODO: Check that datasets are deleted as well? - private boolean verifyAllTablesWereDeleted(Set deletedTables) - { - if (1==1) - return true; - - // Pretend like we deleted from StudyData and StudyDataTemplate tables TODO: why aren't we deleting from these? - Set deletedTableNames = new CaseInsensitiveHashSet("studydata", "studydatatemplate"); - - for (TableInfo t : deletedTables) - { - deletedTableNames.add(t.getName()); - } - - StringBuilder missed = new StringBuilder(); - - for (String tableName : StudySchema.getInstance().getSchema().getTableNames()) - { - if (!deletedTableNames.contains(tableName) && - !"specimen".equalsIgnoreCase(tableName) && !"vial".equalsIgnoreCase(tableName) && !"specimenevent".equalsIgnoreCase(tableName) && - !"site".equalsIgnoreCase(tableName) && !"specimenprimarytype".equalsIgnoreCase(tableName) && - !"specimenderivative".equalsIgnoreCase(tableName) && !"specimenadditive".equalsIgnoreCase(tableName)) - { - missed.append(" "); - missed.append(tableName); - } - } - - if (!missed.isEmpty()) - throw new IllegalStateException("Expected to delete from these tables:" + missed); - - return true; - } - - public @NotNull Collection getParticipantDatasets(Container container, Collection lsids) - { - SimpleFilter filter = new SimpleFilter(); - filter.addClause(new SimpleFilter.InClause(FieldKey.fromParts("LSID"), lsids)); - // We can't use the table layer to map results to our bean class because of the unfortunately named - // "_VisitDate" column in study.StudyData. - - TableInfo sdti = StudySchema.getInstance().getTableInfoStudyData(StudyManager.getInstance().getStudy(container), null); - List pds = new ArrayList<>(); - DatasetDefinition dataset = null; - - try (ResultSet rs = new TableSelector(sdti, filter, new Sort("DatasetId")).getResultSet()) - { - ColumnInfo visitDateCol = sdti.getColumn("_VisitDate"); - while (rs.next()) - { - ParticipantDataset pd = new ParticipantDataset(); - pd.setContainer(container); - int datasetId = rs.getInt("DatasetId"); - if (dataset == null || datasetId != dataset.getDatasetId()) - dataset = getDatasetDefinition(getStudy(container), datasetId); - pd.setDatasetId(datasetId); - pd.setLsid(rs.getString("LSID")); - if (!dataset.isDemographicData()) - { - pd.setSequenceNum(rs.getBigDecimal("SequenceNum")); - pd.setVisitDate(rs.getTimestamp(visitDateCol.getAlias().getId())); - } - pd.setParticipantId(rs.getString("ParticipantId")); - pds.add(pd); - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - - return pds; - } - - - /** - * After changing permissions on the study, we have to scrub the dataset acls to - * remove any groups that no longer have read permission. - * - * UNDONE: move StudyManager into model package (so we can have protected access) - */ - protected void scrubDatasetAcls(Study study, SecurityPolicy newPolicy) - { - //for every principal that plays something other than the RestrictedReaderRole, - //delete that group's role assignments in all dataset policies - Role restrictedReader = RoleManager.getRole(RestrictedReaderRole.class); - - Set resources = new HashSet<>(getDatasetDefinitions(study)); - - Set principals = new HashSet<>(); - - for (RoleAssignment ra : newPolicy.getAssignments()) - { - if (!(ra.getRole().equals(restrictedReader))) - principals.add(SecurityManager.getPrincipal(ra.getUserId())); - } - - SecurityPolicyManager.clearRoleAssignments(resources, principals); - } - - - /** study container only (not dataspace!) */ - public long getParticipantCount(Study study) - { - SQLFragment sql = new SQLFragment("SELECT COUNT(ParticipantId) FROM "); - sql.append(SCHEMA.getTableInfoParticipant(), "p"); - sql.append(" WHERE Container = ?"); - sql.add(study.getContainer()); - return new SqlSelector(StudySchema.getInstance().getSchema(), sql).getObject(Long.class); - } - - public Collection getParticipantIds(Study study, User user) - { - return getParticipantIds(study, user, -1); - } - - /** study container only (not dataspace!) */ - public Collection getParticipantIdsForGroup(Study study, User user, int groupId) - { - return getParticipantIds(study, user, null, groupId, -1); - } - - /** study container only (not dataspace!) */ - public Collection getParticipantIds(Study study, User user, int rowLimit) - { - return getParticipantIds(study, user, null, -1, rowLimit); - } - - public Collection getParticipantIds(Study study, User user, ContainerFilter cf, int rowLimit) - { - return getParticipantIds(study, user, cf, -1, rowLimit); - } - - /** study container only (not dataspace!) */ - private Collection getParticipantIds(Study study, User user, ContainerFilter cf, int participantGroupId, int rowLimit) - { - DbSchema schema = StudySchema.getInstance().getSchema(); - SQLFragment sql = getSQLFragmentForParticipantIds(study, user, cf, participantGroupId, rowLimit, schema, "ParticipantId"); - return new SqlSelector(schema, sql).getCollection(String.class); - } - - private static final String ALTERNATEID_COLUMN_NAME = "AlternateId"; - private static final String DATEOFFSET_COLUMN_NAME = "DateOffset"; - private static final String PTID_COLUMN_NAME = "ParticipantId"; - private static final String CONTAINER_COLUMN_NAME = "Container"; - - public Map getParticipantInfos(Study study, User user, final boolean isShiftDates, final boolean isAlternateIds) - { - DbSchema schema = StudySchema.getInstance().getSchema(); - SQLFragment sql = getSQLFragmentForParticipantIds(study, user, null, -1, -1, schema, - CONTAINER_COLUMN_NAME + ", " + PTID_COLUMN_NAME + ", " + ALTERNATEID_COLUMN_NAME + ", " + DATEOFFSET_COLUMN_NAME); - final Map alternateIdMap = new HashMap<>(); - - new SqlSelector(schema, sql).forEach(rs -> { - String containerId = rs.getString(CONTAINER_COLUMN_NAME); - String participantId = rs.getString(PTID_COLUMN_NAME); - String alternateId = isAlternateIds ? rs.getString(ALTERNATEID_COLUMN_NAME) : participantId; // if !isAlternateIds, use participantId - int dateOffset = isShiftDates ? rs.getInt(DATEOFFSET_COLUMN_NAME) : 0; // if !isDateShift, use 0 shift - alternateIdMap.put(participantId, new ParticipantInfo(containerId, alternateId, dateOffset)); - }); - - return alternateIdMap; - } - - - private SQLFragment getSQLFragmentForParticipantIds(Study study, User user, @Nullable ContainerFilter cf, int participantGroupId, int rowLimit, DbSchema schema, String columns) - { - SQLFragment filter = getParticipantFilter(study, user, cf); - - SQLFragment sql; - if (participantGroupId == -1) - { - sql = new SQLFragment("SELECT " + columns + " FROM " + SCHEMA.getTableInfoParticipant()).append(" WHERE ").append(filter).append(" ORDER BY ParticipantId"); - } - else - { - TableInfo table = StudySchema.getInstance().getTableInfoParticipantGroupMap(); - sql = new SQLFragment("SELECT " + columns + " FROM " + table + " WHERE ").append(filter).append(" AND GroupId = ? ORDER BY ParticipantId").add(participantGroupId); - } - if (rowLimit > 0) - sql = schema.getSqlDialect().limitRows(sql, rowLimit); - return sql; - } - - - private SQLFragment getParticipantFilter(Study study, User user, @Nullable ContainerFilter cf) - { - SQLFragment filter = new SQLFragment(); - if (!study.getShareDatasetDefinitions()) - { - filter.append("Container=").appendValue(study.getContainer()); - } - else - { - if (null == user) - throw new IllegalStateException("provide a user to query the participants table"); - if (null == cf) - cf = new DataspaceContainerFilter(user, study); - filter = cf.getSQLFragment(SCHEMA.getSchema(), new SQLFragment("Container")); - } - return filter; - } - - - public String[] getParticipantIdsForCohort(Study study, int currentCohortId, int rowLimit) - { - DbSchema schema = StudySchema.getInstance().getSchema(); - SQLFragment sql = new SQLFragment("SELECT ParticipantId FROM " + SCHEMA.getTableInfoParticipant() + " WHERE Container = ? AND CurrentCohortId = ? ORDER BY ParticipantId", study.getContainer().getId(), currentCohortId); - - if (rowLimit > 0) - sql = schema.getSqlDialect().limitRows(sql, rowLimit); - - return new SqlSelector(schema, sql).getArray(String.class); - } - - public String[] getParticipantIdsNotInCohorts(Study study) - { - DbSchema schema = StudySchema.getInstance().getSchema(); - SQLFragment sql = new SQLFragment("SELECT ParticipantId FROM " + SCHEMA.getTableInfoParticipant() + " WHERE Container = ? AND CurrentCohortId IS NULL", - study.getContainer().getId()); - - return new SqlSelector(schema, sql).getArray(String.class); - } - - public String[] getParticipantIdsNotInGroupCategory(Study study, User user, int categoryId) - { - return getParticipantIdsNotInGroupCategory(study, user, null, categoryId); - } - - public String[] getParticipantIdsNotInGroupCategory(Study study, User user, @Nullable ContainerFilter cf, int categoryId) - { - TableInfo groupMapTable = StudySchema.getInstance().getTableInfoParticipantGroupMap(); - TableInfo tableInfoParticipantGroup = StudySchema.getInstance().getTableInfoParticipantGroup(); - DbSchema schema = StudySchema.getInstance().getSchema(); - - SQLFragment filter = getParticipantFilter(study,user,cf); - - SQLFragment sql = new SQLFragment("SELECT ParticipantId FROM ").append(SCHEMA.getTableInfoParticipant().getFromSQL("P")) - .append(" WHERE ").append(filter) - .append(" AND ParticipantId NOT IN (SELECT DISTINCT ParticipantId FROM ").append(groupMapTable.getFromSQL("PGM")) - .append(" WHERE GroupId IN (SELECT PG.RowId FROM ").append(tableInfoParticipantGroup.getFromSQL("PG")).append(" WHERE Container = ? AND CategoryId = ?))") - .add(study.getContainer().getId()) - .add(categoryId); - - return new SqlSelector(schema.getScope(), sql).getArray(String.class); - } - - public static final int ALTERNATEID_DEFAULT_NUM_DIGITS = 6; - - public void clearAlternateParticipantIds(Study study) - { - if (study.isDataspaceStudy()) - return; - Collection participantIds = getParticipantIds(study,null); - - for (String participantId : participantIds) - setAlternateId(study, study.getContainer().getId(), participantId, null); - } - - public void generateNeededAlternateParticipantIds(Study study, User user) - { - Map participantInfos = getParticipantInfos(study, user, false, true); - - StudyController.ChangeAlternateIdsForm changeAlternateIdsForm = StudyController.getChangeAlternateIdForm((StudyImpl) study); - String prefix = changeAlternateIdsForm.getPrefix(); - if (null == prefix) - prefix = ""; // So we don't get the string "null" as the prefix - int numDigits = changeAlternateIdsForm.getNumDigits(); - if (numDigits < ALTERNATEID_DEFAULT_NUM_DIGITS) - numDigits = ALTERNATEID_DEFAULT_NUM_DIGITS; // Should not happen, but be safe - - HashSet usedNumbers = new HashSet<>(); - for (ParticipantInfo participantInfo : participantInfos.values()) - { - String alternateId = participantInfo.getAlternateId(); - if (alternateId != null) - { - try - { - if (prefix.isEmpty() || alternateId.startsWith(prefix)) - { - String alternateIdNoPrefix = alternateId.substring(prefix.length()); - usedNumbers.add(alternateIdNoPrefix); - } - } - catch (NumberFormatException x) - { - // It's possible that the id is not an integer after stripping prefix, because it can be - // set explicitly. That's fine, because it won't conflict with what we might generate - } - } - } - - for (Map.Entry entry : participantInfos.entrySet()) - { - ParticipantInfo participantInfo = entry.getValue(); - String alternateId = participantInfo.getAlternateId(); - - if (null == alternateId) - { - String participantId = entry.getKey(); - String newId = nextRandom(usedNumbers, numDigits); - setAlternateId(study, participantInfo.getContainerId(), participantId, prefix + newId); - } - } - } - - public int setImportedAlternateParticipantIds(Study study, DataLoader dl, BatchValidationException errors) throws IOException - { - // Use first line to determine order of columns we care about - // The first column in the data must contain the ones we are seeking - String[][] firstline = dl.getFirstNLines(1); - if (null == firstline || 0 == firstline.length) - return 0; // Unexpected but just in case - - boolean seenParticipantId = false; - boolean seenAlternateIdOrDateOffset = false; - boolean headerError = false; - ColumnDescriptor[] columnDescriptors = new ColumnDescriptor[3]; - for (int i = 0; i < 3 && i < firstline[0].length; i += 1) - { - String header = firstline[0][i]; - switch (header) - { - case PTID_COLUMN_NAME: - columnDescriptors[i] = new ColumnDescriptor(PTID_COLUMN_NAME, String.class); - seenParticipantId = true; - break; - case ALTERNATEID_COLUMN_NAME: - columnDescriptors[i] = new ColumnDescriptor(ALTERNATEID_COLUMN_NAME, String.class); - seenAlternateIdOrDateOffset = true; - break; - case DATEOFFSET_COLUMN_NAME: - columnDescriptors[i] = new ColumnDescriptor(DATEOFFSET_COLUMN_NAME, Integer.class); - seenAlternateIdOrDateOffset = true; - break; - default: - if (i < 2) - headerError = true; - break; - } - if (headerError) - break; - } - - int rowCount = 0; - if (!seenParticipantId || !seenAlternateIdOrDateOffset || headerError) - { - errors.addRowError(new ValidationException("The header row must contain " + PTID_COLUMN_NAME + " and either " + - ALTERNATEID_COLUMN_NAME + ", " + DATEOFFSET_COLUMN_NAME + " or both.")); - } - else - { - assert null != columnDescriptors[0] && null != columnDescriptors[1]; // Since we've seen PTID and 1 other - if (null == columnDescriptors[2]) - columnDescriptors = Arrays.copyOf(columnDescriptors, 2); // Can't hand DataLoader a null column - - // Now get loader to load all rows with correct columns and types - dl.setColumns(columnDescriptors); - dl.setHasColumnHeaders(true); - dl.setThrowOnErrors(true); - dl.setInferTypes(false); - - // Note alternateIds that are already used - Map participantInfos = getParticipantInfos(study, null, true, true); - CaseInsensitiveHashSet usedIds = new CaseInsensitiveHashSet(); - for (ParticipantInfo participantInfo : participantInfos.values()) - { - String alternateId = participantInfo.getAlternateId(); - if (alternateId != null) - { - usedIds.add(alternateId); - } - } - - List> rows = dl.load(); - rowCount = rows.size(); - - // Remove used alternateIds for participantIds that are in the list to be changed - for (Map row : rows) - { - String participantId = Objects.toString(row.get(PTID_COLUMN_NAME), null); - String alternateId = Objects.toString(row.get(ALTERNATEID_COLUMN_NAME), null); - if (null != participantId && null != alternateId) - { - ParticipantInfo participantInfo = participantInfos.get(participantId); - if (null != participantInfo) - { - String currentAlternateId = participantInfo.getAlternateId(); - if (null != currentAlternateId && !alternateId.equalsIgnoreCase(currentAlternateId)) - usedIds.remove(currentAlternateId); // remove as it will get replaced - } - } - } - - try (Transaction transaction = StudySchema.getInstance().getSchema().getScope().ensureTransaction()) - { - for (Map row : rows) - { - String participantId = Objects.toString(row.get(PTID_COLUMN_NAME), null); - if (null == participantId) - { - // ParticipantId must be specified - errors.addRowError(new ValidationException("A ParticipantId must be specified.")); - break; - } - - String alternateId = Objects.toString(row.get(ALTERNATEID_COLUMN_NAME), null); - Integer dateOffset = (null != row.get(DATEOFFSET_COLUMN_NAME)) ? asInteger(row.get(DATEOFFSET_COLUMN_NAME)) : null; - - if (null == alternateId && null == dateOffset) - { - errors.addRowError(new ValidationException("Either " + ALTERNATEID_COLUMN_NAME + " or " + DATEOFFSET_COLUMN_NAME + " must be specified.")); - break; - } - - ParticipantInfo participantInfo = participantInfos.get(participantId); - if (null != participantInfo) - { - String currentAlternateId = participantInfo.getAlternateId(); - if (null != alternateId && !alternateId.equalsIgnoreCase(currentAlternateId) && usedIds.contains(alternateId)) - { - errors.addRowError(new ValidationException("Two participants may not share the same Alternate ID.")); - break; - } - - if ((null != alternateId && !alternateId.equalsIgnoreCase(currentAlternateId)) || - (null != dateOffset && dateOffset != participantInfo.getDateOffset())) - { - - setAlternateIdAndDateOffset(study, participantId, alternateId, dateOffset); - if (null != alternateId) - usedIds.add(alternateId); // Add new id - } - } - else - { - errors.addRowError(new ValidationException("ParticipantID " + participantId + " not found.")); - } - } - - if (!errors.hasErrors()) - transaction.commit(); - } - } - - if (errors.hasErrors()) - return 0; - return rowCount; - } - - private void setAlternateId(Study study, String containerId, String participantId, @Nullable String alternateId) - { - // Set alternateId even if null, because that's how we clear it - SQLFragment sql = new SQLFragment("UPDATE ").append(SCHEMA.getTableInfoParticipant()).append(" SET AlternateId = ? WHERE Container = ? AND ParticipantId = ?") - .addAll(alternateId, containerId, participantId); - new SqlExecutor(StudySchema.getInstance().getSchema()).execute(sql); - StudyManager.getInstance().clearParticipantCache(study.getContainer()); - } - - private void setAlternateIdAndDateOffset(Study study, String participantId, @Nullable String alternateId, @Nullable Integer dateOffset) - { - // Only set alternateId and/or dateOffset if non-null - assert null != participantId; - if (null != alternateId || null != dateOffset) - { - SQLFragment sql = new SQLFragment("UPDATE ").append(SCHEMA.getTableInfoParticipant()).append(" SET "); - boolean needComma = false; - if (null != alternateId) - { - sql.append("AlternateId = ?").add(alternateId); - needComma = true; - } - if (null != dateOffset) - { - if (needComma) - sql.append(", "); - sql.append("DateOffset = ?").add(dateOffset); - } - sql.append(" WHERE Container = ? AND ParticipantId = ?"); - sql.add(study.getContainer()); - sql.add(participantId); - new SqlExecutor(StudySchema.getInstance().getSchema()).execute(sql); - StudyManager.getInstance().clearParticipantCache(study.getContainer()); - } - } - - private String nextRandom(Set usedNumbers, int numDigits) - { - String newId; - do - { - newId = StringUtilsLabKey.getUniquifier(numDigits); - } while (usedNumbers.contains(newId)); - usedNumbers.add(newId); - return newId; - } - - private void parseData(User user, - DatasetDefinition def, - DataLoader loader, - Map columnMap) - throws IOException - { - TableInfo tinfo = def.getTableInfo(user); - - // We're going to lower-case the keys ourselves later, - // so this needs to be case-insensitive - if (!(columnMap instanceof CaseInsensitiveHashMap)) - { - columnMap = new CaseInsensitiveHashMap<>(columnMap); - } - - // StandardDataIteratorBuilder will handle most aliasing, HOWEVER, ... - // columnMap may contain propertyURIs (dataset import job) and labels (GWT import file) - Map nameMap = DataIteratorUtil.createTableMap(tinfo, true); - - // - // create columns to properties map - // - loader.setInferTypes(false); - ColumnDescriptor[] cols = loader.getColumns(); - for (ColumnDescriptor col : cols) - { - String name = col.name.toLowerCase(); - - //Special column name - if ("replace".equals(name)) - { - col.clazz = Boolean.class; - col.name = name; //Lower case - continue; - } - - // let DataIterator do conversions - col.clazz = String.class; - - if (columnMap.containsKey(name)) - name = columnMap.get(name); - - col.name = name; - - ColumnInfo colinfo = nameMap.get(col.name); - if (null != colinfo) - { - col.name = colinfo.getName(); - col.propertyURI = colinfo.getPropertyURI(); - } - } - } - - - public void batchValidateExceptionToList(BatchValidationException errors, List errorStrs) - { - for (ValidationException rowError : errors.getRowErrors()) - { - String rowPrefix = ""; - if (rowError.getRowNumber() >= 0) - rowPrefix = "Row " + rowError.getRowNumber() + " "; - for (ValidationError e : rowError.getErrors()) - errorStrs.add(rowPrefix + e.getMessage()); - } - } - - /** - * @deprecated pass in a DataIteratorContext instead of individual options - */ - @Deprecated - public List importDatasetData(User user, DatasetDefinition def, - DataLoader loader, - Map columnMap, - BatchValidationException errors, - DatasetDefinition.CheckForDuplicates checkDuplicates, - @Nullable DataState defaultQCState, - QueryUpdateService.InsertOption insertOption, - Logger logger, - LookupResolutionType lookupResolutionType, - @Nullable AuditBehaviorType auditBehaviorType) - throws IOException - { - DataIteratorContext context = new DataIteratorContext(errors); - - context.setInsertOption(insertOption); - context.setLookupResolutionType(lookupResolutionType); - - Map options = new HashMap<>(); - options.put(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, auditBehaviorType); - options.put(DatasetUpdateService.Config.AllowImportManagedKey, Boolean.FALSE); - options.put(DatasetUpdateService.Config.CheckForDuplicates, checkDuplicates); - if (defaultQCState != null) - options.put(DatasetUpdateService.Config.DefaultQCState, defaultQCState); - if (logger != null) - options.put(QueryUpdateService.ConfigParameters.Logger, logger); - - context.setConfigParameters(options); - - return importDatasetData(user, def, loader, columnMap, context); - } - - public List importDatasetData(User user, DatasetDefinition def, DataLoader loader, - Map columnMap, DataIteratorContext context) throws IOException - { - parseData(user, def, loader, columnMap); - return def.importDatasetData(user, loader, context); - } - - - /** - * @deprecated pass in a DataIteratorContext instead of individual options - */ - @Deprecated - public List importDatasetData(User user, DatasetDefinition def, - List> data, - BatchValidationException errors, - DatasetDefinition.CheckForDuplicates checkDuplicates, - @Nullable DataState defaultQCState, - Logger logger, - boolean allowImportManagedKey, - boolean skipTriggers) throws IOException - { - if (data.isEmpty()) - return Collections.emptyList(); - - DataIteratorContext context = new DataIteratorContext(errors); - Map options = new HashMap<>(); - - options.put(QueryUpdateService.ConfigParameters.Logger, logger); - options.put(DatasetUpdateService.Config.AllowImportManagedKey, Boolean.valueOf(allowImportManagedKey)); - if (defaultQCState != null) - options.put(DatasetUpdateService.Config.DefaultQCState, defaultQCState); - options.put(DatasetUpdateService.Config.CheckForDuplicates, checkDuplicates); - // if we are being called by QUS we don't want to call triggers twice or resync twice - options.put(QueryUpdateService.ConfigParameters.SkipTriggers, skipTriggers); - options.put(DatasetUpdateService.Config.SkipResyncStudy, skipTriggers); - context.setConfigParameters(options); - - DataLoader loader = new MapLoader(data); - context.setInsertOption(allowImportManagedKey ? QueryUpdateService.InsertOption.INSERT : QueryUpdateService.InsertOption.IMPORT); - - return importDatasetData(user, def, loader, new CaseInsensitiveHashMap<>(), context); - } - - public boolean importDatasetSchemas(StudyImpl study, final User user, SchemaReader reader, BindException errors, boolean createShared, boolean allowDomainUpdates, @Nullable Activity activity) - { - if (errors.hasErrors()) - return false; - - StudyImpl createDatasetStudy = null; - if (createShared) - createDatasetStudy = (StudyImpl)getSharedStudy(study); - if (null == createDatasetStudy) - createDatasetStudy = study; - - List importErrors = new LinkedList<>(); - final Map datasetDefEntryMap = new HashMap<>(); - - // Use a factory to ensure domain URI consistency between imported properties and the dataset. See #7944. - DomainURIFactory factory = name -> { - assert datasetDefEntryMap.containsKey(name); - DatasetDefinitionEntry defEntry = datasetDefEntryMap.get(name); - Container defContainer = defEntry.datasetDefinition.getDefinitionContainer(); - String domainURI = getDomainURI(defEntry.datasetDefinition.getDefinitionContainer(), user, name, defEntry.datasetDefinition.getEntityId()); - return new Pair<>(domainURI, defContainer); - }; - - // We need to build the datasets (but not save) before we create the property descriptors so that - // we can use the unique DomainURI for each dataset as part of the PropertyURI - populateDatasetDefEntryMap(study, createDatasetStudy, reader, user, errors, datasetDefEntryMap); - if (errors.hasErrors()) - return false; - - ImportPropertyDescriptorsList list = reader.getImportPropertyDescriptors(factory, importErrors, study.getContainer()); - if (!importErrors.isEmpty()) - { - for (String error : importErrors) - errors.reject("importDatasetSchemas", error); - return false; - } - - // Check PHI levels; Must check activity level here, because we're in pipeline job, so Compliance can't get activity from HttpContext - /* TODO this list should be consistent across all types, not just List, see ListImporter.createDefinedLists() */ - ComplianceFolderSettings settings = ComplianceService.get().getFolderSettings(createDatasetStudy.getContainer(), User.getAdminServiceUser()); - PhiColumnBehavior columnBehavior = null==settings ? PhiColumnBehavior.show : settings.getPhiColumnBehavior(); - if (PhiColumnBehavior.show != columnBehavior) - { - PHI maxAllowedPhi = ComplianceService.get().getMaxAllowedPhi(createDatasetStudy.getContainer(), user); - if (null != activity && !maxAllowedPhi.isLevelAllowed(activity.getPHI())) - maxAllowedPhi = activity.getPHI(); // Reduce allowed level - - PHI maxContainedPhi = PHI.NotPHI; - for (ImportPropertyDescriptor ipd : list.properties) - { - if (maxContainedPhi.getRank() < ipd.pd.getPHI().getRank()) - maxContainedPhi = ipd.pd.getPHI(); - } - - if (!maxContainedPhi.isLevelAllowed(maxAllowedPhi)) - { - errors.reject(ERROR_MSG, "User's max allowed PHI is '" + maxAllowedPhi.getLabel() + "', but imported datasets contain higher PHI '" + maxContainedPhi.getLabel() + "'."); - return false; - } - } - - for (ImportPropertyDescriptor ipd : list.properties) - { - if (null == ipd.domainName || null == ipd.domainURI) - errors.reject("importDatasetSchemas", "Dataset not specified for property: " + ipd.pd.getName()); - } - if (errors.hasErrors()) - return false; - - StudyManager manager = StudyManager.getInstance(); - - Map domainChangeMap = new CaseInsensitiveHashMap<>(); - if (allowDomainUpdates) - { - // generate the dataset domain changes for existing datasets - buildPropertySaveAndDeleteLists(datasetDefEntryMap, list, domainChangeMap, true); - } - - // now actually create the datasets - for (Map.Entry entry : datasetDefEntryMap.entrySet()) - { - DatasetDefinitionEntry d = entry.getValue(); - DatasetDefinition def = d.datasetDefinition; - - if (d.isNew) - manager.createDatasetDefinition(user, def); - else if (d.isModified) - { - // issue 44363 : in certain situations the dataset domain will need to be saved earlier in order to support - // a change in the key column that may not be in the initial domain - if (manager.isKeyChanged(def)) - { - var tableInfo = def.getTableInfo(user); - if (tableInfo != null && (new TableSelector(def.getTableInfo(user)).getRowCount() > 0)) - { - // throw an error if we are changing keys on a dataset with data - errors.reject(ERROR_MSG, "Unable to change the keys on dataset (" + def.getName() + "), because there is still data present. The dataset should be truncated before the import."); - return false; - } - - if (!def.getUseTimeKeyField()) - { - String keyName = def.getKeyPropertyName(); - Domain domain = def.refreshDomain(); - if (domain != null) - { - _DatasetDomainChange domainChange = domainChangeMap.get(domain.getTypeURI()); - Domain newDomain = domainChange.domain; - if (domain.getStorageTableName() != null && newDomain != null) - { - if (domain.getPropertyByName(keyName) == null && newDomain.getPropertyByName(keyName) != null) - { - try - { - newDomain.save(user); - } - catch (ChangePropertyDescriptorException ex) - { - errors.reject("importDatasetSchemas", ex.getMessage() == null ? ex.toString() : ex.getMessage()); - return false; - } - } - } - } - } - } - manager.updateDatasetDefinition(user, def); - } - - if (d.tags != null) - ReportPropsManager.get().importProperties(def.getEntityId(), def.getDefinitionContainer(), user, d.tags); - } - - // optional param to control whether field additions or deletions are permitted - if (allowDomainUpdates) - { - // Generate dataset domain changes for new datasets - domainChangeMap.clear(); - buildPropertySaveAndDeleteLists(datasetDefEntryMap, list, domainChangeMap, false); - - // Now that we actually have datasets, create or update the domains. This ensures that all domains and - // property changes are saved before adding indices. - ensurePropertiesAndRequiredIndices(reader, datasetDefEntryMap, domainChangeMap, user, errors); - - if (errors.hasErrors()) - return false; - } - - List orderedIds = reader.getDatasetOrder(); - if (null != orderedIds) - { - DatasetReorderer reorderer = new DatasetReorderer(study, user); - reorderer.reorderDatasets(orderedIds); - } - - return true; - } - - // see if we need to delete any columns from an existing domain and create the domain if it doesn't already exist - private boolean deleteAndSaveProperties(User user, BindException errors, _DatasetDomainChange domainChange) - { - // see if we need to delete any columns from the domain - for (DomainProperty p : domainChange.propsToDelete) - { - p.delete(); - } - - try - { - domainChange.domain.save(user); - } - catch (ChangePropertyDescriptorException ex) - { - errors.reject("importDatasetSchemas", ex.getMessage() == null ? ex.toString() : ex.getMessage()); - return false; - } - return true; - } - - /** - * Utility class to track dataset domain changes - */ - private static class _DatasetDomainChange - { - public _DatasetDomainChange() {} - public _DatasetDomainChange(Domain domain) - { - this.domain = domain; - this.propsToDelete = new ArrayList<>(domain.getProperties()); - } - - private Domain domain; - private List propsToDelete = Collections.emptyList(); - } - - private _DatasetDomainChange createDomainChange(String domainURI, String domainName, DatasetDefinitionEntry def, boolean existingDomainsOnly) - { - _DatasetDomainChange domainChange = new _DatasetDomainChange(); - domainChange.domain = PropertyService.get().getDomain(def.datasetDefinition.getDefinitionContainer(), domainURI, true); - if (domainChange.domain == null && existingDomainsOnly) - return null; - else if (domainChange.domain == null) - domainChange.domain = PropertyService.get().createDomain(def.datasetDefinition.getDefinitionContainer(), domainURI, domainName); - - // add all the properties that exist for the domain - domainChange.propsToDelete = new ArrayList<>(domainChange.domain.getProperties()); - - return domainChange; - } - - /** - * Generate the dataset domain changes for the import - * @param domainChangeMap - maps domain URIs to the _DatasetDomainChange object - * @param existingDomainsOnly - if true will only populate the map for existing datasets - */ - private void buildPropertySaveAndDeleteLists(Map datasetDefEntryMap, - ImportPropertyDescriptorsList list, - Map domainChangeMap, - boolean existingDomainsOnly) - { - for (ImportPropertyDescriptor ipd : list.properties) - { - _DatasetDomainChange domainChange = domainChangeMap.computeIfAbsent(ipd.domainURI, (k) -> - createDomainChange(ipd.domainURI, ipd.domainName, datasetDefEntryMap.get(ipd.domainName), existingDomainsOnly)); - - if (domainChange == null) - continue; - - Domain d = domainChange.domain; - // Issue 14569: during study reimport be sure to look for a column has been deleted. - // Look at the existing properties for this dataset's domain and - // remove them as we find them in schema. If there are any properties left after we've - // iterated over all the import properties then we need to delete them - DomainProperty p = d.getPropertyByName(ipd.pd.getName()); - domainChange.propsToDelete.remove(p); - - if (null != p) - { - final String fromPropertyUri = p.getPropertyDescriptor().getPropertyURI(); - boolean fromSystemProp = SystemProperty.getProperties().stream().anyMatch(sp -> - sp.getPropertyURI().equals(fromPropertyUri)); - boolean toSystemProp = SystemProperty.getProperties().stream().anyMatch(sp -> - sp.getPropertyURI().equals(ipd.pd.getPropertyURI())); - boolean propertyUriChange = !fromPropertyUri.equals(ipd.pd.getPropertyURI()); - - // Don't copy values over a system prop, just setup swapping the property descriptor - if (propertyUriChange && toSystemProp) - { - p.setPropertyURI(ipd.pd.getPropertyURI()); - } - else - { - // Enable the domain to make schema changes for this property if required - // by dropping/adding the property and its storage at domain save time - p.setSchemaImport(true); - OntologyManager.updateDomainPropertyFromDescriptor(p, ipd.pd); - } - - // Flag this as a property descriptor swap. EnsurePropertyDescriptor will find correct property ID - // by propertyURI. Ensure correct container/projects set. - if (propertyUriChange && (toSystemProp || fromSystemProp)) - { - p.getPropertyDescriptor().setPropertyId(0); - p.getPropertyDescriptor().setContainer(ipd.pd.getContainer()); - } - } - else - { - // don't add property descriptors for columns with 'global' propertyuri - // TODO: move to conceptURI, and use 'local' propertyURI so each domain can have its own - // propertydescriptor instance - if (ipd.pd.getPropertyURI().startsWith("http://cpas.labkey.com/Study#")) - continue; - p = d.addProperty(); - ipd.pd.copyTo(p.getPropertyDescriptor()); - p.setName(ipd.pd.getName()); - p.setRequired(ipd.pd.isRequired()); // TODO: Redundant? copyTo() already copied required (without involving nullable) - p.setDescription(ipd.pd.getDescription()); - } - - ipd.validators.forEach(p::addValidator); - p.setConditionalFormats(ipd.formats); - p.setDefaultValue(ipd.defaultValue); - } - - //Ensure that each dataset has an entry in the domain map - if (datasetDefEntryMap.size() != domainChangeMap.size()) - { - for (DatasetDefinitionEntry datasetDefinitionEntry : datasetDefEntryMap.values()) - { - if (!domainChangeMap.containsKey(datasetDefinitionEntry.datasetDefinition.getTypeURI())) - { - Domain domain = - PropertyService.get().getDomain( - datasetDefinitionEntry.datasetDefinition.getDefinitionContainer(), - datasetDefinitionEntry.datasetDefinition.getTypeURI(), - true); - if (domain != null) - domainChangeMap.put(datasetDefinitionEntry.datasetDefinition.getTypeURI(), new _DatasetDomainChange(domain)); - } - } - } - } - - private void ensurePropertiesAndRequiredIndices(SchemaReader reader, Map datasetDefEntryMap, Map domainChangeMap, User user, BindException errors) - { - for (SchemaReader.DatasetImportInfo datasetImportInfo : reader.getDatasetInfo().values()) - { - DatasetDefinitionEntry datasetDefinitionEntry = datasetDefEntryMap.get(datasetImportInfo.name); - _DatasetDomainChange domainChange = domainChangeMap.get(datasetDefinitionEntry.datasetDefinition.getTypeURI()); - - if (domainChange != null) - { - Domain domain = domainChange.domain; - boolean shared = datasetDefinitionEntry.datasetDefinition.isShared(); - - if (!domain.isProvisioned() || shared) - { - deleteAndSaveProperties(user, errors, domainChange); - if (!datasetDefinitionEntry.datasetDefinition.isShared()) - { - // Refresh the domain, now that it's provisioned - domain = PropertyService.get().getDomain(domain.getContainer(), domain.getTypeURI()); - if (null == domain) - throw new IllegalStateException("Domain should not be null"); - domain.setPropertyIndices(datasetImportInfo.indices); - StorageProvisioner.get().ensureTableIndices(domain); - } - } - else - { - // If we're changing an existing domain, we may be dropping a column that has an admin-configured - // index. We need to drop the indices first, adjust the properties, and then add the new indices. - // This method allows for that. - domain.setPropertyIndices(datasetImportInfo.indices); - StorageProvisioner.get().ensureTableIndices(domain, () -> deleteAndSaveProperties(user, errors, domainChange)); - } - } - } - } - - public String getDomainURI(Container c, User u, Dataset def) - { - if (null == def) - return getDomainURI(c, u, null, null); - else - return getDomainURI(c, u, def.getName(), def.getEntityId()); - } - - - private boolean populateDatasetDefEntryMap(StudyImpl study, StudyImpl createDatasetStudy, SchemaReader reader, User user, BindException errors, Map defEntryMap) - { - StudyManager manager = StudyManager.getInstance(); - Container c = study.getContainer(); - Map datasetInfoMap = reader.getDatasetInfo(); - - for (Map.Entry entry : datasetInfoMap.entrySet()) - { - int id = entry.getKey().intValue(); - SchemaReader.DatasetImportInfo info = entry.getValue(); - String name = info.name; - String label = info.label; - if (label == null) - { - // Default to using the name as the label if none was explicitly specified - label = name; - } - - // Check for name conflicts - Dataset existingDef = manager.getDatasetDefinitionByLabel(study, label); - - if (existingDef != null && existingDef.getDatasetId() != id) - { - errors.reject("importDatasetSchemas", "Dataset '" + existingDef.getName() + "' is already using the label '" + label + "'"); - return false; - } - - existingDef = manager.getDatasetDefinitionByName(study, name); - - if (existingDef != null && existingDef.getDatasetId() != id) - { - errors.reject("importDatasetSchemas", "Existing " + name + " dataset has id " + existingDef.getDatasetId() + - ", uploaded " + name + " dataset has id " + id); - return false; - } - - if (info.demographicData && (info.keyPropertyName != null)) - { - errors.reject("importDatasetSchemas", "Dataset '" + name + "' has key field set to " + info.keyPropertyName + ". This a demographic dataset therefore cannot have an extra key property."); - return false; - } - - DatasetDefinition def = manager.getDatasetDefinition(study, id); - - if (def == null) - { - def = new DatasetDefinition(createDatasetStudy, id, name, label, null, null, null); - def.setDescription(info.description); - def.setVisitDatePropertyName(info.visitDatePropertyName); - def.setShowByDefault(!info.isHidden); - def.setKeyPropertyName(info.keyPropertyName); - def.setCategory(info.category); - def.setKeyManagementType(info.keyManagementType); - def.setDemographicData(info.demographicData); - def.setType(info.type); - def.setTag(info.tag); - defEntryMap.put(name, new DatasetDefinitionEntry(def, true, info.tags)); - def.setUseTimeKeyField(info.useTimeKeyField); - } - else if (def.isPublishedData()) - { - errors.reject("importDatasetSchemas", "Unable to modify linked data dataset '" + def.getLabel() + "'."); - } - else - { - // TODO: modify shared definition? - boolean canEditDefinition = def.canUpdateDefinition(user); - - if (canEditDefinition) - { - def = def.createMutable(); - def.setLabel(label); - def.setName(name); - def.setDescription(info.description); - if (null == def.getTypeURI()) - { - def.setTypeURI(getDomainURI(c, user, def)); - } - - def.setVisitDatePropertyName(info.visitDatePropertyName); - def.setShowByDefault(!info.isHidden); - def.setKeyPropertyName(info.keyPropertyName); - def.setCategory(info.category); - def.setKeyManagementType(info.keyManagementType); - def.setDemographicData(info.demographicData); - def.setTag(info.tag); - } - else - { - // TODO: warn - // name, label, description, visitdatepropertyname, category - if (def.getKeyManagementType() != info.keyManagementType) - errors.reject("ERROR_MSG", "Key type is not compatible with shared dataset: " + def.getName()); - if (!Strings.CI.equals(def.getKeyPropertyName(), info.keyPropertyName)) - errors.reject("ERROR_MSG", "Key property name is not compatible with shared dataset: " + def.getName()); - if (def.isDemographicData() != info.demographicData) - errors.reject("ERROR_MSG", "Demographic type is not compatible with shared dataset: " + def.getName()); - } - - defEntryMap.put(name, new DatasetDefinitionEntry(def, false, canEditDefinition, info.tags)); - } - } - - return true; - } - - // Detect if this dataset has an old-style URI without the entityid. If so, assign a new type URI to this dataset - // and update the domain descriptor URI - // old: urn:lsid:labkey.com:StudyDataset.Folder-6:DEM - // new: urn:lsid:labkey.com:StudyDataset.Folder-6:DEM-cbffdfa1-f19b-1030-90dd-bf4ca488b2d0 - // Also, the URI will change if the dataset name changes - private void ensureDatasetDefinitionDomain(User user, DatasetDefinition def) - { - String oldURI = def.getTypeURI(); - String newURI = getDomainURI(def.getContainer(), user, def); - - if (Strings.CS.equals(oldURI, newURI)) - return; - - // This dataset has the old uri so upgrade it to use the new URI format - def.setTypeURI(newURI, true /*upgrade*/); - - // fixup the domain - DomainDescriptor dd = OntologyManager.getDomainDescriptor(oldURI, def.getContainer()); - if (null != dd) - { - dd = dd.edit() - .setDomainURI(newURI) - .setName(def.getName()) // Name may have changed too; it's part of URI - .build(); - OntologyManager.ensureDomainDescriptor(dd); - - // since the descriptor has changed, ensure the domain is up-to-date - def.refreshDomain(); - } - } - - private static String getDomainURI(Container c, User u, String name, String id) - { - return DatasetDomainKind.generateDomainURI(name, id, c); - } - - @NotNull - public VisitManager getVisitManager(Study study) - { - @Migrate // TODO: Switch VisitManager() to take Study and get rid of cast - StudyImpl studyImpl = (StudyImpl)study; - switch (study.getTimepointType()) - { - case VISIT: - return new SequenceVisitManager(studyImpl); - case CONTINUOUS: - return new AbsoluteDateVisitManager(studyImpl); - case DATE: - default: - return new RelativeDateVisitManager(studyImpl); - } - } - - public static SQLFragment timePortionFromDateSQL(String dateColumnName) - { - SqlDialect dialect = StudySchema.getInstance().getSqlDialect(); - SQLFragment sql = new SQLFragment(); - if (dialect.isPostgreSQL()) - { - sql.append("to_char(").append(dateColumnName).append(", 'HH24MISS')"); - } - else if (dialect.isSqlServer()) - { - sql.append("FORMAT(").append(dateColumnName).append(", 'HHmmss')"); - } - else - { - sql.append("CAST((").append(dateColumnName).append(") AS VARCHAR(10))"); - } - return sql; - } - - private String getParticipantCacheKey(Container container) - { - return container.getId() + "/" + Participant.class; - } - - /** non-permission checking, non-recursive */ - private Map getParticipantMap(Study study) - { - return _participantCache.get(study.getContainer()); - } - - public void clearParticipantCache(Container container) - { - _participantCache.remove(container); - } - - public Collection getParticipants(Study study) - { - Map participantMap = getParticipantMap(study); - return Collections.unmodifiableCollection(participantMap.values()); - } - - public Participant getParticipant(Study study, String participantId) - { - Map participantMap = getParticipantMap(study); - return participantMap.get(participantId); - } - - public static class ParticipantNotUniqueException extends Exception - { - ParticipantNotUniqueException(String ptid) - { - super("Participant found in more than one study: " + ptid); - } - } - - /* non-permission checking, may return participant from sub folder */ - public Container findParticipant(Study study, String ptid) throws ParticipantNotUniqueException - { - Participant p = getParticipant(study, ptid); - if (null != p) - return study.getContainer(); - else if (!study.isDataspaceStudy()) - return null; - - TableInfo table = StudySchema.getInstance().getTableInfoParticipant(); - ArrayList containers = new SqlSelector(table.getSchema(), new SQLFragment("SELECT container FROM study.participant WHERE participantid=?",ptid)) - .getArrayList(String.class); - if (containers.isEmpty()) - return null; - else if (containers.size() == 1) - return ContainerManager.getForId(containers.get(0)); - throw new ParticipantNotUniqueException(ptid); - } - - - public CustomParticipantView getCustomParticipantView(Study study) - { - if (study == null) - return null; - - Path path = ModuleHtmlView.getStandardPath("participant"); - - for (Module module : study.getContainer().getActiveModules()) - { - if (ModuleHtmlView.exists(module, path)) - { - return CustomParticipantView.create(module, path); - } - } - - String key = new Path(study.getContainer().getId(),CustomParticipantView.class.getName()).toString(); - return (CustomParticipantView) CacheManager.getSharedCache().get(key, study, (k, s) -> - { - SimpleFilter containerFilter = SimpleFilter.createContainerFilter(study.getContainer()); - TableInfo ti = StudySchema.getInstance().getTableInfoParticipantView(); - CustomParticipantView participantView = new TableSelector(ti, containerFilter, null).getObject(CustomParticipantView.class); - if (null == participantView) - return null; - Module studyModule = ModuleLoader.getInstance().getModule("study"); - participantView.setModule(studyModule); - participantView.setTitle(study.getSubjectNounSingular()); - return participantView; - }); - } - - public CustomParticipantView saveCustomParticipantView(Study study, User user, CustomParticipantView view) - { - if (view.isModuleParticipantView()) - throw new IllegalArgumentException("Module-defined participant views should not be saved to the database."); - CustomParticipantView ret; - if (view.getRowId() == null) - { - view.beforeInsert(user, study.getContainer().getId()); - ret = Table.insert(user, StudySchema.getInstance().getTableInfoParticipantView(), view); - } - else - { - view.beforeUpdate(user); - ret = Table.update(user, StudySchema.getInstance().getTableInfoParticipantView(), view, view.getRowId()); - } - CacheManager.getSharedCache().remove(new Path(view.getContainerId(),CustomParticipantView.class.getName()).toString()); - return ret; - } - - public interface ParticipantViewConfig - { - String getParticipantId(); - - int getDatasetId(); - - Map getAliases(); - } - - public WebPartView getParticipantView(Container container, ParticipantViewConfig config) - { - return getParticipantView(container, config, null); - } - - public WebPartView getParticipantView(Container container, ParticipantViewConfig config, BindException errors) - { - StudyImpl study = getStudy(container); - if (study.getTimepointType() == TimepointType.CONTINUOUS) - return new StudyJspView<>(study, "/org/labkey/study/view/participantData.jsp", config, errors); - else - return new StudyJspView<>(study, "/org/labkey/study/view/participantAll.jsp", config, errors); - } - - public WebPartView getParticipantDemographicsView(Container container, ParticipantViewConfig config, BindException errors) - { - return new StudyJspView<>(getStudy(container), "/org/labkey/study/view/participantCharacteristics.jsp", config, errors); - } - - /** - * Called when a dataset has been modified in order to set the modified time, plus any other related actions. - * @param fireNotification - true to fire the changed notification. - */ - public static void datasetModified(DatasetDefinition def, boolean fireNotification) - { - // Issue 19285 - run this as a commit task. This has the benefit of only running per set of batch changes - // under the same transaction and only running if the transaction is committed. If no transaction is active then - // the code is run immediately - DbScope scope = StudySchema.getInstance().getScope(); - scope.addCommitTask(getInstance().getDatasetModifiedRunnable(def, fireNotification), CommitTaskOption.POSTCOMMIT); - } - - public Runnable getDatasetModifiedRunnable(DatasetDefinition def, boolean fireNotification) - { - return new DatasetModifiedRunnable(def, fireNotification); - } - - private static class DatasetModifiedRunnable implements Runnable - { - private final @NotNull - DatasetDefinition _def; - private final boolean _fireNotification; - - private DatasetModifiedRunnable(@NotNull DatasetDefinition def, boolean fireNotification) - { - _def = def; - _fireNotification = fireNotification; - } - - private int getDatasetId() - { - return _def.getDatasetId(); - } - - private Container getContainer() - { - return _def.getContainer(); - } - - @Override - public void run() - { - DatasetDefinition.updateModified(_def, new Date()); - if (_fireNotification) - fireDatasetChanged(_def); - } - - @Override - public boolean equals(Object o) - { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - DatasetModifiedRunnable that = (DatasetModifiedRunnable) o; - if (getDatasetId() != that.getDatasetId()) - return false; - return getContainer().equals(that.getContainer()); - } - - @Override - public int hashCode() - { - int result = getContainer().hashCode(); - result = 31 * result + this.getDatasetId(); - return result; - } - } - - public static void fireDatasetChanged(Dataset def) - { - for (DatasetManager.DatasetListener l : DatasetManager.getListeners()) - { - try - { - l.datasetChanged(def); - } - catch (Throwable t) - { - _log.error("fireDatasetChanged", t); - } - } - } - - // Return a source->alias map for the specified participant - public Map getAliasMap(StudyImpl study, User user, String ptid) - { - @Nullable final TableInfo aliasTable = StudyQuerySchema.createSchema(study, user).getParticipantAliasesTable(); - - if (null == aliasTable) - return Collections.emptyMap(); - - List columns = aliasTable.getColumns(); - SimpleFilter filter = new SimpleFilter(columns.get(0).getFieldKey(), ptid); - - // Return source -> alias map - return new TableSelector(aliasTable, Arrays.asList(columns.get(2), columns.get(1)), filter, null).getValueMap(String.class); - } - - private void unindexDataset(DatasetDefinition ds) - { - String docid = "dataset:" + new Path(ds.getContainer().getId(), String.valueOf(ds.getDatasetId())); - SearchService ss = SearchService.get(); - if (null != ss) - ss.deleteResource(docid); - } - - public static void indexDatasets(SearchService.TaskIndexingQueue queue, Date modifiedSince) - { - Container c = queue.getContainer(); - SimpleFilter filter = (null != c ? SimpleFilter.createContainerFilter(c) : new SimpleFilter()); - if (null != modifiedSince) - filter.addCondition(FieldKey.fromParts("Modified"), modifiedSince, CompareType.DATE_GT); - SQLFragment f = new SQLFragment("SELECT Container, DatasetId FROM " + StudySchema.getInstance().getTableInfoDataset() + " ") - .append(filter.getSQLFragment(StudySchema.getInstance().getSqlDialect())); - - new SqlSelector(StudySchema.getInstance().getSchema(), f).forEach(rs -> - { - String container = rs.getString(1); - int id = rs.getInt(2); - - Container c2 = ContainerManager.getForId(container); - if (null != c2) - { - Study study = StudyManager.getInstance().getStudy(c2); - - if (null != study) - { - DatasetDefinition dsd = StudyManager.getInstance().getDatasetDefinition(study, id); - if (null != dsd) - indexDataset(queue, dsd); - } - } - }); - } - - private static void indexDataset(SearchService.TaskIndexingQueue queue, DatasetDefinition dsd) - { - if (dsd.getType().equals(Dataset.TYPE_PLACEHOLDER)) - return; - if (null == dsd.getTypeURI() || null == dsd.getDomain()) - return; - String docid = "dataset:" + new Path(dsd.getContainer().getId(), String.valueOf(dsd.getDatasetId())); - - StringBuilder body = new StringBuilder(); - Map props = new HashMap<>(); - - props.put(SearchService.PROPERTY.categories.toString(), datasetCategory.toString()); - props.put(SearchService.PROPERTY.title.toString(), StringUtils.defaultIfEmpty(dsd.getLabel(),dsd.getName())); - String name = dsd.getName(); - String label = Strings.CS.equals(dsd.getLabel(),name) ? null : dsd.getLabel(); - String description = dsd.getDescription(); - String tag = dsd.getTag(); - String keywords = StringUtilsLabKey.joinNonBlank(" ", name, label, description, tag); - props.put(SearchService.PROPERTY.keywordsMed.toString(), keywords); - - body.append(keywords).append("\n"); - - StudyQuerySchema schema = StudyQuerySchema.createSchema(dsd.getStudy(), User.getSearchUser(), RoleManager.getRole(ReaderRole.class)); - TableInfo tableInfo = schema.getDatasetTable(dsd, null); - Map columns = QueryService.get().getColumns(tableInfo, tableInfo.getDefaultVisibleColumns()); - String sep = ""; - for (ColumnInfo column : columns.values()) - { - String n = StringUtils.trimToEmpty(column.getName()); - String l = StringUtils.trimToEmpty(column.getLabel()); - if (n.equals(l)) - l = ""; - body.append(sep).append(StringUtilsLabKey.joinNonBlank(" ", n, l)); - sep = ",\n"; - } - - ActionURL view = new ActionURL(StudyController.DatasetAction.class, null); - view.replaceParameter("datasetId", dsd.getDatasetId()); - view.setExtraPath(dsd.getContainer().getId()); - - SimpleDocumentResource r = new SimpleDocumentResource(new Path(docid), docid, - "text/plain", body.toString(), - view, props); - queue.addResource(r); - } - - public static void indexParticipants(SearchService.TaskIndexingQueue queue, @Nullable List ptids, @Nullable Date modifiedSince) - { - if (null != ptids && ptids.isEmpty()) - return; - - Container c = queue.getContainer(); - final StudyImpl study = StudyManager.getInstance().getStudy(c); - if (null == study) - return; - - final String nav = NavTree.toJS(Collections.singleton(new NavTree("study", PageFlowUtil.urlProvider(ProjectUrls.class).getBeginURL(c))), null, false, true).toString(); - - TableInfo participantTable = StudySchema.getInstance().getTableInfoParticipant(); - SQLFragment baseFragment = new SQLFragment(); - baseFragment.append("SELECT Container, ParticipantId FROM "); - baseFragment.append(participantTable, "p"); - - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - - @Nullable final TableInfo aliasTable = StudyQuerySchema.createSchema(study, User.getSearchUser()).getParticipantAliasesTable(); - LastIndexedClause lastIndexedClause = new LastIndexedClause(StudySchema.getInstance().getTableInfoParticipant(), modifiedSince, "p"); - if (!lastIndexedClause.isEmpty()) - { - if (null != aliasTable) - { - // Also reindex participants whose aliases have changed - SQLFragment aliasFragment = new SQLFragment().append("ParticipantId IN (\nSELECT ParticipantId FROM\n") - .append(aliasTable.getFromSQL("aliases")) - .append("WHERE aliases.Modified > p.LastIndexed)"); - filter.addClause(new OrClause(lastIndexedClause, new SQLClause(aliasFragment))); - } - else - { - filter.addClause(lastIndexedClause); - } - } - - baseFragment - .append(" ") - .append(filter.getSQLFragment(participantTable, "p")); - - final ActionURL executeURL = new ActionURL(StudyController.ParticipantAction.class, c); - executeURL.setExtraPath(c.getId()); - - final int BATCH_SIZE = 500; - List> batches = ptids != null ? ListUtils.partition(ptids, BATCH_SIZE) : Collections.singletonList(null); - - batches.forEach(batch -> { - Consumer runnable = (q) -> { - SQLFragment sql; - - if (null != batch) - { - sql = new SQLFragment(baseFragment); // Clone the base fragment before modifying - sql.append(" AND ParticipantId "); - StudySchema.getInstance().getSqlDialect().appendInClauseSql(sql, batch); - } - else - { - sql = baseFragment; - } - - new SqlSelector(StudySchema.getInstance().getSchema(), sql).forEach(rs -> { - final String ptid = rs.getString(2); - String displayTitle = "Study " + study.getLabel() + " -- " + - StudyService.get().getSubjectNounSingular(study.getContainer()) + " " + ptid; - ActionURL execute = executeURL.clone().addParameter("participantId", String.valueOf(ptid)); - Path p = new Path(c.getId(), ptid); - String docid = "participant:" + p; - - String uniqueIds = ptid; - - if (null != aliasTable) - { - // Add all participant aliases as high priority uniqueIds - Map aliasMap = StudyManager.getInstance().getAliasMap(study, User.getSearchUser(), ptid); - - if (!aliasMap.isEmpty()) - uniqueIds = uniqueIds + " " + StringUtils.join(aliasMap.values(), " "); - } - - Map props = new HashMap<>(); - props.put(SearchService.PROPERTY.categories.toString(), subjectCategory.getName()); - props.put(SearchService.PROPERTY.title.toString(), displayTitle); - props.put(SearchService.PROPERTY.identifiersHi.toString(), uniqueIds); - props.put(SearchService.PROPERTY.navtrail.toString(), nav); - - // Index a barebones participant document for now TODO: Figure out if it's safe to include demographic data or not (can all study users see it?) - - // SimpleDocument - SimpleDocumentResource r = new SimpleDocumentResource( - p, docid, - c.getEntityId(), - "text/plain", - displayTitle, - execute, props - ) - { - @Override - public void setLastIndexed(long ms, long modified) - { - StudySchema ss = StudySchema.getInstance(); - SQLFragment update = new SQLFragment("UPDATE ").append(ss.getTableInfoParticipant()).append(" SET LastIndexed = ? WHERE Container = ? AND ParticipantId = ?"); - update.addAll(new Timestamp(ms), c, ptid); - new SqlExecutor(ss.getSchema()).execute(update); - } - }; - queue.addResource(r); - }); - }; - - queue.addRunnable(runnable); - }); - } - - - // make sure we don't over do it with multiple calls to reindex the same study (see reindex()) - // add a level of indirection - // CONSIDER: add some facility like this to SearchService?? - // NOTE: this needs to be reviewed if we use modifiedSince - - final static WeakHashMap> _lastEnumerate = new WeakHashMap<>(); - - public static void _enumerateDocuments(SearchService.TaskIndexingQueue queue, @Nullable Date modifiedSince) - { - Container c = queue.getContainer(); - Consumer runEnumerate = new Consumer<>() - { - public void accept(SearchService.TaskIndexingQueue a) - { - synchronized (_lastEnumerate) - { - Consumer r = _lastEnumerate.get(c); - if (this != r) - return; - _lastEnumerate.remove(c); - } - - Study study = StudyManager.getInstance().getStudy(c); - - if (null != study) - { - StudyManager.indexDatasets(a, modifiedSince); - StudyManager.indexParticipants(a, null, modifiedSince); - // study protocol document - _enumerateProtocolDocuments(queue, study); - } - } - }; - - synchronized (_lastEnumerate) - { - _lastEnumerate.put(c, runEnumerate); - } - - queue.addRunnable(runEnumerate); - } - - - public static void _enumerateProtocolDocuments(SearchService.TaskIndexingQueue queue, @NotNull Study study) - { - AttachmentParent parent = ((StudyImpl)study).getProtocolDocumentAttachmentParent(); - if (null == parent) - return; - - ActionURL begin = PageFlowUtil.urlProvider(ProjectUrls.class).getBeginURL(study.getContainer()); - String nav = NavTree.toJS(Collections.singleton(new NavTree("study", begin)), null, false, true).toString(); - AttachmentService serv = AttachmentService.get(); - Path p = study.getContainer().getParsedPath().append("@study"); - - for (Attachment att : serv.getAttachments(parent)) - { - ActionURL download = StudyController.getProtocolDocumentDownloadURL(study.getContainer(), att.getName()); - - WebdavResource r = serv.getDocumentResource - ( - p.append(att.getName()), - download, - "\"" + att.getName() + "\" -- Protocol document attached to study " + study.getLabel(), - parent, att.getName(), SearchService.fileCategory - ); - r.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); - queue.addResource(r); - } - } - - public List getPublishedStudies(Container sourceStudyContainer) - { - return Collections.unmodifiableList(new TableSelector(StudySchema.getInstance().getTableInfoStudy(), - new SimpleFilter(FieldKey.fromParts("SourceStudyContainerId"), sourceStudyContainer), null).getArrayList(StudyImpl.class)); - } - - // Return collection of current snapshots that are configured to refresh specimens - public Collection getRefreshStudySnapshots() - { - return getStudySnapshots(new SQLFragment(" AND Refresh = ?", Boolean.TRUE)); - } - - // Return collection of all current snapshots - private Collection getStudySnapshots(@Nullable SQLFragment filter) - { - SQLFragment sql = new SQLFragment("SELECT ss.* FROM "); - sql.append(StudySchema.getInstance().getTableInfoStudy(), "s"); - sql.append(" JOIN "); - sql.append(StudySchema.getInstance().getTableInfoStudySnapshot(), "ss"); - sql.append(" ON s.StudySnapshot = ss.RowId AND Source IS NOT NULL AND Destination IS NOT NULL"); - - if (null != filter) - sql.append(filter); - - return new SqlSelector(StudySchema.getInstance().getSchema(), sql).getCollection(StudySnapshot.class); - } - - @Nullable - public StudySnapshot getStudySnapshot(Integer snapshotId) - { - TableSelector selector = new TableSelector(StudySchema.getInstance().getTableInfoStudySnapshot(), new SimpleFilter(FieldKey.fromParts("RowId"), snapshotId), null); - - return selector.getObject(StudySnapshot.class); - } - - /** - * Convert a placeholder or 'ghost' dataset to an actual dataset by renaming the target dataset to the placeholder's name, - * transferring all timepoint requirements from the placeholder to the target and deleting the placeholder dataset. - */ - public DatasetDefinition linkPlaceHolderDataset(StudyImpl study, User user, DatasetDefinition expectationDataset, DatasetDefinition targetDataset) - { - if (expectationDataset == null || targetDataset == null) - throw new IllegalArgumentException("Both expectation DataSet and target DataSet must exist"); - - if (!expectationDataset.getType().equals(Dataset.TYPE_PLACEHOLDER)) - throw new IllegalArgumentException("Only a DataSet of type : placeholder can be linked"); - - if (!targetDataset.getType().equals(Dataset.TYPE_STANDARD)) - throw new IllegalArgumentException("Only a DataSet of type : standard can be linked to"); - - DbScope scope = StudySchema.getInstance().getSchema().getScope(); - - try (Transaction transaction = scope.ensureTransaction()) - { - // transfer any timepoint requirements from the ghost to target - for (VisitDataset vds : expectationDataset.getVisitDatasets()) - { - VisitDatasetType type = vds.isRequired() ? VisitDatasetType.REQUIRED : VisitDatasetType.NOT_ASSOCIATED; - StudyManager.getInstance().updateVisitDatasetMapping(user, study.getContainer(), vds.getVisitRowId(), targetDataset.getDatasetId(), type); - } - - String name = expectationDataset.getName(); - String label = expectationDataset.getLabel(); - - // no need to resync the study, as there should be no data in the expectation dataset - deleteDataset(study, user, expectationDataset, false, null); - - targetDataset = targetDataset.createMutable(); - targetDataset.setName(name); - targetDataset.setLabel(label); - targetDataset.save(user); - - transaction.commit(); - } - - return targetDataset; - } - - public static class CategoryListener implements ViewCategoryListener - { - private final StudyManager _instance; - - private CategoryListener(StudyManager instance) - { - _instance = instance; - } - - @Override - public void categoryDeleted(User user, ViewCategory category) - { - for (DatasetDefinition def : getDatasetsForCategory(category)) - { - def = def.createMutable(); - def.setCategoryId(0); - def.save(user); - } - } - - @Override - public void categoryCreated(User user, ViewCategory category) - {} - - @Override - public void categoryUpdated(User user, ViewCategory category) - { - Container c = ContainerManager.getForId(category.getContainerId()); - if (null != c) - _instance._datasetHelper.clearCache(c); - } - - private Collection getDatasetsForCategory(ViewCategory category) - { - if (category != null) - { - Study study = _instance.getStudy(ContainerManager.getForId(category.getContainerId())); - if (study != null) - { - return _instance._datasetHelper.getDatasetsForCategory(study, category); - } - } - - return Collections.emptyList(); - } - } - - /** - * Get the shared study in the project for the given study (excluding the shared study itself) - */ - @Nullable - public Study getSharedStudy(@NotNull Container c) - { - if (c.isProject()) - return null; - Container p = c.getProject(); - if (null == p) - return null; - Study sharedStudy = getStudy(p); - if (null == sharedStudy) - return null; - if (!sharedStudy.getShareDatasetDefinitions()) - return null; - return sharedStudy; - } - - /** - * Get the shared study in the project for the given study (excluding the shared study itself.) - */ - @Nullable - public Study getSharedStudy(@NotNull Study study) - { - return getSharedStudy(study.getContainer()); - } - - /** - * Get the shared study in the project for the given study - * or just return the current study if no shared study exists. - */ - public @NotNull Study getSharedStudyOrCurrent(@NotNull Study study) - { - Study sharedStudy = getSharedStudy(study); - return sharedStudy != null ? sharedStudy : study; - } - - /** - * Get the Study to use for visits -- either the - * project shared study's container (if shared visits is turned on) - * or the current study container. - */ - @NotNull - public Study getStudyForVisits(@NotNull Study study) - { - Study sharedStudy = getSharedStudy(study); - if (sharedStudy != null && sharedStudy.getShareVisitDefinitions()) - return sharedStudy; - - return study; - } - - /** - * Get the Study to use for VisitTags -- either the - * project shared study's container or the current study container. - */ - @NotNull - public Study getStudyForVisitTag(@NotNull Study study) - { - return getSharedStudyOrCurrent(study); - } - - - /* - * TESTING - */ - - // To see detailed logging from StatementDataIterator, configure org.labkey.study.model.StudyManager$DatasetImportTestCase to level TRACE - private static class Tests {} - public static final Logger TEST_LOGGER = LogManager.getLogger(Tests.class); - - - public static class VisitCreationTestCase extends Assert - { - private static final double DELTA = 1E-8; - - @Test - public void testDateConversion() - { - Date d = new Date(); - String iso = DateUtil.toISO(d.getTime(), true); - DbSchema core = CoreSchema.getInstance().getSchema(); - SQLFragment select = new SQLFragment("SELECT "); - select.append(core.getSqlDialect().getISOFormat(new SQLFragment("?",d))); - String db = new SqlSelector(core, select).getObject(String.class); - // SQL SERVER doesn't quite store millisecond precision - assertEquals(23,iso.length()); - assertEquals(23,db.length()); - assertEquals(iso.substring(0,20), db.substring(0,20)); - String jdbc = (String)JdbcType.VARCHAR.convert(d); - assertEquals(jdbc, iso); - } - - @Test - public void testExistingVisitBased() - { - StudyImpl study = new StudyImpl(); - study.setContainer(JunitUtil.getTestContainer()); - study.setTimepointType(TimepointType.VISIT); - - List existingVisits = new ArrayList<>(3); - existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(1), BigDecimal.valueOf(1), null, Visit.Type.BASELINE)); - existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(2), BigDecimal.valueOf(2), null, Visit.Type.BASELINE)); - existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(2.5), BigDecimal.valueOf(3.0), null, Visit.Type.BASELINE)); - - assertEquals("Should return existing visit", existingVisits.get(0), getInstance().ensureVisitWithoutSaving(study, 1, Visit.Type.BASELINE, existingVisits)); - assertEquals("Should return existing visit", existingVisits.get(1), getInstance().ensureVisitWithoutSaving(study, 2, Visit.Type.BASELINE, existingVisits)); - assertEquals("Should return existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 2.5, Visit.Type.BASELINE, existingVisits)); - assertEquals("Should return existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 3.0, Visit.Type.BASELINE, existingVisits)); - - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.1, Visit.Type.BASELINE, existingVisits), existingVisits, 1.1, 1.1); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 3.001, Visit.Type.BASELINE, existingVisits), existingVisits, 3.001, 3.001); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 4, Visit.Type.BASELINE, existingVisits), existingVisits, 4, 4); - } - - @Test - public void testEmptyVisitBased() - { - StudyImpl study = new StudyImpl(); - study.setContainer(JunitUtil.getTestContainer()); - study.setTimepointType(TimepointType.VISIT); - - List existingVisits = new ArrayList<>(); - - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.1, Visit.Type.BASELINE, existingVisits), existingVisits, 1.1, 1.1); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 3.001, Visit.Type.BASELINE, existingVisits), existingVisits, 3.001, 3.001); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 4, Visit.Type.BASELINE, existingVisits), existingVisits, 4, 4); - } - - @Test - public void testEmptyDateBased() - { - StudyImpl study = new StudyImpl(); - study.setContainer(JunitUtil.getTestContainer()); - study.setTimepointType(TimepointType.DATE); - - List existingVisits = new ArrayList<>(); - - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1, Visit.Type.BASELINE, existingVisits), existingVisits, 1, 1); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, -10, Visit.Type.BASELINE, existingVisits), existingVisits, -10, -10); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.5, Visit.Type.BASELINE, existingVisits), existingVisits, 1.5, 1.5); - - study.setDefaultTimepointDuration(7); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0, Visit.Type.BASELINE, existingVisits), existingVisits, 0, 6); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1, Visit.Type.BASELINE, existingVisits), existingVisits, 0, 6); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 6, Visit.Type.BASELINE, existingVisits), existingVisits, 0, 6); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 7, Visit.Type.BASELINE, existingVisits), existingVisits, 7, 13); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 10, Visit.Type.BASELINE, existingVisits), existingVisits, 7, 13); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 15, Visit.Type.BASELINE, existingVisits), existingVisits, 14, 20); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, -10, Visit.Type.BASELINE, existingVisits), existingVisits, -10, -10); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.5, Visit.Type.BASELINE, existingVisits), existingVisits, 1.5, 1.5); - } - - @Test - public void testExistingDateBased() - { - StudyImpl study = new StudyImpl(); - study.setContainer(JunitUtil.getTestContainer()); - study.setTimepointType(TimepointType.DATE); - - List existingVisits = new ArrayList<>(3); - existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(1), BigDecimal.valueOf(1), null, Visit.Type.BASELINE)); - existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(2), BigDecimal.valueOf(2), null, Visit.Type.BASELINE)); - existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(7), BigDecimal.valueOf(13), null, Visit.Type.BASELINE)); - - assertSame("Should be existing visit", existingVisits.get(0), getInstance().ensureVisitWithoutSaving(study, 1, Visit.Type.BASELINE, existingVisits)); - assertSame("Should be existing visit", existingVisits.get(1), getInstance().ensureVisitWithoutSaving(study, 2, Visit.Type.BASELINE, existingVisits)); - assertSame("Should be existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 7, Visit.Type.BASELINE, existingVisits)); - assertSame("Should be existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 10, Visit.Type.BASELINE, existingVisits)); - assertSame("Should be existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 13, Visit.Type.BASELINE, existingVisits)); - - study.setDefaultTimepointDuration(7); - assertSame("Should be existing visit", existingVisits.get(0), getInstance().ensureVisitWithoutSaving(study, 1, Visit.Type.BASELINE, existingVisits)); - assertSame("Should be existing visit", existingVisits.get(1), getInstance().ensureVisitWithoutSaving(study, 2, Visit.Type.BASELINE, existingVisits)); - assertSame("Should be existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 7, Visit.Type.BASELINE, existingVisits)); - assertSame("Should be existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 10, Visit.Type.BASELINE, existingVisits)); - assertSame("Should be existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 13, Visit.Type.BASELINE, existingVisits)); - } - - @Test - public void testCreationDateBased() - { - StudyImpl study = new StudyImpl(); - study.setContainer(JunitUtil.getTestContainer()); - study.setTimepointType(TimepointType.DATE); - - List existingVisits = new ArrayList<>(4); - existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(1), BigDecimal.valueOf(1), null, Visit.Type.BASELINE)); - existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(2), BigDecimal.valueOf(2), null, Visit.Type.BASELINE)); - existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(7), BigDecimal.valueOf(13), null, Visit.Type.BASELINE)); - existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(62), BigDecimal.valueOf(64), null, Visit.Type.BASELINE)); - - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 3, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 3); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 14, Visit.Type.BASELINE, existingVisits), existingVisits, 14, 14); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, -14, Visit.Type.BASELINE, existingVisits), existingVisits, -14, -14); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0, Visit.Type.BASELINE, existingVisits), existingVisits, 0, 0); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0.5, Visit.Type.BASELINE, existingVisits), existingVisits, 0.5, 0.5); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.5, Visit.Type.BASELINE, existingVisits), existingVisits, 1.5, 1.5); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, -5, Visit.Type.BASELINE, existingVisits), existingVisits, -5, -5); - - study.setDefaultTimepointDuration(7); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 3, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 4, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 5, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 6, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 14, Visit.Type.BASELINE, existingVisits), existingVisits, 14, 20, "Week 3"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 21, Visit.Type.BASELINE, existingVisits), existingVisits, 21, 27, "Week 4"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0, Visit.Type.BASELINE, existingVisits), existingVisits, 0, 0, "Day 0"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0.5, Visit.Type.BASELINE, existingVisits), existingVisits, 0.5, 0.5, "Day 0.5"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.5, Visit.Type.BASELINE, existingVisits), existingVisits, 1.5, 1.5, "Day 1.5"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, -5, Visit.Type.BASELINE, existingVisits), existingVisits, -5, -5, "Day -5"); - - study.setDefaultTimepointDuration(30); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 3, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 4, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 5, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 6, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 14, Visit.Type.BASELINE, existingVisits), existingVisits, 14, 29, "Day 14 - 29"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 21, Visit.Type.BASELINE, existingVisits), existingVisits, 14, 29, "Day 14 - 29"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 29, Visit.Type.BASELINE, existingVisits), existingVisits, 14, 29, "Day 14 - 29"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 30, Visit.Type.BASELINE, existingVisits), existingVisits, 30, 59, "Month 2"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 60, Visit.Type.BASELINE, existingVisits), existingVisits, 60, 61, "Day 60 - 61"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 61, Visit.Type.BASELINE, existingVisits), existingVisits, 60, 61, "Day 60 - 61"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 65, Visit.Type.BASELINE, existingVisits), existingVisits, 65, 89, "Day 65 - 89"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 100, Visit.Type.BASELINE, existingVisits), existingVisits, 90, 119, "Month 4"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0, Visit.Type.BASELINE, existingVisits), existingVisits, 0, 0, "Day 0"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0.5, Visit.Type.BASELINE, existingVisits), existingVisits, 0.5, 0.5, "Day 0.5"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.5, Visit.Type.BASELINE, existingVisits), existingVisits, 1.5, 1.5, "Day 1.5"); - validateNewVisit(getInstance().ensureVisitWithoutSaving(study, -5, Visit.Type.BASELINE, existingVisits), existingVisits, -5, -5, "Day -5"); - } - - @Test - public void testVisitDescription() - { - StudyImpl study = new StudyImpl(); - study.setContainer(JunitUtil.getTestContainer()); - study.setTimepointType(TimepointType.DATE); - - List existingVisits = new ArrayList<>(); - - VisitImpl newVisit = getInstance().ensureVisitWithoutSaving(study, 1, Visit.Type.BASELINE, existingVisits); - newVisit.setDescription("My custom visit description"); - validateNewVisit(newVisit, existingVisits, 1, 1, "Day 1", "My custom visit description"); - } - - private void validateNewVisit(VisitImpl newVisit, List existingVisits, double seqNumMin, double seqNumMax, String label, String description) - { - validateNewVisit(newVisit, existingVisits, seqNumMin, seqNumMax, label); - assertEquals("Descriptions don't match", description, newVisit.getDescription()); - } - - private void validateNewVisit(VisitImpl newVisit, List existingVisits, double seqNumMin, double seqNumMax, String label) - { - validateNewVisit(newVisit, existingVisits, seqNumMin, seqNumMax); - assertEquals("Labels don't match", label, newVisit.getLabel()); - } - - private void validateNewVisit(VisitImpl newVisit, List existingVisits, double seqNumMin, double seqNumMax) - { - for (VisitImpl existingVisit : existingVisits) - { - assertNotSame("Should be a new visit", newVisit, existingVisit); - } - assertEquals("Shouldn't have a rowId yet", 0, newVisit.getRowId()); - assertEquals("Wrong sequenceNumMin", VisitImpl.getSequenceNum(seqNumMin), newVisit.getSequenceNumMin()); - assertEquals("Wrong sequenceNumMax", VisitImpl.getSequenceNum(seqNumMax), newVisit.getSequenceNumMax()); - } - } - - public static class StudySnapshotTestCase extends Assert - { - @Test - public void testComplianceSettings() - { - // We load the SnapshotSettings bean from serialized JSON in the core.StudySnapshot.Settings column. This - // test ensures that we serialize using the latest compliance properties but continue to correctly load - // older snapshots that might specify "removeProtectedColumns":true instead of "phiLevel":. This - // was broken shortly after we migrated to using phiLevel, see #xxxx. - - // phiLevel property takes precedence over legacy properties - testComplianceSettings("\"removeProtectedColumns\":true,\"removePhiColumns\":false,\"phiLevel\":\"Limited\",\"shiftDates\":false,\"useAlternateParticipantIds\":false,\"maskClinic\":false", PHI.Limited); - testComplianceSettings("\"removeProtectedColumns\":false,\"removePhiColumns\":false,\"phiLevel\":\"Limited\",\"shiftDates\":false,\"useAlternateParticipantIds\":false,\"maskClinic\":false", PHI.Limited); - testComplianceSettings("\"phiLevel\":\"Restricted\"", PHI.Restricted); - testComplianceSettings("\"phiLevel\":\"PHI\"", PHI.PHI); - testComplianceSettings("\"phiLevel\":\"Limited\"", PHI.Limited); - testComplianceSettings("\"phiLevel\":\"NotPHI\"", PHI.NotPHI); - - // removeProtectedColumns:true means include no PHI columns - testComplianceSettings("\"removeProtectedColumns\":true,\"shiftDates\":true,\"useAlternateParticipantIds\":true,\"maskClinic\":true", PHI.NotPHI); - testComplianceSettings("\"removeProtectedColumns\":false,\"shiftDates\":true,\"useAlternateParticipantIds\":true,\"maskClinic\":true", PHI.Restricted); - - // removePhiColumns property should have no effect - testComplianceSettings("\"removeProtectedColumns\":true,\"removePhiColumns\":true", PHI.NotPHI); - testComplianceSettings("\"removeProtectedColumns\":true,\"removePhiColumns\":false", PHI.NotPHI); - - // If no properties are specified then include all columns - testComplianceSettings("\"shiftDates\":true,\"useAlternateParticipantIds\":true,\"maskClinic\":true", PHI.Restricted); - testComplianceSettings("", PHI.Restricted); - } - - private static final String JSON_PREFIX = "{\"description\":null,\"participantGroups\":[],\"participants\":null,\"datasets\":[5008,5024,5025,5026,5004,5006,5007],\"datasetRefresh\":true,\"datasetRefreshDelay\":30,\"visits\":null,\"specimenRequestId\":null,\"includeSpecimens\":true,\"specimenRefresh\":true,\"studyObjects\":[],\"lists\":[],\"views\":[],\"reports\":[],\"folderObjects\":[]"; - - private void testComplianceSettings(String settingsJson, PHI expectedLevel) - { - String json = JSON_PREFIX + (StringUtils.isNotEmpty(settingsJson) ? "," + settingsJson + "}" : "}"); - StudySnapshot snapshot = new StudySnapshot(); - snapshot.setSettings(json); - - testSnapshot(snapshot, expectedLevel); - } - - @Test - public void testStoredSnapshots() - { - Collection snapshots = StudyManager.getInstance().getStudySnapshots(null); - - for (StudySnapshot snapshot : snapshots) - { - PHI level = snapshot.getSnapshotSettings().getPhiLevel(); - testSnapshot(snapshot, level); - StudySnapshot snapshotFromRowId = StudyManager.getInstance().getStudySnapshot(snapshot.getRowId()); - testSnapshot(snapshotFromRowId, level); - } - } - - private void testSnapshot(StudySnapshot snapshot, PHI expectedLevel) - { - assertNotNull(snapshot); - assertNotNull(expectedLevel); - SnapshotSettings settings = snapshot.getSnapshotSettings(); - assertNotNull("getPhiLevel() returned null", settings.getPhiLevel()); - assertEquals(expectedLevel, settings.getPhiLevel()); - - // Test the settings JSON that this snapshot generates - String serializedJson = snapshot.getSettings(); - String expectedLevelJson = "\"phiLevel\":\"" + expectedLevel.name() + "\""; - assertTrue("Serialized JSON did not include " + expectedLevelJson, serializedJson.contains(expectedLevelJson)); - assertFalse("Serialized JSON included removeProtectedColumns", serializedJson.contains("removeProtectedColumns")); - assertFalse("Serialized JSON included removePhiColumns", serializedJson.contains("removePhiColumns")); - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.study.model; + +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.Constants; +import org.labkey.api.annotations.Migrate; +import org.labkey.api.assay.AssayService; +import org.labkey.api.attachments.Attachment; +import org.labkey.api.attachments.AttachmentParent; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.cache.BlockingCache; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheLoader; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.collections.LabKeyCollectors; +import org.labkey.api.compliance.ComplianceFolderSettings; +import org.labkey.api.compliance.ComplianceService; +import org.labkey.api.compliance.PhiColumnBehavior; +import org.labkey.api.data.Activity; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DatabaseCache; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DbScope.CommitTaskOption; +import org.labkey.api.data.DbScope.Transaction; +import org.labkey.api.data.Filter; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.LookupResolutionType; +import org.labkey.api.data.PHI; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.PropertyManager.WritablePropertyMap; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SimpleFilter.OrClause; +import org.labkey.api.data.SimpleFilter.SQLClause; +import org.labkey.api.data.Sort; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpdateableTableInfo; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.dataiterator.BeanDataIterator; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DataIteratorUtil; +import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; +import org.labkey.api.dataiterator.Pump; +import org.labkey.api.dataiterator.StandardDataIteratorBuilder; +import org.labkey.api.exceptions.OptimisticConflictException; +import org.labkey.api.exp.ChangePropertyDescriptorException; +import org.labkey.api.exp.DomainDescriptor; +import org.labkey.api.exp.DomainNotFoundException; +import org.labkey.api.exp.DomainURIFactory; +import org.labkey.api.exp.ObjectProperty; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptor; +import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptorsList; +import org.labkey.api.exp.OntologyObject; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.ProvenanceService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.SystemProperty; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleHtmlView; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.portal.ProjectUrls; +import org.labkey.api.qc.DataState; +import org.labkey.api.qc.QCStateManager; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.query.snapshot.QuerySnapshotDefinition; +import org.labkey.api.reader.ColumnDescriptor; +import org.labkey.api.reader.DataLoader; +import org.labkey.api.reader.MapLoader; +import org.labkey.api.reports.model.ReportPropsManager; +import org.labkey.api.reports.model.ViewCategory; +import org.labkey.api.reports.model.ViewCategoryListener; +import org.labkey.api.reports.model.ViewCategoryManager; +import org.labkey.api.search.SearchService; +import org.labkey.api.search.SearchService.LastIndexedClause; +import org.labkey.api.security.RoleAssignment; +import org.labkey.api.security.SecurableResource; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicy; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.User; +import org.labkey.api.security.UserPrincipal; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.security.roles.RestrictedReaderRole; +import org.labkey.api.security.roles.Role; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.specimen.SpecimenManager; +import org.labkey.api.specimen.SpecimenSchema; +import org.labkey.api.specimen.location.LocationCache; +import org.labkey.api.specimen.model.SpecimenTablesProvider; +import org.labkey.api.study.Cohort; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.DataspaceContainerFilter; +import org.labkey.api.study.QueryHelper; +import org.labkey.api.study.QueryHelper.StudyCacheCollections; +import org.labkey.api.study.SpecimenService; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.TimepointType; +import org.labkey.api.study.Visit; +import org.labkey.api.study.Visit.Order; +import org.labkey.api.study.model.ParticipantDataset; +import org.labkey.api.study.model.ParticipantInfo; +import org.labkey.api.studydesign.StudyDesignManager; +import org.labkey.api.studydesign.StudyDesignService; +import org.labkey.api.studydesign.query.AbstractStudyDesignDomainKind; +import org.labkey.api.studydesign.query.StudyPersonnelDomainKind; +import org.labkey.api.studydesign.query.StudyProductAntigenDomainKind; +import org.labkey.api.studydesign.query.StudyProductDomainKind; +import org.labkey.api.studydesign.query.StudyTreatmentDomainKind; +import org.labkey.api.studydesign.query.StudyTreatmentProductDomainKind; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.Path; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.WebPartView; +import org.labkey.api.webdav.SimpleDocumentResource; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.study.StudySchema; +import org.labkey.study.controllers.BaseStudyController.StudyJspView; +import org.labkey.study.controllers.StudyController; +import org.labkey.study.dataset.DatasetAuditProvider; +import org.labkey.study.importer.SchemaReader; +import org.labkey.study.model.StudySnapshot.SnapshotSettings; +import org.labkey.study.query.DatasetTableImpl; +import org.labkey.study.query.DatasetUpdateService; +import org.labkey.study.query.StudyQuerySchema; +import org.labkey.study.visitmanager.AbsoluteDateVisitManager; +import org.labkey.study.visitmanager.RelativeDateVisitManager; +import org.labkey.study.visitmanager.SequenceVisitManager; +import org.labkey.study.visitmanager.VisitManager; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.validation.BindException; + +import java.io.IOException; +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.WeakHashMap; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.labkey.api.action.SpringActionController.ERROR_MSG; +import static org.labkey.api.util.IntegerUtils.asInteger; +import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.PERSONNEL_TABLE_NAME; +import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.PRODUCT_ANTIGEN_TABLE_NAME; +import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.PRODUCT_TABLE_NAME; +import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.TREATMENT_PRODUCT_MAP_TABLE_NAME; +import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.TREATMENT_TABLE_NAME; + +public class StudyManager +{ + public static final SearchService.SearchCategory datasetCategory = new SearchService.SearchCategory("dataset", "Study Datasets"); + public static final SearchService.SearchCategory subjectCategory = new SearchService.SearchCategory("subject", "Study Subjects"); + + private static final Logger _log = LogHelper.getLogger(StudyManager.class, "Dataset operations"); + private static final StudyManager _instance = new StudyManager(); + private static final StudySchema SCHEMA = StudySchema.getInstance(); + private static final String LSID_REQUIRED = "LSID_REQUIRED"; + + private final StudyHelper _studyHelper; + private final VisitHelper _visitHelper; + private final DatasetHelper _datasetHelper; + private final QueryHelper> _cohortHelper; + private final BlockingCache> _sharedProperties; + private final BlockingCache> _participantCache = DatabaseCache.get(StudySchema.getInstance().getScope(), Constants.getMaxContainers(), CacheManager.HOUR, "Participants", (c, argument) -> { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + return Collections.unmodifiableMap( + new TableSelector(StudySchema.getInstance().getTableInfoParticipant(), filter, new Sort("ParticipantId")) + .stream(Participant.class) + .collect(LabKeyCollectors.toLinkedMap(Participant::getParticipantId, participant -> participant)) + ); + }); + + private StudyManager() + { + _studyHelper = new StudyHelper(); + _visitHelper = new VisitHelper(); + _cohortHelper = new QueryHelper<>(() -> StudySchema.getInstance().getTableInfoCohort(), CohortImpl.class, "Label"); + + /* + * Whenever we explicitly invalidate a dataset, unmaterialize it as well. This is probably a little overkill, + * e.g. name change doesn't need to unmaterialize however, this is the best choke point + */ + _datasetHelper = new DatasetHelper(); + + // Cache of PropertyDescriptors found in the Shared container for datasets in the given study Container. + // The shared properties cache will be cleared when the _datasetHelper cache is cleared. + _sharedProperties = CacheManager.getBlockingCache(1000, CacheManager.UNLIMITED, "Study shared properties", + (key, argument) -> + { + Container sharedContainer = ContainerManager.getSharedContainer(); + assert key != sharedContainer; + + Collection defs = _datasetHelper.getCollection(key); + + Set set = new LinkedHashSet<>(); + for (DatasetDefinition def : defs) + { + Domain domain = def.getDomain(); + if (domain == null) + continue; + + for (DomainProperty dp : domain.getProperties()) + if (dp.getContainer().equals(sharedContainer)) + set.add(dp.getPropertyDescriptor()); + } + return Collections.unmodifiableSet(set); + } + ); + + ViewCategoryManager.addCategoryListener(new CategoryListener(this)); + } + + // Study helper is different from other query helpers. There's (at most) one study per container, so we cache a + // single map holding all StudyImpls at the root. Even though we're caching a single object in this cache, it + // provides a fast "has study" check (see Issue 19632) and leverages the DatabaseCache & cache-clearing semantics + // of QueryHelper. + private static class StudyHelper extends QueryHelper> + { + private static final Container ROOT = ContainerManager.getRoot(); + + private StudyHelper() + { + super(() -> StudySchema.getInstance().getTableInfoStudy(), StudyImpl.class, "Label"); + } + + private @NotNull Collection getAllStudies() + { + return getCollections().getCollection(); + } + + private @Nullable StudyImpl getStudy(Container c) + { + return getCollections().get(c.getId()); + } + + @Override + protected TableSelector getTableSelector(Container c) + { + assert c.equals(ROOT); + return new TableSelector(getTableInfo(), null, new Sort(_defaultSortString)); + } + + private StudyCacheCollections getCollections() + { + return getCollections(ROOT); + } + + @Override + public void clearCache(Container c) + { + super.clearCache(ROOT); + } + } + + private static class VisitHelper extends QueryHelper + { + private static final Order DEFAULT_ORDER = Order.DISPLAY; + + private VisitHelper() + { + super(() -> StudySchema.getInstance().getTableInfoVisit(), VisitImpl.class, DEFAULT_ORDER.getSortColumns()); + } + + private Collection getCollection(Container c, Order order) + { + return getCollections(c).getCollection(order); + } + + @Override + protected VisitCollections createCollections(Collection collection) + { + return new VisitCollections(collection); + } + + private static class VisitCollections extends StudyCacheCollections + { + private final Collection _sequenceNumVisits; + private final Collection _chronologicalVisits; + + private VisitCollections(Collection collection) + { + super(collection); + + // I'd prefer to push comparators into Visit.Order, but Visit (in API) doesn't know about the display + // order field. + List sorted = new ArrayList<>(collection); + sorted.sort(Comparator.comparing(VisitImpl::getSequenceNumMin)); + _sequenceNumVisits = Collections.unmodifiableCollection(sorted); + + sorted = new ArrayList<>(collection); + sorted.sort(Comparator.comparing(VisitImpl::getChronologicalOrder).thenComparing(VisitImpl::getSequenceNumMin)); + _chronologicalVisits = Collections.unmodifiableCollection(sorted); + } + + private Collection getCollection(Order order) + { + return order == DEFAULT_ORDER ? getCollection() : order == Order.SEQUENCE_NUM ? _sequenceNumVisits : _chronologicalVisits; + } + } + } + + private class DatasetHelper extends QueryHelper + { + private DatasetHelper() + { + super(() -> StudySchema.getInstance().getTableInfoDataset(), DatasetDefinition.class); + } + + @Override + public void clearCache(Container c) + { + super.clearCache(c); + _sharedProperties.remove(c); + } + + private @Nullable DatasetDefinition getByName(Study study, String name) + { + return getCollections(study)._nameMap.get(name); + } + + private @Nullable DatasetDefinition getByLabel(Study study, String label) + { + return getCollections(study)._labelMap.get(label); + } + + private @Nullable DatasetDefinition getByEntityId(Study study, String entityId) + { + return getCollections(study)._entityIdMap.get(entityId); + } + + private @NotNull List getDatasetsForCategory(Study study, @NotNull ViewCategory category) + { + List ret = getCollections(study)._categoryMap.get(category.getRowId()); + return ret != null ? ret : Collections.emptyList(); + } + + private @NotNull List getDatasetsForCohort(Study study, @NotNull Cohort cohort) + { + return getCollections(study).getDatasetsForCohort(cohort); + } + + @Override + protected DatasetCollections createCollections(Collection collection) + { + return new DatasetCollections(collection); + } + + protected DatasetCollections getCollections(Study study) + { + return super.getCollections(study.getContainer()); + } + + private static class DatasetCollections extends StudyCacheCollections + { + private final Map _nameMap = new CaseInsensitiveHashMap<>(); + private final Map _labelMap = new CaseInsensitiveHashMap<>(); + private final Map _entityIdMap = new HashMap<>(); + + private final Map> _categoryMap; + private final Map> _cohortMap; + private final List _nullCohortDatasets; + + private DatasetCollections(Collection collection) + { + super(collection); + + // study.Dataset has constraints on LOWER(Name) and LOWER(Label), so this code path should never attempt + // to put duplicates into these maps. Use asserts to verify this. + collection.forEach(def -> { + DatasetDefinition old = _nameMap.put(def.getName(), def); assert old == null; + old = _labelMap.put(def.getLabel(), def); assert old == null; + old = _entityIdMap.put(def.getEntityId(), def); assert old == null; + }); + + // Group by (non-null) category ID + _categoryMap = collection.stream() + .filter(def -> def.getCategoryId() != null) + .collect(Collectors.groupingBy(DatasetDefinition::getCategoryId)); + + // Group by cohort + _nullCohortDatasets = collection.stream() + .filter(def -> def.getCohortId() == null) + .toList(); + + _cohortMap = collection.stream() + .filter(def -> def.getCohortId() != null) + .collect(Collectors.groupingBy(DatasetDefinition::getCohortId)); + + // Datasets with null cohort get added to every cohort list. Also, make lists immutable. + if (!_cohortMap.isEmpty()) + { + _cohortMap.keySet().forEach(key -> { + List list = _cohortMap.get(key); + list.addAll(_nullCohortDatasets); + _cohortMap.put(key, Collections.unmodifiableList(list)); + }); + } + } + + private @NotNull List getDatasetsForCohort(Cohort cohort) + { + List ret = _cohortMap.get(cohort.getRowId()); + return ret != null ? ret : _nullCohortDatasets; + } + } + } + + public static StudyManager getInstance() + { + return _instance; + } + + @Nullable + public StudyImpl getStudy(@NotNull Container c) + { + StudyImpl study; + boolean retry = true; + + while (true) + { + study = _studyHelper.getStudy(c); + + // This should be a very fast "has study" check, replacement fix for Issue 19632 + if (null == study) + break; + + assert (study.getContainer().getId().equals(c.getId())); + + // UNDONE: There is a subtle bug in caching, cached objects shouldn't hold onto Container objects + Container freshestContainer = ContainerManager.getForId(c.getId()); + if (study.getContainer() == freshestContainer) + break; + + if (!retry) // we only get one retry + break; + + _log.debug("Clearing cached study for {} as its container reference didn't match the current object from ContainerManager {}", c, freshestContainer); + + _studyHelper.clearCache(c); // Clear the cached "all studies" map + retry = false; + } + + return study; + } + + /** @return all studies in the whole server, unfiltered by permissions and sorted by Label */ + @NotNull + public Collection getAllStudies() + { + return _studyHelper.getAllStudies(); + } + + /** @return all studies under the given root in the container hierarchy (inclusive), unfiltered by permissions */ + @NotNull + public Set getAllStudies(@NotNull Container root) + { + Set result = new LinkedHashSet<>(); + for (StudyImpl study : getAllStudies()) + { + if (study.getContainer().equals(root) || study.getContainer().isDescendant(root)) + { + result.add(study); + } + } + return Collections.unmodifiableSet(result); + } + + /** @return all studies under the given root in the container hierarchy (inclusive), to which the user has at least read permission */ + @NotNull + public Set getAllStudies(@NotNull Container root, @NotNull User user) + { + return getAllStudies(root, user, ReadPermission.class); + } + + /** @return all studies under the given root in the container hierarchy (inclusive), to which the user has at least the specified permission */ + @NotNull + public Set getAllStudies(@NotNull Container root, @NotNull User user, @NotNull Class perm) + { + Set result = new LinkedHashSet<>(); + for (StudyImpl study : getAllStudies()) + { + if (study.getContainer().hasPermission(user, perm) && + (study.getContainer().equals(root) || study.getContainer().isDescendant(root))) + { + result.add(study); + } + } + return Collections.unmodifiableSet(result); + } + + public StudyImpl createStudy(User user, StudyImpl study) + { + Container container = study.getContainer(); + assert null != container; + assert null != user; + if (study.getLsid() == null) + study.initLsid(); + + if (study.getProtocolDocumentEntityId() == null) + study.setProtocolDocumentEntityId(GUID.makeGUID()); + + if (study.getAlternateIdDigits() == 0) + study.setAlternateIdDigits(StudyManager.ALTERNATEID_DEFAULT_NUM_DIGITS); + + try (Transaction transaction = StudySchema.getInstance().getScope().ensureTransaction()) + { + SpecimenSchema.get().getTableInfoLocation(container, user); // This provisioned table is needed for creating the study + study = _studyHelper.create(user, study); + clearCaches(container,false); + + //note: we no longer copy the container's policy to the study upon creation + //instead, we let it inherit the container's policy until the security type + //is changed to one of the advanced options. + + // Force provisioned specimen tables to be created + SpecimenSchema.get().getTableInfoSpecimenPrimaryType(container, user); + SpecimenSchema.get().getTableInfoSpecimenDerivative(container, user); + SpecimenSchema.get().getTableInfoSpecimenAdditive(container, user); + SpecimenSchema.get().getTableInfoSpecimen(container, user); + SpecimenSchema.get().getTableInfoVial(container, user); + SpecimenSchema.get().getTableInfoSpecimenEvent(container, user); + transaction.commit(); + } + StudyDesignManager.get().ensureStudyDesignDomains(container, user); + QueryService.get().updateLastModified(); + ContainerManager.notifyContainerChange(container.getId(), ContainerManager.Property.StudyChange); + return study; + } + + public ValidationException updateStudy(@Nullable User user, StudyImpl study) + { + StudyImpl oldStudy = getStudy(study.getContainer()); + Date oldStartDate = oldStudy.getStartDate(); + _studyHelper.update(user, study, study.getContainer()); + ValidationException errors = new ValidationException(); + + if (oldStudy.getTimepointType() == TimepointType.DATE && !Objects.equals(study.getStartDate(), oldStartDate)) + { + // start date has changed, and datasets may use that value. Uncache. + RelativeDateVisitManager visitManager = (RelativeDateVisitManager) getVisitManager(study); + errors = visitManager.recomputeDates(oldStartDate, user); + clearCaches(study.getContainer(), true); + } + else + { + // Need to get rid of any old copies of the study + clearCaches(study.getContainer(), false); + } + + if (oldStudy.getSecurityType() != study.getSecurityType()) + { + String comment = "Dataset security type changed from " + oldStudy.getSecurityType() + " to " + study.getSecurityType(); + StudyService.get().addStudyAuditEvent(study.getContainer(), user, comment); + } + QueryService.get().updateLastModified(); + return errors; + } + + public void updateStudySnapshot(StudySnapshot snapshot, User user) + { + // For now, "refresh" is the only field that can be updated (plus the Modified fields, which get handled automatically) + Map map = new HashMap<>(); + map.put("refresh", snapshot.isRefresh()); + + Table.update(user, StudySchema.getInstance().getTableInfoStudySnapshot(), map, snapshot.getRowId()); + } + + public void createDatasetDefinition(User user, Container container, int datasetId) + { + createDatasetDefinition(user, new DatasetDefinition(getStudy(container), datasetId)); + } + + public void createDatasetDefinition(User user, DatasetDefinition datasetDefinition) + { + if (datasetDefinition.getDatasetId() <= 0) + throw new IllegalArgumentException("datasetId must be greater than zero."); + DbScope scope = StudySchema.getInstance().getScope(); + + try (Transaction transaction = scope.ensureTransaction()) + { + ensureViewCategory(user, datasetDefinition); + _datasetHelper.create(user, datasetDefinition); + // This method call has the side effect of ensuring that we have a domain. If we don't create it here, + // we're open to a race condition if another thread tries to do something with the dataset's table + // and ends up attempting to create the domain as well + datasetDefinition.getStorageTableInfo(true); + + QueryService.get().updateLastModified(); + transaction.commit(); + } + indexDataset(SearchService.get().defaultTask().getQueue(datasetDefinition.getContainer(), SearchService.PRIORITY.modified), datasetDefinition); + } + + /** + * Temporary shim until we can redo the dataset category UI + */ + private void ensureViewCategory(User user, DatasetDefinition def) + { + ViewCategory category = null; + + if (def.getCategoryId() != null) + category = ViewCategoryManager.getInstance().getCategory(def.getContainer(), def.getCategoryId()); + + if (category == null && def.getCategory() != null) + { + // the imported category name may be encoded to contain subcategory info + String[] parts = ViewCategoryManager.getInstance().decode(def.getCategory()); + category = ViewCategoryManager.getInstance().ensureViewCategory(def.getContainer(), user, parts); + } + + if (category != null) + { + def.setCategoryId(category.getRowId()); + def.setCategory(category.getLabel()); + } + } + + public void updateDatasetDefinition(User user, DatasetDefinition datasetDefinition, List errors) + { + try + { + updateDatasetDefinition(user, datasetDefinition); + } + catch (IllegalArgumentException ex) + { + errors.add(ex.getMessage()); + } + } + + public boolean isKeyChanged(final DatasetDefinition datasetDefinition) + { + DatasetDefinition old = getDatasetDefinition(datasetDefinition.getStudy(), datasetDefinition.getDatasetId()); + if (old != null) + { + return old.isDemographicData() != datasetDefinition.isDemographicData() || + !Strings.CS.equals(old.getKeyPropertyName(), datasetDefinition.getKeyPropertyName()) || + old.getUseTimeKeyField() != datasetDefinition.getUseTimeKeyField(); + } + return false; + } + + /* most users should call the List errors version to avoid uncaught exceptions */ + @Deprecated + public boolean updateDatasetDefinition(User user, final DatasetDefinition datasetDefinition) + { + if (datasetDefinition.isShared()) + { + // check if we're updating the dataset property overrides in a sub-folder + if (!datasetDefinition.getContainer().equals(datasetDefinition.getDefinitionContainer())) + { + return updateDatasetPropertyOverrides(user, datasetDefinition); + } + } + + DbScope scope = StudySchema.getInstance().getScope(); + + try (Transaction transaction = scope.ensureTransaction()) + { + DatasetDefinition old = getDatasetDefinition(datasetDefinition.getStudy(), datasetDefinition.getDatasetId()); + if (null == old) + throw OptimisticConflictException.create(Table.ERROR_DELETED); + + // make sure we reload domain and tableinfo + Domain domain = datasetDefinition.refreshDomain(); + + // Check if the extra key field has changed + boolean isProvisioned = domain != null && domain.getStorageTableName() != null; + boolean isKeyChanged = isKeyChanged(datasetDefinition); + boolean isSharedChanged = old.getDataSharingEnum() != datasetDefinition.getDataSharingEnum(); + if (isProvisioned && isSharedChanged) + { + // let's not change the shared setting if there are existing rows + if (new TableSelector(datasetDefinition.getStorageTableInfo(false)).exists()) + { + throw new IllegalArgumentException("Can't change data sharing setting if there are existing data rows."); + } + } + + if (isProvisioned && isKeyChanged) + { + TableInfo storageTableInfo = datasetDefinition.getStorageTableInfo(false); + + // If so, we need to update the _key column and the LSID + + // Set the _key column to be the value of the selected column + // Change how we build up tableName + String tableName = storageTableInfo.toString(); + SQLFragment updateKeySQL = new SQLFragment("UPDATE " + tableName + " SET _key = "); + if (datasetDefinition.getUseTimeKeyField()) + { + ColumnInfo col = storageTableInfo.getColumn("Date"); + if (null == col) + { + throw new IllegalArgumentException("Cannot find 'Date' column in table: " + tableName); + } + SQLFragment colFrag = col.getValueSql(tableName); + updateKeySQL.append(storageTableInfo.getSqlDialect().getISOFormat(colFrag)); + } + else if (datasetDefinition.getKeyPropertyName() == null) + { + // No column selected, so set it to be null + updateKeySQL.append("NULL"); + } + else + { + ColumnInfo col = storageTableInfo.getColumn(datasetDefinition.getKeyPropertyName()); + if (null == col) + { + throw new IllegalArgumentException("Cannot find 'key' column: " + datasetDefinition.getKeyPropertyName() + " in table: " + tableName); + } + SQLFragment colFrag = col.getValueSql(tableName); + if (col.getJdbcType() == JdbcType.TIMESTAMP) + colFrag = storageTableInfo.getSqlDialect().getISOFormat(colFrag); + updateKeySQL.append(colFrag); + } + + try + { + new SqlExecutor(StudySchema.getInstance().getSchema()).setLogLevel(Level.OFF).execute(updateKeySQL); + + // Now update the LSID column. Note - this needs to be the same as DatasetImportHelper.getURI() + SQLFragment updateLSIDSQL = new SQLFragment("UPDATE " + tableName + " SET lsid = "); + updateLSIDSQL.append(datasetDefinition.generateLSIDSQL()); + new SqlExecutor(StudySchema.getInstance().getSchema()).execute(updateLSIDSQL); + } + catch (DataIntegrityViolationException x) + { + _log.debug("Old Dataset: " + old.getName()); + _log.debug(" Demographic: " + old.isDemographicData()); + _log.debug(" Key: " + old.getKeyPropertyName()); + _log.debug("New Dataset: " + datasetDefinition.getName()); + _log.debug(" Demographic: " + datasetDefinition.isDemographicData()); + _log.debug(" Key: " + datasetDefinition.getKeyPropertyName()); + + if (datasetDefinition.isDemographicData()) + throw new IllegalArgumentException("Can not change dataset type to demographic for dataset " + datasetDefinition.getName()); + else + throw new IllegalArgumentException("Changing the dataset key would result in duplicate keys for dataset " + datasetDefinition.getName()); + } + } + Object[] pk = new Object[]{datasetDefinition.getContainer().getId(), datasetDefinition.getDatasetId()}; + ensureViewCategory(user, datasetDefinition); + ensureDatasetDefinitionDomain(user, datasetDefinition); + _datasetHelper.update(user, datasetDefinition, pk); + + QueryChangeListener.QueryPropertyChange nameChange = null; + if (!old.getName().equals(datasetDefinition.getName())) + { + nameChange = new QueryChangeListener.QueryPropertyChange<>( + QueryService.get().getUserSchema(user, datasetDefinition.getContainer(), StudyQuerySchema.SCHEMA_NAME).getQueryDefForTable(datasetDefinition.getName()), + QueryChangeListener.QueryProperty.Name, + old.getName(), + datasetDefinition.getName() + ); + } + final QueryChangeListener.QueryPropertyChange change = nameChange; + + transaction.addCommitTask(() -> + { + uncache(datasetDefinition); + if (null != change) + { + QueryService.get().fireQueryChanged(user, datasetDefinition.getContainer(), null, new SchemaKey(null, StudyQuerySchema.SCHEMA_NAME), + QueryChangeListener.QueryProperty.Name, Collections.singleton(change)); + } + indexDataset(SearchService.get().defaultTask().getQueue(datasetDefinition.getContainer(), SearchService.PRIORITY.modified), datasetDefinition); + }, CommitTaskOption.POSTCOMMIT); + + // NOTE: not redundant with uncache() in commit task, there may be an active outer transaction + uncache(datasetDefinition); + QueryService.get().updateLastModified(); + transaction.commit(); + } + datasetDefinition.refreshDomain(); + return true; + } + + /** + * Shared dataset may save some of its properties in the current container rather than the dataset definition container. + * Currently allowed overrides: + *
      + *
    • isShownByDefault
      • + *
    + * @return true if successful. + */ + private boolean updateDatasetPropertyOverrides(User user, final DatasetDefinition datasetDefinition) + { + if (!datasetDefinition.isShared() || datasetDefinition.getContainer().isProject()) + { + throw new IllegalArgumentException("Dataset property overrides can only be applied to shared datasets and in sub-containers"); + } + + DbScope scope = StudySchema.getInstance().getScope(); + + try (Transaction transaction = scope.ensureTransaction()) + { + DatasetDefinition old = getDatasetDefinition(datasetDefinition.getStudy(), datasetDefinition.getDatasetId()); + if (null == old) + throw OptimisticConflictException.create(Table.ERROR_DELETED); + + DatasetDefinition original = getDatasetDefinition(datasetDefinition.getDefinitionStudy(), datasetDefinition.getDatasetId()); + + // make sure we reload domain and tableinfo + Domain domain = datasetDefinition.refreshDomain(); + + // Error if any other properties have been changed + if (!Objects.equals(old.getLabel(), datasetDefinition.getLabel())) + { + throw new IllegalArgumentException("Shared dataset label can't be changed"); + } + if (!Objects.equals(old.getCategoryId(), datasetDefinition.getCategoryId())) + { + throw new IllegalArgumentException("Shared dataset category can't be changed"); + } + if (!Objects.equals(old.getCohortId(), datasetDefinition.getCohortId())) + { + throw new IllegalArgumentException("Shared dataset cohort can't be changed"); + } + + // track added and removed properties against the shared dataset in the definition container + Map add = new HashMap<>(); + List remove = new LinkedList<>(); + if (datasetDefinition.isShowByDefault() != original.isShowByDefault()) + add.put("showByDefault", String.valueOf(datasetDefinition.isShowByDefault())); + else + remove.add("showByDefault"); + + // update the override map + Container c = datasetDefinition.getContainer(); + String category = "dataset-overrides:" + datasetDefinition.getDatasetId(); + WritablePropertyMap map = null; + + if (!add.isEmpty()) + { + map = PropertyManager.getWritableProperties(c, category, true); + map.putAll(add); + } + + if (!remove.isEmpty()) + { + if (map == null) + map = PropertyManager.getWritableProperties(c, category, false); + + if (map != null) + { + for (String key : remove) + map.remove(key); + } + } + + // persist change -- if overrides are no longer needed, just remove it + if (map != null) + { + if (map.isEmpty()) + map.delete(); + else + map.save(); + } + + transaction.addCommitTask(() -> { + // And post-commit to make sure that no other threads have reloaded the cache in the meantime + uncache(datasetDefinition); + }, CommitTaskOption.POSTCOMMIT, CommitTaskOption.IMMEDIATE); + transaction.commit(); + } + + return true; + } + + public void deleteDatasetPropertyOverrides(User user, Container c, BindException errors) + { + if (c.isProject()) + { + errors.reject(ERROR_MSG, "can't delete dataset property override from project-level study"); + return; + } + + Study study = getStudy(c); + if (study == null) + { + errors.reject(ERROR_MSG, "study not found"); + return; + } + + if (study.isDataspaceStudy()) + { + errors.reject(ERROR_MSG, "can't delete dataset property override from a shared study"); + return; + } + + Study sharedStudy = getSharedStudy(study); + if (sharedStudy == null) + { + errors.reject(ERROR_MSG, "not a sub-study of a shared study"); + return; + } + + DbScope scope = StudySchema.getInstance().getScope(); + try (Transaction transaction = scope.ensureTransaction()) + { + for (DatasetDefinition dataset : getDatasetDefinitions(study)) + { + if (dataset.isInherited()) + deleteDatasetPropertyOverrides(user, dataset); + } + transaction.commit(); + } + } + + private boolean deleteDatasetPropertyOverrides(User user, final DatasetDefinition datasetDefinition) + { + if (!datasetDefinition.isInherited() || datasetDefinition.getContainer().isProject()) + { + throw new IllegalArgumentException("Dataset property overrides can only be applied to shared datasets and in sub-containers"); + } + + DbScope scope = StudySchema.getInstance().getScope(); + try (Transaction transaction = scope.ensureTransaction()) + { + Container c = datasetDefinition.getContainer(); + String category = "dataset-overrides:" + datasetDefinition.getDatasetId(); + PropertyManager.getNormalStore().deletePropertySet(c, category); + + transaction.addCommitTask(() -> { + // And post-commit to make sure that no other threads have reloaded the cache in the meantime + uncache(datasetDefinition); + }, CommitTaskOption.POSTCOMMIT, CommitTaskOption.IMMEDIATE); + transaction.commit(); + } + + return true; + } + + public boolean isDataUniquePerParticipant(DatasetDefinition dataset) + { + // don't use dataset.getTableInfo() since this method is called during updateDatasetDefinition`() and may be in an inconsistent state + TableInfo t = dataset.getStorageTableInfo(false); + SQLFragment sql = new SQLFragment(); + sql.append("SELECT MAX(n) FROM (SELECT COUNT(*) AS n FROM ").append(t.getFromSQL("DS")).append(" GROUP BY ParticipantId) x"); + Integer maxCount = new SqlSelector(StudySchema.getInstance().getSchema(), sql).getObject(Integer.class); + return maxCount == null || maxCount <= 1; + } + + + public static class VisitCreationException extends RuntimeException + { + public VisitCreationException(String message) + { + super(message); + } + } + + + public VisitImpl createVisit(Study study, User user, VisitImpl visit) + { + return createVisit(study, user, visit, null); + } + + + public VisitImpl createVisit(Study study, User user, VisitImpl visit, @Nullable Collection existingVisits) + { + Study visitStudy = getStudyForVisits(study); + + if (visit.getContainer() != null && !visit.getContainer().getId().equals(visitStudy.getContainer().getId())) + throw new VisitCreationException("Visit container does not match study"); + visit.setContainer(visitStudy.getContainer()); + + if (visit.getSequenceNumMin().compareTo(visit.getSequenceNumMax()) > 0) + throw new VisitCreationException("SequenceNumMin must be less than or equal to SequenceNumMax"); + + if (null == existingVisits) + existingVisits = getVisits(study, Order.SEQUENCE_NUM); + + int prevDisplayOrder = 0; + int prevChronologicalOrder = 0; + + for (VisitImpl existingVisit : existingVisits) + { + if (existingVisit.getSequenceNumMin().compareTo(visit.getSequenceNumMin()) < 0) + { + prevChronologicalOrder = existingVisit.getChronologicalOrder(); + prevDisplayOrder = existingVisit.getDisplayOrder(); + } + + if (existingVisit.getSequenceNumMin().compareTo(existingVisit.getSequenceNumMax()) > 0) + throw new VisitCreationException("Corrupt existing visit " + existingVisit + + ": SequenceNumMin must be less than or equal to SequenceNumMax"); + boolean disjoint = (visit.getSequenceNumMax().compareTo(existingVisit.getSequenceNumMin()) < 0) || (visit.getSequenceNumMin().compareTo(existingVisit.getSequenceNumMax()) > 0); + if (!disjoint) + { + throw new VisitCreationException("New visit " + visit + " overlaps existing visit " + existingVisit); + } + } + + // if our visit doesn't have a display order or chronological order set, but the visit before our new visit + // (based on sequencenum) does, then assign the previous visit's order info to our new visit. This won't always + // be exactly right, but it's better than having all newly created visits appear at the beginning of the display + // and chronological lists: + if (visit.getDisplayOrder() == 0 && prevDisplayOrder > 0) + visit.setDisplayOrder(prevDisplayOrder); + if (visit.getChronologicalOrder() == 0 && prevChronologicalOrder > 0) + visit.setChronologicalOrder(prevChronologicalOrder); + + visit = _visitHelper.create(user, visit); + + if (visit.getRowId() == 0) + throw new VisitCreationException("Visit rowId has not been set properly"); + + return visit; + } + + /** + * Return a visit object regardless of whether it exists in the database but will not insert a new + * record into the database. + */ + public VisitImpl getVisit(Study study, User user, BigDecimal sequenceNum, Visit.Type type) + { + Collection visits = getVisits(study, Order.SEQUENCE_NUM); + return ensureVisitWithoutSaving(study, sequenceNum, type, visits); + } + + /** + * Ensures the existence of a visit for the specified sequence numbers and will insert into the database + * if the visit does not yet exist. + * + * @param failForUndefinedVisits If true, new visits will not be created and an error will be added to the returned + * ValidationException object. + * @return ValidationException which will contain any relevant errors. Callers should check hasErrors on the object. + */ + public @NotNull ValidationException ensureVisits(Study study, User user, Set sequencenums, @Nullable Visit.Type type, + boolean failForUndefinedVisits) + { + Collection visits = getVisits(study, Order.SEQUENCE_NUM); + ValidationException errors = new ValidationException(); + List seqNumFailures = new ArrayList<>(); + + for (BigDecimal sequencenum : sequencenums) + { + VisitImpl result = ensureVisitWithoutSaving(study, sequencenum, type, visits); + if (result.getRowId() == 0) + { + if (!failForUndefinedVisits) + { + createVisit(study, user, result, visits); + // Refresh existing visits to avoid constraint violation, see #44425 + visits = getVisits(study, Order.SEQUENCE_NUM); + } + else + seqNumFailures.add(String.valueOf(sequencenum)); + } + } + + if (!seqNumFailures.isEmpty()) + { + String timepointNoun = study.getTimepointType().isVisitBased() ? "visit" : "timepoint"; + errors.addError(new SimpleValidationError(String.format("Creating new %ss is not allowed for this study. The following %s not currently exist : (%s)", + timepointNoun, timepointNoun + (seqNumFailures.size() > 1 ? "s do" : " does"), + String.join(", ", seqNumFailures)))); + } + return errors; + } + + private VisitImpl ensureVisitWithoutSaving(Study study, double seqNumDouble, @Nullable Visit.Type type, Collection existingVisits) + { + return ensureVisitWithoutSaving(study, VisitImpl.getSequenceNum(seqNumDouble), type, existingVisits); + } + + private VisitImpl ensureVisitWithoutSaving(Study study, BigDecimal sequenceNum, @Nullable Visit.Type type, Collection existingVisits) + { + sequenceNum = VisitImpl.normalizeSequenceNum(sequenceNum); + + // Remember the SequenceNums closest to the requested id in case we need to create one + BigDecimal nextVisit = Visit.MAX_SEQUENCE_NUM; + BigDecimal previousVisit = Visit.MIN_SEQUENCE_NUM; + for (VisitImpl visit : existingVisits) + { + if (visit.getSequenceNumMin().compareTo(sequenceNum) <= 0 && visit.getSequenceNumMax().compareTo(sequenceNum) >= 0) + return visit; + // check to see if our new sequencenum is within the range of an existing visit: + // Check if it's the closest to the requested id, either before or after + if (visit.getSequenceNumMin().compareTo(nextVisit) < 0 && visit.getSequenceNumMin().compareTo(sequenceNum) > 0) + { + nextVisit = visit.getSequenceNumMin(); + } + if (visit.getSequenceNumMax().compareTo(previousVisit) > 0 && visit.getSequenceNumMax().compareTo(sequenceNum) < 0) + { + previousVisit = visit.getSequenceNumMax(); + } + } + BigDecimal visitIdMin = sequenceNum; + BigDecimal visitIdMax = sequenceNum; + String label = null; + if (!study.getTimepointType().isVisitBased()) + { + boolean isFloatingPoint = sequenceNum.stripTrailingZeros().scale() > 0; + + // Do special handling for data-based studies + if (study.getDefaultTimepointDuration() == 1 || isFloatingPoint || sequenceNum.compareTo(BigDecimal.ZERO) < 0) + { + // See if there's a fractional part to the number + if (isFloatingPoint) + { + label = "Day " + VisitImpl.formatSequenceNum(sequenceNum); + } + else + { + // If not, drop the decimal from the default name + label = "Day " + sequenceNum.intValue(); + } + } + else + { + // Try to create a timepoint that spans the default number of days + // For example, if duration is 7 days, do timepoints for days 0-6, 7-13, 14-20, etc + int intervalNumber = sequenceNum.intValue() / study.getDefaultTimepointDuration(); + visitIdMin = BigDecimal.valueOf((long)intervalNumber * study.getDefaultTimepointDuration()); + visitIdMax = BigDecimal.valueOf((long)(intervalNumber + 1) * study.getDefaultTimepointDuration() - 1); + + // Scale the timepoint to be smaller if there are existing timepoints that overlap + // on its desired day range + if (!Visit.MIN_SEQUENCE_NUM.equals(previousVisit)) + { + visitIdMin = visitIdMin.max(previousVisit.add(BigDecimal.ONE)); + } + if (!Visit.MAX_SEQUENCE_NUM.equals(nextVisit)) + { + visitIdMax = visitIdMax.min(nextVisit.subtract(BigDecimal.ONE)); + } + + // Default label is "Day X - Y" + label = "Day " + visitIdMin.intValue() + " - " + visitIdMax.intValue(); + if (visitIdMin.compareTo(visitIdMax) == 0) + { + // Single day timepoint, so don't use the range + label = "Day " + visitIdMin.intValue(); + } + else if (visitIdMin.intValue() == intervalNumber * study.getDefaultTimepointDuration() && + visitIdMax.intValue() == (intervalNumber + 1) * study.getDefaultTimepointDuration() - 1) + { + // The timepoint is the full span for the default duration, so see if we + // should call it "Week" or "Month" + if (study.getDefaultTimepointDuration() == 7) + { + label = "Week " + (intervalNumber + 1); + } + else if (study.getDefaultTimepointDuration() == 30 || study.getDefaultTimepointDuration() == 31) + { + label = "Month " + (intervalNumber + 1); + } + } + } + } + + // create visit in shared study + Study visitStudy = getStudyForVisits(study); + return new VisitImpl(visitStudy.getContainer(), visitIdMin, visitIdMax, label, type); + } + + public void importVisitAliases(Study study, User user, List aliases) throws ValidationException + { + DataIteratorBuilder it = new BeanDataIterator.Builder<>(VisitAlias.class, aliases); + importVisitAliases(study, user, it); + } + + public int importVisitAliases(final Study study, User user, DataIteratorBuilder loader) throws ValidationException + { + TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitAliases(); + DbScope scope = tinfo.getSchema().getScope(); + + // We want to delete and bulk insert in the same transaction + try (Transaction transaction = scope.ensureTransaction()) + { + clearVisitAliases(study); + + DataIteratorContext context = new DataIteratorContext(); + context.setInsertOption(QueryUpdateService.InsertOption.IMPORT); + StandardDataIteratorBuilder etl = StandardDataIteratorBuilder.forInsert(tinfo, loader, study.getContainer(), user, context); + DataIteratorBuilder insert = ((UpdateableTableInfo) tinfo).persistRows(etl, context); + Pump p = new Pump(insert, context); + p.run(); + + if (context.getErrors().hasErrors()) + throw context.getErrors().getRowErrors().get(0); + + transaction.commit(); + + return p.getRowCount(); + } + } + + + public void clearVisitAliases(Study study) + { + SimpleFilter containerFilter = SimpleFilter.createContainerFilter(study.getContainer()); + TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitAliases(); + DbScope scope = tinfo.getSchema().getScope(); + + try (Transaction transaction = scope.ensureTransaction()) + { + Table.delete(tinfo, containerFilter); + transaction.commit(); + } + } + + + public Map getVisitImportMap(Study study, boolean includeStandardMapping) + { + Collection customMapping = getCustomVisitImportMapping(study); + Collection visits = includeStandardMapping ? StudyManager.getInstance().getVisits(study, Order.SEQUENCE_NUM) : Collections.emptyList(); + + Map map = new CaseInsensitiveHashMap<>((customMapping.size() + visits.size()) * 3 / 4); + +// // allow prepended "visit" +// for (Visit visit : visits) +// { +// if (null == visit.getLabel()) +// continue; +// String label = "visit " + visit.getLabel(); +// // Use the **first** instance of each label +// if (!map.containsKey(label)) +// map.put(label, visit.getSequenceNumMin()); +// } + + // Load up standard label -> min sequence number mapping first + for (Visit visit : visits) + { + String label = visit.getLabel(); + + // Use the **first** instance of each label + if (null != label && !map.containsKey(label)) + map.put(label, visit.getSequenceNumMin()); + } + + // Now load custom mapping, overwriting any existing standard labels + for (VisitAlias alias : customMapping) + map.put(alias.getName(), alias.getSequenceNum()); + + return map; + } + + + // Return the custom import mapping (optionally provided by the admin), ordered by sequence num then row id (which + // maintains import order in the case where multiple names map to the same sequence number). + public Collection getCustomVisitImportMapping(Study study) + { + SimpleFilter containerFilter = SimpleFilter.createContainerFilter(study.getContainer()); + TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitAliases(); + + return new TableSelector(tinfo, tinfo.getColumns("Name, SequenceNum"), containerFilter, new Sort("SequenceNum,RowId")).getCollection(VisitAlias.class); + } + + + // Return the standard import mapping (generated from Visit.Label -> Visit.SequenceNumMin), ordered by sequence + // num for display purposes. Include VisitAliases that won't be used, but mark them as overridden. + public Collection getStandardVisitImportMapping(Study study) + { + List list = new LinkedList<>(); + Set labels = new CaseInsensitiveHashSet(); + Map customMap = getVisitImportMap(study, false); + + Collection visits = StudyManager.getInstance().getVisits(study, Order.SEQUENCE_NUM); + + for (Visit visit : visits) + { + String label = visit.getLabel(); + + if (null != label) + { + boolean overridden = labels.contains(label) || customMap.containsKey(label); + list.add(new VisitAlias(label, visit.getSequenceNumMin(), visit.getSequenceString(), overridden)); + + if (!overridden) + labels.add(label); + } + } + + return list; + } + + + public static class VisitAlias + { + private String _name; + private BigDecimal _sequenceNum; + private String _sequenceString; + private boolean _overridden; // For display purposes -- we show all visits and gray out the ones that are not used + + @SuppressWarnings({"UnusedDeclaration"}) // Constructed via reflection by the Table layer + public VisitAlias() + { + } + + public VisitAlias(String name, BigDecimal sequenceNum, @Nullable String sequenceString, boolean overridden) + { + _name = name; + _sequenceNum = sequenceNum; + _sequenceString = sequenceString; + _overridden = overridden; + } + + public VisitAlias(String name, BigDecimal sequenceNum) + { + this(name, VisitImpl.normalizeSequenceNum(sequenceNum), null, false); + } + + public String getName() + { + return _name; + } + + public void setName(String name) + { + _name = name; + } + + public BigDecimal getSequenceNum() + { + return _sequenceNum; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSequenceNum(BigDecimal sequenceNum) + { + _sequenceNum = VisitImpl.normalizeSequenceNum(sequenceNum); + } + + public boolean isOverridden() + { + return _overridden; + } + + public String getSequenceNumString() + { + return VisitImpl.formatSequenceNum(_sequenceNum); + } + + public String getSequenceString() + { + if (null == _sequenceString) + return getSequenceNumString(); + else + return _sequenceString; + } + + public String toString() + { + return _name + " (" + VisitImpl.formatSequenceNum(_sequenceNum) + ")"; + } + } + + + public Map importVisitTags(Study study, User user, List visitTags) throws ValidationException + { + // Import, don't overwrite existing + final Map allVisitTagMap = new HashMap<>(); + final Map newVisitTagMap = new HashMap<>(); + for (VisitTag visitTag : visitTags) + { + newVisitTagMap.put(visitTag.getName(), visitTag); + } + + Container container = getStudyForVisitTag(study).getContainer(); + SimpleFilter containerFilter = SimpleFilter.createContainerFilter(container); + TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTag(); + if (null == tinfo) + throw new IllegalStateException("Study Import/Export expected TableInfo."); + + TableSelector selector = new TableSelector(tinfo, containerFilter, null); + selector.forEach(VisitTag.class, visitTag -> { + allVisitTagMap.put(visitTag.getName(), visitTag); + newVisitTagMap.remove(visitTag.getName()); + }); + + List newVisitTags = new ArrayList<>(newVisitTagMap.values()); + DataIteratorBuilder loader = new BeanDataIterator.Builder<>(VisitTag.class, newVisitTags); + DbScope scope = tinfo.getSchema().getScope(); + + try (Transaction transaction = scope.ensureTransaction()) + { + DataIteratorContext context = new DataIteratorContext(); + context.setInsertOption(QueryUpdateService.InsertOption.IMPORT); + StandardDataIteratorBuilder etl = StandardDataIteratorBuilder.forInsert(tinfo, loader, container, user, context); + DataIteratorBuilder insert = ((UpdateableTableInfo) tinfo).persistRows(etl, context); + Pump p = new Pump(insert, context); + p.run(); + + BatchValidationException errors = context.getErrors(); + if (errors.hasErrors()) + throw errors.getRowErrors().get(0); + + transaction.commit(); + } + allVisitTagMap.putAll(newVisitTagMap); + return allVisitTagMap; + } + + + public Integer createVisitTagMapEntry(User user, Container container, String visitTagName, @NotNull Integer visitId, @Nullable Integer cohortId) + { + TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTagMap(); + Map map = new CaseInsensitiveHashMap<>(); + map.put("visitTag", visitTagName); + map.put("visitId", visitId); + map.put("cohortId", cohortId); + map.put("containerId", container.getId()); + map = Table.insert(user, tinfo, map); + return asInteger(map.get("RowId")); + } + + @Nullable + public String checkSingleUseVisitTag(VisitTag visitTag, @Nullable Integer cohortId, @NotNull List visitTagMapEntries, + @Nullable Integer oldRowId, Container container, User user) + { + for (VisitTagMapEntry visitTagMapEntry : visitTagMapEntries) + if ((null == oldRowId || !oldRowId.equals(visitTagMapEntry.getRowId())) && + ((null == cohortId && null == visitTagMapEntry.getCohortId()) || null != cohortId && cohortId.equals(visitTagMapEntry.getCohortId()))) + { + Cohort cohort = null != cohortId ? getCohortForRowId(container, user, cohortId) : null; + return "Single use visit tag '" + visitTag.getCaption() + + "' may not be used for more than one visit for the same cohort '" + (null != cohort ? cohort.getLabel() : "") + "'."; + } + return null; + } + + public Map getVisitTags(Study study) + { + // TODO: Use QueryHelper? + final Map visitTags = new HashMap<>(); + SimpleFilter containerFilter = SimpleFilter.createContainerFilter(study.getContainer()); + TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTag(); + new TableSelector(tinfo, containerFilter, null).forEach(VisitTag.class, visitTag -> visitTags.put(visitTag.getName(), visitTag)); + return visitTags; + } + + public + @Nullable + VisitTag getVisitTag(Study study, String visitTagName) + { + final List visitTags = new ArrayList<>(); + SimpleFilter filter = SimpleFilter.createContainerFilter(study.getContainer()); + filter.addCondition(FieldKey.fromString("Name"), visitTagName); + TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTag(); + new TableSelector(tinfo, filter, null).forEach(VisitTag.class, visitTags::add); + + if (visitTags.isEmpty()) + return null; + if (visitTags.size() > 1) + throw new IllegalStateException("Expected only one visit tag with given name."); + return visitTags.get(0); + } + + public Map> getVisitTagMapMap(Study study) + { + final Map> visitTagMapMap = new IntHashMap<>(); + SimpleFilter containerFilter = SimpleFilter.createContainerFilter(study.getContainer()); + TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTagMap(); + new TableSelector(tinfo, containerFilter, null).forEach(VisitTagMapEntry.class, visitTagMapEntry -> { + if (!visitTagMapMap.containsKey(visitTagMapEntry.getVisitId())) + visitTagMapMap.put(visitTagMapEntry.getVisitId(), new ArrayList<>()); + visitTagMapMap.get(visitTagMapEntry.getVisitId()).add(visitTagMapEntry); + }); + + return visitTagMapMap; + } + + public Map> getVisitTagToVisitTagMapEntries(Study study) + { + final Map> visitTagToVisitTagMapEntries = new HashMap<>(); + SimpleFilter containerFilter = SimpleFilter.createContainerFilter(study.getContainer()); + TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTagMap(); + new TableSelector(tinfo, containerFilter, null).forEach(VisitTagMapEntry.class, visitTagMapEntry -> { + if (!visitTagToVisitTagMapEntries.containsKey(visitTagMapEntry.getVisitTag())) + visitTagToVisitTagMapEntries.put(visitTagMapEntry.getVisitTag(), new ArrayList<>()); + visitTagToVisitTagMapEntries.get(visitTagMapEntry.getVisitTag()).add(visitTagMapEntry); + }); + + return visitTagToVisitTagMapEntries; + } + + public List getVisitTagMapEntries(Study study, String visitTagName) + { + final List visitTagMapEntries = new ArrayList<>(); + SimpleFilter filter = SimpleFilter.createContainerFilter(study.getContainer()); + filter.addCondition(FieldKey.fromString("VisitTag"), visitTagName); + TableInfo tinfo = StudySchema.getInstance().getTableInfoVisitTagMap(); + new TableSelector(tinfo, filter, null).forEach(VisitTagMapEntry.class, visitTagMapEntries::add); + + return visitTagMapEntries; + } + + public static String makeVisitTagMapKey(String visitTagName, int visitId, @Nullable Integer cohortId) + { + return visitTagName + "/" + visitId + "/" + cohortId; + } + + public void createCohort(Study study, User user, CohortImpl cohort) + { + if (cohort.getContainer() != null && !cohort.getContainer().equals(study.getContainer())) + throw new IllegalArgumentException("Cohort container does not match study"); + cohort.setContainer(study.getContainer()); + + // Lsid requires the row id, which does not get created until this object has been inserted into the db + if (cohort.getLsid() != null) + throw new IllegalStateException("Attempt to create a new cohort with lsid already set"); + cohort.setLsid(LSID_REQUIRED); + cohort = _cohortHelper.create(user, cohort); + + if (cohort.getRowId() == 0) + throw new IllegalStateException("Cohort rowId has not been set properly"); + + cohort.initLsid(); + _cohortHelper.update(user, cohort); + } + + public void deleteVisit(StudyImpl study, VisitImpl visit, User user) + { + deleteVisits(study, Collections.singleton(visit), user, false); + } + + /* + Delete multiple visits; more efficient than calling deleteVisit() in a loop. + */ + public void deleteVisits(StudyImpl study, Collection visits, User user, boolean unused) + { + // Short circuit on empty + if (visits.isEmpty()) + return; + + // Extract visit rowIds + Collection visitIds = CollectionUtils.collect(visits, VisitImpl::getRowId); + + StudySchema schema = StudySchema.getInstance(); + SQLFragment visitInClause = new SQLFragment(); + schema.getSqlDialect().appendInClauseSql(visitInClause, visitIds); + + try (Transaction transaction = schema.getSchema().getScope().ensureTransaction()) + { + if (!unused) + { + for (DatasetDefinition def : study.getDatasets()) + { + TableInfo t = def.getStorageTableInfo(false); + if (null == t) + continue; + + SQLFragment sqlf = new SQLFragment(); + sqlf.append("DELETE FROM "); + sqlf.append(t); + if (schema.getSqlDialect().isSqlServer()) + sqlf.append(" WITH (UPDLOCK)"); + sqlf.append(" WHERE LSID IN (SELECT LSID FROM "); + sqlf.append(t); + sqlf.append(" d, "); + sqlf.append(StudySchema.getInstance().getTableInfoParticipantVisit(), "pv"); + sqlf.append(" WHERE d.ParticipantId = pv.ParticipantId AND d.SequenceNum = pv.SequenceNum AND pv.Container = ?"); + sqlf.add(study.getContainer()); + sqlf.append(" AND pv.VisitRowId ").append(visitInClause).append(')'); + + int count = new SqlExecutor(schema.getSchema()).execute(sqlf); + if (count > 0) + StudyManager.datasetModified(def, true); + } + + for (VisitImpl visit : visits) + { + // Delete specimens first because we may need ParticipantVisit to figure out which specimens + SpecimenManager.get().deleteSpecimensForVisit(visit); + StudyDesignService svc = StudyDesignService.get(); + if (svc != null) + { + svc.deleteTreatmentVisitMapForVisit(study.getContainer(), visit.getRowId()); + svc.deleteAssaySpecimenVisits(study.getContainer(), visit.getRowId()); + } + } + } + + SQLFragment sqlFragParticipantVisit = new SQLFragment("DELETE FROM " + schema.getTableInfoParticipantVisit() + "\n" + + "WHERE Container = ?").add(study.getContainer()); + sqlFragParticipantVisit.append(" AND VisitRowId ").append(visitInClause); + new SqlExecutor(schema.getSchema()).execute(sqlFragParticipantVisit); + + SQLFragment sqlFragVisitMap = new SQLFragment("DELETE FROM " + schema.getTableInfoVisitMap() + "\n" + + "WHERE Container = ?").add(study.getContainer()); + sqlFragVisitMap.append(" AND VisitRowId ").append(visitInClause); + new SqlExecutor(schema.getSchema()).execute(sqlFragVisitMap); + + // UNDONE broken _visitHelper.delete(visit); + try + { + Study visitStudy = getStudyForVisits(study); + Container c = visitStudy.getContainer(); + + try + { + for (VisitImpl visit : visits) + { + Table.delete(schema.getTableInfoVisit(), new Object[]{c, visit.getRowId()}); + } + } + finally + { + _visitHelper.clearCache(c); + } + } + catch (OptimisticConflictException x) + { + /* ignore */ + } + + transaction.commit(); + + getVisitManager(study).updateParticipantVisits(user, study.getDatasets()); + } + } + + public void updateVisit(User user, VisitImpl visit) + { + _visitHelper.update(user, visit, visit.getContainer().getId(), visit.getRowId()); + } + + public void updateCohort(User user, CohortImpl cohort) + { + _cohortHelper.update(user, cohort); + } + + public void updateParticipant(User user, Participant participant) + { + Container c = participant.getContainer(); + Table.update(user, SCHEMA.getTableInfoParticipant(), participant, new Object[]{c.getId(), participant.getParticipantId()}); + _participantCache.remove(c); + } + + public void createVisitDatasetMapping(User user, Container container, int visitId, int datasetId, boolean isRequired) + { + VisitDataset vds = new VisitDataset(container, datasetId, visitId, isRequired); + Table.insert(user, SCHEMA.getTableInfoVisitMap(), vds); + } + + public VisitDataset getVisitDatasetMapping(Container container, int visitRowId, int datasetId) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("VisitRowId"), visitRowId); + filter.addCondition(FieldKey.fromParts("DataSetId"), datasetId); + + Boolean required = new TableSelector(SCHEMA.getTableInfoVisitMap().getColumn("Required"), filter, null).getObject(Boolean.class); + + return (null != required ? new VisitDataset(container, datasetId, visitRowId, required) : null); + } + + public Collection getVisits(Study study, Order order) + { + return getVisits(study, null, null, order); + } + + public Collection getVisits(Study study, @Nullable Cohort cohort, @Nullable User user, Order order) + { + if (study.getTimepointType() == TimepointType.CONTINUOUS) + return Collections.emptyList(); + + Study visitStudy = getStudyForVisits(study); + Collection visits = _visitHelper.getCollection(visitStudy.getContainer(), order); + if (cohort != null && showCohorts(study.getContainer(), user)) + { + // We could cache all combinations of cohort x order instead of filtering on-the-fly, but this seems fast enough + visits = visits.stream() + .filter(visit -> visit.getCohortId() == null || visit.getCohortId() == cohort.getRowId()) + .toList(); + } + + return visits; + } + + public void clearParticipantVisitCaches(Study study) + { + _visitHelper.clearCache(study.getContainer()); + + // clear shared study + Study visitStudy = getStudyForVisits(study); + if (!study.equals(visitStudy)) + _visitHelper.clearCache(visitStudy.getContainer()); + + _participantCache.remove(study.getContainer()); + } + + public VisitImpl getVisitForRowId(Study study, int rowId) + { + Study visitStudy = getStudyForVisits(study); + + return _visitHelper.get(visitStudy.getContainer(), rowId); + } + + /** + * Helper to insert a new QCState and manage some study specific behavior + */ + public DataState insertQCState(User user, DataState state) + { + boolean isFirst = QCStateManager.getInstance().getStates(state.getContainer()).isEmpty(); + DataState newState = QCStateManager.getInstance().insertState(user, state); + if (isFirst) + // switching from zero to more than zero QC states affects the columns in our materialized datasets + // (adding a QC State column), so we unmaterialize them here: + StudyManager.getInstance().clearCaches(state.getContainer(), true); + + return newState; + } + + @Nullable + public DataState getDefaultQCState(StudyImpl study) + { + Long defaultQcStateId = study.getDefaultDirectEntryQCState(); + DataState defaultQCState = null; + if (defaultQcStateId != null) + defaultQCState = QCStateManager.getInstance().getStateForRowId( + study.getContainer(), defaultQcStateId); + return defaultQCState; + } + + private Map getVisitsForDataRows(DatasetDefinition def, Collection dataLsids) + { + final Map visits = new HashMap<>(); + + if (dataLsids == null || dataLsids.isEmpty()) + return visits; + + final Study study = def.getStudy(); + final Study visitStudy = getStudyForVisits(study); + + TableInfo ds = def.getDatasetSchemaTableInfo(null, false); + + SQLFragment sql = new SQLFragment(); + sql.append("SELECT sd.LSID AS LSID, v.RowId AS RowId FROM ").append(ds.getFromSQL("sd")).append("\n" + + "JOIN study.ParticipantVisit pv ON \n" + + "\tsd.SequenceNum = pv.SequenceNum AND\n" + + "\tsd.ParticipantId = pv.ParticipantId\n" + + "JOIN study.Visit v ON\n" + + "\tpv.VisitRowId = v.RowId AND\n" + + "\tpv.Container = ? AND v.Container = ?\n" + + "WHERE sd.lsid "); + sql.add(def.getContainer().getId()); + // shared visit container + sql.add(visitStudy.getContainer().getId()); + + StudySchema.getInstance().getSqlDialect().appendInClauseSql(sql, dataLsids); + + new SqlSelector(StudySchema.getInstance().getSchema(), sql).forEach(rs -> { + String lsid = rs.getString("LSID"); + int visitId = rs.getInt("RowId"); + visits.put(lsid, getVisitForRowId(study, visitId)); + }); + + return visits; + } + + public List getVisitsForDataset(Container container, int datasetId) + { + List visits = new ArrayList<>(); + + DatasetDefinition def = getDatasetDefinition(getStudy(container), datasetId); + TableInfo ds = def.getDatasetSchemaTableInfo(null, false); + + final Study study = def.getStudy(); + final Study visitStudy = getStudyForVisits(study); + + SQLFragment sql = new SQLFragment(); + sql.append("SELECT DISTINCT v.RowId AS RowId FROM ").append(ds.getFromSQL("sd")).append("\n" + + "JOIN study.ParticipantVisit pv ON \n" + + "\tsd.SequenceNum = pv.SequenceNum AND\n" + + "\tsd.ParticipantId = pv.ParticipantId\n" + + "JOIN study.Visit v ON\n" + + "\tpv.VisitRowId = v.RowId AND\n" + + "\tpv.Container = ? AND v.Container = ?\n"); + sql.add(container.getId()); + // shared visit container + sql.add(visitStudy.getContainer().getId()); + + SqlSelector selector = new SqlSelector(StudySchema.getInstance().getSchema(), sql); + for (Integer rowId : selector.getArray(Integer.class)) + { + visits.add(getVisitForRowId(study, rowId)); + } + return visits; + } + + public void updateDataQCState(Container container, User user, int datasetId, Collection lsids, DataState newState, String comments) + { + DbScope scope = StudySchema.getInstance().getSchema().getScope(); + Study study = getStudy(container); + DatasetDefinition def = getDatasetDefinition(study, datasetId); + + Map lsidVisits = null; + if (!def.isDemographicData()) + lsidVisits = getVisitsForDataRows(def, lsids); + List> rows = def.getDatasetRows(user, lsids); + if (rows.isEmpty()) + return; + + Map oldQCStates = new HashMap<>(); + Map newQCStates = new HashMap<>(); + + Set updateLsids = new HashSet<>(); + for (Map row : rows) + { + String lsid = (String) row.get("lsid"); + + Long oldStateId = MapUtils.getLong(row,DatasetTableImpl.QCSTATE_ID_COLNAME); + DataState oldState = null; + if (oldStateId != null) + oldState = QCStateManager.getInstance().getStateForRowId(container, oldStateId); + + // check to see if we're actually changing state. If not, no-op: + if (safeIntegersEqual(newState != null ? newState.getRowId() : null, oldStateId)) + continue; + + updateLsids.add(lsid); + + StringBuilder auditKey = new StringBuilder(StudyService.get().getSubjectNounSingular(container) + " "); + auditKey.append(row.get(StudyService.get().getSubjectColumnName(container))); + if (!def.isDemographicData()) + { + VisitImpl visit = lsidVisits.get(lsid); + auditKey.append(", Visit ").append(visit != null ? visit.getLabel() : "unknown"); + } + String keyProp = def.getKeyPropertyName(); + if (keyProp != null) + { + auditKey.append(", ").append(keyProp).append(" ").append(row.get(keyProp)); + } + + oldQCStates.put(auditKey.toString(), oldState != null ? oldState.getLabel() : "unspecified"); + newQCStates.put(auditKey.toString(), newState != null ? newState.getLabel() : "unspecified"); + } + + if (updateLsids.isEmpty()) + return; + + try (Transaction transaction = scope.ensureTransaction()) + { + // TODO fix updating across study data + SQLFragment sql = new SQLFragment("UPDATE " ).append(def.getStorageTableInfo(false)); + sql.append(" SET QCState = "); + // do string concatenation, rather that using a parameter, for the new state id because Postgres null + // parameters are typed which causes a cast exception trying to set the value back to null (bug 6370) + sql.appendValue(newState != null ? newState.getRowId() : null); + sql.append(", modified = ?"); + sql.add(new Date()); + sql.append("\nWHERE lsid "); + StudySchema.getInstance().getSqlDialect().appendInClauseSql(sql, updateLsids); + + new SqlExecutor(StudySchema.getInstance().getSchema()).execute(sql); + + //def.deleteFromMaterialized(user, updateLsids); + //def.insertIntoMaterialized(user, updateLsids); + + String auditComment = "QC state was changed for " + updateLsids.size() + " record" + + (updateLsids.size() == 1 ? "" : "s") + ". User comment: " + comments; + + DatasetAuditProvider.DatasetAuditEvent event = new DatasetAuditProvider.DatasetAuditEvent(container, auditComment, datasetId); + event.setHasDetails(true); + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldQCStates)); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newQCStates)); + + AuditLogService.get().addEvent(user, event); + clearCaches(container, false); + + transaction.commit(); + } + } + + public static boolean safeIntegersEqual(Integer first, Integer second) + { + if (first == null && second == null) + return true; + if (first == null) + return false; + return first.equals(second); + } + + public static boolean safeIntegersEqual(Long first, Long second) + { + if (first == null && second == null) + return true; + if (first == null) + return false; + return first.equals(second); + } + + public boolean showCohorts(Container container, @Nullable User user) + { + if (user == null) + return false; + + if (user.hasRootAdminPermission()) + return true; + + StudyImpl study = StudyManager.getInstance().getStudy(container); + + if (study == null) + return false; + + Integer cohortDatasetId = study.getParticipantCohortDatasetId(); + if (study.isManualCohortAssignment() || null == cohortDatasetId || -1 == cohortDatasetId) + { + // If we're not reading from a dataset for cohort definition, + // we use the container's permission + return container.hasPermission(user, ReadPermission.class); + } + + // Automatic cohort assignment -- can the user read the source dataset? + DatasetDefinition def = getDatasetDefinition(study, cohortDatasetId); + + if (def != null) + return def.canReadInternal(user); + + return false; + } + + public void assertCohortsViewable(Container container, User user) + { + if (!showCohorts(container, user)) + throw new UnauthorizedException("User does not have permission to view cohort information"); + } + + public Collection getCohorts(Container container, User user) + { + assertCohortsViewable(container, user); + return _cohortHelper.getCollection(container); + } + + public CohortImpl getCurrentCohortForParticipant(Container container, User user, String participantId) + { + assertCohortsViewable(container, user); + Participant participant = getParticipant(getStudy(container), participantId); + if (participant != null && participant.getCurrentCohortId() != null) + return _cohortHelper.get(container, participant.getCurrentCohortId().intValue()); + return null; + } + + public CohortImpl getCohortForRowId(Container container, User user, int rowId) + { + assertCohortsViewable(container, user); + return _cohortHelper.get(container, rowId); + } + + public CohortImpl getCohortByLabel(Container container, User user, String label) + { + assertCohortsViewable(container, user); + + List cohorts = _cohortHelper.getCollection(container).stream() + .filter(cohort -> cohort.getLabel().equals(label)) + .toList(); + + if (cohorts.size() == 1) + return cohorts.get(0); + + return null; + } + + private boolean isCohortInUse(CohortImpl cohort, Container c, TableInfo table, String... columnNames) + { + List params = new ArrayList<>(); + params.add(c.getId()); + + StringBuilder cols = new StringBuilder("("); + String or = ""; + for (String columnName : columnNames) + { + cols.append(or).append(columnName).append(" = ?"); + params.add(cohort.getRowId()); + or = " OR "; + } + cols.append(")"); + + return new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT * FROM " + + table + " WHERE Container = ? AND " + cols, params).exists(); + } + + public boolean isCohortInUse(CohortImpl cohort) + { + Container c = cohort.getContainer(); + Study visitStudy = getStudyForVisits(getStudy(c)); + + return isCohortInUse(cohort, c, StudySchema.getInstance().getTableInfoDataset(), "CohortId") || + isCohortInUse(cohort, c, StudySchema.getInstance().getTableInfoParticipant(), "CurrentCohortId", "InitialCohortId") || + isCohortInUse(cohort, c, StudySchema.getInstance().getTableInfoParticipantVisit(), "CohortId") || + isCohortInUse(cohort, visitStudy.getContainer(), StudySchema.getInstance().getTableInfoVisit(), "CohortId"); + } + + public void deleteCohort(CohortImpl cohort) + { + StudySchema schema = StudySchema.getInstance(); + + try (Transaction transaction = schema.getSchema().getScope().ensureTransaction()) + { + Container container = cohort.getContainer(); + StudyDesignService svc = StudyDesignService.get(); + if (svc != null) + svc.deleteTreatmentVisitMapForCohort(container, cohort.getRowId()); + _cohortHelper.delete(cohort); + + // delete extended properties + String lsid = cohort.getLsid(); + Map resourceProperties = OntologyManager.getPropertyObjects(container, lsid); + if (resourceProperties != null && !resourceProperties.isEmpty()) + { + OntologyManager.deleteOntologyObject(lsid, container, false); + } + + transaction.commit(); + } + } + + public VisitImpl getVisitForSequence(Study study, BigDecimal seqNum) + { + Collection visits = getVisits(study, Order.SEQUENCE_NUM); + for (VisitImpl v : visits) + { + if (v.isInRange(seqNum)) + return v; + } + return null; + } + + public List getDatasetDefinitions(Study study) + { + return getDatasetDefinitions(study, null); + } + + public List getDatasetDefinitions(Study study, @Nullable Cohort cohort, String... types) + { + List local = getDatasetDefinitionsLocal(study, cohort, types); + List shared = Collections.emptyList(); + List combined; + + Study sharedStudy = getSharedStudy(study); + if (null != sharedStudy) + shared = getDatasetDefinitionsLocal(sharedStudy, cohort, types); + + if (shared.isEmpty()) + combined = local; + else + { + // NOTE: it's confusing that both ID and name are unique, manage page should warn about funny inconsistencies + // NOTE: here we'll have LOCAL datasets hide SHARED datasets by both id and name until I have a better idea + CaseInsensitiveHashSet names = new CaseInsensitiveHashSet(); + HashSet ids = new HashSet<>(); + + combined = new ArrayList<>(local.size() + shared.size()); + for (DatasetDefinition dsd : local) + { + combined.add(dsd); + names.add(dsd.getName()); + ids.add(dsd.getDatasetId()); + } + for (DatasetDefinition dsd : shared) + { + if (!names.contains(dsd.getName()) && !ids.contains(dsd.getDatasetId())) + { + DatasetDefinition wrapped = dsd.createLocalDatasetDefinition((StudyImpl) study); + combined.add(wrapped); + } + } + } + + // sort by display order, category, and dataset ID + combined.sort(DATASET_ORDER_COMPARATOR); + + return Collections.unmodifiableList(combined); + } + + public static final Comparator DATASET_ORDER_COMPARATOR = new Comparator<>() + { + @Override + public int compare(DatasetDefinition o1, DatasetDefinition o2) + { + if (o1.getDisplayOrder() != 0 || o2.getDisplayOrder() != 0) + return o1.getDisplayOrder() - o2.getDisplayOrder(); + + if (Strings.CS.equals(o1.getCategory(), o2.getCategory())) + return o1.getDatasetId() - o2.getDatasetId(); + + if (o1.getCategory() != null && o2.getCategory() == null) + return -1; + if (o1.getCategory() == null && o2.getCategory() != null) + return 1; + if (o1.getCategory() != null && o2.getCategory() != null) + return o1.getCategory().compareTo(o2.getCategory()); + + return o1.getDatasetId() - o2.getDatasetId(); + } + }; + + /** + * Get the list of datasets that are 'shadowed' by the list of local dataset definitions or for any local dataset in the study. + * This is pretty much the inverse of getDatasetDefinitions() + * This can be used in the management/admin UI to warn about shadowed datasets + */ + public List getShadowedDatasets(@NotNull Study study, @Nullable List local) + { + if (study.getContainer().isProject()) + return Collections.emptyList(); + + Study sharedStudy = getSharedStudy(study); + if (null == sharedStudy) + return Collections.emptyList(); + + if (null == local) + local = getDatasetDefinitionsLocal(study); + List shared = getDatasetDefinitionsLocal(sharedStudy); + + if (local.isEmpty() || shared.isEmpty()) + return Collections.emptyList(); + + CaseInsensitiveHashSet names = new CaseInsensitiveHashSet(); + HashSet ids = new HashSet<>(); + + for (DatasetDefinition dsd : local) + { + if (dsd.getDefinitionContainer().equals(dsd.getContainer())) + { + names.add(dsd.getName()); + ids.add(dsd.getDatasetId()); + } + } + Map shadowed = new TreeMap<>(); + for (DatasetDefinition dsd : shared) + { + if (names.contains(dsd.getName()) || ids.contains(dsd.getDatasetId())) + shadowed.put(dsd.getDatasetId(), dsd); + } + + return new ArrayList<>(shadowed.values()); + } + + public List getDatasetDefinitionsLocal(Study study) + { + return getDatasetDefinitionsLocal(study, null); + } + + public List getDatasetDefinitionsLocal(Study study, @Nullable Cohort cohort, String... types) + { + Collection ret = cohort != null ? _datasetHelper.getDatasetsForCohort(study, cohort) : _datasetHelper.getCollection(study.getContainer()); + + if (types != null && types.length > 0) + { + Set typeSet = Set.of(types); + ret = ret.stream().filter(def -> typeSet.contains(def.getType())).toList(); + } + + // Make a copy (it's immutable) so that we can sort it. See issue 17875 + return new ArrayList<>(ret); + } + + public Set getSharedProperties(Study study) + { + return _sharedProperties.get(study.getContainer()); + } + + @Nullable + public DatasetDefinition getDatasetDefinition(Study s, int id) + { + DatasetDefinition ds = _datasetHelper.get(s.getContainer(), id); + if (null != ds) + return ds; + + Study sharedStudy = getSharedStudy(s); + if (null == sharedStudy) + return null; + + ds = getDatasetDefinition(sharedStudy, id); + if (null == ds) + return null; + return ds.createLocalDatasetDefinition((StudyImpl) s); + } + + + @Nullable + public DatasetDefinition getDatasetDefinitionByLabel(Study s, String label) + { + if (label == null) + { + return null; + } + + return _datasetHelper.getByLabel(s, label); + } + + + @Nullable + public DatasetDefinition getDatasetDefinitionByEntityId(Study s, String entityId) + { + return _datasetHelper.getByEntityId(s, entityId); + } + + + @Nullable + public DatasetDefinition getDatasetDefinitionByName(Study s, String name) + { + DatasetDefinition def = _datasetHelper.getByName(s, name); + if (def != null) + return def; + + Study sharedStudy = getSharedStudy(s); + if (null == sharedStudy) + return null; + + def = getDatasetDefinitionByName(sharedStudy, name); + if (null == def) + return null; + return def.createLocalDatasetDefinition((StudyImpl) s); + } + + + @Nullable + public DatasetDefinition getDatasetDefinitionByQueryName(Study study, String queryName) + { + // first try resolving the dataset def by name and then by label + DatasetDefinition def = getDatasetDefinitionByName(study, queryName); + if (null != def) + return def; + def = StudyManager.getInstance().getDatasetDefinitionByLabel(study, queryName); + if (null != def) + return def; + + // try shared study + if (study.getContainer().isProject()) + return null; + Study shared = StudyManager.getInstance().getSharedStudy(study); + if (null == shared) + return null; + + // first try resolving the dataset def by name and then by label + def = StudyManager.getInstance().getDatasetDefinitionByName(shared, queryName); + if (null != def) + return def.createLocalDatasetDefinition((StudyImpl) study); + def = StudyManager.getInstance().getDatasetDefinitionByLabel(shared, queryName); + if (null != def) + return def.createLocalDatasetDefinition((StudyImpl) study); + + return null; + } + + + // domainURI -> + private static final Cache> domainCache = CacheManager.getCache(5000, CacheManager.DAY, "Domain->Dataset map"); + + private static final CacheLoader> loader = (domainURI, argument) -> { + SQLFragment sql = new SQLFragment(); + sql.append("SELECT Container, DatasetId FROM study.Dataset WHERE TypeURI=?"); + sql.add(domainURI); + + Map map = new SqlSelector(StudySchema.getInstance().getSchema(), sql).getMap(); + + if (null == map) + return null; + else + return new Pair<>((String)map.get("Container"), asInteger(map.get("DatasetId"))); + }; + + + @Nullable + DatasetDefinition getDatasetDefinition(String domainURI) + { + for (int retry=0 ; retry < 2 ; retry++) + { + Pair p = domainCache.get(domainURI, null, loader); + if (null == p) + return null; + + Container c = ContainerManager.getForId(p.first); + if (c != null) + { + Study study = StudyManager.getInstance().getStudy(c); + if (null != study) + { + DatasetDefinition ret = StudyManager.getInstance().getDatasetDefinition(study, p.second); + if (null != ret && null != ret.getDomain() && Strings.CI.equals(ret.getDomain().getTypeURI(), domainURI)) + return ret; + } + } + domainCache.remove(domainURI); + } + return null; + } + + + public List getDatasetLSIDs(User user, DatasetDefinition def) + { + TableInfo tInfo = def.getTableInfo(user); + return new TableSelector(tInfo.getColumn("lsid")).getArrayList(String.class); + } + + + public void uncache(DatasetDefinition def) + { + if (null == def) + return; + + _log.debug("Uncaching dataset: " + def.getName(), new Throwable()); + + _datasetHelper.clearCache(def.getContainer()); + String uri = def.getTypeURI(); + if (null != uri) + domainCache.remove(uri); + + // Also clear caches of subjects and visits; changes to this dataset may have affected this data: + clearParticipantVisitCaches(def.getStudy()); + } + + public Map getRequiredMap(Study study) + { + TableInfo tableVisitMap = StudySchema.getInstance().getTableInfoVisitMap(); + final HashMap map = new HashMap<>(); + + new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT DatasetId, VisitRowId, Required FROM " + tableVisitMap + " WHERE Container = ?", + study.getContainer()).forEach(rs -> map.put(new VisitMapKey(rs.getInt(1), rs.getInt(2)), rs.getBoolean(3))); + + return map; + } + + private static final String VISITMAP_JOIN_BY_VISIT = """ + SELECT d.*, vm.Required FROM study.Visit v, study.DataSet d, study.VisitMap vm + WHERE v.RowId = vm.VisitRowId AND vm.DataSetId = d.DataSetId AND v.Container = vm.Container AND + vm.Container = d.Container AND v.Container = ? AND v.RowId = ? + ORDER BY d.DisplayOrder, d.DataSetId"""; + + private static final String VISITMAP_JOIN_BY_DATASET = """ + SELECT vm.VisitRowId, vm.Required + FROM study.VisitMap vm JOIN study.Visit v ON vm.VisitRowId = v.RowId + WHERE vm.Container = ? AND vm.DataSetId = ? + ORDER BY v.DisplayOrder, v.RowId"""; + + List getMapping(final VisitImpl visit) + { + if (visit.getContainer() == null) + throw new IllegalStateException("Visit has no container"); + + final List visitDatasets = new ArrayList<>(); + + new SqlSelector(StudySchema.getInstance().getSchema(), VISITMAP_JOIN_BY_VISIT, + visit.getContainer(), visit.getRowId()).forEach(rs -> { + int datasetId = rs.getInt("DataSetId"); + boolean isRequired = rs.getBoolean("Required"); + visitDatasets.add(new VisitDataset(visit.getContainer(), datasetId, visit.getRowId(), isRequired)); + }); + + return visitDatasets; + } + + + public List getMapping(final Dataset dataset) + { + final List visitDatasets = new ArrayList<>(); + + new SqlSelector(StudySchema.getInstance().getSchema(), VISITMAP_JOIN_BY_DATASET, + dataset.getContainer(), dataset.getDatasetId()).forEach(rs -> { + int visitRowId = rs.getInt("VisitRowId"); + boolean isRequired = rs.getBoolean("Required"); + visitDatasets.add(new VisitDataset(dataset.getContainer(), dataset.getDatasetId(), visitRowId, isRequired)); + }); + + return visitDatasets; + } + + + public void updateVisitDatasetMapping(User user, Container container, int visitId, + int datasetId, VisitDatasetType type) + { + VisitDataset vds = getVisitDatasetMapping(container, visitId, datasetId); + if (vds == null) + { + if (type != VisitDatasetType.NOT_ASSOCIATED) + { + // need to insert a new VisitMap entry: + createVisitDatasetMapping(user, container, visitId, + datasetId, type == VisitDatasetType.REQUIRED); + } + } + else if (type == VisitDatasetType.NOT_ASSOCIATED) + { + // need to remove an existing VisitMap entry: + Table.delete(SCHEMA.getTableInfoVisitMap(), + new Object[] { container.getId(), visitId, datasetId}); + } + else if ((VisitDatasetType.OPTIONAL == type && vds.isRequired()) || + (VisitDatasetType.REQUIRED == type && !vds.isRequired())) + { + Map required = new HashMap<>(1); + required.put("Required", VisitDatasetType.REQUIRED == type ? Boolean.TRUE : Boolean.FALSE); + Table.update(user, SCHEMA.getTableInfoVisitMap(), required, + new Object[]{container.getId(), visitId, datasetId}); + } + } + + public long getNumDatasetRows(User user, Dataset dataset) + { + TableInfo sdTable = dataset.getTableInfo(user); + return new TableSelector(sdTable).getRowCount(); + } + + + /** + * Delete all rows from a dataset or just those newer than the cutoff date. + */ + public int purgeDataset(DatasetDefinition dataset, @Nullable Date cutoff) + { + return dataset.deleteRows(cutoff); + } + + /** + * delete a dataset definition along with associated type, data, visitmap entries + * @param performStudyResync whether to kick off our normal bookkeeping. If the whole study is being deleted, + * we don't need to bother doing this, for example. + */ + public void deleteDataset(StudyImpl study, User user, DatasetDefinition ds, boolean performStudyResync, @Nullable String auditUserComment) + { + try (Transaction transaction = StudySchema.getInstance().getScope().ensureTransaction()) + { + if (!ds.canDeleteDefinition(user)) + throw new UnauthorizedException("Can't delete dataset: " + ds.getName()); + + // When the dataset is deleted, the provenance rows should be cleaned up + ProvenanceService pvs = ProvenanceService.get(); + + Collection allDatasetLsids = pvs.getDatasetProvenanceLsids(user, ds); + + allDatasetLsids.forEach(lsid -> { + Set protocolApplications = pvs.getProtocolApplications(lsid); + + OntologyObject expObject = OntologyManager.getOntologyObject(null, lsid); + if (null != expObject) + { + pvs.deleteObjectProvenance(expObject.getObjectId()); + } + + if (!protocolApplications.isEmpty()) + { + ExperimentService expService = ExperimentService.get(); + protocolApplications.forEach(protocolApp -> { + ExpRun run = expService.getExpProtocolApplication(protocolApp).getRun(); + expService.deleteExperimentRunsByRowIds(study.getContainer(), user, run.getRowId()); + }); + } + }); + + try + { + deleteDatasetType(study, user, ds, auditUserComment); + + QuerySnapshotDefinition def = QueryService.get().getSnapshotDef(study.getContainer(), StudySchema.getInstance().getSchemaName(), ds.getName()); + if (def != null) + def.delete(user); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + new SqlExecutor(StudySchema.getInstance().getSchema()).execute("DELETE FROM " + SCHEMA.getTableInfoVisitMap() + "\n" + + "WHERE Container=? AND DatasetId=?", study.getContainer(), ds.getDatasetId()); + + // UNDONE: This is broken + // _datasetHelper.delete(ds); + new SqlExecutor(StudySchema.getInstance().getSchema()).execute("DELETE FROM " + StudySchema.getInstance().getTableInfoDataset() + "\n" + + "WHERE Container=? AND DatasetId=?", study.getContainer(), ds.getDatasetId()); + _datasetHelper.clearCache(study.getContainer()); + + SecurityPolicyManager.deletePolicy(ds); + + if (safeIntegersEqual(ds.getDatasetId(), study.getParticipantCohortDatasetId())) + CohortManager.getInstance().setManualCohortAssignment(study, user, Collections.emptyMap()); + + if (performStudyResync) + { + // This dataset may have contained the only references to some subjects or visits; as a result, we need + // to re-sync the participant and participant/visit tables. (Issue 12447) + // Don't provide the deleted dataset in the list of modified datasets; deletion doesn't count as a modification + // within VisitManager, and passing in the empty set ensures that all subject/visit info will be recalculated. + getVisitManager(study).updateParticipantVisits(user, Collections.emptySet()); + } + + SchemaKey schemaPath = SchemaKey.fromParts(SCHEMA.getSchemaName()); + QueryService.get().fireQueryDeleted(user, study.getContainer(), null, schemaPath, Collections.singleton(ds.getName())); + new DatasetDefinition.DatasetAuditHandler(ds).addAuditEvent(user, study.getContainer(), AuditBehaviorType.DETAILED, "Dataset deleted: " + ds.getName(), null); + + transaction.addCommitTask(() -> + unindexDataset(ds), + CommitTaskOption.POSTCOMMIT + ); + + transaction.commit(); + } + } + + /** delete a dataset type and data + * does not clear typeURI as we're about to delete the dataset + */ + private void deleteDatasetType(Study study, User user, DatasetDefinition ds, @Nullable String auditUserComment) + { + assert StudySchema.getInstance().getSchema().getScope().isTransactionActive(); + + if (null == ds) + return; + + if (!ds.canDeleteDefinition(user)) + throw new IllegalStateException("Can't delete dataset: " + ds.getName()); + + Domain domain = ds.getDomain(); + if (domain == null) + return; + + DomainAuditProvider.DomainAuditEvent event = new DomainAuditProvider.DomainAuditEvent(study.getContainer(), String.format("The domain %s was deleted", domain.getName())); + event.setUserComment(auditUserComment); + event.setDomainUri(domain.getTypeURI()); + event.setDomainName(domain.getName()); + AuditLogService.get().addEvent(user, event); + + StorageProvisioner.get().drop(domain); + + if (ds.getTypeURI() != null) + { + try + { + OntologyManager.deleteType(ds.getTypeURI(), study.getContainer()); + } + catch (DomainNotFoundException x) + { + // continue + } + } + } + + // Any container can be passed here (whether it contains a study or not). + public void clearCaches(Container c, boolean unmaterializeDatasets) + { + Study study = getStudy(c); + _studyHelper.clearCache(c); + _visitHelper.clearCache(c); + LocationCache.clear(c); + AssayService.get().clearProtocolCache(); + if (unmaterializeDatasets && null != study) + for (DatasetDefinition def : getDatasetDefinitions(study)) + uncache(def); + // Aggressive, but datasets are cached with container objects that might go stale, for example, when moving a + // folder tree to another parent, the datasets in subfolders will be left with invalid paths. See FolderTest. + _datasetHelper.clearCache(); + _cohortHelper.clearCache(c); + _participantCache.remove(c); + } + + public void deleteAllStudyData(Container c, User user) + { + // No need to delete individual participants if the whole study is going away + VisitManager.cancelParticipantPurge(c); + + // Before we delete any data, we need to go fetch the Dataset definitions. + StudyImpl study = StudyManager.getInstance().getStudy(c); + List dsds; + if (study == null) // no study in this folder + dsds = Collections.emptyList(); + else + dsds = study.getDatasets(); + + // get the list of study design tables + List studyDesignTables = getStudyDesignTables(c, user); + + DbScope scope = StudySchema.getInstance().getSchema().getScope(); + + Set deletedTables = new HashSet<>(); + SimpleFilter containerFilter = SimpleFilter.createContainerFilter(c); + + try (Transaction transaction = scope.ensureTransaction()) + { + StudyDesignManager.get().deleteStudyDesignData(c, deletedTables); + + for (DatasetDefinition dsd : dsds) + { + if (dsd.getContainer().equals(dsd.getDefinitionContainer())) + deleteDataset(study, user, dsd, false, null); + else + dsd.deleteAllRows(user); + } + + // + // specimens + // + SpecimenService ss = SpecimenService.get(); + if (null != ss) + ss.deleteAllSpecimenData(c, deletedTables, user); + + // Since study creates these tables, study needs to delete them + new SpecimenTablesProvider(c, null, null).deleteTables(); + LocationCache.clear(c); + + // + // metadata + // + Table.delete(SCHEMA.getTableInfoVisitMap(), containerFilter); + assert deletedTables.add(SCHEMA.getTableInfoVisitMap()); + Table.delete(StudySchema.getInstance().getTableInfoUploadLog(), containerFilter); + assert deletedTables.add(StudySchema.getInstance().getTableInfoUploadLog()); + Table.delete(_datasetHelper.getTableInfo(), containerFilter); + _datasetHelper.clearCache(c); + assert deletedTables.add(_datasetHelper.getTableInfo()); + Table.delete(_visitHelper.getTableInfo(), containerFilter); + _visitHelper.clearCache(c); + assert deletedTables.add(_visitHelper.getTableInfo()); + Table.delete(_studyHelper.getTableInfo(), containerFilter); + _studyHelper.clearCache(c); + assert deletedTables.add(_studyHelper.getTableInfo()); + + // participant lists + Table.delete(ParticipantGroupManager.getTableInfoParticipantGroupMap(), containerFilter); + assert deletedTables.add(ParticipantGroupManager.getTableInfoParticipantGroupMap()); + Table.delete(ParticipantGroupManager.getTableInfoParticipantGroup(), containerFilter); + assert deletedTables.add(ParticipantGroupManager.getTableInfoParticipantGroup()); + Table.delete(StudySchema.getInstance().getTableInfoParticipantCategory(), containerFilter); + assert deletedTables.add(StudySchema.getInstance().getTableInfoParticipantCategory()); + ParticipantGroupManager.getInstance().clearCache(c); + + // + // participant data (OntologyManager will take care of properties) + // + // Table.delete(StudySchema.getInstance().getTableInfoStudyData(null), containerFilter); + //assert deletedTables.add(StudySchema.getInstance().getTableInfoStudyData(null)); + Table.delete(StudySchema.getInstance().getTableInfoParticipantVisit(), containerFilter); + assert deletedTables.add(StudySchema.getInstance().getTableInfoParticipantVisit()); + Table.delete(StudySchema.getInstance().getTableInfoVisitAliases(), containerFilter); + assert deletedTables.add(StudySchema.getInstance().getTableInfoVisitAliases()); + Table.delete(SCHEMA.getTableInfoParticipant(), containerFilter); + _participantCache.remove(c); + assert deletedTables.add(SCHEMA.getTableInfoParticipant()); + Table.delete(_cohortHelper.getTableInfo(), containerFilter); + _cohortHelper.clearCache(c); + assert deletedTables.add(StudySchema.getInstance().getTableInfoCohort()); + Table.delete(StudySchema.getInstance().getTableInfoParticipantView(), containerFilter); + assert deletedTables.add(StudySchema.getInstance().getTableInfoParticipantView()); + + // participant group cohort union view + assert deletedTables.add(StudySchema.getInstance().getSchema().getTable(StudyQuerySchema.PARTICIPANT_GROUP_COHORT_UNION_TABLE_NAME)); + + // Specimen comments + Table.delete(SpecimenSchema.get().getTableInfoSpecimenComment(), containerFilter); + assert deletedTables.add(SpecimenSchema.get().getTableInfoSpecimenComment()); + + deleteStudyDesignData(c, user, studyDesignTables); + + Table.delete(StudySchema.getInstance().getTableInfoVisitTag(), containerFilter); + assert deletedTables.add(StudySchema.getInstance().getTableInfoVisitTag()); + Table.delete(StudySchema.getInstance().getTableInfoVisitTagMap(), containerFilter); + assert deletedTables.add(StudySchema.getInstance().getTableInfoVisitTagMap()); + + // dataset tables + for (DatasetDefinition dsd : dsds) + { + fireDatasetChanged(dsd); + } + + // Clear this container ID from any source and destination columns of study snapshots. Then delete any + // study snapshots that are orphaned (both source and destination are gone). + SqlExecutor executor = new SqlExecutor(StudySchema.getInstance().getSchema()); + executor.execute(getStudySnapshotUpdateSql(c, "Source")); + executor.execute(getStudySnapshotUpdateSql(c, "Destination")); + + Filter orphanedFilter = new SimpleFilter + ( + new CompareType.CompareClause(FieldKey.fromParts("Source"), CompareType.ISBLANK, null), + new CompareType.CompareClause(FieldKey.fromParts("Destination"), CompareType.ISBLANK, null) + ); + Table.delete(StudySchema.getInstance().getTableInfoStudySnapshot(), orphanedFilter); + + assert deletedTables.add(StudySchema.getInstance().getTableInfoStudySnapshot()); + + transaction.commit(); + } + + ContainerManager.notifyContainerChange(c.getId(), ContainerManager.Property.StudyChange); + + // + // trust and verify... but only when asserts are on + // + + assert verifyAllTablesWereDeleted(deletedTables); + } + + private List getStudyDesignTables(Container c, User user) + { + List studyDesignTables = new ArrayList<>(); + UserSchema schema = QueryService.get().getUserSchema(user, c, StudyQuerySchema.SCHEMA_NAME); + + addIfProvisioned(studyDesignTables, schema, new StudyProductDomainKind(), PRODUCT_TABLE_NAME); + addIfProvisioned(studyDesignTables, schema, new StudyProductAntigenDomainKind(), PRODUCT_ANTIGEN_TABLE_NAME); + addIfProvisioned(studyDesignTables, schema, new StudyTreatmentProductDomainKind(), TREATMENT_PRODUCT_MAP_TABLE_NAME); + addIfProvisioned(studyDesignTables, schema, new StudyTreatmentDomainKind(), TREATMENT_TABLE_NAME); + addIfProvisioned(studyDesignTables, schema, new StudyPersonnelDomainKind(), PERSONNEL_TABLE_NAME); + + return studyDesignTables; + } + + private void addIfProvisioned(List studyDesignTables, UserSchema schema, AbstractStudyDesignDomainKind domainKind, String tableName) + { + // Might not be provisioned (e.g., if this isn't a study) + Domain domain = domainKind.getDomain(schema.getContainer(), tableName); + + if (null != domain) + studyDesignTables.add(schema.getTable(tableName)); + } + + private void deleteStudyDesignData(Container c, User user, List studyDesignTables) + { + for (TableInfo tinfo : studyDesignTables) + { + if (tinfo instanceof FilteredTable) + { + Table.delete(((FilteredTable)tinfo).getRealTable(), new SimpleFilter(FieldKey.fromParts("Container"), c)); + } + } + } + + private SQLFragment getStudySnapshotUpdateSql(Container c, String columnName) + { + SQLFragment sql = new SQLFragment(); + sql.append("UPDATE "); + sql.append(StudySchema.getInstance().getTableInfoStudySnapshot()); + sql.append(" SET "); + sql.append(columnName); + sql.append(" = NULL WHERE "); + sql.append(columnName); + sql.append(" = ?"); + sql.add(c); + + return sql; + } + + // TODO: Check that datasets are deleted as well? + private boolean verifyAllTablesWereDeleted(Set deletedTables) + { + if (1==1) + return true; + + // Pretend like we deleted from StudyData and StudyDataTemplate tables TODO: why aren't we deleting from these? + Set deletedTableNames = new CaseInsensitiveHashSet("studydata", "studydatatemplate"); + + for (TableInfo t : deletedTables) + { + deletedTableNames.add(t.getName()); + } + + StringBuilder missed = new StringBuilder(); + + for (String tableName : StudySchema.getInstance().getSchema().getTableNames()) + { + if (!deletedTableNames.contains(tableName) && + !"specimen".equalsIgnoreCase(tableName) && !"vial".equalsIgnoreCase(tableName) && !"specimenevent".equalsIgnoreCase(tableName) && + !"site".equalsIgnoreCase(tableName) && !"specimenprimarytype".equalsIgnoreCase(tableName) && + !"specimenderivative".equalsIgnoreCase(tableName) && !"specimenadditive".equalsIgnoreCase(tableName)) + { + missed.append(" "); + missed.append(tableName); + } + } + + if (!missed.isEmpty()) + throw new IllegalStateException("Expected to delete from these tables:" + missed); + + return true; + } + + public @NotNull Collection getParticipantDatasets(Container container, Collection lsids) + { + SimpleFilter filter = new SimpleFilter(); + filter.addClause(new SimpleFilter.InClause(FieldKey.fromParts("LSID"), lsids)); + // We can't use the table layer to map results to our bean class because of the unfortunately named + // "_VisitDate" column in study.StudyData. + + TableInfo sdti = StudySchema.getInstance().getTableInfoStudyData(StudyManager.getInstance().getStudy(container), null); + List pds = new ArrayList<>(); + DatasetDefinition dataset = null; + + try (ResultSet rs = new TableSelector(sdti, filter, new Sort("DatasetId")).getResultSet()) + { + ColumnInfo visitDateCol = sdti.getColumn("_VisitDate"); + while (rs.next()) + { + ParticipantDataset pd = new ParticipantDataset(); + pd.setContainer(container); + int datasetId = rs.getInt("DatasetId"); + if (dataset == null || datasetId != dataset.getDatasetId()) + dataset = getDatasetDefinition(getStudy(container), datasetId); + pd.setDatasetId(datasetId); + pd.setLsid(rs.getString("LSID")); + if (!dataset.isDemographicData()) + { + pd.setSequenceNum(rs.getBigDecimal("SequenceNum")); + pd.setVisitDate(rs.getTimestamp(visitDateCol.getAlias().getId())); + } + pd.setParticipantId(rs.getString("ParticipantId")); + pds.add(pd); + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + + return pds; + } + + + /** + * After changing permissions on the study, we have to scrub the dataset acls to + * remove any groups that no longer have read permission. + * + * UNDONE: move StudyManager into model package (so we can have protected access) + */ + protected void scrubDatasetAcls(Study study, SecurityPolicy newPolicy) + { + //for every principal that plays something other than the RestrictedReaderRole, + //delete that group's role assignments in all dataset policies + Role restrictedReader = RoleManager.getRole(RestrictedReaderRole.class); + + Set resources = new HashSet<>(getDatasetDefinitions(study)); + + Set principals = new HashSet<>(); + + for (RoleAssignment ra : newPolicy.getAssignments()) + { + if (!(ra.getRole().equals(restrictedReader))) + principals.add(SecurityManager.getPrincipal(ra.getUserId())); + } + + SecurityPolicyManager.clearRoleAssignments(resources, principals); + } + + + /** study container only (not dataspace!) */ + public long getParticipantCount(Study study) + { + SQLFragment sql = new SQLFragment("SELECT COUNT(ParticipantId) FROM "); + sql.append(SCHEMA.getTableInfoParticipant(), "p"); + sql.append(" WHERE Container = ?"); + sql.add(study.getContainer()); + return new SqlSelector(StudySchema.getInstance().getSchema(), sql).getObject(Long.class); + } + + public Collection getParticipantIds(Study study, User user) + { + return getParticipantIds(study, user, -1); + } + + /** study container only (not dataspace!) */ + public Collection getParticipantIdsForGroup(Study study, User user, int groupId) + { + return getParticipantIds(study, user, null, groupId, -1); + } + + /** study container only (not dataspace!) */ + public Collection getParticipantIds(Study study, User user, int rowLimit) + { + return getParticipantIds(study, user, null, -1, rowLimit); + } + + public Collection getParticipantIds(Study study, User user, ContainerFilter cf, int rowLimit) + { + return getParticipantIds(study, user, cf, -1, rowLimit); + } + + /** study container only (not dataspace!) */ + private Collection getParticipantIds(Study study, User user, ContainerFilter cf, int participantGroupId, int rowLimit) + { + DbSchema schema = StudySchema.getInstance().getSchema(); + SQLFragment sql = getSQLFragmentForParticipantIds(study, user, cf, participantGroupId, rowLimit, schema, "ParticipantId"); + return new SqlSelector(schema, sql).getCollection(String.class); + } + + private static final String ALTERNATEID_COLUMN_NAME = "AlternateId"; + private static final String DATEOFFSET_COLUMN_NAME = "DateOffset"; + private static final String PTID_COLUMN_NAME = "ParticipantId"; + private static final String CONTAINER_COLUMN_NAME = "Container"; + + public Map getParticipantInfos(Study study, User user, final boolean isShiftDates, final boolean isAlternateIds) + { + DbSchema schema = StudySchema.getInstance().getSchema(); + SQLFragment sql = getSQLFragmentForParticipantIds(study, user, null, -1, -1, schema, + CONTAINER_COLUMN_NAME + ", " + PTID_COLUMN_NAME + ", " + ALTERNATEID_COLUMN_NAME + ", " + DATEOFFSET_COLUMN_NAME); + final Map alternateIdMap = new HashMap<>(); + + new SqlSelector(schema, sql).forEach(rs -> { + String containerId = rs.getString(CONTAINER_COLUMN_NAME); + String participantId = rs.getString(PTID_COLUMN_NAME); + String alternateId = isAlternateIds ? rs.getString(ALTERNATEID_COLUMN_NAME) : participantId; // if !isAlternateIds, use participantId + int dateOffset = isShiftDates ? rs.getInt(DATEOFFSET_COLUMN_NAME) : 0; // if !isDateShift, use 0 shift + alternateIdMap.put(participantId, new ParticipantInfo(containerId, alternateId, dateOffset)); + }); + + return alternateIdMap; + } + + + private SQLFragment getSQLFragmentForParticipantIds(Study study, User user, @Nullable ContainerFilter cf, int participantGroupId, int rowLimit, DbSchema schema, String columns) + { + SQLFragment filter = getParticipantFilter(study, user, cf); + + SQLFragment sql; + if (participantGroupId == -1) + { + sql = new SQLFragment("SELECT " + columns + " FROM " + SCHEMA.getTableInfoParticipant()).append(" WHERE ").append(filter).append(" ORDER BY ParticipantId"); + } + else + { + TableInfo table = StudySchema.getInstance().getTableInfoParticipantGroupMap(); + sql = new SQLFragment("SELECT " + columns + " FROM " + table + " WHERE ").append(filter).append(" AND GroupId = ? ORDER BY ParticipantId").add(participantGroupId); + } + if (rowLimit > 0) + sql = schema.getSqlDialect().limitRows(sql, rowLimit); + return sql; + } + + + private SQLFragment getParticipantFilter(Study study, User user, @Nullable ContainerFilter cf) + { + SQLFragment filter = new SQLFragment(); + if (!study.getShareDatasetDefinitions()) + { + filter.append("Container=").appendValue(study.getContainer()); + } + else + { + if (null == user) + throw new IllegalStateException("provide a user to query the participants table"); + if (null == cf) + cf = new DataspaceContainerFilter(user, study); + filter = cf.getSQLFragment(SCHEMA.getSchema(), new SQLFragment("Container")); + } + return filter; + } + + + public String[] getParticipantIdsForCohort(Study study, int currentCohortId, int rowLimit) + { + DbSchema schema = StudySchema.getInstance().getSchema(); + SQLFragment sql = new SQLFragment("SELECT ParticipantId FROM " + SCHEMA.getTableInfoParticipant() + " WHERE Container = ? AND CurrentCohortId = ? ORDER BY ParticipantId", study.getContainer().getId(), currentCohortId); + + if (rowLimit > 0) + sql = schema.getSqlDialect().limitRows(sql, rowLimit); + + return new SqlSelector(schema, sql).getArray(String.class); + } + + public String[] getParticipantIdsNotInCohorts(Study study) + { + DbSchema schema = StudySchema.getInstance().getSchema(); + SQLFragment sql = new SQLFragment("SELECT ParticipantId FROM " + SCHEMA.getTableInfoParticipant() + " WHERE Container = ? AND CurrentCohortId IS NULL", + study.getContainer().getId()); + + return new SqlSelector(schema, sql).getArray(String.class); + } + + public String[] getParticipantIdsNotInGroupCategory(Study study, User user, int categoryId) + { + return getParticipantIdsNotInGroupCategory(study, user, null, categoryId); + } + + public String[] getParticipantIdsNotInGroupCategory(Study study, User user, @Nullable ContainerFilter cf, int categoryId) + { + TableInfo groupMapTable = StudySchema.getInstance().getTableInfoParticipantGroupMap(); + TableInfo tableInfoParticipantGroup = StudySchema.getInstance().getTableInfoParticipantGroup(); + DbSchema schema = StudySchema.getInstance().getSchema(); + + SQLFragment filter = getParticipantFilter(study,user,cf); + + SQLFragment sql = new SQLFragment("SELECT ParticipantId FROM ").append(SCHEMA.getTableInfoParticipant().getFromSQL("P")) + .append(" WHERE ").append(filter) + .append(" AND ParticipantId NOT IN (SELECT DISTINCT ParticipantId FROM ").append(groupMapTable.getFromSQL("PGM")) + .append(" WHERE GroupId IN (SELECT PG.RowId FROM ").append(tableInfoParticipantGroup.getFromSQL("PG")).append(" WHERE Container = ? AND CategoryId = ?))") + .add(study.getContainer().getId()) + .add(categoryId); + + return new SqlSelector(schema.getScope(), sql).getArray(String.class); + } + + public static final int ALTERNATEID_DEFAULT_NUM_DIGITS = 6; + + public void clearAlternateParticipantIds(Study study) + { + if (study.isDataspaceStudy()) + return; + Collection participantIds = getParticipantIds(study,null); + + for (String participantId : participantIds) + setAlternateId(study, study.getContainer().getId(), participantId, null); + } + + public void generateNeededAlternateParticipantIds(Study study, User user) + { + Map participantInfos = getParticipantInfos(study, user, false, true); + + StudyController.ChangeAlternateIdsForm changeAlternateIdsForm = StudyController.getChangeAlternateIdForm((StudyImpl) study); + String prefix = changeAlternateIdsForm.getPrefix(); + if (null == prefix) + prefix = ""; // So we don't get the string "null" as the prefix + int numDigits = changeAlternateIdsForm.getNumDigits(); + if (numDigits < ALTERNATEID_DEFAULT_NUM_DIGITS) + numDigits = ALTERNATEID_DEFAULT_NUM_DIGITS; // Should not happen, but be safe + + HashSet usedNumbers = new HashSet<>(); + for (ParticipantInfo participantInfo : participantInfos.values()) + { + String alternateId = participantInfo.getAlternateId(); + if (alternateId != null) + { + try + { + if (prefix.isEmpty() || alternateId.startsWith(prefix)) + { + String alternateIdNoPrefix = alternateId.substring(prefix.length()); + usedNumbers.add(alternateIdNoPrefix); + } + } + catch (NumberFormatException x) + { + // It's possible that the id is not an integer after stripping prefix, because it can be + // set explicitly. That's fine, because it won't conflict with what we might generate + } + } + } + + for (Map.Entry entry : participantInfos.entrySet()) + { + ParticipantInfo participantInfo = entry.getValue(); + String alternateId = participantInfo.getAlternateId(); + + if (null == alternateId) + { + String participantId = entry.getKey(); + String newId = nextRandom(usedNumbers, numDigits); + setAlternateId(study, participantInfo.getContainerId(), participantId, prefix + newId); + } + } + } + + public int setImportedAlternateParticipantIds(Study study, DataLoader dl, BatchValidationException errors) throws IOException + { + // Use first line to determine order of columns we care about + // The first column in the data must contain the ones we are seeking + String[][] firstline = dl.getFirstNLines(1); + if (null == firstline || 0 == firstline.length) + return 0; // Unexpected but just in case + + boolean seenParticipantId = false; + boolean seenAlternateIdOrDateOffset = false; + boolean headerError = false; + ColumnDescriptor[] columnDescriptors = new ColumnDescriptor[3]; + for (int i = 0; i < 3 && i < firstline[0].length; i += 1) + { + String header = firstline[0][i]; + switch (header) + { + case PTID_COLUMN_NAME: + columnDescriptors[i] = new ColumnDescriptor(PTID_COLUMN_NAME, String.class); + seenParticipantId = true; + break; + case ALTERNATEID_COLUMN_NAME: + columnDescriptors[i] = new ColumnDescriptor(ALTERNATEID_COLUMN_NAME, String.class); + seenAlternateIdOrDateOffset = true; + break; + case DATEOFFSET_COLUMN_NAME: + columnDescriptors[i] = new ColumnDescriptor(DATEOFFSET_COLUMN_NAME, Integer.class); + seenAlternateIdOrDateOffset = true; + break; + default: + if (i < 2) + headerError = true; + break; + } + if (headerError) + break; + } + + int rowCount = 0; + if (!seenParticipantId || !seenAlternateIdOrDateOffset || headerError) + { + errors.addRowError(new ValidationException("The header row must contain " + PTID_COLUMN_NAME + " and either " + + ALTERNATEID_COLUMN_NAME + ", " + DATEOFFSET_COLUMN_NAME + " or both.")); + } + else + { + assert null != columnDescriptors[0] && null != columnDescriptors[1]; // Since we've seen PTID and 1 other + if (null == columnDescriptors[2]) + columnDescriptors = Arrays.copyOf(columnDescriptors, 2); // Can't hand DataLoader a null column + + // Now get loader to load all rows with correct columns and types + dl.setColumns(columnDescriptors); + dl.setHasColumnHeaders(true); + dl.setThrowOnErrors(true); + dl.setInferTypes(false); + + // Note alternateIds that are already used + Map participantInfos = getParticipantInfos(study, null, true, true); + CaseInsensitiveHashSet usedIds = new CaseInsensitiveHashSet(); + for (ParticipantInfo participantInfo : participantInfos.values()) + { + String alternateId = participantInfo.getAlternateId(); + if (alternateId != null) + { + usedIds.add(alternateId); + } + } + + List> rows = dl.load(); + rowCount = rows.size(); + + // Remove used alternateIds for participantIds that are in the list to be changed + for (Map row : rows) + { + String participantId = Objects.toString(row.get(PTID_COLUMN_NAME), null); + String alternateId = Objects.toString(row.get(ALTERNATEID_COLUMN_NAME), null); + if (null != participantId && null != alternateId) + { + ParticipantInfo participantInfo = participantInfos.get(participantId); + if (null != participantInfo) + { + String currentAlternateId = participantInfo.getAlternateId(); + if (null != currentAlternateId && !alternateId.equalsIgnoreCase(currentAlternateId)) + usedIds.remove(currentAlternateId); // remove as it will get replaced + } + } + } + + try (Transaction transaction = StudySchema.getInstance().getSchema().getScope().ensureTransaction()) + { + for (Map row : rows) + { + String participantId = Objects.toString(row.get(PTID_COLUMN_NAME), null); + if (null == participantId) + { + // ParticipantId must be specified + errors.addRowError(new ValidationException("A ParticipantId must be specified.")); + break; + } + + String alternateId = Objects.toString(row.get(ALTERNATEID_COLUMN_NAME), null); + Integer dateOffset = (null != row.get(DATEOFFSET_COLUMN_NAME)) ? asInteger(row.get(DATEOFFSET_COLUMN_NAME)) : null; + + if (null == alternateId && null == dateOffset) + { + errors.addRowError(new ValidationException("Either " + ALTERNATEID_COLUMN_NAME + " or " + DATEOFFSET_COLUMN_NAME + " must be specified.")); + break; + } + + ParticipantInfo participantInfo = participantInfos.get(participantId); + if (null != participantInfo) + { + String currentAlternateId = participantInfo.getAlternateId(); + if (null != alternateId && !alternateId.equalsIgnoreCase(currentAlternateId) && usedIds.contains(alternateId)) + { + errors.addRowError(new ValidationException("Two participants may not share the same Alternate ID.")); + break; + } + + if ((null != alternateId && !alternateId.equalsIgnoreCase(currentAlternateId)) || + (null != dateOffset && dateOffset != participantInfo.getDateOffset())) + { + + setAlternateIdAndDateOffset(study, participantId, alternateId, dateOffset); + if (null != alternateId) + usedIds.add(alternateId); // Add new id + } + } + else + { + errors.addRowError(new ValidationException("ParticipantID " + participantId + " not found.")); + } + } + + if (!errors.hasErrors()) + transaction.commit(); + } + } + + if (errors.hasErrors()) + return 0; + return rowCount; + } + + private void setAlternateId(Study study, String containerId, String participantId, @Nullable String alternateId) + { + // Set alternateId even if null, because that's how we clear it + SQLFragment sql = new SQLFragment("UPDATE ").append(SCHEMA.getTableInfoParticipant()).append(" SET AlternateId = ? WHERE Container = ? AND ParticipantId = ?") + .addAll(alternateId, containerId, participantId); + new SqlExecutor(StudySchema.getInstance().getSchema()).execute(sql); + StudyManager.getInstance().clearParticipantCache(study.getContainer()); + } + + private void setAlternateIdAndDateOffset(Study study, String participantId, @Nullable String alternateId, @Nullable Integer dateOffset) + { + // Only set alternateId and/or dateOffset if non-null + assert null != participantId; + if (null != alternateId || null != dateOffset) + { + SQLFragment sql = new SQLFragment("UPDATE ").append(SCHEMA.getTableInfoParticipant()).append(" SET "); + boolean needComma = false; + if (null != alternateId) + { + sql.append("AlternateId = ?").add(alternateId); + needComma = true; + } + if (null != dateOffset) + { + if (needComma) + sql.append(", "); + sql.append("DateOffset = ?").add(dateOffset); + } + sql.append(" WHERE Container = ? AND ParticipantId = ?"); + sql.add(study.getContainer()); + sql.add(participantId); + new SqlExecutor(StudySchema.getInstance().getSchema()).execute(sql); + StudyManager.getInstance().clearParticipantCache(study.getContainer()); + } + } + + private String nextRandom(Set usedNumbers, int numDigits) + { + String newId; + do + { + newId = StringUtilsLabKey.getUniquifier(numDigits); + } while (usedNumbers.contains(newId)); + usedNumbers.add(newId); + return newId; + } + + private void parseData(User user, + DatasetDefinition def, + DataLoader loader, + Map columnMap) + throws IOException + { + TableInfo tinfo = def.getTableInfo(user); + + // We're going to lower-case the keys ourselves later, + // so this needs to be case-insensitive + if (!(columnMap instanceof CaseInsensitiveHashMap)) + { + columnMap = new CaseInsensitiveHashMap<>(columnMap); + } + + // StandardDataIteratorBuilder will handle most aliasing, HOWEVER, ... + // columnMap may contain propertyURIs (dataset import job) and labels (GWT import file) + Map nameMap = DataIteratorUtil.createTableMap(tinfo, true); + + // + // create columns to properties map + // + loader.setInferTypes(false); + ColumnDescriptor[] cols = loader.getColumns(); + for (ColumnDescriptor col : cols) + { + String name = col.name.toLowerCase(); + + //Special column name + if ("replace".equals(name)) + { + col.clazz = Boolean.class; + col.name = name; //Lower case + continue; + } + + // let DataIterator do conversions + col.clazz = String.class; + + if (columnMap.containsKey(name)) + name = columnMap.get(name); + + col.name = name; + + ColumnInfo colinfo = nameMap.get(col.name); + if (null != colinfo) + { + col.name = colinfo.getName(); + col.propertyURI = colinfo.getPropertyURI(); + } + } + } + + + public void batchValidateExceptionToList(BatchValidationException errors, List errorStrs) + { + for (ValidationException rowError : errors.getRowErrors()) + { + String rowPrefix = ""; + if (rowError.getRowNumber() >= 0) + rowPrefix = "Row " + rowError.getRowNumber() + " "; + for (ValidationError e : rowError.getErrors()) + errorStrs.add(rowPrefix + e.getMessage()); + } + } + + /** + * @deprecated pass in a DataIteratorContext instead of individual options + */ + @Deprecated + public List importDatasetData(User user, DatasetDefinition def, + DataLoader loader, + Map columnMap, + BatchValidationException errors, + DatasetDefinition.CheckForDuplicates checkDuplicates, + @Nullable DataState defaultQCState, + QueryUpdateService.InsertOption insertOption, + Logger logger, + LookupResolutionType lookupResolutionType, + @Nullable AuditBehaviorType auditBehaviorType) + throws IOException + { + DataIteratorContext context = new DataIteratorContext(errors); + + context.setInsertOption(insertOption); + context.setLookupResolutionType(lookupResolutionType); + + Map options = new HashMap<>(); + options.put(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, auditBehaviorType); + options.put(DatasetUpdateService.Config.AllowImportManagedKey, Boolean.FALSE); + options.put(DatasetUpdateService.Config.CheckForDuplicates, checkDuplicates); + if (defaultQCState != null) + options.put(DatasetUpdateService.Config.DefaultQCState, defaultQCState); + if (logger != null) + options.put(QueryUpdateService.ConfigParameters.Logger, logger); + + context.setConfigParameters(options); + + return importDatasetData(user, def, loader, columnMap, context); + } + + public List importDatasetData(User user, DatasetDefinition def, DataLoader loader, + Map columnMap, DataIteratorContext context) throws IOException + { + parseData(user, def, loader, columnMap); + return def.importDatasetData(user, loader, context); + } + + + /** + * @deprecated pass in a DataIteratorContext instead of individual options + */ + @Deprecated + public List importDatasetData(User user, DatasetDefinition def, + List> data, + BatchValidationException errors, + DatasetDefinition.CheckForDuplicates checkDuplicates, + @Nullable DataState defaultQCState, + Logger logger, + boolean allowImportManagedKey, + boolean skipTriggers) throws IOException + { + if (data.isEmpty()) + return Collections.emptyList(); + + DataIteratorContext context = new DataIteratorContext(errors); + Map options = new HashMap<>(); + + options.put(QueryUpdateService.ConfigParameters.Logger, logger); + options.put(DatasetUpdateService.Config.AllowImportManagedKey, Boolean.valueOf(allowImportManagedKey)); + if (defaultQCState != null) + options.put(DatasetUpdateService.Config.DefaultQCState, defaultQCState); + options.put(DatasetUpdateService.Config.CheckForDuplicates, checkDuplicates); + // if we are being called by QUS we don't want to call triggers twice or resync twice + options.put(QueryUpdateService.ConfigParameters.SkipTriggers, skipTriggers); + options.put(DatasetUpdateService.Config.SkipResyncStudy, skipTriggers); + context.setConfigParameters(options); + + DataLoader loader = new MapLoader(data); + context.setInsertOption(allowImportManagedKey ? QueryUpdateService.InsertOption.INSERT : QueryUpdateService.InsertOption.IMPORT); + + return importDatasetData(user, def, loader, new CaseInsensitiveHashMap<>(), context); + } + + public boolean importDatasetSchemas(StudyImpl study, final User user, SchemaReader reader, BindException errors, boolean createShared, boolean allowDomainUpdates, @Nullable Activity activity) + { + if (errors.hasErrors()) + return false; + + StudyImpl createDatasetStudy = null; + if (createShared) + createDatasetStudy = (StudyImpl)getSharedStudy(study); + if (null == createDatasetStudy) + createDatasetStudy = study; + + List importErrors = new LinkedList<>(); + final Map datasetDefEntryMap = new HashMap<>(); + + // Use a factory to ensure domain URI consistency between imported properties and the dataset. See #7944. + DomainURIFactory factory = name -> { + assert datasetDefEntryMap.containsKey(name); + DatasetDefinitionEntry defEntry = datasetDefEntryMap.get(name); + Container defContainer = defEntry.datasetDefinition.getDefinitionContainer(); + String domainURI = getDomainURI(defEntry.datasetDefinition.getDefinitionContainer(), user, name, defEntry.datasetDefinition.getEntityId()); + return new Pair<>(domainURI, defContainer); + }; + + // We need to build the datasets (but not save) before we create the property descriptors so that + // we can use the unique DomainURI for each dataset as part of the PropertyURI + populateDatasetDefEntryMap(study, createDatasetStudy, reader, user, errors, datasetDefEntryMap); + if (errors.hasErrors()) + return false; + + ImportPropertyDescriptorsList list = reader.getImportPropertyDescriptors(factory, importErrors, study.getContainer()); + if (!importErrors.isEmpty()) + { + for (String error : importErrors) + errors.reject("importDatasetSchemas", error); + return false; + } + + // Check PHI levels; Must check activity level here, because we're in pipeline job, so Compliance can't get activity from HttpContext + /* TODO this list should be consistent across all types, not just List, see ListImporter.createDefinedLists() */ + ComplianceFolderSettings settings = ComplianceService.get().getFolderSettings(createDatasetStudy.getContainer(), User.getAdminServiceUser()); + PhiColumnBehavior columnBehavior = null==settings ? PhiColumnBehavior.show : settings.getPhiColumnBehavior(); + if (PhiColumnBehavior.show != columnBehavior) + { + PHI maxAllowedPhi = ComplianceService.get().getMaxAllowedPhi(createDatasetStudy.getContainer(), user); + if (null != activity && !maxAllowedPhi.isLevelAllowed(activity.getPHI())) + maxAllowedPhi = activity.getPHI(); // Reduce allowed level + + PHI maxContainedPhi = PHI.NotPHI; + for (ImportPropertyDescriptor ipd : list.properties) + { + if (maxContainedPhi.getRank() < ipd.pd.getPHI().getRank()) + maxContainedPhi = ipd.pd.getPHI(); + } + + if (!maxContainedPhi.isLevelAllowed(maxAllowedPhi)) + { + errors.reject(ERROR_MSG, "User's max allowed PHI is '" + maxAllowedPhi.getLabel() + "', but imported datasets contain higher PHI '" + maxContainedPhi.getLabel() + "'."); + return false; + } + } + + for (ImportPropertyDescriptor ipd : list.properties) + { + if (null == ipd.domainName || null == ipd.domainURI) + errors.reject("importDatasetSchemas", "Dataset not specified for property: " + ipd.pd.getName()); + } + if (errors.hasErrors()) + return false; + + StudyManager manager = StudyManager.getInstance(); + + Map domainChangeMap = new CaseInsensitiveHashMap<>(); + if (allowDomainUpdates) + { + // generate the dataset domain changes for existing datasets + buildPropertySaveAndDeleteLists(datasetDefEntryMap, list, domainChangeMap, true); + } + + // now actually create the datasets + for (Map.Entry entry : datasetDefEntryMap.entrySet()) + { + DatasetDefinitionEntry d = entry.getValue(); + DatasetDefinition def = d.datasetDefinition; + + if (d.isNew) + manager.createDatasetDefinition(user, def); + else if (d.isModified) + { + // issue 44363 : in certain situations the dataset domain will need to be saved earlier in order to support + // a change in the key column that may not be in the initial domain + if (manager.isKeyChanged(def)) + { + var tableInfo = def.getTableInfo(user); + if (tableInfo != null && (new TableSelector(def.getTableInfo(user)).getRowCount() > 0)) + { + // throw an error if we are changing keys on a dataset with data + errors.reject(ERROR_MSG, "Unable to change the keys on dataset (" + def.getName() + "), because there is still data present. The dataset should be truncated before the import."); + return false; + } + + if (!def.getUseTimeKeyField()) + { + String keyName = def.getKeyPropertyName(); + Domain domain = def.refreshDomain(); + if (domain != null) + { + _DatasetDomainChange domainChange = domainChangeMap.get(domain.getTypeURI()); + Domain newDomain = domainChange.domain; + if (domain.getStorageTableName() != null && newDomain != null) + { + if (domain.getPropertyByName(keyName) == null && newDomain.getPropertyByName(keyName) != null) + { + try + { + newDomain.save(user); + } + catch (ChangePropertyDescriptorException ex) + { + errors.reject("importDatasetSchemas", ex.getMessage() == null ? ex.toString() : ex.getMessage()); + return false; + } + } + } + } + } + } + manager.updateDatasetDefinition(user, def); + } + + if (d.tags != null) + ReportPropsManager.get().importProperties(def.getEntityId(), def.getDefinitionContainer(), user, d.tags); + } + + // optional param to control whether field additions or deletions are permitted + if (allowDomainUpdates) + { + // Generate dataset domain changes for new datasets + domainChangeMap.clear(); + buildPropertySaveAndDeleteLists(datasetDefEntryMap, list, domainChangeMap, false); + + // Now that we actually have datasets, create or update the domains. This ensures that all domains and + // property changes are saved before adding indices. + ensurePropertiesAndRequiredIndices(reader, datasetDefEntryMap, domainChangeMap, user, errors); + + if (errors.hasErrors()) + return false; + } + + List orderedIds = reader.getDatasetOrder(); + if (null != orderedIds) + { + DatasetReorderer reorderer = new DatasetReorderer(study, user); + reorderer.reorderDatasets(orderedIds); + } + + return true; + } + + // see if we need to delete any columns from an existing domain and create the domain if it doesn't already exist + private boolean deleteAndSaveProperties(User user, BindException errors, _DatasetDomainChange domainChange) + { + // see if we need to delete any columns from the domain + for (DomainProperty p : domainChange.propsToDelete) + { + p.delete(); + } + + try + { + domainChange.domain.save(user); + } + catch (ChangePropertyDescriptorException ex) + { + errors.reject("importDatasetSchemas", ex.getMessage() == null ? ex.toString() : ex.getMessage()); + return false; + } + return true; + } + + /** + * Utility class to track dataset domain changes + */ + private static class _DatasetDomainChange + { + public _DatasetDomainChange() {} + public _DatasetDomainChange(Domain domain) + { + this.domain = domain; + this.propsToDelete = new ArrayList<>(domain.getProperties()); + } + + private Domain domain; + private List propsToDelete = Collections.emptyList(); + } + + private _DatasetDomainChange createDomainChange(String domainURI, String domainName, DatasetDefinitionEntry def, boolean existingDomainsOnly) + { + _DatasetDomainChange domainChange = new _DatasetDomainChange(); + domainChange.domain = PropertyService.get().getDomain(def.datasetDefinition.getDefinitionContainer(), domainURI, true); + if (domainChange.domain == null && existingDomainsOnly) + return null; + else if (domainChange.domain == null) + domainChange.domain = PropertyService.get().createDomain(def.datasetDefinition.getDefinitionContainer(), domainURI, domainName); + + // add all the properties that exist for the domain + domainChange.propsToDelete = new ArrayList<>(domainChange.domain.getProperties()); + + return domainChange; + } + + /** + * Generate the dataset domain changes for the import + * @param domainChangeMap - maps domain URIs to the _DatasetDomainChange object + * @param existingDomainsOnly - if true will only populate the map for existing datasets + */ + private void buildPropertySaveAndDeleteLists(Map datasetDefEntryMap, + ImportPropertyDescriptorsList list, + Map domainChangeMap, + boolean existingDomainsOnly) + { + for (ImportPropertyDescriptor ipd : list.properties) + { + _DatasetDomainChange domainChange = domainChangeMap.computeIfAbsent(ipd.domainURI, (k) -> + createDomainChange(ipd.domainURI, ipd.domainName, datasetDefEntryMap.get(ipd.domainName), existingDomainsOnly)); + + if (domainChange == null) + continue; + + Domain d = domainChange.domain; + // Issue 14569: during study reimport be sure to look for a column has been deleted. + // Look at the existing properties for this dataset's domain and + // remove them as we find them in schema. If there are any properties left after we've + // iterated over all the import properties then we need to delete them + DomainProperty p = d.getPropertyByName(ipd.pd.getName()); + domainChange.propsToDelete.remove(p); + + if (null != p) + { + final String fromPropertyUri = p.getPropertyDescriptor().getPropertyURI(); + boolean fromSystemProp = SystemProperty.getProperties().stream().anyMatch(sp -> + sp.getPropertyURI().equals(fromPropertyUri)); + boolean toSystemProp = SystemProperty.getProperties().stream().anyMatch(sp -> + sp.getPropertyURI().equals(ipd.pd.getPropertyURI())); + boolean propertyUriChange = !fromPropertyUri.equals(ipd.pd.getPropertyURI()); + + // Don't copy values over a system prop, just setup swapping the property descriptor + if (propertyUriChange && toSystemProp) + { + p.setPropertyURI(ipd.pd.getPropertyURI()); + } + else + { + // Enable the domain to make schema changes for this property if required + // by dropping/adding the property and its storage at domain save time + p.setSchemaImport(true); + OntologyManager.updateDomainPropertyFromDescriptor(p, ipd.pd); + } + + // Flag this as a property descriptor swap. EnsurePropertyDescriptor will find correct property ID + // by propertyURI. Ensure correct container/projects set. + if (propertyUriChange && (toSystemProp || fromSystemProp)) + { + p.getPropertyDescriptor().setPropertyId(0); + p.getPropertyDescriptor().setContainer(ipd.pd.getContainer()); + } + } + else + { + // don't add property descriptors for columns with 'global' propertyuri + // TODO: move to conceptURI, and use 'local' propertyURI so each domain can have its own + // propertydescriptor instance + if (ipd.pd.getPropertyURI().startsWith("http://cpas.labkey.com/Study#")) + continue; + p = d.addProperty(); + ipd.pd.copyTo(p.getPropertyDescriptor()); + p.setName(ipd.pd.getName()); + p.setRequired(ipd.pd.isRequired()); // TODO: Redundant? copyTo() already copied required (without involving nullable) + p.setDescription(ipd.pd.getDescription()); + } + + ipd.validators.forEach(p::addValidator); + p.setConditionalFormats(ipd.formats); + p.setDefaultValue(ipd.defaultValue); + } + + //Ensure that each dataset has an entry in the domain map + if (datasetDefEntryMap.size() != domainChangeMap.size()) + { + for (DatasetDefinitionEntry datasetDefinitionEntry : datasetDefEntryMap.values()) + { + if (!domainChangeMap.containsKey(datasetDefinitionEntry.datasetDefinition.getTypeURI())) + { + Domain domain = + PropertyService.get().getDomain( + datasetDefinitionEntry.datasetDefinition.getDefinitionContainer(), + datasetDefinitionEntry.datasetDefinition.getTypeURI(), + true); + if (domain != null) + domainChangeMap.put(datasetDefinitionEntry.datasetDefinition.getTypeURI(), new _DatasetDomainChange(domain)); + } + } + } + } + + private void ensurePropertiesAndRequiredIndices(SchemaReader reader, Map datasetDefEntryMap, Map domainChangeMap, User user, BindException errors) + { + for (SchemaReader.DatasetImportInfo datasetImportInfo : reader.getDatasetInfo().values()) + { + DatasetDefinitionEntry datasetDefinitionEntry = datasetDefEntryMap.get(datasetImportInfo.name); + _DatasetDomainChange domainChange = domainChangeMap.get(datasetDefinitionEntry.datasetDefinition.getTypeURI()); + + if (domainChange != null) + { + Domain domain = domainChange.domain; + boolean shared = datasetDefinitionEntry.datasetDefinition.isShared(); + + if (!domain.isProvisioned() || shared) + { + deleteAndSaveProperties(user, errors, domainChange); + if (!datasetDefinitionEntry.datasetDefinition.isShared()) + { + // Refresh the domain, now that it's provisioned + domain = PropertyService.get().getDomain(domain.getContainer(), domain.getTypeURI()); + if (null == domain) + throw new IllegalStateException("Domain should not be null"); + domain.setPropertyIndices(datasetImportInfo.indices); + StorageProvisioner.get().ensureTableIndices(domain); + } + } + else + { + // If we're changing an existing domain, we may be dropping a column that has an admin-configured + // index. We need to drop the indices first, adjust the properties, and then add the new indices. + // This method allows for that. + domain.setPropertyIndices(datasetImportInfo.indices); + StorageProvisioner.get().ensureTableIndices(domain, () -> deleteAndSaveProperties(user, errors, domainChange)); + } + } + } + } + + public String getDomainURI(Container c, User u, Dataset def) + { + if (null == def) + return getDomainURI(c, u, null, null); + else + return getDomainURI(c, u, def.getName(), def.getEntityId()); + } + + + private boolean populateDatasetDefEntryMap(StudyImpl study, StudyImpl createDatasetStudy, SchemaReader reader, User user, BindException errors, Map defEntryMap) + { + StudyManager manager = StudyManager.getInstance(); + Container c = study.getContainer(); + Map datasetInfoMap = reader.getDatasetInfo(); + + for (Map.Entry entry : datasetInfoMap.entrySet()) + { + int id = entry.getKey().intValue(); + SchemaReader.DatasetImportInfo info = entry.getValue(); + String name = info.name; + String label = info.label; + if (label == null) + { + // Default to using the name as the label if none was explicitly specified + label = name; + } + + // Check for name conflicts + Dataset existingDef = manager.getDatasetDefinitionByLabel(study, label); + + if (existingDef != null && existingDef.getDatasetId() != id) + { + errors.reject("importDatasetSchemas", "Dataset '" + existingDef.getName() + "' is already using the label '" + label + "'"); + return false; + } + + existingDef = manager.getDatasetDefinitionByName(study, name); + + if (existingDef != null && existingDef.getDatasetId() != id) + { + errors.reject("importDatasetSchemas", "Existing " + name + " dataset has id " + existingDef.getDatasetId() + + ", uploaded " + name + " dataset has id " + id); + return false; + } + + if (info.demographicData && (info.keyPropertyName != null)) + { + errors.reject("importDatasetSchemas", "Dataset '" + name + "' has key field set to " + info.keyPropertyName + ". This a demographic dataset therefore cannot have an extra key property."); + return false; + } + + DatasetDefinition def = manager.getDatasetDefinition(study, id); + + if (def == null) + { + def = new DatasetDefinition(createDatasetStudy, id, name, label, null, null, null); + def.setDescription(info.description); + def.setVisitDatePropertyName(info.visitDatePropertyName); + def.setShowByDefault(!info.isHidden); + def.setKeyPropertyName(info.keyPropertyName); + def.setCategory(info.category); + def.setKeyManagementType(info.keyManagementType); + def.setDemographicData(info.demographicData); + def.setType(info.type); + def.setTag(info.tag); + defEntryMap.put(name, new DatasetDefinitionEntry(def, true, info.tags)); + def.setUseTimeKeyField(info.useTimeKeyField); + } + else if (def.isPublishedData()) + { + errors.reject("importDatasetSchemas", "Unable to modify linked data dataset '" + def.getLabel() + "'."); + } + else + { + // TODO: modify shared definition? + boolean canEditDefinition = def.canUpdateDefinition(user); + + if (canEditDefinition) + { + def = def.createMutable(); + def.setLabel(label); + def.setName(name); + def.setDescription(info.description); + if (null == def.getTypeURI()) + { + def.setTypeURI(getDomainURI(c, user, def)); + } + + def.setVisitDatePropertyName(info.visitDatePropertyName); + def.setShowByDefault(!info.isHidden); + def.setKeyPropertyName(info.keyPropertyName); + def.setCategory(info.category); + def.setKeyManagementType(info.keyManagementType); + def.setDemographicData(info.demographicData); + def.setTag(info.tag); + } + else + { + // TODO: warn + // name, label, description, visitdatepropertyname, category + if (def.getKeyManagementType() != info.keyManagementType) + errors.reject("ERROR_MSG", "Key type is not compatible with shared dataset: " + def.getName()); + if (!Strings.CI.equals(def.getKeyPropertyName(), info.keyPropertyName)) + errors.reject("ERROR_MSG", "Key property name is not compatible with shared dataset: " + def.getName()); + if (def.isDemographicData() != info.demographicData) + errors.reject("ERROR_MSG", "Demographic type is not compatible with shared dataset: " + def.getName()); + } + + defEntryMap.put(name, new DatasetDefinitionEntry(def, false, canEditDefinition, info.tags)); + } + } + + return true; + } + + // Detect if this dataset has an old-style URI without the entityid. If so, assign a new type URI to this dataset + // and update the domain descriptor URI + // old: urn:lsid:labkey.com:StudyDataset.Folder-6:DEM + // new: urn:lsid:labkey.com:StudyDataset.Folder-6:DEM-cbffdfa1-f19b-1030-90dd-bf4ca488b2d0 + // Also, the URI will change if the dataset name changes + private void ensureDatasetDefinitionDomain(User user, DatasetDefinition def) + { + String oldURI = def.getTypeURI(); + String newURI = getDomainURI(def.getContainer(), user, def); + + if (Strings.CS.equals(oldURI, newURI)) + return; + + // This dataset has the old uri so upgrade it to use the new URI format + def.setTypeURI(newURI, true /*upgrade*/); + + // fixup the domain + DomainDescriptor dd = OntologyManager.getDomainDescriptor(oldURI, def.getContainer()); + if (null != dd) + { + dd = dd.edit() + .setDomainURI(newURI) + .setName(def.getName()) // Name may have changed too; it's part of URI + .build(); + OntologyManager.ensureDomainDescriptor(dd); + + // since the descriptor has changed, ensure the domain is up-to-date + def.refreshDomain(); + } + } + + private static String getDomainURI(Container c, User u, String name, String id) + { + return DatasetDomainKind.generateDomainURI(name, id, c); + } + + @NotNull + public VisitManager getVisitManager(Study study) + { + @Migrate // TODO: Switch VisitManager() to take Study and get rid of cast + StudyImpl studyImpl = (StudyImpl)study; + switch (study.getTimepointType()) + { + case VISIT: + return new SequenceVisitManager(studyImpl); + case CONTINUOUS: + return new AbsoluteDateVisitManager(studyImpl); + case DATE: + default: + return new RelativeDateVisitManager(studyImpl); + } + } + + public static SQLFragment timePortionFromDateSQL(String dateColumnName) + { + SqlDialect dialect = StudySchema.getInstance().getSqlDialect(); + SQLFragment sql = new SQLFragment(); + if (dialect.isPostgreSQL()) + { + sql.append("to_char(").append(dateColumnName).append(", 'HH24MISS')"); + } + else if (dialect.isSqlServer()) + { + sql.append("FORMAT(").append(dateColumnName).append(", 'HHmmss')"); + } + else + { + sql.append("CAST((").append(dateColumnName).append(") AS VARCHAR(10))"); + } + return sql; + } + + private String getParticipantCacheKey(Container container) + { + return container.getId() + "/" + Participant.class; + } + + /** non-permission checking, non-recursive */ + private Map getParticipantMap(Study study) + { + return _participantCache.get(study.getContainer()); + } + + public void clearParticipantCache(Container container) + { + _participantCache.remove(container); + } + + public Collection getParticipants(Study study) + { + Map participantMap = getParticipantMap(study); + return Collections.unmodifiableCollection(participantMap.values()); + } + + public Participant getParticipant(Study study, String participantId) + { + Map participantMap = getParticipantMap(study); + return participantMap.get(participantId); + } + + public static class ParticipantNotUniqueException extends Exception + { + ParticipantNotUniqueException(String ptid) + { + super("Participant found in more than one study: " + ptid); + } + } + + /* non-permission checking, may return participant from sub folder */ + public Container findParticipant(Study study, String ptid) throws ParticipantNotUniqueException + { + Participant p = getParticipant(study, ptid); + if (null != p) + return study.getContainer(); + else if (!study.isDataspaceStudy()) + return null; + + TableInfo table = StudySchema.getInstance().getTableInfoParticipant(); + ArrayList containers = new SqlSelector(table.getSchema(), new SQLFragment("SELECT container FROM study.participant WHERE participantid=?",ptid)) + .getArrayList(String.class); + if (containers.isEmpty()) + return null; + else if (containers.size() == 1) + return ContainerManager.getForId(containers.get(0)); + throw new ParticipantNotUniqueException(ptid); + } + + + public CustomParticipantView getCustomParticipantView(Study study) + { + if (study == null) + return null; + + Path path = ModuleHtmlView.getStandardPath("participant"); + + for (Module module : study.getContainer().getActiveModules()) + { + if (ModuleHtmlView.exists(module, path)) + { + return CustomParticipantView.create(module, path); + } + } + + String key = new Path(study.getContainer().getId(),CustomParticipantView.class.getName()).toString(); + return (CustomParticipantView) CacheManager.getSharedCache().get(key, study, (k, s) -> + { + SimpleFilter containerFilter = SimpleFilter.createContainerFilter(study.getContainer()); + TableInfo ti = StudySchema.getInstance().getTableInfoParticipantView(); + CustomParticipantView participantView = new TableSelector(ti, containerFilter, null).getObject(CustomParticipantView.class); + if (null == participantView) + return null; + Module studyModule = ModuleLoader.getInstance().getModule("study"); + participantView.setModule(studyModule); + participantView.setTitle(study.getSubjectNounSingular()); + return participantView; + }); + } + + public CustomParticipantView saveCustomParticipantView(Study study, User user, CustomParticipantView view) + { + if (view.isModuleParticipantView()) + throw new IllegalArgumentException("Module-defined participant views should not be saved to the database."); + CustomParticipantView ret; + if (view.getRowId() == null) + { + view.beforeInsert(user, study.getContainer().getId()); + ret = Table.insert(user, StudySchema.getInstance().getTableInfoParticipantView(), view); + } + else + { + view.beforeUpdate(user); + ret = Table.update(user, StudySchema.getInstance().getTableInfoParticipantView(), view, view.getRowId()); + } + CacheManager.getSharedCache().remove(new Path(view.getContainerId(),CustomParticipantView.class.getName()).toString()); + return ret; + } + + public interface ParticipantViewConfig + { + String getParticipantId(); + + int getDatasetId(); + + Map getAliases(); + } + + public WebPartView getParticipantView(Container container, ParticipantViewConfig config) + { + return getParticipantView(container, config, null); + } + + public WebPartView getParticipantView(Container container, ParticipantViewConfig config, BindException errors) + { + StudyImpl study = getStudy(container); + if (study.getTimepointType() == TimepointType.CONTINUOUS) + return new StudyJspView<>(study, "/org/labkey/study/view/participantData.jsp", config, errors); + else + return new StudyJspView<>(study, "/org/labkey/study/view/participantAll.jsp", config, errors); + } + + public WebPartView getParticipantDemographicsView(Container container, ParticipantViewConfig config, BindException errors) + { + return new StudyJspView<>(getStudy(container), "/org/labkey/study/view/participantCharacteristics.jsp", config, errors); + } + + /** + * Called when a dataset has been modified in order to set the modified time, plus any other related actions. + * @param fireNotification - true to fire the changed notification. + */ + public static void datasetModified(DatasetDefinition def, boolean fireNotification) + { + // Issue 19285 - run this as a commit task. This has the benefit of only running per set of batch changes + // under the same transaction and only running if the transaction is committed. If no transaction is active then + // the code is run immediately + DbScope scope = StudySchema.getInstance().getScope(); + scope.addCommitTask(getInstance().getDatasetModifiedRunnable(def, fireNotification), CommitTaskOption.POSTCOMMIT); + } + + public Runnable getDatasetModifiedRunnable(DatasetDefinition def, boolean fireNotification) + { + return new DatasetModifiedRunnable(def, fireNotification); + } + + private static class DatasetModifiedRunnable implements Runnable + { + private final @NotNull + DatasetDefinition _def; + private final boolean _fireNotification; + + private DatasetModifiedRunnable(@NotNull DatasetDefinition def, boolean fireNotification) + { + _def = def; + _fireNotification = fireNotification; + } + + private int getDatasetId() + { + return _def.getDatasetId(); + } + + private Container getContainer() + { + return _def.getContainer(); + } + + @Override + public void run() + { + DatasetDefinition.updateModified(_def, new Date()); + if (_fireNotification) + fireDatasetChanged(_def); + } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + DatasetModifiedRunnable that = (DatasetModifiedRunnable) o; + if (getDatasetId() != that.getDatasetId()) + return false; + return getContainer().equals(that.getContainer()); + } + + @Override + public int hashCode() + { + int result = getContainer().hashCode(); + result = 31 * result + this.getDatasetId(); + return result; + } + } + + public static void fireDatasetChanged(Dataset def) + { + for (DatasetManager.DatasetListener l : DatasetManager.getListeners()) + { + try + { + l.datasetChanged(def); + } + catch (Throwable t) + { + _log.error("fireDatasetChanged", t); + } + } + } + + // Return a source->alias map for the specified participant + public Map getAliasMap(StudyImpl study, User user, String ptid) + { + @Nullable final TableInfo aliasTable = StudyQuerySchema.createSchema(study, user).getParticipantAliasesTable(); + + if (null == aliasTable) + return Collections.emptyMap(); + + List columns = aliasTable.getColumns(); + SimpleFilter filter = new SimpleFilter(columns.get(0).getFieldKey(), ptid); + + // Return source -> alias map + return new TableSelector(aliasTable, Arrays.asList(columns.get(2), columns.get(1)), filter, null).getValueMap(String.class); + } + + private void unindexDataset(DatasetDefinition ds) + { + String docid = "dataset:" + new Path(ds.getContainer().getId(), String.valueOf(ds.getDatasetId())); + SearchService.get().deleteResource(docid); + } + + public static void indexDatasets(SearchService.TaskIndexingQueue queue, Date modifiedSince) + { + Container c = queue.getContainer(); + SimpleFilter filter = (null != c ? SimpleFilter.createContainerFilter(c) : new SimpleFilter()); + if (null != modifiedSince) + filter.addCondition(FieldKey.fromParts("Modified"), modifiedSince, CompareType.DATE_GT); + SQLFragment f = new SQLFragment("SELECT Container, DatasetId FROM " + StudySchema.getInstance().getTableInfoDataset() + " ") + .append(filter.getSQLFragment(StudySchema.getInstance().getSqlDialect())); + + new SqlSelector(StudySchema.getInstance().getSchema(), f).forEach(rs -> + { + String container = rs.getString(1); + int id = rs.getInt(2); + + Container c2 = ContainerManager.getForId(container); + if (null != c2) + { + Study study = StudyManager.getInstance().getStudy(c2); + + if (null != study) + { + DatasetDefinition dsd = StudyManager.getInstance().getDatasetDefinition(study, id); + if (null != dsd) + indexDataset(queue, dsd); + } + } + }); + } + + private static void indexDataset(SearchService.TaskIndexingQueue queue, DatasetDefinition dsd) + { + if (dsd.getType().equals(Dataset.TYPE_PLACEHOLDER)) + return; + if (null == dsd.getTypeURI() || null == dsd.getDomain()) + return; + String docid = "dataset:" + new Path(dsd.getContainer().getId(), String.valueOf(dsd.getDatasetId())); + + StringBuilder body = new StringBuilder(); + Map props = new HashMap<>(); + + props.put(SearchService.PROPERTY.categories.toString(), datasetCategory.toString()); + props.put(SearchService.PROPERTY.title.toString(), StringUtils.defaultIfEmpty(dsd.getLabel(),dsd.getName())); + String name = dsd.getName(); + String label = Strings.CS.equals(dsd.getLabel(),name) ? null : dsd.getLabel(); + String description = dsd.getDescription(); + String tag = dsd.getTag(); + String keywords = StringUtilsLabKey.joinNonBlank(" ", name, label, description, tag); + props.put(SearchService.PROPERTY.keywordsMed.toString(), keywords); + + body.append(keywords).append("\n"); + + StudyQuerySchema schema = StudyQuerySchema.createSchema(dsd.getStudy(), User.getSearchUser(), RoleManager.getRole(ReaderRole.class)); + TableInfo tableInfo = schema.getDatasetTable(dsd, null); + Map columns = QueryService.get().getColumns(tableInfo, tableInfo.getDefaultVisibleColumns()); + String sep = ""; + for (ColumnInfo column : columns.values()) + { + String n = StringUtils.trimToEmpty(column.getName()); + String l = StringUtils.trimToEmpty(column.getLabel()); + if (n.equals(l)) + l = ""; + body.append(sep).append(StringUtilsLabKey.joinNonBlank(" ", n, l)); + sep = ",\n"; + } + + ActionURL view = new ActionURL(StudyController.DatasetAction.class, null); + view.replaceParameter("datasetId", dsd.getDatasetId()); + view.setExtraPath(dsd.getContainer().getId()); + + SimpleDocumentResource r = new SimpleDocumentResource(new Path(docid), docid, + "text/plain", body.toString(), + view, props); + queue.addResource(r); + } + + public static void indexParticipants(SearchService.TaskIndexingQueue queue, @Nullable List ptids, @Nullable Date modifiedSince) + { + if (null != ptids && ptids.isEmpty()) + return; + + Container c = queue.getContainer(); + final StudyImpl study = StudyManager.getInstance().getStudy(c); + if (null == study) + return; + + final String nav = NavTree.toJS(Collections.singleton(new NavTree("study", PageFlowUtil.urlProvider(ProjectUrls.class).getBeginURL(c))), null, false, true).toString(); + + TableInfo participantTable = StudySchema.getInstance().getTableInfoParticipant(); + SQLFragment baseFragment = new SQLFragment(); + baseFragment.append("SELECT Container, ParticipantId FROM "); + baseFragment.append(participantTable, "p"); + + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + + @Nullable final TableInfo aliasTable = StudyQuerySchema.createSchema(study, User.getSearchUser()).getParticipantAliasesTable(); + LastIndexedClause lastIndexedClause = new LastIndexedClause(StudySchema.getInstance().getTableInfoParticipant(), modifiedSince, "p"); + if (!lastIndexedClause.isEmpty()) + { + if (null != aliasTable) + { + // Also reindex participants whose aliases have changed + SQLFragment aliasFragment = new SQLFragment().append("ParticipantId IN (\nSELECT ParticipantId FROM\n") + .append(aliasTable.getFromSQL("aliases")) + .append("WHERE aliases.Modified > p.LastIndexed)"); + filter.addClause(new OrClause(lastIndexedClause, new SQLClause(aliasFragment))); + } + else + { + filter.addClause(lastIndexedClause); + } + } + + baseFragment + .append(" ") + .append(filter.getSQLFragment(participantTable, "p")); + + final ActionURL executeURL = new ActionURL(StudyController.ParticipantAction.class, c); + executeURL.setExtraPath(c.getId()); + + final int BATCH_SIZE = 500; + List> batches = ptids != null ? ListUtils.partition(ptids, BATCH_SIZE) : Collections.singletonList(null); + + batches.forEach(batch -> { + Consumer runnable = (q) -> { + SQLFragment sql; + + if (null != batch) + { + sql = new SQLFragment(baseFragment); // Clone the base fragment before modifying + sql.append(" AND ParticipantId "); + StudySchema.getInstance().getSqlDialect().appendInClauseSql(sql, batch); + } + else + { + sql = baseFragment; + } + + new SqlSelector(StudySchema.getInstance().getSchema(), sql).forEach(rs -> { + final String ptid = rs.getString(2); + String displayTitle = "Study " + study.getLabel() + " -- " + + StudyService.get().getSubjectNounSingular(study.getContainer()) + " " + ptid; + ActionURL execute = executeURL.clone().addParameter("participantId", String.valueOf(ptid)); + Path p = new Path(c.getId(), ptid); + String docid = "participant:" + p; + + String uniqueIds = ptid; + + if (null != aliasTable) + { + // Add all participant aliases as high priority uniqueIds + Map aliasMap = StudyManager.getInstance().getAliasMap(study, User.getSearchUser(), ptid); + + if (!aliasMap.isEmpty()) + uniqueIds = uniqueIds + " " + StringUtils.join(aliasMap.values(), " "); + } + + Map props = new HashMap<>(); + props.put(SearchService.PROPERTY.categories.toString(), subjectCategory.getName()); + props.put(SearchService.PROPERTY.title.toString(), displayTitle); + props.put(SearchService.PROPERTY.identifiersHi.toString(), uniqueIds); + props.put(SearchService.PROPERTY.navtrail.toString(), nav); + + // Index a barebones participant document for now TODO: Figure out if it's safe to include demographic data or not (can all study users see it?) + + // SimpleDocument + SimpleDocumentResource r = new SimpleDocumentResource( + p, docid, + c.getEntityId(), + "text/plain", + displayTitle, + execute, props + ) + { + @Override + public void setLastIndexed(long ms, long modified) + { + StudySchema ss = StudySchema.getInstance(); + SQLFragment update = new SQLFragment("UPDATE ").append(ss.getTableInfoParticipant()).append(" SET LastIndexed = ? WHERE Container = ? AND ParticipantId = ?"); + update.addAll(new Timestamp(ms), c, ptid); + new SqlExecutor(ss.getSchema()).execute(update); + } + }; + queue.addResource(r); + }); + }; + + queue.addRunnable(runnable); + }); + } + + + // make sure we don't over do it with multiple calls to reindex the same study (see reindex()) + // add a level of indirection + // CONSIDER: add some facility like this to SearchService?? + // NOTE: this needs to be reviewed if we use modifiedSince + + final static WeakHashMap> _lastEnumerate = new WeakHashMap<>(); + + public static void _enumerateDocuments(SearchService.TaskIndexingQueue queue, @Nullable Date modifiedSince) + { + Container c = queue.getContainer(); + Consumer runEnumerate = new Consumer<>() + { + public void accept(SearchService.TaskIndexingQueue a) + { + synchronized (_lastEnumerate) + { + Consumer r = _lastEnumerate.get(c); + if (this != r) + return; + _lastEnumerate.remove(c); + } + + Study study = StudyManager.getInstance().getStudy(c); + + if (null != study) + { + StudyManager.indexDatasets(a, modifiedSince); + StudyManager.indexParticipants(a, null, modifiedSince); + // study protocol document + _enumerateProtocolDocuments(queue, study); + } + } + }; + + synchronized (_lastEnumerate) + { + _lastEnumerate.put(c, runEnumerate); + } + + queue.addRunnable(runEnumerate); + } + + + public static void _enumerateProtocolDocuments(SearchService.TaskIndexingQueue queue, @NotNull Study study) + { + AttachmentParent parent = ((StudyImpl)study).getProtocolDocumentAttachmentParent(); + if (null == parent) + return; + + ActionURL begin = PageFlowUtil.urlProvider(ProjectUrls.class).getBeginURL(study.getContainer()); + String nav = NavTree.toJS(Collections.singleton(new NavTree("study", begin)), null, false, true).toString(); + AttachmentService serv = AttachmentService.get(); + Path p = study.getContainer().getParsedPath().append("@study"); + + for (Attachment att : serv.getAttachments(parent)) + { + ActionURL download = StudyController.getProtocolDocumentDownloadURL(study.getContainer(), att.getName()); + + WebdavResource r = serv.getDocumentResource + ( + p.append(att.getName()), + download, + "\"" + att.getName() + "\" -- Protocol document attached to study " + study.getLabel(), + parent, att.getName(), SearchService.fileCategory + ); + r.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); + queue.addResource(r); + } + } + + public List getPublishedStudies(Container sourceStudyContainer) + { + return Collections.unmodifiableList(new TableSelector(StudySchema.getInstance().getTableInfoStudy(), + new SimpleFilter(FieldKey.fromParts("SourceStudyContainerId"), sourceStudyContainer), null).getArrayList(StudyImpl.class)); + } + + // Return collection of current snapshots that are configured to refresh specimens + public Collection getRefreshStudySnapshots() + { + return getStudySnapshots(new SQLFragment(" AND Refresh = ?", Boolean.TRUE)); + } + + // Return collection of all current snapshots + private Collection getStudySnapshots(@Nullable SQLFragment filter) + { + SQLFragment sql = new SQLFragment("SELECT ss.* FROM "); + sql.append(StudySchema.getInstance().getTableInfoStudy(), "s"); + sql.append(" JOIN "); + sql.append(StudySchema.getInstance().getTableInfoStudySnapshot(), "ss"); + sql.append(" ON s.StudySnapshot = ss.RowId AND Source IS NOT NULL AND Destination IS NOT NULL"); + + if (null != filter) + sql.append(filter); + + return new SqlSelector(StudySchema.getInstance().getSchema(), sql).getCollection(StudySnapshot.class); + } + + @Nullable + public StudySnapshot getStudySnapshot(Integer snapshotId) + { + TableSelector selector = new TableSelector(StudySchema.getInstance().getTableInfoStudySnapshot(), new SimpleFilter(FieldKey.fromParts("RowId"), snapshotId), null); + + return selector.getObject(StudySnapshot.class); + } + + /** + * Convert a placeholder or 'ghost' dataset to an actual dataset by renaming the target dataset to the placeholder's name, + * transferring all timepoint requirements from the placeholder to the target and deleting the placeholder dataset. + */ + public DatasetDefinition linkPlaceHolderDataset(StudyImpl study, User user, DatasetDefinition expectationDataset, DatasetDefinition targetDataset) + { + if (expectationDataset == null || targetDataset == null) + throw new IllegalArgumentException("Both expectation DataSet and target DataSet must exist"); + + if (!expectationDataset.getType().equals(Dataset.TYPE_PLACEHOLDER)) + throw new IllegalArgumentException("Only a DataSet of type : placeholder can be linked"); + + if (!targetDataset.getType().equals(Dataset.TYPE_STANDARD)) + throw new IllegalArgumentException("Only a DataSet of type : standard can be linked to"); + + DbScope scope = StudySchema.getInstance().getSchema().getScope(); + + try (Transaction transaction = scope.ensureTransaction()) + { + // transfer any timepoint requirements from the ghost to target + for (VisitDataset vds : expectationDataset.getVisitDatasets()) + { + VisitDatasetType type = vds.isRequired() ? VisitDatasetType.REQUIRED : VisitDatasetType.NOT_ASSOCIATED; + StudyManager.getInstance().updateVisitDatasetMapping(user, study.getContainer(), vds.getVisitRowId(), targetDataset.getDatasetId(), type); + } + + String name = expectationDataset.getName(); + String label = expectationDataset.getLabel(); + + // no need to resync the study, as there should be no data in the expectation dataset + deleteDataset(study, user, expectationDataset, false, null); + + targetDataset = targetDataset.createMutable(); + targetDataset.setName(name); + targetDataset.setLabel(label); + targetDataset.save(user); + + transaction.commit(); + } + + return targetDataset; + } + + public static class CategoryListener implements ViewCategoryListener + { + private final StudyManager _instance; + + private CategoryListener(StudyManager instance) + { + _instance = instance; + } + + @Override + public void categoryDeleted(User user, ViewCategory category) + { + for (DatasetDefinition def : getDatasetsForCategory(category)) + { + def = def.createMutable(); + def.setCategoryId(0); + def.save(user); + } + } + + @Override + public void categoryCreated(User user, ViewCategory category) + {} + + @Override + public void categoryUpdated(User user, ViewCategory category) + { + Container c = ContainerManager.getForId(category.getContainerId()); + if (null != c) + _instance._datasetHelper.clearCache(c); + } + + private Collection getDatasetsForCategory(ViewCategory category) + { + if (category != null) + { + Study study = _instance.getStudy(ContainerManager.getForId(category.getContainerId())); + if (study != null) + { + return _instance._datasetHelper.getDatasetsForCategory(study, category); + } + } + + return Collections.emptyList(); + } + } + + /** + * Get the shared study in the project for the given study (excluding the shared study itself) + */ + @Nullable + public Study getSharedStudy(@NotNull Container c) + { + if (c.isProject()) + return null; + Container p = c.getProject(); + if (null == p) + return null; + Study sharedStudy = getStudy(p); + if (null == sharedStudy) + return null; + if (!sharedStudy.getShareDatasetDefinitions()) + return null; + return sharedStudy; + } + + /** + * Get the shared study in the project for the given study (excluding the shared study itself.) + */ + @Nullable + public Study getSharedStudy(@NotNull Study study) + { + return getSharedStudy(study.getContainer()); + } + + /** + * Get the shared study in the project for the given study + * or just return the current study if no shared study exists. + */ + public @NotNull Study getSharedStudyOrCurrent(@NotNull Study study) + { + Study sharedStudy = getSharedStudy(study); + return sharedStudy != null ? sharedStudy : study; + } + + /** + * Get the Study to use for visits -- either the + * project shared study's container (if shared visits is turned on) + * or the current study container. + */ + @NotNull + public Study getStudyForVisits(@NotNull Study study) + { + Study sharedStudy = getSharedStudy(study); + if (sharedStudy != null && sharedStudy.getShareVisitDefinitions()) + return sharedStudy; + + return study; + } + + /** + * Get the Study to use for VisitTags -- either the + * project shared study's container or the current study container. + */ + @NotNull + public Study getStudyForVisitTag(@NotNull Study study) + { + return getSharedStudyOrCurrent(study); + } + + + /* + * TESTING + */ + + // To see detailed logging from StatementDataIterator, configure org.labkey.study.model.StudyManager$DatasetImportTestCase to level TRACE + private static class Tests {} + public static final Logger TEST_LOGGER = LogManager.getLogger(Tests.class); + + + public static class VisitCreationTestCase extends Assert + { + private static final double DELTA = 1E-8; + + @Test + public void testDateConversion() + { + Date d = new Date(); + String iso = DateUtil.toISO(d.getTime(), true); + DbSchema core = CoreSchema.getInstance().getSchema(); + SQLFragment select = new SQLFragment("SELECT "); + select.append(core.getSqlDialect().getISOFormat(new SQLFragment("?",d))); + String db = new SqlSelector(core, select).getObject(String.class); + // SQL SERVER doesn't quite store millisecond precision + assertEquals(23,iso.length()); + assertEquals(23,db.length()); + assertEquals(iso.substring(0,20), db.substring(0,20)); + String jdbc = (String)JdbcType.VARCHAR.convert(d); + assertEquals(jdbc, iso); + } + + @Test + public void testExistingVisitBased() + { + StudyImpl study = new StudyImpl(); + study.setContainer(JunitUtil.getTestContainer()); + study.setTimepointType(TimepointType.VISIT); + + List existingVisits = new ArrayList<>(3); + existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(1), BigDecimal.valueOf(1), null, Visit.Type.BASELINE)); + existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(2), BigDecimal.valueOf(2), null, Visit.Type.BASELINE)); + existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(2.5), BigDecimal.valueOf(3.0), null, Visit.Type.BASELINE)); + + assertEquals("Should return existing visit", existingVisits.get(0), getInstance().ensureVisitWithoutSaving(study, 1, Visit.Type.BASELINE, existingVisits)); + assertEquals("Should return existing visit", existingVisits.get(1), getInstance().ensureVisitWithoutSaving(study, 2, Visit.Type.BASELINE, existingVisits)); + assertEquals("Should return existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 2.5, Visit.Type.BASELINE, existingVisits)); + assertEquals("Should return existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 3.0, Visit.Type.BASELINE, existingVisits)); + + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.1, Visit.Type.BASELINE, existingVisits), existingVisits, 1.1, 1.1); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 3.001, Visit.Type.BASELINE, existingVisits), existingVisits, 3.001, 3.001); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 4, Visit.Type.BASELINE, existingVisits), existingVisits, 4, 4); + } + + @Test + public void testEmptyVisitBased() + { + StudyImpl study = new StudyImpl(); + study.setContainer(JunitUtil.getTestContainer()); + study.setTimepointType(TimepointType.VISIT); + + List existingVisits = new ArrayList<>(); + + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.1, Visit.Type.BASELINE, existingVisits), existingVisits, 1.1, 1.1); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 3.001, Visit.Type.BASELINE, existingVisits), existingVisits, 3.001, 3.001); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 4, Visit.Type.BASELINE, existingVisits), existingVisits, 4, 4); + } + + @Test + public void testEmptyDateBased() + { + StudyImpl study = new StudyImpl(); + study.setContainer(JunitUtil.getTestContainer()); + study.setTimepointType(TimepointType.DATE); + + List existingVisits = new ArrayList<>(); + + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1, Visit.Type.BASELINE, existingVisits), existingVisits, 1, 1); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, -10, Visit.Type.BASELINE, existingVisits), existingVisits, -10, -10); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.5, Visit.Type.BASELINE, existingVisits), existingVisits, 1.5, 1.5); + + study.setDefaultTimepointDuration(7); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0, Visit.Type.BASELINE, existingVisits), existingVisits, 0, 6); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1, Visit.Type.BASELINE, existingVisits), existingVisits, 0, 6); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 6, Visit.Type.BASELINE, existingVisits), existingVisits, 0, 6); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 7, Visit.Type.BASELINE, existingVisits), existingVisits, 7, 13); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 10, Visit.Type.BASELINE, existingVisits), existingVisits, 7, 13); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 15, Visit.Type.BASELINE, existingVisits), existingVisits, 14, 20); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, -10, Visit.Type.BASELINE, existingVisits), existingVisits, -10, -10); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.5, Visit.Type.BASELINE, existingVisits), existingVisits, 1.5, 1.5); + } + + @Test + public void testExistingDateBased() + { + StudyImpl study = new StudyImpl(); + study.setContainer(JunitUtil.getTestContainer()); + study.setTimepointType(TimepointType.DATE); + + List existingVisits = new ArrayList<>(3); + existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(1), BigDecimal.valueOf(1), null, Visit.Type.BASELINE)); + existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(2), BigDecimal.valueOf(2), null, Visit.Type.BASELINE)); + existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(7), BigDecimal.valueOf(13), null, Visit.Type.BASELINE)); + + assertSame("Should be existing visit", existingVisits.get(0), getInstance().ensureVisitWithoutSaving(study, 1, Visit.Type.BASELINE, existingVisits)); + assertSame("Should be existing visit", existingVisits.get(1), getInstance().ensureVisitWithoutSaving(study, 2, Visit.Type.BASELINE, existingVisits)); + assertSame("Should be existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 7, Visit.Type.BASELINE, existingVisits)); + assertSame("Should be existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 10, Visit.Type.BASELINE, existingVisits)); + assertSame("Should be existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 13, Visit.Type.BASELINE, existingVisits)); + + study.setDefaultTimepointDuration(7); + assertSame("Should be existing visit", existingVisits.get(0), getInstance().ensureVisitWithoutSaving(study, 1, Visit.Type.BASELINE, existingVisits)); + assertSame("Should be existing visit", existingVisits.get(1), getInstance().ensureVisitWithoutSaving(study, 2, Visit.Type.BASELINE, existingVisits)); + assertSame("Should be existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 7, Visit.Type.BASELINE, existingVisits)); + assertSame("Should be existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 10, Visit.Type.BASELINE, existingVisits)); + assertSame("Should be existing visit", existingVisits.get(2), getInstance().ensureVisitWithoutSaving(study, 13, Visit.Type.BASELINE, existingVisits)); + } + + @Test + public void testCreationDateBased() + { + StudyImpl study = new StudyImpl(); + study.setContainer(JunitUtil.getTestContainer()); + study.setTimepointType(TimepointType.DATE); + + List existingVisits = new ArrayList<>(4); + existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(1), BigDecimal.valueOf(1), null, Visit.Type.BASELINE)); + existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(2), BigDecimal.valueOf(2), null, Visit.Type.BASELINE)); + existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(7), BigDecimal.valueOf(13), null, Visit.Type.BASELINE)); + existingVisits.add(new VisitImpl(null, BigDecimal.valueOf(62), BigDecimal.valueOf(64), null, Visit.Type.BASELINE)); + + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 3, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 3); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 14, Visit.Type.BASELINE, existingVisits), existingVisits, 14, 14); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, -14, Visit.Type.BASELINE, existingVisits), existingVisits, -14, -14); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0, Visit.Type.BASELINE, existingVisits), existingVisits, 0, 0); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0.5, Visit.Type.BASELINE, existingVisits), existingVisits, 0.5, 0.5); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.5, Visit.Type.BASELINE, existingVisits), existingVisits, 1.5, 1.5); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, -5, Visit.Type.BASELINE, existingVisits), existingVisits, -5, -5); + + study.setDefaultTimepointDuration(7); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 3, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 4, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 5, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 6, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 14, Visit.Type.BASELINE, existingVisits), existingVisits, 14, 20, "Week 3"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 21, Visit.Type.BASELINE, existingVisits), existingVisits, 21, 27, "Week 4"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0, Visit.Type.BASELINE, existingVisits), existingVisits, 0, 0, "Day 0"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0.5, Visit.Type.BASELINE, existingVisits), existingVisits, 0.5, 0.5, "Day 0.5"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.5, Visit.Type.BASELINE, existingVisits), existingVisits, 1.5, 1.5, "Day 1.5"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, -5, Visit.Type.BASELINE, existingVisits), existingVisits, -5, -5, "Day -5"); + + study.setDefaultTimepointDuration(30); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 3, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 4, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 5, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 6, Visit.Type.BASELINE, existingVisits), existingVisits, 3, 6, "Day 3 - 6"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 14, Visit.Type.BASELINE, existingVisits), existingVisits, 14, 29, "Day 14 - 29"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 21, Visit.Type.BASELINE, existingVisits), existingVisits, 14, 29, "Day 14 - 29"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 29, Visit.Type.BASELINE, existingVisits), existingVisits, 14, 29, "Day 14 - 29"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 30, Visit.Type.BASELINE, existingVisits), existingVisits, 30, 59, "Month 2"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 60, Visit.Type.BASELINE, existingVisits), existingVisits, 60, 61, "Day 60 - 61"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 61, Visit.Type.BASELINE, existingVisits), existingVisits, 60, 61, "Day 60 - 61"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 65, Visit.Type.BASELINE, existingVisits), existingVisits, 65, 89, "Day 65 - 89"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 100, Visit.Type.BASELINE, existingVisits), existingVisits, 90, 119, "Month 4"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0, Visit.Type.BASELINE, existingVisits), existingVisits, 0, 0, "Day 0"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 0.5, Visit.Type.BASELINE, existingVisits), existingVisits, 0.5, 0.5, "Day 0.5"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, 1.5, Visit.Type.BASELINE, existingVisits), existingVisits, 1.5, 1.5, "Day 1.5"); + validateNewVisit(getInstance().ensureVisitWithoutSaving(study, -5, Visit.Type.BASELINE, existingVisits), existingVisits, -5, -5, "Day -5"); + } + + @Test + public void testVisitDescription() + { + StudyImpl study = new StudyImpl(); + study.setContainer(JunitUtil.getTestContainer()); + study.setTimepointType(TimepointType.DATE); + + List existingVisits = new ArrayList<>(); + + VisitImpl newVisit = getInstance().ensureVisitWithoutSaving(study, 1, Visit.Type.BASELINE, existingVisits); + newVisit.setDescription("My custom visit description"); + validateNewVisit(newVisit, existingVisits, 1, 1, "Day 1", "My custom visit description"); + } + + private void validateNewVisit(VisitImpl newVisit, List existingVisits, double seqNumMin, double seqNumMax, String label, String description) + { + validateNewVisit(newVisit, existingVisits, seqNumMin, seqNumMax, label); + assertEquals("Descriptions don't match", description, newVisit.getDescription()); + } + + private void validateNewVisit(VisitImpl newVisit, List existingVisits, double seqNumMin, double seqNumMax, String label) + { + validateNewVisit(newVisit, existingVisits, seqNumMin, seqNumMax); + assertEquals("Labels don't match", label, newVisit.getLabel()); + } + + private void validateNewVisit(VisitImpl newVisit, List existingVisits, double seqNumMin, double seqNumMax) + { + for (VisitImpl existingVisit : existingVisits) + { + assertNotSame("Should be a new visit", newVisit, existingVisit); + } + assertEquals("Shouldn't have a rowId yet", 0, newVisit.getRowId()); + assertEquals("Wrong sequenceNumMin", VisitImpl.getSequenceNum(seqNumMin), newVisit.getSequenceNumMin()); + assertEquals("Wrong sequenceNumMax", VisitImpl.getSequenceNum(seqNumMax), newVisit.getSequenceNumMax()); + } + } + + public static class StudySnapshotTestCase extends Assert + { + @Test + public void testComplianceSettings() + { + // We load the SnapshotSettings bean from serialized JSON in the core.StudySnapshot.Settings column. This + // test ensures that we serialize using the latest compliance properties but continue to correctly load + // older snapshots that might specify "removeProtectedColumns":true instead of "phiLevel":. This + // was broken shortly after we migrated to using phiLevel, see #xxxx. + + // phiLevel property takes precedence over legacy properties + testComplianceSettings("\"removeProtectedColumns\":true,\"removePhiColumns\":false,\"phiLevel\":\"Limited\",\"shiftDates\":false,\"useAlternateParticipantIds\":false,\"maskClinic\":false", PHI.Limited); + testComplianceSettings("\"removeProtectedColumns\":false,\"removePhiColumns\":false,\"phiLevel\":\"Limited\",\"shiftDates\":false,\"useAlternateParticipantIds\":false,\"maskClinic\":false", PHI.Limited); + testComplianceSettings("\"phiLevel\":\"Restricted\"", PHI.Restricted); + testComplianceSettings("\"phiLevel\":\"PHI\"", PHI.PHI); + testComplianceSettings("\"phiLevel\":\"Limited\"", PHI.Limited); + testComplianceSettings("\"phiLevel\":\"NotPHI\"", PHI.NotPHI); + + // removeProtectedColumns:true means include no PHI columns + testComplianceSettings("\"removeProtectedColumns\":true,\"shiftDates\":true,\"useAlternateParticipantIds\":true,\"maskClinic\":true", PHI.NotPHI); + testComplianceSettings("\"removeProtectedColumns\":false,\"shiftDates\":true,\"useAlternateParticipantIds\":true,\"maskClinic\":true", PHI.Restricted); + + // removePhiColumns property should have no effect + testComplianceSettings("\"removeProtectedColumns\":true,\"removePhiColumns\":true", PHI.NotPHI); + testComplianceSettings("\"removeProtectedColumns\":true,\"removePhiColumns\":false", PHI.NotPHI); + + // If no properties are specified then include all columns + testComplianceSettings("\"shiftDates\":true,\"useAlternateParticipantIds\":true,\"maskClinic\":true", PHI.Restricted); + testComplianceSettings("", PHI.Restricted); + } + + private static final String JSON_PREFIX = "{\"description\":null,\"participantGroups\":[],\"participants\":null,\"datasets\":[5008,5024,5025,5026,5004,5006,5007],\"datasetRefresh\":true,\"datasetRefreshDelay\":30,\"visits\":null,\"specimenRequestId\":null,\"includeSpecimens\":true,\"specimenRefresh\":true,\"studyObjects\":[],\"lists\":[],\"views\":[],\"reports\":[],\"folderObjects\":[]"; + + private void testComplianceSettings(String settingsJson, PHI expectedLevel) + { + String json = JSON_PREFIX + (StringUtils.isNotEmpty(settingsJson) ? "," + settingsJson + "}" : "}"); + StudySnapshot snapshot = new StudySnapshot(); + snapshot.setSettings(json); + + testSnapshot(snapshot, expectedLevel); + } + + @Test + public void testStoredSnapshots() + { + Collection snapshots = StudyManager.getInstance().getStudySnapshots(null); + + for (StudySnapshot snapshot : snapshots) + { + PHI level = snapshot.getSnapshotSettings().getPhiLevel(); + testSnapshot(snapshot, level); + StudySnapshot snapshotFromRowId = StudyManager.getInstance().getStudySnapshot(snapshot.getRowId()); + testSnapshot(snapshotFromRowId, level); + } + } + + private void testSnapshot(StudySnapshot snapshot, PHI expectedLevel) + { + assertNotNull(snapshot); + assertNotNull(expectedLevel); + SnapshotSettings settings = snapshot.getSnapshotSettings(); + assertNotNull("getPhiLevel() returned null", settings.getPhiLevel()); + assertEquals(expectedLevel, settings.getPhiLevel()); + + // Test the settings JSON that this snapshot generates + String serializedJson = snapshot.getSettings(); + String expectedLevelJson = "\"phiLevel\":\"" + expectedLevel.name() + "\""; + assertTrue("Serialized JSON did not include " + expectedLevelJson, serializedJson.contains(expectedLevelJson)); + assertFalse("Serialized JSON included removeProtectedColumns", serializedJson.contains("removeProtectedColumns")); + assertFalse("Serialized JSON included removePhiColumns", serializedJson.contains("removePhiColumns")); + } + } +} diff --git a/wiki/src/org/labkey/wiki/WikiController.java b/wiki/src/org/labkey/wiki/WikiController.java index 2ca4951b390..ba44a15aa15 100644 --- a/wiki/src/org/labkey/wiki/WikiController.java +++ b/wiki/src/org/labkey/wiki/WikiController.java @@ -152,9 +152,7 @@ protected BaseWikiPermissions getPermissions() { VBox vbox = new VBox(); - SearchService ss = SearchService.get(); - if (null != ss) - vbox.addView(ss.getSearchView(false, 0, false, true)); + vbox.addView(SearchService.get().getSearchView(false, 0, false, true)); for (WikiPartFactory factory : WikiManager.get().getWikiPartFactories()) { @@ -1185,9 +1183,7 @@ public ModelAndView getView(WikiNameForm form, BindException errors) //set new page title to be the name. _wikiversion.setTitle(name); // check if this is a search result hit - SearchService ss = SearchService.get(); - if (ss != null) - ss.notFound(getViewContext().getActionURL()); + SearchService.get().notFound(getViewContext().getActionURL()); } else { diff --git a/wiki/src/org/labkey/wiki/WikiModule.java b/wiki/src/org/labkey/wiki/WikiModule.java index 60685e15a97..f2484d48da5 100644 --- a/wiki/src/org/labkey/wiki/WikiModule.java +++ b/wiki/src/org/labkey/wiki/WikiModule.java @@ -1,258 +1,255 @@ -/* - * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.wiki; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.admin.sitevalidation.SiteValidationService; -import org.labkey.api.announcements.CommSchema; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DatabaseMigrationService; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationHandler; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.TableInfo; -import org.labkey.api.module.CodeOnlyModule; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.wiki.WikiRendererType; -import org.labkey.api.wiki.WikiService; -import org.labkey.wiki.export.WikiImporterFactory; -import org.labkey.wiki.export.WikiWriterFactory; -import org.labkey.wiki.model.Wiki; -import org.labkey.wiki.model.WikiType; -import org.labkey.wiki.model.WikiVersion; -import org.labkey.wiki.query.WikiSchema; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Provides a simple wiki with multiple rendering engine options. - */ -public class WikiModule extends CodeOnlyModule implements SearchService.DocumentProvider -{ - public static final String WEB_PART_NAME = "Wiki"; - - // package logger for use with logger-manage.view - private static final Logger _logPackage = LogManager.getLogger(WikiModule.class.getPackage().getName()); - - private static final Logger _log = LogManager.getLogger(WikiModule.class); - - @Override - public String getName() - { - return "Wiki"; - } - - @Override - protected void init() - { - addController("wiki", WikiController.class, "attachments"); - - WikiService.setInstance(WikiManager.get()); - - AttachmentService.get().registerAttachmentType(WikiType.get()); - - SiteValidationService svc = SiteValidationService.get(); - if (null != svc) - { - svc.registerProviderFactory(getName(), new WikiValidationProviderFactory()); - } - } - - @Override - @NotNull - protected Collection createWebPartFactories() - { - return List.of( - new MenuWikiWebPartFactory(), - new WikiTOCFactory(), - new WikiWebPartFactory() - ); - } - - @Override - public void doStartup(ModuleContext moduleContext) - { - ContainerManager.addContainerListener(new WikiContainerListener()); -// WebdavService.get().addProvider(new WikiWebdavProvider()); - - // Don't check ModuleLoader.isNewInstall() here to support trial AMIs, where _newInstall may be true even though - // scripts and bootstrap() have already run - if (moduleContext.isNewInstall()) - bootstrap(moduleContext); - - SearchService ss = SearchService.get(); - if (null != ss) - { - ss.addSearchCategory(WikiManager.searchCategory); - ss.addDocumentProvider(this); - } - - FolderSerializationRegistry.get().addFactories(new WikiWriterFactory(), new WikiImporterFactory()); - - WikiSchema.register(this); - WikiController.registerAdminConsoleLinks(); - DatabaseMigrationService.get().registerHandler(CommSchema.getInstance().getSchema(), new DefaultMigrationHandler() - { - @Override - public void beforeSchema(DbSchema targetSchema) - { - new SqlExecutor(targetSchema).execute("ALTER TABLE comm.Pages DROP CONSTRAINT FK_Pages_PageVersions"); - } - - @Override - public List getTablesToCopy(DbSchema targetSchema) - { - List tablesToCopy = super.getTablesToCopy(targetSchema); - tablesToCopy.add(targetSchema.getTable("Pages")); - tablesToCopy.add(targetSchema.getTable("PageVersions")); - - return tablesToCopy; - } - - @Override - public void afterSchema(DbSchema targetSchema) - { - new SqlExecutor(targetSchema).execute("ALTER TABLE comm.Pages ADD CONSTRAINT FK_Pages_PageVersions FOREIGN KEY (PageVersionId) REFERENCES comm.PageVersions (RowId)"); - } - }); - } - - private void bootstrap(ModuleContext moduleContext) - { - if (ModuleLoader.getInstance().shouldInsertData()) - { - Container supportContainer = ContainerManager.getDefaultSupportContainer(); - Container homeContainer = ContainerManager.getHomeContainer(); - Container sharedContainer = ContainerManager.getSharedContainer(); - String defaultPageName = "default"; - - loadWikiContent(homeContainer, moduleContext.getUpgradeUser(), defaultPageName, "Welcome to LabKey Server", "/org/labkey/wiki/welcomeWiki.txt"); - loadWikiContent(supportContainer, moduleContext.getUpgradeUser(), defaultPageName, "Welcome to LabKey Support", "/org/labkey/wiki/supportWiki.txt"); - loadWikiContent(sharedContainer, moduleContext.getUpgradeUser(), defaultPageName, "Shared Resources", "/org/labkey/wiki/sharedWiki.txt"); - - addWebPart(supportContainer, defaultPageName); - addWebPart(sharedContainer, defaultPageName); - - // Add a wiki webpart with the default content. - addWebPart(homeContainer, defaultPageName); - } - } - - private void addWebPart(@Nullable Container c, String wikiName) - { - if (c != null) - { - Map wikiProps = new HashMap<>(); - wikiProps.put("webPartContainer", c.getId()); - wikiProps.put("name", wikiName); - addWebPart(WEB_PART_NAME, c, HttpView.BODY, 0, wikiProps); - } - } - - @Override - @NotNull - public Collection getSummary(Container c) - { - Collection list = new LinkedList<>(); - try - { - int count = WikiSelectManager.getPageCount(c); - if (count > 0) - list.add(count + " Wiki Page" + (count > 1 ? "s" : "")); - } - catch (Exception x) - { - list.add(x.toString()); - } - return list; - } - - private void loadWikiContent(@Nullable Container c, User user, String name, String title, String resource) - { - if (c != null) - { - Wiki wiki = new Wiki(c, name); - WikiVersion wikiversion = new WikiVersion(); - wikiversion.setTitle(title); - - InputStream is = getClass().getResourceAsStream(resource); - String body = PageFlowUtil.getStreamContentsAsString(is); - wikiversion.setBody(body); - - wikiversion.setRendererTypeEnum(WikiRendererType.HTML); - - try - { - getWikiManager().insertWiki(user, c, wiki, wikiversion, null, false, null); - } - catch (IOException e) - { - _log.error("Failed to insert wiki in " + c.getPath() + " from " + resource, e); - } - } - } - - - @Override - @NotNull - public Set getIntegrationTests() - { - return Set.of( - WikiManager.TestCase.class - ); - } - - - @Override - public void enumerateDocuments(SearchService.TaskIndexingQueue queue, @Nullable Date modifiedSince) - { - queue.addRunnable((q) -> - getWikiManager().indexWikis(q, modifiedSince, null)); - } - - - @Override - public void indexDeleted() - { - new SqlExecutor(CommSchema.getInstance().getSchema()).execute("UPDATE comm.pages SET lastIndexed=NULL"); - } - - - private WikiManager getWikiManager() - { - return WikiManager.get(); - } -} +/* + * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.wiki; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.admin.sitevalidation.SiteValidationService; +import org.labkey.api.announcements.CommSchema; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DatabaseMigrationService; +import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationHandler; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.TableInfo; +import org.labkey.api.module.CodeOnlyModule; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.wiki.WikiRendererType; +import org.labkey.api.wiki.WikiService; +import org.labkey.wiki.export.WikiImporterFactory; +import org.labkey.wiki.export.WikiWriterFactory; +import org.labkey.wiki.model.Wiki; +import org.labkey.wiki.model.WikiType; +import org.labkey.wiki.model.WikiVersion; +import org.labkey.wiki.query.WikiSchema; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Provides a simple wiki with multiple rendering engine options. + */ +public class WikiModule extends CodeOnlyModule implements SearchService.DocumentProvider +{ + public static final String WEB_PART_NAME = "Wiki"; + + // package logger for use with logger-manage.view + private static final Logger _logPackage = LogManager.getLogger(WikiModule.class.getPackage().getName()); + + private static final Logger _log = LogManager.getLogger(WikiModule.class); + + @Override + public String getName() + { + return "Wiki"; + } + + @Override + protected void init() + { + addController("wiki", WikiController.class, "attachments"); + + WikiService.setInstance(WikiManager.get()); + + AttachmentService.get().registerAttachmentType(WikiType.get()); + + SiteValidationService svc = SiteValidationService.get(); + if (null != svc) + { + svc.registerProviderFactory(getName(), new WikiValidationProviderFactory()); + } + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + return List.of( + new MenuWikiWebPartFactory(), + new WikiTOCFactory(), + new WikiWebPartFactory() + ); + } + + @Override + public void doStartup(ModuleContext moduleContext) + { + ContainerManager.addContainerListener(new WikiContainerListener()); +// WebdavService.get().addProvider(new WikiWebdavProvider()); + + // Don't check ModuleLoader.isNewInstall() here to support trial AMIs, where _newInstall may be true even though + // scripts and bootstrap() have already run + if (moduleContext.isNewInstall()) + bootstrap(moduleContext); + + SearchService ss = SearchService.get(); + ss.addSearchCategory(WikiManager.searchCategory); + ss.addDocumentProvider(this); + + FolderSerializationRegistry.get().addFactories(new WikiWriterFactory(), new WikiImporterFactory()); + + WikiSchema.register(this); + WikiController.registerAdminConsoleLinks(); + DatabaseMigrationService.get().registerHandler(CommSchema.getInstance().getSchema(), new DefaultMigrationHandler() + { + @Override + public void beforeSchema(DbSchema targetSchema) + { + new SqlExecutor(targetSchema).execute("ALTER TABLE comm.Pages DROP CONSTRAINT FK_Pages_PageVersions"); + } + + @Override + public List getTablesToCopy(DbSchema targetSchema) + { + List tablesToCopy = super.getTablesToCopy(targetSchema); + tablesToCopy.add(targetSchema.getTable("Pages")); + tablesToCopy.add(targetSchema.getTable("PageVersions")); + + return tablesToCopy; + } + + @Override + public void afterSchema(DbSchema targetSchema) + { + new SqlExecutor(targetSchema).execute("ALTER TABLE comm.Pages ADD CONSTRAINT FK_Pages_PageVersions FOREIGN KEY (PageVersionId) REFERENCES comm.PageVersions (RowId)"); + } + }); + } + + private void bootstrap(ModuleContext moduleContext) + { + if (ModuleLoader.getInstance().shouldInsertData()) + { + Container supportContainer = ContainerManager.getDefaultSupportContainer(); + Container homeContainer = ContainerManager.getHomeContainer(); + Container sharedContainer = ContainerManager.getSharedContainer(); + String defaultPageName = "default"; + + loadWikiContent(homeContainer, moduleContext.getUpgradeUser(), defaultPageName, "Welcome to LabKey Server", "/org/labkey/wiki/welcomeWiki.txt"); + loadWikiContent(supportContainer, moduleContext.getUpgradeUser(), defaultPageName, "Welcome to LabKey Support", "/org/labkey/wiki/supportWiki.txt"); + loadWikiContent(sharedContainer, moduleContext.getUpgradeUser(), defaultPageName, "Shared Resources", "/org/labkey/wiki/sharedWiki.txt"); + + addWebPart(supportContainer, defaultPageName); + addWebPart(sharedContainer, defaultPageName); + + // Add a wiki webpart with the default content. + addWebPart(homeContainer, defaultPageName); + } + } + + private void addWebPart(@Nullable Container c, String wikiName) + { + if (c != null) + { + Map wikiProps = new HashMap<>(); + wikiProps.put("webPartContainer", c.getId()); + wikiProps.put("name", wikiName); + addWebPart(WEB_PART_NAME, c, HttpView.BODY, 0, wikiProps); + } + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + Collection list = new LinkedList<>(); + try + { + int count = WikiSelectManager.getPageCount(c); + if (count > 0) + list.add(count + " Wiki Page" + (count > 1 ? "s" : "")); + } + catch (Exception x) + { + list.add(x.toString()); + } + return list; + } + + private void loadWikiContent(@Nullable Container c, User user, String name, String title, String resource) + { + if (c != null) + { + Wiki wiki = new Wiki(c, name); + WikiVersion wikiversion = new WikiVersion(); + wikiversion.setTitle(title); + + InputStream is = getClass().getResourceAsStream(resource); + String body = PageFlowUtil.getStreamContentsAsString(is); + wikiversion.setBody(body); + + wikiversion.setRendererTypeEnum(WikiRendererType.HTML); + + try + { + getWikiManager().insertWiki(user, c, wiki, wikiversion, null, false, null); + } + catch (IOException e) + { + _log.error("Failed to insert wiki in " + c.getPath() + " from " + resource, e); + } + } + } + + + @Override + @NotNull + public Set getIntegrationTests() + { + return Set.of( + WikiManager.TestCase.class + ); + } + + + @Override + public void enumerateDocuments(SearchService.TaskIndexingQueue queue, @Nullable Date modifiedSince) + { + queue.addRunnable((q) -> + getWikiManager().indexWikis(q, modifiedSince, null)); + } + + + @Override + public void indexDeleted() + { + new SqlExecutor(CommSchema.getInstance().getSchema()).execute("UPDATE comm.pages SET lastIndexed=NULL"); + } + + + private WikiManager getWikiManager() + { + return WikiManager.get(); + } +} From a415fe2807a04721a401261142965787a029bedd Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Fri, 12 Sep 2025 17:44:03 -0700 Subject: [PATCH 2/2] Fix build --- search/src/org/labkey/search/SearchModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search/src/org/labkey/search/SearchModule.java b/search/src/org/labkey/search/SearchModule.java index 1bf6a6baa37..ea05e096f2d 100644 --- a/search/src/org/labkey/search/SearchModule.java +++ b/search/src/org/labkey/search/SearchModule.java @@ -188,7 +188,7 @@ public void startBackgroundThreads() { // Execute any reindexing operations in the background to not block startup, Issue #48960 JobRunner.getDefault().execute(() -> { - _searchIndexStartupHandler.reindexIfNeeded(ss); + _searchIndexStartupHandler.reindexIfNeeded(SearchService.get()); SearchService.get().start(); DavCrawler.getInstance().start(); });